"""Tests for zombie process cleanup — verifies processes spawned by tools
are properly reaped when agent sessions end.

Reproduction for issue #7131: zombie process accumulation on long-running
gateway deployments.
"""

import os
import signal
import subprocess
import sys
import threading



def _spawn_sleep(seconds: float = 60) -> subprocess.Popen:
    """Spawn a portable long-lived Python sleep process (no shell wrapper)."""
    return subprocess.Popen(
        [sys.executable, "-c", f"import time; time.sleep({seconds})"],
    )


def _pid_alive(pid: int) -> bool:
    """Return True if a process with the given PID is still running."""
    try:
        os.kill(pid, 0)
        return True
    except (ProcessLookupError, PermissionError):
        return False


class TestZombieReproduction:
    """Demonstrate that subprocesses survive when cleanup is not called."""

    def test_orphaned_processes_survive_without_cleanup(self):
        """REPRODUCTION: processes spawned directly survive if no one kills
        them — this models the gap that causes zombie accumulation when
        the gateway drops agent references without calling close()."""
        pids = []

        try:
            for _ in range(3):
                proc = _spawn_sleep(60)
                pids.append(proc.pid)

            for pid in pids:
                assert _pid_alive(pid), f"PID {pid} should be alive after spawn"

            # Simulate "session end" by just dropping the reference
            del proc  # noqa: F821

            # BUG: processes are still alive after reference is dropped
            for pid in pids:
                assert _pid_alive(pid), (
                    f"PID {pid} died after ref drop — "
                    f"expected it to survive (demonstrating the bug)"
                )
        finally:
            for pid in pids:
                try:
                    os.kill(pid, signal.SIGKILL)
                except (ProcessLookupError, PermissionError):
                    pass

    def test_explicit_terminate_reaps_processes(self):
        """Explicitly terminating+waiting on Popen handles works.
        This models what ProcessRegistry.kill_process does internally."""
        procs = []

        try:
            for _ in range(3):
                proc = _spawn_sleep(60)
                procs.append(proc)

            for proc in procs:
                assert _pid_alive(proc.pid)

            for proc in procs:
                proc.terminate()
                proc.wait(timeout=5)

            for proc in procs:
                assert proc.returncode is not None, (
                    f"PID {proc.pid} should have exited after terminate+wait"
                )
        finally:
            for proc in procs:
                try:
                    proc.kill()
                    proc.wait(timeout=1)
                except Exception:
                    pass


class TestAgentCloseMethod:
    """Verify AIAgent.close() exists, is idempotent, and calls cleanup."""

    def test_close_calls_cleanup_functions(self):
        """close() should release every session-owned execution backend."""
        from unittest.mock import patch

        with patch("run_agent.AIAgent.__init__", return_value=None):
            from run_agent import AIAgent
            agent = AIAgent.__new__(AIAgent)
            agent.session_id = "test-close-cleanup"
            agent._active_children = []
            agent._active_children_lock = threading.Lock()
            agent.client = None

            with patch("tools.process_registry.process_registry") as mock_registry, \
                 patch("run_agent.cleanup_vm") as mock_cleanup_vm, \
                 patch("run_agent.cleanup_browser") as mock_cleanup_browser, \
                 patch("tools.computer_use.release_computer_use_session") as mock_cleanup_cua:
                agent.close()

                mock_registry.kill_all.assert_called_once_with(
                    task_id="test-close-cleanup"
                )
                mock_cleanup_vm.assert_called_once_with("test-close-cleanup")
                mock_cleanup_browser.assert_called_once_with("test-close-cleanup")
                mock_cleanup_cua.assert_called_once_with("test-close-cleanup")

    def test_close_is_idempotent(self):
        """close() can be called multiple times without error."""
        from unittest.mock import patch

        with patch("run_agent.AIAgent.__init__", return_value=None):
            from run_agent import AIAgent
            agent = AIAgent.__new__(AIAgent)
            agent.session_id = "test-close-idempotent"
            agent._active_children = []
            agent._active_children_lock = threading.Lock()
            agent.client = None

            agent.close()
            agent.close()
            agent.close()

    def test_close_releases_computer_use_when_earlier_cleanup_fails(self):
        """One failed cleanup step must not strand the computer-use session."""
        from unittest.mock import patch

        with patch("run_agent.AIAgent.__init__", return_value=None):
            from run_agent import AIAgent
            agent = AIAgent.__new__(AIAgent)
            agent.session_id = "test-close-after-failure"
            agent._active_children = []
            agent._active_children_lock = threading.Lock()
            agent.client = None

            with patch(
                "tools.process_registry.process_registry.kill_all",
                side_effect=RuntimeError("process cleanup failed"),
            ), patch(
                "tools.computer_use.release_computer_use_session",
            ) as mock_cleanup_cua:
                agent.close()

            mock_cleanup_cua.assert_called_once_with(
                "test-close-after-failure"
            )

    def test_soft_client_release_preserves_computer_use_session(self):
        """Cache eviction is not a hard session boundary."""
        from unittest.mock import patch

        with patch("run_agent.AIAgent.__init__", return_value=None):
            from run_agent import AIAgent
            agent = AIAgent.__new__(AIAgent)
            agent.session_id = "test-soft-release"
            agent._active_children = []
            agent._active_children_lock = threading.Lock()
            agent.client = None

            with patch(
                "tools.computer_use.release_computer_use_session",
            ) as mock_cleanup_cua:
                agent.release_clients()

            mock_cleanup_cua.assert_not_called()

    def test_close_propagates_to_children(self):
        """close() should call close() on all active child agents."""
        from unittest.mock import MagicMock, patch

        with patch("run_agent.AIAgent.__init__", return_value=None):
            from run_agent import AIAgent
            agent = AIAgent.__new__(AIAgent)
            agent.session_id = "test-close-children"
            agent._active_children_lock = threading.Lock()
            agent.client = None

            child_1 = MagicMock()
            child_2 = MagicMock()
            agent._active_children = [child_1, child_2]

            agent.close()

            child_1.close.assert_called_once()
            child_2.close.assert_called_once()
            assert agent._active_children == []

    def test_close_ends_owned_session_row(self):
        """close() finalizes the agent's owned SQLite session row."""
        from unittest.mock import MagicMock, patch

        with patch("run_agent.AIAgent.__init__", return_value=None):
            from run_agent import AIAgent
            agent = AIAgent.__new__(AIAgent)
            agent.session_id = "test-close-session-row"
            agent._active_children = []
            agent._active_children_lock = threading.Lock()
            agent.client = None
            agent._end_session_on_close = True
            agent._session_db = MagicMock()

            agent.close()

            agent._session_db.end_session.assert_called_once_with(
                "test-close-session-row", "agent_close"
            )

    def test_close_skips_session_end_for_forwarded_continuation_agents(self):
        """Helper agents that handed session ownership forward opt out."""
        from unittest.mock import MagicMock, patch

        with patch("run_agent.AIAgent.__init__", return_value=None):
            from run_agent import AIAgent
            agent = AIAgent.__new__(AIAgent)
            agent.session_id = "test-close-forwarded-session"
            agent._active_children = []
            agent._active_children_lock = threading.Lock()
            agent.client = None
            agent._end_session_on_close = False
            agent._session_db = MagicMock()

            agent.close()

            agent._session_db.end_session.assert_not_called()

    def test_close_session_end_noops_without_session_db(self):
        """close() is a no-op for session finalization when no DB is wired in."""
        from unittest.mock import patch

        with patch("run_agent.AIAgent.__init__", return_value=None):
            from run_agent import AIAgent
            agent = AIAgent.__new__(AIAgent)
            agent.session_id = "test-close-no-db"
            agent._active_children = []
            agent._active_children_lock = threading.Lock()
            agent.client = None
            # No _session_db / _end_session_on_close attributes at all —
            # getattr defaults must keep close() from raising.
            agent.close()  # must not raise

    def test_close_survives_partial_failures(self):
        """close() continues cleanup even if one step fails."""
        from unittest.mock import patch

        with patch("run_agent.AIAgent.__init__", return_value=None):
            from run_agent import AIAgent
            agent = AIAgent.__new__(AIAgent)
            agent.session_id = "test-close-partial"
            agent._active_children = []
            agent._active_children_lock = threading.Lock()
            agent.client = None

            with patch(
                "tools.process_registry.process_registry"
            ) as mock_reg, patch(
                "run_agent.cleanup_vm"
            ) as mock_vm, patch(
                "run_agent.cleanup_browser"
            ) as mock_browser:
                mock_reg.kill_all.side_effect = RuntimeError("boom")

                agent.close()

                mock_vm.assert_called_once()
                mock_browser.assert_called_once()


class TestGatewayCleanupWiring:
    """Verify gateway lifecycle calls close() on agents."""

    def test_gateway_stop_calls_close(self):
        """gateway stop() should call close() on all running agents."""
        import asyncio
        import threading
        from unittest.mock import MagicMock, patch

        from gateway.run import GatewayRunner

        runner = object.__new__(GatewayRunner)
        runner._running = True
        runner._running_agents = {}
        runner._running_agents_ts = {}
        runner.adapters = {}
        runner._background_tasks = set()
        runner._pending_messages = {}
        runner._pending_approvals = {}
        runner._pending_model_notes = {}
        runner._shutdown_event = asyncio.Event()
        runner._exit_reason = None
        runner._exit_code = None
        runner._stop_task = None
        runner._draining = False
        runner._restart_requested = False
        runner._restart_task_started = False
        runner._restart_detached = False
        runner._restart_via_service = False
        runner._restart_drain_timeout = 0.1
        runner._voice_mode = {}
        runner._session_model_overrides = {}
        runner._update_prompt_pending = {}
        runner._busy_input_mode = "interrupt"
        runner._agent_cache = {}
        runner._agent_cache_lock = threading.Lock()
        runner._shutdown_all_gateway_honcho = lambda: None
        runner._update_runtime_status = MagicMock()

        mock_agent_1 = MagicMock()
        mock_agent_2 = MagicMock()
        runner._running_agents = {
            "session-1": mock_agent_1,
            "session-2": mock_agent_2,
        }

        loop = asyncio.new_event_loop()
        try:
            with patch("gateway.status.remove_pid_file"), \
                 patch("gateway.status.write_runtime_status"), \
                 patch("tools.terminal_tool.cleanup_all_environments"), \
                 patch("tools.browser_tool.cleanup_all_browsers"):
                loop.run_until_complete(GatewayRunner.stop(runner))
        finally:
            loop.close()

        mock_agent_1.close.assert_called()
        mock_agent_2.close.assert_called()

    def test_evict_does_not_call_close(self):
        """_evict_cached_agent() should NOT call close() — it's also used
        for non-destructive refreshes (model switch, branch, fallback)."""
        import threading
        from unittest.mock import MagicMock

        from gateway.run import GatewayRunner

        runner = object.__new__(GatewayRunner)
        runner._agent_cache_lock = threading.Lock()

        mock_agent = MagicMock()
        runner._agent_cache = {"session-key": (mock_agent, 12345)}

        GatewayRunner._evict_cached_agent(runner, "session-key")

        mock_agent.close.assert_not_called()
        assert "session-key" not in runner._agent_cache


class TestDelegationCleanup:
    """Verify subagent delegation cleans up child agents."""

    def test_run_single_child_calls_close(self, monkeypatch, tmp_path):
        """_run_single_child finally block should call close() on child."""
        from unittest.mock import MagicMock
        from hermes_constants import (
            get_hermes_home,
            reset_hermes_home_override,
            set_hermes_home_override,
        )
        from hermes_cli.observability import relay_runtime
        from tools.delegate_tool import _run_single_child

        parent = MagicMock()
        parent._active_children = []
        parent._active_children_lock = threading.Lock()

        child = MagicMock()
        child.session_id = "child-session"
        child._delegate_saved_tool_names = ["tool1"]
        observed = {}

        def run_conversation(**_kwargs):
            observed["hermes_home"] = get_hermes_home()
            raise RuntimeError("test abort")

        child.run_conversation.side_effect = run_conversation
        relay_host = MagicMock()
        monkeypatch.setattr(relay_runtime, "get_runtime", lambda **kwargs: relay_host)

        parent._active_children.append(child)

        profile_home = tmp_path / "profile-a"
        token = set_hermes_home_override(profile_home)
        try:
            result = _run_single_child(
                task_index=0,
                goal="test goal",
                child=child,
                parent_agent=parent,
            )
        finally:
            reset_hermes_home_override(token)

        child.close.assert_called_once()
        assert observed["hermes_home"] == profile_home
        relay_host.unregister_subagent.assert_called_once_with(
            {"child_session_id": "child-session"}
        )
        assert child not in parent._active_children
        assert result["status"] == "error"

    def test_active_child_turn_owns_relay_scope_cleanup(self, monkeypatch):
        from unittest.mock import MagicMock

        from hermes_cli.observability import relay_runtime
        from tools.delegate_tool import _run_single_child

        parent = MagicMock()
        parent._active_children = []
        parent._active_children_lock = threading.Lock()
        child = MagicMock()
        child.session_id = "active-child-session"
        child._delegate_saved_tool_names = ["tool1"]
        child.run_conversation.side_effect = RuntimeError("test abort")
        parent._active_children.append(child)
        relay_host = MagicMock()
        monkeypatch.setattr(relay_runtime, "get_runtime", lambda **kwargs: relay_host)
        monkeypatch.setattr(
            relay_runtime.SESSION_COORDINATOR,
            "has_active_turn",
            lambda **_kwargs: True,
        )

        result = _run_single_child(
            task_index=0,
            goal="test active turn cleanup",
            child=child,
            parent_agent=parent,
        )

        assert result["status"] == "error"
        relay_host.unregister_subagent.assert_not_called()

    def test_timed_out_child_keeps_relay_session_until_its_turn_exits(
        self, monkeypatch, tmp_path
    ):
        from unittest.mock import MagicMock

        from agent import relay_runtime
        from hermes_constants import (
            reset_hermes_home_override,
            set_hermes_home_override,
        )
        from tools.delegate_tool import _run_single_child

        relay_runtime._reset_for_tests()
        profile_home = tmp_path / "profile-timeout"
        profile_token = set_hermes_home_override(profile_home)
        child_started = threading.Event()
        release_child = threading.Event()
        child_finished = threading.Event()
        parent = MagicMock()
        parent._active_children = []
        parent._active_children_lock = threading.Lock()
        child = MagicMock()
        child.session_id = "timed-out-child"
        child._delegate_saved_tool_names = ["tool1"]
        child.get_activity_summary.return_value = {"api_call_count": 1}
        parent._active_children.append(child)
        relay_host = MagicMock()
        monkeypatch.setattr(relay_runtime, "get_runtime", lambda **_kwargs: relay_host)
        monkeypatch.setattr("tools.delegate_tool._get_child_timeout", lambda: 0.1)

        def run_conversation(**kwargs):
            lease = relay_runtime.SESSION_COORDINATOR.acquire_conversation(
                profile_key=relay_runtime.current_profile_key(),
                session_id=child.session_id,
                platform="subagent",
            )
            turn = relay_runtime.SESSION_COORDINATOR.begin_turn(
                lease,
                turn_id="timed-out-child-turn",
                task_id=kwargs["task_id"],
            )
            child_started.set()
            try:
                release_child.wait(timeout=5)
                return {
                    "final_response": "late result",
                    "completed": True,
                    "interrupted": False,
                    "api_calls": 1,
                    "messages": [],
                }
            finally:
                relay_runtime.SESSION_COORDINATOR.end_turn(
                    turn,
                    outcome="cancelled",
                )
                relay_runtime.SESSION_COORDINATOR.release_conversation(lease)
                child_finished.set()

        child.run_conversation.side_effect = run_conversation
        try:
            result = _run_single_child(
                task_index=0,
                goal="test timed-out turn cleanup",
                child=child,
                parent_agent=parent,
            )

            assert child_started.is_set()
            assert result["status"] == "timeout"
            assert relay_runtime.SESSION_COORDINATOR.has_active_turn(
                profile_key=str(profile_home),
                session_id=child.session_id,
            )
            relay_host.unregister_subagent.assert_not_called()

            release_child.set()
            assert child_finished.wait(timeout=5)
            assert not relay_runtime.SESSION_COORDINATOR.has_active_turn(
                profile_key=str(profile_home),
                session_id=child.session_id,
            )
        finally:
            release_child.set()
            reset_hermes_home_override(profile_token)
            relay_runtime._reset_for_tests()