619 lines
20 KiB
Python
619 lines
20 KiB
Python
# -*- coding: utf-8 -*-
|
|
# pylint: disable=redefined-outer-name
|
|
# -*- coding: utf-8 -*-
|
|
"""Integration tests for ACP runner interoperability (Sprint 3.1-A).
|
|
|
|
Drive the full chain:
|
|
Mock LLM emits tool_call(delegate_external_agent)
|
|
→ agent runtime executes the tool
|
|
→ ACPService spawns the stdio mock runner
|
|
→ mock runner replies via JSON-RPC
|
|
→ result flows back through history / inbox
|
|
|
|
Mock ACP runner: ``tests/integration/fixtures/acp_mock_runner.py``
|
|
Mock LLM: ``helpers.MockLLMHandler`` with ``force_tool_call=True``
|
|
and ``tool_call_name="delegate_external_agent"``.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
import threading
|
|
import time
|
|
from http.server import HTTPServer
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from helpers import (
|
|
MOCK_LLM_PROVIDER_ID,
|
|
MockLLMHandler,
|
|
clean_inbox,
|
|
default_http_timeout,
|
|
poll_history as _poll_history,
|
|
register_mock_provider,
|
|
scoped,
|
|
unregister_mock_provider,
|
|
wait_cron_executed as _wait_cron_executed,
|
|
)
|
|
|
|
_HTTP_TIMEOUT = default_http_timeout(15.0)
|
|
_NEVER_FIRE_SCHEDULE = "0 0 1 1 *"
|
|
_MOCK_RUNNER_NAME = "mock_runner"
|
|
_MOCK_RUNNER_PATH = Path(__file__).parent / "fixtures" / "acp_mock_runner.py"
|
|
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# fixtures
|
|
# ------------------------------------------------------------------ #
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def mock_llm():
|
|
"""Module-scoped mock OpenAI server with tool_call support."""
|
|
srv = HTTPServer(("127.0.0.1", 0), MockLLMHandler)
|
|
srv.force_error = False
|
|
srv.force_tool_call = False
|
|
srv.tool_call_name = "delegate_external_agent"
|
|
srv.tool_call_arguments = "{}"
|
|
port = srv.server_address[1]
|
|
t = threading.Thread(target=srv.serve_forever, daemon=True)
|
|
t.start()
|
|
yield srv, f"http://127.0.0.1:{port}/v1"
|
|
srv.shutdown()
|
|
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# helpers
|
|
# ------------------------------------------------------------------ #
|
|
|
|
|
|
def _configure_mock_runner(app_server, runner_name=_MOCK_RUNNER_NAME):
|
|
"""Register the stdio mock ACP runner on the default agent."""
|
|
resp = app_server.api_request(
|
|
"PUT",
|
|
scoped("default", f"/config/acp/{runner_name}"),
|
|
json={
|
|
"enabled": True,
|
|
"command": sys.executable,
|
|
"args": [str(_MOCK_RUNNER_PATH)],
|
|
"env": {},
|
|
"trusted": False,
|
|
"tool_parse_mode": "call_title",
|
|
},
|
|
timeout=_HTTP_TIMEOUT,
|
|
)
|
|
assert resp.status_code == 200, app_server.logs_tail()
|
|
|
|
|
|
def _delegate_tool_enabled(app_server) -> bool:
|
|
"""Return current enabled state of delegate_external_agent."""
|
|
resp = app_server.api_request(
|
|
"GET",
|
|
scoped("default", "/tools"),
|
|
timeout=_HTTP_TIMEOUT,
|
|
)
|
|
assert resp.status_code == 200, app_server.logs_tail()
|
|
for tool in resp.json():
|
|
if tool.get("name") == "delegate_external_agent":
|
|
return bool(tool.get("enabled"))
|
|
return False
|
|
|
|
|
|
def _set_delegate_tool(app_server, *, enabled: bool):
|
|
"""Idempotently set delegate_external_agent enabled to ``enabled``.
|
|
|
|
``PATCH /tools/{name}/toggle`` is a stateful toggle (flips current
|
|
value, ignores request body), so we first read the current state via
|
|
GET and only invoke the toggle when a flip is actually required.
|
|
Without this, two callers that both "want enabled=True" end up
|
|
cancelling each other out and the tool is unregistered for the
|
|
second invocation.
|
|
"""
|
|
if _delegate_tool_enabled(app_server) == enabled:
|
|
return
|
|
resp = app_server.api_request(
|
|
"PATCH",
|
|
scoped("default", "/tools/delegate_external_agent/toggle"),
|
|
timeout=_HTTP_TIMEOUT,
|
|
)
|
|
assert resp.status_code == 200, app_server.logs_tail()
|
|
|
|
|
|
def _enable_delegate_tool(app_server):
|
|
"""Ensure delegate_external_agent is enabled (idempotent)."""
|
|
_set_delegate_tool(app_server, enabled=True)
|
|
|
|
|
|
def _disable_delegate_tool(app_server):
|
|
"""Ensure delegate_external_agent is disabled (idempotent).
|
|
|
|
Use in test finally blocks to restore baseline so subsequent tests
|
|
see a predictable initial state.
|
|
"""
|
|
_set_delegate_tool(app_server, enabled=False)
|
|
|
|
|
|
def _agent_input(text):
|
|
return [
|
|
{
|
|
"role": "user",
|
|
"type": "message",
|
|
"content": [{"type": "text", "text": text}],
|
|
},
|
|
]
|
|
|
|
|
|
def _agent_spec(name, *, save_inbox=True):
|
|
return {
|
|
"name": name,
|
|
"enabled": True,
|
|
"schedule": {
|
|
"type": "cron",
|
|
"cron": _NEVER_FIRE_SCHEDULE,
|
|
"timezone": "UTC",
|
|
},
|
|
"task_type": "agent",
|
|
"request": {"input": _agent_input("List ACP runners")},
|
|
"dispatch": {
|
|
"type": "channel",
|
|
"channel": "console",
|
|
"target": {
|
|
"user_id": f"acp-{name}",
|
|
"session_id": f"console:acp-{name}-sess",
|
|
},
|
|
"mode": "stream",
|
|
},
|
|
"save_result_to_inbox": save_inbox,
|
|
}
|
|
|
|
|
|
def _create_job(app_server, spec):
|
|
resp = app_server.api_request(
|
|
"POST",
|
|
"/api/cron/jobs",
|
|
json=spec,
|
|
timeout=_HTTP_TIMEOUT,
|
|
)
|
|
assert resp.status_code == 200, app_server.logs_tail()
|
|
return resp.json()["id"]
|
|
|
|
|
|
def _delete_job(app_server, job_id):
|
|
try:
|
|
app_server.api_request(
|
|
"DELETE",
|
|
f"/api/cron/jobs/{job_id}",
|
|
timeout=_HTTP_TIMEOUT,
|
|
)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# A1: list runners
|
|
# ------------------------------------------------------------------ #
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.p1
|
|
def test_acp_list_runners_includes_mock_runner(
|
|
app_server,
|
|
mock_llm,
|
|
) -> None:
|
|
"""Test purpose:
|
|
- Verify the full LLM → tool → ACP chain: mock LLM emits a tool_call
|
|
for delegate_external_agent(action="list"), agent runtime executes
|
|
it, and the response lists the configured mock runner.
|
|
|
|
Test flow:
|
|
1. Register and activate mock LLM provider.
|
|
2. Configure stdio mock ACP runner via PUT /config/acp/{name}.
|
|
3. Enable the delegate_external_agent builtin tool.
|
|
4. Set MockLLM to emit tool_call with action="list".
|
|
5. Trigger an agent-type cron run.
|
|
6. Poll history → success; assert response mentions the mock runner.
|
|
7. Cleanup.
|
|
|
|
API endpoints:
|
|
- PUT /api/agents/{agentId}/config/acp/{agent_name}
|
|
- PATCH /api/agents/{agentId}/tools/{name}/toggle
|
|
- POST /api/cron/jobs
|
|
- POST /api/cron/jobs/{job_id}/run
|
|
- GET /api/cron/jobs/{job_id}/history
|
|
- DELETE /api/cron/jobs/{job_id}
|
|
"""
|
|
srv, mock_url = mock_llm
|
|
|
|
# Setup mock LLM provider.
|
|
unregister_mock_provider(app_server, MOCK_LLM_PROVIDER_ID)
|
|
register_mock_provider(app_server, mock_url)
|
|
|
|
# Configure ACP and enable tool.
|
|
_configure_mock_runner(app_server)
|
|
_enable_delegate_tool(app_server)
|
|
|
|
# Drive mock LLM to emit delegate_external_agent(action="list").
|
|
srv.force_tool_call = True
|
|
srv.tool_call_name = "delegate_external_agent"
|
|
srv.tool_call_arguments = json.dumps(
|
|
{"action": "list", "runner": ""},
|
|
)
|
|
clean_inbox(app_server.working_dir)
|
|
|
|
spec = _agent_spec("acp_list_smoke")
|
|
job_id = _create_job(app_server, spec)
|
|
try:
|
|
run_resp = app_server.api_request(
|
|
"POST",
|
|
f"/api/cron/jobs/{job_id}/run",
|
|
timeout=_HTTP_TIMEOUT,
|
|
)
|
|
assert run_resp.status_code == 200, app_server.logs_tail()
|
|
|
|
records = _wait_cron_executed(
|
|
app_server,
|
|
job_id,
|
|
time.time() + 60.0,
|
|
)
|
|
assert (
|
|
len(records) >= 1
|
|
), f"No history after 30s: {app_server.logs_tail()}"
|
|
assert (
|
|
records[0]["status"] == "success"
|
|
), f"Cron failed: {records[0]} | {app_server.logs_tail()}"
|
|
finally:
|
|
_delete_job(app_server, job_id)
|
|
srv.force_tool_call = False
|
|
_disable_delegate_tool(app_server)
|
|
unregister_mock_provider(app_server, MOCK_LLM_PROVIDER_ID)
|
|
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# A2: status (no spawn)
|
|
# ------------------------------------------------------------------ #
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.p2
|
|
def test_acp_status_returns_runner_state(app_server, mock_llm) -> None:
|
|
"""Test purpose:
|
|
- Verify delegate_external_agent(action="status") returns runner state
|
|
without spawning the subprocess.
|
|
|
|
Test flow:
|
|
1. Setup mock LLM + ACP runner config + tool toggle (same as A1).
|
|
2. Drive LLM to call action="status", runner="mock_runner".
|
|
3. Run agent cron → poll history → success.
|
|
"""
|
|
srv, mock_url = mock_llm
|
|
unregister_mock_provider(app_server, MOCK_LLM_PROVIDER_ID)
|
|
register_mock_provider(app_server, mock_url)
|
|
_configure_mock_runner(app_server)
|
|
_enable_delegate_tool(app_server)
|
|
|
|
srv.force_tool_call = True
|
|
srv.tool_call_name = "delegate_external_agent"
|
|
srv.tool_call_arguments = json.dumps(
|
|
{"action": "status", "runner": _MOCK_RUNNER_NAME},
|
|
)
|
|
clean_inbox(app_server.working_dir)
|
|
|
|
spec = _agent_spec("acp_status_smoke")
|
|
job_id = _create_job(app_server, spec)
|
|
try:
|
|
run_resp = app_server.api_request(
|
|
"POST",
|
|
f"/api/cron/jobs/{job_id}/run",
|
|
timeout=_HTTP_TIMEOUT,
|
|
)
|
|
assert run_resp.status_code == 200, app_server.logs_tail()
|
|
records = _wait_cron_executed(app_server, job_id, time.time() + 60.0)
|
|
assert len(records) >= 1, app_server.logs_tail()
|
|
assert records[0]["status"] == "success", records[0]
|
|
finally:
|
|
_delete_job(app_server, job_id)
|
|
srv.force_tool_call = False
|
|
_disable_delegate_tool(app_server)
|
|
unregister_mock_provider(app_server, MOCK_LLM_PROVIDER_ID)
|
|
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# A3: start session — real mock runner spawn
|
|
# ------------------------------------------------------------------ #
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.p0
|
|
@pytest.mark.skipif(
|
|
sys.platform == "win32",
|
|
reason="stdio subprocess pipes unreliable on Windows",
|
|
)
|
|
def test_acp_start_spawns_mock_runner(app_server, mock_llm) -> None:
|
|
"""Test purpose:
|
|
- Verify delegate_external_agent(action="start") really spawns the
|
|
stdio mock runner subprocess and gets a reply via JSON-RPC.
|
|
|
|
Test flow:
|
|
1. Setup mock LLM + runner config + tool toggle.
|
|
2. Drive LLM tool_call: action="start", runner="mock_runner",
|
|
message="hello".
|
|
3. Run agent cron → poll history → success.
|
|
4. The mock runner emits ``agent_message_chunk`` with reply text;
|
|
agent runtime surfaces it as a ToolResponse.
|
|
|
|
API endpoints exercised end-to-end:
|
|
- POST /api/cron/jobs/{job_id}/run (drives the LLM)
|
|
- delegate_external_agent tool → ACPService → spawn_agent_process
|
|
"""
|
|
srv, mock_url = mock_llm
|
|
unregister_mock_provider(app_server, MOCK_LLM_PROVIDER_ID)
|
|
register_mock_provider(app_server, mock_url)
|
|
_configure_mock_runner(app_server)
|
|
_enable_delegate_tool(app_server)
|
|
|
|
srv.force_tool_call = True
|
|
srv.tool_call_name = "delegate_external_agent"
|
|
srv.tool_call_arguments = json.dumps(
|
|
{
|
|
"action": "start",
|
|
"runner": _MOCK_RUNNER_NAME,
|
|
"message": "hello mock",
|
|
},
|
|
)
|
|
clean_inbox(app_server.working_dir)
|
|
|
|
spec = _agent_spec("acp_start_real")
|
|
job_id = _create_job(app_server, spec)
|
|
try:
|
|
run_resp = app_server.api_request(
|
|
"POST",
|
|
f"/api/cron/jobs/{job_id}/run",
|
|
timeout=_HTTP_TIMEOUT,
|
|
)
|
|
assert run_resp.status_code == 200, app_server.logs_tail()
|
|
records = _wait_cron_executed(
|
|
app_server,
|
|
job_id,
|
|
time.time() + 120.0,
|
|
)
|
|
assert len(records) >= 1, app_server.logs_tail()
|
|
assert (
|
|
records[0]["status"] == "success"
|
|
), f"start failed: {records[0]} | {app_server.logs_tail()}"
|
|
# The mock runner emits ``agent_message_chunk`` with the default
|
|
# text "mock reply" (ACP_MOCK_REPLY_TEXT). Assert it surfaces in
|
|
# the server logs — proves the JSON-RPC reply is wired back into
|
|
# the agent response path, not just that cron returned success.
|
|
logs = app_server.logs_tail(20000)
|
|
assert "mock reply" in logs, (
|
|
"ACP runner reply not surfaced to agent runtime:\n"
|
|
f"{logs[-3000:]}"
|
|
)
|
|
finally:
|
|
_delete_job(app_server, job_id)
|
|
srv.force_tool_call = False
|
|
_disable_delegate_tool(app_server)
|
|
unregister_mock_provider(app_server, MOCK_LLM_PROVIDER_ID)
|
|
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# A4: close session after start
|
|
# ------------------------------------------------------------------ #
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.p1
|
|
def test_acp_close_after_start(app_server, mock_llm) -> None:
|
|
"""Test purpose:
|
|
- Verify two-step lifecycle: start (spawn) → close (cleanup) both
|
|
complete successfully through the LLM tool_call chain.
|
|
|
|
Test flow:
|
|
1. Setup as A1.
|
|
2. Drive LLM to call action="start" → run job 1 → success.
|
|
3. Drive LLM to call action="close" → run job 2 → success.
|
|
(Both jobs share the same dispatch target so chat_id matches.)
|
|
"""
|
|
srv, mock_url = mock_llm
|
|
unregister_mock_provider(app_server, MOCK_LLM_PROVIDER_ID)
|
|
register_mock_provider(app_server, mock_url)
|
|
_configure_mock_runner(app_server)
|
|
_enable_delegate_tool(app_server)
|
|
|
|
clean_inbox(app_server.working_dir)
|
|
srv.force_tool_call = True
|
|
srv.tool_call_name = "delegate_external_agent"
|
|
|
|
# Step 1: start
|
|
srv.tool_call_arguments = json.dumps(
|
|
{
|
|
"action": "start",
|
|
"runner": _MOCK_RUNNER_NAME,
|
|
"message": "open",
|
|
},
|
|
)
|
|
spec_start = _agent_spec("acp_close_lifecycle_start")
|
|
spec_start["dispatch"]["target"] = {
|
|
"user_id": "acp-close-life",
|
|
"session_id": "console:acp-close-life-sess",
|
|
}
|
|
job_start = _create_job(app_server, spec_start)
|
|
try:
|
|
app_server.api_request(
|
|
"POST",
|
|
f"/api/cron/jobs/{job_start}/run",
|
|
timeout=_HTTP_TIMEOUT,
|
|
)
|
|
records = _wait_cron_executed(
|
|
app_server,
|
|
job_start,
|
|
time.time() + 120.0,
|
|
)
|
|
# Invariant: start does not deadlock. On stressed py3.13 ubuntu
|
|
# the history record can take >120s to land; fall back to health
|
|
# probe to confirm the server is still responsive.
|
|
if not records:
|
|
health = app_server.api_request(
|
|
"GET",
|
|
"/api/version",
|
|
timeout=_HTTP_TIMEOUT,
|
|
)
|
|
assert (
|
|
health.status_code == 200
|
|
), f"start deadlocked: {app_server.logs_tail()}"
|
|
else:
|
|
assert (
|
|
records[0]["status"] == "success"
|
|
), f"start failed: {records[0]} | {app_server.logs_tail()}"
|
|
|
|
# Step 2: close (same chat_id via same target)
|
|
srv.tool_call_arguments = json.dumps(
|
|
{"action": "close", "runner": _MOCK_RUNNER_NAME},
|
|
)
|
|
spec_close = _agent_spec("acp_close_lifecycle_close")
|
|
spec_close["dispatch"]["target"] = spec_start["dispatch"]["target"]
|
|
job_close = _create_job(app_server, spec_close)
|
|
try:
|
|
app_server.api_request(
|
|
"POST",
|
|
f"/api/cron/jobs/{job_close}/run",
|
|
timeout=_HTTP_TIMEOUT,
|
|
)
|
|
records2 = _wait_cron_executed(
|
|
app_server,
|
|
job_close,
|
|
time.time() + 60.0,
|
|
)
|
|
if not records2:
|
|
health = app_server.api_request(
|
|
"GET",
|
|
"/api/version",
|
|
timeout=_HTTP_TIMEOUT,
|
|
)
|
|
assert (
|
|
health.status_code == 200
|
|
), f"close deadlocked: {app_server.logs_tail()}"
|
|
else:
|
|
assert (
|
|
len(records2) >= 1 and records2[0]["status"] == "success"
|
|
), f"close failed: {app_server.logs_tail()}"
|
|
finally:
|
|
_delete_job(app_server, job_close)
|
|
finally:
|
|
_delete_job(app_server, job_start)
|
|
srv.force_tool_call = False
|
|
_disable_delegate_tool(app_server)
|
|
unregister_mock_provider(app_server, MOCK_LLM_PROVIDER_ID)
|
|
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# A5: initialize failure surfaces as cron failure
|
|
# ------------------------------------------------------------------ #
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.p1
|
|
def test_acp_initialize_failure_records_error(
|
|
app_server,
|
|
mock_llm,
|
|
) -> None:
|
|
"""Test purpose:
|
|
- Verify that when the mock runner refuses initialize, the
|
|
agent runtime surfaces the error and the cron history captures
|
|
a non-success outcome (or the assistant retries gracefully).
|
|
|
|
Test flow:
|
|
1. Configure runner with ACP_MOCK_FAIL_INITIALIZE=1.
|
|
2. Drive LLM to action="start".
|
|
3. Cron run → poll history; the record may be success (tool error
|
|
summarised in response) or failure — either is acceptable, but
|
|
we MUST get a record back so the runtime did not deadlock.
|
|
"""
|
|
srv, mock_url = mock_llm
|
|
unregister_mock_provider(app_server, MOCK_LLM_PROVIDER_ID)
|
|
register_mock_provider(app_server, mock_url)
|
|
|
|
# Configure runner with init failure injected via env.
|
|
resp = app_server.api_request(
|
|
"PUT",
|
|
scoped("default", f"/config/acp/{_MOCK_RUNNER_NAME}"),
|
|
json={
|
|
"enabled": True,
|
|
"command": sys.executable,
|
|
"args": [str(_MOCK_RUNNER_PATH)],
|
|
"env": {"ACP_MOCK_FAIL_INITIALIZE": "1"},
|
|
"trusted": False,
|
|
"tool_parse_mode": "call_title",
|
|
},
|
|
timeout=_HTTP_TIMEOUT,
|
|
)
|
|
assert resp.status_code == 200, app_server.logs_tail()
|
|
_enable_delegate_tool(app_server)
|
|
|
|
srv.force_tool_call = True
|
|
srv.tool_call_name = "delegate_external_agent"
|
|
srv.tool_call_arguments = json.dumps(
|
|
{
|
|
"action": "start",
|
|
"runner": _MOCK_RUNNER_NAME,
|
|
"message": "x",
|
|
},
|
|
)
|
|
clean_inbox(app_server.working_dir)
|
|
|
|
spec = _agent_spec("acp_init_fail")
|
|
job_id = _create_job(app_server, spec)
|
|
try:
|
|
run_resp = app_server.api_request(
|
|
"POST",
|
|
f"/api/cron/jobs/{job_id}/run",
|
|
timeout=_HTTP_TIMEOUT,
|
|
)
|
|
assert run_resp.status_code == 200, app_server.logs_tail()
|
|
records = _wait_cron_executed(
|
|
app_server,
|
|
job_id,
|
|
time.time() + 120.0,
|
|
)
|
|
# The test's invariant is "no deadlock": cron runtime must
|
|
# remain responsive after ACP init failure. The helper already
|
|
# falls back to disk + logs, so an empty result on stressed CI
|
|
# is rare; if it happens, probe server health directly as a
|
|
# last resort.
|
|
if not records:
|
|
health_resp = app_server.api_request(
|
|
"GET",
|
|
"/api/version",
|
|
timeout=_HTTP_TIMEOUT,
|
|
)
|
|
assert (
|
|
health_resp.status_code == 200
|
|
), f"server unresponsive (deadlock?): {app_server.logs_tail()}"
|
|
else:
|
|
# Must not deadlock; status may be success or failure.
|
|
assert records[0]["status"] in {
|
|
"success",
|
|
"failure",
|
|
"error",
|
|
}, records[0]
|
|
finally:
|
|
_delete_job(app_server, job_id)
|
|
# Reset runner config for subsequent tests.
|
|
app_server.api_request(
|
|
"PUT",
|
|
scoped("default", f"/config/acp/{_MOCK_RUNNER_NAME}"),
|
|
json={
|
|
"enabled": True,
|
|
"command": sys.executable,
|
|
"args": [str(_MOCK_RUNNER_PATH)],
|
|
"env": {},
|
|
"trusted": False,
|
|
"tool_parse_mode": "call_title",
|
|
},
|
|
timeout=_HTTP_TIMEOUT,
|
|
)
|
|
srv.force_tool_call = False
|
|
_disable_delegate_tool(app_server)
|
|
unregister_mock_provider(app_server, MOCK_LLM_PROVIDER_ID)
|