NeoAnd/adk-python
1
0
Fork 0
mirror of https://github.com/EvolutionAPI/adk-python.git synced 2025-07-14 01:41:25 -06:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix eval unittests failure

Browse Source 
PiperOrigin-RevId: 757872670
This commit is contained in:
Xiang (Sean) Zhou 2025-05-12 12:41:44 -07:00 committed by Copybara-Service
parent c39f24f072
commit 5462862795
2 changed files with 233 additions and 183 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
src/google/adk/cli/cli_tools_click.py
View File

@ -269,25 +269,20 @@ def cli_eval(
eval_set_to_evals = parse_and_get_evals_to_run(eval_set_file_path)
async def _collect_async_gen(
async_gen_coroutine: Coroutine[
AsyncGenerator[EvalResult, None], None, None
],
) -> list[EvalResult]:
return [result async for result in async_gen_coroutine]
try:
eval_results = asyncio.run(
_collect_async_gen(
run_evals(
async def _collect_eval_results() -> list[EvalResult]:
return [
result
async for result in run_evals(
eval_set_to_evals,
root_agent,
reset_func,
eval_metrics,
print_detailed_results=print_detailed_results,
)
)
)
]
try:
eval_results = asyncio.run(_collect_eval_results())
except ModuleNotFoundError:
raise click.ClickException(MISSING_EVAL_DEPENDENCIES_MESSAGE)

111
tests/unittests/cli/utils/test_cli_tools_click.py
View File

@ -18,15 +18,18 @@
from __future__ import annotations
import builtins
from pathlib import Path
from types import SimpleNamespace
from typing import Any
from typing import Dict
from typing import List
from typing import Tuple
import click
from click.testing import CliRunner
from google.adk.cli import cli_tools_click
import pytest
from google.adk.cli import cli_tools_click
from pathlib import Path
from typing import Any, Dict, List, Tuple
from types import SimpleNamespace
from click.testing import CliRunner
# Helpers
class _Recorder:
@ -52,7 +55,9 @@ def test_validate_exclusive_allows_single() -> None:
"""Providing exactly one exclusive option should pass."""
ctx = click.Context(cli_tools_click.main)
param = SimpleNamespace(name="replay")
assert cli_tools_click.validate_exclusive(ctx, param, "file.json") == "file.json"
assert (
cli_tools_click.validate_exclusive(ctx, param, "file.json") == "file.json"
)
def test_validate_exclusive_blocks_multiple() -> None:
@ -70,7 +75,9 @@ def test_validate_exclusive_blocks_multiple() -> None:
# cli create
def test_cli_create_cmd_invokes_run_cmd(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
def test_cli_create_cmd_invokes_run_cmd(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""`adk create` should forward arguments to cli_create.run_cmd."""
rec = _Recorder()
monkeypatch.setattr(cli_tools_click.cli_create, "run_cmd", rec)
@ -87,11 +94,15 @@ def test_cli_create_cmd_invokes_run_cmd(tmp_path: Path, monkeypatch: pytest.Monk
# cli run
@pytest.mark.asyncio
async def test_cli_run_invokes_run_cli(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
async def test_cli_run_invokes_run_cli(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""`adk run` should call run_cli via asyncio.run with correct parameters."""
rec = _Recorder()
monkeypatch.setattr(cli_tools_click, "run_cli", lambda **kwargs: rec(kwargs))
monkeypatch.setattr(cli_tools_click.asyncio, "run", lambda coro: coro) # pass-through
monkeypatch.setattr(
cli_tools_click.asyncio, "run", lambda coro: coro
) # pass-through
# create dummy agent directory
agent_dir = tmp_path / "agent"
@ -106,7 +117,9 @@ async def test_cli_run_invokes_run_cli(tmp_path: Path, monkeypatch: pytest.Monke
# cli deploy cloud_run
def test_cli_deploy_cloud_run_success(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
def test_cli_deploy_cloud_run_success(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Successful path should call cli_deploy.to_cloud_run once."""
rec = _Recorder()
monkeypatch.setattr(cli_tools_click.cli_deploy, "to_cloud_run", rec)
@ -130,8 +143,11 @@ def test_cli_deploy_cloud_run_success(tmp_path: Path, monkeypatch: pytest.Monkey
assert rec.calls, "cli_deploy.to_cloud_run must be invoked"
def test_cli_deploy_cloud_run_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
def test_cli_deploy_cloud_run_failure(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Exception from to_cloud_run should be caught and surfaced via click.secho."""
def _boom(*_a: Any, **_k: Any) -> None: # noqa: D401
raise RuntimeError("boom")
@ -144,14 +160,18 @@ def test_cli_deploy_cloud_run_failure(tmp_path: Path, monkeypatch: pytest.Monkey
agent_dir = tmp_path / "agent3"
agent_dir.mkdir()
runner = CliRunner()
result = runner.invoke(cli_tools_click.main, ["deploy", "cloud_run", str(agent_dir)])
result = runner.invoke(
cli_tools_click.main, ["deploy", "cloud_run", str(agent_dir)]
)
assert result.exit_code == 0
assert any("Deploy failed: boom" in m for m in captured)
# cli eval
def test_cli_eval_missing_deps_raises(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
def test_cli_eval_missing_deps_raises(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""If cli_eval sub-module is missing, command should raise ClickException."""
# Ensure .cli_eval is not importable
orig_import = builtins.__import__
@ -171,17 +191,28 @@ def _patch_uvicorn(monkeypatch: pytest.MonkeyPatch) -> _Recorder:
rec = _Recorder()
class _DummyServer:
def __init__(self, *a: Any, **k: Any) -> None: ...
def __init__(self, *a: Any, **k: Any) -> None:
...
def run(self) -> None:
rec()
monkeypatch.setattr(cli_tools_click.uvicorn, "Config", lambda *a, **k: object())
monkeypatch.setattr(cli_tools_click.uvicorn, "Server", lambda *_a, **_k: _DummyServer())
monkeypatch.setattr(cli_tools_click, "get_fast_api_app", lambda **_k: object())
monkeypatch.setattr(
cli_tools_click.uvicorn, "Config", lambda *a, **k: object()
)
monkeypatch.setattr(
cli_tools_click.uvicorn, "Server", lambda *_a, **_k: _DummyServer()
)
monkeypatch.setattr(
cli_tools_click, "get_fast_api_app", lambda **_k: object()
)
return rec
def test_cli_web_invokes_uvicorn(tmp_path: Path, _patch_uvicorn: _Recorder) -> None:
def test_cli_web_invokes_uvicorn(
tmp_path: Path, _patch_uvicorn: _Recorder
) -> None:
"""`adk web` should configure and start uvicorn.Server.run."""
agents_dir = tmp_path / "agents"
agents_dir.mkdir()
@ -191,7 +222,9 @@ def test_cli_web_invokes_uvicorn(tmp_path: Path, _patch_uvicorn: _Recorder) -> N
assert _patch_uvicorn.calls, "uvicorn.Server.run must be called"
def test_cli_api_server_invokes_uvicorn(tmp_path: Path, _patch_uvicorn: _Recorder) -> None:
def test_cli_api_server_invokes_uvicorn(
tmp_path: Path, _patch_uvicorn: _Recorder
) -> None:
"""`adk api_server` should configure and start uvicorn.Server.run."""
agents_dir = tmp_path / "agents_api"
agents_dir.mkdir()
@ -201,17 +234,24 @@ def test_cli_api_server_invokes_uvicorn(tmp_path: Path, _patch_uvicorn: _Recorde
assert _patch_uvicorn.calls, "uvicorn.Server.run must be called"
def test_cli_eval_success_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
def test_cli_eval_success_path(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Test the success path of `adk eval` by fully executing it with a stub module, up to summary generation."""
import sys, types
import asyncio
import sys
import types
# stub cli_eval module
stub = types.ModuleType("google.adk.cli.cli_eval")
class _EvalMetric:
def __init__(self, metric_name: str, threshold: float) -> None: ...
def __init__(self, metric_name: str, threshold: float) -> None:
...
class _EvalResult:
def __init__(self, eval_set_file: str, final_eval_status: str) -> None:
self.eval_set_file = eval_set_file
self.final_eval_status = final_eval_status
@ -229,11 +269,24 @@ def test_cli_eval_success_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch)
stub.get_root_agent = lambda _p: object()
stub.try_get_reset_func = lambda _p: None
stub.parse_and_get_evals_to_run = lambda _paths: {"set1.json": ["e1", "e2"]}
stub.run_evals = lambda *_a, **_k: iter(
[_EvalResult("set1.json", "PASSED"), _EvalResult("set1.json", "FAILED")]
)
monkeypatch.setattr(cli_tools_click.asyncio, "run", lambda coro: list(coro))
# Create an async generator function for run_evals
async def mock_run_evals(*_a, **_k):
yield _EvalResult("set1.json", "PASSED")
yield _EvalResult("set1.json", "FAILED")
stub.run_evals = mock_run_evals
# Replace asyncio.run with a function that properly handles coroutines
def mock_asyncio_run(coro):
# Create a new event loop
loop = asyncio.new_event_loop()
try:
return loop.run_until_complete(coro)
finally:
loop.close()
monkeypatch.setattr(cli_tools_click.asyncio, "run", mock_asyncio_run)
# inject stub
sys.modules["google.adk.cli.cli_eval"] = stub
@ -244,7 +297,9 @@ def test_cli_eval_success_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch)
(agent_dir / "__init__.py").touch()
# inject monkeypatch
monkeypatch.setattr(cli_tools_click.envs, "load_dotenv_for_agent", lambda *a, **k: None)
monkeypatch.setattr(
cli_tools_click.envs, "load_dotenv_for_agent", lambda *a, **k: None
)
runner = CliRunner()
result = runner.invoke(