Add browser dashboard, run tracing, and logged navigation decisions
- dashboard.py: stdlib http.server + JSON API (tree/trace/dream/status, POST submit/relabel); static/ single-page Redux-DevTools-style inspector - trace.py: runs.jsonl lifecycle events keyed by run_id - scheduler/skills/skill: every decision carries run_id; navigation choices now logged (navigate:category/leaf); requeues stamp meta.parent_run - cli: 'dashboard' subcommand; config.json untracked per-machine (see config.example.json)
This commit is contained in:
@@ -0,0 +1,148 @@
|
||||
"""Dashboard API tests against a real stdlib HTTP server on an ephemeral port.
|
||||
|
||||
The scheduler is constructed with the lazy SemIfEngine (never loaded), so this
|
||||
runs anywhere without SemIf. Submit degrades to a JSON error, which is the
|
||||
expected behaviour on the thin dev box; relabel works against a seeded log.
|
||||
"""
|
||||
|
||||
import json
|
||||
import threading
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from http.server import ThreadingHTTPServer
|
||||
|
||||
from semif_agent.dashboard import DashboardHandler
|
||||
from semif_agent.decisions import DecisionRequest, DecisionResult, Option
|
||||
from semif_agent.engine import EngineConfig, SemIfEngine
|
||||
from semif_agent.llm import LLMClient
|
||||
from semif_agent.log import DecisionLog
|
||||
from semif_agent.scheduler import Scheduler
|
||||
from semif_agent.trace import TraceLog
|
||||
|
||||
|
||||
def build_scheduler(tmp_path):
|
||||
log = DecisionLog(str(tmp_path / "decisions.jsonl"))
|
||||
trace = TraceLog(str(tmp_path / "runs.jsonl"))
|
||||
engine = SemIfEngine(EngineConfig())
|
||||
llm = LLMClient(base_url="http://localhost:1/v1", model="test")
|
||||
scheduler = Scheduler(
|
||||
engine=engine,
|
||||
llm=llm,
|
||||
log=log,
|
||||
config={"skills": {}},
|
||||
trace=trace,
|
||||
)
|
||||
return scheduler
|
||||
|
||||
|
||||
def seed_decision(log, decision_id="abc123"):
|
||||
request = DecisionRequest(
|
||||
id=decision_id,
|
||||
state="some state",
|
||||
question="Pick one?",
|
||||
options=[Option("a", "A."), Option("b", "B.")],
|
||||
)
|
||||
result = DecisionResult(
|
||||
request=request, option_ids=["a", "b"], probabilities=[0.3, 0.7]
|
||||
)
|
||||
log.append(request, result, extra={"phase": "gate", "run_id": "run-1"})
|
||||
|
||||
|
||||
class Server:
|
||||
def __init__(self, scheduler):
|
||||
handler = type("Handler", (DashboardHandler,), {"scheduler": scheduler})
|
||||
self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), handler)
|
||||
self.thread = threading.Thread(target=self.httpd.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
self.port = self.httpd.server_address[1]
|
||||
|
||||
def get(self, path):
|
||||
with urllib.request.urlopen(f"http://127.0.0.1:{self.port}{path}") as res:
|
||||
return res.status, json.loads(res.read().decode("utf-8"))
|
||||
|
||||
def post(self, path, payload):
|
||||
req = urllib.request.Request(
|
||||
f"http://127.0.0.1:{self.port}{path}",
|
||||
data=json.dumps(payload).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req) as res:
|
||||
return res.status, json.loads(res.read().decode("utf-8"))
|
||||
|
||||
def close(self):
|
||||
self.httpd.shutdown()
|
||||
self.httpd.server_close()
|
||||
|
||||
|
||||
def test_tree_endpoint(tmp_path):
|
||||
server = Server(build_scheduler(tmp_path))
|
||||
try:
|
||||
status, payload = server.get("/api/tree")
|
||||
assert status == 200
|
||||
assert "email" in payload["categories"]
|
||||
assert any(s["name"] == "email.compose" for s in payload["categories"]["email"])
|
||||
finally:
|
||||
server.close()
|
||||
|
||||
|
||||
def test_trace_endpoint_empty(tmp_path):
|
||||
server = Server(build_scheduler(tmp_path))
|
||||
try:
|
||||
status, payload = server.get("/api/trace")
|
||||
assert status == 200
|
||||
assert payload["runs"] == []
|
||||
finally:
|
||||
server.close()
|
||||
|
||||
|
||||
def test_submit_without_engine_returns_error_json(tmp_path):
|
||||
scheduler = build_scheduler(tmp_path)
|
||||
server = Server(scheduler)
|
||||
try:
|
||||
status, payload = server.post("/api/submit", {"text": "do something"})
|
||||
assert status == 200
|
||||
assert payload["status"] == "error"
|
||||
assert "engine" in payload["detail"]
|
||||
finally:
|
||||
server.close()
|
||||
|
||||
|
||||
def test_relabel_roundtrip(tmp_path):
|
||||
scheduler = build_scheduler(tmp_path)
|
||||
seed_decision(scheduler.log)
|
||||
server = Server(scheduler)
|
||||
try:
|
||||
status, payload = server.get("/api/trace")
|
||||
assert status == 200
|
||||
assert payload["runs"]
|
||||
assert payload["runs"][0]["decisions"][0]["id"] == "abc123"
|
||||
|
||||
status, payload = server.post("/api/relabel", {"id": "abc123", "outcome": "b"})
|
||||
assert status == 200
|
||||
assert payload["ok"] is True
|
||||
|
||||
status, payload = server.get("/api/trace")
|
||||
row = payload["runs"][0]["decisions"][0]
|
||||
assert row["observed_outcome"] == "b"
|
||||
assert row["label_source"] == "human"
|
||||
assert row["cost"]["weight"] == 3.0
|
||||
|
||||
status, payload = server.post("/api/relabel", {"id": "nope", "outcome": "a"})
|
||||
assert payload["ok"] is False
|
||||
finally:
|
||||
server.close()
|
||||
|
||||
|
||||
def test_submit_trace_event_recorded_even_when_engine_missing(tmp_path):
|
||||
scheduler = build_scheduler(tmp_path)
|
||||
server = Server(scheduler)
|
||||
try:
|
||||
server.post("/api/submit", {"text": "hello"})
|
||||
status, payload = server.get("/api/trace")
|
||||
runs = payload["runs"]
|
||||
assert len(runs) == 1
|
||||
kinds = [e["kind"] for e in runs[0]["events"]]
|
||||
assert "submit" in kinds
|
||||
finally:
|
||||
server.close()
|
||||
Reference in New Issue
Block a user