Add browser dashboard, run tracing, and logged navigation decisions
- dashboard.py: stdlib http.server + JSON API (tree/trace/dream/status, POST submit/relabel); static/ single-page Redux-DevTools-style inspector - trace.py: runs.jsonl lifecycle events keyed by run_id - scheduler/skills/skill: every decision carries run_id; navigation choices now logged (navigate:category/leaf); requeues stamp meta.parent_run - cli: 'dashboard' subcommand; config.json untracked per-machine (see config.example.json)
This commit is contained in:
Denton Social
@@ -39,6 +39,7 @@ def test_pipeline_end_to_end(tmp_path):
|
||||
config = load_config()
|
||||
require_real(config)
|
||||
config["log"] = str(tmp_path / "decisions.jsonl")
|
||||
config["trace"] = str(tmp_path / "runs.jsonl")
|
||||
scheduler, config = build_scheduler(config)
|
||||
|
||||
inputs = [
|
||||
@@ -55,6 +56,16 @@ def test_pipeline_end_to_end(tmp_path):
|
||||
rows = scheduler.log.read()
|
||||
assert len(rows) > 0, "expected SemIf decisions to be logged"
|
||||
|
||||
phases = [r.get("extra", {}).get("phase") for r in rows]
|
||||
assert "navigate:category" in phases, "navigation decisions must be logged"
|
||||
assert "navigate:leaf" in phases, "navigation decisions must be logged"
|
||||
for row in rows:
|
||||
assert row.get("extra", {}).get("run_id"), "every decision must carry a run_id"
|
||||
|
||||
trace_rows = scheduler.trace.read()
|
||||
assert any(r["kind"] == "submit" for r in trace_rows)
|
||||
assert any(r["kind"] == "assessed" for r in trace_rows)
|
||||
|
||||
report = dream(scheduler.log)
|
||||
assert report.cross_entropy is not None
|
||||
print(report.render())
|
||||
@@ -70,6 +81,7 @@ def test_busy_choice_path(tmp_path):
|
||||
config = load_config()
|
||||
require_real(config)
|
||||
config["log"] = str(tmp_path / "decisions.jsonl")
|
||||
config["trace"] = str(tmp_path / "runs.jsonl")
|
||||
scheduler, config = build_scheduler(config)
|
||||
|
||||
scheduler.busy("driving on the freeway", skill="driving")
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Dashboard API tests against a real stdlib HTTP server on an ephemeral port.
|
||||
|
||||
The scheduler is constructed with the lazy SemIfEngine (never loaded), so this
|
||||
runs anywhere without SemIf. Submit degrades to a JSON error, which is the
|
||||
expected behaviour on the thin dev box; relabel works against a seeded log.
|
||||
"""
|
||||
|
||||
import json
|
||||
import threading
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from http.server import ThreadingHTTPServer
|
||||
|
||||
from semif_agent.dashboard import DashboardHandler
|
||||
from semif_agent.decisions import DecisionRequest, DecisionResult, Option
|
||||
from semif_agent.engine import EngineConfig, SemIfEngine
|
||||
from semif_agent.llm import LLMClient
|
||||
from semif_agent.log import DecisionLog
|
||||
from semif_agent.scheduler import Scheduler
|
||||
from semif_agent.trace import TraceLog
|
||||
|
||||
|
||||
def build_scheduler(tmp_path):
|
||||
log = DecisionLog(str(tmp_path / "decisions.jsonl"))
|
||||
trace = TraceLog(str(tmp_path / "runs.jsonl"))
|
||||
engine = SemIfEngine(EngineConfig())
|
||||
llm = LLMClient(base_url="http://localhost:1/v1", model="test")
|
||||
scheduler = Scheduler(
|
||||
engine=engine,
|
||||
llm=llm,
|
||||
log=log,
|
||||
config={"skills": {}},
|
||||
trace=trace,
|
||||
)
|
||||
return scheduler
|
||||
|
||||
|
||||
def seed_decision(log, decision_id="abc123"):
|
||||
request = DecisionRequest(
|
||||
id=decision_id,
|
||||
state="some state",
|
||||
question="Pick one?",
|
||||
options=[Option("a", "A."), Option("b", "B.")],
|
||||
)
|
||||
result = DecisionResult(
|
||||
request=request, option_ids=["a", "b"], probabilities=[0.3, 0.7]
|
||||
)
|
||||
log.append(request, result, extra={"phase": "gate", "run_id": "run-1"})
|
||||
|
||||
|
||||
class Server:
|
||||
def __init__(self, scheduler):
|
||||
handler = type("Handler", (DashboardHandler,), {"scheduler": scheduler})
|
||||
self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), handler)
|
||||
self.thread = threading.Thread(target=self.httpd.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
self.port = self.httpd.server_address[1]
|
||||
|
||||
def get(self, path):
|
||||
with urllib.request.urlopen(f"http://127.0.0.1:{self.port}{path}") as res:
|
||||
return res.status, json.loads(res.read().decode("utf-8"))
|
||||
|
||||
def post(self, path, payload):
|
||||
req = urllib.request.Request(
|
||||
f"http://127.0.0.1:{self.port}{path}",
|
||||
data=json.dumps(payload).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req) as res:
|
||||
return res.status, json.loads(res.read().decode("utf-8"))
|
||||
|
||||
def close(self):
|
||||
self.httpd.shutdown()
|
||||
self.httpd.server_close()
|
||||
|
||||
|
||||
def test_tree_endpoint(tmp_path):
|
||||
server = Server(build_scheduler(tmp_path))
|
||||
try:
|
||||
status, payload = server.get("/api/tree")
|
||||
assert status == 200
|
||||
assert "email" in payload["categories"]
|
||||
assert any(s["name"] == "email.compose" for s in payload["categories"]["email"])
|
||||
finally:
|
||||
server.close()
|
||||
|
||||
|
||||
def test_trace_endpoint_empty(tmp_path):
|
||||
server = Server(build_scheduler(tmp_path))
|
||||
try:
|
||||
status, payload = server.get("/api/trace")
|
||||
assert status == 200
|
||||
assert payload["runs"] == []
|
||||
finally:
|
||||
server.close()
|
||||
|
||||
|
||||
def test_submit_without_engine_returns_error_json(tmp_path):
|
||||
scheduler = build_scheduler(tmp_path)
|
||||
server = Server(scheduler)
|
||||
try:
|
||||
status, payload = server.post("/api/submit", {"text": "do something"})
|
||||
assert status == 200
|
||||
assert payload["status"] == "error"
|
||||
assert "engine" in payload["detail"]
|
||||
finally:
|
||||
server.close()
|
||||
|
||||
|
||||
def test_relabel_roundtrip(tmp_path):
|
||||
scheduler = build_scheduler(tmp_path)
|
||||
seed_decision(scheduler.log)
|
||||
server = Server(scheduler)
|
||||
try:
|
||||
status, payload = server.get("/api/trace")
|
||||
assert status == 200
|
||||
assert payload["runs"]
|
||||
assert payload["runs"][0]["decisions"][0]["id"] == "abc123"
|
||||
|
||||
status, payload = server.post("/api/relabel", {"id": "abc123", "outcome": "b"})
|
||||
assert status == 200
|
||||
assert payload["ok"] is True
|
||||
|
||||
status, payload = server.get("/api/trace")
|
||||
row = payload["runs"][0]["decisions"][0]
|
||||
assert row["observed_outcome"] == "b"
|
||||
assert row["label_source"] == "human"
|
||||
assert row["cost"]["weight"] == 3.0
|
||||
|
||||
status, payload = server.post("/api/relabel", {"id": "nope", "outcome": "a"})
|
||||
assert payload["ok"] is False
|
||||
finally:
|
||||
server.close()
|
||||
|
||||
|
||||
def test_submit_trace_event_recorded_even_when_engine_missing(tmp_path):
|
||||
scheduler = build_scheduler(tmp_path)
|
||||
server = Server(scheduler)
|
||||
try:
|
||||
server.post("/api/submit", {"text": "hello"})
|
||||
status, payload = server.get("/api/trace")
|
||||
runs = payload["runs"]
|
||||
assert len(runs) == 1
|
||||
kinds = [e["kind"] for e in runs[0]["events"]]
|
||||
assert "submit" in kinds
|
||||
finally:
|
||||
server.close()
|
||||
@@ -0,0 +1,33 @@
|
||||
from semif_agent.trace import TraceLog
|
||||
|
||||
|
||||
def test_append_read_roundtrip(tmp_path):
|
||||
trace = TraceLog(str(tmp_path / "runs.jsonl"))
|
||||
trace.append("submit", "run-a", text="hello")
|
||||
trace.append("queued", "run-a", weight=0.5, label="medium")
|
||||
trace.append("submit", "run-b", text="world")
|
||||
|
||||
rows = trace.read()
|
||||
assert len(rows) == 3
|
||||
assert rows[0]["kind"] == "submit"
|
||||
assert rows[0]["run_id"] == "run-a"
|
||||
assert rows[0]["text"] == "hello"
|
||||
assert rows[1]["label"] == "medium"
|
||||
|
||||
|
||||
def test_runs_groups_by_run_id_preserving_order(tmp_path):
|
||||
trace = TraceLog(str(tmp_path / "runs.jsonl"))
|
||||
trace.append("submit", "run-a", text="x")
|
||||
trace.append("submit", "run-b", text="y")
|
||||
trace.append("assessed", "run-a", success=True)
|
||||
|
||||
runs = trace.runs()
|
||||
assert list(runs) == ["run-a", "run-b"]
|
||||
assert len(runs["run-a"]) == 2
|
||||
assert runs["run-b"][0]["text"] == "y"
|
||||
|
||||
|
||||
def test_read_missing_file_is_empty(tmp_path):
|
||||
trace = TraceLog(str(tmp_path / "none.jsonl"))
|
||||
assert trace.read() == []
|
||||
assert trace.runs() == {}
|
||||
Reference in New Issue
Block a user