diff --git a/.gitignore b/.gitignore index d5a355e..68dd7bb 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,8 @@ __pycache__/ *.pyc .venv/ data/decisions.jsonl -data/drafts/ \ No newline at end of file +data/runs.jsonl +data/drafts/ +.pytest_cache/ +config.json +*.log \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index 6081348..f9d57cc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,19 +17,25 @@ supplied options; an LLM is used only for generation and self-assessment. ## Architecture map ``` -cli.py argparse: run (REPL / --script), dream, skills, status, relabel +cli.py argparse: run (REPL / --script), dream, skills, status, relabel, + dashboard scheduler.py gate -> choice(tau) -> score -> queue; preempt + requeue queue.py urgency max-heap (desc weight, FIFO seq), age pulls toward 1.0 skills.py tree + registry (email.compose, response.reject, tracking.check), - navigation = SemIf choices per level, create_skill branch (stub) + navigation = SemIf choices per level (logged), create_skill + branch (stub) skill.py loop: observe -> predict -> act -> observe -> assess (LLM) engine.py SemIfEngine -> semif_phase1.llamacpp_backend (lazy import) llm.py OpenAI-compatible client for self-assessment (stdlib urllib) log.py decisions.jsonl rows {state, question, options, predicted_probs, selected, observed_outcome, label_source} +trace.py runs.jsonl lifecycle events keyed by run_id (submit/queued/ + preempted/assessed/...); decisions reference run_id in extra dream.py NLL of observed outcome per row; weighted CE, accuracy, ECE decisions.py contract dataclasses (Option, DecisionRequest, DecisionResult, Request) +dashboard.py stdlib http.server + JSON API (tree/trace/dream/status + + POST submit/relabel); static/ frontend served at / ``` ## Run / verify @@ -37,6 +43,17 @@ decisions.py contract dataclasses (Option, DecisionRequest, DecisionResult, Dev machine is a thin client (no GPU, ~1.4G disk): only pure stdlib unit tests run here (`python3 -m pytest tests/ -q --ignore=tests/integration`). +## Git / sync + +- Canonical repo lives on Gitea: `git.manyworlds.fit` (SSH on port 22, key + `~/.ssh/id_ed25519` registered there). The box `guppy` keeps a working copy + at `~/semif-agent`; the dev machine at `~/Repos/semif-agent`. Push to Gitea, + pull on each side — never rsync/tar the code. +- **`config.json` is gitignored and per-machine** (dev and the box use different + engine/LLM paths). Copy `config.example.json` to `config.json` and edit. + `data/decisions.jsonl`, `data/runs.jsonl`, and `data/drafts/` are runtime + artifacts and gitignored too. + The AMD box `guppy` (`abby@192.168.8.181`) is the real run target. Key facts: - ssh key `~/.ssh/id_ed25519` is passphrase-protected. Load it into an agent at @@ -54,7 +71,13 @@ The AMD box `guppy` (`abby@192.168.8.181`) is the real run target. Key facts: ~/semif-venv/bin/python -m semif_agent.cli run --script demo.jsonl ~/semif-venv/bin/python -m semif_agent.cli dream # cost report ~/semif-venv/bin/python -m semif_agent.cli relabel + ~/semif-venv/bin/python -m semif_agent.cli dashboard --port 8765 ``` +- Dashboard: browser UI on http://localhost:8765/. It works in live mode on + the box (submit runs the real engine + LLM) and in replay mode anywhere + (reads decisions.jsonl + runs.jsonl; submit degrades to a JSON error without + the engine). Relabeling in the UI writes a human override (3x weight in + dream) via `POST /api/relabel`. - Integration tests (real engine + real LLM) only run on the box: `~/semif-venv/bin/python -m pytest tests/integration -q -s` They take ~100s (model load ~34s). Run them in the background and poll — @@ -65,7 +88,7 @@ The AMD box `guppy` (`abby@192.168.8.181`) is the real run target. Key facts: ### v1 (done) Core loop, urgency queue, skill tree, skill loop with real SemIf + real LLM self-assessment, decision logging, `dream` cost pass, REPL + JSONL CLI, -unit tests (16) + box integration tests (2). +unit tests (24) + box integration tests (2). ### v2 - Real fine-tuning from `decisions.jsonl` at a regular interval ("dreaming"): @@ -79,6 +102,8 @@ unit tests (16) + box integration tests (2). - Event/timer intake sources beyond typed input. - Concurrency: SemIf shared-state mode (`score_shared` / `SerialPrefixScorer`) for parallel decisions; single execution slot remains for processes. +- Dashboard: run-requeue cross-linking (child run references its parent), + scheduler sim controls (busy/idle/tau) as a first-class panel. ### Later / open questions - Safety/authority: which inputs may interrupt high-stakes processes; is @@ -136,6 +161,9 @@ unit tests (16) + box integration tests (2). - `DecisionLog.append` labels a row with the *selected* option by default (self-consistent, near-zero cost). Real labels come from `relabel` (human, weight 3x in `dream`) — failures alone don't produce correct labels. +- Navigation decisions ARE logged (`navigate:category`, `navigate:leaf` in + skills.py) and therefore count toward dream cost. This is intended per the + design; don't silently drop them. - Queue ordering: urgency desc, then FIFO (`seq`). Recency is stored but is NOT in the sort key (it's anti-correlated with FIFO). Ageing pulls weights toward the max (1.0) so low items catch up; uniform additive boosts do nothing. @@ -146,6 +174,8 @@ unit tests (16) + box integration tests (2). ## Testing - `python3 -m pytest tests/ -q --ignore=tests/integration` — anywhere, fast. + Includes the dashboard API tests (`tests/test_dashboard_api.py`), which spin + up the stdlib HTTP server on an ephemeral port with the engine never loaded. - `tests/integration/` — box only; requires real SemIf + real ollama. - After touching scheduler/skills/engine, re-run both; the integration tests are the only end-to-end verification. \ No newline at end of file diff --git a/config.json b/config.example.json similarity index 57% rename from config.json rename to config.example.json index 42e8186..d6430c9 100644 --- a/config.json +++ b/config.example.json @@ -1,4 +1,5 @@ { + "_comment": "Per-machine config. config.json is gitignored; copy this file to config.json and adjust paths. The box (guppy) keeps its own config.json with local paths.", "tau": 0.6, "max_reentries": 3, "queue": {"max_size": 100, "age_rate": 0.01}, @@ -6,16 +7,18 @@ "backend": "llamacpp", "source": "Qwen/Qwen3.5-4B", "revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", - "gguf": "/mnt/models/Qwen3.5-4B-Q4_K_M.gguf", + "gguf": "/home/abby/models/Qwen3.5-4B-Q4_K_M.gguf", "context_tokens": 4096, - "threads": null + "threads": 8 }, - "llm": {"base_url": "http://localhost:11434/v1", "model": "qwen2.5:3b"}, + "llm": {"base_url": "http://localhost:11434/v1", "model": "qwen3.5:4b"}, "skills": { "email": {"cost_budget": 1.0}, "contacts": "data/contacts.json", "drafts": "data/drafts", "packages": "data/packages.json" }, - "log": "data/decisions.jsonl" + "log": "data/decisions.jsonl", + "trace": "data/runs.jsonl", + "dashboard": {"port": 8765} } \ No newline at end of file diff --git a/semif_agent/cli.py b/semif_agent/cli.py index 16c107b..cac7228 100644 --- a/semif_agent/cli.py +++ b/semif_agent/cli.py @@ -23,6 +23,7 @@ from .llm import LLMClient from .log import DecisionLog from .scheduler import Scheduler from .skills import build_skills, build_tree, tree_summary +from .trace import TraceLog def load_config(path: str = "config.json") -> dict: @@ -45,6 +46,7 @@ def build_scheduler(config: dict) -> tuple[Scheduler, dict]: model=config.get("llm", {}).get("model", "qwen2.5:3b"), ) log = DecisionLog(config.get("log", "data/decisions.jsonl")) + trace = TraceLog(config.get("trace", "data/runs.jsonl")) scheduler = Scheduler( engine=engine, llm=llm, @@ -52,6 +54,7 @@ def build_scheduler(config: dict) -> tuple[Scheduler, dict]: config=config, tau=float(config.get("tau", 0.6)), max_reentries=int(config.get("max_reentries", 3)), + trace=trace, ) return scheduler, config @@ -116,6 +119,8 @@ def scripted(scheduler: Scheduler, path: str) -> None: def main(argv: list[str] | None = None) -> int: + argv = list(argv) if argv is not None else list(sys.argv[1:]) + config = load_config(_extract_config_path(argv)) parser = argparse.ArgumentParser(prog="semif-agent") parser.add_argument("--config", default="config.json") sub = parser.add_subparsers(dest="command") @@ -131,8 +136,15 @@ def main(argv: list[str] | None = None) -> int: relabel_p.add_argument("id") relabel_p.add_argument("outcome") + dash_p = sub.add_parser("dashboard", help="run the local browser dashboard") + dash_p.add_argument("--port", type=int, default=int(config.get("dashboard", {}).get("port", 8765))) + dash_p.add_argument( + "--replay", + action="store_true", + help="replay mode: do not warm the decision engine", + ) + args = parser.parse_args(argv) - config = load_config(args.config) scheduler, config = build_scheduler(config) if args.command == "run": @@ -149,10 +161,27 @@ def main(argv: list[str] | None = None) -> int: elif args.command == "relabel": ok = scheduler.log.relabel(args.id, args.outcome) print("relabeled." if ok else f"no row with id {args.id}") + elif args.command == "dashboard": + from .dashboard import serve + + if args.replay: + print("replay mode: reading decision log + trace; engine not warmed.") + serve(scheduler, port=args.port) else: parser.print_help() return 0 +def _extract_config_path(argv: list[str]) -> str: + """Pull --config out of argv before the full parser runs, so the dashboard + subcommand can read dashboard.port from config for its default.""" + for index, arg in enumerate(argv): + if arg == "--config" and index + 1 < len(argv): + return argv[index + 1] + if arg.startswith("--config="): + return arg.split("=", 1)[1] + return "config.json" + + if __name__ == "__main__": sys.exit(main()) \ No newline at end of file diff --git a/semif_agent/dashboard.py b/semif_agent/dashboard.py new file mode 100644 index 0000000..8301981 --- /dev/null +++ b/semif_agent/dashboard.py @@ -0,0 +1,235 @@ +"""Local browser dashboard for the SemIf agent. + +A pure-stdlib HTTP server on localhost serving a Redux-DevTools-style +inspector over the agent's decision flow. Reads the decision log +(`decisions.jsonl`) plus the run lifecycle trace (`runs.jsonl`), exposes the +static skill tree, the dream cost report, and two write endpoints: submit a +request and relabel a decision (human override). + +The scheduler's engine and LLM are built lazily, so the dashboard runs on the +thin dev box in replay mode (reads logs; submit degrades to a JSON error) and +in live mode on the box with SemIf + a local LLM. + + python -m semif_agent.cli dashboard [--port 8765] +""" + +from __future__ import annotations + +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +from .dream import dream +from .scheduler import Scheduler +from .skills import build_skills, build_tree + +STATIC_DIR = Path(__file__).parent / "static" + +JSON_HEADERS = {"Content-Type": "application/json; charset=utf-8"} + + +def _phase(row: dict) -> str: + return (row.get("extra") or {}).get("phase", "") + + +def _enrich_decision(row: dict, costs: dict) -> dict: + out = dict(row) + out["phase"] = _phase(row) + cost = costs.get(row.get("id")) + if cost is not None: + out["cost"] = cost + return out + + +def build_payload(scheduler: Scheduler) -> dict: + """Group logged decisions by run, attach lifecycle events and dream costs.""" + events_by_run = scheduler.trace.runs() + rows = scheduler.log.read() + + costs = {} + for cost_row in dream(scheduler.log).rows: + costs[cost_row.decision_id] = { + "predicted": cost_row.predicted, + "nll": cost_row.nll, + "weight": cost_row.weight, + "correct": cost_row.correct, + } + + by_run: dict[str, dict] = {} + for row in rows: + run_id = (row.get("extra") or {}).get("run_id", "?") + run = by_run.setdefault( + run_id, {"run_id": run_id, "events": [], "decisions": [], "first_ts": None} + ) + run["first_ts"] = row["ts"] if run["first_ts"] is None else min(run["first_ts"], row["ts"]) + run["decisions"].append(_enrich_decision(row, costs)) + for run_id, events in events_by_run.items(): + run = by_run.setdefault( + run_id, {"run_id": run_id, "events": [], "decisions": [], "first_ts": None} + ) + run["events"] = events + event_ts = [e["ts"] for e in events if "ts" in e] + if event_ts: + earliest = min(event_ts) + run["first_ts"] = earliest if run["first_ts"] is None else min(run["first_ts"], earliest) + + runs = sorted(by_run.values(), key=lambda r: (r["first_ts"] is None, r["first_ts"] or 0)) + for run in runs: + run["decisions"].sort(key=lambda d: d["ts"]) + return {"runs": runs} + + +def build_dream_report(scheduler: Scheduler) -> dict: + report = dream(scheduler.log) + return { + "rows": len(report.rows), + "skipped": report.skipped, + "human_overrides": report.human_overrides, + "cross_entropy": report.cross_entropy, + "accuracy": report.accuracy, + "ece": report.ece, + } + + +def build_status(scheduler: Scheduler) -> dict: + current = ( + {"skill": scheduler.current.skill, "request_id": scheduler.current.request.id} + if scheduler.current + else None + ) + queue = [ + {"id": request.id, "weight": weight, "text": request.text[:80]} + for weight, request in scheduler.queue.items() + ] + return { + "current": current, + "queue": queue, + "tau": scheduler.tau, + "queue_max": scheduler.queue.max_size, + } + + +def build_tree_payload(scheduler: Scheduler) -> dict: + tree = scheduler.tree + return { + "categories": { + category: [{"name": skill.name, "description": skill.description} for skill in skills] + for category, skills in sorted(tree.items()) + } + } + + +class DashboardHandler(BaseHTTPRequestHandler): + scheduler: Scheduler + lock: threading.Lock = threading.Lock() + + server_version = "semif-dashboard/0.1" + + # ---- helpers ---- + + def _send(self, code: int, payload: dict, headers: dict | None = None): + body = json.dumps(payload).encode("utf-8") + self.send_response(code) + for key, value in JSON_HEADERS.items(): + self.send_header(key, value) + if headers: + for key, value in headers.items(): + self.send_header(key, value) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _send_error(self, code: int, message: str): + self._send(code, {"error": message}) + + def _read_json(self) -> dict: + length = int(self.headers.get("Content-Length") or 0) + if length <= 0: + return {} + return json.loads(self.rfile.read(length).decode("utf-8")) + + def _serve_static(self, rel: str): + if not rel or rel == "/": + rel = "index.html" + target = (STATIC_DIR / rel).resolve() + if not str(target).startswith(str(STATIC_DIR.resolve())): + self._send_error(403, "forbidden") + return + if not target.is_file(): + self._send_error(404, "not found") + return + body = target.read_bytes() + content_type = { + ".html": "text/html; charset=utf-8", + ".js": "application/javascript; charset=utf-8", + ".css": "text/css; charset=utf-8", + }.get(target.suffix, "application/octet-stream") + self.send_response(200) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + # ---- routing ---- + + def do_GET(self): + path = self.path.split("?", 1)[0].rstrip("/") or "/" + if path == "/" or path.startswith("/static/"): + rel = path.removeprefix("/static/").removeprefix("/") + self._serve_static(rel) + return + if path == "/api/trace": + self._send(200, build_payload(self.scheduler)) + return + if path == "/api/tree": + self._send(200, build_tree_payload(self.scheduler)) + return + if path == "/api/dream": + self._send(200, build_dream_report(self.scheduler)) + return + if path == "/api/status": + self._send(200, build_status(self.scheduler)) + return + self._send_error(404, "no such endpoint") + + def do_POST(self): + path = self.path.split("?", 1)[0] + if path == "/api/submit": + with self.lock: + try: + body = self._read_json() + status, detail = self.scheduler.submit( + str(body.get("text", "")), source=str(body.get("source", "dashboard")) + ) + except Exception as exc: + self._send_error(500, str(exc)) + return + self._send(200, {"status": status, "detail": detail}) + return + if path == "/api/relabel": + with self.lock: + try: + body = self._read_json() + ok = self.scheduler.log.relabel(str(body.get("id", "")), str(body.get("outcome", ""))) + except Exception as exc: + self._send_error(500, str(exc)) + return + self._send(200, {"ok": ok}) + return + self._send_error(404, "no such endpoint") + + def log_message(self, format, *args): + pass + + +def serve(scheduler: Scheduler, port: int = 8765, host: str = "127.0.0.1") -> None: + handler = type("Handler", (DashboardHandler,), {"scheduler": scheduler, "lock": threading.Lock()}) + server = ThreadingHTTPServer((host, port), handler) + print(f"semif dashboard on http://{host}:{port}/ (Ctrl-C to stop)") + try: + server.serve_forever() + except KeyboardInterrupt: + pass + finally: + server.server_close() \ No newline at end of file diff --git a/semif_agent/scheduler.py b/semif_agent/scheduler.py index fd87721..74fab93 100644 --- a/semif_agent/scheduler.py +++ b/semif_agent/scheduler.py @@ -24,6 +24,7 @@ from .skills import ( compose_state, navigate, ) +from .trace import TraceLog GATE_YES = "yes" CHOICE_INTERRUPT = "interrupt" @@ -60,10 +61,12 @@ class Scheduler: config: dict, tau: float = 0.6, max_reentries: int = 3, + trace: TraceLog | None = None, ): self.engine = engine self.llm = llm self.log = log + self.trace = trace if trace is not None else TraceLog() self.config = config self.tau = tau self.max_reentries = max_reentries @@ -86,7 +89,9 @@ class Scheduler: options=[Option(GATE_YES, "Yes, it is actionable."), Option("no", "No, it is not.")], ) result = self.engine.call(decision) - self.log.append(decision, result, extra={"phase": "gate"}) + self.log.append( + decision, result, extra={"phase": "gate", "run_id": request.id} + ) return result.prob(GATE_YES) >= self.tau def _choice(self, request: Request, current: Process) -> bool: @@ -99,7 +104,11 @@ class Scheduler: ], ) result = self.engine.call(decision) - self.log.append(decision, result, extra={"phase": "choice", "current": current.skill}) + self.log.append( + decision, + result, + extra={"phase": "choice", "current": current.skill, "run_id": request.id}, + ) return result.prob(CHOICE_INTERRUPT) >= self.tau def _score(self, request: Request, current: str | None = None) -> tuple[float, str]: @@ -109,7 +118,9 @@ class Scheduler: options=[Option(option_id, description) for option_id, description in URGENCY_OPTIONS], ) result = self.engine.call(decision) - self.log.append(decision, result, extra={"phase": "score"}) + self.log.append( + decision, result, extra={"phase": "score", "run_id": request.id} + ) label = result.selected return URGENCY_WEIGHTS[label], label @@ -126,7 +137,9 @@ class Scheduler: def _submit(self, text: str, source: str = "typed") -> tuple[str, str]: request = Request(text, source=source) + self.trace.append("submit", request.id, text=text, source=source) if not self._contains_request(request): + self.trace.append("dropped", request.id, reason="no actionable request") return "dropped", "no actionable request" if self.current is None: @@ -134,6 +147,7 @@ class Scheduler: self.current = Process(request=request, skill="(scheduling)", weight=weight) outcome = self._dispatch(request) self.current = None + self.trace.append("ran", request.id, skill=outcome.skill, summary=outcome.summary) return "running", f"[{label}] {outcome.summary}" interrupt = self._choice(request, self.current) @@ -142,14 +156,18 @@ class Scheduler: previous.request.resume["from_skill"] = previous.skill self.queue.push(previous.request, previous.weight) self.current = Process(request=request, skill="(scheduling)", weight=1.0) + self.trace.append("preempted", request.id, preempted=previous.skill) outcome = self._dispatch(request) self.current = None + self.trace.append("ran", request.id, skill=outcome.skill, summary=outcome.summary) return "preempted", f"interrupted {previous.skill}; {outcome.summary}" weight, label = self._score(request, current=self.current.skill) ok = self.queue.push(request, weight) if not ok: + self.trace.append("rejected", request.id, reason="queue is full") return "rejected", "queue is full" + self.trace.append("queued", request.id, weight=weight, label=label) return "queued", f"urgency {label} (weight {weight:.2f})" def busy(self, text: str, skill: str = "(driving)") -> None: @@ -166,29 +184,44 @@ class Scheduler: results = [] while self.current is None and len(self.queue) > 0: request = self.queue.pop() + self.trace.append("dequeued", request.id) self.current = Process(request=request, skill="(scheduling)", weight=0.0) try: outcome = self._dispatch(request) except EngineUnavailable as exc: outcome = DispatchResult(kind="error", summary=f"engine unavailable: {exc}") self.current = None + self.trace.append("ran", request.id, skill=outcome.skill, summary=outcome.summary) results.append(("ran", f"[{request.id}] {outcome.summary}")) return results # ---- dispatch ---- def _dispatch(self, request: Request) -> DispatchResult: - navigation = navigate(self.engine, request, self.tree) + navigation = navigate(self.engine, self.log, request, self.tree) if isinstance(navigation, CreateSkill): + self.trace.append("create_skill", request.id, category=navigation.category) return DispatchResult( kind="create_skill", summary="skill authoring via opencode is deferred to v2; request logged.", ) outcome = self.runner.run(navigation, request) if outcome.error: + self.trace.append( + "error", request.id, skill=navigation.name, message=outcome.error + ) return DispatchResult(kind="error", summary=f"skill error: {outcome.error}") + self.trace.append( + "assessed", + request.id, + skill=navigation.name, + success=outcome.success, + summary=outcome.summary, + updated_request=outcome.updated_request, + ) if outcome.updated_request and request.reentries < self.max_reentries: self.queue.push(_requeue(request, outcome.updated_request), 0.5) + self.trace.append("requeued", request.id, text=outcome.updated_request) return DispatchResult( kind="ran", summary=f"{navigation.name}: {'ok' if outcome.success else 'failed'} — {outcome.summary}", @@ -209,4 +242,5 @@ class Scheduler: def _requeue(request: Request, updated_text: str) -> Request: updated = Request(updated_text, source="requeue") updated.reentries = request.reentries + 1 + updated.meta["parent_run"] = request.id return updated \ No newline at end of file diff --git a/semif_agent/skill.py b/semif_agent/skill.py index ed9759c..316a3ae 100644 --- a/semif_agent/skill.py +++ b/semif_agent/skill.py @@ -52,7 +52,16 @@ class SkillRunner: run_ok = assessment.success decisions = getattr(prediction, "decisions", []) for decision, result in decisions: - self.log.append(decision, result, extra={"skill": skill.name, "run_ok": run_ok}) + self.log.append( + decision, + result, + extra={ + "phase": "predict", + "skill": skill.name, + "run_ok": run_ok, + "run_id": request.id, + }, + ) return RunResult( skill=skill.name, diff --git a/semif_agent/skills.py b/semif_agent/skills.py index 3b19d2e..4584194 100644 --- a/semif_agent/skills.py +++ b/semif_agent/skills.py @@ -17,6 +17,7 @@ from typing import Callable from .decisions import DecisionRequest, Option, Request from .engine import SemIfEngine +from .log import DecisionLog @dataclass @@ -158,8 +159,13 @@ def build_tree(skills: list[Skill]) -> dict[str, list[Skill]]: return tree -def navigate(engine: SemIfEngine, request: Request, tree: dict[str, list[Skill]]) -> Skill | CreateSkill: - """Descend the tree one SemIf choice per level.""" +def navigate( + engine: SemIfEngine, + log: DecisionLog, + request: Request, + tree: dict[str, list[Skill]], +) -> Skill | CreateSkill: + """Descend the tree one SemIf choice per level. Every choice is logged.""" categories = sorted(tree.keys()) create = Option("create_skill", "Create a new skill for this.") top = DecisionRequest( @@ -168,6 +174,7 @@ def navigate(engine: SemIfEngine, request: Request, tree: dict[str, list[Skill]] options=[Option(c, c) for c in categories] + [create], ) top_result = engine.call(top) + log.append(top, top_result, extra={"phase": "navigate:category", "run_id": request.id}) category = top_result.selected if category == "create_skill": return CreateSkill(category=None) @@ -178,6 +185,7 @@ def navigate(engine: SemIfEngine, request: Request, tree: dict[str, list[Skill]] options=[Option(s.name, s.description) for s in skills] + [create], ) leaf_result = engine.call(leaf) + log.append(leaf, leaf_result, extra={"phase": "navigate:leaf", "run_id": request.id}) pick = leaf_result.selected if pick == "create_skill": return CreateSkill(category=category) diff --git a/semif_agent/static/app.js b/semif_agent/static/app.js new file mode 100644 index 0000000..b9cafef --- /dev/null +++ b/semif_agent/static/app.js @@ -0,0 +1,550 @@ +"use strict"; + +const $ = (sel) => document.querySelector(sel); + +const state = { + runs: [], + tree: { categories: {} }, + status: { current: null, queue: [], tau: 0.6 }, + dream: {}, + selectedRunId: null, + selectedDecisionId: null, + phaseFilter: "", + scrub: 0, + flowSteps: [], +}; + +async function getJSON(url, opts) { + const res = await fetch(url, opts); + if (!res.ok) throw new Error(`${url}: ${res.status}`); + return res.json(); +} + +async function refreshAll() { + const [trace, tree, status, dream] = await Promise.all([ + getJSON("/api/trace"), + getJSON("/api/tree"), + getJSON("/api/status"), + getJSON("/api/dream"), + ]); + state.runs = trace.runs; + state.tree = tree; + state.status = status; + state.dream = dream; + if (state.selectedRunId && !state.runs.some((r) => r.run_id === state.selectedRunId)) { + state.selectedRunId = null; + state.selectedDecisionId = null; + } + render(); +} + +function flash(msg) { + $("#mode-badge").textContent = msg; +} + +/* ---------------- phase helpers ---------------- */ + +function phaseClass(phase) { + if (phase.startsWith("navigate:")) return "navigate"; + return phase; +} + +function shortPhase(phase) { + return phase.replace("navigate:", "nav:"); +} + +const ALL_PHASES = ["gate", "choice", "score", "navigate:category", "navigate:leaf", "predict"]; + +function optionProbs(row) { + return (row.options || []).map((o) => ({ + id: o.id, + description: o.description, + p: (row.predicted_probs || {})[o.id] || 0, + })); +} + +/* ---------------- timeline ---------------- */ + +function renderTimeline() { + const el = $("#timeline"); + el.innerHTML = ""; + for (const run of state.runs) { + const decisions = run.decisions.filter( + (d) => !state.phaseFilter || d.phase === state.phaseFilter + ); + if (state.phaseFilter && decisions.length === 0) continue; + + const group = document.createElement("div"); + group.className = "run-group"; + + const head = document.createElement("div"); + head.className = "run-head"; + head.textContent = run.decisions[0] ? run.decisions[0].state.slice(0, 60) : run.run_id; + head.title = run.run_id; + head.addEventListener("click", () => { + state.selectedRunId = run.run_id; + state.selectedDecisionId = null; + state.scrub = 0; + render(); + }); + group.appendChild(head); + + if (state.selectedRunId === run.run_id) { + for (const evt of run.events) { + group.appendChild(eventRow(evt)); + } + for (const d of decisions) { + group.appendChild(decisionRow(d)); + } + } else { + group.appendChild(dimRow(`${decisions.length} decisions`)); + } + el.appendChild(group); + } +} + +function dimRow(text) { + const div = document.createElement("div"); + div.className = "trow event"; + div.textContent = text; + return div; +} + +function eventRow(evt) { + const div = document.createElement("div"); + div.className = "trow event"; + div.textContent = evt.kind; + if (evt.kind === "assessed") { + div.textContent = `assessed → ${evt.success ? "ok" : "fail"}`; + div.title = evt.summary || ""; + } else if (evt.kind === "queued") { + div.textContent = `queued (${evt.label})`; + } else if (evt.kind === "preempted") { + div.textContent = `preempted ${evt.preempted}`; + } + return div; +} + +function decisionRow(d) { + const div = document.createElement("div"); + div.className = "trow" + (d.id === state.selectedDecisionId ? " selected" : ""); + div.appendChild(pill(d.phase)); + if (d.label_source === "human") div.appendChild(star()); + const label = document.createElement("span"); + label.textContent = `${d.selected} ${(d.predicted_probs[d.selected] || 0).toFixed(2)}`; + div.appendChild(label); + if (d.cost && !d.cost.correct) div.appendChild(mark("x")); + div.title = d.question; + div.addEventListener("click", () => { + state.selectedRunId = (d.extra || {}).run_id; + state.selectedDecisionId = d.id; + render(); + }); + return div; +} + +function pill(text) { + const span = document.createElement("span"); + span.className = `pill ${phaseClass(text)}`; + span.textContent = shortPhase(text); + return span; +} + +function star() { + const span = document.createElement("span"); + span.className = "human-star"; + span.textContent = "★"; + return span; +} + +function mark(kind) { + const span = document.createElement("span"); + span.className = kind === "x" ? "x" : "chk"; + span.textContent = kind === "x" ? "✗" : "✓"; + return span; +} + +/* ---------------- flow graph ---------------- */ + +function buildFlowSteps(run) { + const items = []; + for (const evt of run.events) items.push({ kind: "event", ts: evt.ts, data: evt }); + for (const d of run.decisions) items.push({ kind: "decision", ts: d.ts, data: d }); + items.sort((a, b) => a.ts - b.ts); + return items; +} + +function renderFlow() { + const el = $("#flow"); + const run = state.runs.find((r) => r.run_id === state.selectedRunId); + $("#run-label").textContent = run ? run.run_id : ""; + $("#scrubber").max = 0; + $("#scrubber").value = 0; + state.flowSteps = []; + el.innerHTML = ""; + + if (!run) { + const empty = document.createElement("div"); + empty.className = "muted"; + empty.textContent = "select a run from the timeline"; + el.appendChild(empty); + return; + } + + const steps = buildFlowSteps(run); + state.flowSteps = steps; + $("#scrubber").max = Math.max(0, steps.length - 1); + $("#scrubber").value = 0; + + for (let i = 0; i < steps.length; i++) { + if (i > 0) el.appendChild(edge(steps[i - 1].data)); + const node = steps[i].kind === "event" ? eventNode(steps[i].data) : decisionNode(steps[i].data); + node.dataset.step = i; + if (i > state.scrub) node.classList.add("dim"); + el.appendChild(node); + } +} + +function edge(prev) { + const div = document.createElement("div"); + div.className = "edge"; + if (prev.kind === "decision") { + const p = (prev.predicted_probs || {})[prev.selected] || 0; + if (p >= 0.6) div.classList.add("hot"); + } + return div; +} + +function decisionNode(d) { + const node = document.createElement("div"); + const ok = d.cost ? d.cost.correct : null; + node.className = "node" + (d.id === state.selectedDecisionId ? " selected" : ""); + if (ok === true) node.classList.add("ok"); + if (ok === false) node.classList.add("fail"); + + const head = document.createElement("div"); + head.className = "node-head"; + head.appendChild(pill(d.phase)); + const q = document.createElement("div"); + q.className = "node-question"; + q.textContent = d.question; + head.appendChild(q); + if (d.label_source === "human") head.appendChild(star()); + const id = document.createElement("div"); + id.className = "node-id"; + id.textContent = d.id; + head.appendChild(id); + node.appendChild(head); + + for (const opt of optionProbs(d)) { + node.appendChild(optionRow(opt, d)); + } + + const meta = document.createElement("div"); + meta.className = "node-meta"; + const sel = document.createElement("span"); + sel.textContent = `selected: ${d.selected}`; + meta.appendChild(sel); + if (d.label_source === "human") { + const obs = document.createElement("span"); + obs.textContent = `human override → ${d.observed_outcome}`; + obs.style.color = "var(--human)"; + meta.appendChild(obs); + } + if (d.cost) { + const nll = document.createElement("span"); + nll.className = "cost-nll"; + nll.textContent = `nll ${d.cost.nll.toFixed(3)} ×${d.cost.weight}`; + meta.appendChild(nll); + } + const ex = d.extra || {}; + const timing = document.createElement("span"); + timing.textContent = ex.total_seconds ? `${ex.total_seconds.toFixed(2)}s` : ""; + meta.appendChild(timing); + node.appendChild(meta); + + node.addEventListener("click", () => { + state.selectedDecisionId = d.id; + state.selectedRunId = (d.extra || {}).run_id; + render(); + }); + return node; +} + +function optionRow(opt, d) { + const row = document.createElement("div"); + const cls = ["opt-row"]; + if (opt.id === d.selected) cls.push("selected"); + row.className = cls.join(" "); + + const label = document.createElement("span"); + label.className = "opt-label"; + label.textContent = opt.id; + label.title = opt.description; + row.appendChild(label); + + const barTrack = document.createElement("span"); + barTrack.className = "opt-bar-track"; + const bar = document.createElement("span"); + bar.className = "opt-bar"; + bar.style.width = `${Math.max(opt.p * 100, 1)}%`; + barTrack.appendChild(bar); + row.appendChild(barTrack); + + const marks = document.createElement("span"); + marks.className = "opt-marks"; + if (opt.id === d.observed_outcome && opt.id !== d.selected) marks.appendChild(mark("chk")); + row.appendChild(marks); + + const pct = document.createElement("span"); + pct.className = "opt-pct"; + pct.textContent = `${(opt.p * 100).toFixed(0)}%`; + row.appendChild(pct); + return row; +} + +function eventNode(evt) { + const node = document.createElement("div"); + node.className = "node event-node"; + if (evt.kind === "assessed") node.classList.add(evt.success ? "ok" : "fail"); + const kind = document.createElement("div"); + kind.className = "evt-kind"; + kind.textContent = evt.kind; + node.appendChild(kind); + const body = document.createElement("div"); + body.className = "node-question"; + if (evt.kind === "assessed") { + body.textContent = evt.summary || ""; + if (evt.updated_request) { + const req = document.createElement("div"); + req.className = "muted"; + req.textContent = `→ requeued: ${evt.updated_request}`; + node.appendChild(req); + } + } else if (evt.kind === "queued") { + body.textContent = `urgency ${evt.label} (weight ${Number(evt.weight || 0).toFixed(2)})`; + } else if (evt.kind === "preempted") { + body.textContent = `interrupted ${evt.preempted}, requeued with state`; + } else if (evt.kind === "dropped") { + body.textContent = evt.reason || ""; + } else { + body.textContent = evt.summary || evt.text || ""; + } + node.appendChild(body); + return node; +} + +/* ---------------- inspector ---------------- */ + +function selectedDecision() { + if (!state.selectedDecisionId) return null; + for (const run of state.runs) { + for (const d of run.decisions) { + if (d.id === state.selectedDecisionId) return d; + } + } + return null; +} + +function renderInspector() { + const el = $("#inspector"); + const d = selectedDecision(); + $("#relabel-btn").disabled = !d; + if (!d) { + el.innerHTML = '
click a decision node to inspect it
'; + return; + } + el.innerHTML = ""; + el.appendChild(kv("id", d.id)); + el.appendChild(kv("phase", d.phase || "—")); + el.appendChild(kv("state", d.state)); + const pre = document.createElement("pre"); + pre.className = "json"; + const copy = { ...d }; + delete copy.cost; + pre.textContent = JSON.stringify(copy, null, 2); + el.appendChild(pre); +} + +function kv(k, v) { + const div = document.createElement("div"); + div.className = "kv"; + div.innerHTML = `${k}: `; + div.appendChild(document.createTextNode(v)); + return div; +} + +/* ---------------- skill tree ---------------- */ + +function renderTree() { + const el = $("#skill-tree"); + el.innerHTML = ""; + const cats = state.tree.categories || {}; + for (const [cat, skills] of Object.entries(cats)) { + const div = document.createElement("div"); + div.className = "cat"; + const name = document.createElement("div"); + name.className = "cat-name"; + name.textContent = cat; + div.appendChild(name); + for (const skill of skills) { + const row = document.createElement("div"); + row.className = "skill-row"; + const nm = document.createElement("span"); + nm.textContent = skill.name; + const desc = document.createElement("span"); + desc.className = "sdesc"; + desc.textContent = skill.description; + row.appendChild(nm); + row.appendChild(desc); + div.appendChild(row); + } + el.appendChild(div); + } +} + +/* ---------------- status / dream ---------------- */ + +function renderStatus() { + const el = $("#status"); + el.innerHTML = ""; + const grid = document.createElement("div"); + grid.className = "stat-grid"; + + const current = state.status.current + ? `${state.status.current.skill} (${state.status.current.request_id})` + : "idle"; + grid.appendChild(stat("current", current)); + grid.appendChild(stat("queue", String(state.status.queue.length))); + grid.appendChild(stat("tau", String(state.status.tau))); + grid.appendChild(stat("rows", String(state.dream.rows))); + + const acc = state.dream.accuracy; + const ce = state.dream.cross_entropy; + const ece = state.dream.ece; + grid.appendChild(stat("acc", acc == null ? "n/a" : acc.toFixed(3))); + grid.appendChild(stat("CE", ce == null ? "n/a" : ce.toFixed(4))); + grid.appendChild(stat("ECE", ece == null ? "n/a" : ece.toFixed(4))); + grid.appendChild(stat("human", String(state.dream.human_overrides))); + + el.appendChild(grid); + const q = document.createElement("div"); + q.className = "muted"; + q.style.marginTop = "8px"; + q.textContent = state.status.queue + .map((item) => `${item.id} w=${item.weight.toFixed(2)} ${item.text}`) + .join("\n") || "queue empty"; + el.appendChild(q); +} + +function stat(k, v) { + const div = document.createElement("div"); + div.className = "stat"; + div.innerHTML = `${k}
${v}`; + return div; +} + +/* ---------------- scrubber ---------------- */ + +function onScrub() { + state.scrub = Number($("#scrubber").value); + document.querySelectorAll("#flow .node").forEach((node) => { + node.classList.toggle("dim", Number(node.dataset.step) > state.scrub); + }); +} + +/* ---------------- relabel modal ---------------- */ + +function openRelabel() { + const d = selectedDecision(); + if (!d) return; + $("#relabel-id").textContent = `${d.id} — ${d.question}`; + const sel = $("#relabel-options"); + sel.innerHTML = ""; + for (const o of d.options) { + const opt = document.createElement("option"); + opt.value = o.id; + opt.textContent = `${o.id} — ${o.description}`; + sel.appendChild(opt); + } + sel.value = d.observed_outcome; + $("#relabel-modal").classList.remove("hidden"); +} + +async function applyRelabel() { + const d = selectedDecision(); + if (!d) return; + const outcome = $("#relabel-options").value; + const res = await getJSON("/api/relabel", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ id: d.id, outcome }), + }); + $("#relabel-modal").classList.add("hidden"); + if (res.ok) { + flash(`relabeled ${d.id} → ${outcome}`); + await refreshAll(); + } else { + flash("relabel failed"); + } +} + +/* ---------------- wiring ---------------- */ + +$("#submit-form").addEventListener("submit", async (e) => { + e.preventDefault(); + const text = $("#query-input").value.trim(); + if (!text) return; + try { + const res = await getJSON("/api/submit", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ text }), + }); + flash(`[${res.status}] ${res.detail}`); + } catch (err) { + flash(`submit failed: ${err.message}`); + } + $("#query-input").value = ""; + await refreshAll(); +}); + +$("#refresh-btn").addEventListener("click", async () => { + try { + await refreshAll(); + } catch (err) { + flash(`refresh failed: ${err.message}`); + } +}); + +$("#phase-filter").addEventListener("change", (e) => { + state.phaseFilter = e.target.value; + render(); +}); + +$("#scrubber").addEventListener("input", onScrub); +$("#relabel-btn").addEventListener("click", openRelabel); +$("#relabel-cancel").addEventListener("click", () => $("#relabel-modal").classList.add("hidden")); +$("#relabel-apply").addEventListener("click", applyRelabel); + +function render() { + renderTimeline(); + renderFlow(); + renderInspector(); + renderTree(); + renderStatus(); +} + +function init() { + const filter = $("#phase-filter"); + for (const p of ALL_PHASES) { + const opt = document.createElement("option"); + opt.value = p; + opt.textContent = p; + filter.appendChild(opt); + } + refreshAll().catch((err) => flash(`failed to load: ${err.message}`)); + setInterval(() => refreshAll().catch(() => {}), 4000); +} + +init(); \ No newline at end of file diff --git a/semif_agent/static/index.html b/semif_agent/static/index.html new file mode 100644 index 0000000..b43139a --- /dev/null +++ b/semif_agent/static/index.html @@ -0,0 +1,80 @@ + + + + + + SemIf Agent Dashboard + + + +
+
SemIf agent dashboard
+
…
+
+ +
+
+ + +
+ +
+ +
+ + +
+
+

Decision flow

+ +
+ +
+
+
+ + +
+ + + + + + \ No newline at end of file diff --git a/semif_agent/static/style.css b/semif_agent/static/style.css new file mode 100644 index 0000000..816f22d --- /dev/null +++ b/semif_agent/static/style.css @@ -0,0 +1,346 @@ +:root { + --bg: #14161b; + --panel: #1c1f27; + --panel-2: #232733; + --line: #2e3340; + --text: #d7dae0; + --muted: #8b93a3; + --accent: #4f8cff; + --ok: #3ecf8e; + --fail: #ff6b6b; + --warn: #ffc94d; + --human: #c792ea; + font-size: 14px; +} + +* { box-sizing: border-box; } + +html, body { + margin: 0; + height: 100%; + background: var(--bg); + color: var(--text); + font-family: "SF Mono", "Cascadia Code", "JetBrains Mono", Consolas, monospace; +} + +.muted { color: var(--muted); } +.spacer { flex: 1; } + +.topbar { + display: flex; + align-items: center; + gap: 12px; + padding: 10px 16px; + background: var(--panel); + border-bottom: 1px solid var(--line); + font-size: 15px; + font-weight: 700; +} + +.badge { + padding: 2px 10px; + border: 1px solid var(--line); + border-radius: 12px; + color: var(--muted); + font-weight: 400; + font-size: 12px; +} + +.inputbar { + display: flex; + gap: 8px; + padding: 10px 16px; + background: var(--panel-2); + border-bottom: 1px solid var(--line); +} + +.inputbar form { display: flex; flex: 1; gap: 8px; } + +#query-input { + flex: 1; + background: var(--bg); + color: var(--text); + border: 1px solid var(--line); + border-radius: 6px; + padding: 8px 12px; + outline: none; +} + +#query-input:focus { border-color: var(--accent); } + +button { + background: var(--panel-2); + color: var(--text); + border: 1px solid var(--line); + border-radius: 6px; + padding: 8px 14px; + cursor: pointer; +} + +button:hover { border-color: var(--accent); } +button:disabled { opacity: 0.45; cursor: default; } +button.small { padding: 4px 8px; font-size: 12px; } +button.primary { background: var(--accent); border-color: var(--accent); color: #fff; } + +.layout { + display: grid; + grid-template-columns: 300px 1fr 360px; + height: calc(100vh - 98px); +} + +.panel { + overflow: hidden; + display: flex; + flex-direction: column; + border-right: 1px solid var(--line); + background: var(--panel); +} + +.panel-head { + display: flex; + align-items: center; + gap: 10px; + padding: 8px 12px; + border-bottom: 1px solid var(--line); + background: var(--panel-2); +} + +.panel-head h2 { + margin: 0; + font-size: 12px; + text-transform: uppercase; + letter-spacing: 0.08em; + color: var(--muted); +} + +select { + background: var(--bg); + color: var(--text); + border: 1px solid var(--line); + border-radius: 6px; + padding: 4px 6px; + font-size: 12px; +} + +/* ---------- timeline ---------- */ + +.timeline { + overflow-y: auto; + flex: 1; + padding: 8px; +} + +.timeline .run-group { margin-bottom: 10px; } + +.timeline .run-head { + display: flex; + align-items: center; + gap: 8px; + cursor: pointer; + padding: 5px 8px; + border-radius: 6px; + background: var(--panel-2); + font-size: 12px; +} + +.timeline .run-head:hover { border-color: var(--accent); } + +.timeline .trow { + display: flex; + align-items: center; + gap: 6px; + padding: 4px 8px; + border-left: 2px solid var(--line); + margin-left: 10px; + cursor: pointer; + font-size: 12px; +} + +.timeline .trow:hover { background: var(--panel-2); } +.timeline .trow.selected { background: var(--panel-2); border-left-color: var(--accent); } +.timeline .trow.event { color: var(--muted); cursor: default; } + +.pill { + border-radius: 3px; + padding: 0 5px; + font-size: 10px; + text-transform: uppercase; + white-space: nowrap; +} + +.pill.gate { background: #2b3a52; color: #9ec1ff; } +.pill.choice { background: #4a3a2b; color: #ffd59e; } +.pill.score { background: #3a2b4a; color: #d59eff; } +.pill.navigate { background: #2b4a3a; color: #9effd5; } +.pill.predict { background: #2b464a; color: #9eeaff; } +.pill.assess { background: #4a2b2b; color: #ff9e9e; } + +.human-star { color: var(--human); } + +/* ---------- flow graph ---------- */ + +.flow { + overflow-y: auto; + flex: 1; + padding: 16px; + display: flex; + flex-direction: column; + align-items: center; + gap: 0; +} + +.edge { + width: 2px; + height: 18px; + background: var(--line); + margin: 0 auto; +} + +.edge.hot { background: var(--accent); } + +.node { + width: 420px; + max-width: 100%; + background: var(--panel-2); + border: 1px solid var(--line); + border-radius: 8px; + padding: 8px 10px; + cursor: pointer; +} + +.node:hover { border-color: var(--accent); } +.node.selected { border-color: var(--accent); box-shadow: 0 0 0 2px rgba(79, 140, 255, 0.25); } +.node.dim { opacity: 0.35; } +.node.ok { border-left: 3px solid var(--ok); } +.node.fail { border-left: 3px solid var(--fail); } + +.node-head { + display: flex; + align-items: center; + gap: 8px; + margin-bottom: 6px; +} + +.node-head .pill { font-size: 10px; } + +.node-question { + flex: 1; + font-size: 12px; + font-weight: 600; +} + +.node-question .q { color: var(--text); } + +.node-id { font-size: 10px; color: var(--muted); } + +.opt-row { + display: flex; + align-items: center; + gap: 8px; + font-size: 11px; + margin: 3px 0; + position: relative; +} + +.opt-row .opt-label { width: 130px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } + +.opt-bar-track { + flex: 1; + height: 14px; + background: var(--bg); + border-radius: 3px; + overflow: hidden; +} + +.opt-bar { height: 100%; background: var(--line); transition: width 0.2s; } + +.opt-row.selected .opt-bar { background: var(--accent); } +.opt-row.selected .opt-label { color: var(--text); font-weight: 700; } +.opt-row.observed .opt-bar { background: var(--human); } +.opt-row .opt-pct { width: 48px; text-align: right; color: var(--muted); } +.opt-row.selected .opt-pct { color: var(--accent); } + +.opt-marks { position: absolute; right: 102px; } +.opt-row .chk { color: var(--ok); } +.opt-row .x { color: var(--fail); } + +.node-meta { + margin-top: 6px; + font-size: 10px; + color: var(--muted); + display: flex; + gap: 10px; + flex-wrap: wrap; +} + +.cost-nll { color: var(--warn); } + +.node.event-node { + width: 360px; + background: var(--bg); + cursor: default; +} + +.node.event-node .evt-kind { text-transform: uppercase; color: var(--muted); font-size: 10px; } + +/* ---------- inspector / tree / status ---------- */ + +.right { border-right: none; } +.inspector, .status { overflow-y: auto; padding: 10px; font-size: 11px; } +.skill-tree { overflow-y: auto; padding: 10px; font-size: 12px; flex: 1; } + +pre.json { + margin: 0; + white-space: pre-wrap; + word-break: break-all; + color: var(--text); +} + +.kv { margin: 2px 0; } +.kv .k { color: var(--muted); } + +.cat { margin-bottom: 10px; } +.cat .cat-name { font-weight: 700; margin-bottom: 4px; } +.skill-row { display: flex; gap: 6px; align-items: baseline; padding-left: 8px; font-size: 11px; } +.skill-row .sdesc { color: var(--muted); } + +.stat-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 4px; } +.stat-grid .stat { background: var(--panel-2); border-radius: 4px; padding: 4px 6px; } +.stat-grid .stat .v { font-weight: 700; } + +/* ---------- scrubber ---------- */ + +.scrub-label { display: flex; align-items: center; gap: 6px; font-size: 11px; color: var(--muted); } +#scrubber { width: 180px; } + +/* ---------- modal ---------- */ + +.modal { + position: fixed; + inset: 0; + background: rgba(0, 0, 0, 0.6); + display: flex; + align-items: center; + justify-content: center; +} + +.modal.hidden { display: none; } + +.modal-box { + background: var(--panel-2); + border: 1px solid var(--line); + border-radius: 10px; + padding: 18px; + min-width: 320px; + display: flex; + flex-direction: column; + gap: 12px; +} + +.modal-box h3 { margin: 0; } + +.modal-box label { display: flex; flex-direction: column; gap: 4px; font-size: 12px; color: var(--muted); } + +.modal-actions { display: flex; justify-content: flex-end; gap: 8px; } + +.hidden { display: none !important; } \ No newline at end of file diff --git a/semif_agent/trace.py b/semif_agent/trace.py new file mode 100644 index 0000000..f4c34b9 --- /dev/null +++ b/semif_agent/trace.py @@ -0,0 +1,44 @@ +"""Run lifecycle tracing: the events around each request's trip through the +scheduler. + +Decision rows live in `decisions.jsonl` (SemIf-compatible). Lifecycle events +(submit, dropped, queued, preempted, assessed, requeued, ...) live here in +`runs.jsonl`, keyed by `run_id` (the request id) so the dashboard can rebuild +each run's flow without touching the decision-log contract. Both files are +append-only; replays read them in order. +""" + +from __future__ import annotations + +import json +import time +from pathlib import Path + + +class TraceLog: + def __init__(self, path: str = "data/runs.jsonl"): + self.path = Path(path) + + def append(self, kind: str, run_id: str, **fields) -> None: + row = {"kind": kind, "run_id": run_id, "ts": time.time(), **fields} + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.path.open("a") as handle: + handle.write(json.dumps(row) + "\n") + + def read(self) -> list[dict]: + if not self.path.is_file(): + return [] + rows = [] + with self.path.open("r") as handle: + for line in handle: + line = line.strip() + if line: + rows.append(json.loads(line)) + return rows + + def runs(self) -> dict[str, list[dict]]: + """Group events by run_id, preserving insertion order of first sighting.""" + grouped: dict[str, list[dict]] = {} + for row in self.read(): + grouped.setdefault(row.get("run_id", ""), []).append(row) + return grouped \ No newline at end of file diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py index 7263690..948d299 100644 --- a/tests/integration/test_pipeline.py +++ b/tests/integration/test_pipeline.py @@ -39,6 +39,7 @@ def test_pipeline_end_to_end(tmp_path): config = load_config() require_real(config) config["log"] = str(tmp_path / "decisions.jsonl") + config["trace"] = str(tmp_path / "runs.jsonl") scheduler, config = build_scheduler(config) inputs = [ @@ -55,6 +56,16 @@ def test_pipeline_end_to_end(tmp_path): rows = scheduler.log.read() assert len(rows) > 0, "expected SemIf decisions to be logged" + phases = [r.get("extra", {}).get("phase") for r in rows] + assert "navigate:category" in phases, "navigation decisions must be logged" + assert "navigate:leaf" in phases, "navigation decisions must be logged" + for row in rows: + assert row.get("extra", {}).get("run_id"), "every decision must carry a run_id" + + trace_rows = scheduler.trace.read() + assert any(r["kind"] == "submit" for r in trace_rows) + assert any(r["kind"] == "assessed" for r in trace_rows) + report = dream(scheduler.log) assert report.cross_entropy is not None print(report.render()) @@ -70,6 +81,7 @@ def test_busy_choice_path(tmp_path): config = load_config() require_real(config) config["log"] = str(tmp_path / "decisions.jsonl") + config["trace"] = str(tmp_path / "runs.jsonl") scheduler, config = build_scheduler(config) scheduler.busy("driving on the freeway", skill="driving") diff --git a/tests/test_dashboard_api.py b/tests/test_dashboard_api.py new file mode 100644 index 0000000..451430e --- /dev/null +++ b/tests/test_dashboard_api.py @@ -0,0 +1,148 @@ +"""Dashboard API tests against a real stdlib HTTP server on an ephemeral port. + +The scheduler is constructed with the lazy SemIfEngine (never loaded), so this +runs anywhere without SemIf. Submit degrades to a JSON error, which is the +expected behaviour on the thin dev box; relabel works against a seeded log. +""" + +import json +import threading +import urllib.error +import urllib.request +from http.server import ThreadingHTTPServer + +from semif_agent.dashboard import DashboardHandler +from semif_agent.decisions import DecisionRequest, DecisionResult, Option +from semif_agent.engine import EngineConfig, SemIfEngine +from semif_agent.llm import LLMClient +from semif_agent.log import DecisionLog +from semif_agent.scheduler import Scheduler +from semif_agent.trace import TraceLog + + +def build_scheduler(tmp_path): + log = DecisionLog(str(tmp_path / "decisions.jsonl")) + trace = TraceLog(str(tmp_path / "runs.jsonl")) + engine = SemIfEngine(EngineConfig()) + llm = LLMClient(base_url="http://localhost:1/v1", model="test") + scheduler = Scheduler( + engine=engine, + llm=llm, + log=log, + config={"skills": {}}, + trace=trace, + ) + return scheduler + + +def seed_decision(log, decision_id="abc123"): + request = DecisionRequest( + id=decision_id, + state="some state", + question="Pick one?", + options=[Option("a", "A."), Option("b", "B.")], + ) + result = DecisionResult( + request=request, option_ids=["a", "b"], probabilities=[0.3, 0.7] + ) + log.append(request, result, extra={"phase": "gate", "run_id": "run-1"}) + + +class Server: + def __init__(self, scheduler): + handler = type("Handler", (DashboardHandler,), {"scheduler": scheduler}) + self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), handler) + self.thread = threading.Thread(target=self.httpd.serve_forever, daemon=True) + self.thread.start() + self.port = self.httpd.server_address[1] + + def get(self, path): + with urllib.request.urlopen(f"http://127.0.0.1:{self.port}{path}") as res: + return res.status, json.loads(res.read().decode("utf-8")) + + def post(self, path, payload): + req = urllib.request.Request( + f"http://127.0.0.1:{self.port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req) as res: + return res.status, json.loads(res.read().decode("utf-8")) + + def close(self): + self.httpd.shutdown() + self.httpd.server_close() + + +def test_tree_endpoint(tmp_path): + server = Server(build_scheduler(tmp_path)) + try: + status, payload = server.get("/api/tree") + assert status == 200 + assert "email" in payload["categories"] + assert any(s["name"] == "email.compose" for s in payload["categories"]["email"]) + finally: + server.close() + + +def test_trace_endpoint_empty(tmp_path): + server = Server(build_scheduler(tmp_path)) + try: + status, payload = server.get("/api/trace") + assert status == 200 + assert payload["runs"] == [] + finally: + server.close() + + +def test_submit_without_engine_returns_error_json(tmp_path): + scheduler = build_scheduler(tmp_path) + server = Server(scheduler) + try: + status, payload = server.post("/api/submit", {"text": "do something"}) + assert status == 200 + assert payload["status"] == "error" + assert "engine" in payload["detail"] + finally: + server.close() + + +def test_relabel_roundtrip(tmp_path): + scheduler = build_scheduler(tmp_path) + seed_decision(scheduler.log) + server = Server(scheduler) + try: + status, payload = server.get("/api/trace") + assert status == 200 + assert payload["runs"] + assert payload["runs"][0]["decisions"][0]["id"] == "abc123" + + status, payload = server.post("/api/relabel", {"id": "abc123", "outcome": "b"}) + assert status == 200 + assert payload["ok"] is True + + status, payload = server.get("/api/trace") + row = payload["runs"][0]["decisions"][0] + assert row["observed_outcome"] == "b" + assert row["label_source"] == "human" + assert row["cost"]["weight"] == 3.0 + + status, payload = server.post("/api/relabel", {"id": "nope", "outcome": "a"}) + assert payload["ok"] is False + finally: + server.close() + + +def test_submit_trace_event_recorded_even_when_engine_missing(tmp_path): + scheduler = build_scheduler(tmp_path) + server = Server(scheduler) + try: + server.post("/api/submit", {"text": "hello"}) + status, payload = server.get("/api/trace") + runs = payload["runs"] + assert len(runs) == 1 + kinds = [e["kind"] for e in runs[0]["events"]] + assert "submit" in kinds + finally: + server.close() \ No newline at end of file diff --git a/tests/test_trace.py b/tests/test_trace.py new file mode 100644 index 0000000..b703f7a --- /dev/null +++ b/tests/test_trace.py @@ -0,0 +1,33 @@ +from semif_agent.trace import TraceLog + + +def test_append_read_roundtrip(tmp_path): + trace = TraceLog(str(tmp_path / "runs.jsonl")) + trace.append("submit", "run-a", text="hello") + trace.append("queued", "run-a", weight=0.5, label="medium") + trace.append("submit", "run-b", text="world") + + rows = trace.read() + assert len(rows) == 3 + assert rows[0]["kind"] == "submit" + assert rows[0]["run_id"] == "run-a" + assert rows[0]["text"] == "hello" + assert rows[1]["label"] == "medium" + + +def test_runs_groups_by_run_id_preserving_order(tmp_path): + trace = TraceLog(str(tmp_path / "runs.jsonl")) + trace.append("submit", "run-a", text="x") + trace.append("submit", "run-b", text="y") + trace.append("assessed", "run-a", success=True) + + runs = trace.runs() + assert list(runs) == ["run-a", "run-b"] + assert len(runs["run-a"]) == 2 + assert runs["run-b"][0]["text"] == "y" + + +def test_read_missing_file_is_empty(tmp_path): + trace = TraceLog(str(tmp_path / "none.jsonl")) + assert trace.read() == [] + assert trace.runs() == {} \ No newline at end of file