From 440e49e76e1b07d7b99ad8b86ea6d2bc8f7d0c8c Mon Sep 17 00:00:00 2001 From: Denton Social Date: Thu, 24 Sep 2026 00:36:35 -0500 Subject: [PATCH] Author runnable skill bodies via OpenAI-compatible codegen model create_skill now writes a real predict/act body: the small decision model still authors title + description (engine.generate), then a larger OpenAI-compatible model (default qwen38-iq3s) writes the runnable code against the SKILL.md contract. Bodies persist to data/skills//.py, are hot-loaded via importlib, merged into the running tree, and the request re-dispatches to the new leaf. The dashboard decision-flow view shows the title/description with a writing badge while the body is being written. Codegen failure degrades to a navigable stub. --- .gitignore | 1 + AGENTS.md | 41 +++++- IDEA.md | 4 +- SKILL.md | 132 +++++++++++++++++ config.example.json | 7 + semif_agent/cli.py | 10 ++ semif_agent/codegen.py | 169 ++++++++++++++++++++++ semif_agent/scheduler.py | 99 +++++++++++-- semif_agent/skills.py | 109 ++++++++++++++ semif_agent/static/app.js | 35 +++++ semif_agent/static/style.css | 20 +++ tests/integration/test_pipeline.py | 44 +++++- tests/test_codegen.py | 219 +++++++++++++++++++++++++++++ tests/test_dashboard_api.py | 38 +++++ 14 files changed, 910 insertions(+), 18 deletions(-) create mode 100644 SKILL.md create mode 100644 semif_agent/codegen.py create mode 100644 tests/test_codegen.py diff --git a/.gitignore b/.gitignore index 5e7f776..7f8a391 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ __pycache__/ data/decisions.jsonl data/runs.jsonl data/categories.json +data/skills/ data/drafts/ .pytest_cache/ config.json diff --git a/AGENTS.md b/AGENTS.md index 63e55be..37bf176 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,9 +24,12 @@ queue.py urgency max-heap (desc weight, FIFO seq), age pulls toward 1.0 skills.py tree + registry (email.compose, response.reject, tracking.check), navigation = SemIf choices per level (logged), create_category and create_skill author + register stubs via the decision model - in generation mode + in generation mode; SkillBodyStore + materialize_skill persist + and hot-load runnable skill bodies from data/skills/ skill.py loop: observe -> predict -> act -> observe -> assess (LLM) engine.py SemIfEngine -> semif_phase1.llamacpp_backend (lazy import) +codegen.py CodegenClient (OpenAI-compatible) writes runnable skill bodies + against SKILL.md; parse/validate (compile + predict/act) llm.py OpenAI-compatible client for self-assessment (stdlib urllib) log.py decisions.jsonl rows {state, question, options, predicted_probs, selected, observed_outcome, label_source} @@ -116,12 +119,16 @@ unit tests (24) + box integration tests (2). validate (accuracy/ECE on a held-out slice, prompt-hash regression), swap the pinned model revision. GPU offload: train on a beefier GPU; the running agent keeps a frozen inference revision until a swap validates. -- `create_skill` branch: now live, mirroring `create_category`. The decision +- `create_skill` branch: live, mirroring `create_category`. The decision model, driven in normal generation mode via `SemIfEngine.generate`, proposes a specific skill title + description for the chosen category; the stub is persisted to `data/categories.json` (under that category's `skills` list) and - merged into the running tree as a leaf. Still deferred: a real skill body — - opencode authoring at a tree leaf remains future work. + merged into the running tree as a leaf. Since Sep 2026 the leaf also gets a + real runnable body: a larger OpenAI-compatible model (`codegen`, default + `qwen38-iq3s`) writes `predict`/`act` code against `SKILL.md`, persisted to + `data/skills/` and hot-loaded, then the request re-dispatches to the new + skill. Authoring is still a single pass — validating/reusing written bodies + across runs is future work. - Queue persistence (durable across restarts). - Event/timer intake sources beyond typed input. - Concurrency: SemIf shared-state mode (`score_shared` / `SerialPrefixScorer`) @@ -175,6 +182,25 @@ unit tests (24) + box integration tests (2). - If generation hangs with no log output, restart the service (`sudo systemctl restart ollama`) — the ROCm runner can wedge. +### codegen (skill bodies, box) +- Skill **bodies** are written by a separate OpenAI-compatible model, configured + under `codegen` in config.json (default model `qwen38-iq3s`, the 12G 27B + IQ3_S GGUF — huge/slow; a 3-bit 27B write can take 30-120s). Title + + description for new skills still come from the **small** decision model + (`engine.generate`); only the runnable code body uses codegen. +- Bodies are persisted to `data/skills//.py` (gitignored) and + loaded back at startup via `importlib`, so skills stay runnable across + restarts. `SKILL.md` at the repo root is the contract the codegen model is + prompted with — change it only with intent, it shapes every generated body. +- **Trust boundary**: generated skill code is executed locally (it is imported + as a module and its `predict`/`act` run in-process). The box is the intended + target; treat the endpoint as trusted. +- Flow in `scheduler._create_skill`: small model authors title+description → + trace `skill_writing` (dashboard shows title/description + a "writing skill + body…" badge) → sync codegen write → `materialize_skill` → hot-merge into the + tree → bounded re-dispatch so the request runs the new skill. Codegen failure + leaves a navigable stub and returns a graceful `create_skill` result. + ### Code principles - **No mocking.** The decision engine is always real SemIf; the LLM is always a real endpoint. Pure unit tests touch data-structure math only (queue ordering, @@ -199,7 +225,8 @@ unit tests (24) + box integration tests (2). - `python3 -m pytest tests/ -q --ignore=tests/integration` — anywhere, fast. Includes the dashboard API tests (`tests/test_dashboard_api.py`), which spin - up the stdlib HTTP server on an ephemeral port with the engine never loaded. + up the stdlib HTTP server on an ephemeral port with the engine never loaded, + and `tests/test_codegen.py` for prompt/parse/validate + body store round-trips. - `tests/integration/` — box only; requires real SemIf + real ollama. -- After touching scheduler/skills/engine, re-run both; the integration tests are - the only end-to-end verification. \ No newline at end of file +- After touching scheduler/skills/codegen/engine, re-run both; the integration + tests are the only end-to-end verification. \ No newline at end of file diff --git a/IDEA.md b/IDEA.md index f7b09ef..3ee52ab 100644 --- a/IDEA.md +++ b/IDEA.md @@ -41,7 +41,7 @@ All decisions are SemIf calls: `{state, question, options[]}`. State is the curr - Structure: categories → skills → actions. Top level listed at each level. - Navigation is a chain of SemIf choices, one per level, descending until a leaf skill matches. -- Navigation offers a `create_category` suggestion at the category level and a `create_skill` suggestion at the leaf level. Both are live: the decision model, driven in normal generation mode, authors a title + description (broad bucket for a category, single specific action for a skill), and the stub is persisted to the category registry and merged into the running tree. The new skill becomes a leaf immediately; a real skill body (opencode authoring a skill manifest) remains deferred to v2. +- Navigation offers a `create_category` suggestion at the category level and a `create_skill` suggestion at the leaf level. Both are live: the decision model, driven in normal generation mode, authors a title + description (broad bucket for a category, single specific action for a skill), and the stub is persisted to the category registry and merged into the running tree. For a new skill the stub is then promoted to a runnable body: a separate, larger OpenAI-compatible model writes the `predict`/`act` code against the `SKILL.md` contract, persisted under `data/skills/` and hot-loaded, and the request re-dispatches to the new leaf. ### Skill manifest - name, category, description, allowed inputs, action list, cost budget, decision log reference. @@ -97,7 +97,7 @@ Every skill run follows the same loop: - contains_request: yes - interrupt_current: yes - skill_selection: tracking => (no leaf) => create_skill -- opencode authors `tracking.check_delivery` from the input + registry conventions → manifest registered → requeue → skill_selection: tracking => check_delivery(input) +- the codegen model authors `tracking.check_delivery` (title+description from the decision model; `predict`/`act` body from a larger OpenAI-compatible model per `SKILL.md`) → body persisted + registered → requeue → skill_selection: tracking => check_delivery(input) ### Example 5 (self-assessment + dream) - current_process: email compose diff --git a/SKILL.md b/SKILL.md new file mode 100644 index 0000000..59e17f5 --- /dev/null +++ b/SKILL.md @@ -0,0 +1,132 @@ +# SKILL.md — the contract for new skills + +This file is the authoritative spec for what a new skill is and how its code +body must be written. It is fed verbatim to the code-generation model so every +generated skill is consistent, and it is read by humans who want to know what a +good skill looks like. + +## What a skill is + +A skill is a **leaf** in the agent's skill tree, reached by a chain of SemIf +decisions (category -> skill). It is one specific, single-purpose action the +agent can take — never a broad bucket (that is a category's job). It runs the +standard skill loop: observe -> predict -> act -> observe -> assess. + +A skill is two things: + +1. **A manifest** — the registry entry that makes it navigable and describes + what it does. +2. **A code body** — a runnable Python module implementing the `predict` and + `act` phases. + +## Manifest schema + +Registered in `data/categories.json` (and mirrored in the running tree). Fields: + +| Field | Meaning | +| -------------------- | -------------------------------------------------------------------- | +| `name` | `category.skill` dotted snake_case id. Lowercase letters, digits, `_` and `.` only (`[a-z0-9_]+(?:\.[a-z0-9_]+)*`). Single purpose. | +| `category` | The top-level category bucket the skill lives under. | +| `description` | One to two sentences: what the skill does, for navigation. | +| `allowed_inputs` | What the skill accepts / needs as input context. | +| `actions` | The concrete actions the skill takes. | +| `cost_budget` | Relative budget for one run; a run that exceeds it fails fast. | +| `decision_log_ref` | Reference to the decision rows this skill logged during a run. | + +Only `name` and `description` are required for a stub; the rest fill in as the +skill is exercised. + +## Code body contract + +The generated module is persisted to `data/skills//.py` and +imported at runtime. It must satisfy **all** of the following: + +### Required functions + +```python +def predict(ctx, request) -> Prediction: + """Forecast + make any SemIf sub-decisions. Return the prediction.""" + +def act(ctx, request, prediction) -> ActionResult: + """Execute the action. Return the result + new state.""" +``` + +- `ctx` is an `ActionContext` with `ctx.engine` (the real SemIf engine) and + `ctx.config` (the agent config dict). +- `request` is the `Request` being handled. +- `Prediction(text: str, decisions: list)` and + `ActionResult(action_log: str, new_state: str)` are imported from + `semif_agent.skills`; return those exact types. `decisions` carries any + `(DecisionRequest, DecisionResult)` pairs made during predict so they are + logged as training rows. + +### Rules (hard requirements) + +- **Stdlib only.** No third-party imports, no files outside the project. The + core agent is pure-stdlib and runs on the thin dev box. +- **No mocking.** Sub-decisions use the real engine: build a + `DecisionRequest(state, question, options=[Option(id, description), ...])` + and call `ctx.engine.call(decision)`; return it inside `Prediction.decisions`. +- **Never swallow the request.** If the skill cannot act, return an + `ActionResult` with a short `action_log` explaining why and set `new_state` + back to `request.text`. +- **Write files under configured data dirs only** (e.g. `ctx.config["drafts"]`), + never anywhere else on disk. +- **Fail fast on budget.** Keep the work small; do not loop or retry in code. +- **Names match the manifest.** The module is imported as its manifest name; + the functions are `predict` and `act` exactly. + +## Conventions + +- Single purpose, single file, single module. +- Avoid duplicating an existing skill in the same category. +- `predict` resolves ambiguity (arguments, recipients, targets) with SemIf + sub-decisions, mirroring how `email.compose` resolves its recipient. +- `act` performs the concrete action and writes a human-readable `action_log` + that the self-assessment LLM can judge. + +## Acceptance criteria + +A generated skill is accepted only if: + +1. It compiles (`compile(..., "exec")` succeeds) and defines both `predict` + and `act`. +2. Its `name` matches the manifest regex and its `category` is given. +3. Its body imports nothing outside the stdlib and the agent package. +4. It uses `ctx.engine` (never mocks) and returns proper `Prediction` / + `ActionResult` types. +5. It is single-purpose and does not duplicate an existing category leaf. + +## Worked example + +`email.compose` resolves its recipient with a SemIf sub-decision, then writes a +draft file: + +```python +def predict(ctx, request): + contacts = read_contacts(ctx.config) # config-driven, local data + decision = DecisionRequest( + state=f"{request.text} [current process: none]", + question="Which contact is the intended recipient?", + options=[Option(c["name"], c.get("description", "")) for c in contacts] + + [Option("none", "None of the listed contacts.")], + ) + result = ctx.engine.call(decision) + return Prediction(text=f"recipient is {result.selected}", + decisions=[(decision, result)]) + +def act(ctx, request, prediction): + recipient = prediction.text.removeprefix("recipient is ") + drafts = Path(ctx.config.get("drafts", "data/drafts")) + drafts.mkdir(parents=True, exist_ok=True) + target = drafts / f"{request.id}.txt" + target.write_text(f"To: {recipient}\nBody: {request.text}\n") + return ActionResult( + action_log=f"email.compose: wrote draft {target} for {recipient!r}.", + new_state=f"Draft written to {target.name} for {recipient}.", + ) +``` + +Write skill bodies in this shape: resolve ambiguity in `predict` via +`ctx.engine`, do the work in `act`, keep both stdlib-only, and return the +proper types. \ No newline at end of file diff --git a/config.example.json b/config.example.json index a6e0dbd..351bea9 100644 --- a/config.example.json +++ b/config.example.json @@ -12,6 +12,13 @@ "threads": 8 }, "llm": {"base_url": "http://localhost:11434/v1", "model": "qwen3.5:4b"}, + "codegen": { + "_comment": "OpenAI-compatible model that writes runnable skill bodies. Larger/slower than the decision or self-assessment model.", + "base_url": "http://localhost:11434/v1", + "model": "qwen38-iq3s", + "timeout": 600 + }, + "skill_bodies": "data/skills", "skills": { "email": {"cost_budget": 1.0}, "contacts": "data/contacts.json", diff --git a/semif_agent/cli.py b/semif_agent/cli.py index 7df0613..157e4fb 100644 --- a/semif_agent/cli.py +++ b/semif_agent/cli.py @@ -18,6 +18,7 @@ import sys from pathlib import Path from .dream import dream as run_dream +from .codegen import CodegenClient from .engine import EngineConfig, EngineUnavailable, SemIfEngine from .llm import LLMClient from .log import DecisionLog @@ -45,6 +46,14 @@ def build_scheduler(config: dict) -> tuple[Scheduler, dict]: base_url=config.get("llm", {}).get("base_url", "http://localhost:11434/v1"), model=config.get("llm", {}).get("model", "qwen2.5:3b"), ) + codegen_cfg = config.get("codegen", {}) + codegen = CodegenClient( + base_url=codegen_cfg.get( + "base_url", config.get("llm", {}).get("base_url", "http://localhost:11434/v1") + ), + model=codegen_cfg.get("model", "qwen38-iq3s"), + timeout=float(codegen_cfg.get("timeout", 600.0)), + ) log = DecisionLog(config.get("log", "data/decisions.jsonl")) trace = TraceLog(config.get("trace", "data/runs.jsonl")) scheduler = Scheduler( @@ -55,6 +64,7 @@ def build_scheduler(config: dict) -> tuple[Scheduler, dict]: tau=float(config.get("tau", 0.6)), max_reentries=int(config.get("max_reentries", 3)), trace=trace, + codegen=codegen, ) return scheduler, config diff --git a/semif_agent/codegen.py b/semif_agent/codegen.py new file mode 100644 index 0000000..6d83f9a --- /dev/null +++ b/semif_agent/codegen.py @@ -0,0 +1,169 @@ +"""Skill code-body generation through an OpenAI-compatible API. + +The decision engine authors a skill's title + description with the small model +(engine.generate). Writing the runnable body is a separate step: a larger +OpenAI-compatible model (e.g. qwen38-iq3s on ollama) is prompted with the +SKILL.md contract plus the request and the existing tree, and must reply with +valid Python implementing `predict` / `act`. Stdlib-only HTTP, mirroring +`llm.py`. +""" + +from __future__ import annotations + +import ast +import json +import urllib.error +import urllib.request +from pathlib import Path + +from .decisions import Request +from .skills import SkillDraft, tree_summary + +DEFAULT_CONTRACT = Path(__file__).resolve().parent.parent / "SKILL.md" + + +class CodegenError(RuntimeError): + """The codegen endpoint could not be reached.""" + + +def read_skill_contract(path: str | None = None) -> str: + contract = Path(path) if path else DEFAULT_CONTRACT + if not contract.is_file(): + raise CodegenError(f"skill contract not found: {contract}") + return contract.read_text() + + +class CodegenClient: + """Minimal OpenAI-compatible chat client for writing skill bodies.""" + + def __init__(self, base_url: str, model: str, timeout: float = 600.0): + self.base_url = base_url.rstrip("/") + self.model = model + self.timeout = timeout + + def chat(self, messages: list[dict], max_tokens: int = 2048, temperature: float = 0.0) -> str: + url = f"{self.base_url}/chat/completions" + body = json.dumps( + { + "model": self.model, + "messages": messages, + "temperature": temperature, + "max_tokens": max_tokens, + } + ).encode("utf-8") + request = urllib.request.Request( + url, data=body, headers={"Content-Type": "application/json"} + ) + try: + with urllib.request.urlopen(request, timeout=self.timeout) as response: + payload = json.loads(response.read().decode("utf-8")) + except urllib.error.URLError as exc: + raise CodegenError( + f"codegen endpoint unreachable at {url}: {exc}. Is your local server running?" + ) from exc + return payload["choices"][0]["message"]["content"] + + +def build_skill_body_prompt( + request: Request, + category: str, + draft: SkillDraft, + tree: dict, + contract: str, +) -> list[dict]: + """Messages for the code-generation model. + + The small model already chose the title + description; the big model only + writes the runnable body against the SKILL.md contract, informed by the + request, the category, and the existing skills so it avoids duplication. + """ + existing = ", ".join(s.name for s in tree.get(category, [])) or "(none)" + system = ( + "You write runnable skill bodies for a local agent. The contract below " + "is authoritative: follow it exactly.\n\n" + f"{contract}" + ) + user = ( + f"Request: {request.text}\n" + f"Category: {category}\n" + f"Skill name: {draft.name}\n" + f"Skill description: {draft.description}\n" + f"Existing skills in this category: {existing}\n" + f"Existing categories:\n{tree_summary(tree)}\n" + "Write the Python module body now. Reply with ONLY valid Python code " + "defining `predict` and `act`. No prose, no markdown fences, no JSON." + ) + return [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ] + + +def parse_skill_body(raw: str) -> str: + """Extract and validate a Python skill body from the model's reply. + + Accepts bare code, ```fenced``` code, or JSON {"code": "..."}. The body + must parse and must define module-level `predict` and `act` functions. + Returns the cleaned source. Raises ValueError otherwise. + """ + text = raw.strip() + if "code" in text[:400] and "{" in text and "}" in text: + start, end = text.find("{"), text.rfind("}") + try: + parsed = json.loads(text[start : end + 1]) + candidate = parsed.get("code") + if isinstance(candidate, str): + text = candidate.strip() + except (ValueError, AttributeError): + pass + for fence in ("```python", "```py", "```"): + start = text.find(fence) + if start != -1: + text = text[start + len(fence):] + close = text.rfind("```") + if close != -1: + text = text[:close] + break + text = text.strip() + if not text: + raise ValueError("skill body is empty") + try: + module = ast.parse(text, filename="") + except SyntaxError as exc: + raise ValueError(f"skill body is not valid Python: {exc}") from exc + names = {node.name for node in module.body if isinstance(node, ast.FunctionDef)} + for required in ("predict", "act"): + if required not in names: + raise ValueError(f"skill body must define a module-level `{required}` function") + return text + + +def generate_skill_body( + client: CodegenClient, + request: Request, + category: str, + draft: SkillDraft, + tree: dict, + contract: str | None = None, + max_tokens: int = 2048, +) -> str: + """Author a skill body with the big model; retries once on invalid output.""" + contract_text = contract if contract is not None else read_skill_contract() + messages = build_skill_body_prompt(request, category, draft, tree, contract_text) + last_error: Exception | None = None + for attempt in range(2): + try: + raw = client.chat(messages, max_tokens=max_tokens) + return parse_skill_body(raw) + except ValueError as exc: + last_error = exc + messages = messages + [ + { + "role": "user", + "content": ( + f"That was rejected: {exc}. Reply with ONLY the Python code " + "now — no prose, no fences, no JSON." + ), + } + ] + raise ValueError(f"skill body rejected twice: {last_error}") \ No newline at end of file diff --git a/semif_agent/scheduler.py b/semif_agent/scheduler.py index f3e3086..8d409c5 100644 --- a/semif_agent/scheduler.py +++ b/semif_agent/scheduler.py @@ -9,6 +9,7 @@ from __future__ import annotations from dataclasses import dataclass, field +from .codegen import CodegenClient, CodegenError, generate_skill_body from .decisions import DecisionRequest, Option, Request from .engine import SemIfEngine from .llm import LLMClient @@ -21,12 +22,15 @@ from .skills import ( CreateCategory, CreateSkill, Skill, + SkillBodyStore, build_skills, build_tree, compose_state, generate_category, generate_skill, + materialize_skill, merge_registry, + merge_skill_bodies, navigate, ) from .trace import TraceLog @@ -55,6 +59,7 @@ class DispatchResult: summary: str skill: str | None = None decisions_logged: int = 0 + body_written: bool = False class Scheduler: @@ -67,6 +72,7 @@ class Scheduler: tau: float = 0.6, max_reentries: int = 3, trace: TraceLog | None = None, + codegen: CodegenClient | None = None, ): self.engine = engine self.llm = llm @@ -75,6 +81,7 @@ class Scheduler: self.config = config self.tau = tau self.max_reentries = max_reentries + self.codegen = codegen self.queue = UrgencyQueue( max_size=int(config.get("queue", {}).get("max_size", 100)), age_rate=float(config.get("queue", {}).get("age_rate", 0.0)), @@ -82,7 +89,9 @@ class Scheduler: self.skills = build_skills(config) self.tree = build_tree(self.skills) self.registry = CategoryRegistry(config.get("category_registry", "data/categories.json")) + self.body_store = SkillBodyStore(config.get("skill_bodies", "data/skills")) merge_registry(self.tree, self.registry.read()) + merge_skill_bodies(self.tree, self.body_store, self.registry.read()) self.ctx = ActionContext(engine=self.engine, config=config) self.runner = SkillRunner(self.ctx, self.llm, self.log) self.current: Process | None = None @@ -210,22 +219,32 @@ class Scheduler: # ---- dispatch ---- - def _dispatch(self, request: Request) -> DispatchResult: + def _dispatch(self, request: Request, _depth: int = 0) -> DispatchResult: navigation = navigate(self.engine, self.log, self.trace, request, self.tree) if isinstance(navigation, CreateCategory): return self._create_category(request) if isinstance(navigation, CreateSkill): - return self._create_skill(request, navigation.category) - outcome = self.runner.run(navigation, request) + created = self._create_skill(request, navigation.category) + if created.kind == "create_skill" and created.body_written and _depth < self.max_reentries: + requeued = _requeue(request, request.text) + self.trace.append( + "requeued", request.id, text=request.text, reason="skill created" + ) + return self._dispatch(requeued, _depth=_depth + 1) + return created + return self._run_skill(navigation, request) + + def _run_skill(self, skill: Skill, request: Request) -> DispatchResult: + outcome = self.runner.run(skill, request) if outcome.error: self.trace.append( - "error", request.id, skill=navigation.name, message=outcome.error + "error", request.id, skill=skill.name, message=outcome.error ) return DispatchResult(kind="error", summary=f"skill error: {outcome.error}") self.trace.append( "assessed", request.id, - skill=navigation.name, + skill=skill.name, success=outcome.success, summary=outcome.summary, updated_request=outcome.updated_request, @@ -235,8 +254,8 @@ class Scheduler: self.trace.append("requeued", request.id, text=outcome.updated_request) return DispatchResult( kind="ran", - summary=f"{navigation.name}: {'ok' if outcome.success else 'failed'} — {outcome.summary}", - skill=navigation.name, + summary=f"{skill.name}: {'ok' if outcome.success else 'failed'} — {outcome.summary}", + skill=skill.name, decisions_logged=outcome.decisions_logged, ) @@ -275,7 +294,14 @@ class Scheduler: ) def _create_skill(self, request: Request, category: str) -> DispatchResult: - """Author a new skill leaf stub with the decision model in generation mode.""" + """Author a new skill leaf with the decision model in generation mode. + + The small model writes the title + description; a larger OpenAI- + compatible model then writes the runnable body against SKILL.md. The + stub is registered first so the leaf is navigable even if the body + write fails; a successful write is merged into the tree as a runnable + skill and the request re-dispatches to it. + """ from .engine import EngineUnavailable try: @@ -296,21 +322,78 @@ class Scheduler: kind="error", summary=f"create_skill failed: {draft.name} already exists", ) + self.registry.register_skill(category, draft.name, draft.description) self.tree.setdefault(category, []).append( Skill(name=draft.name, category=category, description=draft.description) ) + self.trace.append( + "skill_writing", + request.id, + category=category, + skill=draft.name, + description=draft.description, + model=self.codegen.model if self.codegen else None, + ) + + if self.codegen is None: + self.trace.append( + "skill_created", + request.id, + category=category, + skill=draft.name, + description=draft.description, + body=None, + written=False, + ) + return DispatchResult( + kind="create_skill", + summary=f"created stub {category}.{draft.name}: {draft.description} (no codegen configured)", + skill=draft.name, + ) + + try: + draft.code = generate_skill_body( + self.codegen, request, category, draft, self.tree + ) + skill = materialize_skill(draft, category, self.body_store) + except (CodegenError, ValueError) as exc: + self.trace.append( + "error", + request.id, + phase="create_skill", + category=category, + message=f"skill body write failed: {exc}", + ) + return DispatchResult( + kind="create_skill", + summary=f"created stub {category}.{draft.name}: {draft.description} (body write failed: {exc})", + skill=draft.name, + ) + + skills = self.tree.setdefault(category, []) + for index, existing in enumerate(skills): + if existing.name == draft.name: + skills[index] = skill + break + else: + skills.append(skill) + skills.sort(key=lambda s: s.name) + body_path = self.body_store.body_path(category, draft.name).as_posix() self.trace.append( "skill_created", request.id, category=category, skill=draft.name, description=draft.description, + body=body_path, + written=True, ) return DispatchResult( kind="create_skill", summary=f"created skill {category}.{draft.name}: {draft.description}", skill=draft.name, + body_written=True, ) def status(self) -> str: diff --git a/semif_agent/skills.py b/semif_agent/skills.py index 68d1752..438a8b0 100644 --- a/semif_agent/skills.py +++ b/semif_agent/skills.py @@ -12,6 +12,7 @@ Only the real skills live here; navigation uses the real decision engine. from __future__ import annotations +import importlib.util import json import os import re @@ -95,6 +96,7 @@ class SkillDraft: name: str description: str + code: str = "" class CategoryRegistry: @@ -130,6 +132,113 @@ class CategoryRegistry: self.path.write_text(json.dumps(categories, indent=2) + "\n") +class SkillBodyStore: + """Persists runnable skill bodies as one Python file per skill. + + Layout: //.py. Bodies are written by the codegen step + and loaded back at startup so skills stay runnable across restarts. + """ + + def __init__(self, path: str = "data/skills"): + self.path = Path(path) + + def write(self, category: str, name: str, code: str) -> Path: + directory = self.path / category + directory.mkdir(parents=True, exist_ok=True) + target = directory / f"{name}.py" + target.write_text(code.rstrip() + "\n") + return target + + def body_path(self, category: str, name: str) -> Path: + return self.path / category / f"{name}.py" + + def list_bodies(self) -> list[tuple[str, str]]: + if not self.path.is_dir(): + return [] + bodies = [] + for directory in sorted(self.path.iterdir()): + if not directory.is_dir(): + continue + for module in sorted(directory.glob("*.py")): + bodies.append((directory.name, module.stem)) + return bodies + + +def load_skill_module(category: str, name: str, base: str = "data/skills"): + """Import a persisted skill body and return its module.""" + path = Path(base) / category / f"{name}.py" + module_name = f"_skill_{category}_{name}".replace("-", "_") + spec = importlib.util.spec_from_file_location(module_name, path) + if spec is None or spec.loader is None: + raise ValueError(f"cannot load skill module: {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def materialize_skill( + draft: SkillDraft, category: str, store: SkillBodyStore +) -> Skill: + """Persist the draft's code body and build a runnable Skill from it.""" + if not draft.code: + raise ValueError(f"skill {draft.name} has no code body to materialize") + store.write(category, draft.name, draft.code) + try: + module = load_skill_module(category, draft.name, store.path) + except Exception as exc: + raise ValueError(f"skill {category}.{draft.name} body failed to import: {exc}") from exc + if not callable(getattr(module, "predict", None)) or not callable( + getattr(module, "act", None) + ): + raise ValueError(f"skill {category}.{draft.name} body must define predict and act") + return Skill( + name=draft.name, + category=category, + description=draft.description, + predict=module.predict, + act=module.act, + ) + + +def merge_skill_bodies( + tree: dict[str, list[Skill]], store: SkillBodyStore, registry: dict[str, dict] +) -> int: + """Upgrade persisted skill bodies in the tree to runnable skills. + + A body file makes a stub leaf executable; where the registry entry was lost + (or never written), the category is created and the description falls back + to the skill name. Returns the number of skills made runnable. + """ + upgraded = 0 + for category, name in store.list_bodies(): + description = "" + entry = registry.get(category, {}) + for skill in entry.get("skills", []): + if skill.get("name") == name: + description = skill.get("description", "") + try: + module = load_skill_module(category, name, store.path) + except Exception: + continue + skill = Skill( + name=name, + category=category, + description=description or name, + predict=module.predict, + act=module.act, + ) + skills = tree.setdefault(category, []) + for index, existing in enumerate(skills): + if existing.name == name: + skills[index] = skill + break + else: + skills.append(skill) + skills.sort(key=lambda s: s.name) + upgraded += 1 + return upgraded + + def compose_state(request: Request, current: str | None = None) -> str: parts = [request.text] if current: diff --git a/semif_agent/static/app.js b/semif_agent/static/app.js index b9cafef..f663da4 100644 --- a/semif_agent/static/app.js +++ b/semif_agent/static/app.js @@ -121,6 +121,11 @@ function eventRow(evt) { div.textContent = `queued (${evt.label})`; } else if (evt.kind === "preempted") { div.textContent = `preempted ${evt.preempted}`; + } else if (evt.kind === "skill_writing") { + div.textContent = `writing skill ${evt.skill} — ${evt.description} (${evt.model || "codegen"})`; + } else if (evt.kind === "skill_created") { + div.textContent = `created ${evt.skill}${evt.written ? " (body written)" : " (stub)"}`; + div.title = evt.body || evt.description || ""; } return div; } @@ -327,6 +332,36 @@ function eventNode(evt) { body.textContent = `interrupted ${evt.preempted}, requeued with state`; } else if (evt.kind === "dropped") { body.textContent = evt.reason || ""; + } else if (evt.kind === "skill_writing") { + node.classList.add("writing"); + const title = document.createElement("div"); + title.className = "skill-title"; + title.textContent = evt.skill; + body.appendChild(title); + const desc = document.createElement("div"); + desc.className = "muted"; + desc.textContent = evt.description || ""; + body.appendChild(desc); + const badge = document.createElement("span"); + badge.className = "writing-badge"; + badge.textContent = "writing skill body…"; + node.appendChild(badge); + } else if (evt.kind === "skill_created") { + const title = document.createElement("div"); + title.className = "skill-title"; + title.textContent = evt.skill; + body.appendChild(title); + const desc = document.createElement("div"); + desc.className = "muted"; + desc.textContent = evt.description || ""; + body.appendChild(desc); + if (evt.body) { + const p = document.createElement("div"); + p.className = "muted"; + p.textContent = `body: ${evt.body}`; + node.appendChild(p); + } + node.classList.add(evt.written ? "ok" : "stub"); } else { body.textContent = evt.summary || evt.text || ""; } diff --git a/semif_agent/static/style.css b/semif_agent/static/style.css index 816f22d..565e898 100644 --- a/semif_agent/static/style.css +++ b/semif_agent/static/style.css @@ -283,6 +283,26 @@ select { .node.event-node .evt-kind { text-transform: uppercase; color: var(--muted); font-size: 10px; } +.node.event-node .skill-title { font-weight: 700; } +.node.event-node.writing { border-left: 3px solid var(--warn); } +.node.event-node.stub { border-left: 3px solid var(--muted); } + +.writing-badge { + display: inline-block; + margin-top: 8px; + padding: 2px 8px; + border: 1px solid var(--warn); + border-radius: 10px; + color: var(--warn); + font-size: 10px; + animation: writing-pulse 1.2s ease-in-out infinite; +} + +@keyframes writing-pulse { + 0%, 100% { opacity: 1; } + 50% { opacity: 0.35; } +} + /* ---------- inspector / tree / status ---------- */ .right { border-right: none; } diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py index c59fedd..e610092 100644 --- a/tests/integration/test_pipeline.py +++ b/tests/integration/test_pipeline.py @@ -12,10 +12,20 @@ from pathlib import Path import pytest from semif_agent.cli import build_scheduler, load_config +from semif_agent.codegen import CodegenClient, generate_skill_body from semif_agent.decisions import Request from semif_agent.dream import dream from semif_agent.engine import EngineUnavailable -from semif_agent.skills import CategoryDraft, SkillDraft, generate_category, generate_skill +from semif_agent.skills import ( + CategoryDraft, + SkillBodyStore, + SkillDraft, + build_skills, + build_tree, + generate_category, + generate_skill, + materialize_skill, +) def require_real(config: dict): @@ -140,6 +150,38 @@ def test_generate_skill(tmp_path): assert draft.name and draft.description +def test_generate_skill_body_codegen(tmp_path): + """A real OpenAI-compatible model writes a runnable skill body. + + Slow: uses the big codegen model (qwen38-iq3s by default). Run this one in + the background and poll — long-lived ssh sessions get SIGHUP'd. + """ + config = load_config() + require_real(config) + codegen_cfg = config.get("codegen", {}) + client = CodegenClient( + base_url=codegen_cfg.get("base_url", "http://localhost:11434/v1"), + model=codegen_cfg.get("model", "qwen38-iq3s"), + timeout=float(codegen_cfg.get("timeout", 600.0)), + ) + tree = build_tree(build_skills({"skills": {}})) + draft = SkillDraft( + name="check_service", + description="Check whether a service is reachable.", + ) + code = generate_skill_body( + client, + Request("is my home server reachable right now?"), + "tracking", + draft, + tree, + ) + print(f"generated {len(code)} bytes of skill body") + store = SkillBodyStore(str(tmp_path / "skills")) + skill = materialize_skill(draft, "tracking", store) + assert callable(skill.predict) and callable(skill.act) + + def test_create_skill_empty_category_does_not_wedge(tmp_path): """A dispatch that lands on an empty category must not leave the scheduler wedged. diff --git a/tests/test_codegen.py b/tests/test_codegen.py new file mode 100644 index 0000000..faea347 --- /dev/null +++ b/tests/test_codegen.py @@ -0,0 +1,219 @@ +"""Pure-stdlib tests for skill code-body generation. + +Prompt building, draft parsing/validation, body persistence + import, and +tree hot-merge all run without SemIf or a real LLM. The only network usage is a +throwaway stdlib HTTP server that stands in for an OpenAI-compatible endpoint — +the CodegenClient itself is real, not mocked. +""" + +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest + +from semif_agent.codegen import ( + CodegenClient, + build_skill_body_prompt, + generate_skill_body, + parse_skill_body, + read_skill_contract, +) +from semif_agent.decisions import Request +from semif_agent.skills import ( + SkillBodyStore, + SkillDraft, + build_skills, + build_tree, + load_skill_module, + materialize_skill, + merge_skill_bodies, + merge_registry, +) + +GOOD_BODY = """\ +from semif_agent.decisions import DecisionRequest, Option +from semif_agent.skills import ActionResult, Prediction + +def predict(ctx, request): + return Prediction(text="ok", decisions=[]) + +def act(ctx, request, prediction): + return ActionResult(action_log="probe ran", new_state=request.text) +""" + + +def test_read_skill_contract_loads_contract(): + text = read_skill_contract() + assert "predict" in text and "act" in text + assert "data/skills" in text + + +def test_build_skill_body_prompt_includes_contract_request_and_draft(): + tree = build_tree(build_skills({"skills": {}})) + draft = SkillDraft(name="probe", description="Probe the service.") + messages = build_skill_body_prompt( + Request("check if the service is up"), "tracking", draft, tree, "THE CONTRACT" + ) + assert messages[0]["role"] == "system" + assert "THE CONTRACT" in messages[0]["content"] + joined = messages[1]["content"] + assert "check if the service is up" in joined + assert "probe" in joined + assert "tracking.check" in joined + + +@pytest.mark.parametrize( + "raw", + [ + GOOD_BODY, + "```python\n" + GOOD_BODY + "\n```", + json.dumps({"code": GOOD_BODY}), + "Here you go:\n```python\n" + GOOD_BODY + "\n```\nHope that helps.", + 'Sure: ' + json.dumps({"code": GOOD_BODY}) + ' (that was it)', + ], +) +def test_parse_skill_body_accepts_forms(raw): + code = parse_skill_body(raw) + assert "def predict" in code and "def act" in code + + +def test_parse_skill_body_rejects_empty(): + with pytest.raises(ValueError): + parse_skill_body("") + + +def test_parse_skill_body_rejects_invalid_python(): + with pytest.raises(ValueError): + parse_skill_body("def predict(:\n pass") + + +def test_parse_skill_body_rejects_missing_functions(): + with pytest.raises(ValueError): + parse_skill_body("def predict(ctx, request):\n return None") + + +def test_parse_skill_body_rejects_missing_act(): + with pytest.raises(ValueError): + parse_skill_body("def predict(ctx, request):\n return None\nx = 1") + + +def test_body_store_roundtrip(tmp_path): + store = SkillBodyStore(str(tmp_path / "skills")) + assert store.list_bodies() == [] + store.write("tracking", "probe", GOOD_BODY) + assert store.list_bodies() == [("tracking", "probe")] + target = store.body_path("tracking", "probe") + assert target.is_file() + assert "def predict" in target.read_text() + + +def test_load_skill_module_exposes_predict_act(tmp_path): + store = SkillBodyStore(str(tmp_path / "skills")) + store.write("tracking", "probe", GOOD_BODY) + module = load_skill_module("tracking", "probe", store.path) + assert callable(module.predict) and callable(module.act) + + +def test_materialize_skill_builds_runnable_skill(tmp_path): + store = SkillBodyStore(str(tmp_path / "skills")) + draft = SkillDraft(name="probe", description="Probe the service.", code=GOOD_BODY) + skill = materialize_skill(draft, "tracking", store) + assert skill.name == "probe" + assert skill.category == "tracking" + assert callable(skill.predict) and callable(skill.act) + + +def test_materialize_skill_requires_code(tmp_path): + store = SkillBodyStore(str(tmp_path / "skills")) + draft = SkillDraft(name="probe", description="Probe the service.") + with pytest.raises(ValueError): + materialize_skill(draft, "tracking", store) + + +def test_materialize_skill_rejects_import_failure(tmp_path): + store = SkillBodyStore(str(tmp_path / "skills")) + bad = "def predict(ctx, request):\n return None\n" + draft = SkillDraft(name="probe", description="Probe.", code=bad) + with pytest.raises(ValueError): + materialize_skill(draft, "tracking", store) + + +def test_merge_skill_bodies_upgrades_stub(tmp_path): + store = SkillBodyStore(str(tmp_path / "skills")) + store.write("tracking", "probe", GOOD_BODY) + tree = build_tree(build_skills({"skills": {}})) + registry = {"tracking": {"description": "", "skills": [{"name": "probe", "description": "Probe."}]}} + merge_registry(tree, registry) + upgraded = merge_skill_bodies(tree, store, registry) + assert upgraded == 1 + skill = next(s for s in tree["tracking"] if s.name == "probe") + assert callable(skill.predict) and callable(skill.act) + assert skill.description == "Probe." + + +def test_merge_skill_bodies_creates_missing_category(tmp_path): + store = SkillBodyStore(str(tmp_path / "skills")) + store.write("brand_new", "ping", GOOD_BODY) + tree = build_tree(build_skills({"skills": {}})) + upgraded = merge_skill_bodies(tree, store, {}) + assert upgraded == 1 + assert tree["brand_new"][0].name == "ping" + + +class _FakeOpenAI(BaseHTTPRequestHandler): + reply: str = GOOD_BODY + + def do_POST(self): + length = int(self.headers.get("Content-Length") or 0) + self.rfile.read(length) + body = json.dumps({"choices": [{"message": {"content": self.reply}}]}).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format, *args): + pass + + +def _fake_server(reply: str) -> tuple[ThreadingHTTPServer, str]: + handler = type("Handler", (_FakeOpenAI,), {"reply": reply}) + httpd = ThreadingHTTPServer(("127.0.0.1", 0), handler) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + return httpd, f"http://127.0.0.1:{httpd.server_address[1]}/v1" + + +def test_generate_skill_body_end_to_end(tmp_path): + httpd, base = _fake_server(GOOD_BODY) + try: + client = CodegenClient(base_url=base, model="test", timeout=10) + tree = build_tree(build_skills({"skills": {}})) + draft = SkillDraft(name="probe", description="Probe the service.") + code = generate_skill_body(client, Request("is the service up?"), "tracking", draft, tree) + assert "def predict" in code and "def act" in code + finally: + httpd.shutdown() + httpd.server_close() + + +def test_generate_skill_body_retries_then_fails(tmp_path): + httpd, base = _fake_server("this is not python at all") + try: + client = CodegenClient(base_url=base, model="test", timeout=10) + tree = build_tree(build_skills({"skills": {}})) + draft = SkillDraft(name="probe", description="Probe the service.") + with pytest.raises(ValueError): + generate_skill_body(client, Request("is the service up?"), "tracking", draft, tree) + finally: + httpd.shutdown() + httpd.server_close() + + +def test_codegen_client_unreachable_raises(tmp_path): + client = CodegenClient(base_url="http://127.0.0.1:1/v1", model="test", timeout=2) + tree = build_tree(build_skills({"skills": {}})) + draft = SkillDraft(name="probe", description="Probe the service.") + with pytest.raises(Exception): + generate_skill_body(client, Request("is the service up?"), "tracking", draft, tree) \ No newline at end of file diff --git a/tests/test_dashboard_api.py b/tests/test_dashboard_api.py index 451430e..c67c6eb 100644 --- a/tests/test_dashboard_api.py +++ b/tests/test_dashboard_api.py @@ -144,5 +144,43 @@ def test_submit_trace_event_recorded_even_when_engine_missing(tmp_path): assert len(runs) == 1 kinds = [e["kind"] for e in runs[0]["events"]] assert "submit" in kinds + finally: + server.close() + + +def test_skill_writing_and_created_events_in_payload(tmp_path): + scheduler = build_scheduler(tmp_path) + scheduler.trace.append("submit", "run-9", text="track my package") + scheduler.trace.append( + "skill_writing", + "run-9", + category="tracking", + skill="track_live", + description="Follow a package in real time.", + model="qwen38-iq3s", + ) + scheduler.trace.append( + "skill_created", + "run-9", + category="tracking", + skill="track_live", + description="Follow a package in real time.", + body="data/skills/tracking/track_live.py", + written=True, + ) + server = Server(scheduler) + try: + status, payload = server.get("/api/trace") + assert status == 200 + run = next(r for r in payload["runs"] if r["run_id"] == "run-9") + kinds = [e["kind"] for e in run["events"]] + assert "skill_writing" in kinds and "skill_created" in kinds + writing = next(e for e in run["events"] if e["kind"] == "skill_writing") + assert writing["skill"] == "track_live" + assert "real time" in writing["description"] + assert writing["model"] == "qwen38-iq3s" + created = next(e for e in run["events"] if e["kind"] == "skill_created") + assert created["written"] is True + assert created["body"] == "data/skills/tracking/track_live.py" finally: server.close() \ No newline at end of file