Automate create_category via normal-mode generation of the decision model

.gitignore

@@ -3,6 +3,7 @@ __pycache__/
 .venv/
 data/decisions.jsonl
 data/runs.jsonl
+data/categories.json
 data/drafts/
 .pytest_cache/
 config.json

AGENTS.md

@@ -22,8 +22,9 @@ cli.py          argparse: run (REPL / --script), dream, skills, status, relabel,
 scheduler.py    gate -> choice(tau) -> score -> queue; preempt + requeue
 queue.py        urgency max-heap (desc weight, FIFO seq), age pulls toward 1.0
 skills.py       tree + registry (email.compose, response.reject, tracking.check),
-                navigation = SemIf choices per level (logged), create_skill
-                branch (stub)
+                navigation = SemIf choices per level (logged), create_category
+                authors + registers a category stub via the decision model in
+                generation mode; create_skill branch (stub)
 skill.py        loop: observe -> predict -> act -> observe -> assess (LLM)
 engine.py       SemIfEngine -> semif_phase1.llamacpp_backend (lazy import)
 llm.py          OpenAI-compatible client for self-assessment (stdlib urllib)
@@ -111,9 +112,12 @@ unit tests (24) + box integration tests (2).
   validate (accuracy/ECE on a held-out slice, prompt-hash regression), swap the
   pinned model revision. GPU offload: train on a beefier GPU; the running agent
   keeps a frozen inference revision until a swap validates.
-- `create_skill` / `create_category` branches: navigation logs a suggestion event
-  (state, query, SemIf output) to the trace — currently a stub; opencode
-  authoring at a tree leaf is deferred.
+- `create_skill` branch: navigation logs a suggestion event (state, query, SemIf
+  output) to the trace — currently a stub; opencode authoring at a tree leaf is
+  deferred. (`create_category` is live: the decision model, driven in normal
+  generation mode via `SemIfEngine.generate`, proposes a broad title +
+  description, and the stub is persisted to `data/categories.json` and merged
+  into the running tree.)
 - Queue persistence (durable across restarts).
 - Event/timer intake sources beyond typed input.
 - Concurrency: SemIf shared-state mode (`score_shared` / `SerialPrefixScorer`)

IDEA.md

@@ -41,7 +41,7 @@ All decisions are SemIf calls: `{state, question, options[]}`. State is the curr
 
 - Structure: categories → skills → actions. Top level listed at each level.
 - Navigation is a chain of SemIf choices, one per level, descending until a leaf skill matches.
-- Navigation offers a `create_category` suggestion at the category level and a `create_skill` suggestion at the leaf level. Both are stubs that log a suggestion event (state, query, SemIf output) to the trace — **opencode authors the skill** (its only role) and drops a skill manifest into the registry, deferred to v2. The new skill becomes a leaf immediately.
+- Navigation offers a `create_category` suggestion at the category level and a `create_skill` suggestion at the leaf level. `create_category` is live: the decision model, driven in normal generation mode, authors a broad title + description, and the stub is persisted to the category registry and merged into the running tree. `create_skill` logs a suggestion event (state, query, SemIf output) to the trace — **opencode authors the skill** (its only role) and drops a skill manifest into the registry, deferred to v2. The new skill becomes a leaf immediately.
 
 ### Skill manifest
 - name, category, description, allowed inputs, action list, cost budget, decision log reference.

config.example.json

@@ -20,5 +20,6 @@
   },
   "log": "data/decisions.jsonl",
   "trace": "data/runs.jsonl",
+  "category_registry": "data/categories.json",
   "dashboard": {"port": 8765, "host": "0.0.0.0"}
 }

semif_agent/engine.py

@@ -91,3 +91,27 @@ class SemIfEngine:
                 "total_seconds": result.get("total_seconds"),
             },
         )
+
+    def generate(
+        self,
+        messages: list[dict],
+        temperature: float = 0.2,
+        max_tokens: int = 256,
+    ) -> str:
+        """Drive the pinned decision model in the normal way: text generation.
+
+        SemIf scoring reads option logits directly; this instead uses the
+        underlying llama.cpp chat-completion endpoint on the same loaded model,
+        e.g. for skill-tree authoring. Each call resets the KV cache by
+        default, so interleaving scoring and generation on one model is safe.
+        """
+        model, tokenizer, metadata = self._ensure_loaded()
+        try:
+            reply = model.create_chat_completion(
+                messages=messages,
+                temperature=temperature,
+                max_tokens=max_tokens,
+            )
+        except Exception as exc:
+            raise EngineUnavailable(f"generation failed: {exc}") from exc
+        return reply["choices"][0]["message"]["content"].strip()

semif_agent/scheduler.py

@@ -17,11 +17,14 @@ from .queue import UrgencyQueue
 from .skill import SkillRunner
 from .skills import (
     ActionContext,
+    CategoryRegistry,
+    CreateCategory,
     CreateSkill,
     Skill,
     build_skills,
     build_tree,
     compose_state,
+    generate_category,
     navigate,
 )
 from .trace import TraceLog
@@ -76,6 +79,9 @@ class Scheduler:
         )
         self.skills = build_skills(config)
         self.tree = build_tree(self.skills)
+        self.registry = CategoryRegistry(config.get("category_registry", "data/categories.json"))
+        for category in self.registry.read():
+            self.tree.setdefault(category, [])
         self.ctx = ActionContext(engine=self.engine, config=config)
         self.runner = SkillRunner(self.ctx, self.llm, self.log)
         self.current: Process | None = None
@@ -199,6 +205,8 @@ class Scheduler:
 
     def _dispatch(self, request: Request) -> DispatchResult:
         navigation = navigate(self.engine, self.log, self.trace, request, self.tree)
+        if isinstance(navigation, CreateCategory):
+            return self._create_category(request)
         if isinstance(navigation, CreateSkill):
             return DispatchResult(
                 kind="create_skill",
@@ -228,6 +236,40 @@ class Scheduler:
             decisions_logged=outcome.decisions_logged,
         )
 
+    def _create_category(self, request: Request) -> DispatchResult:
+        """Author a new category stub with the decision model in generation mode."""
+        from .engine import EngineUnavailable
+
+        try:
+            draft = generate_category(self.engine, request, self.tree)
+        except (EngineUnavailable, ValueError) as exc:
+            self.trace.append("error", request.id, phase="create_category", message=str(exc))
+            return DispatchResult(kind="error", summary=f"create_category failed: {exc}")
+        if draft.name in self.tree:
+            self.trace.append(
+                "error",
+                request.id,
+                phase="create_category",
+                message=f"category {draft.name} already exists",
+            )
+            return DispatchResult(
+                kind="error",
+                summary=f"create_category failed: {draft.name} already exists",
+            )
+        self.registry.register(draft.name, draft.description)
+        self.tree[draft.name] = []
+        self.trace.append(
+            "category_created",
+            request.id,
+            category=draft.name,
+            description=draft.description,
+        )
+        return DispatchResult(
+            kind="create_category",
+            summary=f"created category {draft.name}: {draft.description}",
+            skill=draft.name,
+        )
+
     def status(self) -> str:
         lines = []
         current = f"{self.current.skill} ({self.current.request.id})" if self.current else "idle"

semif_agent/skills.py

@@ -2,8 +2,10 @@
 
 A skill is a leaf reached by a chain of SemIf choices (category -> skill).
 The category level carries a "create_category" branch and the leaf level a
-"create_skill" branch; both are stubs that log a suggestion event to the trace
-(deferred to v2 — no actual authoring yet).
+"create_skill" branch. create_category is live: the decision model is driven in
+normal generation mode to propose a title + description for a broad new
+category, which is persisted to a category registry and becomes a stub in the
+tree. create_skill is still a stub that logs a suggestion event (deferred).
 
 Only the real skills live here; navigation uses the real decision engine.
 """
@@ -12,12 +14,14 @@ from __future__ import annotations
 
 import json
 import os
+import re
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Callable
 
 from .decisions import DecisionRequest, Option, Request
 from .engine import SemIfEngine
+from .llm import LLMClient
 from .log import DecisionLog
 from .trace import TraceLog
 
@@ -58,14 +62,53 @@ class Skill:
 
 @dataclass
 class CreateSkill:
-    """Stub for a missing-category/skill suggestion at a tree level.
+    """Suggestion that the current category needs a new skill.
 
     Navigation logs the suggestion event to the trace; actual authoring is
-    deferred to v2. `category` is None for a new-category suggestion, else the
-    category that needs the new skill.
+    deferred to v2. `category` names the category that needs the new skill.
     """
 
-    category: str | None = None
+    category: str
+
+
+@dataclass
+class CreateCategory:
+    """Suggestion that the request needs a brand-new top-level category.
+
+    Unlike CreateSkill this is handled live: the decision model is used in
+    normal generation mode to author the category stub.
+    """
+
+
+@dataclass
+class CategoryDraft:
+    """An authored category stub: a broad bucket for future skills."""
+
+    name: str
+    description: str
+
+
+class CategoryRegistry:
+    """Persisted category stubs, one file on disk.
+
+    Format: {name: {"description": str, "skills": []}}. The empty skills list is
+    the slot that create_skill will fill later; for now a stub category has no
+    leaves.
+    """
+
+    def __init__(self, path: str = "data/categories.json"):
+        self.path = Path(path)
+
+    def read(self) -> dict[str, dict]:
+        if not self.path.is_file():
+            return {}
+        return json.loads(self.path.read_text())
+
+    def register(self, name: str, description: str) -> None:
+        categories = self.read()
+        categories[name] = {"description": description, "skills": []}
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.path.write_text(json.dumps(categories, indent=2) + "\n")
 
 
 def compose_state(request: Request, current: str | None = None) -> str:
@@ -172,12 +215,12 @@ def navigate(
     trace: TraceLog,
     request: Request,
     tree: dict[str, list[Skill]],
-) -> Skill | CreateSkill:
+) -> Skill | CreateCategory | CreateSkill:
     """Descend the tree one SemIf choice per level. Every choice is logged.
 
-    The category level offers a "create_category" branch and the leaf level a
-    "create_skill" branch; both log a suggestion event to the trace and return
-    a CreateSkill stub (actual authoring is deferred to v2).
+    The category level offers a "create_category" branch (handled live by
+    dispatch) and the leaf level a "create_skill" branch (still a stub); both
+    log a suggestion event to the trace.
     """
     categories = sorted(tree.keys())
     create_category = Option("create_category", "Suggest a new category for this.")
@@ -199,7 +242,7 @@ def navigate(
             selected=top_result.selected,
             probs=top_result.probs,
         )
-        return CreateSkill(category=None)
+        return CreateCategory()
     skills = tree[category]
     create_skill = Option("create_skill", "Suggest creating a new skill.")
     leaf = DecisionRequest(
@@ -231,3 +274,51 @@ def tree_summary(tree: dict[str, list[Skill]]) -> str:
         names = ", ".join(s.name for s in tree[category])
         lines.append(f"  {category}: {names}")
     return "\n".join(lines)
+
+
+def build_category_prompt(request: Request, tree: dict[str, list[Skill]]) -> list[dict]:
+    """Chat messages for the decision model used as the category author.
+
+    The category must be a general bucket that many tools could fit under, not
+    a single skill. The existing tree is included so the model avoids duplicating
+    categories and stays broad enough to be useful.
+    """
+    system = (
+        "You are the skill-tree authoring step of a local agent. A request did "
+        "not fit any existing category. Propose one new top-level category of "
+        "tools/skills that would encompass this request. It must be broad enough "
+        "that many tools could fit under it — a general-purpose bucket, not a "
+        "single skill. Reply with JSON only: "
+        '{"title": "<short lowercase snake_case id, no spaces>", '
+        '"description": "<one to two sentence purpose>"}'
+    )
+    user = (
+        f"Request: {request.text}\n"
+        f"Existing categories and their skills:\n{tree_summary(tree)}\n"
+        "Proposed new category (JSON only):"
+    )
+    return [
+        {"role": "system", "content": system},
+        {"role": "user", "content": user},
+    ]
+
+
+def parse_category_draft(raw: str) -> CategoryDraft:
+    """Parse the model's JSON reply into a CategoryDraft."""
+    parsed = LLMClient._parse_json(raw)
+    title = str(parsed.get("title", "")).strip()
+    description = str(parsed.get("description", "")).strip()
+    if not title or not description:
+        raise ValueError(f"category draft missing title/description: {raw!r}")
+    name = re.sub(r"\s+", "_", title.lower())
+    if not name.replace("_", "").isalnum():
+        raise ValueError(f"category title must be snake_case alnum: {title!r}")
+    return CategoryDraft(name=name, description=description)
+
+
+def generate_category(
+    engine: SemIfEngine, request: Request, tree: dict[str, list[Skill]]
+) -> CategoryDraft:
+    """Author a new category stub with the decision model in generation mode."""
+    raw = engine.generate(build_category_prompt(request, tree))
+    return parse_category_draft(raw)

tests/integration/test_pipeline.py

@@ -12,8 +12,10 @@ from pathlib import Path
 import pytest
 
 from semif_agent.cli import build_scheduler, load_config
+from semif_agent.decisions import Request
 from semif_agent.dream import dream
 from semif_agent.engine import EngineUnavailable
+from semif_agent.skills import CategoryDraft, generate_category
 
 
 def require_real(config: dict):
@@ -89,3 +91,34 @@ def test_busy_choice_path(tmp_path):
     print(f"[{status}] {detail}")
     assert status in ("preempted", "queued", "dropped")
     scheduler.idle()
+
+
+def test_engine_generation_normal_mode(tmp_path):
+    """The pinned decision model must also generate text in the normal way."""
+    config = load_config()
+    require_real(config)
+    scheduler, config = build_scheduler(config)
+    out = scheduler.engine.generate(
+        [
+            {"role": "system", "content": "Reply with the single word ok."},
+            {"role": "user", "content": "say ok"},
+        ],
+        max_tokens=16,
+    )
+    print(f"generation: {out!r}")
+    assert isinstance(out, str) and out.strip()
+
+
+def test_generate_category(tmp_path):
+    """Authoring a category stub through the real decision model."""
+    config = load_config()
+    require_real(config)
+    scheduler, config = build_scheduler(config)
+    draft = generate_category(
+        scheduler.engine,
+        Request("tell me if my package was delivered"),
+        scheduler.tree,
+    )
+    print(f"draft: {draft.name!r} — {draft.description!r}")
+    assert isinstance(draft, CategoryDraft)
+    assert draft.name and draft.description

tests/test_skills.py

@@ -0,0 +1,93 @@
+"""Pure-stdlib tests for skill-tree authoring: category prompts, draft parsing,
+the category registry, and the error path when the engine is unavailable.
+
+No mocking: engine-dependent success paths are exercised only by the box
+integration tests against the real decision model.
+"""
+
+import pytest
+
+from semif_agent.decisions import Request
+from semif_agent.engine import EngineConfig, EngineUnavailable, SemIfEngine
+from semif_agent.skills import (
+    CategoryDraft,
+    CategoryRegistry,
+    build_category_prompt,
+    build_skills,
+    build_tree,
+    generate_category,
+    parse_category_draft,
+)
+
+
+def test_build_category_prompt_contains_request_and_tree():
+    tree = build_tree(build_skills({"skills": {}}))
+    messages = build_category_prompt(Request("tracking for my drone delivery"), tree)
+    assert messages[0]["role"] == "system"
+    assert "category" in messages[0]["content"]
+    joined = messages[1]["content"]
+    assert "tracking for my drone delivery" in joined
+    assert "email: email.compose" in joined
+
+
+def test_parse_category_draft_plain_json():
+    draft = parse_category_draft(
+        '{"title": "delivery", "description": "Track and manage package deliveries."}'
+    )
+    assert draft.name == "delivery"
+    assert "package" in draft.description
+
+
+def test_parse_category_draft_json_in_prose():
+    draft = parse_category_draft(
+        'Sure! Here you go:\n{"title": "home_automation", "description": "Control '
+        'lights, locks, and appliances around the house."}\nHope that helps.'
+    )
+    assert draft.name == "home_automation"
+
+
+def test_parse_category_draft_normalizes_title():
+    draft = parse_category_draft(
+        '{"title": "Home Automation", "description": "Control household devices."}'
+    )
+    assert draft.name == "home_automation"
+
+
+def test_parse_category_draft_missing_fields_raises():
+    with pytest.raises(ValueError):
+        parse_category_draft('{"title": "only_title"}')
+    with pytest.raises(ValueError):
+        parse_category_draft("not json at all")
+
+
+def test_parse_category_draft_rejects_unclean_title():
+    with pytest.raises(ValueError):
+        parse_category_draft('{"title": "ca$h!", "description": "nope"}')
+
+
+def test_category_registry_roundtrip(tmp_path):
+    registry = CategoryRegistry(str(tmp_path / "categories.json"))
+    assert registry.read() == {}
+    registry.register("delivery", "Track and manage package deliveries.")
+    registry.register("delivery", "Track and manage package deliveries, re-registered.")
+    loaded = registry.read()
+    assert set(loaded) == {"delivery"}
+    assert loaded["delivery"]["description"] == (
+        "Track and manage package deliveries, re-registered."
+    )
+    assert loaded["delivery"]["skills"] == []
+
+
+def test_generate_category_without_engine_raises():
+    engine = SemIfEngine(EngineConfig())
+    with pytest.raises(EngineUnavailable):
+        generate_category(engine, Request("anything"), {})
+
+
+def test_build_tree_includes_registry_stubs(tmp_path):
+    registry = CategoryRegistry(str(tmp_path / "categories.json"))
+    registry.register("delivery", "Track and manage package deliveries.")
+    tree = build_tree(build_skills({"skills": {}}))
+    for category in registry.read():
+        tree.setdefault(category, [])
+    assert tree["delivery"] == []