deterministically execute create_skill from create_category, execute skill after creation

This commit is contained in:
Denton Social
2026-09-24 02:31:12 -05:00
parent 5bdfa1d91d
commit 300c20714b
5 changed files with 90 additions and 21 deletions
+10 -6
View File
@@ -126,9 +126,11 @@ unit tests (24) + box integration tests (2).
merged into the running tree as a leaf. Since Sep 2026 the leaf also gets a
real runnable body: a larger OpenAI-compatible model (`codegen`, default
`qwen38-iq3s`) writes `predict`/`act` code against `SKILL.md`, persisted to
`data/skills/` and hot-loaded, then the request re-dispatches to the new
skill. Authoring is still a single pass — validating/reusing written bodies
across runs is future work.
`data/skills/` and hot-loaded, then the newly created leaf is executed
directly so the request that prompted creation is answered. A request that
prompted a whole new category runs the same chain deterministically:
`create_category` → `create_skill` → run. Authoring is still a single pass —
validating/reusing written bodies across runs is future work.
- Queue persistence (durable across restarts).
- Event/timer intake sources beyond typed input.
- Concurrency: SemIf shared-state mode (`score_shared` / `SerialPrefixScorer`)
@@ -202,11 +204,13 @@ unit tests (24) + box integration tests (2).
- **Trust boundary**: generated skill code is executed locally (it is imported
as a module and its `predict`/`act` run in-process). The box is the intended
target; treat the endpoint as trusted.
- Flow in `scheduler._create_skill`: small model authors title+description →
- Flow in `scheduler._dispatch_skill`: small model authors title+description →
trace `skill_writing` (dashboard shows title/description + a "writing skill
body…" badge) → sync codegen write → `materialize_skill` → hot-merge into the
tree → bounded re-dispatch so the request runs the new skill. Codegen failure
leaves a navigable stub and returns a graceful `create_skill` result.
tree → the new leaf runs directly so the request is answered. `create_category`
runs the same chain after authoring the category (`create_category` →
`create_skill` → run). Codegen failure leaves a navigable stub and returns a
graceful `create_skill` result.
### Code principles
- **No mocking.** The decision engine is always real SemIf; the LLM is always a
+1 -1
View File
@@ -41,7 +41,7 @@ All decisions are SemIf calls: `{state, question, options[]}`. State is the curr
- Structure: categories → skills → actions. Top level listed at each level.
- Navigation is a chain of SemIf choices, one per level, descending until a leaf skill matches.
- Navigation offers a `create_category` suggestion at the category level and a `create_skill` suggestion at the leaf level. Both are live: the decision model, driven in normal generation mode, authors a title + description (broad bucket for a category, single specific action for a skill), and the stub is persisted to the category registry and merged into the running tree. For a new skill the stub is then promoted to a runnable body: a separate, larger OpenAI-compatible model writes the `predict`/`act` code against the `SKILL.md` contract, persisted under `data/skills/` and hot-loaded, and the request re-dispatches to the new leaf.
- Navigation offers a `create_category` suggestion at the category level and a `create_skill` suggestion at the leaf level. Both are live: the decision model, driven in normal generation mode, authors a title + description (broad bucket for a category, single specific action for a skill), and the stub is persisted to the category registry and merged into the running tree. For a new skill the stub is then promoted to a runnable body: a separate, larger OpenAI-compatible model writes the `predict`/`act` code against the `SKILL.md` contract, persisted under `data/skills/` and hot-loaded. The newly created leaf is then executed directly (no re-dispatch through navigation) so the request that prompted creation is answered: `create_category``create_skill` → run, or `create_skill` → run.
### Skill manifest
- name, category, description, allowed inputs, action list, cost budget, decision log reference.
+25 -12
View File
@@ -55,7 +55,7 @@ class Process:
@dataclass
class DispatchResult:
kind: str # ran | create_skill | error
kind: str # ran | create_category | create_skill | error
summary: str
skill: str | None = None
decisions_logged: int = 0
@@ -219,21 +219,34 @@ class Scheduler:
# ---- dispatch ----
def _dispatch(self, request: Request, _depth: int = 0) -> DispatchResult:
def _dispatch(self, request: Request) -> DispatchResult:
navigation = navigate(self.engine, self.log, self.trace, request, self.tree)
if isinstance(navigation, CreateCategory):
return self._create_category(request)
created = self._create_category(request)
if created.kind != "create_category":
return created
return self._dispatch_skill(request, created.skill)
if isinstance(navigation, CreateSkill):
created = self._create_skill(request, navigation.category)
if created.kind == "create_skill" and created.body_written and _depth < self.max_reentries:
requeued = _requeue(request, request.text)
self.trace.append(
"requeued", request.id, text=request.text, reason="skill created"
)
return self._dispatch(requeued, _depth=_depth + 1)
return created
return self._dispatch_skill(request, navigation.category)
return self._run_skill(navigation, request)
def _dispatch_skill(self, request: Request, category: str) -> DispatchResult:
"""create_skill in `category`, then run the new skill so the request is answered.
The created leaf is executed directly, not via a re-dispatch that would
re-run navigation on a tree that just changed.
"""
created = self._create_skill(request, category)
if created.kind != "create_skill" or not created.body_written:
return created
skill = next(
(s for s in self.tree.get(category, []) if s.name == created.skill),
None,
)
if skill is None:
return created
return self._run_skill(skill, request)
def _run_skill(self, skill: Skill, request: Request) -> DispatchResult:
outcome = self.runner.run(skill, request)
if outcome.error:
@@ -300,7 +313,7 @@ class Scheduler:
compatible model then writes the runnable body against SKILL.md. The
stub is registered first so the leaf is navigable even if the body
write fails; a successful write is merged into the tree as a runnable
skill and the request re-dispatches to it.
skill and executed directly by _dispatch_skill.
"""
from .engine import EngineUnavailable
+29 -1
View File
@@ -206,4 +206,32 @@ def test_create_skill_empty_category_does_not_wedge(tmp_path):
status, detail = scheduler.submit("tell me if my package was delivered")
print(f"[{status}] {detail}")
assert scheduler.current is None
assert scheduler.current is None
def test_create_category_chain_runs_new_skill(tmp_path):
"""A request that needs a brand-new category must end with a skill run.
Deterministic chain: create_category -> create_skill in the new category ->
run that skill (the leaf answers the request, not the category stub). Slow:
uses real codegen (~7 min). Run in the background.
"""
config = load_config()
require_real(config)
config["log"] = str(tmp_path / "decisions.jsonl")
config["trace"] = str(tmp_path / "runs.jsonl")
config["category_registry"] = str(tmp_path / "categories.json")
config["skill_bodies"] = str(tmp_path / "skills")
scheduler, config = build_scheduler(config)
scheduler.tree = {}
status, detail = scheduler.submit("track my drone delivery in real time")
print(f"[{status}] {detail}")
rows = scheduler.trace.read()
kinds = [e["kind"] for e in rows]
assert "category_created" in kinds, "category stub must be authored first"
created = next(e for e in rows if e["kind"] == "skill_created")
assert created["written"] is True, "codegen must produce a runnable body"
assessed = next(e for e in rows if e["kind"] == "assessed")
assert assessed["skill"] == created["skill"], "the created skill must run"
+25 -1
View File
@@ -9,7 +9,9 @@ import pytest
from semif_agent.decisions import Request
from semif_agent.engine import EngineConfig, EngineUnavailable, SemIfEngine
from semif_agent.llm import LLMClient
from semif_agent.log import DecisionLog
from semif_agent.scheduler import Scheduler
from semif_agent.skills import (
CategoryDraft,
CategoryRegistry,
@@ -198,4 +200,26 @@ def test_navigate_empty_tree_short_circuits(tmp_path):
result = navigate(None, log, trace, Request("anything"), {})
assert isinstance(result, CreateCategory)
assert log.read() == []
assert any(e["kind"] == "create_category" for e in trace.read())
assert any(e["kind"] == "create_category" for e in trace.read())
def test_dispatch_create_category_without_engine_returns_error(tmp_path):
"""An empty tree short-circuits to CreateCategory; without an engine the
category authoring fails gracefully instead of leaving the scheduler wedged."""
log = DecisionLog(str(tmp_path / "decisions.jsonl"))
trace = TraceLog(str(tmp_path / "runs.jsonl"))
scheduler = Scheduler(
engine=SemIfEngine(EngineConfig()),
llm=LLMClient(base_url="http://localhost:1/v1", model="test"),
log=log,
config={
"skills": {},
"category_registry": str(tmp_path / "categories.json"),
"skill_bodies": str(tmp_path / "skills"),
},
trace=trace,
)
scheduler.tree = {}
result = scheduler._dispatch(Request("anything"))
assert result.kind == "error"
assert "create_category failed" in result.summary