Author runnable skill bodies via OpenAI-compatible codegen model
create_skill now writes a real predict/act body: the small decision model still authors title + description (engine.generate), then a larger OpenAI-compatible model (default qwen38-iq3s) writes the runnable code against the SKILL.md contract. Bodies persist to data/skills/<cat>/<name>.py, are hot-loaded via importlib, merged into the running tree, and the request re-dispatches to the new leaf. The dashboard decision-flow view shows the title/description with a writing badge while the body is being written. Codegen failure degrades to a navigable stub.
This commit is contained in:
Denton Social
@@ -12,10 +12,20 @@ from pathlib import Path
|
||||
import pytest
|
||||
|
||||
from semif_agent.cli import build_scheduler, load_config
|
||||
from semif_agent.codegen import CodegenClient, generate_skill_body
|
||||
from semif_agent.decisions import Request
|
||||
from semif_agent.dream import dream
|
||||
from semif_agent.engine import EngineUnavailable
|
||||
from semif_agent.skills import CategoryDraft, SkillDraft, generate_category, generate_skill
|
||||
from semif_agent.skills import (
|
||||
CategoryDraft,
|
||||
SkillBodyStore,
|
||||
SkillDraft,
|
||||
build_skills,
|
||||
build_tree,
|
||||
generate_category,
|
||||
generate_skill,
|
||||
materialize_skill,
|
||||
)
|
||||
|
||||
|
||||
def require_real(config: dict):
|
||||
@@ -140,6 +150,38 @@ def test_generate_skill(tmp_path):
|
||||
assert draft.name and draft.description
|
||||
|
||||
|
||||
def test_generate_skill_body_codegen(tmp_path):
|
||||
"""A real OpenAI-compatible model writes a runnable skill body.
|
||||
|
||||
Slow: uses the big codegen model (qwen38-iq3s by default). Run this one in
|
||||
the background and poll — long-lived ssh sessions get SIGHUP'd.
|
||||
"""
|
||||
config = load_config()
|
||||
require_real(config)
|
||||
codegen_cfg = config.get("codegen", {})
|
||||
client = CodegenClient(
|
||||
base_url=codegen_cfg.get("base_url", "http://localhost:11434/v1"),
|
||||
model=codegen_cfg.get("model", "qwen38-iq3s"),
|
||||
timeout=float(codegen_cfg.get("timeout", 600.0)),
|
||||
)
|
||||
tree = build_tree(build_skills({"skills": {}}))
|
||||
draft = SkillDraft(
|
||||
name="check_service",
|
||||
description="Check whether a service is reachable.",
|
||||
)
|
||||
code = generate_skill_body(
|
||||
client,
|
||||
Request("is my home server reachable right now?"),
|
||||
"tracking",
|
||||
draft,
|
||||
tree,
|
||||
)
|
||||
print(f"generated {len(code)} bytes of skill body")
|
||||
store = SkillBodyStore(str(tmp_path / "skills"))
|
||||
skill = materialize_skill(draft, "tracking", store)
|
||||
assert callable(skill.predict) and callable(skill.act)
|
||||
|
||||
|
||||
def test_create_skill_empty_category_does_not_wedge(tmp_path):
|
||||
"""A dispatch that lands on an empty category must not leave the scheduler wedged.
|
||||
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
"""Pure-stdlib tests for skill code-body generation.
|
||||
|
||||
Prompt building, draft parsing/validation, body persistence + import, and
|
||||
tree hot-merge all run without SemIf or a real LLM. The only network usage is a
|
||||
throwaway stdlib HTTP server that stands in for an OpenAI-compatible endpoint —
|
||||
the CodegenClient itself is real, not mocked.
|
||||
"""
|
||||
|
||||
import json
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
import pytest
|
||||
|
||||
from semif_agent.codegen import (
|
||||
CodegenClient,
|
||||
build_skill_body_prompt,
|
||||
generate_skill_body,
|
||||
parse_skill_body,
|
||||
read_skill_contract,
|
||||
)
|
||||
from semif_agent.decisions import Request
|
||||
from semif_agent.skills import (
|
||||
SkillBodyStore,
|
||||
SkillDraft,
|
||||
build_skills,
|
||||
build_tree,
|
||||
load_skill_module,
|
||||
materialize_skill,
|
||||
merge_skill_bodies,
|
||||
merge_registry,
|
||||
)
|
||||
|
||||
GOOD_BODY = """\
|
||||
from semif_agent.decisions import DecisionRequest, Option
|
||||
from semif_agent.skills import ActionResult, Prediction
|
||||
|
||||
def predict(ctx, request):
|
||||
return Prediction(text="ok", decisions=[])
|
||||
|
||||
def act(ctx, request, prediction):
|
||||
return ActionResult(action_log="probe ran", new_state=request.text)
|
||||
"""
|
||||
|
||||
|
||||
def test_read_skill_contract_loads_contract():
|
||||
text = read_skill_contract()
|
||||
assert "predict" in text and "act" in text
|
||||
assert "data/skills" in text
|
||||
|
||||
|
||||
def test_build_skill_body_prompt_includes_contract_request_and_draft():
|
||||
tree = build_tree(build_skills({"skills": {}}))
|
||||
draft = SkillDraft(name="probe", description="Probe the service.")
|
||||
messages = build_skill_body_prompt(
|
||||
Request("check if the service is up"), "tracking", draft, tree, "THE CONTRACT"
|
||||
)
|
||||
assert messages[0]["role"] == "system"
|
||||
assert "THE CONTRACT" in messages[0]["content"]
|
||||
joined = messages[1]["content"]
|
||||
assert "check if the service is up" in joined
|
||||
assert "probe" in joined
|
||||
assert "tracking.check" in joined
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw",
|
||||
[
|
||||
GOOD_BODY,
|
||||
"```python\n" + GOOD_BODY + "\n```",
|
||||
json.dumps({"code": GOOD_BODY}),
|
||||
"Here you go:\n```python\n" + GOOD_BODY + "\n```\nHope that helps.",
|
||||
'Sure: ' + json.dumps({"code": GOOD_BODY}) + ' (that was it)',
|
||||
],
|
||||
)
|
||||
def test_parse_skill_body_accepts_forms(raw):
|
||||
code = parse_skill_body(raw)
|
||||
assert "def predict" in code and "def act" in code
|
||||
|
||||
|
||||
def test_parse_skill_body_rejects_empty():
|
||||
with pytest.raises(ValueError):
|
||||
parse_skill_body("")
|
||||
|
||||
|
||||
def test_parse_skill_body_rejects_invalid_python():
|
||||
with pytest.raises(ValueError):
|
||||
parse_skill_body("def predict(:\n pass")
|
||||
|
||||
|
||||
def test_parse_skill_body_rejects_missing_functions():
|
||||
with pytest.raises(ValueError):
|
||||
parse_skill_body("def predict(ctx, request):\n return None")
|
||||
|
||||
|
||||
def test_parse_skill_body_rejects_missing_act():
|
||||
with pytest.raises(ValueError):
|
||||
parse_skill_body("def predict(ctx, request):\n return None\nx = 1")
|
||||
|
||||
|
||||
def test_body_store_roundtrip(tmp_path):
|
||||
store = SkillBodyStore(str(tmp_path / "skills"))
|
||||
assert store.list_bodies() == []
|
||||
store.write("tracking", "probe", GOOD_BODY)
|
||||
assert store.list_bodies() == [("tracking", "probe")]
|
||||
target = store.body_path("tracking", "probe")
|
||||
assert target.is_file()
|
||||
assert "def predict" in target.read_text()
|
||||
|
||||
|
||||
def test_load_skill_module_exposes_predict_act(tmp_path):
|
||||
store = SkillBodyStore(str(tmp_path / "skills"))
|
||||
store.write("tracking", "probe", GOOD_BODY)
|
||||
module = load_skill_module("tracking", "probe", store.path)
|
||||
assert callable(module.predict) and callable(module.act)
|
||||
|
||||
|
||||
def test_materialize_skill_builds_runnable_skill(tmp_path):
|
||||
store = SkillBodyStore(str(tmp_path / "skills"))
|
||||
draft = SkillDraft(name="probe", description="Probe the service.", code=GOOD_BODY)
|
||||
skill = materialize_skill(draft, "tracking", store)
|
||||
assert skill.name == "probe"
|
||||
assert skill.category == "tracking"
|
||||
assert callable(skill.predict) and callable(skill.act)
|
||||
|
||||
|
||||
def test_materialize_skill_requires_code(tmp_path):
|
||||
store = SkillBodyStore(str(tmp_path / "skills"))
|
||||
draft = SkillDraft(name="probe", description="Probe the service.")
|
||||
with pytest.raises(ValueError):
|
||||
materialize_skill(draft, "tracking", store)
|
||||
|
||||
|
||||
def test_materialize_skill_rejects_import_failure(tmp_path):
|
||||
store = SkillBodyStore(str(tmp_path / "skills"))
|
||||
bad = "def predict(ctx, request):\n return None\n"
|
||||
draft = SkillDraft(name="probe", description="Probe.", code=bad)
|
||||
with pytest.raises(ValueError):
|
||||
materialize_skill(draft, "tracking", store)
|
||||
|
||||
|
||||
def test_merge_skill_bodies_upgrades_stub(tmp_path):
|
||||
store = SkillBodyStore(str(tmp_path / "skills"))
|
||||
store.write("tracking", "probe", GOOD_BODY)
|
||||
tree = build_tree(build_skills({"skills": {}}))
|
||||
registry = {"tracking": {"description": "", "skills": [{"name": "probe", "description": "Probe."}]}}
|
||||
merge_registry(tree, registry)
|
||||
upgraded = merge_skill_bodies(tree, store, registry)
|
||||
assert upgraded == 1
|
||||
skill = next(s for s in tree["tracking"] if s.name == "probe")
|
||||
assert callable(skill.predict) and callable(skill.act)
|
||||
assert skill.description == "Probe."
|
||||
|
||||
|
||||
def test_merge_skill_bodies_creates_missing_category(tmp_path):
|
||||
store = SkillBodyStore(str(tmp_path / "skills"))
|
||||
store.write("brand_new", "ping", GOOD_BODY)
|
||||
tree = build_tree(build_skills({"skills": {}}))
|
||||
upgraded = merge_skill_bodies(tree, store, {})
|
||||
assert upgraded == 1
|
||||
assert tree["brand_new"][0].name == "ping"
|
||||
|
||||
|
||||
class _FakeOpenAI(BaseHTTPRequestHandler):
|
||||
reply: str = GOOD_BODY
|
||||
|
||||
def do_POST(self):
|
||||
length = int(self.headers.get("Content-Length") or 0)
|
||||
self.rfile.read(length)
|
||||
body = json.dumps({"choices": [{"message": {"content": self.reply}}]}).encode("utf-8")
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, format, *args):
|
||||
pass
|
||||
|
||||
|
||||
def _fake_server(reply: str) -> tuple[ThreadingHTTPServer, str]:
|
||||
handler = type("Handler", (_FakeOpenAI,), {"reply": reply})
|
||||
httpd = ThreadingHTTPServer(("127.0.0.1", 0), handler)
|
||||
threading.Thread(target=httpd.serve_forever, daemon=True).start()
|
||||
return httpd, f"http://127.0.0.1:{httpd.server_address[1]}/v1"
|
||||
|
||||
|
||||
def test_generate_skill_body_end_to_end(tmp_path):
|
||||
httpd, base = _fake_server(GOOD_BODY)
|
||||
try:
|
||||
client = CodegenClient(base_url=base, model="test", timeout=10)
|
||||
tree = build_tree(build_skills({"skills": {}}))
|
||||
draft = SkillDraft(name="probe", description="Probe the service.")
|
||||
code = generate_skill_body(client, Request("is the service up?"), "tracking", draft, tree)
|
||||
assert "def predict" in code and "def act" in code
|
||||
finally:
|
||||
httpd.shutdown()
|
||||
httpd.server_close()
|
||||
|
||||
|
||||
def test_generate_skill_body_retries_then_fails(tmp_path):
|
||||
httpd, base = _fake_server("this is not python at all")
|
||||
try:
|
||||
client = CodegenClient(base_url=base, model="test", timeout=10)
|
||||
tree = build_tree(build_skills({"skills": {}}))
|
||||
draft = SkillDraft(name="probe", description="Probe the service.")
|
||||
with pytest.raises(ValueError):
|
||||
generate_skill_body(client, Request("is the service up?"), "tracking", draft, tree)
|
||||
finally:
|
||||
httpd.shutdown()
|
||||
httpd.server_close()
|
||||
|
||||
|
||||
def test_codegen_client_unreachable_raises(tmp_path):
|
||||
client = CodegenClient(base_url="http://127.0.0.1:1/v1", model="test", timeout=2)
|
||||
tree = build_tree(build_skills({"skills": {}}))
|
||||
draft = SkillDraft(name="probe", description="Probe the service.")
|
||||
with pytest.raises(Exception):
|
||||
generate_skill_body(client, Request("is the service up?"), "tracking", draft, tree)
|
||||
@@ -144,5 +144,43 @@ def test_submit_trace_event_recorded_even_when_engine_missing(tmp_path):
|
||||
assert len(runs) == 1
|
||||
kinds = [e["kind"] for e in runs[0]["events"]]
|
||||
assert "submit" in kinds
|
||||
finally:
|
||||
server.close()
|
||||
|
||||
|
||||
def test_skill_writing_and_created_events_in_payload(tmp_path):
|
||||
scheduler = build_scheduler(tmp_path)
|
||||
scheduler.trace.append("submit", "run-9", text="track my package")
|
||||
scheduler.trace.append(
|
||||
"skill_writing",
|
||||
"run-9",
|
||||
category="tracking",
|
||||
skill="track_live",
|
||||
description="Follow a package in real time.",
|
||||
model="qwen38-iq3s",
|
||||
)
|
||||
scheduler.trace.append(
|
||||
"skill_created",
|
||||
"run-9",
|
||||
category="tracking",
|
||||
skill="track_live",
|
||||
description="Follow a package in real time.",
|
||||
body="data/skills/tracking/track_live.py",
|
||||
written=True,
|
||||
)
|
||||
server = Server(scheduler)
|
||||
try:
|
||||
status, payload = server.get("/api/trace")
|
||||
assert status == 200
|
||||
run = next(r for r in payload["runs"] if r["run_id"] == "run-9")
|
||||
kinds = [e["kind"] for e in run["events"]]
|
||||
assert "skill_writing" in kinds and "skill_created" in kinds
|
||||
writing = next(e for e in run["events"] if e["kind"] == "skill_writing")
|
||||
assert writing["skill"] == "track_live"
|
||||
assert "real time" in writing["description"]
|
||||
assert writing["model"] == "qwen38-iq3s"
|
||||
created = next(e for e in run["events"] if e["kind"] == "skill_created")
|
||||
assert created["written"] is True
|
||||
assert created["body"] == "data/skills/tracking/track_live.py"
|
||||
finally:
|
||||
server.close()
|
||||
Reference in New Issue
Block a user