From 9e365446a38f466a150843bb464a9732c31ac667 Mon Sep 17 00:00:00 2001 From: Denton Social Date: Thu, 24 Sep 2026 01:20:18 -0500 Subject: [PATCH] Drop max_tokens cap on codegen; qwen3 reasoning truncation left content empty qwen38-iq3s reasons extensively before emitting the skill body. A max_tokens cap truncated the hidden reasoning (finish_reason: length) leaving content empty, so the body write failed with 'skill body is empty'. Omit max_tokens so the model runs to completion (~7 min); reasoning is filtered automatically since only content is read. Client timeout default raised to 1200s. --- AGENTS.md | 13 ++++++++++--- config.example.json | 4 ++-- semif_agent/codegen.py | 35 +++++++++++++++++++++++------------ tests/test_codegen.py | 32 +++++++++++++++++++++++++++++--- 4 files changed, 64 insertions(+), 20 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 37bf176..7eb8a3c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -185,9 +185,16 @@ unit tests (24) + box integration tests (2). ### codegen (skill bodies, box) - Skill **bodies** are written by a separate OpenAI-compatible model, configured under `codegen` in config.json (default model `qwen38-iq3s`, the 12G 27B - IQ3_S GGUF — huge/slow; a 3-bit 27B write can take 30-120s). Title + - description for new skills still come from the **small** decision model - (`engine.generate`); only the runnable code body uses codegen. + IQ3_S GGUF — huge/slow). Title + description for new skills still come from + the **small** decision model (`engine.generate`); only the runnable code body + uses codegen. +- **Do NOT cap `max_tokens`** on the codegen call. qwen38-iq3s reasons first + and a cap truncates the hidden reasoning, leaving `content` empty + (`finish_reason: length`) and the body write fails with "skill body is + empty". Unbounded, it runs to completion in ~7 min (~40k chars of reasoning + then the code); the client reads only `content`, so reasoning is filtered + automatically. The client default timeout is 1200s — raise `codegen.timeout` + in config if a harder prompt needs more. - Bodies are persisted to `data/skills//.py` (gitignored) and loaded back at startup via `importlib`, so skills stay runnable across restarts. `SKILL.md` at the repo root is the contract the codegen model is diff --git a/config.example.json b/config.example.json index 351bea9..3e0d2c1 100644 --- a/config.example.json +++ b/config.example.json @@ -13,10 +13,10 @@ }, "llm": {"base_url": "http://localhost:11434/v1", "model": "qwen3.5:4b"}, "codegen": { - "_comment": "OpenAI-compatible model that writes runnable skill bodies. Larger/slower than the decision or self-assessment model.", + "_comment": "OpenAI-compatible model that writes runnable skill bodies. Larger/slower than the decision or self-assessment model. No max_tokens cap: qwen38-iq3s reasons extensively (~7 min) before emitting the body; reasoning is filtered automatically. timeout is seconds.", "base_url": "http://localhost:11434/v1", "model": "qwen38-iq3s", - "timeout": 600 + "timeout": 1200 }, "skill_bodies": "data/skills", "skills": { diff --git a/semif_agent/codegen.py b/semif_agent/codegen.py index 6d83f9a..97415ac 100644 --- a/semif_agent/codegen.py +++ b/semif_agent/codegen.py @@ -34,23 +34,34 @@ def read_skill_contract(path: str | None = None) -> str: class CodegenClient: - """Minimal OpenAI-compatible chat client for writing skill bodies.""" + """Minimal OpenAI-compatible chat client for writing skill bodies. - def __init__(self, base_url: str, model: str, timeout: float = 600.0): + No token cap by default: Qwen3-style models reason first and the cap + truncates the hidden reasoning, leaving `content` empty. Omit `max_tokens` + so the model runs to completion; the reasoning is filtered automatically + because only `content` is read. + """ + + def __init__(self, base_url: str, model: str, timeout: float = 1200.0): self.base_url = base_url.rstrip("/") self.model = model self.timeout = timeout - def chat(self, messages: list[dict], max_tokens: int = 2048, temperature: float = 0.0) -> str: + def chat( + self, + messages: list[dict], + max_tokens: int | None = None, + temperature: float = 0.0, + ) -> str: url = f"{self.base_url}/chat/completions" - body = json.dumps( - { - "model": self.model, - "messages": messages, - "temperature": temperature, - "max_tokens": max_tokens, - } - ).encode("utf-8") + payload: dict = { + "model": self.model, + "messages": messages, + "temperature": temperature, + } + if max_tokens is not None: + payload["max_tokens"] = max_tokens + body = json.dumps(payload).encode("utf-8") request = urllib.request.Request( url, data=body, headers={"Content-Type": "application/json"} ) @@ -145,7 +156,7 @@ def generate_skill_body( draft: SkillDraft, tree: dict, contract: str | None = None, - max_tokens: int = 2048, + max_tokens: int | None = None, ) -> str: """Author a skill body with the big model; retries once on invalid output.""" contract_text = contract if contract is not None else read_skill_contract() diff --git a/tests/test_codegen.py b/tests/test_codegen.py index faea347..9b9f39d 100644 --- a/tests/test_codegen.py +++ b/tests/test_codegen.py @@ -163,10 +163,12 @@ def test_merge_skill_bodies_creates_missing_category(tmp_path): class _FakeOpenAI(BaseHTTPRequestHandler): reply: str = GOOD_BODY + received: list = [] def do_POST(self): length = int(self.headers.get("Content-Length") or 0) - self.rfile.read(length) + raw = self.rfile.read(length).decode("utf-8") + type(self).received.append(json.loads(raw)) body = json.dumps({"choices": [{"message": {"content": self.reply}}]}).encode("utf-8") self.send_response(200) self.send_header("Content-Type", "application/json") @@ -179,7 +181,7 @@ class _FakeOpenAI(BaseHTTPRequestHandler): def _fake_server(reply: str) -> tuple[ThreadingHTTPServer, str]: - handler = type("Handler", (_FakeOpenAI,), {"reply": reply}) + handler = type("Handler", (_FakeOpenAI,), {"reply": reply, "received": []}) httpd = ThreadingHTTPServer(("127.0.0.1", 0), handler) threading.Thread(target=httpd.serve_forever, daemon=True).start() return httpd, f"http://127.0.0.1:{httpd.server_address[1]}/v1" @@ -216,4 +218,28 @@ def test_codegen_client_unreachable_raises(tmp_path): tree = build_tree(build_skills({"skills": {}})) draft = SkillDraft(name="probe", description="Probe the service.") with pytest.raises(Exception): - generate_skill_body(client, Request("is the service up?"), "tracking", draft, tree) \ No newline at end of file + generate_skill_body(client, Request("is the service up?"), "tracking", draft, tree) + + +def test_chat_omits_max_tokens_by_default(): + httpd, base = _fake_server(GOOD_BODY) + try: + client = CodegenClient(base_url=base, model="test", timeout=10) + client.chat([{"role": "user", "content": "hi"}]) + body = httpd.RequestHandlerClass.received[0] + assert "max_tokens" not in body + finally: + httpd.shutdown() + httpd.server_close() + + +def test_chat_includes_max_tokens_when_set(): + httpd, base = _fake_server(GOOD_BODY) + try: + client = CodegenClient(base_url=base, model="test", timeout=10) + client.chat([{"role": "user", "content": "hi"}], max_tokens=512) + body = httpd.RequestHandlerClass.received[0] + assert body["max_tokens"] == 512 + finally: + httpd.shutdown() + httpd.server_close() \ No newline at end of file