diff --git a/semif_agent/engine.py b/semif_agent/engine.py index f5eb842..c16f484 100644 --- a/semif_agent/engine.py +++ b/semif_agent/engine.py @@ -95,23 +95,54 @@ class SemIfEngine: def generate( self, messages: list[dict], - temperature: float = 0.2, + temperature: float = 0.0, max_tokens: int = 256, ) -> str: """Drive the pinned decision model in the normal way: text generation. - SemIf scoring reads option logits directly; this instead uses the - underlying llama.cpp chat-completion endpoint on the same loaded model, - e.g. for skill-tree authoring. Each call resets the KV cache by - default, so interleaving scoring and generation on one model is safe. + SemIf scoring reads option logits directly; this instead autoregressively + samples from the same llama.cpp context, e.g. for skill-tree authoring. + Generation decodes the chat template through the backend's low-level + context (there is no high-level chat-completion object on the CPU + backend), stopping at the tokenizer's eos token. The KV cache is cleared + at the start, so interleaving scoring and generation on one model is safe. """ model, tokenizer, metadata = self._ensure_loaded() try: - reply = model.create_chat_completion( - messages=messages, - temperature=temperature, - max_tokens=max_tokens, + import numpy + + engine = model.engine + prompt_ids = tokenizer.apply_chat_template( + messages, + tokenize=True, + add_generation_prompt=True, + enable_thinking=False, ) + engine.clear() + logits = engine._decode(prompt_ids, 0, 0, True) + generated: list[int] = [] + rng = numpy.random.default_rng() + for position in range(max_tokens): + token = _sample_token(logits, temperature, rng) + if token == tokenizer.eos_token_id: + break + generated.append(token) + logits = engine._decode([token], len(prompt_ids) + position, 0, True) + return tokenizer.decode(generated).strip() + except EngineUnavailable: + raise except Exception as exc: raise EngineUnavailable(f"generation failed: {exc}") from exc - return reply["choices"][0]["message"]["content"].strip() + + +def _sample_token(logits, temperature: float, rng) -> int: + """Pick the next token from next-position logits: greedy or temperature.""" + import numpy + + if temperature <= 0.0: + return int(numpy.argmax(logits)) + scaled = numpy.asarray(logits, dtype=numpy.float64) / max(temperature, 1e-6) + scaled = scaled - scaled.max() + probabilities = numpy.exp(scaled) + probabilities /= probabilities.sum() + return int(rng.choice(probabilities.size, p=probabilities)) diff --git a/tests/test_skills.py b/tests/test_skills.py index bfc2309..78eb6b6 100644 --- a/tests/test_skills.py +++ b/tests/test_skills.py @@ -84,6 +84,12 @@ def test_generate_category_without_engine_raises(): generate_category(engine, Request("anything"), {}) +def test_generate_without_engine_raises(): + engine = SemIfEngine(EngineConfig()) + with pytest.raises(EngineUnavailable): + engine.generate([{"role": "user", "content": "hi"}]) + + def test_build_tree_includes_registry_stubs(tmp_path): registry = CategoryRegistry(str(tmp_path / "categories.json")) registry.register("delivery", "Track and manage package deliveries.")