print reasoning tokens to console during testing
This commit is contained in:
@@ -216,9 +216,11 @@ unit tests (24) + box integration tests (2).
|
|||||||
`codegen.timeout` in config for harder prompts.
|
`codegen.timeout` in config for harder prompts.
|
||||||
- Set `codegen.stream: true` to echo the codegen output as an SSE token stream
|
- Set `codegen.stream: true` to echo the codegen output as an SSE token stream
|
||||||
to stdout during body writes — including the chain-of-thought, so a long
|
to stdout during body writes — including the chain-of-thought, so a long
|
||||||
(~7 min) write shows live progress. Echoing is console-only; the returned
|
(~7 min) write shows live progress. The client reads reasoning from either
|
||||||
content is identical either way. Integration tests already force streaming;
|
`reasoning` (ollama) or `reasoning_content` (other OpenAI-compatible
|
||||||
see it with `-s` on the box.
|
backends) — do not drop one for the other. Echoing is console-only; the
|
||||||
|
returned content is identical either way. Integration tests already force
|
||||||
|
streaming; see it with `-s` on the box.
|
||||||
|
|
||||||
### Code principles
|
### Code principles
|
||||||
- **No mocking.** The decision engine is always real SemIf; the LLM is always a
|
- **No mocking.** The decision engine is always real SemIf; the LLM is always a
|
||||||
|
|||||||
@@ -97,8 +97,9 @@ class CodegenClient:
|
|||||||
"""Read an OpenAI-compatible SSE stream, echo tokens to stdout.
|
"""Read an OpenAI-compatible SSE stream, echo tokens to stdout.
|
||||||
|
|
||||||
Only `content` deltas are accumulated into the returned body;
|
Only `content` deltas are accumulated into the returned body;
|
||||||
`reasoning_content` (chain-of-thought) is echoed to the console but
|
reasoning (chain-of-thought) is echoed to the console but never part
|
||||||
never part of the result.
|
of the result. Backends disagree on the field name: ollama streams it
|
||||||
|
as `reasoning`, DeepSeek/vllm-style as `reasoning_content`, so read both.
|
||||||
"""
|
"""
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
@@ -117,7 +118,7 @@ class CodegenClient:
|
|||||||
choice = chunk.get("choices", [{}])[0]
|
choice = chunk.get("choices", [{}])[0]
|
||||||
delta = choice.get("delta", {}) or {}
|
delta = choice.get("delta", {}) or {}
|
||||||
text = delta.get("content") or ""
|
text = delta.get("content") or ""
|
||||||
reasoning = delta.get("reasoning_content") or ""
|
reasoning = delta.get("reasoning_content") or delta.get("reasoning") or ""
|
||||||
if text or reasoning:
|
if text or reasoning:
|
||||||
sys.stdout.write(text + reasoning)
|
sys.stdout.write(text + reasoning)
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
|
|||||||
+15
-6
@@ -257,9 +257,14 @@ def _sse_frame(payload: dict) -> str:
|
|||||||
|
|
||||||
|
|
||||||
class _StreamingOpenAI(BaseHTTPRequestHandler):
|
class _StreamingOpenAI(BaseHTTPRequestHandler):
|
||||||
"""Replies with an OpenAI-compatible SSE token stream (COT then content)."""
|
"""Replies with an OpenAI-compatible SSE token stream (COT then content).
|
||||||
|
|
||||||
|
Reasoning field name matches the backend: ollama emits `reasoning`,
|
||||||
|
DeepSeek/vllm-style `reasoning_content`. Defaults to ollama's.
|
||||||
|
"""
|
||||||
|
|
||||||
reasoning: str = "thinking about the body..."
|
reasoning: str = "thinking about the body..."
|
||||||
|
reasoning_key: str = "reasoning"
|
||||||
content: str = GOOD_BODY
|
content: str = GOOD_BODY
|
||||||
received: list = []
|
received: list = []
|
||||||
|
|
||||||
@@ -271,11 +276,12 @@ class _StreamingOpenAI(BaseHTTPRequestHandler):
|
|||||||
self.send_header("Content-Type", "text/event-stream")
|
self.send_header("Content-Type", "text/event-stream")
|
||||||
self.end_headers()
|
self.end_headers()
|
||||||
reasoning = type(self).reasoning
|
reasoning = type(self).reasoning
|
||||||
|
reasoning_key = type(self).reasoning_key
|
||||||
content = type(self).content
|
content = type(self).content
|
||||||
step = max(len(reasoning) // 4, 1)
|
step = max(len(reasoning) // 4, 1)
|
||||||
for i in range(0, len(reasoning), step):
|
for i in range(0, len(reasoning), step):
|
||||||
frame = _sse_frame(
|
frame = _sse_frame(
|
||||||
{"choices": [{"delta": {"reasoning_content": reasoning[i : i + step]}}]}
|
{"choices": [{"delta": {reasoning_key: reasoning[i : i + step]}}]}
|
||||||
)
|
)
|
||||||
self.wfile.write(frame.encode("utf-8"))
|
self.wfile.write(frame.encode("utf-8"))
|
||||||
step = max(len(content) // 4, 1)
|
step = max(len(content) // 4, 1)
|
||||||
@@ -292,8 +298,10 @@ class _StreamingOpenAI(BaseHTTPRequestHandler):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
def _streaming_server() -> tuple[ThreadingHTTPServer, str]:
|
def _streaming_server(reasoning_key: str = "reasoning") -> tuple[ThreadingHTTPServer, str]:
|
||||||
handler = type("Handler", (_StreamingOpenAI,), {"received": []})
|
handler = type(
|
||||||
|
"Handler", (_StreamingOpenAI,), {"reasoning_key": reasoning_key, "received": []}
|
||||||
|
)
|
||||||
httpd = ThreadingHTTPServer(("127.0.0.1", 0), handler)
|
httpd = ThreadingHTTPServer(("127.0.0.1", 0), handler)
|
||||||
threading.Thread(target=httpd.serve_forever, daemon=True).start()
|
threading.Thread(target=httpd.serve_forever, daemon=True).start()
|
||||||
return httpd, f"http://127.0.0.1:{httpd.server_address[1]}/v1"
|
return httpd, f"http://127.0.0.1:{httpd.server_address[1]}/v1"
|
||||||
@@ -312,8 +320,9 @@ def test_chat_stream_accumulates_full_content():
|
|||||||
httpd.server_close()
|
httpd.server_close()
|
||||||
|
|
||||||
|
|
||||||
def test_chat_stream_verbose_echoes_tokens(capsys):
|
@pytest.mark.parametrize("reasoning_key", ["reasoning", "reasoning_content"])
|
||||||
httpd, base = _streaming_server()
|
def test_chat_stream_verbose_echoes_tokens(reasoning_key, capsys):
|
||||||
|
httpd, base = _streaming_server(reasoning_key=reasoning_key)
|
||||||
try:
|
try:
|
||||||
client = CodegenClient(base_url=base, model="test", timeout=10, stream=True)
|
client = CodegenClient(base_url=base, model="test", timeout=10, stream=True)
|
||||||
out = client.chat([{"role": "user", "content": "hi"}])
|
out = client.chat([{"role": "user", "content": "hi"}])
|
||||||
|
|||||||
Reference in New Issue
Block a user