Add basic version: SemIf decision engine, urgency queue, skill tree, skill loop, decision log, dream pass, CLI

tests/integration/test_pipeline.py

@@ -0,0 +1,79 @@
+"""End-to-end pipeline test. Run ONLY on the box with real SemIf + real LLM.
+
+    python -m pytest tests/integration -q
+
+The decision engine is real (llamacpp GGUF) and self-assessment is a real
+local LLM endpoint. If either is unavailable this fails loudly — no mocking.
+"""
+
+import json
+from pathlib import Path
+
+import pytest
+
+from semif_agent.cli import build_scheduler, load_config
+from semif_agent.dream import dream
+from semif_agent.engine import EngineUnavailable
+
+
+def require_real(config: dict):
+    from semif_agent.engine import SemIfEngine, EngineConfig
+
+    engine = SemIfEngine(
+        EngineConfig(
+            backend=config.get("engine", {}).get("backend", "llamacpp"),
+            source=config.get("engine", {}).get("source", ""),
+            revision=config.get("engine", {}).get("revision", ""),
+            gguf=config.get("engine", {}).get("gguf", ""),
+            context_tokens=int(config.get("engine", {}).get("context_tokens", 4096)),
+            threads=config.get("engine", {}).get("threads"),
+        )
+    )
+    try:
+        engine._ensure_loaded()
+    except EngineUnavailable as exc:
+        pytest.fail(f"real engine unavailable: {exc}")
+
+
+def test_pipeline_end_to_end(tmp_path):
+    config = load_config()
+    require_real(config)
+    config["log"] = str(tmp_path / "decisions.jsonl")
+    scheduler, config = build_scheduler(config)
+
+    inputs = [
+        "send my girlfriend an email that says I'm going to be late to the party",
+        "tell me if my package was delivered",
+    ]
+    for text in inputs:
+        status, detail = scheduler.submit(text)
+        print(f"[{status}] {detail}")
+        assert status in ("running", "preempted", "queued", "rejected")
+
+    scheduler.run_queue()
+
+    rows = scheduler.log.read()
+    assert len(rows) > 0, "expected SemIf decisions to be logged"
+
+    report = dream(scheduler.log)
+    assert report.cross_entropy is not None
+    print(report.render())
+
+    skills = scheduler.status()
+    assert "queue:" in skills
+
+    contacts_path = Path(config.get("skills", {}).get("contacts", "data/contacts.json"))
+    assert contacts_path.is_file()
+
+
+def test_busy_choice_path(tmp_path):
+    config = load_config()
+    require_real(config)
+    config["log"] = str(tmp_path / "decisions.jsonl")
+    scheduler, config = build_scheduler(config)
+
+    scheduler.busy("driving on the freeway", skill="driving")
+    status, detail = scheduler.submit("send my girlfriend an email that says I'm going to be late")
+    print(f"[{status}] {detail}")
+    assert status in ("preempted", "queued", "dropped")
+    scheduler.idle()

tests/test_contract.py

@@ -0,0 +1,42 @@
+from semif_agent.decisions import DecisionRequest, DecisionResult, Option, Request
+
+
+def test_to_semif_row_shape():
+    request = DecisionRequest(
+        state="some state",
+        question="some question?",
+        options=[Option("a", "A."), Option("b", "B.")],
+        id="abc",
+    )
+    row = request.to_semif_row()
+    assert row["id"] == "abc"
+    assert row["state"] == "some state"
+    assert row["question"] == "some question?"
+    assert row["options"] == [
+        {"id": "a", "description": "A."},
+        {"id": "b", "description": "B."},
+    ]
+
+
+def test_result_selected_and_probs():
+    request = DecisionRequest(
+        state="s",
+        question="q",
+        options=[Option("a", "A."), Option("b", "B.")],
+    )
+    result = DecisionResult(
+        request=request, option_ids=["a", "b"], probabilities=[0.3, 0.7]
+    )
+    assert result.selected == "b"
+    assert result.probs == {"a": 0.3, "b": 0.7}
+    assert result.prob("a") == 0.3
+
+
+def test_request_requeue_preserves_state():
+    original = Request(text="t", priority=0.7)
+    original.resume["from_skill"] = "email.compose"
+    updated = original.copy_for_requeue()
+    assert updated.id == original.id
+    assert updated.priority == original.priority
+    assert updated.resume["from_skill"] == "email.compose"
+    assert updated.reentries == original.reentries + 1

tests/test_dream.py

@@ -0,0 +1,89 @@
+import math
+
+import pytest
+
+from semif_agent.decisions import DecisionRequest, DecisionResult, Option
+from semif_agent.dream import HUMAN_WEIGHT, SELF_WEIGHT, dream
+from semif_agent.log import DecisionLog
+
+
+def make_result(probs):
+    request = DecisionRequest(
+        state="state",
+        question="question",
+        options=[Option(key, key) for key in probs],
+    )
+    result = DecisionResult(
+        request=request,
+        option_ids=list(probs),
+        probabilities=list(probs.values()),
+    )
+    return request, result
+
+
+def test_cross_entropy_matches_hand_calculation(tmp_path):
+    log = DecisionLog(str(tmp_path / "decisions.jsonl"))
+    probs = {"yes": 0.8, "no": 0.2}
+    request, result = make_result(probs)
+    log.append(request, result, label="no")
+    report = dream(log)
+    assert report.cross_entropy == pytest.approx(-math.log(0.2))
+    assert report.rows[0].nll == pytest.approx(-math.log(0.2))
+    assert report.rows[0].weight == HUMAN_WEIGHT
+
+
+def test_default_label_is_selected(tmp_path):
+    log = DecisionLog(str(tmp_path / "decisions.jsonl"))
+    request, result = make_result({"yes": 0.9, "no": 0.1})
+    log.append(request, result)
+    report = dream(log)
+    row = report.rows[0]
+    assert row.observed == "yes"
+    assert row.correct is True
+    assert row.weight == SELF_WEIGHT
+
+
+def test_accuracy_and_ece(tmp_path):
+    log = DecisionLog(str(tmp_path / "decisions.jsonl"))
+    for probs, label in [
+        ({"yes": 0.9, "no": 0.1}, "yes"),
+        ({"yes": 0.6, "no": 0.4}, "no"),
+        ({"yes": 0.9, "no": 0.1}, "yes"),
+    ]:
+        request, result = make_result(probs)
+        log.append(request, result, label=label)
+    report = dream(log)
+    assert report.accuracy == pytest.approx(2 / 3)
+    assert report.ece is not None and 0.0 <= report.ece <= 1.0
+
+
+def test_relabel_human_override(tmp_path):
+    log = DecisionLog(str(tmp_path / "decisions.jsonl"))
+    request, result = make_result({"yes": 0.9, "no": 0.1})
+    log.append(request, result)
+    assert log.relabel(request.id, "no") is True
+    rows = log.read()
+    assert rows[0]["observed_outcome"] == "no"
+    assert rows[0]["label_source"] == "human"
+    assert log.relabel("missing", "yes") is False
+
+
+def test_skips_unlabeled(tmp_path):
+    log = DecisionLog(str(tmp_path / "decisions.jsonl"))
+    log.path.parent.mkdir(parents=True, exist_ok=True)
+    log.path.write_text('{"id": "x", "predicted_probs": {"a": 1.0}}\n')
+    report = dream(log)
+    assert report.skipped == 1
+    assert report.rows == []
+    assert report.cross_entropy is None
+
+
+def test_clamped_probability_never_zero(tmp_path):
+    log = DecisionLog(str(tmp_path / "decisions.jsonl"))
+    log.path.parent.mkdir(parents=True, exist_ok=True)
+    log.path.write_text(
+        '{"id": "x", "predicted_probs": {"a": 0.0, "b": 1.0}, '
+        '"selected": "b", "observed_outcome": "a", "label_source": "human"}\n'
+    )
+    report = dream(log)
+    assert report.rows[0].nll == pytest.approx(-math.log(1e-9))

tests/test_queue.py

@@ -0,0 +1,74 @@
+import pytest
+
+from semif_agent.queue import UrgencyQueue
+from semif_agent.decisions import Request
+
+
+def make(text, weight):
+    return Request(text=text), weight
+
+
+def test_order_by_weight_desc():
+    q = UrgencyQueue()
+    req_a, _ = make("low", 0.25)
+    req_c, _ = make("critical", 1.0)
+    req_h, _ = make("high", 0.75)
+    q.push(req_a, 0.25)
+    q.push(req_c, 1.0)
+    q.push(req_h, 0.75)
+    assert q.pop().text == "critical"
+    assert q.pop().text == "high"
+    assert q.pop().text == "low"
+    assert q.pop() is None
+
+
+def test_fifo_tiebreak():
+    q = UrgencyQueue()
+    req_a, _ = make("first", 0.5)
+    req_b, _ = make("second", 0.5)
+    q.push(req_a, 0.5)
+    q.push(req_b, 0.5)
+    assert q.pop().text == "first"
+    assert q.pop().text == "second"
+
+
+def test_fifo_beats_recency():
+    q = UrgencyQueue()
+    old, _ = make("older", 0.5)
+    new, _ = make("newer", 0.5)
+    q.push(old, 0.5, recency=1.0)
+    q.push(new, 0.5, recency=2.0)
+    assert q.pop().text == "older"
+
+
+def test_peek_does_not_remove():
+    q = UrgencyQueue()
+    req, _ = make("peek", 0.9)
+    q.push(req, 0.9)
+    assert q.peek().text == "peek"
+    assert len(q) == 1
+
+
+def test_bounds_reject():
+    q = UrgencyQueue(max_size=2)
+    assert q.push(make("a", 1.0)[0], 1.0)
+    assert q.push(make("b", 1.0)[0], 1.0)
+    assert not q.push(make("c", 1.0)[0], 1.0)
+
+
+def test_age_raises_priority():
+    q = UrgencyQueue(age_rate=1.0)
+    old, _ = make("aging", 0.2)
+    new, _ = make("busy", 1.0)
+    q.push(old, 0.2)
+    q.push(new, 1.0)
+    q.age(dt=10.0)
+    assert q.peek().text == "aging"
+
+
+def test_items_sorted():
+    q = UrgencyQueue()
+    q.push(make("b", 0.5)[0], 0.5)
+    q.push(make("a", 1.0)[0], 1.0)
+    weights = [weight for weight, _ in q.items()]
+    assert weights == [1.0, 0.5]