probbit/python/test_persona.py at main · BitmapAsset/probbit · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
"""Tests for the persona functions of python/probbit.py (stdlib unittest, Python 3.9; run: python3 python/test_persona.py).
They drive `probbit persona` through the wrapper and check its documents against the goldens in examples/persona/golden/."""
import json, os, unittest
import probbit

EX = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "examples", "persona")
FX = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "probbit-cli", "tests", "fixtures", "persona")


def load(name):
    with open(os.path.join(EX, name), encoding="utf-8") as f:
        return json.load(f)


def golden(name, f):
    with open(os.path.join(EX, "golden", name, f), encoding="utf-8") as fh:
        return [json.loads(x) for x in fh.read().splitlines()] if f.endswith(".jsonl") else json.load(fh)


class Persona(unittest.TestCase):
    def test_init_turn_and_replay_equal_the_goldens(self):
        script = load("workday.json")["turns"]
        for name in ("ops-engineer", "tutor", "trader-assistant"):
            path = os.path.join(EX, name + ".yaml")
            st = probbit.persona_init(path)
            self.assertEqual(st, golden(name, "state0.json"))
            stances = []
            for inputs in script:
                r = probbit.persona_turn(path, st, inputs)
                stances.append(r["stance"]); st = r["state"]
            self.assertEqual(stances, golden(name, "workday.jsonl"), name)
            self.assertEqual(st, golden(name, "final-state.json"))
            self.assertEqual(probbit.persona_replay(path, {"turns": script}), stances)

    def test_a_dict_persona_is_the_same_individual(self):
        doc = load("tutor.json")
        a = probbit.persona_init(doc, seed=3); b = probbit.persona_init(os.path.join(EX, "tutor.yaml"), seed=3)
        self.assertEqual(a, b); self.assertEqual(a["seed"], 3)
        self.assertNotEqual(a["genes"], probbit.persona_init(doc, seed=4)["genes"])  # another seed, another individual
        r = probbit.persona_turn(doc, a, {"loss": True, "sentiment": "negative"}, timing=True)
        self.assertEqual(r["stance"]["stance"]["humour"]["level"], "none")  # the habit: no jokes on a loss
        self.assertEqual(r["stance"]["habits"]["violations"], 0); self.assertIn("timing", r["stance"]); self.assertLessEqual(r["stance"]["line_tokens"], 40)

    def test_goal_signals_pass_through_unchanged(self):
        # drives (docs/persona.md 2.9): the same signature; `goals` is one more input, handed to the engine as given
        path = os.path.join(FX, "drives-adversary.json")
        with open(os.path.join(FX, "drives-adversary-script.json"), encoding="utf-8") as f:
            script = json.load(f)["turns"]
        with open(os.path.join(FX, "drives-adversary-replay.jsonl"), encoding="utf-8") as f:
            gold = [json.loads(x) for x in f.read().splitlines()]
        st = probbit.persona_init(path, seed=4); self.assertIn("drives", st)
        stances = []
        for inputs in script:
            r = probbit.persona_turn(path, st, inputs)
            stances.append(r["stance"]); st = r["state"]
        self.assertEqual(stances, gold)
        self.assertEqual(probbit.persona_replay(path, {"turns": script}, seed=4), gold)
        r = probbit.persona_turn(path, st, {"goals": {"chores": {"deadline_hours": 5}}, "security": True})
        self.assertEqual(r["stance"]["pursue"]["goal"], "chores")  # the must-do habit
        self.assertEqual(r["stance"]["inputs"]["goals"], {"chores": {"deadline_hours": 5}})  # logged as given
        with self.assertRaises(probbit.ProbbitInputError) as e:
            probbit.persona_turn(path, st, {"goals": {"sleep": {"cue": True}}})
        self.assertEqual(e.exception.path, "inputs.goals.sleep")

    def test_errors_are_typed(self):
        doc = load("ops-engineer.json")
        st = probbit.persona_init(doc)
        with self.assertRaises(probbit.ProbbitInputError) as e:
            probbit.persona_turn(doc, st, {"stakes": 3})
        self.assertEqual((e.exception.code, e.exception.path), ("persona", "inputs.stakes"))
        bad = dict(doc, traits=[dict(doc["traits"][0], sprad=1)] + doc["traits"][1:])
        with self.assertRaises(probbit.ProbbitInputError) as e:
            probbit.persona_init(bad)
        self.assertEqual((e.exception.code, e.exception.path), ("persona", "traits[0].sprad"))
        with self.assertRaises(probbit.ProbbitInputError) as e:  # a state of another persona
            probbit.persona_turn(load("tutor.json"), st, {})
        self.assertEqual(e.exception.path, "state.persona.digest")
        edited = dict(st, turn=st["turn"] + 1)
        with self.assertRaises(probbit.ProbbitInputError) as e:
            probbit.persona_turn(doc, edited, {})
        self.assertEqual(e.exception.path, "state.digest")

    def test_a_refusal_is_an_answer_with_the_habits_only(self):
        doc = load("tutor.json"); doc["engine"] = dict(doc.get("engine") or {}, op="sample", sweeps=8)
        st = probbit.persona_init(doc)
        r = probbit.persona_turn(doc, st, {"loss": True})
        s = r["stance"]
        self.assertIn(s["status"], ("refused", "partial", "fallback")); self.assertTrue(s["escalate"])
        unvouched = [t for t, e in s["stance"].items() if not e["released"]]
        self.assertTrue(unvouched)
        self.assertIn("no jokes, be kind", s["line"])  # a habit in force is always safe to state

    def test_a_state_rewritten_by_python_still_reads(self):
        # Python writes 0.0 where the CLI wrote 0: the canonical number rule makes it the same state (same digest)
        doc = load("trader-assistant.json")
        st = json.loads(json.dumps(probbit.persona_init(doc)), parse_int=float)
        self.assertEqual(probbit.persona_turn(doc, st, {"error": True})["stance"]["turn"], 0)


    def test_fuzz_and_prove_on_the_0_5_0_tutor(self):
        # the tutor as it shipped in 0.5.0: one upset message can make an individual playful; its loss habit holds by construction
        tutor = os.path.join(EX, "..", "..", "probbit-cli", "tests", "fixtures", "persona", "tutor-0.5.0.yaml")
        upset = {"when": {"sentiment": "negative"}, "then": {"humour": {"at_most": "light"}}}
        f = probbit.persona_fuzz(tutor, never=upset, seeds="0-4", scripts=10, grid=[0, 1])
        self.assertEqual(f["probbit_persona_fuzz"], 1)
        self.assertTrue(f["found"])
        self.assertEqual(f["properties"][0]["shortest"]["script"], [{"sentiment": "negative"}])
        p = probbit.persona_prove(tutor, props=[dict(upset, id="not_playful"), {"id": "loss", "when": {"loss": True}, "then": {"humour": ["none"]}}], seeds=[0, 1])
        self.assertEqual(p["probbit_persona_prove"], 1)
        self.assertEqual([x["verdict"] for x in p["properties"]], ["unknown", "held_by_construction"])
        with self.assertRaises(probbit.ProbbitInputError) as e:
            probbit.persona_prove(tutor, never={"when": {"sentimentx": "negative"}, "then": {"humour": ["none"]}})
        self.assertEqual(e.exception.code, "persona")
        with self.assertRaises(TypeError):
            probbit.persona_fuzz(tutor)


class Live(unittest.TestCase):
    def test_live_event_logs_a_strand_that_verifies(self):
        import tempfile
        tutor = os.path.join(EX, "tutor.yaml")
        with tempfile.TemporaryDirectory() as d:
            strand = os.path.join(d, "pip.strand")
            r = probbit.live_event(tutor, None, {"loss": True}, seed=2, strand=strand)
            self.assertEqual(r["stance"]["stance"]["humour"]["level"], "none")
            self.assertEqual((r["strand"]["events"], r["stance"]["inputs"]["loss"]), (1, True))
            for ev in ({"praise": True, "elapsed_hours": 2.5}, {"sentiment": "negative", "elapsed_hours": 12}):
                r = probbit.live_event(tutor, r["state"], ev, strand=strand)
            self.assertEqual(r["strand"]["events"], 3)
            v = probbit.live_verify(strand)
            self.assertEqual((v["ok"], v["events"], v["last_line"], v["final_state"]), (True, 3, r["strand"]["head"], r["state"]["digest"]))
            with open(strand, encoding="utf-8", newline="") as f:  # bytes as written: a strand is verified byte for byte (no CRLF on Windows)
                text = f.read()
            with open(strand, "w", encoding="utf-8", newline="") as f:
                f.write(text.replace('"elapsed_hours":2.5', '"elapsed_hours":2.6'))
            self.assertEqual(probbit.live_verify(strand), {"ok": False, "line": 3, "diverges": "the stance differs"})
            with self.assertRaises(probbit.ProbbitInputError) as e:
                probbit.live_event(tutor, r["state"], {"elapsed_hours": -1})
            self.assertEqual(e.exception.path, "events[0].event.elapsed_hours")


if __name__ == "__main__":
    unittest.main()