The p4 models: directions, measured

The runs so far (nested-store.md and the Colab results under .docs/p4/nested-store-results/) point to five directions. This notebook measures each one and ends with a single report:

	Direction	What this notebook measures
D1	Keep the JSON ranker as the default	its leave-one-pool-out ranking against plain word overlap, scored against the store's labels and each teacher's
D2	Fix pretraining before more architecture	the single .pt ranker at pretraining budgets from 30 s to 6 min, with early stopping on held-back pools: which budget ranks best, and whether any of them raises the JSON ranker
D3	Keep the tree for what it proves	update size, forgetting and query time per arm, from the Q-matrix suite already run
D4	Labels are the main lever	two teachers through the Anthropic API (Claude Fable 5.1 and Claude Opus 5.5) label every pool; models trained on each label set and scored against each; how far the label sets agree
D5	Settle whether the φ structure matters	the pathways notebook's verdicts, read from its run on Drive

The Anthropic API is used wherever it serves the work: the two teachers, and the closing written report, which Claude drafts from the measured JSON, with the numbers printed beside it. The run reuses an earlier run's teacher labels when the pairs match exactly, so nothing is labelled twice.

Drive. As the other notebooks: everything is saved to Drive as it is produced, and Run all after a disconnect resumes where it stopped. To compare with the Q-matrix run, the store is pinned to the commit that run used, and its pairs file is reused (FROM_RUN).

Secrets:

• GITHUB_TOKEN
• ANTHROPIC_API_KEY
• ANTHROPIC_WORKSPACE_ID, if your key is not scoped to a workspace

Runtime: the T4 if you can get it (directions_colab_t4.ipynb); the CPU works. D2 pretrains five budgets on two seeds, about 27 minutes of pretraining a seed, then fine-tunes for each label set.

0. Settings

Edit, then Runtime → Run all. A new RUN name starts a fresh run; the same name resumes.

DRIVE_DIR = "/content/drive/MyDrive/phi-colab"   # everything this notebook keeps lives under DRIVE_DIR/RUN
RUN = "directions-1"
FROM_RUN = "q-matrix-t4"              # an earlier Q-matrix run: its store pin, pairs and teacher labels are reused
PATHWAYS_RUN = "pathways-t4"          # the pathways run D5 reads its verdicts from

STORE_REPO = "ezexe/AudioPlugin-p4"
STORE_BRANCH = "master"
STORE_PATH = ""
PHI_REF = "main"
P4_REF = "main"

BUDGETS = [0.5, 1, 2, 4, 6]           # D2: pretraining minutes for the single .pt ranker
SEEDS = [0, 1]
EPOCHS = 10
FOLDS = 4
CHECK_S = 20                          # early stopping: seconds between dev checks

TEACHERS = {"fable": "claude-fable-5-1", "opus": "claude-opus-5-5"}   # D4: name -> model, labelled through the API
TEACHER_EFFORT = "high"
TEACHER_MAX_USD = 25.0                # per teacher: nothing is spent when its estimate is above this
REPORT_MODEL = "claude-fable-5-1"     # writes the closing report from the measured JSON

DEVICE = "auto"                       # auto: the GPU when the runtime has one; "cuda" requires it; "cpu" forces the CPU

PUSH_RESULTS = False                  # True: commit the results into ezexe/phi on PUSH_BRANCH (a PR is yours to open)
PUSH_BRANCH = "colab/directions-results"

1. Drive, state and logging

Mounts Drive and creates the run's folders. state.json records each finished step; every command's output is appended to a log on Drive as it runs.

import datetime, json, os, pathlib, shutil, subprocess, sys, time

try:
    from google.colab import drive
    drive.mount("/content/drive")
    IN_COLAB = True
except ImportError:                    # run elsewhere: DRIVE_DIR is an ordinary folder
    IN_COLAB = False

ROOT = pathlib.Path(DRIVE_DIR) / RUN
for d in ("results", "cache", "logs", "models", "pairs"):
    (ROOT / d).mkdir(parents=True, exist_ok=True)
STATE = ROOT / "state.json"
WORK = pathlib.Path("/content/work" if IN_COLAB else ROOT.parent / (RUN + "-work"))
WORK.mkdir(parents=True, exist_ok=True)


def now():
    return datetime.datetime.now().strftime("%Y-%m-%d %H:%M")


def save_json(path, obj):
    """Write through a temporary file and rename, so a disconnect mid-write never leaves half a file on Drive."""
    path = pathlib.Path(path)
    tmp = path.with_name(path.name + ".tmp")
    tmp.write_text(json.dumps(obj, indent=1), encoding="utf-8")
    os.replace(tmp, path)


def load_state():
    try:
        return json.loads(STATE.read_text(encoding="utf-8"))
    except (OSError, ValueError):
        return {"created": now(), "done": {}}


def mark(step, **info):
    st = load_state()
    st.setdefault("done", {})[step] = {"at": now(), **info}
    save_json(STATE, st)


def done(step):
    return step in load_state().get("done", {})


HEARTBEAT_S = 120                      # a line when a command has printed nothing for this long


def run(cmd, log, cwd=None, env=None, check=True):
    """Run a command, echo its output here with the time and append it to logs/<log>.log on Drive line by line;
    after HEARTBEAT_S of silence, say it is still running and what it last said."""
    import threading
    path = ROOT / "logs" / f"{log}.log"
    started = time.time()
    last = {"t": started, "line": "(started)"}
    with open(path, "a", encoding="utf-8") as f:
        f.write(f"\n# {now()} $ {' '.join(map(str, cmd))}\n")
        f.flush()
        p = subprocess.Popen([str(c) for c in cmd], cwd=cwd,
                             env={**os.environ, "PYTHONUNBUFFERED": "1", **(env or {})},
                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1)
        stop = threading.Event()

        def beat():
            while not stop.wait(15):
                if time.time() - last["t"] >= HEARTBEAT_S:
                    print(f"{time.strftime('%H:%M:%S')} ... {log} still running ({(time.time() - started) / 60:.0f} min); "
                          f"last: {last['line'][:100]}", flush=True)
                    last["t"] = time.time()
        threading.Thread(target=beat, daemon=True).start()
        for line in p.stdout:
            last["t"], last["line"] = time.time(), line.strip()
            print(f"{time.strftime('%H:%M:%S')} {line}", end="", flush=True)
            f.write(line)
            f.flush()
        code = p.wait()
        stop.set()
        f.write(f"# exit {code} after {(time.time() - started) / 60:.1f} min\n")
    if check and code != 0:
        raise RuntimeError(f"{cmd[0]} exited {code}; see {path}")
    return code


print("run folder:", ROOT)
print("done so far:", ", ".join(load_state().get("done", {})) or "nothing")

1b. Reuse the earlier run

The first session copies FROM_RUN's store pin and pairs file into this run, so D4 can reuse that run's teacher labels and the numbers stay comparable. If FROM_RUN isn't on Drive, this run pins and exports its own.

prior = pathlib.Path(DRIVE_DIR) / FROM_RUN
st = load_state()
if FROM_RUN and (prior / "state.json").exists() and "store" not in st.get("pins", {}):
    pst = json.loads((prior / "state.json").read_text())
    if pst.get("pins", {}).get("store"):
        st.setdefault("pins", {})["store"] = pst["pins"]["store"]
        save_json(STATE, st)
        print(f"store pinned to {FROM_RUN}'s commit {pst['pins']['store'][:8]}")
    if (prior / "pairs" / "pairs.jsonl").exists() and not (ROOT / "pairs" / "pairs.jsonl").exists():
        shutil.copy2(prior / "pairs" / "pairs.jsonl", ROOT / "pairs" / "pairs.jsonl")
        mark("pairs", from_run=FROM_RUN)
        print(f"pairs copied from {FROM_RUN}")
else:
    print("from run:", "already applied" if "store" in st.get("pins", {}) else f"{FROM_RUN} not found on Drive; this run pins its own")

2. Repositories, pinned

Clones ezexe/phi, ezexe/p4 and the store into the runtime's disk. The first session records each commit in state.json; later sessions check out those same commits, so a resumed run measures the same code on the same data. Delete pins from state.json to move to newer commits.

def token():
    if not IN_COLAB:
        return os.environ.get("GITHUB_TOKEN", "")
    from google.colab import userdata
    try:
        return userdata.get("GITHUB_TOKEN") or ""
    except Exception:                  # no such secret, or access not granted to this notebook
        return ""


def url(repo):
    t = token()
    return f"https://x-access-token:{t}@github.com/{repo}.git" if t else f"https://github.com/{repo}.git"


def checkout(repo, ref, dest, pin):
    """Clone with full history (the shards are cut by when git first added each file) but no LFS objects, then
    check out the pinned commit, or `ref` on the first session."""
    env = {"GIT_LFS_SKIP_SMUDGE": "1", "GIT_TERMINAL_PROMPT": "0"}
    if not (dest / ".git").exists():
        r = subprocess.run(["git", "clone", "-q", "--filter=blob:none", "--no-checkout", url(repo), str(dest)],
                           env={**os.environ, **env}, capture_output=True, text=True)
        if r.returncode != 0:          # the error may hold the URL, so the token is stripped before printing
            raise RuntimeError(f"cloning {repo} failed: " + r.stderr.replace(token() or "\0", "***"))
    target = pin or f"origin/{ref}"
    subprocess.run(["git", "-C", str(dest), "checkout", "-q", "--force", target], env={**os.environ, **env}, check=True)
    return subprocess.run(["git", "-C", str(dest), "rev-parse", "HEAD"], capture_output=True, text=True).stdout.strip()


st = load_state()
pins = st.get("pins", {})
PHI = WORK / "phi"
P4 = WORK / "p4"
pins["phi"] = checkout("ezexe/phi", PHI_REF, PHI, pins.get("phi"))
pins["p4"] = checkout("ezexe/p4", P4_REF, P4, pins.get("p4"))
if STORE_PATH:
    STORE = pathlib.Path(STORE_PATH)
else:
    STORE = WORK / "store"
    pins["store"] = checkout(STORE_REPO, STORE_BRANCH, STORE, pins.get("store"))
st = load_state()
st["pins"] = pins
save_json(STATE, st)

PLUGIN = P4 / "lbiza" / "plugins" / "vlds"
PICK = PLUGIN / "scripts" / "pick_model.py"
SUITE = PHI / "scripts" / "session" / "nested_suite.py"
run([sys.executable, "-m", "pip", "install", "-q", "--no-deps", "-e", str(PHI)], "setup")
sys.path.insert(0, str(PHI))

# the training device, checked once per runtime: a short pretrain and fine-tune on it, the CPU if that fails
import torch
import phi_rank as pr
want = "cpu" if DEVICE == "cpu" else ("cuda" if torch.cuda.is_available() else "cpu")
if DEVICE == "cuda" and want != "cuda":
    raise RuntimeError("DEVICE = 'cuda' but this runtime has no GPU: Runtime -> Change runtime type -> T4 GPU, then Run all")
if want == "cuda":
    try:
        os.environ["PHI_DEVICE"] = "cuda"
        m = pr.Ranker(1 << 13, 16)
        texts = [f"the ring buffer drain rule {i}" for i in range(64)]
        assert pr.pretrain(m, list(zip(texts, texts[1:])), 0.05, log=lambda *_: None) > 0
        pr.finetune(m, [{"task": "the ring buffer", "rows": [{"text": t, "label": int(i < 3)}
                                                               for i, t in enumerate(texts)]}], epochs=1)
        assert next(m.parameters()).device.type == "cpu"
    except Exception as e:                 # under auto a broken CUDA path must not stop the run: the CPU still measures
        if DEVICE == "cuda":
            raise RuntimeError(f"GPU self-check failed: {type(e).__name__}: {e}") from e
        print(f"GPU self-check failed ({type(e).__name__}: {e}); training on the CPU")
        want = "cpu"
os.environ["PHI_DEVICE"] = want
gpu = torch.cuda.get_device_name(0) if want == "cuda" else ""
print(f"training on: {want} {gpu}".strip(), f"| CPU cores: {os.cpu_count()}")
st = load_state()
devices = st.setdefault("devices", [])
if devices and devices[-1]["device"] != (gpu or want):
    print(f"note: earlier sessions of this run trained on {devices[-1]['device']}. Runs already on Drive keep "
          "their models; new runs will differ in how many steps fit the time budget.")
devices.append({"at": now(), "device": gpu or want, "cpus": os.cpu_count()})
save_json(STATE, st)

SESSION = next(p for p in sorted((STORE / "sessions").iterdir()) if (p / "vlds").is_dir()) / "vlds"
print("pins:", json.dumps(pins))
print("store:", STORE, "| a session store for the plugin:", SESSION.relative_to(STORE))

4. The store's labelled pairs

The vlds plugin exports every labelled pool of the store (pick_model.py pairs). The file is kept on Drive.

PAIRS = ROOT / "pairs" / "pairs.jsonl"
if not (done("pairs") and PAIRS.exists()):
    run([sys.executable, str(PICK), "--store", str(SESSION), "pairs", "--jsonl", str(PAIRS)], "pairs")
    n = sum(1 for _ in open(PAIRS, encoding="utf-8"))
    mark("pairs", rows=n)
pools = len({json.loads(l)["pool_id"] for l in open(PAIRS, encoding="utf-8")})
print(f"{pools} labelled pools in {PAIRS}")
assert pools >= FOLDS, "too few labelled pools to fold"

4b. The Anthropic API

Installs the SDK and reads ANTHROPIC_API_KEY and ANTHROPIC_WORKSPACE_ID from the environment or the Colab secrets into the environment the scripts run in. Neither is printed or saved.

run([sys.executable, "-m", "pip", "install", "-q", "anthropic"], "setup")
for name in ("ANTHROPIC_API_KEY", "ANTHROPIC_WORKSPACE_ID"):
    if not os.environ.get(name) and IN_COLAB:
        from google.colab import userdata
        try:
            os.environ[name] = userdata.get(name) or ""
        except Exception:                  # no such secret, or notebook access not granted
            pass
API = bool(os.environ.get("ANTHROPIC_API_KEY"))
TEACH_PY = PHI / "scripts" / "session" / "teacher.py"
print("Anthropic API:", "ready" if API else "no ANTHROPIC_API_KEY: the teachers and the written report are skipped",
      "| workspace header:", "set" if os.environ.get("ANTHROPIC_WORKSPACE_ID") else "not set")

D4a. The teachers label every pool

Each teacher in TEACHERS labels every pool through teacher.py label: the input is counted first, nothing is spent above TEACHER_MAX_USD, and each pool is saved to Drive as it lands. When FROM_RUN used the same pairs file, its Fable labels are copied instead of paid for again.

import hashlib


def sha(path):
    return hashlib.sha1(pathlib.Path(path).read_bytes()).hexdigest()


TEACH_DIRS = {}
for name, model in TEACHERS.items():
    d = ROOT / "teacher" / name
    d.mkdir(parents=True, exist_ok=True)
    old = pathlib.Path(DRIVE_DIR) / FROM_RUN / "results" / "teacher"
    same_pairs = (pathlib.Path(DRIVE_DIR) / FROM_RUN / "pairs" / "pairs.jsonl").exists() and \
        sha(pathlib.Path(DRIVE_DIR) / FROM_RUN / "pairs" / "pairs.jsonl") == sha(PAIRS)
    if model == "claude-fable-5-1" and old.is_dir() and same_pairs and not any(d.iterdir()):
        for f in old.glob("*.json"):
            shutil.copy2(f, d / f.name)
        print(f"teacher {name}: {len(list(d.glob('*.json')))} pools copied from {FROM_RUN}")
    if API:
        run([sys.executable, str(TEACH_PY), "label", "--pairs", PAIRS, "--out-dir", d, "--model", model,
             "--effort", TEACHER_EFFORT, "--max-usd", TEACHER_MAX_USD], f"teacher-{name}", check=False)
    if any(d.glob("*.json")):
        TEACH_DIRS[name] = d
print("label sets for D4:", ["store"] + list(TEACH_DIRS))

D1, D2, D4. The measured legs

One directions_suite.py run over every budget and seed, resumable through its cache on Drive. Every 6th pool is held back as the dev set early stopping reads, and kept out of every score.

DIRECTIONS = PHI / "scripts" / "session" / "directions_suite.py"
out = ROOT / "results" / "directions.json"
if not done("directions"):
    cmd = [sys.executable, str(DIRECTIONS), "--p4", str(STORE), "--pairs", str(PAIRS), "--plugin", str(PLUGIN),
           "--budgets", *BUDGETS, "--seeds", *SEEDS, "--epochs", EPOCHS, "--folds", FOLDS, "--check-s", CHECK_S,
           "--cache", ROOT / "cache", "--out", out]
    for name, d in TEACH_DIRS.items():
        cmd += ["--teacher", f"{name}={d}"]
    run(cmd, "directions")
    mark("directions")
directions = json.loads(out.read_text())
print("agreement between label sets (F1):", directions["agreement_f1"])
print("JSON ranker:", json.dumps(directions["json_ranker"]))

D3. The tree's structural properties

Read from the Q-matrix suite already run: FROM_RUN on Drive, or the copy committed to the repository.

import glob, statistics
src = sorted((pathlib.Path(DRIVE_DIR) / FROM_RUN / "results").glob("suite-k*-s*.json"))
if not src:
    src = sorted((PHI / ".docs" / "p4" / "nested-store-results").glob(f"colab-{FROM_RUN}/suite-k*-s*.json"))
streams = [(json.loads(p.read_text())["k"], r["stream"]) for p in src for r in json.loads(p.read_text())["runs"]]
d3 = {}
if streams:
    arms = sorted(streams[0][1])
    for arm in arms:
        d3[arm] = {m: round(statistics.mean(s[arm][m] for _k, s in streams), 4)
                   for m in ("msg_bytes", "forget", "update_s", "query_ms")}
    d3_claims = {
        "time-routed nodes forget nothing (every run)": all(s["routed"]["forget"] == 0 for _k, s in streams),
        "a routed update is smaller than one large model's (every run)":
            all(s["qrouted"]["msg_bytes"] < s["big"]["msg_bytes"] for _k, s in streams),
    }
    print(f"D3 from {len(streams)} suite runs:")
    for arm in ("big", "zeck_ens", "routed", "qrouted"):
        print(f"  {arm:>10}: update {d3[arm]['msg_bytes']:,.0f} B, forget {d3[arm]['forget']}, query {d3[arm]['query_ms']} ms")
    print(d3_claims)
else:
    d3_claims = {}
    print(f"D3: no suite results for {FROM_RUN} on Drive or in the repository")

D5. Does the φ structure matter?

The pathways notebook's verdicts, read from PATHWAYS_RUN on Drive. If that run hasn't finished, D5 reads not measured: run the pathways notebook, then this cell again.

pw = pathlib.Path(DRIVE_DIR) / PATHWAYS_RUN / "summary.json"
d5 = json.loads(pw.read_text()) if pw.exists() else None
if d5:
    for c, v in d5["claims"].items():
        print(f"  {v:>14}  {c}")
else:
    print(f"D5: {PATHWAYS_RUN} has no summary on Drive yet: not measured")

The direction report

One verdict per direction, the settings the verdicts imply, a plot of D2, and the written report Claude drafts from summary.json. The drafted report shows each number beside its source, so check them against the JSON.

import statistics
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from IPython.display import Markdown, Image, display


def verdict(ok, measured=True):
    return "not measured" if not measured else ("holds" if ok else "does not hold")


runs = directions["runs"]
jr = directions["json_ranker"]
budgets = sorted({r["budget_min"] for r in runs})
by_budget = {b: [r for r in runs if r["budget_min"] == b] for b in budgets}
mean_q = lambda b, key: round(statistics.mean(r["quality"].get(key, float("nan")) for r in by_budget[b]), 4)  # noqa: E731
best_b = max(budgets, key=lambda b: mean_q(b, "store->store")) if budgets else None
raises = {b: all(r.get("deployed", 0) > jr["store"]["json_ranker"] for r in by_budget[b]) for b in budgets}
teachers = [n for n in directions["label_sets"] if n != "store"]
ag = directions["agreement_f1"]

claims = {
    "D1 the JSON ranker ranks better than plain word overlap (store's labels)":
        verdict(jr["store"]["json_ranker"] > jr["store"]["term_overlap"], "store" in jr),
    "D2 a shorter pretraining budget ranks better than the longest":
        verdict(best_b is not None and best_b < max(budgets)
                and mean_q(best_b, "store->store") > mean_q(max(budgets), "store->store"), bool(budgets)),
    "D2 some pretraining budget's .pt raises the JSON ranker on every seed":
        verdict(any(raises.values()), bool(budgets)),
    **{f"D3 {c}": verdict(v) for c, v in d3_claims.items()},
    **{f"D4 the {t} labels are more learnable than the store's (trained and scored on themselves, best budget)":
       verdict(mean_q(best_b, f"{t}->{t}") > mean_q(best_b, "store->store"), best_b is not None) for t in teachers},
    **{f"D4 training on the {t} labels predicts the store's labels at least as well as training on them":
       verdict(mean_q(best_b, f"{t}->store") >= mean_q(best_b, "store->store"), best_b is not None) for t in teachers},
}
if len(teachers) >= 2:
    a, b = teachers[:2]
    claims["D4 the two teachers agree with each other more than either agrees with the store's labels"] = verdict(
        (ag.get(f"{a}~{b}") or 0) > max(ag.get(f"store~{a}") or 0, ag.get(f"store~{b}") or 0))
if d5:
    claims.update({f"D5 {c}": v for c, v in d5["claims"].items()})
else:
    claims["D5 the φ structure matters"] = "not measured"

settings = {
    "pool-effort": "low or unnamed (the JSON ranker alone)" if not any(raises.values())
                   else f"medium or high, with the .pt pretrained {min(b for b in budgets if raises[b]):g} min",
    "pretraining budget": f"{best_b:g} min (best held-out ranking)" if best_b is not None else "not measured",
    "labels": ("train on the teachers' labels: they are more learnable than the store's" if teachers and all(
                   claims.get(f"D4 the {t} labels are more learnable than the store's (trained and scored on themselves, "
                              "best budget)") == "holds" for t in teachers)
               else "keep the store's labels: no teacher's were more learnable" if teachers
               else "the store's labels (no teacher ran)"),
}
summary = {"run": RUN, "at": now(), "pins": load_state().get("pins"), "devices": load_state().get("devices"),
           "from_run": FROM_RUN, "pathways_run": PATHWAYS_RUN, "claims": claims, "implied_settings": settings,
           "json_ranker": jr, "agreement_f1": ag,
           "by_budget": {f"{b:g}": {k: mean_q(b, k) for k in runs[0]["quality"]} | {
               "deployed": round(statistics.mean(r.get("deployed", 0) for r in by_budget[b]), 4),
               "steps": round(statistics.mean(r["early_stopping"].get("steps", 0) for r in by_budget[b])),
               "best_steps": round(statistics.mean(r["early_stopping"].get("best_steps", 0) for r in by_budget[b]))}
               for b in budgets} if runs else {},
           "d3": d3, "d5": d5 and {"claims": d5["claims"], "by_k": d5.get("by_k")}}
save_json(ROOT / "summary.json", summary)

lines = [f"# The p4 models' directions: {RUN}", "", f"Pinned commits: `{json.dumps(summary['pins'])}`", "",
         "## Verdicts", ""] + [f"- {c}: **{v}**" for c, v in claims.items()]
lines += ["", "## Implied settings", ""] + [f"- **{k}:** {v}" for k, v in settings.items()]
lines += ["", "## D2 by pretraining budget (mean over seeds)", "",
          "| budget (min) | steps | best dev step | " + " | ".join(runs[0]["quality"]) + " | JSON + .pt |",
          "|---|---|---|" + "---|" * len(runs[0]["quality"]) + "---|"] if runs else []
for b, row in summary["by_budget"].items():
    lines.append(f"| {b} | {row['steps']:,} | {row['best_steps']:,} | "
                 + " | ".join(str(row[k]) for k in runs[0]["quality"]) + f" | {row['deployed']} |")
lines += ["", f"JSON ranker alone: {jr['store']['json_ranker']} (word overlap {jr['store']['term_overlap']}); "
          f"label-set agreement (F1): {ag}"]
(ROOT / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8")

if runs:
    fig, ax = plt.subplots(figsize=(7, 4))
    for key in runs[0]["quality"]:
        if key.split("->")[0] == key.split("->")[1]:
            ax.plot(budgets, [mean_q(b, key) for b in budgets], marker="o", label=f"trained and scored on {key.split('->')[0]}")
    ax.axhline(jr["store"]["json_ranker"], color="#888", ls=":", label="JSON ranker (store labels)")
    ax.set(xlabel="pretraining budget (min)", ylabel="held-out r-precision", title="D2: does longer pretraining help?")
    ax.legend(fontsize=8)
    fig.tight_layout()
    fig.savefig(ROOT / "summary.png", dpi=140)
display(Markdown((ROOT / "summary.md").read_text()))
if runs:
    display(Image(str(ROOT / "summary.png")))

if API and not (ROOT / "report.md").exists():
    run([sys.executable, str(TEACH_PY), "report", "--summary", ROOT / "summary.json", "--out", ROOT / "report.md",
         "--model", REPORT_MODEL], "report", check=False)
if (ROOT / "report.md").exists():
    display(Markdown((ROOT / "report.md").read_text()))

8. Optional: commit the results into ezexe/phi

With PUSH_RESULTS = True and a GITHUB_TOKEN that can write to ezexe/phi, this cell:

• copies the results into .docs/p4/nested-store-results/colab-<RUN>/
• commits them on PUSH_BRANCH
• pushes the branch

Opening a pull request and updating nested-store.md stay yours. This cell runs once per run; delete push from state.json to push again.

if PUSH_RESULTS and not done("push"):
    if not token():
        raise RuntimeError("PUSH_RESULTS needs a GITHUB_TOKEN secret that can write to ezexe/phi")
    dest = WORK / "phi-push"
    if dest.exists():
        shutil.rmtree(dest)
    subprocess.run(["git", "clone", "-q", "--depth", "1", url("ezexe/phi"), str(dest)], check=True,
                   capture_output=True)
    subprocess.run(["git", "-C", str(dest), "checkout", "-q", "-B", PUSH_BRANCH], check=True)
    out = dest / ".docs" / "p4" / "nested-store-results" / f"colab-{RUN}"
    out.mkdir(parents=True, exist_ok=True)
    for p in list((ROOT / "results").glob("*.json")) + [ROOT / "summary.json", ROOT / "summary.md", ROOT / "summary.png",
              ROOT / "report.md"]:
        if p.exists():
            shutil.copy2(p, out / p.name)
    git = ["git", "-C", str(dest), "-c", "user.name=phi colab", "-c", "user.email=phi-colab@users.noreply.github.com"]
    subprocess.run(git + ["add", "-A"], check=True)
    subprocess.run(git + ["commit", "-q", "-m", f"nested-store results from Colab run {RUN}"], check=True)
    r = subprocess.run(["git", "-C", str(dest), "push", "-q", "-u", "origin", PUSH_BRANCH], capture_output=True, text=True)
    if r.returncode != 0:
        raise RuntimeError("push failed: " + r.stderr.replace(token(), "***"))
    mark("push", branch=PUSH_BRANCH)
    print(f"pushed to {PUSH_BRANCH}: open a pull request on https://github.com/ezexe/phi/compare/{PUSH_BRANCH}")
else:
    print("push:", "done earlier" if done("push") else "off (PUSH_RESULTS = False)")