# ╔══════════════════════════════════════════════════════════════════╗
# ║ Rig Conductor — labels.train ║
# ║ « from a label-set version to a registered model on Aoraki » ║
# ╠══════════════════════════════════════════════════════════════════╣
# ║ split here → upload → sleap_nn_train.sh → eval.json → registry. ║
# ╚══════════════════════════════════════════════════════════════════╝
"""Fine-tune a pose model on a label-set version and decide whether it replaces
the current one.
Promotion rule (Decisions 2026-11-11): the new model is offered as the default
only if it is not worse than the current model on any validation set and better
on at least one; Bart confirms it in the wizard.
"""
from __future__ import annotations
import json
import re
import shlex
import subprocess
from dataclasses import dataclass, field, fields
from datetime import date
from pathlib import Path, PurePosixPath
import yaml
from rigconductor import ConductorError, sleapenv
from rigconductor.config import ConductorConfig
from rigconductor.constants import LABEL_TEST_SHARE, NN_ENV_REMOTE, SSH_TIMEOUT_S, TRAIN_EPOCHS
from rigconductor.labels.store import LabelSet
TRAIN_SCRIPTS = ( # thermokourt repo → $PROJ/scripts
"slurm/sleap_nn_train.sh",
"slurm/sleap_track.sh",
"sleap/nn_config.py",
"sleap/nn_eval.py",
)
EPOCH_RE = re.compile(r"Epoch (\d+)")
METRIC = "mAP"
HEADS = ("centroid", "centered_instance") # trained one after the other
# training_log.csv → "epoch,train loss,val loss,seconds" (column names differ by version)
CURVE_AWK = (
"NR==1{for(i=1;i<=NF;i++){"
'if($i=="epoch")e=i;'
'if($i=="train/loss"||$i=="train_loss")t=i;'
'if($i=="val/loss"||$i=="val_loss")v=i;'
'if($i=="train/time")s=i'
"};next}"
'{print $e","(t?$t:"")","(v?$v:"")","(s?$s:"")}'
)
# the early-stopping patience (the scheduler has a "patience" too, earlier in the file)
PATIENCE_AWK = "/early_stopping:/{f=1} f&&/patience:/{print $2; exit}"
[docs]
@dataclass
class TrainPlan:
set_name: str
version: int
run: str
base_run: str
compare_run: str
remote_dir: str = ""
val: dict[str, str] = field(default_factory=dict) # scale → remote file
job_id: str = ""
excluded: list[str] = field(default_factory=list) # recordings left out of training
shared: list[str] = field(default_factory=list) # recordings split by frame
[docs]
@classmethod
def from_entry(cls, entry: dict) -> TrainPlan:
"""Rebuild a plan from its record in the label set's ``models`` list."""
return cls(**{f.name: entry[f.name] for f in fields(cls) if f.name in entry})
[docs]
def project_root(cfg: ConductorConfig, profile: str) -> str:
"""Aoraki project folder (the registry lives in ``<project>/models/``)."""
explicit = cfg.profile(profile).get("project_root")
if explicit:
return explicit
reg = cfg.registry_remote(profile)
if not reg:
raise ConductorError(f"profile '{profile}' has no model registry on the cluster")
return str(PurePosixPath(reg).parents[1])
[docs]
def run_name(lset: LabelSet, version: int, today: date | None = None) -> str:
"""``<set>_v<N>_<date>``, with b, c, … added when that run exists already."""
name = f"{lset.name}_v{version}_{(today or date.today()):%Y%m%d}"
taken = {m.get("run") for m in lset.data["models"]}
suffixes = iter("bcdefghijklmnopqrstuvwxyz")
out = name
while out in taken:
out = name + next(suffixes)
return out
def _ssh(cfg: ConductorConfig, remote: str, run=subprocess.run, timeout=SSH_TIMEOUT_S) -> str:
r = run(
["ssh", "-o", "BatchMode=yes", cfg.cluster.ssh_host, remote],
capture_output=True,
text=True,
timeout=timeout,
)
if r.returncode != 0:
raise ConductorError(f"Aoraki: {(r.stderr or r.stdout).strip()[-400:]}")
return r.stdout
def _rsync(cfg: ConductorConfig, files: list[Path], remote_dir: str, run=subprocess.run) -> None:
_ssh(cfg, f"mkdir -p {shlex.quote(remote_dir)}", run=run)
r = run(
["rsync", "-a", *[str(f) for f in files], f"{cfg.cluster.ssh_host}:{remote_dir}/"],
capture_output=True,
text=True,
)
if r.returncode != 0:
raise ConductorError(f"upload to Aoraki failed: {r.stderr.strip()[-400:]}")
# ── steps ─────────────────────────────────────────────────────────
[docs]
def summarise(lset: LabelSet, version: int, env, log=None, progress=None) -> list[dict]:
"""Per-recording facts of a version (cached: version files never change)."""
out_dir = lset.path / "train" / f"v{version}"
cache = out_dir / "summary.json"
if cache.is_file():
rows = json.loads(cache.read_text(encoding="utf-8"))
if all(Path(r["example"]).is_file() for r in rows):
return rows
res = sleapenv.run_task(
env,
"summary",
{"slp": str(lset.version_file(version)), "out_dir": str(out_dir / "examples")},
log=log,
progress=progress,
)
cache.write_text(json.dumps(res["recordings"], indent=1), encoding="utf-8")
return res["recordings"]
[docs]
def prepare(
lset: LabelSet, version: int, env, exclude=(), log=None, progress=None
) -> dict[str, Path]:
"""Split a version into train / validation files (by recording, or by
frame for the recordings set to *All*).
Args:
exclude: Recordings left out of training altogether.
"""
out_dir = lset.path / "train" / f"v{version}"
out_dir.mkdir(parents=True, exist_ok=True)
for old in out_dir.glob("*.pkg.slp"):
old.unlink()
res = sleapenv.run_task(
env,
"split",
{
"slp": str(lset.version_file(version)),
"val": lset.data["validation"],
"exclude": list(exclude),
"shared": lset.shared(),
"test_share": LABEL_TEST_SHARE,
"out_dir": str(out_dir),
},
log=log,
progress=progress,
)
files = {part: out_dir / f"{part}.pkg.slp" for part, n in res["counts"].items() if n}
if "train" not in files:
raise ConductorError("no training frames left after holding out the validation recordings")
return files
[docs]
def upload(
cfg: ConductorConfig,
profile: str,
lset: LabelSet,
plan: TrainPlan,
files: dict[str, Path],
scripts_root: Path,
run=subprocess.run,
) -> TrainPlan:
"""Send the split and the training scripts to Aoraki; fills in remote paths."""
proj = project_root(cfg, profile)
plan.remote_dir = f"{proj}/sleap_labels/{lset.name}/v{plan.version}"
_rsync(cfg, list(files.values()), plan.remote_dir, run=run)
plan.val = {
part[len("val_") :]: f"{plan.remote_dir}/{p.name}"
for part, p in files.items()
if part.startswith("val_")
}
for sub in ("slurm", "sleap"):
mine = [scripts_root / s for s in TRAIN_SCRIPTS if s.startswith(sub + "/")]
_rsync(cfg, mine, f"{proj}/scripts/{sub}", run=run)
return plan
def submit_command(proj: str, plan: TrainPlan, epochs: int = TRAIN_EPOCHS) -> str:
vals = " ".join(shlex.quote(f"{k}={v}") for k, v in sorted(plan.val.items()))
return (
f"cd {shlex.quote(proj)} && mkdir -p logs && EPOCHS={int(epochs)} "
f"NN_ENV={shlex.quote(proj + '/' + NN_ENV_REMOTE)} sbatch --parsable "
f"scripts/slurm/sleap_nn_train.sh {shlex.quote(plan.remote_dir + '/train.pkg.slp')} "
f"{shlex.quote(plan.run)} {shlex.quote(plan.base_run)} {shlex.quote(plan.compare_run)} "
f"{vals}"
).rstrip()
def submit(
cfg: ConductorConfig, profile: str, plan: TrainPlan, epochs=TRAIN_EPOCHS, run=subprocess.run
):
out = _ssh(cfg, submit_command(project_root(cfg, profile), plan, epochs), run=run)
m = re.search(r"^(\d+)", out.strip().splitlines()[-1] if out.strip() else "")
if not m:
raise ConductorError(f"sbatch gave no job id: {out[-300:]!r}")
plan.job_id = m.group(1)
return plan
def status_command(proj: str, plan: TrainPlan) -> str:
q = shlex.quote
models = f"{proj}/models/{plan.run}"
log = f"{proj}/logs/sleap-nn-train_{plan.job_id}.log"
curves = "".join(
f"echo CURVE={head}; awk -F, {q(CURVE_AWK)} {q(f'{models}.{head}/training_log.csv')}"
f" 2>/dev/null; echo PATIENCE=$(awk {q(PATIENCE_AWK)} "
f"{q(f'{models}.{head}/training_config.yaml')} 2>/dev/null); "
"echo CURVE_END; "
for head in HEADS
)
return (
f"echo STATE=$(sacct -j {q(plan.job_id)} -n -X -o State 2>/dev/null | head -n1 | xargs); "
f"test -f {q(models + '.done')} && echo DONE; "
f"test -f {q(models + '.failed')} && echo FAILED=$(cat {q(models + '.failed')}); "
f"echo TRAINED=$(grep -c ' trained in ' {q(log)} 2>/dev/null); "
+ curves
+ f"tail -c 4000 {q(log)} 2>/dev/null | tr '\\r' '\\n' | tail -n 20"
)
def _num(text: str) -> float | None:
try:
return float(text)
except ValueError:
return None
def _curve_row(line: str) -> tuple | None:
"""``epoch,train,val,seconds`` → (epoch, train, val, seconds); missing values None."""
parts = (line.split(",") + ["", "", "", ""])[:4]
epoch = _num(parts[0])
if epoch is None:
return None
return (int(epoch), *(_num(x) for x in parts[1:]))
[docs]
def parse_status(text: str) -> dict:
"""State, done/failed flags, loss curves per head, last epoch seen and a log tail.
``curves`` maps a head to rows (epoch, train loss, val loss, seconds);
``patience`` a head to its early-stopping patience (epochs without a better
validation loss); ``trained`` is how many heads have finished training.
"""
st = {
"state": "",
"done": False,
"failed": "",
"epoch": None,
"trained": 0,
"curves": {},
"patience": {},
"tail": [],
}
head = None
for line in text.splitlines():
if head is not None:
if line == "CURVE_END":
head = None
elif line.startswith("PATIENCE="):
n = _num(line[len("PATIENCE=") :].strip())
if n is not None:
st["patience"][head] = int(n)
elif (row := _curve_row(line)) is not None:
st["curves"].setdefault(head, []).append(row)
elif line.startswith("CURVE="):
head = line[len("CURVE=") :].strip()
elif line.startswith("TRAINED="):
st["trained"] = int(_num(line[len("TRAINED=") :].strip() or "0") or 0)
elif line.startswith("STATE="):
st["state"] = line[len("STATE=") :].strip()
elif line == "DONE":
st["done"] = True
elif line.startswith("FAILED="):
st["failed"] = line[len("FAILED=") :].strip() or "failed"
else:
st["tail"].append(line)
for m in EPOCH_RE.finditer(line):
st["epoch"] = int(m.group(1))
if st["state"].startswith(("FAILED", "CANCELLED", "TIMEOUT", "OUT_OF_ME", "NODE_FAIL")):
st["failed"] = st["failed"] or st["state"]
return st
def status(cfg: ConductorConfig, profile: str, plan: TrainPlan, run=subprocess.run) -> dict:
return parse_status(_ssh(cfg, status_command(project_root(cfg, profile), plan), run=run))
def fetch_eval(cfg: ConductorConfig, profile: str, run_id: str, run=subprocess.run) -> list[dict]:
path = f"{project_root(cfg, profile)}/models/{run_id}.eval.json"
return json.loads(_ssh(cfg, f"cat {shlex.quote(path)}", run=run))
# ── judging and registering ───────────────────────────────────────
[docs]
def compare(rows: list[dict], new: str, old: str, metric: str = METRIC) -> dict:
"""Side-by-side numbers per validation set and the promotion verdict."""
by = {(r["model"], r["set"]): r for r in rows}
sets = sorted({r["set"] for r in rows})
table, better, worse = [], 0, 0
for s in sets:
a = (by.get((new, s)) or {}).get(metric)
b = (by.get((old, s)) or {}).get(metric)
table.append(
{"set": s, "new": a, "old": b, "frames": (by.get((new, s)) or {}).get("frames", 0)}
)
if a is None or b is None:
continue
better += a > b
worse += a < b
ok = bool(sets) and better > 0 and worse == 0 and new != old
return {"metric": metric, "table": table, "promotable": ok}
[docs]
def registry_entry(
run_id: str,
base: dict,
lset: LabelSet,
version: int,
verdict: dict,
promote: bool,
today: date | None = None,
excluded: list[str] | None = None,
shared: list[str] | None = None,
) -> dict:
"""Registry entry for a trained model (expects inherited from its base)."""
frames = sum(r["frames"] or 0 for r in verdict["table"])
sets = {r["set"]: r["new"] for r in verdict["table"] if r["new"] is not None}
entry = {
"id": run_id,
"organism": base.get("organism", "drosophila"),
"kind": base.get("kind", "sleap-topdown"),
"run": run_id,
"env": "sleap-nn",
"trained": str(today or date.today()),
"validation": {
"frames": frames,
METRIC: max(sets.values()) if sets else None,
"sets": sets,
},
"grayscale_input": base.get("grayscale_input", True),
"expects": dict(base.get("expects") or {}),
"label_set": {"name": lset.name, "version": version},
"notes": f"sleap-nn fine-tune of {base.get('id')} on {lset.name} v{version}",
}
if excluded:
entry["label_set"]["left_out"] = sorted(excluded)
if shared:
entry["label_set"]["split_by_frame"] = sorted(shared)
entry["label_set"]["test_share"] = LABEL_TEST_SHARE
entry["expects"].pop("fps", None) # trained on several recordings
if promote:
entry["default"] = True
return entry
[docs]
def append_registry(cfg: ConductorConfig, profile: str, entry: dict, run=subprocess.run) -> None:
"""Append ``entry`` to the registry on Aoraki (backup kept next to it)."""
reg = cfg.registry_remote(profile)
text = "\n" + yaml.safe_dump([entry], sort_keys=False)
q = shlex.quote(reg)
remote = (
f"cp {q} {q}.bak_$(date +%Y%m%d%H%M%S) && cat >> {q} <<'RIGCONDUCTOR_EOF'\n"
f"{text}RIGCONDUCTOR_EOF\n"
)
_ssh(cfg, remote, run=run)
# ── models on this PC (for pre-filling) ───────────────────────────
def local_models_dir() -> Path:
from rigconductor.constants import LOCAL_MODELS_DIR
return Path(LOCAL_MODELS_DIR).expanduser()
[docs]
def local_runs(root: Path | None = None) -> list[str]:
"""Model runs with both halves (centroid + centred instance) on this PC."""
root = root or local_models_dir()
if not root.is_dir():
return []
cen = {p.name[: -len(".centroid")] for p in root.glob("*.centroid") if p.is_dir()}
ci = {p.name[: -len(".centered_instance")] for p in root.glob("*.centered_instance")}
return sorted(cen & ci)
def model_dirs(run_id: str, root: Path | None = None) -> list[str]:
root = root or local_models_dir()
return [str(root / f"{run_id}.centroid"), str(root / f"{run_id}.centered_instance")]
[docs]
def fetch_model(cfg: ConductorConfig, profile: str, run_id: str, run=subprocess.run) -> None:
"""Copy a model pair from Aoraki to this PC (only the files inference needs)."""
proj = project_root(cfg, profile)
dest = local_models_dir()
dest.mkdir(parents=True, exist_ok=True)
for half in ("centroid", "centered_instance"):
r = run(
[
"rsync",
"-a",
"--exclude",
"*.slp",
"--exclude",
"viz",
f"{cfg.cluster.ssh_host}:{proj}/models/{run_id}.{half}",
f"{dest}/",
],
capture_output=True,
text=True,
)
if r.returncode != 0:
raise ConductorError(f"could not fetch {run_id}.{half}: {r.stderr.strip()[-300:]}")