# ╔══════════════════════════════════════════════════════════════════╗
# ║ Rig Conductor — labels.worker ║
# ║ « the half that runs inside the SLEAP 1.6 environment » ║
# ╠══════════════════════════════════════════════════════════════════╣
# ║ Started as `<sleap python> -I worker.py <task> <args.json>`; ║
# ║ prints @progress lines and one @result JSON line at the end. ║
# ╚══════════════════════════════════════════════════════════════════╝
"""SLEAP-side tasks of the labelling wizard.
The Conductor never imports sleap-io itself (it lives in its own uv env with
PyTorch). It writes the arguments to a JSON file and runs this script with the
SLEAP env's Python; see :mod:`rigconductor.sleapenv`.
Tasks: ``inspect``, ``seed``, ``select``, ``build``, ``check``, ``harvest``,
``merge``, ``summary``, ``split``. Every frame in a label set is identified by a hash of its pixels,
so provenance survives SLEAP re-saving, merging and splitting.
"""
from __future__ import annotations
import hashlib
import json
import re
import subprocess
import sys
import tempfile
from collections import Counter
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[2])) # rigconductor (pure modules)
import numpy as np # noqa: E402
from rigconductor.labels import rules, select # noqa: E402
PROV_KEY = "rigconductor"
CLIP_RE = re.compile(r"(?P<rec>.+)_arena_(?P<arena>\d+)")
# ── helpers ───────────────────────────────────────────────────────
def emit_progress(frac: float, detail: str = "") -> None:
print(f"@progress label {frac:.3f} {detail}", flush=True)
def emit_result(obj: dict) -> None:
print("@result " + json.dumps(obj), flush=True)
def as_gray(img: np.ndarray) -> np.ndarray:
img = np.asarray(img)
if img.ndim == 3:
img = img[..., 0]
return np.ascontiguousarray(img.astype(np.uint8))
def frame_hash(img: np.ndarray) -> str:
return hashlib.sha1(as_gray(img).tobytes()).hexdigest()[:16]
[docs]
def clip_scale(video) -> str:
"""Camera scale of an arena clip from its width (arena + ThermoKourt padding)."""
from rigconductor.constants import TK_CROP_PADDING
from rigconductor.labels.store import scale_name
return scale_name(video.shape[2] / (1 + 2 * TK_CROP_PADDING))
def provenance(labels) -> dict:
return dict((labels.provenance or {}).get(PROV_KEY, {}).get("frames", {}))
def set_provenance(labels, frames: dict, **extra) -> None:
labels.provenance[PROV_KEY] = {**extra, "frames": frames}
def user_frames(labels) -> list:
return [lf for lf in labels.labeled_frames if lf.user_instances]
def hashes_of(labels, frames: list) -> list[str]:
return [frame_hash(lf.video[lf.frame_idx]) for lf in frames]
[docs]
def stats(labels, prov: dict) -> dict:
"""Counts for the registry: frames, flies, per recording, scale, labeller."""
lfs = user_frames(labels)
hs = hashes_of(labels, lfs)
info = [prov.get(h, {}) for h in hs]
return {
"frames": len(lfs),
"instances": sum(len(lf.user_instances) for lf in lfs),
"recordings": dict(Counter(i.get("recording", "?") for i in info)),
"scales": dict(Counter(i.get("scale", "?") for i in info)),
"labellers": dict(Counter(i.get("labeller", "?") for i in info)),
"frames_index": [
[i.get("recording", "?"), int(i.get("arena", -1)), int(i.get("source_frame", -1))]
for i in info
],
}
[docs]
def keep_user_only(labels, frames: list) -> None:
"""Reduce ``labels`` to ``frames`` with only hand-placed instances."""
for lf in frames:
lf.instances = list(lf.user_instances)
labels.labeled_frames = frames
labels.suggestions = []
labels.clean(frames=True, skeletons=True, tracks=True, videos=True)
def save_embedded(labels, out: str, embed: str = "user") -> None:
import sleap_io as sio
Path(out).parent.mkdir(parents=True, exist_ok=True)
sio.save_slp(labels, out, embed=embed, verbose=False)
# ── tasks ─────────────────────────────────────────────────────────
[docs]
def frame_states(labels) -> dict:
"""Frames to label (suggestions and labelled frames) by state.
``labelled``: hand-placed instances; ``predicted``: only model predictions;
``empty``: nothing yet.
"""
vid = {id(v): i for i, v in enumerate(labels.videos)}
keys = {(vid[id(s.video)], s.frame_idx) for s in labels.suggestions}
by = {(vid[id(lf.video)], lf.frame_idx): lf for lf in labels.labeled_frames}
keys |= set(by)
user = sum(1 for k in keys if k in by and by[k].user_instances)
pred = sum(
1 for k in keys if k in by and not by[k].user_instances and by[k].predicted_instances
)
return {
"total": len(keys),
"labelled": user,
"predicted": pred,
"empty": len(keys) - user - pred,
}
def task_inspect(a: dict) -> dict:
import sleap_io as sio
labels = sio.load_slp(a["slp"])
lfs = labels.labeled_frames
return {
"states": frame_states(labels),
"frames": len(lfs),
"user_frames": len(user_frames(labels)),
"user_instances": sum(len(lf.user_instances) for lf in lfs),
"predicted_instances": sum(len(lf.predicted_instances) for lf in lfs),
"videos": len(labels.videos),
"nodes": labels.skeletons[0].node_names if labels.skeletons else [],
"provenance_frames": len(provenance(labels)),
}
[docs]
def clip_info(lf, scale: str | None, labeller: str, source: str) -> dict:
"""Provenance of a frame labelled straight on an arena clip (not a package)."""
stem = Path(lf.video.filename).stem
m = CLIP_RE.match(stem)
return {
"recording": m["rec"] if m else stem,
"arena": int(m["arena"]) if m else -1,
"source_frame": int(lf.frame_idx),
"scale": scale or clip_scale(lf.video),
"labeller": labeller,
"package": source,
"reason": source,
}
[docs]
def task_seed(a: dict) -> dict:
"""First version of a set from an existing SLEAP project (hand labels only)."""
import sleap_io as sio
labels = sio.load_slp(a["src"])
lfs = user_frames(labels)
prov = {}
for k, lf in enumerate(lfs):
prov[frame_hash(lf.video[lf.frame_idx])] = clip_info(
lf, a.get("scale"), a.get("labeller", "seed"), "seed"
)
emit_progress((k + 1) / max(len(lfs), 1), "reading frames")
keep_user_only(labels, lfs)
labels.skeletons = labels.skeletons[:1] if labels.skeletons else labels.skeletons
sk = labels.skeletons[0]
for lf in lfs:
for inst in lf.instances:
inst.skeleton = sk
set_provenance(labels, prov, set=a.get("set", ""), version=1)
save_embedded(labels, a["out"])
return stats(sio.load_slp(a["out"]), prov)
def _pose_arrays(path: str):
import h5py
with h5py.File(path, "r") as h:
nodes = [n.decode() if isinstance(n, bytes) else str(n) for n in h["node_names"][:]]
return h["tracks"][:], h["point_scores"][:], h["track_occupancy"][:], nodes
[docs]
def task_select(a: dict) -> dict:
"""Pick frames per arena clip."""
import sleap_io as sio
exclude = {(r, int(ar), int(f)) for r, ar, f in a.get("exclude", [])}
out = []
clips = a["clips"]
for k, c in enumerate(clips):
fps = float(c.get("fps") or 25.0)
gap = max(1, int(round(a.get("min_gap_s", 2.0) * fps)))
banned = {f for r, ar, f in exclude if r == c["recording"] and ar == int(c["arena"])}
unsure = contact = None
n_frames = None
if c.get("pose") and Path(c["pose"]).is_file():
tracks, scores, occ, nodes = _pose_arrays(c["pose"])
n_frames = tracks.shape[-1]
unsure = select.unsure_scores(scores, occ, int(a.get("n_expected", 3)))
contact = select.contact_scores(tracks, nodes)
if n_frames is None:
n_frames = int(sio.load_video(c["clip"]).shape[0])
picks = select.pick_frames(
n_frames,
int(a["n"]),
unsure,
contact,
a.get("mix"),
gap,
banned,
seed=int(a.get("seed", 0)) + k,
)
for p in picks:
out.append(
{
**{k2: c[k2] for k2 in ("recording", "arena", "clip", "scale")},
"frame": p.frame,
"reason": p.reason,
"score": round(p.score, 3),
}
)
emit_progress((k + 1) / len(clips), f"{c['recording']} arena {int(c['arena']):02d}")
return {"frames": out}
def _write_images(group: list[dict], invert: bool, tmp: Path, tag: str):
"""Grab the frames of one clip, return PNG paths and their hashes."""
import imageio.v3 as iio
import sleap_io as sio
video = sio.load_video(group[0]["clip"])
paths, hashes = [], []
for k, f in enumerate(group):
img = as_gray(video[int(f["frame"])])
if invert:
img = 255 - img
p = tmp / f"{tag}_{k:04d}.png"
iio.imwrite(p, img)
paths.append(str(p))
hashes.append(frame_hash(img))
video.close()
return paths, hashes
def _prefill(raw: str, models: list[str], n_animals: int, tmp: Path):
import sleap_io as sio
nn = Path(sys.executable).with_name("sleap-nn")
pred = tmp / "pred.slp"
cmd = [
str(nn),
"track",
"-i",
raw,
"--only_suggested_frames",
"-n",
str(n_animals),
"-o",
str(pred),
]
for m in models:
cmd += ["-m", m]
r = subprocess.run(cmd, capture_output=True, text=True)
if r.returncode != 0 or not pred.is_file():
raise RuntimeError(f"pre-fill failed: {(r.stderr or r.stdout)[-800:]}")
return sio.load_slp(str(pred))
[docs]
def task_build(a: dict) -> dict:
"""Write a labelling package: chosen frames embedded, optional pre-fill."""
import sleap_io as sio
skel = sio.load_slp(a["skeleton_from"]).skeletons[0]
invert = a.get("polarity", "inverted") == "inverted"
groups: dict[tuple, list[dict]] = {}
for f in a["frames"]:
groups.setdefault((f["recording"], int(f["arena"]), f["clip"]), []).append(f)
tmp = Path(tempfile.mkdtemp(prefix="rc_pkg_"))
videos, suggestions, prov = [], [], {}
for k, (key, group) in enumerate(sorted(groups.items())):
group = sorted(group, key=lambda f: int(f["frame"]))
paths, hashes = _write_images(group, invert, tmp, f"v{k:03d}")
v = sio.Video.from_filename(paths)
videos.append(v)
for i, (f, h) in enumerate(zip(group, hashes, strict=True)):
suggestions.append(sio.SuggestionFrame(video=v, frame_idx=i))
prov[h] = {
"recording": f["recording"],
"arena": int(f["arena"]),
"source_frame": int(f["frame"]),
"scale": f.get("scale", "?"),
"reason": f.get("reason", ""),
"package": a["package"],
}
emit_progress(0.6 * (k + 1) / len(groups), f"frames of {key[0]} arena {key[1]:02d}")
labels = sio.Labels(skeletons=[skel], videos=videos)
labels.suggestions = suggestions
set_provenance(labels, prov, set=a.get("set", ""), package=a["package"])
n_pred = 0
if a.get("prefill"):
raw = str(tmp / "raw.pkg.slp")
save_embedded(labels, raw, embed="suggestions")
emit_progress(0.7, "pre-filling with the model")
pred = _prefill(raw, a["prefill"], int(a.get("n_animals", 3)), tmp)
for lf in pred.labeled_frames:
vi = pred.videos.index(lf.video)
for inst in lf.instances:
inst.skeleton = skel
labels.append(
sio.LabeledFrame(
video=videos[vi], frame_idx=lf.frame_idx, instances=list(lf.instances)
)
)
n_pred += len(lf.instances)
emit_progress(0.9, "writing the package")
save_embedded(labels, a["out"], embed="suggestions")
return {"frames": len(suggestions), "videos": len(videos), "predicted": n_pred, "hashes": prov}
def _returned_frames(labels, pkg_frames: dict):
"""Labelled frames of a returned package, each with its hash and package info."""
out = []
for lf in labels.labeled_frames:
h = frame_hash(lf.video[lf.frame_idx])
out.append((lf, h, pkg_frames.get(h)))
return out
[docs]
def task_check(a: dict) -> dict:
"""Judge every frame of a returned package; nothing is merged here."""
import sleap_io as sio
labels = sio.load_slp(a["returned"])
nodes = labels.skeletons[0].node_names if labels.skeletons else []
problem = rules.skeleton_problem(nodes, a["nodes"])
if problem:
return {"skeleton": problem, "frames": [], "summary": {}}
return _judge(_returned_frames(labels, a["package_frames"]), nodes, a)
def _live_instances(lf) -> list:
"""Instances on a frame, minus predictions a labeller already corrected.
Double-clicking a prediction in SLEAP makes a labelled copy and keeps the
prediction hidden underneath; that hidden one is not a half-corrected fly.
"""
replaced = {id(i.from_predicted) for i in lf.user_instances if i.from_predicted is not None}
return [i for i in lf.instances if id(i) not in replaced]
def _judge(items: list, nodes: list[str], a: dict) -> dict:
"""Rule check of (frame, hash, info) items; the report the Back page shows."""
rows = []
for k, (lf, h, info) in enumerate(items):
if info is None:
v = rules.Verdict("rejected", ["frame is not part of this package"])
else:
img = lf.video[lf.frame_idx]
insts = [
rules.Inst(i.numpy(), user=type(i).__name__ == "Instance")
for i in _live_instances(lf)
]
v = rules.check_frame(
insts,
nodes,
img.shape[:2],
tuple(a.get("allowed_counts", [3])),
a.get("n_females", 1),
)
rows.append(
{
"hash": h,
**(info or {}),
"status": v.status,
"reasons": v.reasons,
"warnings": v.warnings,
"flies": len(lf.user_instances),
}
)
emit_progress((k + 1) / max(len(items), 1), "checking frames")
summary = {s: sum(1 for r in rows if r["status"] == s) for s in ("accepted", "rejected")}
summary["warnings"] = sum(1 for r in rows if r["warnings"] and r["status"] == "accepted")
summary["unlabelled"] = len(a["package_frames"]) - summary["accepted"] - summary["rejected"]
return {"skeleton": None, "frames": rows, "summary": summary}
[docs]
def task_harvest(a: dict) -> dict:
"""Frames hand-labelled in a SLEAP project since they were last taken into the set.
Frames already in the set (by pixel hash) are skipped; the rest get
provenance from their clip names and the usual rule check.
"""
import sleap_io as sio
labels = sio.load_slp(a["slp"])
nodes = labels.skeletons[0].node_names if labels.skeletons else []
problem = rules.skeleton_problem(nodes, a["nodes"])
if problem:
return {"skeleton": problem, "frames": [], "summary": {}, "package_frames": {}}
known = set(provenance(sio.load_slp(a["base"]))) if a.get("base") else set()
items, frames = [], {}
lfs = user_frames(labels)
for k, lf in enumerate(lfs):
h = frame_hash(lf.video[lf.frame_idx])
if h not in known:
frames[h] = clip_info(lf, a.get("scale"), "?", a.get("source", "project"))
items.append((lf, h, frames[h]))
emit_progress(0.5 * (k + 1) / max(len(lfs), 1), "finding new frames")
rep = _judge(items, nodes, {**a, "package_frames": frames})
rep["summary"]["unlabelled"] = 0
rep["summary"]["already_in_set"] = len(lfs) - len(items)
return {**rep, "package_frames": frames}
[docs]
def task_merge(a: dict) -> dict:
"""New version = base set + accepted frames of a returned package."""
import sleap_io as sio
ret = sio.load_slp(a["returned"])
accepted = set(a["accepted"])
items = [
(lf, h, info)
for lf, h, info in _returned_frames(ret, a["package_frames"])
if h in accepted and info is not None
]
new_prov = {
h: {**info, "labeller": a.get("labeller", "?"), "labelled": a.get("date", "")}
for _, h, info in items
}
keep_user_only(ret, [lf for lf, _, _ in items])
if a.get("base"):
base = sio.load_slp(a["base"])
prov = provenance(base)
keep_user_only(base, user_frames(base))
for lf in ret.labeled_frames: # appended, not merged: SLEAP would match videos
for inst in lf.instances: # by source clip and mix up frame numbers
inst.skeleton = base.skeletons[0]
inst.track = None
base.append(lf)
labels = base
else:
labels, prov = ret, {}
labels.skeletons = labels.skeletons[:1]
prov.update(new_prov)
set_provenance(labels, prov, set=a.get("set", ""), version=a.get("version"))
save_embedded(labels, a["out"])
out = sio.load_slp(a["out"])
return {**stats(out, prov), "added": len(items)}
def _example_image(img: np.ndarray, instances: list) -> np.ndarray:
"""Frame as RGB with every labelled point as a coloured dot (one colour per fly)."""
from rigconductor.constants import WONG
colours = [WONG[k] for k in ("orange", "sky_blue", "bluish_green", "yellow", "vermillion")]
rgb = np.repeat(img[..., None], 3, axis=2)
h, w = img.shape
for k, pts in enumerate(instances):
for x, y in pts:
if not (np.isfinite(x) and np.isfinite(y)):
continue
xi, yi = int(round(x)), int(round(y))
rgb[max(0, yi - 3) : min(h, yi + 4), max(0, xi - 3) : min(w, xi + 4)] = colours[
k % len(colours)
]
return rgb
def _new_summary(name: str) -> dict:
return {
"recording": name,
"frames": 0,
"flies": Counter(),
"sizes": Counter(),
"scales": Counter(),
"arenas": set(),
"labellers": Counter(),
}
[docs]
def task_summary(a: dict) -> dict:
"""Per recording: frames, flies per frame, image sizes and one example frame."""
import imageio.v3 as iio
import sleap_io as sio
labels = sio.load_slp(a["slp"])
prov = provenance(labels)
lfs = user_frames(labels)
out_dir = Path(a["out_dir"])
out_dir.mkdir(parents=True, exist_ok=True)
recs: dict[str, dict] = {}
best: dict[str, tuple] = {}
for k, lf in enumerate(lfs):
img = as_gray(lf.video[lf.frame_idx])
info = prov.get(frame_hash(img), {})
name = info.get("recording", "?")
r = recs.setdefault(name, _new_summary(name))
n = len(lf.user_instances)
r["frames"] += 1
r["flies"][n] += 1
r["sizes"][f"{img.shape[1]}×{img.shape[0]}"] += 1
r["scales"][info.get("scale", "?")] += 1
r["arenas"].add(int(info.get("arena", -1)))
r["labellers"][info.get("labeller", "?")] += 1
if name not in best or n > best[name][0]:
best[name] = (n, img, [i.numpy() for i in lf.user_instances], info)
emit_progress((k + 1) / max(len(lfs), 1), "reading frames")
rows = []
for i, (name, r) in enumerate(sorted(recs.items())):
_, img, pts, info = best[name]
example = out_dir / f"example_{i:03d}.png"
iio.imwrite(example, _example_image(img, pts))
rows.append(
{
"recording": name,
"frames": r["frames"],
"instances": sum(n * c for n, c in r["flies"].items()),
"flies": {str(n): c for n, c in sorted(r["flies"].items())},
"sizes": dict(r["sizes"]),
"scale": r["scales"].most_common(1)[0][0],
"arenas": sorted(x for x in r["arenas"] if x >= 0),
"labellers": dict(r["labellers"]),
"example": str(example),
"example_from": [int(info.get("arena", -1)), int(info.get("source_frame", -1))],
}
)
return {"recordings": rows}
[docs]
def hash_share(h: str) -> float:
"""A frame's fixed place in [0, 1) from its pixel hash (same in every version)."""
return int(h[:8], 16) / 16**8
[docs]
def task_split(a: dict) -> dict:
"""Train file + one validation (test) file per scale.
Validation recordings are held out whole. Recordings in ``shared``
(recording → scale) are split by frame: a ``test_share`` of their frames,
chosen by pixel hash so a frame stays on its side in later versions, joins
the validation file of their scale; the rest is trained on.
"""
import sleap_io as sio
src = a["slp"]
labels = sio.load_slp(src)
prov = provenance(labels)
lfs = user_frames(labels)
hs = hashes_of(labels, lfs)
rec_of = [prov.get(h, {}).get("recording", "?") for h in hs]
val_sets = {name: set(recs) for name, recs in (a.get("val") or {}).items()}
held = set().union(*val_sets.values()) if val_sets else set()
left_out = set(a.get("exclude") or []) - held
shared = {r: s for r, s in (a.get("shared") or {}).items() if r not in held | left_out}
share = float(a.get("test_share", 0.2))
tested = {
i: shared[r] for i, r in enumerate(rec_of) if r in shared and hash_share(hs[i]) < share
}
parts = {
"train": [i for i, r in enumerate(rec_of) if r not in held | left_out and i not in tested]
}
for name in sorted(set(val_sets) | set(tested.values())):
recs = val_sets.get(name, set())
parts[f"val_{name}"] = [
i for i, r in enumerate(rec_of) if r in recs or tested.get(i) == name
]
counts = {}
for part, idx in parts.items():
counts[part] = len(idx)
if not idx:
continue
sub = sio.load_slp(src)
sub_lfs = user_frames(sub)
keep_user_only(sub, [sub_lfs[i] for i in idx])
save_embedded(sub, str(Path(a["out_dir"]) / f"{part}.pkg.slp"))
return {
"counts": counts,
"left_out": sum(r in left_out for r in rec_of),
"shared_test": len(tested),
}
TASKS = {
"inspect": task_inspect,
"seed": task_seed,
"select": task_select,
"build": task_build,
"check": task_check,
"harvest": task_harvest,
"merge": task_merge,
"summary": task_summary,
"split": task_split,
}
def main(argv: list[str] | None = None) -> int:
argv = sys.argv[1:] if argv is None else argv
if len(argv) != 2 or argv[0] not in TASKS:
print(f"usage: worker.py {{{','.join(TASKS)}}} ARGS.json", file=sys.stderr)
return 2
args = json.loads(Path(argv[1]).read_text(encoding="utf-8"))
emit_result(TASKS[argv[0]](args))
return 0
if __name__ == "__main__":
sys.exit(main())