Source code for rigconductor.compat

# ╔══════════════════════════════════════════════════════════════════╗
# ║  Rig Conductor — compat                                          ║
# ║  « does this AI model fit these videos? »                        ║
# ╠══════════════════════════════════════════════════════════════════╣
# ║  Model-vs-clips and model-vs-camera checks against the           ║
# ║  registry's 'expects' block, before recording and submit.        ║
# ╚══════════════════════════════════════════════════════════════════╝
"""Model-vs-data check: does the chosen AI model expect the kind of video we are sending?

Run before every cluster submission. Each registry entry may carry an ``expects`` block
describing the data the model was trained on::

    expects:
      channels: 1            # 1 = grayscale, 3 = colour
      polarity: bright       # animals bright on dark ("bright") or dark on light ("dark")
      invert: true           # the pipeline inverts our clips before inference
      min_side_px: 224       # clips smaller than this cannot hold one instance crop
      fps: [25, 30]          # frame rates the model was trained at (tolerance 20 %)
      input_size: 1024       # model resizes every frame to this (resize_and_pad_to_target);
                             # clips more than 10 % off change the animals' apparent size

Missing keys are not checked. Problems found on 2026-11-05/11 that motivated this:
flies13 was trained on bright flies on dark (ours are dark on white, so every
prediction failed), and fine-tuning crashed because our frames were 3-channel
and the base weights 1-channel. Then the fine-tune learned nothing because the
inherited config resized our 512 px clips to flies13's 1024 px, doubling the flies.
"""

from __future__ import annotations

import json
import subprocess
from dataclasses import dataclass
from pathlib import Path

from rigconductor.doctor import Check
from rigconductor.models import Model

# Pose models see single frames, so a frame-rate mismatch only warns for them;
# behaviour classifiers learn durations and speeds, so it blocks.
POSE_KINDS = {"sleap-topdown", "sleap-bottomup", "dlc"}

GRAY_FORMATS = {"gray", "gray8", "gray16le", "gray10le", "gray12le", "y8"}


[docs] @dataclass class ClipInfo: path: Path width: int height: int fps: float pix_fmt: str mean_intensity: float | None # 0..255 of the first decoded frame @property def channels_stored(self) -> int: return 1 if self.pix_fmt in GRAY_FORMATS else 3 @property def polarity(self) -> str | None: """'dark' animals on a light background when the frame is mostly bright.""" if self.mean_intensity is None: return None return "dark" if self.mean_intensity > 127 else "bright"
def probe_clip(path: Path, run=subprocess.run) -> ClipInfo: r = run( [ "ffprobe", "-v", "error", "-select_streams", "v:0", "-of", "json", "-show_entries", "stream=width,height,r_frame_rate,pix_fmt", str(path), ], capture_output=True, text=True, ) s = json.loads(r.stdout or "{}").get("streams", [{}])[0] num, _, den = str(s.get("r_frame_rate", "0/1")).partition("/") fps = float(num) / float(den or 1) if float(den or 1) else 0.0 g = run( [ "ffmpeg", "-v", "error", "-i", str(path), "-frames:v", "1", "-f", "rawvideo", "-pix_fmt", "gray", "-", ], capture_output=True, ) data = g.stdout if isinstance(g.stdout, bytes | bytearray) else b"" mean = sum(data) / len(data) if data else None return ClipInfo( Path(path), int(s.get("width", 0)), int(s.get("height", 0)), fps, str(s.get("pix_fmt", "")), mean, )
[docs] def check_model_vs_clips( model: Model, clips: list[ClipInfo], invert: bool | None = None ) -> list[Check]: """Compare the model's ``expects`` block with the probed clips. ``invert`` is what the pipeline will do to the clips before inference (``None`` = take the model's ``expects.invert``).""" exp = model.expects or {} out: list[Check] = [] if not clips: return [Check("Model vs data", False, "no clips to check", "Run the cut step first")] if not exp: return [ Check( "Model vs data", False, f"model {model.id} has no 'expects' block", "Add the training-data description to the registry entry", ) ] name = f"{model.id} vs clips" pol = {c.polarity for c in clips if c.polarity} if "polarity" in exp and pol: inv = exp.get("invert", False) if invert is None else invert seen = {("bright" if p == "dark" else "dark") if inv else p for p in pol} ok = seen == {exp["polarity"]} out.append( Check( f"{name}: polarity", ok, f"model wants {exp['polarity']} animals, clips give {sorted(seen)}" f"{' after inversion' if inv else ''}", "" if ok else "Toggle inversion for this model or pick a model trained on this lighting", ) ) if "channels" in exp: chans = {c.channels_stored for c in clips} ok = chans == {exp["channels"]} or (exp["channels"] == 1 and model.grayscale_input) conv = " (model converts to gray)" if ok and chans != {1} else "" out.append( Check( f"{name}: channels", ok, f"model trained on {exp['channels']}-channel input, clips stored as " f"{sorted(chans)}-channel{conv}", "" if ok else "Set ensure_grayscale in the model's config, or re-encode the clips as gray", ) ) if "min_side_px" in exp: small = [c.path.name for c in clips if min(c.width, c.height) < exp["min_side_px"]] out.append( Check( f"{name}: frame size", not small, f"all clips ≥ {exp['min_side_px']} px" if not small else f"{len(small)} clips smaller than {exp['min_side_px']} px", "" if not small else "Increase the arena crop padding", ) ) if "input_size" in exp: want = float(exp["input_size"]) off = sorted( {max(c.width, c.height) for c in clips if abs(max(c.width, c.height) / want - 1) > 0.1} ) out.append( Check( f"{name}: input size", not off, f"model resizes frames to {want:.0f} px; clips are " f"{sorted({max(c.width, c.height) for c in clips})} px", "" if not off else "Animals would be rescaled before inference: retrain with this clip size " "or set the model's target size to the clips", ) ) if "fps" in exp: lo, hi = (exp["fps"], exp["fps"]) if isinstance(exp["fps"], int | float) else exp["fps"] bad = sorted({round(c.fps) for c in clips if not lo * 0.8 <= c.fps <= hi * 1.2}) rates = sorted({round(c.fps) for c in clips}) out.append( Check( f"{name}: frame rate", not bad, f"model trained at {lo}–{hi} fps; clips at {rates} fps", "" if not bad else "Pose models look at single frames, so this is usually fine; scores " "use the real frame rate. Retrain with these clips if poses get worse" if model.kind in POSE_KINDS else "Record at the model's frame rate, or retrain with " "frames at this rate (speeds and bout durations depend on it)", blocking=model.kind not in POSE_KINDS, ) ) return out
# ── before recording: model vs camera + preset ──────────────────────────
[docs] @dataclass class CameraInfo: pixel_format: str | None # e.g. "Mono8", "BayerRG8", "RGB8" fps: float | None # AcquisitionFrameRate model_name: str | None = None @property def colour(self) -> bool | None: if not self.pixel_format: return None return not self.pixel_format.lower().startswith("mono")
[docs] def camera_info(client) -> CameraInfo: """Read what matters from a MotifClient (missing settings → None).""" def get(name): try: return client.read(name) except Exception: # noqa: BLE001 return None fps = get("AcquisitionFrameRate") return CameraInfo(get("PixelFormat"), float(fps) if fps else None, get("DeviceModelName"))
CHANNELS = "channels" CAMERA_FPS = "camera fps"
[docs] def check_model_vs_camera(model: Model, cam: CameraInfo, preset) -> list[Check]: """Checks to run on the first page, before anything is recorded. ``Check.name`` ends in ``CHANNELS`` or ``CAMERA_FPS`` for the two problems the GUI offers a fix for (convert to grayscale / set the camera frame rate).""" out: list[Check] = [] exp = model.expects or {} name = f"{model.id} vs camera" if cam.colour is not None and "channels" in exp: gray_out = (not cam.colour) or preset.gray if exp["channels"] == 1: ok = gray_out or model.grayscale_input detail = ( "camera is mono" if not cam.colour else "colour camera, preset converts to grayscale" if preset.gray else "colour camera, model converts to gray itself" if ok else "colour camera, but this model reads grayscale only" ) hint = "" if ok else "Convert the videos to grayscale, or pick a colour model" else: ok = not gray_out detail = f"model wants {exp['channels']}-channel colour, " + ( "camera is mono" if not cam.colour else "preset converts to grayscale" ) if ok: detail = "colour camera, colour model" hint = ( "" if ok else "Pick a grayscale model" + ("" if not cam.colour else " or switch the preset to colour") ) out.append(Check(f"{name}: {CHANNELS}", ok, detail, hint)) if cam.fps and preset.fps: ok = abs(cam.fps - preset.fps) <= 0.5 out.append( Check( f"{name}: {CAMERA_FPS}", ok, f"camera runs at {cam.fps:g} fps, preset '{preset.key}' needs {preset.fps:g} fps", "" if ok else f"Set the camera to {preset.fps:g} fps. Otherwise the clips play " f"{cam.fps / preset.fps:.2f}× too slow and every speed and duration is wrong", ) ) if cam.fps and "fps" in exp: lo, hi = (exp["fps"], exp["fps"]) if isinstance(exp["fps"], int | float) else exp["fps"] want = preset.fps or cam.fps ok = lo * 0.8 <= want <= hi * 1.2 out.append( Check( f"{name}: model fps", ok, f"model trained at {exp['fps']} fps, recording at {want:g} fps", "" if ok else "Pose models look at single frames, so this is usually fine; motion " "blur and how far animals move between frames differ from the training data" if model.kind in POSE_KINDS else "Pick a model trained at this frame rate", blocking=model.kind not in POSE_KINDS, ) ) return out