# ╔══════════════════════════════════════════════════════════════════╗
# ║ Rig Conductor — cluster_status ║
# ║ « what every task of a SLURM array job is doing » ║
# ╠══════════════════════════════════════════════════════════════════╣
# ║ One sacct call per refresh. Qt-free, so the GUI monitor and tests ║
# ║ share the parsing. ║
# ╚══════════════════════════════════════════════════════════════════╝
"""SLURM array job status from ``sacct``."""
from __future__ import annotations
import re
import shlex
import subprocess
from collections import Counter
from dataclasses import dataclass
from rigconductor.constants import SSH_TIMEOUT_S, STALE_AFTER_S
SACCT_FIELDS = "JobID,State,NodeList,Elapsed,ExitCode"
FAILED_STATES = {
"FAILED",
"TIMEOUT",
"OUT_OF_MEMORY",
"NODE_FAIL",
"CANCELLED",
"BOOT_FAIL",
"DEADLINE",
"PREEMPTED",
"REVOKED",
}
_TASK_RE = re.compile(r"^(\d+)_(\d+|\[[^\]]+\])$")
[docs]
@dataclass
class TaskState:
"""One array task.
Attributes:
task: Array index (= arena / unit index for Conductor jobs).
state: SLURM state, e.g. ``RUNNING``, ``CANCELLED by 123`` → ``CANCELLED``.
node: Node it runs or ran on ('' while pending).
elapsed: ``HH:MM:SS`` as sacct prints it.
exit_code: ``0:0`` etc.
"""
task: int
state: str
node: str = ""
elapsed: str = ""
exit_code: str = ""
@property
def category(self) -> str:
"""pending · running · done · failed."""
if self.state in ("PENDING", "REQUEUED", "SUSPENDED", "CONFIGURING"):
return "pending"
if self.state in ("RUNNING", "COMPLETING"):
return "running"
if self.state == "COMPLETED":
return "done"
return "failed" if self.state in FAILED_STATES else "pending"
[docs]
def expand_array_spec(spec: str) -> list[int]:
"""``[3-5,8%2]`` → ``[3, 4, 5, 8]``."""
out: list[int] = []
for part in spec.strip("[]").split("%")[0].split(","):
if "-" in part:
a, b = part.split("-")
out.extend(range(int(a), int(b) + 1))
elif part:
out.append(int(part))
return out
[docs]
def parse_sacct(text: str, job_id: str) -> list[TaskState]:
"""Parse ``sacct -X -n -P -o SACCT_FIELDS`` output for one array job."""
tasks: dict[int, TaskState] = {}
for line in text.splitlines():
cols = line.split("|")
if len(cols) < 5:
continue
jid, state, node, elapsed, code = cols[:5]
m = _TASK_RE.match(jid)
if not m or m.group(1) != str(job_id):
continue
state = state.split()[0]
node = "" if node in ("None assigned", "(null)") else node
idx = m.group(2)
for t in expand_array_spec(idx) if idx.startswith("[") else [int(idx)]:
tasks[t] = TaskState(t, state, node, elapsed, code)
return [tasks[k] for k in sorted(tasks)]
[docs]
def summarise(tasks: list[TaskState]) -> dict:
"""Counts per category and running tasks per node."""
counts = Counter(t.category for t in tasks)
nodes = Counter(t.node for t in tasks if t.category == "running" and t.node)
return {
"pending": counts["pending"],
"running": counts["running"],
"done": counts["done"],
"failed": counts["failed"],
"total": len(tasks),
"nodes": dict(sorted(nodes.items())),
}
[docs]
def fetch_tasks(
ssh_host: str, job_id: str, ssh_cmd: str = "ssh -o BatchMode=yes", run=subprocess.run
) -> list[TaskState]:
"""Ask the cluster for every task of ``job_id`` (one ssh round trip)."""
remote = f"sacct -j {shlex.quote(str(job_id))} -X -n -P -o {SACCT_FIELDS}"
r = run(
[*shlex.split(ssh_cmd), ssh_host, remote],
capture_output=True,
text=True,
timeout=SSH_TIMEOUT_S,
)
if r.returncode != 0:
raise RuntimeError(r.stderr.strip()[-200:] or f"ssh exit {r.returncode}")
return parse_sacct(r.stdout, str(job_id))
# ── pull-queue jobs (sleap_submit.sh / sleap_worker.sh) ─────────────────
QUEUE_MARK = "@@QUEUE"
REASON_MARK = "@@REASON"
STALE = "stale" # TaskState.exit_code prefix for a claim without heartbeat
WORKER_FIELDS = "JobID,State,NodeList,Elapsed,ExitCode,Partition"
[docs]
@dataclass
class WorkerState:
"""One GPU worker job.
Attributes:
job_id: SLURM job id.
state: SLURM state.
partition: GPU partition it was sent to.
node: Node it runs or ran on.
elapsed: ``HH:MM:SS``.
reason: Why a pending job waits (squeue ``%r``, e.g. ``Resources``).
"""
job_id: str
state: str
partition: str = ""
node: str = ""
elapsed: str = ""
reason: str = ""
@property
def category(self) -> str:
return TaskState(0, self.state).category
@property
def card(self) -> str:
"""``aoraki_gpu_L4_24GB`` → ``L4``."""
return self.partition.removeprefix("aoraki_gpu_").split("_")[0] or self.partition
[docs]
def queue_listing_command(pose_dir: str) -> str:
"""Shell snippet that lists a trial's outputs and queue state in one go."""
d = shlex.quote(pose_dir)
return (
f"cd {d} 2>/dev/null && {{ ls -1; ls -1 _queue 2>/dev/null | sed 's|^|_queue/|'; "
'for c in _queue/claim_*; do [ -e "$c/owner" ] && '
'echo "OWNER $c $(cat "$c/owner") $(stat -c %Y "$c")"; done; '
'echo "NOW $(date +%s)"; }'
)
def parse_workers(text: str, job_ids: list[str]) -> list[WorkerState]:
out = []
for line in text.splitlines():
cols = line.split("|")
if len(cols) >= 6 and cols[0] in job_ids:
node = "" if cols[2] in ("None assigned", "(null)") else cols[2]
out.append(WorkerState(cols[0], cols[1].split()[0], cols[5], node, cols[3]))
return sorted(out, key=lambda w: job_ids.index(w.job_id))
def _hms(seconds: int) -> str:
h, rest = divmod(max(0, seconds), 3600)
return f"{h:02d}:{rest // 60:02d}:{rest % 60:02d}"
[docs]
def parse_queue(
text: str,
trial: str,
indices: list[int],
workers_done: bool,
stale_after_s: float = STALE_AFTER_S,
) -> list[TaskState]:
"""Per-arena state from :func:`queue_listing_command` output.
``workers_done``: no worker is pending or running any more, so a clip that is
neither finished nor parked will not be picked up again (shown as failed).
A claim whose heartbeat is older than ``stale_after_s`` is reported with
``exit_code`` "stale …": its worker died or hangs; another worker (or a rerun)
takes it over.
"""
names, owners, now = set(), {}, None
for line in text.splitlines():
if line.startswith("OWNER "):
parts = line.split()
if len(parts) >= 5:
beat = int(parts[5]) if len(parts) >= 6 else int(parts[4])
owners[parts[1].split("_")[-1]] = (parts[3], int(parts[4]), beat)
elif line.startswith("NOW "):
now = int(line.split()[1])
elif line:
names.add(line.strip())
out = []
for i in indices:
nn = f"{i:02d}"
stem = f"{trial}_arena_{nn}"
if f"{stem}.h5" in names:
out.append(TaskState(i, "COMPLETED"))
elif f"{stem}.empty" in names:
out.append(TaskState(i, "COMPLETED", exit_code="no animals"))
elif f"_queue/parked_{nn}" in names:
out.append(TaskState(i, "FAILED", exit_code="gave up after 3 tries"))
elif f"_queue/claim_{nn}" in names:
node, start, beat = owners.get(nn, ("", now or 0, now or 0))
quiet = (now or beat) - beat
note = f"{STALE}: no heartbeat for {quiet // 60} min" if quiet > stale_after_s else ""
out.append(TaskState(i, "RUNNING", node, _hms((now or start) - start), note))
elif workers_done:
out.append(TaskState(i, "FAILED", exit_code="not tracked"))
else:
retry = f"_queue/strikes_{nn}" in names
out.append(TaskState(i, "PENDING", exit_code="retrying" if retry else ""))
return out
[docs]
@dataclass
class JobStatus:
workers: list[WorkerState]
arenas: list[TaskState]
[docs]
def fetch_queue_status(
ssh_host: str,
job_ids: str,
pose_dir: str,
trial: str,
indices: list[int],
ssh_cmd: str = "ssh -o BatchMode=yes",
run=subprocess.run,
) -> JobStatus:
"""Workers (sacct) and arenas (queue files) in one ssh round trip."""
ids = [j for j in str(job_ids).split(",") if j]
jobs = shlex.quote(",".join(ids))
remote = (
f"sacct -j {jobs} -X -n -P -o {WORKER_FIELDS}; "
f"echo {REASON_MARK}; squeue -j {jobs} -h -t PENDING -o '%i|%r' 2>/dev/null; "
f"echo {QUEUE_MARK}; {queue_listing_command(pose_dir)}"
)
r = run(
[*shlex.split(ssh_cmd), ssh_host, remote],
capture_output=True,
text=True,
timeout=SSH_TIMEOUT_S,
)
if r.returncode != 0 and QUEUE_MARK not in r.stdout:
raise RuntimeError(r.stderr.strip()[-200:] or f"ssh exit {r.returncode}")
head, _, queue_part = r.stdout.partition(QUEUE_MARK)
sacct_part, _, reason_part = head.partition(REASON_MARK)
workers = parse_workers(sacct_part, ids)
reasons = dict(x.split("|", 1) for x in reason_part.split() if "|" in x)
for w in workers:
w.reason = reasons.get(w.job_id, "")
done = bool(workers) and all(w.category in ("done", "failed") for w in workers)
return JobStatus(workers, parse_queue(queue_part, trial, indices, done))
# ── actions ───────────────────────────────────────────────────────────
def _ssh(ssh_host: str, ssh_cmd: str, remote: str, run=subprocess.run) -> str:
r = run(
[*shlex.split(ssh_cmd), ssh_host, remote],
capture_output=True,
text=True,
timeout=SSH_TIMEOUT_S,
)
if r.returncode != 0:
raise RuntimeError(r.stderr.strip()[-300:] or f"ssh exit {r.returncode}")
return r.stdout
[docs]
def stop_run(
ssh_host: str,
job_ids: str,
pose_dir: str | None,
ssh_cmd: str = "ssh -o BatchMode=yes",
run=subprocess.run,
) -> str:
"""Cancel every job of a run and release its claimed arenas, so a rerun can
take them at once. Finished arenas (.h5 / .empty) are kept."""
ids = shlex.quote(",".join(j for j in str(job_ids).split(",") if j))
# scancel exits 1 on job ids that already finished, so cancel the live ones only
remote = (
f"alive=$(squeue -j {ids} -h -o %i 2>/dev/null | paste -sd,); "
'[ -z "$alive" ] || scancel "$alive" 2>/dev/null'
)
if pose_dir:
remote += f"; rm -rf {shlex.quote(pose_dir)}/_queue/claim_*"
remote += f"; squeue -j {ids} -h -o '%i %T' 2>/dev/null; true"
return _ssh(ssh_host, ssh_cmd, remote, run)
[docs]
def arena_log(
ssh_host: str,
pose_dir: str,
index: int,
lines: int = 80,
ssh_cmd: str = "ssh -o BatchMode=yes",
run=subprocess.run,
) -> str:
"""Last lines of one arena's SLEAP log on the cluster."""
log = f"{pose_dir}/_queue/arena_{index:02d}.log"
remote = (
f"tail -n {int(lines)} {shlex.quote(log)} 2>/dev/null || echo 'no log yet for this arena'"
)
return _ssh(ssh_host, ssh_cmd, remote, run)