#!/bin/sh
# gpumon.sh — one screen of GPU state for fl (RX 7600 XT · amdgpu · Vulkan backend).
#
#   watch -c -d -n 1 ~/.config/llamacpp/gpumon.sh    (fish abbr `vramx`, fl/.config/fish/host.fish)
#   ~/.config/llamacpp/gpumon.sh                      (one shot)
#
# Why not rocm-smi: `--showmemuse` gives percentages and hides GTT. The freeze mode on this box is
# VRAM overcommit spilling into GTT (system RAM) — see the config.ini header — so GTT and the
# absolute free-VRAM figure (the ≥ 2.5 GB gate) are the numbers that matter. amdgpu exposes both
# in sysfs: instant, no ROCm needed. Per-process VRAM comes from /proc/<pid>/fdinfo (kernel
# drm-usage-stats), readable only for your own processes unless root. llama.service runs as the
# login user (anon), so its children show up; if they ever don't:
#   systemctl show -p User llama.service      and run this with sudo.
# No absolute GTT alarm on purpose: bench.py judges GTT as a delta over a per-model baseline
# (a browser-heavy desktop legitimately sits on a few GB) — `watch -d` shows the movement.
#
# GPUMON_HOST overrides the router host (default 127.0.0.1) — for a dry run from another box.

dev=/sys/class/drm/renderD128/device
host=${GPUMON_HOST:-127.0.0.1}
port=11343

echo "=== GPU (amdgpu sysfs) ==="
if [ -r "$dev/mem_info_vram_used" ]; then
  temp=$(cat "$dev"/hwmon/hwmon*/temp1_input 2>/dev/null | head -n 1)
  awk -v u="$(cat "$dev/mem_info_vram_used")" -v t="$(cat "$dev/mem_info_vram_total")" \
      -v g="$(cat "$dev/mem_info_gtt_used" 2>/dev/null)" -v gt="$(cat "$dev/mem_info_gtt_total" 2>/dev/null)" \
      -v b="$(cat "$dev/gpu_busy_percent" 2>/dev/null)" -v m="$(cat "$dev/mem_busy_percent" 2>/dev/null)" \
      -v c="${temp:-0}" 'BEGIN {
    G = 1073741824
    free = (t - u) / G
    flag = free < 1.5 ? "   <<< HARD FLOOR (bench.py guard)" : free < 2.5 ? "   <-- below the 2.5 GB target" : ""
    printf "VRAM  %5.2f / %5.2f GiB   free %5.2f GiB%s\n", u / G, t / G, free, flag
    printf "GTT   %5.2f / %5.2f GiB   (VRAM spilled to RAM — must not grow while a model is resident)\n", g / G, gt / G
    printf "busy  gpu %3d%%   mem %3d%%   edge %d°C\n", b, m, c / 1000
  }'
else
  echo "no amdgpu sysfs under $dev — not the fl box, or the GPU is not renderD128"
fi

echo
echo "=== processes holding the GPU  (VRAM from fdinfo · RSS · uptime; own-user only unless root) ==="
# fuser puts the PID list on stdout (the pretty table of -v goes to stderr, hence no -v here).
pids=$(fuser /dev/dri/renderD128 /dev/dri/card* 2>/dev/null | tr ' ' '\n' | grep -E '^[0-9]+$' | sort -un)
if [ -z "$pids" ]; then
  echo "(nothing visible — no /dev/dri here, or the GPU users belong to another user)"
else
  for p in $pids; do
    [ -r "/proc/$p/fdinfo" ] || continue
    # one drm client can be reachable through several fds — dedupe by drm-client-id, then sum
    vram=$(awk 'FNR == 1 { cid = "" }
                /^drm-client-id:/ { cid = $2 }
                /^drm-(total|memory)-vram:/ { v[cid] = $2 }
                END { for (k in v) s += v[k]; printf "%.2f", s / 1048576 }' /proc/"$p"/fdinfo/* 2>/dev/null)
    name=$(cat /proc/"$p"/comm 2>/dev/null)
    alias=$(tr '\0' '\n' < /proc/"$p"/cmdline 2>/dev/null | awk '$0 == "--alias" { getline; print; exit }')
    set -- $(ps -o rss=,etime= -p "$p" 2>/dev/null | awk '{ printf "%.1f %s", $1 / 1048576, $2 }')
    printf '%7s GiB  %7s  %-16s  rss %5s GiB  up %-11s %s\n' "$vram" "$p" "$name" "${1:-?}" "${2:-?}" "$alias"
  done | sort -rn | head -n 15
fi

echo
echo "=== router $host:$port — preset status (anything not unloaded; the sysfs line above is the truth for VRAM) ==="
if out=$(curl -sf -m 1 "http://$host:$port/models" 2>/dev/null); then
  printf '%s' "$out" \
    | jq -r '.data[] | select(.status.value != "unloaded") | "\(.status.value)\t\(.id)"' 2>/dev/null \
    | { grep . || echo "(nothing resident)"; }
else
  echo "(router not answering — systemctl status llama.service)"
fi