Coja
93b4f695ce
Re-export of private HEAD 73b4a65: default-model flip + 8k thinking budget in the aichat/opencode/pi client configs, the Ornith-1.5-35B-A3B preset and its gate bench in fl/.config/llamacpp, and the fl fish abbrs `vram` / `vramx` with the new gpumon.sh (sysfs VRAM/GTT monitor).
71 lines
3.9 KiB
Bash
Executable File
71 lines
3.9 KiB
Bash
Executable File
#!/bin/sh
|
|
# gpumon.sh — one screen of GPU state for fl (RX 7600 XT · amdgpu · Vulkan backend).
|
|
#
|
|
# watch -c -d -n 1 ~/.config/llamacpp/gpumon.sh (fish abbr `vramx`, fl/.config/fish/host.fish)
|
|
# ~/.config/llamacpp/gpumon.sh (one shot)
|
|
#
|
|
# Why not rocm-smi: `--showmemuse` gives percentages and hides GTT. The freeze mode on this box is
|
|
# VRAM overcommit spilling into GTT (system RAM) — see the config.ini header — so GTT and the
|
|
# absolute free-VRAM figure (the ≥ 2.5 GB gate) are the numbers that matter. amdgpu exposes both
|
|
# in sysfs: instant, no ROCm needed. Per-process VRAM comes from /proc/<pid>/fdinfo (kernel
|
|
# drm-usage-stats), readable only for your own processes unless root. llama.service runs as the
|
|
# login user (anon), so its children show up; if they ever don't:
|
|
# systemctl show -p User llama.service and run this with sudo.
|
|
# No absolute GTT alarm on purpose: bench.py judges GTT as a delta over a per-model baseline
|
|
# (a browser-heavy desktop legitimately sits on a few GB) — `watch -d` shows the movement.
|
|
#
|
|
# GPUMON_HOST overrides the router host (default 127.0.0.1) — for a dry run from another box.
|
|
|
|
dev=/sys/class/drm/renderD128/device
|
|
host=${GPUMON_HOST:-127.0.0.1}
|
|
port=11343
|
|
|
|
echo "=== GPU (amdgpu sysfs) ==="
|
|
if [ -r "$dev/mem_info_vram_used" ]; then
|
|
temp=$(cat "$dev"/hwmon/hwmon*/temp1_input 2>/dev/null | head -n 1)
|
|
awk -v u="$(cat "$dev/mem_info_vram_used")" -v t="$(cat "$dev/mem_info_vram_total")" \
|
|
-v g="$(cat "$dev/mem_info_gtt_used" 2>/dev/null)" -v gt="$(cat "$dev/mem_info_gtt_total" 2>/dev/null)" \
|
|
-v b="$(cat "$dev/gpu_busy_percent" 2>/dev/null)" -v m="$(cat "$dev/mem_busy_percent" 2>/dev/null)" \
|
|
-v c="${temp:-0}" 'BEGIN {
|
|
G = 1073741824
|
|
free = (t - u) / G
|
|
flag = free < 1.5 ? " <<< HARD FLOOR (bench.py guard)" : free < 2.5 ? " <-- below the 2.5 GB target" : ""
|
|
printf "VRAM %5.2f / %5.2f GiB free %5.2f GiB%s\n", u / G, t / G, free, flag
|
|
printf "GTT %5.2f / %5.2f GiB (VRAM spilled to RAM — must not grow while a model is resident)\n", g / G, gt / G
|
|
printf "busy gpu %3d%% mem %3d%% edge %d°C\n", b, m, c / 1000
|
|
}'
|
|
else
|
|
echo "no amdgpu sysfs under $dev — not the fl box, or the GPU is not renderD128"
|
|
fi
|
|
|
|
echo
|
|
echo "=== processes holding the GPU (VRAM from fdinfo · RSS · uptime; own-user only unless root) ==="
|
|
# fuser puts the PID list on stdout (the pretty table of -v goes to stderr, hence no -v here).
|
|
pids=$(fuser /dev/dri/renderD128 /dev/dri/card* 2>/dev/null | tr ' ' '\n' | grep -E '^[0-9]+$' | sort -un)
|
|
if [ -z "$pids" ]; then
|
|
echo "(nothing visible — no /dev/dri here, or the GPU users belong to another user)"
|
|
else
|
|
for p in $pids; do
|
|
[ -r "/proc/$p/fdinfo" ] || continue
|
|
# one drm client can be reachable through several fds — dedupe by drm-client-id, then sum
|
|
vram=$(awk 'FNR == 1 { cid = "" }
|
|
/^drm-client-id:/ { cid = $2 }
|
|
/^drm-(total|memory)-vram:/ { v[cid] = $2 }
|
|
END { for (k in v) s += v[k]; printf "%.2f", s / 1048576 }' /proc/"$p"/fdinfo/* 2>/dev/null)
|
|
name=$(cat /proc/"$p"/comm 2>/dev/null)
|
|
alias=$(tr '\0' '\n' < /proc/"$p"/cmdline 2>/dev/null | awk '$0 == "--alias" { getline; print; exit }')
|
|
set -- $(ps -o rss=,etime= -p "$p" 2>/dev/null | awk '{ printf "%.1f %s", $1 / 1048576, $2 }')
|
|
printf '%7s GiB %7s %-16s rss %5s GiB up %-11s %s\n' "$vram" "$p" "$name" "${1:-?}" "${2:-?}" "$alias"
|
|
done | sort -rn | head -n 15
|
|
fi
|
|
|
|
echo
|
|
echo "=== router $host:$port — preset status (anything not unloaded; the sysfs line above is the truth for VRAM) ==="
|
|
if out=$(curl -sf -m 1 "http://$host:$port/models" 2>/dev/null); then
|
|
printf '%s' "$out" \
|
|
| jq -r '.data[] | select(.status.value != "unloaded") | "\(.status.value)\t\(.id)"' 2>/dev/null \
|
|
| { grep . || echo "(nothing resident)"; }
|
|
else
|
|
echo "(router not answering — systemctl status llama.service)"
|
|
fi
|