594 lines
42 KiB
INI
594 lines
42 KiB
INI
# llama.cpp model config — AMD RX 7600 XT · 16 GB (17.16 GB total) · backend = Arch llama.cpp-vulkan
|
||
# (RADV, device Vulkan0 — NOT ROCm/HIP; rocm-smi is used for monitoring only)
|
||
# ═════════════════════════════════════════════════════════════════════════════
|
||
# ⚠⚠ VRAM SAFETY — READ THIS. This GPU ALSO DRIVES THE DISPLAY. If a model asks for
|
||
# more VRAM than is free, the amdgpu driver does NOT OOM cleanly — it spills into system RAM
|
||
# (GTT), which starves the desktop and FREEZES THE WHOLE PC (reboot required).
|
||
# → TARGET ≥ ~2.5 GB free (≤ ~14.5 GB used). HARD FLOOR 1.5 GB (bench.py guard stops there).
|
||
# Idle desktop uses ~1.3 GB of 17.16 — but a browser/leftover model can hold 1-3 GB more,
|
||
# so measure with a CLEAN baseline (bench.py prints "baseline used"; close GPU apps first;
|
||
# sleep-idle keeps a model resident up to 5 min after last use).
|
||
# → Verify with bench.py → ledger: bench-results.md (latest) + bench-history.md (all runs).
|
||
# → If a model is tight: LOWER ctx-size, or RAISE n-cpu-moe (MoE), or use a smaller quant.
|
||
# The values below are deliberately CONSERVATIVE after freeze incidents — raise ctx
|
||
# only after bench.py confirms the headroom.
|
||
# ═════════════════════════════════════════════════════════════════════════════
|
||
# Speed notes: decode is memory-bandwidth-bound (~288 GB/s). Prefer MoE (few active
|
||
# params). For MoE that doesn't fit, n-cpu-moe offloads expert FFNs to CPU (48 GB RAM) —
|
||
# this is FASTER than letting VRAM spill to GTT, and frees VRAM. threads = 6 (5600X cores).
|
||
# Runtime: `sudo rocm-smi --setperflevel high`; `watch -n1 rocm-smi --showmeminfo vram`.
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Coding models
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
[Qwen3-Coder-Next-UD-IQ3_XXS]
|
||
# ⭐ Big-context coder: Qwen3-Coder-Next 80B-A3B (3B active), Qwen3-Next hybrid attn → cheap KV.
|
||
# Weights ~30 GB: ALL experts on CPU (n-cpu-moe 99), only attention+KV on GPU → low VRAM, safe.
|
||
# Needs a CURRENT llama.cpp build (Qwen3-Next arch). ⬇ DOWNLOAD unsloth/Qwen3-Coder-Next-GGUF:UD-IQ3_XXS.
|
||
# Froze the PC before at n-cpu-moe 48 / ctx 131072 (overcommit) — keep it conservative, verify, then raise ctx.
|
||
model = /home/anon/software/models/Qwen3-Coder-Next-UD-IQ3_XXS.gguf
|
||
ctx-size = 131072 # ✓ VERIFIED @131072/moe99 (2026-07-16): 15.7 t/s, 6.0 GB used / 11.2 free —
|
||
# full 128k at a comfortable margin. Settled.
|
||
n-gpu-layers = 99
|
||
n-cpu-moe = 99 # ALL experts on CPU — settled AGAIN 2026-08-08 after a two-round
|
||
# experiment, CLOSED: GPU experts barely move this model.
|
||
# moe56 (04:11) = NO-OP: 6.0 used ≈ moe99's 5.8, all numbers flat →
|
||
# only ~48 MoE layers exist; moe ≥ 48 ≡ all-on-CPU. (Also: the old
|
||
# "moe48/131k froze the PC" lore was ctx-KV overcommit in a pre-
|
||
# quantized-KV config, not expert placement.)
|
||
# moe40 (04:37) = 8 layers on GPU, 10.2 used / 6.9 free: prefill
|
||
# 33→36 (+9%), decode 16.4 (noise) — 4.2 GB for a rounding error.
|
||
# The shallow curve says don't chase it: cache-reuse already makes
|
||
# growing sessions incremental, and the 6 GB footprint IS the niche.
|
||
# MEASURED (07-16): moe99 = 14.3-16.4 t/s vs moe36@65536 = 16.9.
|
||
threads = 6
|
||
# no-mmap TRIED AND REVERTED 2026-07-18: llama.cpp's log hint suggested it (CPU experts +
|
||
# mmap = page-fault churn), but reading 26 GB fully resident exceeds the router's load
|
||
# patience → "failed to load". Retry only if the router ever gets a load-timeout knob.
|
||
flash-attn = on
|
||
cache-type-k = q8_0
|
||
cache-type-v = q4_0
|
||
batch-size = 2048
|
||
ubatch-size = 256 # small compute buffer = less VRAM
|
||
cache-reuse = 256
|
||
defrag-thold = 0.1
|
||
temp = 0.7 # unsloth default 1.0/top-p0.95/top-k40/min-p0.01 — lower = more deterministic
|
||
top-p = 0.95
|
||
top-k = 40
|
||
min-p = 0.01
|
||
jinja = on
|
||
sleep-idle-seconds = 300
|
||
|
||
# ─── RETIRED 2026-07-16: [DeepSeek-Coder-V2-Lite-Instruct-Q8_0] ──────────────────────────
|
||
# Mid-2024 model, outclassed by GLM-4.7-Flash + both Qwen coders. Bench kept skipping it
|
||
# (~15.3 GB fixed footprint; measured 2026-07-03 that ctx/moe knobs barely move it — only fix
|
||
# was a Q4_K_M requant, not worth it as the 4th-best coder). Removed from opencode/pi/aichat.
|
||
# File can be deleted from the models dir: DeepSeek-Coder-V2-Lite-Instruct-Q5_K_M.gguf
|
||
|
||
# ─── RETIRED 2026-08-06: [Qwen3-Coder-30B-Instruct-IQ4_XS] ───────────────────────────────
|
||
# Lost the 2026-08-06 quant race to UD-Q3_K_XL@moe10 (31.3 vs 23.0 t/s, +36%, three
|
||
# consistent sweeps; it also failed twice to load on ~3.2 GB evening baselines at moe20 =
|
||
# 14.1-14.3 used). Clients rewired to Qwen3-Coder-30B-Instruct-UD-Q3_K_XL same day — the
|
||
# extended quality trial was deliberately skipped (user call; revert path = git, one commit).
|
||
# ⚠ OWUI chats/presets saved against this id need re-picking after the restart.
|
||
# File can be deleted from the models dir: Qwen3-Coder-30B-A3B-Instruct-IQ4_XS.gguf
|
||
# Reference numbers: moe24 = 21.1 t/s, 11.9/5.3 · moe20 = 23.2 (2026-07-16), 22.0-23.0 (2026-08-05/06)
|
||
|
||
[Qwen3-Coder-30B-Instruct-UD-Q3_K_XL]
|
||
# ⭐ MAIN AGENT CODER — took the slot 2026-08-06 after winning the quant race: 31.3 t/s @moe10
|
||
# vs IQ4_XS@moe20 = 23.0 (+36%, three consistent sweeps); Q3_K_S dominated (tombstone below).
|
||
# Same 32k ctx and sampling as the retired IQ4_XS — clients only needed the id swap.
|
||
# ⚠ QUALITY TRIAL SKIPPED (user call, 2026-08-06): Q3-on-MoE damage looks like wrong API
|
||
# signatures / off-by-one logic, NOT word salad — if opencode sessions smell off, the
|
||
# IQ4_XS preset + client wiring are one `git revert` away (and the .gguf may still be on
|
||
# disk). UD protects attn/shared tensors; the experts are still 3-bit.
|
||
# RACE NUMBERS: moe24 = 22.2 · moe16 = 25.1 (07-16) / 25.5 / 26.3 · moe10 = 31.3 (15.0 used /
|
||
# 2.1 free @3.1 GB baseline; replicate 30.9 @1.8 free under a 13.9 leftover-model baseline).
|
||
model = /home/anon/software/models/Qwen3-Coder-30B-A3B-Instruct-UD-Q3_K_XL.gguf
|
||
ctx-size = 32768
|
||
n-gpu-layers = 99
|
||
n-cpu-moe = 12 # ✓ VERIFIED @moe12 (2026-08-07, 2.9 GB EVENING baseline): 27.0 t/s,
|
||
# 14.2 used / 3.0 free — loaded fine in exactly the regime that killed
|
||
# IQ4_XS@moe20 twice; the insurance works. Cost vs moe10 (31.3 clean race)
|
||
# was ~4 t/s, more than the ~1 estimated — evening compute contention
|
||
# likely eats 2-3 of that; confirmed: CLEAN (2026-08-08 02:45, 1.9
|
||
# baseline) = 29.5 t/s, 13.2 used / 4.0 free — evening cost was indeed
|
||
# contention. Settled: moe12 = 29.5 clean / 27.0 evening, always loads.
|
||
threads = 6
|
||
flash-attn = on
|
||
cache-type-k = q8_0
|
||
cache-type-v = q4_0
|
||
batch-size = 2048
|
||
ubatch-size = 1024 # ✓ SETTLED 2026-08-08: prefill 74 → 89-92 t/s (+24%), decode unchanged,
|
||
# buffer +0.5 GB. FINAL GATE PASSED (14:43 sweep, 3.0 GB dirty baseline =
|
||
# evening-class): loaded and ran at 14.6 used / 2.6 free. If a worse
|
||
# evening (3.3+) ever fails the load: revert to 512, or pay one moe
|
||
# step (12→13).
|
||
cache-reuse = 256
|
||
defrag-thold = 0.1
|
||
temp = 0.15
|
||
top-p = 0.8
|
||
top-k = 20
|
||
min-p = 0
|
||
repeat-penalty = 1.05
|
||
jinja = on
|
||
sleep-idle-seconds = 300
|
||
|
||
# ─── RETIRED 2026-08-06: [Qwen3-Coder-30B-Instruct-Q3_K_S] ───────────────────────────────
|
||
# Dominated in the quant race: 30.9 t/s @moe8 (15.2 used / 2.0 free, replicate 30.9) = a tie
|
||
# with UD-Q3_K_XL@moe10 (31.3) at equal headroom, from a strictly worse quant (plain Q3_K_S,
|
||
# no UD-protected attn/shared tensors). No axis where it wins. Was never client-wired.
|
||
# File can be deleted from the models dir: Qwen3-Coder-30B-A3B-Instruct-Q3_K_S.gguf
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# General Qwen models (35B-A3B is fast MoE; the dense 27Bs are inherently VRAM-tight)
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
[Qwen3.6-35B-A3B-MTP-UD-IQ3_XXS]
|
||
# ⭐ Fast NON-THINKING daily driver: Qwen3.6-35B-A3B (MoE, 3B active) — the "bigger Qwen3
|
||
# Instruct". Compact precise answers, no reasoning preamble. (Alias kept for clients.)
|
||
# File is flat + named without "MTP" (unsloth MTP-repo naming). If the load log complains
|
||
# there is no MTP head in this file, comment the two spec- lines out (plain repo file).
|
||
model = /home/anon/software/models/Qwen3.6-35B-A3B-UD-IQ3_XXS.gguf
|
||
mmproj = /home/anon/software/models/mmproj-Qwen3.6-35B-A3B-F16.gguf # vision ✓ VERIFIED 2026-07-18:
|
||
# 13.9 used / 3.3 free (2.8 dirty baseline), decode unchanged at 36-37 t/s.
|
||
spec-type = draft-mtp
|
||
spec-draft-n-max = 2
|
||
reasoning-budget = 0 # hard-disable thinking → instant compact answers. Delete to allow /think.
|
||
# ⚠ if startup says "option 'reasoning_budget' not recognized" → remove this line.
|
||
ctx-size = 24576 # raised from 16384 (bench @16384: 4.8 GB free) — bench to confirm ≥2.5 free
|
||
n-gpu-layers = 99
|
||
n-cpu-moe = 16 # MEASURED @24: 19.8 t/s (MTP working!), 10.6 used / 6.6 free → pull experts onto GPU
|
||
threads = 6
|
||
flash-attn = on
|
||
cache-type-k = q8_0
|
||
cache-type-v = q4_0
|
||
batch-size = 2048
|
||
ubatch-size = 512
|
||
cache-reuse = 256
|
||
defrag-thold = 0.1
|
||
temp = 0.7
|
||
top-p = 0.8
|
||
top-k = 20
|
||
min-p = 0
|
||
repeat-penalty = 1.05
|
||
jinja = on
|
||
sleep-idle-seconds = 120
|
||
|
||
[Qwen3.6-35B-A3B-Thinking]
|
||
# Same GGUF as above with reasoning ENABLED (no reasoning-budget line) — fills the "thinking
|
||
# model" slot at zero disk cost. Qwen thinking-mode sampling: temp 0.6 / top-p 0.95, no
|
||
# repeat-penalty. Slower per answer (reasoning preamble) — use for hard problems.
|
||
model = /home/anon/software/models/Qwen3.6-35B-A3B-UD-IQ3_XXS.gguf
|
||
mmproj = /home/anon/software/models/mmproj-Qwen3.6-35B-A3B-F16.gguf # vision ✓ VERIFIED 2026-07-18:
|
||
# 13.9 used / 3.3 free (2.8 dirty baseline), decode unchanged at 36-37 t/s.
|
||
spec-type = draft-mtp
|
||
spec-draft-n-max = 2
|
||
ctx-size = 24576 # thinking eats ctx — same footprint as the non-thinking preset
|
||
n-gpu-layers = 99
|
||
n-cpu-moe = 16
|
||
threads = 6
|
||
flash-attn = on
|
||
cache-type-k = q8_0
|
||
cache-type-v = q4_0
|
||
batch-size = 2048
|
||
ubatch-size = 512
|
||
cache-reuse = 256
|
||
defrag-thold = 0.1
|
||
temp = 0.6
|
||
top-p = 0.95
|
||
top-k = 20
|
||
min-p = 0
|
||
jinja = on
|
||
sleep-idle-seconds = 120
|
||
|
||
# ─── RETIRED 2026-07-03: [Qwen3.6-27B-UD-IQ3_XXS] ────────────────────────────────────────
|
||
# The 27B files (plain AND MTP) were deleted from the models dir — preset removed so the
|
||
# router doesn't advertise a dead model. To bring it back: re-download (MTP-repo files are
|
||
# named WITHOUT "MTP"), restore the preset, re-add the id to opencode/pi/aichat lists.
|
||
|
||
# ─── RETIRED 2026-07-16: [Qwen3.5-27B-UD-Q3_K_XL] ────────────────────────────────────────
|
||
# Dense 27B: 16.2 GB used / 0.9 free even @8192 (under the 1.5 floor) and ~10 t/s ceiling on
|
||
# this card. Role covered by Qwen3.6-35B-A3B (MoE, 2× faster). Removed from opencode/pi/aichat.
|
||
# File can be deleted from the models dir: Qwen3.5-27B-UD-Q3_K_XL.gguf
|
||
|
||
[Qwen3.5-9B-UD-Q6_K_XL]
|
||
# Small fast Qwen + MTP. Replaces Qwen3.5-9B-Q8_0 (retired 2026-07-16: Q8 was quality-overkill
|
||
# and bandwidth-slow at 17.2 t/s — delete Qwen3.5-9B-Q8_0.gguf; also delete the interim
|
||
# Qwen3.5-4B-UD-Q6_K_XL.gguf, superseded by this).
|
||
# ⬇ FILE MUST COME FROM unsloth/Qwen3.5-9B-MTP-GGUF (8.99 GB — MTP head embedded, filename
|
||
# identical to the plain repo!). If the load log complains there is no MTP head in this
|
||
# file, it's the plain-repo file — comment the two spec- lines out.
|
||
# ✓ VERIFIED (2026-07-16): 32.2 t/s (MTP working — nearly 2× the Q8's 17.2), prefill 83,
|
||
# 12.4 used / 4.8 free @32768. TTFT ~3.7 s = the model thinking (reasoning streams first).
|
||
model = /home/anon/software/models/Qwen3.5-9B-UD-Q6_K_XL.gguf
|
||
mmproj = /home/anon/software/models/Qwen3.5-9B-MTP-mmproj-F16.gguf # vision (from the MTP repo)
|
||
spec-type = draft-mtp
|
||
spec-draft-n-max = 3 # ⚗→✓ KEPT (04:11): 33.8 t/s vs 32.0-33.1 @n=2 — small real win.
|
||
# Revert to 2 if a future sweep drops below 32.
|
||
ctx-size = 32768 # raise toward 65536 once bench confirms free ≥2.5 GB
|
||
n-gpu-layers = 99
|
||
flash-attn = on
|
||
cache-type-k = q8_0
|
||
cache-type-v = q4_0 # q4 V to save VRAM
|
||
batch-size = 2048
|
||
ubatch-size = 512
|
||
cache-reuse = 256
|
||
defrag-thold = 0.1
|
||
temp = 0.7
|
||
top-p = 0.8
|
||
top-k = 20
|
||
min-p = 0
|
||
repeat-penalty = 1.05
|
||
jinja = on
|
||
sleep-idle-seconds = 60
|
||
|
||
[Qwen3.8-27B-UD-IQ3_XXS]
|
||
# ⚠ CLEAN-DESKTOP SPECIALTY — KEPT by user decision 2026-09-10 despite failing the ≥22 t/s speed
|
||
# gate (14.5 t/s MTP / 10.9 base; 2.8 GB free @1.5 baseline @16k). Pick it ONLY on a quiet
|
||
# desktop: at a 3+ GB evening baseline it lands ~1 GB free = GTT-spill/freeze territory. Role:
|
||
# slow-but-smart dense-27B answers + vision. Origin (added 2026-08-16): dense 27B, but NOT like the two
|
||
# retired dense 27Bs: hybrid attention (only 16 of 64 layers full-attn, rest linear/SSM →
|
||
# cheap KV, Coder-Next-style) AND an embedded MTP head — VERIFIED in the gguf header:
|
||
# qwen35.nextn_predict_layers = 1, block_count = 65 (64+1), arch "qwen35" = the same
|
||
# draft-mtp path the working 9B preset uses. Hybrid reasoner (thinking ON by default,
|
||
# reasoning_effort tunable via chat-template-kwargs if a -low alias is ever wanted),
|
||
# vision, n_ctx_train = 262144.
|
||
# Speed math: dense 11.9 GB weights on ~288 GB/s ≈ 14-16 t/s base; MTP should lift to
|
||
# ~22-30 = roster-competitive IF the draft engages. If it doesn't, this lands in
|
||
# retired-dense-27B territory (~10-14) → retire again, quality won't pay for the speed.
|
||
# ⚠ BENCH HISTORY — still no green row, decode/MTP UNMEASURED after 5 attempts:
|
||
# · 08-16 14:59/15:19/15:29 (3.3-3.5 DIRTY baseline): 16.5-16.7 used / 0.4-0.7 free, guard
|
||
# skipped gen every time. Implied footprint 13.2 GB — MISLEADING, see below.
|
||
# · 08-17 10:31: ERROR conn-refused = bench raced llama.service coming up after boot.
|
||
# · 08-17 10:38 (1.9 near-clean baseline): 15.9 used / 1.3 free — guard floor missed by
|
||
# 0.2. TRUE footprint = 14.0 GB over baseline; the dirty runs looked 0.8 smaller because
|
||
# ROCm silently spilled that much to GTT (the exact freeze precursor the guards watch).
|
||
# · 08-17 10:43 (post-trim, 10.0 leftover start, partial drain): 15.8 / 1.4 — floor missed
|
||
# by 0.1. Trim verified live via router args; ~2.4 effective baseline poisoned it again.
|
||
# · 08-17 10:53 (1.9 baseline, NO leftovers — a genuine desktop-idle run): 15.8 / 1.4,
|
||
# floor missed by 0.1 again. DECISIVE: trimmed footprint 13.9 vs untrimmed 14.0 — both
|
||
# knobs together saved 0.1 GB. ctx/ubatch do NOT move this model (fixed SSM-state +
|
||
# big-vocab buffers). Absolute best case ≈ 2.0 free at a pristine 1.3 baseline @16k —
|
||
# permanently below the 2.5 target. → RETIREMENT RECOMMENDED 2026-08-17; decode/MTP
|
||
# never measured, academic at these margins. Third dense ~27B to die on this card,
|
||
# first one killed by fixed buffers rather than bandwidth.
|
||
# Consequences: full-ctx even on a 1.3 CLEAN night = ~15.3 used / 1.9 free (runs, but
|
||
# under the 2.5 target). EVENING regime can never fit this quant — that needs ≤ UD-IQ2_M,
|
||
# a quality gutting. → 08-17: TRIMMED FOR THE GATE BENCH: ctx 24576→16384 (~-0.25) +
|
||
# ubatch 512→256 (~-0.3-0.5) → predicted ~15.2 used / ~2.0 free @1.9 baseline. Clients
|
||
# still say 24576 — deliberately NOT re-wired for a temporary trim; do not use from
|
||
# clients anyway until this passes (a busy-desktop pick is one tab from the GTT freeze).
|
||
# DECISION GATE (superseded by the 09-10 user decision below): MTP ≥ ~22 t/s keeps it (clean-desktop specialty, restore ctx
|
||
# 24576 + re-check), anything less → retire + delete (dense 27Bs die on this card).
|
||
# ↻ REOPENED 2026-09-09 — the file on disk is STALE: unsloth re-quantized the whole repo as
|
||
# "Dynamic v3" on 2026-08-19, three days AFTER the 08-16 download (~10% better accuracy at
|
||
# equal size, per unsloth). Verified via the HF tree API at the pre/post commits:
|
||
# UD-IQ3_XXS 11.91 → 10.93 GB (SAME filename — re-download overwrites in place, no client
|
||
# or section changes); UD-Q2_K_XL 10.68 → 9.83; UD-IQ2_M DELETED (gone as an option);
|
||
# UD-IQ2_S 8.37 / UD-IQ2_XXS 7.27 are new 2-bit tiers. Header range-read 09-09: v3
|
||
# IQ3_XXS and Q2_K_XL both still carry nextn_predict_layers = 1 (MTP head embedded);
|
||
# unsloth stripped the head from ≤ IQ2_S only and ships MTP/mtp-Qwen3.8-27B-Q4_0.gguf
|
||
# as a separate model-draft for those (gemma-4 style).
|
||
# Predicted with v3 IQ3_XXS (footprint 14.0 − 0.98 ≈ 13.0; bench.py GB = decimal):
|
||
# @1.9 real-idle baseline ≈ 14.8–15.0 used / 2.2–2.4 free · @1.3 pristine ≈ 14.2 / ~2.9.
|
||
# Clears the 1.5 floor by ~0.8 even at 1.9 → the gen bench can FINALLY run; still ~0.1–0.3
|
||
# under the 2.5 target on a real idle desktop, clears it outright on a pristine one. Overhead datum: a third-party paired bench measured
|
||
# draft-mtp at ~+0.75 GB on this model (unsloth docs: "1-2 GB extra headroom") — a good
|
||
# part of the "fixed" ~2.1 GB is MTP + KV, not only SSM-state/vocab buffers.
|
||
# NEXT STEP (recommended, NOT yet done): re-download v3 IQ3_XXS on fl (verify 10.93 GB /
|
||
# 10.18 GiB before restarting), dotsync, restart llama.service, clean-baseline bench.
|
||
# Then the gate above decides — with one refinement: free 2.0–2.5 at ≥22 t/s = user's call
|
||
# between "clean-desktop specialty" and stepping down to v3 UD-Q2_K_XL (predicted ~13.7 /
|
||
# 3.5 @1.9 — needs the section-rename round across the six client files; sequence it
|
||
# after any in-flight client edits). Decode < 22 → retire regardless of VRAM.
|
||
# No in-family alternative: the Qwen3.8 lineup is 27B dense, Flash-Next 125B-A6B (smallest
|
||
# GGUF 72.5 GB, 75 GB RAM floor even with mmap'd n-gram tables — out on 48+16),
|
||
# 2.4T-A95B and Max. No 3.8 MoE in the 35B-A3B class exists; watch HF.
|
||
# ✦ GATE BENCH RAN 2026-09-10 02:36 (v3 file, sha verified; 2.1 baseline): 14.9 used / 2.2 free
|
||
# (footprint 12.8 — VRAM half of the gate PASSED, borderline) but decode 14.8 t/s, prefill 19,
|
||
# TTFT 11.7 s → speed half FAILED. Server log: MTP head loaded, draft acceptance 0.80
|
||
# (157/195, mean len 2.6) — so MTP IS engaging; at 2.6 tok/step that is ~175 ms per step ≈
|
||
# 3–4× a plain forward pass. NOTE the box runs the Arch llama.cpp-vulkan package (RADV,
|
||
# device Vulkan0) — NOT HIP/ROCm (this header said "ROCm" until 09-10); rocm-smi = monitoring only.
|
||
# Suspects: the 48 GatedDeltaNet layers on the Vulkan backend (upstream #20354: missing/slow
|
||
# GDN shader → CPU fallback) and/or recurrent-state checkpoint/restore per speculative step
|
||
# (~150 MiB for this model; reported net-negative on hybrid 27Bs). The hybrid 35B-A3B does
|
||
# 37 t/s on the same backend, so it is size-specific, not Vulkan-generic.
|
||
# A/B NEXT: [Qwen3.8-27B-UD-IQ3_XXS-nomtp] (TEMP alias, used 03:19 then removed) measures
|
||
# base decode; base ≥ 18 → MTP is the net negative here, run without it (frees ~0.75 GB);
|
||
# base ≈ 9 → the GDN path is the wall → retire. Also read "graph splits" from the load log.
|
||
# cache_reuse is inert here ("not supported by multimodal") — true for every mmproj preset.
|
||
# Re-run 03:07 @0.9 PRISTINE baseline: 15.0 t/s / 17 pp / 13.9 used / 3.3 free (footprint 13.0)
|
||
# — reproducible; the VRAM half is now fully closed, only the speed half is open.
|
||
# ✦ A/B 2026-09-10 03:19 (@1.5 baseline, same run): MTP 14.5 t/s / 30 pp / 14.3 used / 2.8 free
|
||
# vs NO-MTP 10.9 t/s / 34 pp / 12.8 used / 4.4 free. → MTP WORKS (+33%, matches the sudoingX
|
||
# paired benches) and costs 1.5 GB VRAM; the wall is the BASE: 10.9 t/s = 119 GB/s effective
|
||
# (41% of the card's 288) — the 48 GatedDeltaNet layers on the Vulkan/RADV backend, exactly
|
||
# the #20354 datum (Qwen3.5-27B ≈ 11.8 t/s on Strix Halo, either backend). Prefill 30-34 on
|
||
# a fully-GPU model is the same wall (the hybrid 35B-A3B daily driver's 64 pp is GDN-bound
|
||
# too). "graphs reused" 97-260 in the log → not a graph-rebuild problem. Nothing in this
|
||
# preset can move it; a bigger K-quant (faster dequant on Vulkan) does not fit. Perfect-
|
||
# kernel ceiling on this card ≈ 20 base / ~27 MTP — an upstream Vulkan gated_delta_net
|
||
# shader improvement COULD flip it; nothing else can.
|
||
# VERDICT 2026-09-10: gate FAILED on speed (14.5 < 22) with the VRAM half passed (2.8 free
|
||
# @1.5). Per the gate → retire. Re-probe only after a llama.cpp-vulkan bump whose changelog
|
||
# touches gated_delta_net / Vulkan GDN: `./bench.py -m Qwen3.8-27B-UD-IQ3_XXS`.
|
||
# → USER DECISION 2026-09-10: KEEP as clean-desktop specialty (overrides the gate). ctx restored
|
||
# to 24576 (clients already say 24576), ubatch stays 256 (prefill is GDN-bound, the bigger
|
||
# buffer buys nothing), MTP stays (+33% for 1.5 GB). ✓ CONFIRMED @24576 2026-09-10 03:51
|
||
# (1.6 baseline): 15.1 t/s / 31 pp / 14.6 used / 2.5 free — footprint 13.0, exactly as
|
||
# predicted; free = the 2.5 target at a normal idle, ~2.8 pristine, ~0.8 on a 3.3 evening.
|
||
# ⚠ Needs a CURRENT llama.cpp build: first "qwen35" file HERE with ssm.*/hybrid-attn keys
|
||
# (the 9B is regular-attention qwen35). If the load fails with unknown-architecture /
|
||
# missing-tensor errors, update llama.cpp on fl before touching this preset.
|
||
# ⬇ DOWNLOAD from unsloth/Qwen3.8-27B-GGUF (NO separate MTP repo — unlike Qwen3.5/3.6, the
|
||
# head ships in the MAIN repo's ggufs for ≥ Q2_K_XL; ≤ IQ2_S need MTP/mtp-Qwen3.8-27B-Q4_0.gguf):
|
||
# Qwen3.8-27B-UD-IQ3_XXS.gguf (10.93 GB Dynamic v3 since 2026-08-19 — the 08-16 download
|
||
# is the 11.91 GB v2; RE-FETCH. UD-IQ4_XS 14.3 GB and
|
||
# UD-Q3_K_XL 13.15 = dead on arrival, no n-cpu-moe escape)
|
||
# mmproj-Qwen3.8-27B-F16.gguf ← mmproj-F16.gguf (928 MB — unchanged; rename; watch trailing spaces)
|
||
model = /home/anon/software/models/Qwen3.8-27B-UD-IQ3_XXS.gguf
|
||
mmproj = /home/anon/software/models/mmproj-Qwen3.8-27B-F16.gguf
|
||
spec-type = draft-mtp
|
||
spec-draft-n-max = 2 # ⚠ if the load log says this file has no MTP head, comment the two
|
||
# spec- lines out — and expect ~14 t/s, probably not worth keeping.
|
||
ctx-size = 24576 # RESTORED 2026-09-10 (16384 was the 08-17 gate-bench trim): matches
|
||
# the clients. KV is cheap (16 full-attn layers × 4 kv-heads × 256
|
||
# head_dim ≈ 26 KB/token at q8/q4): 16k→24k cost +0.3 → MEASURED
|
||
# 09-10 @1.6 baseline: 14.6 used / 2.5 free. n_ctx_train 262144.
|
||
n-gpu-layers = 99 # dense: no n-cpu-moe escape — ctx/quant are the only knobs
|
||
threads = 6
|
||
flash-attn = on
|
||
cache-type-k = q8_0
|
||
cache-type-v = q4_0 # hybrid arch takes mixed KV like Coder-Next (GLM's same-type
|
||
# rejection was MLA-specific). If load fails at context creation:
|
||
# try v = q8_0, then delete both lines (f16 — KV small enough here).
|
||
batch-size = 2048
|
||
ubatch-size = 256 # KEPT at 256 (512 before the 08-17 trim): prefill is GDN-bound on the
|
||
# Vulkan backend (30-34 t/s either way), so the bigger compute buffer
|
||
# buys nothing measurable and VRAM is the binding constraint here.
|
||
cache-reuse = 256
|
||
defrag-thold = 0.1
|
||
no-mmproj-offload = true # vision encoder on CPU → ~1 GB freed, same as the 26B
|
||
temp = 1.0 # card: thinking mode 1.0/0.95/20/0 — thinking IS the default mode
|
||
top-p = 0.95
|
||
top-k = 20
|
||
min-p = 0
|
||
jinja = on
|
||
sleep-idle-seconds = 120
|
||
|
||
# ─── RETIRED 2026-07-16: [Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-IQ3_M] ──────────────
|
||
# Dense: 10 t/s, 2.0 GB free (under the 2.5 target) and unfixable — bandwidth-bound, smaller
|
||
# quant would cost the quality that justified it. Removed from opencode/pi/aichat.
|
||
# File can be deleted from the models dir: Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-IQ3_M.gguf
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Gemma models
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
[gemma-4-26B-A4B-it-UD-IQ4_XS]
|
||
# MoE (4B active) + MTP + vision. ~13.5 GB weights → tight. Projector on CPU (no-mmproj-offload).
|
||
# From unsloth/gemma-4-26B-A4B-it-qat-GGUF: mmproj-F16.gguf + mtp-gemma-4-26B-A4B-it.gguf.
|
||
# ⬇ SOURCES — local name ← upstream (PLAIN repo unsloth/gemma-4-26B-A4B-it-GGUF; the qat
|
||
# repo has NO IQ4_XS, its only main is qat-UD-Q4_K_XL 14.25 GB — see decision note below):
|
||
# gemma-4-26B-A4B-it-UD-IQ4_XS.gguf ← gemma-4-26B-A4B-it-UD-IQ4_XS.gguf (13.6 GB)
|
||
# ✓ current: the 2026-08-05 dl has the Jul 17 template
|
||
# mmproj-gemma-4-26B-A4B-it-F16.gguf ← mmproj-F16.gguf (1.19 GB)
|
||
# mtp-gemma-4-26B-A4B-it.gguf ← MTP/mtp-gemma-4-26B-A4B-it-Q8_0.gguf (462 MB)
|
||
# Keep LOCAL names stable; re-fetch the upstream names above when repos update. Pair main
|
||
# and drafter from the SAME repo — the two repos' drafters are different extractions.
|
||
# ✓ FIXED 2026-08-08 03:15: matched plain-repo sidecars cured the "Invalid input batch."
|
||
# 500s — root cause CONFIRMED as the cross-repo drafter mismatch (old qat-repo mtp under
|
||
# the new plain main). Re-benched 32.8 t/s @moe13, 12.1 used / 5.0 free, MTP ✓ — same
|
||
# speed as pre-update (32.7), template fix now in place. The *.gguf.bak sidecars are
|
||
# deletable once vision + opencode tool-calling are smoke-tested.
|
||
# (QAT quality upgrade = separate decision: main → qat-UD-Q4_K_XL, +0.7 GB, moe 13→~16, and
|
||
# a section/client id rename round. Not part of this fix.)
|
||
# ⚠ vision + MTP together is heavy — if it OOMs/freezes, add n-cpu-moe or drop MTP (keep vision).
|
||
model = /home/anon/software/models/gemma-4-26B-A4B-it-UD-IQ4_XS.gguf
|
||
mmproj = /home/anon/software/models/mmproj-gemma-4-26B-A4B-it-F16.gguf
|
||
model-draft = /home/anon/software/models/mtp-gemma-4-26B-A4B-it.gguf
|
||
spec-type = draft-mtp
|
||
spec-draft-n-max = 2
|
||
reasoning-budget = 0 # added 2026-08-09: the Jul-17 official template makes gemma-4 DELIBERATE
|
||
# on hard prompts (probe: 700 tok = 100% reasoning_content, content
|
||
# EMPTY, finish=length) — that ate pi's 4096 maxTokens cap and returned
|
||
# truncated nothing. Same fix as the 35B daily driver. ⚠ if startup says
|
||
# option not recognized, remove; if it loads but a hard probe still emits
|
||
# reasoning_content, the gemma template ignores the budget → remove and
|
||
# raise client maxTokens to 8192 instead.
|
||
ctx-size = 24576 # MEASURED @16384/moe16: 29.6 t/s, 10.7 used / 6.5 free → sliding-window KV is
|
||
# cheap, take the ctx back (clients already assume 24576)
|
||
# n_ctx_train = 262144 (08-08 load log) — huge ctx headroom exists if a
|
||
# long-context vision need ever appears; bench stepwise, free ≥ 2.5.
|
||
n-gpu-layers = 99
|
||
n-cpu-moe = 13 # MEASURED @16: 26.4 t/s, 11.9 used / 5.2 free → pull ~3 expert layers onto GPU for speed
|
||
threads = 6
|
||
flash-attn = on
|
||
jinja = on # OpenAI-style tool calling (opencode build/plan default to this preset)
|
||
cache-type-k = q8_0
|
||
cache-type-v = q4_0
|
||
batch-size = 2048
|
||
ubatch-size = 512
|
||
cache-reuse = 256
|
||
defrag-thold = 0.1
|
||
no-mmproj-offload = true # vision encoder on CPU → frees ~1 GB
|
||
temp = 1.0
|
||
top-p = 0.95
|
||
top-k = 64
|
||
sleep-idle-seconds = 60
|
||
|
||
[gemma-4-E4B-it-UD-Q8_K_XL]
|
||
# ~5-7 GB — small/fast, lots of headroom, safe at big ctx.
|
||
# Vision: the HF repo ships mmproj-{BF16,F16,F32}.gguf — use F16 (990 MB), saved AS the exact
|
||
# path below (rename it; if the path doesn't exist the model fails to load / vision silently
|
||
# missing depending on router). Verify: startup log shows the mmproj load, and an image request
|
||
# answers instead of erroring "image input is not supported".
|
||
# NOTE: repo got Google's chat-template update ~2026-05 — if the main gguf predates that, re-download it too.
|
||
model = /home/anon/software/models/gemma-4-E4B-it-UD-Q8_K_XL.gguf
|
||
mmproj = /home/anon/software/models/mmproj-gemma-4-E4B-it.gguf
|
||
model-draft = /home/anon/software/models/mtp-gemma-4-E4B-it.gguf # ✓ MEASURED (2026-07-18): 57.3 t/s —
|
||
# +50% over 38.4 without the draft, 10.5 used / 6.6 free. Fastest model on the box.
|
||
spec-type = draft-mtp
|
||
spec-draft-n-max = 2
|
||
reasoning-budget = 0 # added 2026-08-09, same reason as the 26B: gemma-4 deliberates on hard
|
||
# prompts since the Jul-17 template (this is what caused the 20-second
|
||
# aichat -e "think" preambles on 2026-08-05). Fast models answer NOW.
|
||
# ⚠ same caveats as the 26B — remove if not recognized / not honored.
|
||
ctx-size = 98304 # ⚗→✓ CLOSED 2026-08-08: 58.2-60.4 t/s, 10.3-10.6 used / 6.5-6.9 free
|
||
# @98304, and n_ctx_train = 131072 confirmed (upstream config.json
|
||
# max_position_embeddings) — 98304 is inside training range, no RoPE
|
||
# risk. 131072 is the hard ceiling if ever needed (~+0.5-1 GB KV).
|
||
n-gpu-layers = 99
|
||
flash-attn = on
|
||
jinja = on # OpenAI-style tool calling (doc-writer agent defaults to this preset)
|
||
cache-type-k = q8_0
|
||
cache-type-v = q8_0
|
||
batch-size = 2048
|
||
ubatch-size = 1024
|
||
cache-reuse = 256
|
||
defrag-thold = 0.1
|
||
temp = 1.0
|
||
top-p = 0.95
|
||
top-k = 64
|
||
sleep-idle-seconds = 30
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# GLM
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
[GLM-4.7-Flash-UD-Q4_K_XL]
|
||
# 30B-A3B MoE, primary coder. Upgraded UD-Q3_K_XL (~14 GB) → UD-Q4_K_XL (17.5 GB, unsloth's
|
||
# recommended quant) 2026-07-16, benched clean same day → the old GLM-4.7-Flash-UD-Q3_K_XL.gguf
|
||
# can be deleted.
|
||
# Tuning trajectory: Q3@moe9 = 28.2 t/s → Q4@moe24 = 17.9 (measured 2026-07-16) → now moe18,
|
||
# lower stepwise toward 16 while bench keeps free ≥2.5 GB.
|
||
model = /home/anon/software/models/GLM-4.7-Flash-UD-Q4_K_XL.gguf
|
||
ctx-size = 24576 # MEASURED @16384/moe24 (Q3): 9.5 GB used, 7.6 free
|
||
# n_ctx_train = 202752 (08-08 load log) — MLA KV is compact, ctx could
|
||
# go far higher if opencode subagent sessions ever outgrow 24k.
|
||
n-gpu-layers = 99
|
||
n-cpu-moe = 22 # lowered 24→22 on 2026-08-08: both step-down gates passed post-KV-fix —
|
||
# clean sweep @24 = 20.5 t/s, 12.0 used / 5.2 free (02:45, 1.9 baseline);
|
||
# elevated-baseline load @24 = 18.7, 13.0/4.2 (@2.9). moe22 est ~21.5
|
||
# clean / ~19.5-20 evening, ~12.7 used clean / ~14.0 evening → inside the
|
||
# coder-proven evening envelope (14.2 loads). ✓ CLEAN VERIFIED (02:58,
|
||
# 2.0 baseline): 21.5 t/s, 12.8 used / 4.4 free — est was 21.5, exact.
|
||
# Evening observation pending; revert to 24 if an evening bench shows
|
||
# free < 2.5. moe20 stays gated (~14.4-14.7 evening = borderline vs the
|
||
# coder's 15.2 evening death).
|
||
# (08-05/07 "failures" were the mixed-KV rejection below, not VRAM.)
|
||
# Trajectory: Q3@moe9 = 28.2 → Q4@moe24 = 17.9 (07-16) → 18.7/20.5 post-KV-fix.
|
||
threads = 6
|
||
flash-attn = on
|
||
cache-type-k = q8_0
|
||
cache-type-v = q8_0 # was q4_0 — BROKE 2026-08-08: a llama.cpp upgrade (the one deprecating
|
||
# defrag-thold) enforces same-type K/V for this model (deepseek2/MLA
|
||
# conversion; MLA KV is compact, mixed types unsupported). Every load
|
||
# since failed at context creation regardless of VRAM. Only THIS preset
|
||
# was affected — the other k-q8/v-q4 presets are regular-attention archs.
|
||
# If q8/q8 still refuses: delete both cache-type lines (f16 default; MLA
|
||
# KV is small enough that the cost is minor at 24576 ctx).
|
||
batch-size = 2048
|
||
ubatch-size = 512
|
||
cache-reuse = 256
|
||
defrag-thold = 0.1
|
||
# MTP: GLM-4.7-Flash HAS an MTP head upstream (num_nextn_predict_layers: 1) but the GGUF
|
||
# conversion (deepseek2 arch) drops those tensors and llama.cpp draft-mtp only supports
|
||
# Qwen3.5/3.6 + Gemma 4 so far (PRs #22673/#23398). No mtp-*.gguf exists to download —
|
||
# recheck when llama.cpp lands GLM MTP support.
|
||
temp = 0.6 # unsloth rec: 0.7/top-p 1.0 (tool-calling) or 1.0/0.95 (general); no repeat-penalty
|
||
top-p = 0.95
|
||
min-p = 0.01
|
||
jinja = on
|
||
reasoning-preserve = true # ⚗ option ACCEPTED by the build (loaded + ran 20.9 t/s, 04:11 sweep).
|
||
# BEHAVIOR flag — judge multi-turn subagent quality in opencode over the
|
||
# next days; remove if reasoning bleed ever pollutes final answers.
|
||
sleep-idle-seconds = 60
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Utility & candidates
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
[Qwen3-Embedding-0.6B]
|
||
# Embeddings-only, for RAG/semantic search (OWUI etc. via /v1/embeddings) — NOT a chat model,
|
||
# so it is deliberately absent from the opencode/pi/aichat chat lists.
|
||
# Qwen3-Embedding requires last-token pooling. ~0.7 GB — tiny, loads fast.
|
||
# (Q8_0 is effectively lossless for a 0.6B embedder — the f16 file is redundant, delete it.)
|
||
# CPU-ONLY on purpose: a 0.6B embeds fast on 6 cores, and 0 VRAM means it can never stack
|
||
# with a chat model into an overcommit (and never suffers/causes eviction pressure).
|
||
model = /home/anon/software/models/Qwen3-Embedding-0.6B-Q8_0.gguf
|
||
embeddings = true
|
||
pooling = last
|
||
ctx-size = 8192
|
||
n-gpu-layers = 0
|
||
flash-attn = on
|
||
batch-size = 8192 # embed long chunks in one pass
|
||
ubatch-size = 1024
|
||
threads = 6
|
||
sleep-idle-seconds = 60
|
||
|
||
[gpt-oss-20b]
|
||
# OpenAI gpt-oss-20b — MoE 21B / 3.6B active, reasoning + tool use (enabled 2026-07-16, NOT
|
||
# BENCHED yet). OpenAI post-trained it in MXFP4, so there is exactly ONE sensible GGUF:
|
||
# ggml-org/gpt-oss-20b-GGUF → gpt-oss-20b-mxfp4.gguf (~12.1 GB). Requants are never better —
|
||
# the old gpt-oss-20b-Q3_K_M.gguf can be deleted.
|
||
# Harmony chat template → jinja required. Default KV types kept (gpt-oss attention sinks have
|
||
# had issues with quantized KV — verify before adding cache-type lines).
|
||
model = /home/anon/software/models/gpt-oss-20b-mxfp4.gguf
|
||
ctx-size = 65536 # ⚗→✓ VERIFIED @65536 (04:11): 37.3/37.4 t/s unchanged, 14.0 used / 3.2
|
||
# free — the fast long-context option is real (37 t/s vs Coder-Next 16).
|
||
# 131072 is OFF the table: 32k→65k cost +1.2 GB (KV slope steepens past
|
||
# the sliding window) → 131k would land under 2.5 free. Client caps
|
||
# raised 32k→64k on 2026-08-08.
|
||
n-gpu-layers = 99
|
||
n-cpu-moe = 4 # ✓ SETTLED. MEASURED @6: 31.3 t/s · @4 (2026-07-16): 38.1 t/s, 12.7 used /
|
||
# 4.5 free — fastest model in the lineup. TTFT (~3-7 s) is the model THINKING
|
||
# (harmony reasoning streams as reasoning_content first) — not a slow load.
|
||
threads = 6
|
||
flash-attn = on
|
||
batch-size = 2048
|
||
ubatch-size = 512
|
||
cache-reuse = 256
|
||
defrag-thold = 0.1
|
||
temp = 1.0
|
||
top-p = 1.0
|
||
jinja = on
|
||
sleep-idle-seconds = 120
|
||
|
||
[gpt-oss-20b-low]
|
||
# Same GGUF, reasoning effort LOW — zero disk cost (like the 35B-Thinking alias, inverted).
|
||
# Cuts the multi-second thinking preamble for light/interactive use; keep [gpt-oss-20b]
|
||
# (default = medium effort) for hard problems. ⚠ if startup says the option is not
|
||
# recognized, the llama.cpp build predates chat-template-kwargs — comment that line out.
|
||
model = /home/anon/software/models/gpt-oss-20b-mxfp4.gguf
|
||
chat-template-kwargs = {"reasoning_effort":"low"}
|
||
ctx-size = 65536 # ⚗→✓ VERIFIED @65536 (04:11): 37.3/37.4 t/s unchanged, 14.0 used / 3.2
|
||
# free — the fast long-context option is real (37 t/s vs Coder-Next 16).
|
||
# 131072 is OFF the table: 32k→65k cost +1.2 GB (KV slope steepens past
|
||
# the sliding window) → 131k would land under 2.5 free. Client caps
|
||
# raised 32k→64k on 2026-08-08.
|
||
n-gpu-layers = 99
|
||
n-cpu-moe = 4
|
||
threads = 6
|
||
flash-attn = on
|
||
batch-size = 2048
|
||
ubatch-size = 512
|
||
cache-reuse = 256
|
||
defrag-thold = 0.1
|
||
temp = 1.0
|
||
top-p = 1.0
|
||
jinja = on
|
||
sleep-idle-seconds = 60
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Notes
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# n-cpu-moe N : offload the first N MoE layers' experts to CPU. Higher = less VRAM (safer) but
|
||
# slower. On this display-driving GPU, a bit of offload is FAR better than a freeze.
|
||
# Only applies to MoE models (dense models ignore it — reduce ctx instead).
|
||
# override-tensor : finer control, e.g. `blk\.(1[2-9]|[2-9][0-9])\.ffn_.*_exps=CPU`.
|
||
# After ANY change: reload the server and run bench.py — keep every "VRAM free" ≥ ~2.5 GB.
|