coja/dots
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Update] bulk update

Browse Source
This commit is contained in:
coja
2026-09-17 20:45:06 +02:00
committed by Coja
parent 3ec2503f38
commit ed7e34552d
158 changed files with 3258 additions and 2419 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fl/.config/llamacpp/config.ini
+138 -2
View File
@@ -1,7 +1,8 @@
# llama.cpp model config — AMD RX 7600 XT (ROCm) · 16 GB (17.16 GB total)
# llama.cpp model config — AMD RX 7600 XT · 16 GB (17.16 GB total) · backend = Arch llama.cpp-vulkan
# (RADV, device Vulkan0 — NOT ROCm/HIP; rocm-smi is used for monitoring only)
# ═════════════════════════════════════════════════════════════════════════════
# ⚠⚠ VRAM SAFETY — READ THIS. This GPU ALSO DRIVES THE DISPLAY. If a model asks for
# more VRAM than is free, ROCm does NOT OOM cleanly — it spills into system RAM
# more VRAM than is free, the amdgpu driver does NOT OOM cleanly — it spills into system RAM
# (GTT), which starves the desktop and FREEZES THE WHOLE PC (reboot required).
# → TARGET ≥ ~2.5 GB free (≤ ~14.5 GB used). HARD FLOOR 1.5 GB (bench.py guard stops there).
# Idle desktop uses ~1.3 GB of 17.16 — but a browser/leftover model can hold 1-3 GB more,
@@ -223,6 +224,141 @@ repeat-penalty = 1.05
jinja = on
sleep-idle-seconds = 60
[Qwen3.8-27B-UD-IQ3_XXS]
# ⚠ CLEAN-DESKTOP SPECIALTY — KEPT by user decision 2026-09-10 despite failing the ≥22 t/s speed
# gate (14.5 t/s MTP / 10.9 base; 2.8 GB free @1.5 baseline @16k). Pick it ONLY on a quiet
# desktop: at a 3+ GB evening baseline it lands ~1 GB free = GTT-spill/freeze territory. Role:
# slow-but-smart dense-27B answers + vision. Origin (added 2026-08-16): dense 27B, but NOT like the two
# retired dense 27Bs: hybrid attention (only 16 of 64 layers full-attn, rest linear/SSM →
# cheap KV, Coder-Next-style) AND an embedded MTP head — VERIFIED in the gguf header:
# qwen35.nextn_predict_layers = 1, block_count = 65 (64+1), arch "qwen35" = the same
# draft-mtp path the working 9B preset uses. Hybrid reasoner (thinking ON by default,
# reasoning_effort tunable via chat-template-kwargs if a -low alias is ever wanted),
# vision, n_ctx_train = 262144.
# Speed math: dense 11.9 GB weights on ~288 GB/s ≈ 14-16 t/s base; MTP should lift to
# ~22-30 = roster-competitive IF the draft engages. If it doesn't, this lands in
# retired-dense-27B territory (~10-14) → retire again, quality won't pay for the speed.
# ⚠ BENCH HISTORY — still no green row, decode/MTP UNMEASURED after 5 attempts:
# · 08-16 14:59/15:19/15:29 (3.3-3.5 DIRTY baseline): 16.5-16.7 used / 0.4-0.7 free, guard
# skipped gen every time. Implied footprint 13.2 GB — MISLEADING, see below.
# · 08-17 10:31: ERROR conn-refused = bench raced llama.service coming up after boot.
# · 08-17 10:38 (1.9 near-clean baseline): 15.9 used / 1.3 free — guard floor missed by
# 0.2. TRUE footprint = 14.0 GB over baseline; the dirty runs looked 0.8 smaller because
# ROCm silently spilled that much to GTT (the exact freeze precursor the guards watch).
# · 08-17 10:43 (post-trim, 10.0 leftover start, partial drain): 15.8 / 1.4 — floor missed
# by 0.1. Trim verified live via router args; ~2.4 effective baseline poisoned it again.
# · 08-17 10:53 (1.9 baseline, NO leftovers — a genuine desktop-idle run): 15.8 / 1.4,
# floor missed by 0.1 again. DECISIVE: trimmed footprint 13.9 vs untrimmed 14.0 — both
# knobs together saved 0.1 GB. ctx/ubatch do NOT move this model (fixed SSM-state +
# big-vocab buffers). Absolute best case ≈ 2.0 free at a pristine 1.3 baseline @16k —
# permanently below the 2.5 target. → RETIREMENT RECOMMENDED 2026-08-17; decode/MTP
# never measured, academic at these margins. Third dense ~27B to die on this card,
# first one killed by fixed buffers rather than bandwidth.
# Consequences: full-ctx even on a 1.3 CLEAN night = ~15.3 used / 1.9 free (runs, but
# under the 2.5 target). EVENING regime can never fit this quant — that needs ≤ UD-IQ2_M,
# a quality gutting. → 08-17: TRIMMED FOR THE GATE BENCH: ctx 24576→16384 (~-0.25) +
# ubatch 512→256 (~-0.3-0.5) → predicted ~15.2 used / ~2.0 free @1.9 baseline. Clients
# still say 24576 — deliberately NOT re-wired for a temporary trim; do not use from
# clients anyway until this passes (a busy-desktop pick is one tab from the GTT freeze).
# DECISION GATE (superseded by the 09-10 user decision below): MTP ≥ ~22 t/s keeps it (clean-desktop specialty, restore ctx
# 24576 + re-check), anything less → retire + delete (dense 27Bs die on this card).
# ↻ REOPENED 2026-09-09 — the file on disk is STALE: unsloth re-quantized the whole repo as
# "Dynamic v3" on 2026-08-19, three days AFTER the 08-16 download (~10% better accuracy at
# equal size, per unsloth). Verified via the HF tree API at the pre/post commits:
# UD-IQ3_XXS 11.91 → 10.93 GB (SAME filename — re-download overwrites in place, no client
# or section changes); UD-Q2_K_XL 10.68 → 9.83; UD-IQ2_M DELETED (gone as an option);
# UD-IQ2_S 8.37 / UD-IQ2_XXS 7.27 are new 2-bit tiers. Header range-read 09-09: v3
# IQ3_XXS and Q2_K_XL both still carry nextn_predict_layers = 1 (MTP head embedded);
# unsloth stripped the head from ≤ IQ2_S only and ships MTP/mtp-Qwen3.8-27B-Q4_0.gguf
# as a separate model-draft for those (gemma-4 style).
# Predicted with v3 IQ3_XXS (footprint 14.0 − 0.98 ≈ 13.0; bench.py GB = decimal):
# @1.9 real-idle baseline ≈ 14.8–15.0 used / 2.2–2.4 free · @1.3 pristine ≈ 14.2 / ~2.9.
# Clears the 1.5 floor by ~0.8 even at 1.9 → the gen bench can FINALLY run; still ~0.1–0.3
# under the 2.5 target on a real idle desktop, clears it outright on a pristine one. Overhead datum: a third-party paired bench measured
# draft-mtp at ~+0.75 GB on this model (unsloth docs: "1-2 GB extra headroom") — a good
# part of the "fixed" ~2.1 GB is MTP + KV, not only SSM-state/vocab buffers.
# NEXT STEP (recommended, NOT yet done): re-download v3 IQ3_XXS on fl (verify 10.93 GB /
# 10.18 GiB before restarting), dotsync, restart llama.service, clean-baseline bench.
# Then the gate above decides — with one refinement: free 2.0–2.5 at ≥22 t/s = user's call
# between "clean-desktop specialty" and stepping down to v3 UD-Q2_K_XL (predicted ~13.7 /
# 3.5 @1.9 — needs the section-rename round across the six client files; sequence it
# after any in-flight client edits). Decode < 22 → retire regardless of VRAM.
# No in-family alternative: the Qwen3.8 lineup is 27B dense, Flash-Next 125B-A6B (smallest
# GGUF 72.5 GB, 75 GB RAM floor even with mmap'd n-gram tables — out on 48+16),
# 2.4T-A95B and Max. No 3.8 MoE in the 35B-A3B class exists; watch HF.
# ✦ GATE BENCH RAN 2026-09-10 02:36 (v3 file, sha verified; 2.1 baseline): 14.9 used / 2.2 free
# (footprint 12.8 — VRAM half of the gate PASSED, borderline) but decode 14.8 t/s, prefill 19,
# TTFT 11.7 s → speed half FAILED. Server log: MTP head loaded, draft acceptance 0.80
# (157/195, mean len 2.6) — so MTP IS engaging; at 2.6 tok/step that is ~175 ms per step ≈
# 3–4× a plain forward pass. NOTE the box runs the Arch llama.cpp-vulkan package (RADV,
# device Vulkan0) — NOT HIP/ROCm (this header said "ROCm" until 09-10); rocm-smi = monitoring only.
# Suspects: the 48 GatedDeltaNet layers on the Vulkan backend (upstream #20354: missing/slow
# GDN shader → CPU fallback) and/or recurrent-state checkpoint/restore per speculative step
# (~150 MiB for this model; reported net-negative on hybrid 27Bs). The hybrid 35B-A3B does
# 37 t/s on the same backend, so it is size-specific, not Vulkan-generic.
# A/B NEXT: [Qwen3.8-27B-UD-IQ3_XXS-nomtp] (TEMP alias, used 03:19 then removed) measures
# base decode; base ≥ 18 → MTP is the net negative here, run without it (frees ~0.75 GB);
# base ≈ 9 → the GDN path is the wall → retire. Also read "graph splits" from the load log.
# cache_reuse is inert here ("not supported by multimodal") — true for every mmproj preset.
# Re-run 03:07 @0.9 PRISTINE baseline: 15.0 t/s / 17 pp / 13.9 used / 3.3 free (footprint 13.0)
# — reproducible; the VRAM half is now fully closed, only the speed half is open.
# ✦ A/B 2026-09-10 03:19 (@1.5 baseline, same run): MTP 14.5 t/s / 30 pp / 14.3 used / 2.8 free
# vs NO-MTP 10.9 t/s / 34 pp / 12.8 used / 4.4 free. → MTP WORKS (+33%, matches the sudoingX
# paired benches) and costs 1.5 GB VRAM; the wall is the BASE: 10.9 t/s = 119 GB/s effective
# (41% of the card's 288) — the 48 GatedDeltaNet layers on the Vulkan/RADV backend, exactly
# the #20354 datum (Qwen3.5-27B ≈ 11.8 t/s on Strix Halo, either backend). Prefill 30-34 on
# a fully-GPU model is the same wall (the hybrid 35B-A3B daily driver's 64 pp is GDN-bound
# too). "graphs reused" 97-260 in the log → not a graph-rebuild problem. Nothing in this
# preset can move it; a bigger K-quant (faster dequant on Vulkan) does not fit. Perfect-
# kernel ceiling on this card ≈ 20 base / ~27 MTP — an upstream Vulkan gated_delta_net
# shader improvement COULD flip it; nothing else can.
# VERDICT 2026-09-10: gate FAILED on speed (14.5 < 22) with the VRAM half passed (2.8 free
# @1.5). Per the gate → retire. Re-probe only after a llama.cpp-vulkan bump whose changelog
# touches gated_delta_net / Vulkan GDN: `./bench.py -m Qwen3.8-27B-UD-IQ3_XXS`.
# → USER DECISION 2026-09-10: KEEP as clean-desktop specialty (overrides the gate). ctx restored
# to 24576 (clients already say 24576), ubatch stays 256 (prefill is GDN-bound, the bigger
# buffer buys nothing), MTP stays (+33% for 1.5 GB). ✓ CONFIRMED @24576 2026-09-10 03:51
# (1.6 baseline): 15.1 t/s / 31 pp / 14.6 used / 2.5 free — footprint 13.0, exactly as
# predicted; free = the 2.5 target at a normal idle, ~2.8 pristine, ~0.8 on a 3.3 evening.
# ⚠ Needs a CURRENT llama.cpp build: first "qwen35" file HERE with ssm.*/hybrid-attn keys
# (the 9B is regular-attention qwen35). If the load fails with unknown-architecture /
# missing-tensor errors, update llama.cpp on fl before touching this preset.
# ⬇ DOWNLOAD from unsloth/Qwen3.8-27B-GGUF (NO separate MTP repo — unlike Qwen3.5/3.6, the
# head ships in the MAIN repo's ggufs for ≥ Q2_K_XL; ≤ IQ2_S need MTP/mtp-Qwen3.8-27B-Q4_0.gguf):
# Qwen3.8-27B-UD-IQ3_XXS.gguf (10.93 GB Dynamic v3 since 2026-08-19 — the 08-16 download
# is the 11.91 GB v2; RE-FETCH. UD-IQ4_XS 14.3 GB and
# UD-Q3_K_XL 13.15 = dead on arrival, no n-cpu-moe escape)
# mmproj-Qwen3.8-27B-F16.gguf ← mmproj-F16.gguf (928 MB — unchanged; rename; watch trailing spaces)
model = /home/anon/software/models/Qwen3.8-27B-UD-IQ3_XXS.gguf
mmproj = /home/anon/software/models/mmproj-Qwen3.8-27B-F16.gguf
spec-type = draft-mtp
spec-draft-n-max = 2 # ⚠ if the load log says this file has no MTP head, comment the two
# spec- lines out — and expect ~14 t/s, probably not worth keeping.
ctx-size = 24576 # RESTORED 2026-09-10 (16384 was the 08-17 gate-bench trim): matches
# the clients. KV is cheap (16 full-attn layers × 4 kv-heads × 256
# head_dim ≈ 26 KB/token at q8/q4): 16k→24k cost +0.3 → MEASURED
# 09-10 @1.6 baseline: 14.6 used / 2.5 free. n_ctx_train 262144.
n-gpu-layers = 99 # dense: no n-cpu-moe escape — ctx/quant are the only knobs
threads = 6
flash-attn = on
cache-type-k = q8_0
cache-type-v = q4_0 # hybrid arch takes mixed KV like Coder-Next (GLM's same-type
# rejection was MLA-specific). If load fails at context creation:
# try v = q8_0, then delete both lines (f16 — KV small enough here).
batch-size = 2048
ubatch-size = 256 # KEPT at 256 (512 before the 08-17 trim): prefill is GDN-bound on the
# Vulkan backend (30-34 t/s either way), so the bigger compute buffer
# buys nothing measurable and VRAM is the binding constraint here.
cache-reuse = 256
defrag-thold = 0.1
no-mmproj-offload = true # vision encoder on CPU → ~1 GB freed, same as the 26B
temp = 1.0 # card: thinking mode 1.0/0.95/20/0 — thinking IS the default mode
top-p = 0.95
top-k = 20
min-p = 0
jinja = on
sleep-idle-seconds = 120
# ─── RETIRED 2026-07-16: [Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-IQ3_M] ──────────────
# Dense: 10 t/s, 2.0 GB free (under the 2.5 target) and unfixable — bandwidth-bound, smaller
# quant would cost the quality that justified it. Removed from opencode/pi/aichat.