diff --git a/common/.config/claude/StatusBar.md b/common/.config/claude/StatusBar.md index 4e0420c..228f6d9 100644 --- a/common/.config/claude/StatusBar.md +++ b/common/.config/claude/StatusBar.md @@ -8,7 +8,7 @@ resize), trailing (lowest-priority) segments are dropped from the right until it Wide โ€” everything: ``` -๐Ÿ“ ~/.dots โ”‚ ๐ŸŒฟ .dots main* โ”‚ ๐Ÿค– Fable 5 (1M) ยท xhigh โ”‚ ๐Ÿ“ 84% (843k) โ”‚ ๐Ÿ“Š 34% 2h54m โ”‚ ๐Ÿ’ฐ $0.40 $12.00/h โ”‚ โ™ป๏ธ 91% 1h โ”‚ ๐Ÿ’พ 81% +๐Ÿ“ ~/.dots โ”‚ ๐ŸŒฟ .dots main* โ”‚ ๐Ÿค– Fable 5 (1M) ยท xhigh โ”‚ ๐Ÿ“ 84% (843k) โ”‚ ๐Ÿ“Š 34% 2h54m โ”‚ ๐Ÿ’ฐ $0.40 $12.00/h โ”‚ ๐Ÿ”ข โ†‘63M โ†“595k โ”‚ โ™ป๏ธ 91% 1h โ”‚ ๐Ÿ’พ 81% ``` Narrow โ€” trailing segments trimmed: @@ -32,6 +32,7 @@ is supplementary to the main colored value. | ๐Ÿ“ | **Context** | `% of context window used` + `(Nk)` tokens | usage comes from the payload's `context_window` (`used_percentage` + `current_usage`, Claude Code 2.1.x); the transcript's last `message.usage` is only a fallback. Window from `context_window.context_window_size`, then `(1M โ€ฆ)` in the display name, else 200k โ€” **not** from `model.id`, which arrives with any `[1m]` suffix already stripped. Adds a red **โš compact** at โ‰ฅ80% | | ๐Ÿ“Š | **5h usage** | `% of the 5-hour rolling limit used` + time until it resets | from `rate_limits.five_hour`; Pro/Max only, and absent until the first API response of a session | | ๐Ÿ’ฐ | **Cost** | `$` session cost so far + `$/h` burn rate | burn rate shown once the session exceeds ~30s | +| ๐Ÿ”ข | **Session tokens** | `โ†‘` input + `โ†“` output tokens used so far this session (cumulative, `k`/`M`) | summed over the transcript's assistant `message.usage` blocks, de-duplicated by message id (one line per content block shares the same usage). Input = `input + cache_creation + cache_read` โ€” everything billed as input, so on a 1M session it dwarfs output (every call re-reads the cached prefix). Main transcript only: subagent usage lives in separate sidechain files. Incremental: a per-session state file `~/.claude/cache/statusline.tokens..json` remembers the byte offset already summed, so each refresh parses only the new tail (cold scan of a 6.5 MB transcript โ‰ˆ 70 ms, warm โ‰ˆ 0.1 ms); stale state files are pruned after 7 days once 50+ pile up. Omitted when there is no transcript; `โ†‘` cyan, `โ†“` green, no thresholds | | โ™ป๏ธ | **Prompt cache** | hit `%` (higher is better) + cache TTL (`5m` / `1h`), yellow `ยทcold` once it has expired | from the payload's `prompt_cache` (2.1.260+: `hit_ratio` 0โ€“1, `ttl`, `warm`); computed from usage on older payloads. A `1h` TTL is what keeps a 10-min `/pr-loop` cadence warm | | ๐Ÿ’พ | **Disk** | `% used` of the filesystem at the cwd | from `statvfs` | diff --git a/common/.config/claude/keybindings.json b/common/.config/claude/keybindings.json index d867671..ddc88f3 100644 --- a/common/.config/claude/keybindings.json +++ b/common/.config/claude/keybindings.json @@ -5,15 +5,21 @@ { "context": "Chat", "bindings": { - "shift+enter": "chat:newline" + "shift+enter": "chat:newline", + "ctrl+x v": "app:toggleTranscript" + } + }, + { + "context": "Transcript", + "bindings": { + "ctrl+x v": "app:toggleTranscript" } }, { "context": "Global", "bindings": { "alt+j": "scroll:lineDown", - "alt+k": "scroll:lineUp", - "ctrl+x v": "app:toggleTranscript" + "alt+k": "scroll:lineUp" } }, { diff --git a/common/.config/claude/statusline.py b/common/.config/claude/statusline.py index 5b4072a..b8ac001 100755 --- a/common/.config/claude/statusline.py +++ b/common/.config/claude/statusline.py @@ -4,7 +4,7 @@ One line: vim mode | dir (repo-relative, hidden at the root) | git(repo/branch +dirty +ahead/behind) | model (+effort) | context% (+compact warn) | 5h usage (+reset eta) | cost (+burn rate) | -prompt-cache hit% (+ttl, cold marker) | disk. If the +session tokens (cumulative in/out) | prompt-cache hit% (+ttl, cold marker) | disk. If the rendered line would overflow the terminal, trailing (lowest-priority) segments are dropped until it fits. (Dormant, re-addable in main(): ram, cpu%, temp -- retired 2026-07-26 because the @@ -156,6 +156,133 @@ def last_usage(data): return {} +# --- session token totals (transcript, incremental) --------------------------- +TOKENS_STATE_DIR = os.path.expanduser("~/.claude/cache") +TOKENS_RECENT_IDS = 20 # message ids remembered across runs for de-duplication +TOKENS_STATE_MAX_AGE = 7 * 86400 # prune per-session state files older than this + + +def fmt_tokens(n): + """1234 -> 1.2k, 843000 -> 843k, 1_234_567 -> 1.2M, 12_345_678 -> 12M. + + Thresholds sit at the rounding boundaries (0.9995, 9.95) so a value never prints as + "10.0k" or "1000k" -- it rolls into the shorter form / the next unit instead.""" + n = float(n) + for unit, div in (("M", 1e6), ("k", 1e3)): + v = n / div + if v < 0.9995: + continue + return f"{v:.0f}{unit}" if v >= 9.95 else f"{v:.1f}{unit}" + return f"{n:.0f}" + + +def _tokens_state_path(data, path): + sid = data.get("session_id") or os.path.splitext(os.path.basename(path))[0] + sid = re.sub(r"[^A-Za-z0-9_-]", "_", str(sid))[:80] + return os.path.join(TOKENS_STATE_DIR, f"statusline.tokens.{sid}.json") + + +def _prune_tokens_state(keep): + """Drop stale per-session state files (one scandir, only when the dir has piled up).""" + try: + entries = [e for e in os.scandir(TOKENS_STATE_DIR) if e.name.startswith("statusline.tokens.")] + if len(entries) < 50: + return + cutoff = time.time() - TOKENS_STATE_MAX_AGE + for e in entries: + if e.path != keep and e.stat().st_mtime < cutoff: + os.unlink(e.path) + except Exception: + pass + + +def session_tokens(data): + """Cumulative (input, output) tokens this session, summed over the transcript's + assistant `message.usage` blocks -- or None when there is no transcript. + + Input = input + cache_creation + cache_read (everything the API billed as input), so + on a 1M session it dwarfs output: every call re-reads the whole cached prefix. + Only the main transcript is counted; subagent usage lives in separate sidechain files. + + Incremental: the status line runs on every refresh and transcripts grow to tens of + MB, so a per-session state file (~/.claude/cache/statusline.tokens..json) + remembers the byte offset already summed and only the new tail is parsed. Lines are + de-duplicated by message id: the transcript writes one line per content block of a + turn, all carrying the same id and the same usage block. Only complete lines are + consumed (the writer may be mid-line); a shrunken file (truncate/rotate) resets. + """ + path = data.get("transcript_path") + if not path or not os.path.exists(path): + return None + state_path = _tokens_state_path(data, path) + state = {"path": path, "offset": 0, "in": 0, "out": 0, "ids": []} + try: + with open(state_path) as fh: + s = json.load(fh) + if ( + isinstance(s, dict) + and s.get("path") == path + and all(isinstance(s.get(k), int) and s[k] >= 0 for k in ("offset", "in", "out")) + and isinstance(s.get("ids"), list) + ): + state = s + except Exception: + pass + + size = os.path.getsize(path) + if size < state["offset"]: # truncated or rotated under us: start over + state = {"path": path, "offset": 0, "in": 0, "out": 0, "ids": []} + if size == state["offset"]: + return state["in"], state["out"] + + with open(path, "rb") as fh: + fh.seek(state["offset"]) + chunk = fh.read() + end = chunk.rfind(b"\n") + if end < 0: # no complete new line yet + return state["in"], state["out"] + text = chunk[: end + 1].decode("utf-8", "ignore") + + tin, tout = state["in"], state["out"] + ids = [i for i in state["ids"] if isinstance(i, str)] + seen = set(ids) + for line in text.splitlines(): + if '"usage"' not in line: + continue + try: + obj = json.loads(line) + except Exception: + continue + msg = obj.get("message") if isinstance(obj, dict) else None + usage = msg.get("usage") if isinstance(msg, dict) else None + if not isinstance(usage, dict): + continue + mid = msg.get("id") or obj.get("requestId") or obj.get("uuid") + if not isinstance(mid, str) or mid in seen: + continue + seen.add(mid) + ids.append(mid) + + def num(k): + v = usage.get(k, 0) + return int(v) if isinstance(v, (int, float)) else 0 + + tin += num("input_tokens") + num("cache_creation_input_tokens") + num("cache_read_input_tokens") + tout += num("output_tokens") + + state = {"path": path, "offset": state["offset"] + end + 1, "in": tin, "out": tout, "ids": ids[-TOKENS_RECENT_IDS:]} + try: + os.makedirs(TOKENS_STATE_DIR, exist_ok=True) + tmp = f"{state_path}.{os.getpid()}.tmp" + with open(tmp, "w") as fh: + json.dump(state, fh) + os.replace(tmp, state_path) # atomic: a concurrent refresh never reads a half-written file + _prune_tokens_state(state_path) + except Exception: + pass + return tin, tout + + # --- row 1: work ------------------------------------------------------------ def vim_segment(data): """Vim editor mode as a compact [I]/[N]/[V] (only present with editorMode: vim). @@ -376,6 +503,20 @@ def velocity_segment(data): return "" +def tokens_segment(data): + """Cumulative session tokens: โ†‘ input (incl. cache reads/writes) and โ†“ output.""" + try: + totals = session_tokens(data) + if not totals: + return "" + tin, tout = totals + if not (tin or tout): + return "" + return "๐Ÿ”ข " + color(f"โ†‘{fmt_tokens(tin)}", "cyan") + " " + color(f"โ†“{fmt_tokens(tout)}", "green") + except Exception: + return "" + + def cache_segment(usage, data=None): """Prompt-cache hit % (higher is better) + the cache TTL, `ยทcold` when the cache has expired. Hit % from the 2.1.260+ payload `prompt_cache.hit_ratio` (a 0..1 fraction, null @@ -583,6 +724,7 @@ def main(): context_segment(data, usage), usage_segment(data), cost_segment(data), + tokens_segment(data), cache_segment(usage, data), ) if s] system = [s for s in ( diff --git a/common/.config/lazygit/config.yml b/common/.config/lazygit/config.yml index 21d6337..67ce84c 100644 --- a/common/.config/lazygit/config.yml +++ b/common/.config/lazygit/config.yml @@ -41,7 +41,7 @@ gui: # If true, wrap lines in the staging view to the width of the view. This # makes it much easier to work with diffs that have long lines, e.g. # paragraphs of markdown text. - wrapLinesInStagingView: true + wrapLinesInDiffView: true # One of 'auto' (default) | 'en' | 'zh-CN' | 'zh-TW' | 'pl' | 'nl' | 'ja' | 'ko' | 'ru' language: auto # Format used when displaying time e.g. commit time. diff --git a/common/.config/misc/clis.txt b/common/.config/misc/clis.txt index 1347b19..82f909f 100644 --- a/common/.config/misc/clis.txt +++ b/common/.config/misc/clis.txt @@ -30,7 +30,9 @@ pake bwrap dog nmap +rustscan traceroute fuck atool aria2c + diff --git a/common/.config/misc/tuis.txt b/common/.config/misc/tuis.txt index 822c4c0..3442dbe 100644 --- a/common/.config/misc/tuis.txt +++ b/common/.config/misc/tuis.txt @@ -25,6 +25,7 @@ gittype toofan pastel hexyl +binsider tmux zellij nvim @@ -61,7 +62,7 @@ bookokrat tennis llmfit termdraw -torlink +torlnk riptide quien eilmeldung @@ -73,3 +74,4 @@ netwatch gpk circumflex darya +usbtree diff --git a/common/.pi/README.md b/common/.pi/README.md index 95f6a35..2f36c04 100644 --- a/common/.pi/README.md +++ b/common/.pi/README.md @@ -175,7 +175,8 @@ Beyond pointing at local models, these values deviate from pi's defaults: | `compaction.keepRecentTokens` | `4000` | 20000 | keep less verbatim, to pay for the larger reserve and leave room after a compaction for the next one | | `npmCommand` | `fnm exec --using=22 -- npm` | `npm` | pin extension npm installs to Node 22 via fnm | | `retry` / `httpIdleTimeoutMs` | long (1h provider timeout, 10m idle) | shorter | local models can be slow to first token | -| `defaultThinkingLevel` | `off` | โ€” | most local models here are non-reasoning | +| `defaultThinkingLevel` | `medium` | โ€” | the default model is a thinking preset (`Qwen3.6-35B-A3B-Thinking`); Alt+T cycles it per session | +| `thinkingBudgets` | `512 / 1024 / 2048 / 3072` (minimal / low / medium / high) | 1024 / 2048 / 8192 / 16384 | per-request cap on the `` block, sent only for models whose `compat.thinkingTokenBudgetField` is set (`Ornith`, as `thinking_budget_tokens`). pi clamps the budget to `maxTokens โˆ’ 1024`, so on a 4096-token reply the stock values collapse to 3072 and leave the answer 1024 tokens โ€” the recipe for "Response was truncated before completion." | | `packages` | `["npm:pi-vim"]` | `[]` | the modal editor โ€” see the pi-vim section above | | `piVim` | mode colors, `clipboardMirror: "yank"`, `:` โ†’ pi bridge | โ€” | package config, not a pi setting; ignored if the package is removed | | `collapseChangelog` | `true` | `false` | condensed changelog after an upgrade instead of the full screen | diff --git a/common/.pi/agent/models.json b/common/.pi/agent/models.json index 6819163..dde1a9b 100644 --- a/common/.pi/agent/models.json +++ b/common/.pi/agent/models.json @@ -109,7 +109,7 @@ "id": "Ornith-1.5-35B-A3B-IQ3_XXS", "name": "Ornith 1.5 35B ยท 24k ยท vision โ€” agentic coder, thinking (~32 t/s)", "reasoning": true, - "compat": { "thinkingFormat": "qwen-chat-template" }, + "compat": { "thinkingFormat": "qwen-chat-template", "thinkingTokenBudgetField": "thinking_budget_tokens" }, "input": ["text", "image"], "contextWindow": 24576, "maxTokens": 4096, @@ -244,7 +244,7 @@ "id": "Ornith-1.5-35B-A3B-IQ3_XXS", "name": "Ornith 1.5 35B ยท 24k ยท vision โ€” agentic coder, thinking (~32 t/s)", "reasoning": true, - "compat": { "thinkingFormat": "qwen-chat-template" }, + "compat": { "thinkingFormat": "qwen-chat-template", "thinkingTokenBudgetField": "thinking_budget_tokens" }, "input": ["text", "image"], "contextWindow": 24576, "maxTokens": 4096, diff --git a/common/.pi/agent/settings.json b/common/.pi/agent/settings.json index 8374a9e..acf5d68 100644 --- a/common/.pi/agent/settings.json +++ b/common/.pi/agent/settings.json @@ -1,7 +1,13 @@ { - "defaultProvider": "duskadiy", - "defaultModel": "Qwen3.6-35B-A3B-Thinking", + "defaultProvider": "localcpp", + "defaultModel": "Ornith-1.5-35B-A3B-IQ3_XXS", "defaultThinkingLevel": "medium", + "thinkingBudgets": { + "minimal": 512, + "low": 1024, + "medium": 2048, + "high": 3072 + }, "theme": "catppuccin-mocha", "quietStartup": false, "collapseChangelog": true, @@ -58,7 +64,7 @@ "ex": "mode" } }, - "lastChangelogVersion": "0.85.1", + "lastChangelogVersion": "1.0.0", "packages": [ "npm:pi-vim" ] diff --git a/fl/.config/fish/host.fish b/fl/.config/fish/host.fish index aaf5454..60fe8aa 100644 --- a/fl/.config/fish/host.fish +++ b/fl/.config/fish/host.fish @@ -1,2 +1,11 @@ -# fl runs the llama.cpp router as a systemd service (llama.service) โ€” no shell abbr needed. +# fl runs the llama.cpp router as a systemd service (llama.service) โ€” no launch abbr needed. # Presets: ~/.config/llamacpp/config.ini; see ~/.config/llamacpp/README.md. + +## GPU monitors โ€” the RX 7600 XT also drives the display, so VRAM headroom is a safety matter +## (config.ini header: keep โ‰ฅ 2.5 GB free; GTT growth = VRAM spilling to RAM = the freeze precursor). +# vram : quick rocm-smi % view + who holds the render node. fuser prints its table on STDERR, +# so it must be 2>&1 โ€” with 2>/dev/null the "LLM GPU usage" section is always empty. +abbr vram 'watch -c -d -n 1 \'echo "=== VRAM ==="; rocm-smi --showmemuse --showuse 2>/dev/null | grep -E "GPU use|Allocated|Read/Write"; echo ""; echo "=== LLM GPU usage ==="; fuser -v /dev/dri/renderD128 2>&1 | grep -iE "COMMAND|llama"\'' +# vramx: absolute GiB used/free + GTT + temp straight from amdgpu sysfs (no ROCm), per-process VRAM +# via fdinfo, and which preset the router has resident. Body = gpumon.sh (runs bare too). +abbr vramx 'watch -c -d -n 1 ~/.config/llamacpp/gpumon.sh' diff --git a/fl/.config/llamacpp/README.md b/fl/.config/llamacpp/README.md index 1b3587a..63cd8f1 100644 --- a/fl/.config/llamacpp/README.md +++ b/fl/.config/llamacpp/README.md @@ -4,7 +4,7 @@ llama.cpp in **router mode** serving ~10 model presets from one 16 GB GPU. Last full tune: **2026-07-16** (decode numbers measured then via `bench.py`, unless a preset comment in `config.ini` says otherwise). Last full clean sweep: **2026-08-08 03:22** โ€” all 11 presets loaded and ran green (baseline 2.0 GB) after the week's coder-race/KV-fix/gemma-sidecar round. -- **Box**: AMD RX 7600 XT 16 GB (~288 GB/s; backend = Arch `llama.cpp-vulkan` / RADV, **not** ROCm โ€” +- **Box**: AMD RX 7600 XT 16 GB (~288 GB/s; backend = Arch `llama-cpp` 0.5.0 + `ggml-vulkan` / RADV = llama.cpp **b11146** since 2026-10-02, **not** ROCm โ€” `rocm-smi` is only the VRAM monitor) + Ryzen 5600X (6 cores) + 48 GB RAM. โš  The GPU **also drives the display** โ€” see [VRAM safety](#vram-safety-the-golden-rules). - **Endpoints**: LAN `http://$SERVER:11343/v1` ยท remote `https:///api/v1` @@ -35,6 +35,14 @@ symlinks, so on the server `dotsync` (pull; re-stows only if files were added/re `sudo systemctl restart llama.service` โ€” the router reads presets once at startup. After ANY preset change: restart + `./bench.py -m ` and check the guards. +**Upgrading llama.cpp itself** (2026-10-02, b10615 โ†’ b11146): Arch replaced `llama.cpp-vulkan` +with a unified `llama-cpp` package on top of the system `ggml`, whose backends are split out. +`sudo pacman -Syu ggml llama-cpp ggml-vulkan` โ€” name **`ggml`** explicitly (it conflicts with +the old `ggml-cpu` without replacing it, and the removal prompt defaults to N), and **`ggml-vulkan`** +explicitly (only an optdepend โ€” without it llama-server silently runs on the CPU backend). +Then restart, and `llama-server --list-devices` must show `Vulkan0`. `whisper-cpp` moves to the +same `ggml` (CPU backends are inside the base package) โ€” restart its unit too. + ## VRAM safety (the golden rules) The amdgpu driver does not OOM cleanly โ€” an overcommitted model spills into GTT (system RAM), @@ -62,7 +70,7 @@ starves the desktop and freezes the PC (reboot). Hard-won rules: | `GLM-4.7-Flash-UD-Q4_K_XL` | **21.5** @moe22 | 24k | reasoning | quality coder (opencode subagents); KV fix + moe22 โœ“ verified clean 08-08 | | `GLM-4.7-Flash-UD-Q3_K_XL` | **26.0** @moe12 (2.0 daytime baseline; ~28 clean) | 24k | reasoning | **fast coder** โ€” re-added 09-21 because Q4's 21.5 dragged in agent loops; 3-bit, evening-safe at moe12 (13.4 used / 3.8 free @2.0); Q4 stays for hard reviews | | `Qwen3-Coder-Next-UD-IQ3_XXS` | 16.0 | **128k** | 80B-A3B | long-session coder (128k ctx) | -| `Qwen3.8-27B-UD-IQ3_XXS` | 14.5 (10.9 w/o MTP) | 24k | vision, MTP, hybrid reasoner | โš  **clean-desktop specialty** (kept 09-10 over the โ‰ฅ22 gate) โ€” dense-27B quality, slow: its Gated-DeltaNet layers are Vulkan-bound (prefill ~30). 2.5 GB free @1.6 baseline @24k (โœ“ 09-10); **never on a busy evening desktop** (~1 GB free = GTT freeze) | +| `Qwen3.8-27B-UD-IQ3_XXS` | **15.5** (10.9 w/o MTP) | 24k | vision, MTP, hybrid reasoner @ **xhigh** | โš  **clean-desktop specialty** (kept 09-10 over the โ‰ฅ22 gate) โ€” dense-27B quality, slow: its Gated-DeltaNet layers are Vulkan-bound (prefill ~35). 2.5 GB free @1.7 baseline @24k (โœ“ 10-02 on b11146, `-n 512`, acceptance 0.84). The *felt* slowness is the **xhigh thinking preamble** (17.5 s before the first answer token on a trivial prompt; `reasoning_effort=low` measured 6.45 s at the same decode on 10-02 โ€” siblings benched then removed by decision, the kwarg is one line if ever wanted); **never on a busy evening desktop** (~1 GB free = GTT freeze) | | `Qwen3-Embedding-0.6B` | 33 (CPU) | 8k | CPU-only, `/v1/embeddings` | RAG/search embedder (not a chat model) | | `Ornith-1.5-35B-A3B-IQ3_XXS` | **32.1** @moe16 | 24k | MTP, vision, thinking | **agentic coder** (added 09-21) โ€” Qwen3.5-MoE derivative claiming +15 Terminal-Bench / +5.6 SWE-bench over Qwen3.6-35B; the only coder with MTP; 13.8 used / 3.3 free @2.0 | @@ -144,6 +152,7 @@ the router, but chats/presets saved against the old id need re-picking. 2026-09-21: `GLM-4.7-Flash-UD-Q3_K_XL` (โšก fast coder) added to all six client files next to the Q4. 2026-09-21: `Ornith-1.5-35B-A3B-IQ3_XXS` (agentic coder, thinking + vision) added to all six client files after passing its gate (32.1 t/s @moe16). 2026-09-21: default model โ†’ `Qwen3.6-35B-A3B-Thinking` everywhere (opencode global + plan/build/auto, pi defaultModel + thinking level medium, aichat common/lw/wm) โ€” gemma-4-26B refused routine sysadmin work. Thinking preset gets `reasoning-budget 8192`. wm aichat caught up (GLM Q3, Ornith). +2026-10-12: Ornith in pi: "Response was truncated before completion." = `finish_reason: length` โ€” the `` block counts against pi's 4096 `max_tokens` and the preset has no reasoning-budget. Fixed client-side: pi sends `thinking_budget_tokens` per thinking level on Ornith (`compat.thinkingTokenBudgetField` + `settings.json > thinkingBudgets`, common + lw). The preset deliberately keeps NO CLI reasoning-budget โ€” it would override the per-request field (see the preset comment). ## Tuning cheat-sheet @@ -183,8 +192,17 @@ verifies a clean card with rocm-smi before every load, records a โš  row and ski generation when a model lands too tight (free < 1.5 GB or GTT ballooning). Results: `bench-results.md` (latest) + `bench-history.md` (append-only; diff runs there). +**Reading TTFT**: bench.py stamps it on the first non-empty `content` delta, so for thinking +models TTFT = prompt eval **plus the whole reasoning preamble**, and `0.00` means the answer never +started within `-n` tokens (thinking ate the budget). It is the column that compares +reasoning-effort siblings; for stable MTP decode numbers use `-n 512`. + ## Watch list - **GLM MTP in llama.cpp** โ€” the single biggest pending win (primary coder ~22 โ†’ ~30+). +- **DFlash2 for Qwen3.8-27B** โ€” `z-lab/Qwen3.8-27B-DFlash2-GGUF` drafter, upstream 1.81ร— vs the + MTP head's 1.33ร— here (b11146 has the support). Blockers: #27408 (spec + mmproj โ†’ image requests + 500, so only as a text-only sibling preset) and #28158 (Vulkan OOB draft token, open). Trial = + `spec-type draft-dflash`, `spec-draft-n-max 3`, no mmproj; n-max 7 does not fit the VRAM envelope. - **Coder-Next 256k**: if the load log's `n_ctx_train` says 262144, ctx can likely double again for ~1 GB (KV measured nearly flat). diff --git a/fl/.config/llamacpp/config.ini b/fl/.config/llamacpp/config.ini index 146f7e6..b70ea7e 100644 --- a/fl/.config/llamacpp/config.ini +++ b/fl/.config/llamacpp/config.ini @@ -299,6 +299,7 @@ sleep-idle-seconds = 60 # (157/195, mean len 2.6) โ€” so MTP IS engaging; at 2.6 tok/step that is ~175 ms per step โ‰ˆ # 3โ€“4ร— a plain forward pass. NOTE the box runs the Arch llama.cpp-vulkan package (RADV, # device Vulkan0) โ€” NOT HIP/ROCm (this header said "ROCm" until 09-10); rocm-smi = monitoring only. +# [10-02: that package is gone โ€” now Arch `llama-cpp` 0.5.0 (b11146) + `ggml-vulkan`, see RE-PROBE below] # Suspects: the 48 GatedDeltaNet layers on the Vulkan backend (upstream #20354: missing/slow # GDN shader โ†’ CPU fallback) and/or recurrent-state checkpoint/restore per speculative step # (~150 MiB for this model; reported net-negative on hybrid 27Bs). The hybrid 35B-A3B does @@ -327,6 +328,57 @@ sleep-idle-seconds = 60 # buffer buys nothing), MTP stays (+33% for 1.5 GB). โœ“ CONFIRMED @24576 2026-09-10 03:51 # (1.6 baseline): 15.1 t/s / 31 pp / 14.6 used / 2.5 free โ€” footprint 13.0, exactly as # predicted; free = the 2.5 target at a normal idle, ~2.8 pristine, ~0.8 on a 3.3 evening. +# โœฆ RE-PROBE 2026-10-02 03:11 on llama.cpp b11146 = Arch `llama-cpp 0.5.0` + `ggml-vulkan 0.25.3` +# (the `llama.cpp-vulkan b10615` package is gone; `ggml` conflicts with the old `ggml-cpu` and +# does not replace it โ†’ name it: `sudo pacman -Syu ggml llama-cpp ggml-vulkan`, answer y twice; +# `ggml-vulkan` is only an optdepend โ€” without it the server silently runs on CPU). +# 4.0 DIRTY baseline: 14.8 t/s / 18 pp / 14.4 used / 2.8 free. Decode IDENTICAL to 09-10 +# (14.5โ€“15.1) โ†’ the build moves nothing on speed, as predicted. VRAM columns INVALID this run: +# 14.4 used at a 4.0 baseline = model "added" 10.4 GB < its own 10.93 GB file โ€” amdgpu evicted +# the desktop's allocations to GTT to make room (the 08-16 masking effect, 3 GB this time); +# footprint stays 13.0 from the clean 09-10 runs. TTFT 0.00 โ€” decoded in the 03:34 entry +# below (NOT a bench quirk, as this line first claimed). +# Re-run 03:16 @1.8 normal-idle baseline: 13.8 t/s / 33 pp / 14.7 used / 2.4 free โ†’ footprint +# 12.9, VRAM half exactly as on 09-10 (2.5 @1.6); 13.8 is inside the ยฑ15% MTP acceptance +# swing (14.5โ€“15.1 before). b11146 confirmed NEUTRAL on both speed and VRAM for this preset. +# Third run 03:19 @1.7: 12.5 t/s / 38 pp / TTFT 17.1 / 14.7 used / 2.4 free. Three back-to-back +# runs โ†’ decode BAND 12.5โ€“14.8 with VRAM identical each time; prefill 18โ†’33โ†’38 = Vulkan +# pipeline warm-up on the first load after the restart, decode drift = acceptance at temp 1.0 +# over 256 tokens (ยฑ15% documented) and possibly GPU heat after 3 consecutive loads. +# โœฆ DEFINITIVE 2026-10-02 03:34, -n 512 @1.7 baseline: 15.5 t/s / 38 pp / TTFT 20.6 / 14.7 used / +# 2.5 free. Server log: eval 442 tok @ 15.50 t/s (64.5 ms/tok), draft acceptance 0.842 +# (278/330), mean len 2.68, graphs reused 163 โ†’ MTP = 1.42ร— over the 10.9 base. The 256-token +# runs under-read (short window + acceptance noise). REFERENCE: 15.5 (10.9 w/o MTP). +# TTFT DECODED (bench.py stamps ttft on the first non-empty `content` delta; reasoning deltas +# do not count): TTFT = prompt eval + the whole THINKING preamble. 17.1 s / 20.6 s โ‰ˆ 230-300 +# tokens of xhigh thinking on the bench's trivial prompt; "0.00" on the 256-token runs = +# thinking consumed the entire budget and no answer was ever produced. The GGUF's embedded +# template (header read 10-02) defaults reasoning_effort to XHIGH โ€” heaviest of +# xhigh/medium/low โ€” and this preset passes nothing, so every answer pays it. That, not the +# 15 t/s, is why it feels slow. โ†’ sibling presets benched 03:49 (next entry), then removed. +# โœฆ SIBLING BENCH 2026-10-02 03:49, -n 512 @1.9: xhigh 16.2 t/s / TTFT 17.5 s / 14.9 used / 2.2 +# free; -low 15.9 t/s / TTFT 6.45 s / 15.0 / 2.1 โ†’ reasoning_effort=low cuts the preamble 63% +# at identical decode. -medium SKIPPED by the guard (15.7 used / 1.4 free at load) = VRAM +# drain lag after two back-to-back loads, not a medium cost (same weights). โ†’ USER DECISION +# 10-02: both siblings REMOVED from the config. The lever remains one line on THIS preset โ€” +# chat-template-kwargs = {"reasoning_effort":"low"} (or "medium") โ€” or per request via +# chat_template_kwargs; xhigh stays the default here. +# What b11146 DOES carry for this preset: #27812 (merged 08-28) fixes the Vulkan graph +# optimizer reordering nodes that alias Qwen3.8's recurrent state โ€” on b10615 that meant +# possible wrong tokens at temp 0 and acceptance numbers "that mean nothing"; everything from +# 10-02 on is from a correct build. Also DFlash2 support (#27342, 08-27). NOT in b11146: +# RDNA3 int8 coopmat prefill kernels (#27952, 09-24 โ€” K-quant/q8 tensors only, not IQ3_XXS +# blocks anyway) and the GDN tune (#29476, 09-29 โ€” an Intel fix, AMD measured flat). +# CORRECTION to the 09-10 suspects: the Vulkan GDN shader has existed since 2026-03 +# (#20334 โ†’ #20662 โ†’ #24581); there is no CPU fallback โ€” 10.9 base IS the RDNA3 kernel +# ceiling. The remaining lever is speculative acceptance LENGTH, not the kernel: +# z-lab/Qwen3.8-27B-DFlash2-GGUF (Q4_K_M 1.14 GB drafter, mrope-fixed 08-24) โ€” upstream +# 10.4 โ†’ 18.9 t/s (1.81ร—, AL โ‰ˆ 5) on this model vs MTP's 1.33ร— here (AL 2.6). Costs: the +# drafter + ~150 MiB recurrent rollback plane PER draft token โ†’ spec-draft-n-max 3โ€“4 fits +# the MTP envelope, the default 7 does not. DFlash + mmproj = open #27408 (image requests +# stall/500) โ†’ any trial is a separate TEXT-ONLY preset; this one keeps MTP + vision. +# Open on Vulkan for MTP and DFlash alike: #28158 โ€” intermittent draft token id 248320 +# (= n_vocab) โ†’ HTTP 500 "Invalid input batch", mostly after very long prompts. If seen: that. # โš  Needs a CURRENT llama.cpp build: first "qwen35" file HERE with ssm.*/hybrid-attn keys # (the 9B is regular-attention qwen35). If the load fails with unknown-architecture / # missing-tensor errors, update llama.cpp on fl before touching this preset. @@ -493,6 +545,13 @@ sleep-idle-seconds = 30 # Role: agentic coder with reasoning โ€” the only coder here with MTP (Coder-30B 30, GLM Q3 26). # Instruct alias (reasoning-budget = 0, same file) still optional โ€” add if agent loops want # the ~4 s thinking TTFT gone. +# โš  NO reasoning-budget LINE IN THIS PRESET, ON PURPOSE (2026-10-12): pi caps the think block +# PER REQUEST (`thinking_budget_tokens`, from pi settings.json > thinkingBudgets, tested on +# b11429: budget 40 โ†’ trace cut at ~40 tokens, answer still arrives, finish_reason stop) and +# llama.cpp IGNORES that field whenever the preset sets a CLI --reasoning-budget. Adding one +# here "like the Thinking preset" silently disables the pi knob and brings back pi's +# "Response was truncated before completion." (= finish_reason length: think block + answer +# > the client's 4096 max_tokens). An instruct alias (budget 0) in a SEPARATE section is fine. # โฌ‡ DOWNLOAD from bartowski/Ornith-1.5-35B-A3B-GGUF (imatrix; the official repo has only Q4_K_M+ # โ‰ฅ 21.7 GB, unsloth has no 1.5 quants yet): # Ornith-1.5-35B-A3B-IQ3_XXS.gguf 15,340,447,392 B ยท sha256 8918ccb9ee29abe3875efec0c3e86f0e35ef0b8a03e3f0e1c422869859a518d0 diff --git a/fl/.config/llamacpp/gpumon.sh b/fl/.config/llamacpp/gpumon.sh new file mode 100755 index 0000000..60e891b --- /dev/null +++ b/fl/.config/llamacpp/gpumon.sh @@ -0,0 +1,70 @@ +#!/bin/sh +# gpumon.sh โ€” one screen of GPU state for fl (RX 7600 XT ยท amdgpu ยท Vulkan backend). +# +# watch -c -d -n 1 ~/.config/llamacpp/gpumon.sh (fish abbr `vramx`, fl/.config/fish/host.fish) +# ~/.config/llamacpp/gpumon.sh (one shot) +# +# Why not rocm-smi: `--showmemuse` gives percentages and hides GTT. The freeze mode on this box is +# VRAM overcommit spilling into GTT (system RAM) โ€” see the config.ini header โ€” so GTT and the +# absolute free-VRAM figure (the โ‰ฅ 2.5 GB gate) are the numbers that matter. amdgpu exposes both +# in sysfs: instant, no ROCm needed. Per-process VRAM comes from /proc//fdinfo (kernel +# drm-usage-stats), readable only for your own processes unless root. llama.service runs as the +# login user (anon), so its children show up; if they ever don't: +# systemctl show -p User llama.service and run this with sudo. +# No absolute GTT alarm on purpose: bench.py judges GTT as a delta over a per-model baseline +# (a browser-heavy desktop legitimately sits on a few GB) โ€” `watch -d` shows the movement. +# +# GPUMON_HOST overrides the router host (default 127.0.0.1) โ€” for a dry run from another box. + +dev=/sys/class/drm/renderD128/device +host=${GPUMON_HOST:-127.0.0.1} +port=11343 + +echo "=== GPU (amdgpu sysfs) ===" +if [ -r "$dev/mem_info_vram_used" ]; then + temp=$(cat "$dev"/hwmon/hwmon*/temp1_input 2>/dev/null | head -n 1) + awk -v u="$(cat "$dev/mem_info_vram_used")" -v t="$(cat "$dev/mem_info_vram_total")" \ + -v g="$(cat "$dev/mem_info_gtt_used" 2>/dev/null)" -v gt="$(cat "$dev/mem_info_gtt_total" 2>/dev/null)" \ + -v b="$(cat "$dev/gpu_busy_percent" 2>/dev/null)" -v m="$(cat "$dev/mem_busy_percent" 2>/dev/null)" \ + -v c="${temp:-0}" 'BEGIN { + G = 1073741824 + free = (t - u) / G + flag = free < 1.5 ? " <<< HARD FLOOR (bench.py guard)" : free < 2.5 ? " <-- below the 2.5 GB target" : "" + printf "VRAM %5.2f / %5.2f GiB free %5.2f GiB%s\n", u / G, t / G, free, flag + printf "GTT %5.2f / %5.2f GiB (VRAM spilled to RAM โ€” must not grow while a model is resident)\n", g / G, gt / G + printf "busy gpu %3d%% mem %3d%% edge %dยฐC\n", b, m, c / 1000 + }' +else + echo "no amdgpu sysfs under $dev โ€” not the fl box, or the GPU is not renderD128" +fi + +echo +echo "=== processes holding the GPU (VRAM from fdinfo ยท RSS ยท uptime; own-user only unless root) ===" +# fuser puts the PID list on stdout (the pretty table of -v goes to stderr, hence no -v here). +pids=$(fuser /dev/dri/renderD128 /dev/dri/card* 2>/dev/null | tr ' ' '\n' | grep -E '^[0-9]+$' | sort -un) +if [ -z "$pids" ]; then + echo "(nothing visible โ€” no /dev/dri here, or the GPU users belong to another user)" +else + for p in $pids; do + [ -r "/proc/$p/fdinfo" ] || continue + # one drm client can be reachable through several fds โ€” dedupe by drm-client-id, then sum + vram=$(awk 'FNR == 1 { cid = "" } + /^drm-client-id:/ { cid = $2 } + /^drm-(total|memory)-vram:/ { v[cid] = $2 } + END { for (k in v) s += v[k]; printf "%.2f", s / 1048576 }' /proc/"$p"/fdinfo/* 2>/dev/null) + name=$(cat /proc/"$p"/comm 2>/dev/null) + alias=$(tr '\0' '\n' < /proc/"$p"/cmdline 2>/dev/null | awk '$0 == "--alias" { getline; print; exit }') + set -- $(ps -o rss=,etime= -p "$p" 2>/dev/null | awk '{ printf "%.1f %s", $1 / 1048576, $2 }') + printf '%7s GiB %7s %-16s rss %5s GiB up %-11s %s\n' "$vram" "$p" "$name" "${1:-?}" "${2:-?}" "$alias" + done | sort -rn | head -n 15 +fi + +echo +echo "=== router $host:$port โ€” preset status (anything not unloaded; the sysfs line above is the truth for VRAM) ===" +if out=$(curl -sf -m 1 "http://$host:$port/models" 2>/dev/null); then + printf '%s' "$out" \ + | jq -r '.data[] | select(.status.value != "unloaded") | "\(.status.value)\t\(.id)"' 2>/dev/null \ + | { grep . || echo "(nothing resident)"; } +else + echo "(router not answering โ€” systemctl status llama.service)" +fi diff --git a/lw/.pi/agent/models.json b/lw/.pi/agent/models.json index 3c287e8..0c8cf25 100644 --- a/lw/.pi/agent/models.json +++ b/lw/.pi/agent/models.json @@ -194,7 +194,7 @@ "id": "Ornith-1.5-35B-A3B-IQ3_XXS", "name": "Ornith 1.5 35B ยท 24k ยท vision โ€” agentic coder, thinking (~32 t/s)", "reasoning": true, - "compat": { "thinkingFormat": "qwen-chat-template" }, + "compat": { "thinkingFormat": "qwen-chat-template", "thinkingTokenBudgetField": "thinking_budget_tokens" }, "input": ["text", "image"], "contextWindow": 24576, "maxTokens": 2048, @@ -329,7 +329,7 @@ "id": "Ornith-1.5-35B-A3B-IQ3_XXS", "name": "Ornith 1.5 35B ยท 24k ยท vision โ€” agentic coder, thinking (~32 t/s)", "reasoning": true, - "compat": { "thinkingFormat": "qwen-chat-template" }, + "compat": { "thinkingFormat": "qwen-chat-template", "thinkingTokenBudgetField": "thinking_budget_tokens" }, "input": ["text", "image"], "contextWindow": 24576, "maxTokens": 2048, diff --git a/lw/.pi/agent/settings.json b/lw/.pi/agent/settings.json index bee358c..9fef69d 100644 --- a/lw/.pi/agent/settings.json +++ b/lw/.pi/agent/settings.json @@ -2,6 +2,12 @@ "defaultProvider": "duskadiy", "defaultModel": "Qwen3.6-35B-A3B-Thinking", "defaultThinkingLevel": "medium", + "thinkingBudgets": { + "minimal": 256, + "low": 512, + "medium": 1024, + "high": 1024 + }, "theme": "catppuccin-mocha", "quietStartup": false, "collapseChangelog": true,