[{"data":1,"prerenderedAt":2368},["ShallowReactive",2],{"site-info":3,"post-quantization-kernel-dispatch":99,"related-zh-TW-quantization-kernel-dispatch":1400},{"id":4,"company":5,"extension":16,"meta":17,"services":18,"social":96,"stem":97,"__hash__":98},"site\u002Fsite.yml",{"name":6,"nameEn":7,"email":8,"address":9,"addressEn":10,"lineUrl":11,"lineId":12,"duns":13,"youtube":14,"youtubeId":15},"銡策數位有限公司","Jecision Digital Co., Ltd.","contact@jecision.com","231 新北市新店區北新路二段 262 號 3 樓之 3","3F.-3, No. 262, Sec. 2, Beixin Rd., Xindian Dist., New Taipei City 231, Taiwan","https:\u002F\u002Fline.me\u002FR\u002Fti\u002Fp\u002F@374qnumi","@374qnumi","963905705","https:\u002F\u002Fwww.youtube.com\u002Fshorts\u002F6UE0WTmyjsg","6UE0WTmyjsg","yml",{},[19,43,69],{"key":20,"zh":21,"en":32},"audit",{"title":22,"tagline":23,"summary":24,"points":25,"deliverable":29,"detail":30,"duration":31},"企業 AI 導入健檢","先看清楚，再花錢","60 分鐘線上訪談，盤點最耗時、最依賴人力的營運環節；我們會告訴你哪些工作現在就能交給 AI、哪些還不成熟。",[26,27,28],"現況盤點：梳理日常營運中最耗時、最依賴人力的環節","痛點診斷：判斷哪些工作現在就能交給 AI，哪些建議先保留人力","可行性評估：依你的業務特性與預算，篩選適合的模型與工具","《企業 AI 導入建議報告書》（PDF）","含 1–2 個「開發成本最低、成效最明顯」的優先項目、技術棧建議、預期節省的人力時間與下一步","訪談 60 分鐘，報告書 3 個工作天內交付",{"title":33,"tagline":34,"summary":35,"points":36,"deliverable":40,"detail":41,"duration":42},"AI Readiness Audit","Look before you spend","A 60-minute remote audit of the workflows that eat the most time and people-power — and an honest answer on what AI can take over today, and what it cannot.",[37,38,39],"Audit the daily workflows that consume the most time and headcount","Triage which tasks AI can own today, and which should stay human","Shortlist the models and tools that fit your budget and constraints","AI Adoption Recommendation Report (PDF)","One or two highest-ROI entry points, a recommended tech stack, expected hours saved, and the concrete next step","60-minute call, report delivered within 3 business days",{"key":44,"zh":45,"en":56},"prompt",{"title":46,"tagline":47,"summary":48,"points":49,"deliverable":53,"detail":54,"duration":55},"提示詞工程","讓 AI 穩定產出，而不是碰運氣","針對你的實際任務設計提示詞與多步驟工作流，並做極限測試，確保換了資料、換了人操作也不會走鐘。",[50,51,52],"客製提示詞：ChatGPT、Claude、Gemini、Midjourney 等主流工具的專屬模板","除錯與優化：修掉幻覺、格式跑掉、時好時壞的問題","工作流建置：把複雜任務拆成多步驟提示詞與可重複的 SOP","提示詞文件或設定檔","附使用說明與範例，並提供 1 對 1 教學，確保你與團隊能自己維護","依任務複雜度報價，通常 3–7 個工作天",{"title":57,"tagline":58,"summary":59,"points":60,"deliverable":66,"detail":67,"duration":68},"Prompt Engineering","Make AI produce reliably, not by luck","Prompt sets and multi-step workflows designed for your actual tasks, stress-tested so they keep working when the data or the operator changes.",[61,62,64],"Custom prompts for ChatGPT, Claude, Gemini, Midjourney and friends",{"Debugging and tuning":63},"kill hallucinations, broken formats and flaky output",{"Workflow design":65},"break complex work into repeatable multi-step SOPs","Prompt library or exportable config","Ships with usage notes and examples, plus a 1:1 walkthrough so your team can maintain it","Quoted by complexity, typically 3–7 business days",{"key":70,"zh":71,"en":82},"poc",{"title":72,"tagline":73,"summary":74,"points":75,"deliverable":79,"detail":80,"duration":81},"極速 PoC","五天，看到能跑的東西","不談架構藍圖，直接做一個能動的雛形。五個工作天內交付線上可操作的智能體，用你真實的資料驗證。",[76,77,78],"Day 1：15 分鐘對焦，鎖定單一痛點與資料盤點","Day 2–4：模型串接、邏輯設計與雛形開發","Day 5：Demo 展示與 AI 擴展建議書","可線上操作的智能體雛形","交付範圍為概念驗證與展示，不含後續維運或大規模伺服器部署；要進階成完整系統會另行報價","5 個工作天",{"title":83,"tagline":84,"summary":85,"points":86,"deliverable":93,"detail":94,"duration":95},"Rapid PoC","Five days to something that runs","No architecture decks — a working prototype built in five business days, validated against your real data.",[87,89,91],{"Day 1":88},"15-minute scoping call, one pain point, data inventory",{"Day 2–4":90},"model integration, logic design, prototype build",{"Day 5":92},"live demo plus an AI expansion recommendation","A runnable online agent prototype","Proof of concept and demo scope only; ongoing operations and large-scale deployment are quoted separately","5 business days",[],"site","XTwARfVAOcrXNiKehsM6V6cdkd2HXy4d5yZVrSWrI3Q",{"zh":100,"en":800},{"id":101,"title":102,"author":103,"body":104,"cover":780,"coverAlt":781,"date":782,"description":783,"draft":784,"extension":785,"featured":784,"lang":786,"meta":787,"navigation":177,"ogImage":788,"path":789,"seo":790,"slug":791,"stem":792,"tags":793,"translationKey":791,"updated":798,"__hash__":799},"blog\u002Fblog\u002Fzh-TW\u002Fquantization-kernel-dispatch.md","量化會不會加速，取決於你怎麼寫 kernel","Jecision",{"type":105,"value":106,"toc":768},"minimark",[107,114,118,121,124,127,130,133,137,140,143,146,149,152,155,158,163,166,183,187,190,193,261,264,267,270,273,280,286,292,295,391,394,397,400,404,407,410,480,483,486,489,492,499,502,506,509,512,562,565,568,571,574,578,581,587,593,599,605,666,669,691,695,698,701,715,718,726,744,747],[108,109,110],"blockquote",{},[111,112,113],"p",{},"數字全部來自 arXiv 2605.05699v1（2026-05-07）。作者是 Illumina 的 Mohamed Amine Bergach；這是單人預印本，未同儕審查。我們沒有在 Apple M1 上獨立重跑；「我認為」的地方會標出來。",[115,116,117],"h2",{"id":117},"先講一個畫面",[111,119,120],{},"你在筆電上跑 1B 模型，context 從 256 拉到 4K，解碼變慢、KV cache（解碼時累積的 K\u002FV 快取）越長越吃記憶體。這篇要幫你看懂兩組數字：那個「慢 17%」跟「快 8%」分別在比什麼。還有下次看 benchmark 要問哪四件事，以及什麼時候該留 fp16 退路。",[111,122,123],{},"上網查，答案通常是「量化」。但下一句總是「會變慢」。",[111,125,126],{},"有一篇論文把這句話拆開了。同一份 int4（4-bit 整數）KV cache 量化，eager 版要四次 kernel 啟動（dispatch）。在 360M 模型上，這讓延遲比 SRHT（用 Hadamard 做隨機旋轉的對照法）高 17%。把四步融成兩顆 kernel 後，SmolLM 的同一組對照變成低 1%，1.7B 低 5%；Qwen 仍慢 1%。端到端對 fp16，Gemma-3 1B 在 256 前綴低 8.2%。",[111,128,129],{},"但先別高興。換到 Qwen2.5-1.5B 的 2K 前綴，它只吐兩個 token 就 EOS（提前結束）。這 8% 有平台、前綴、架構的前提，後面會把邊界畫出來。",[111,131,132],{},"這篇要把兩個常被混在一起的問題拆開。一個是實作稅：同一個演算法，eager 版四次 dispatch、融合版兩顆 kernel。另一個是部署帳：int4 cache 對 fp16，端到端到底快多少、什麼時候會壞。",[115,134,136],{"id":135},"先看那個慢-17","先看那個「慢 17%」",[111,138,139],{},"論文的方法叫 rotate-then-quantize。KV cache 裡的每個 K\u002FV 向量，先做一個隨機旋轉，再壓成 int4；attention 讀取時，再解量化、反旋轉回來。",[111,141,142],{},"這裡的旋轉是 SRFT（sign-randomized FFT）：先乘上一組固定的隨機正負號，再做 FFT。目的是打散離群值。模型的 K\u002FV 向量裡，少數很大的值會把量化尺度撐大，其他值就被壓成 0；先旋轉，讓每個座標看起來接近高斯分布，均勻量化才不會被少數大值拖垮。",[111,144,145],{},"問題在「四步」怎麼寫。eager 版把旋轉、量化、解量化、反旋轉拆成四個 PyTorch\u002FMPS 呼叫；每一層、每一個解碼步，就是四次 kernel dispatch。",[111,147,148],{},"dispatch 是什麼？就是 GPU 每啟動一顆 kernel 的排隊與搬資料成本。計算本身可能只有幾微秒，但四次啟動、四次把資料搬進搬出，加起來就變成主成本。",[111,150,151],{},"論文量到的代價是：SmolLM2-360M 延遲高 17%、1.7B 高 16%、Qwen2.5-1.5B 高 12%。基準是 SRHT（sign-randomized Hadamard transform），另一種隨機旋轉，不是 fp16。",[111,153,154],{},"為什麼拿 SRHT 當基準？因為 SRFT 和 SRHT 的量化品質在誤差內，差別只在「旋轉用 FFT 還是 Hadamard」。拿它對照，才能把「旋轉本身的成本」和「四步拆開的實作稅」分開。",[111,156,157],{},"論文有一句話是整段的骨架，我照原意翻：",[108,159,160],{},[111,161,162],{},"成本在 dispatch，不在計算。底層的 rfft 和 irfft 各只要個位數微秒，但四步分開呼叫，把這個開銷乘了上去。",[111,164,165],{},"這就是為什麼同樣的演算法，換一個寫法會從 +17% 變成 −1%。不是量化變好了，是四次排隊變成兩次（如圖 1）。",[167,168,171,172,171,179],"figure",{"className":169},[170],"article-figure","\n  ",[173,174],"video",{"src":175,"poster":176,"autoPlay":177,"loop":177,"muted":177,"playsInline":177,"ariaLabel":178},"\u002Fimages\u002Fblog\u002Fquantization-kernel-dispatch\u002Fkernel-fusion.mp4","\u002Fimages\u002Fblog\u002Fquantization-kernel-dispatch\u002Fkernel-fusion-poster.jpg",true,"左邊是 eager 版四次 kernel dispatch，右邊是融合成 forward 與 inverse 兩顆 kernel",[180,181,182],"figcaption",{},"圖 1：同一份量化，eager 版每層每步四次 dispatch；融合版把前兩步與後兩步各收成一顆 kernel。",[115,184,186],{"id":185},"把四步融成兩顆還不夠","把四步融成兩顆，還不夠",[111,188,189],{},"融合的做法，是把前兩步和後兩步各自塞進一顆 Metal kernel（Apple 的 GPU 程式）：forward 做旋轉加量化，inverse 做解量化加反旋轉。資料在中間不用回 device memory，留在 GPU 內部的快速記憶體。",[111,191,192],{},"同一組對照，SmolLM 反轉，Qwen 仍慢 1%：",[194,195,196,215],"table",{},[197,198,199],"thead",{},[200,201,202,206,209,212],"tr",{},[203,204,205],"th",{},"模型",[203,207,208],{},"SRHT 基準",[203,210,211],{},"eager Δ",[203,213,214],{},"融合 SRFT（Δ）",[216,217,218,233,247],"tbody",{},[200,219,220,224,227,230],{},[221,222,223],"td",{},"SmolLM2-360M",[221,225,226],{},"37.4 ms\u002Ftok",[221,228,229],{},"+17%",[221,231,232],{},"37.0 ms\u002Ftok（−1%）",[200,234,235,238,241,244],{},[221,236,237],{},"SmolLM2-1.7B",[221,239,240],{},"222.2 ms\u002Ftok",[221,242,243],{},"+16%",[221,245,246],{},"211.9 ms\u002Ftok（−5%）",[200,248,249,252,255,258],{},[221,250,251],{},"Qwen2.5-1.5B",[221,253,254],{},"84.9 ms\u002Ftok",[221,256,257],{},"+12%",[221,259,260],{},"85.7 ms\u002Ftok（+1%）",[111,262,263],{},"（Δ 相對 SRHT，百分比取整；256 token 前綴加 128 token 生成；Apple M1。不是對 fp16。）",[111,265,266],{},"1.7B 的差距比 360M 大。eager 的罰則在兩個模型上差不多（+17% 對 +16%）。差別在融合後：1.7B 每層的 KV 維度是 2048，360M 只有 320。融合 kernel 的固定啟動成本被更多工作攤薄，所以它從 −1% 變成 −5%。",[111,268,269],{},"到這裡，故事看起來很乾淨：四步變兩顆，慢 17% 變快 1%（360M）。但如果你看到這裡就下結論「量化一定快」，會漏掉下一段。",[111,271,272],{},"端到端對 fp16 的 −8.2%，不是只有融合。它還需要三個條件。",[111,274,275,279],{},[276,277,278],"strong",{},"第一，量化配方。"," 4-bit 如果只用 per-token（每個 token 一個尺度），Qwen 的 ΔPPL 會炸到 +7975。作者改成 per-channel λ 加 per-group abs-max（每個 channel 一個縮放係數，每 32 個值一組取絕對值最大），把同一個數字壓到 +638.6。這個配方是在同一顆融合 kernel 裡做的。",[111,281,282,285],{},[276,283,284],{},"第二，dequant-prefix cache。"," 解量化過的前綴先存起來，不要每一步重算；最近 16 個 token 保留 fp16，只有這個視窗滿了才失效。沒有它，Qwen 在 1024 前綴是 +2.4% 慢；有了它，才變成 −1.6% 快（表內回算；論文文字寫 −0.7%）。",[111,287,288,291],{},[276,289,290],{},"第三，模型架構。"," Gemma-3 1B 有 sliding-window 層：大部分層只看固定視窗、用 fp16，只有少數 full-attention 層（看完整前綴）要付 dequant 成本。所以它的優勢可以撐到 4096；Qwen 是 28 層 full-attention，成本隨前綴線性成長。",[111,293,294],{},"把這三件事放回去，端到端的表長這樣：",[194,296,297,312],{},[197,298,299],{},[200,300,301,303,306,309],{},[203,302,205],{},[203,304,305],{},"前綴",[203,307,308],{},"fp16 → int4（ms\u002Ftok）",[203,310,311],{},"延遲差",[216,313,314,328,341,354,367,379],{},[200,315,316,319,322,325],{},[221,317,318],{},"Gemma-3 1B",[221,320,321],{},"256",[221,323,324],{},"71.2 → 65.4",[221,326,327],{},"−8.2%",[200,329,330,332,335,338],{},[221,331,318],{},[221,333,334],{},"1024",[221,336,337],{},"83.3 → 82.5",[221,339,340],{},"−0.9%",[200,342,343,345,348,351],{},[221,344,318],{},[221,346,347],{},"2048",[221,349,350],{},"129.5 → 121.3",[221,352,353],{},"−6.3%",[200,355,356,358,361,364],{},[221,357,318],{},[221,359,360],{},"4096",[221,362,363],{},"188.5 → 182.4",[221,365,366],{},"−3.2%",[200,368,369,371,373,376],{},[221,370,251],{},[221,372,321],{},[221,374,375],{},"90.7 → 88.3",[221,377,378],{},"−2.6%",[200,380,381,383,385,388],{},[221,382,251],{},[221,384,334],{},[221,386,387],{},"137.1 → 134.9",[221,389,390],{},"−1.6%",[111,392,393],{},"（Δ 相對 fp16；Apple M1；256 token 前綴加 128 token 生成。Qwen 2048 不在表裡，原因見下一節。Qwen 1024 的 −1.6% 是用表內 137.1 → 134.9 回算；論文文字寫 −0.7%，我們以表內數字為準。）",[111,395,396],{},"記憶體方面，Qwen 是 3.0–3.1×；Gemma 的 19.5× 不是同層比較（full-attention 層內約 3.2×），不能直接跟 Qwen 比。",[111,398,399],{},"−8.2% 是 256 前綴的最佳情況。同一個模型拉到 4096，只剩 −3.2%。Qwen 拉到 1024，只剩 −1.6%。「量化快 8%」不是一個常數，是一條會隨前綴長度變化的曲線。",[115,401,403],{"id":402},"快之前先看它掉了多少","快之前，先看它掉了多少",[111,405,406],{},"速度之外，還有一本帳：品質。量化對品質的影響，通常看困惑度（perplexity，PPL）的變化；ΔPPL 是量化後減 fp16 的差。",[111,408,409],{},"4-bit per-token 的 ΔPPL 如下（Cosmopedia，8192 個 held-out token）：",[194,411,412,430],{},[197,413,414],{},[200,415,416,418,421,424,427],{},[203,417,205],{},[203,419,420],{},"FP16 PPL",[203,422,423],{},"不旋轉（identity）",[203,425,426],{},"SRHT",[203,428,429],{},"SRFT",[216,431,432,449,465],{},[200,433,434,437,440,443,446],{},[221,435,436],{},"SmolLM2-135M",[221,438,439],{},"6.61",[221,441,442],{},"+1.96 ± 0.15",[221,444,445],{},"+0.34 ± 0.01",[221,447,448],{},"+0.38 ± 0.06",[200,450,451,453,456,459,462],{},[221,452,223],{},[221,454,455],{},"4.87",[221,457,458],{},"+0.97 ± 0.08",[221,460,461],{},"+0.20 ± 0.03",[221,463,464],{},"+0.21 ± 0.02",[200,466,467,469,472,475,478],{},[221,468,237],{},[221,470,471],{},"3.77",[221,473,474],{},"+1.04",[221,476,477],{},"+0.16",[221,479,477],{},[111,481,482],{},"（前兩列是 3 個 seed 的平均 ± 1σ；1.7B 因為計算成本只跑 1 個 seed。6-bit 與 8-bit 在這組測試裡無損。）",[111,484,485],{},"旋轉不是可選項。135M 不旋轉是 +1.96，旋轉後是 +0.38；差了大約 5 倍。這也解釋了為什麼「量化會壞」和「量化只是變慢」常常是同一件事的兩面：如果你的旋轉沒做對，4-bit 的誤差會直接反映在 PPL 上。",[111,487,488],{},"6-bit 和 8-bit 無損，4-bit 才是取捨點。3-bit 的絕對誤差更大，但這篇沒有把 3-bit 當賣點，我也不建議讀者從 3-bit 開始。",[111,490,491],{},"Qwen 的 4-bit 是另一個故事。Python reference 的 per-token 是 +7975.3，Metal 的 per-token 是 +7727.9；加上 per-channel λ 加 per-group g32 之後，融合版是 +638.6。這是 12.5 倍的改善，跟 Python reference 的 +654.6 同級。",[111,493,494,495,498],{},"但 +638.6 仍然是 +638.6。",[276,496,497],{},"12.5 倍的改善，不等於可以上線。"," 論文把這個叫做「關掉災難」；我認為，它比較像是把火從整棟房子縮到一個房間，離安全還很遠。如果你要在 Qwen 上開 4-bit，請用任務指標驗收，不要只看 PPL 的相對改善。",[111,500,501],{},"Gemma 的結果比較好：per-token +18.7，融合後 +3.6。Python reference 是 −5.2，差距 +8.8 來自 kernel 的 fp32 reduction 與 rounding ordering，不是演算法。這一點作者自己做了兩個 ablation 排除。",[115,503,505],{"id":504},"它真的會壞qwen-2k-只吐兩個-token","它真的會壞：Qwen 2K 只吐兩個 token",[111,507,508],{},"現在回到那個「先別高興」。",[111,510,511],{},"Qwen2.5-1.5B 在 2048 前綴，int4 只生成 2 個 token 就 EOS；fp16 是 64 個。",[194,513,514,527],{},[197,515,516],{},[200,517,518,521,524],{},[203,519,520],{},"項目",[203,522,523],{},"fp16",[203,525,526],{},"int4",[216,528,529,540,551],{},[200,530,531,534,537],{},[221,532,533],{},"生成 token 數（2048 前綴）",[221,535,536],{},"64",[221,538,539],{},"2（EOS）",[200,541,542,545,548],{},[221,543,544],{},"峰值記憶體",[221,546,547],{},"4447 MB",[221,549,550],{},"5527 MB",[200,552,553,556,559],{},[221,554,555],{},"作者退路",[221,557,558],{},"—",[221,560,561],{},"約 1.5K 以上退回 fp16",[111,563,564],{},"原因不只一個。dequant-prefix cache 加上 int4 storage，讓峰值記憶體從 4447 拉到 5527 MB（+24%）；再加上 28 層 full-attention 的量化噪聲累積，logits 被推去提前 EOS。",[111,566,567],{},"我認為，讓 1024 變快的那個設計，也是讓 2048 變壞的設計之一。prefix cache 把解量化成本從「前綴越長越慢」攤成固定成本，但它同時佔記憶體；在 Qwen 這種全注意力堆疊上，兩件事一起發生，就爆了。",[111,569,570],{},"Gemma 為什麼沒事？它的 sliding-window 層用 fp16，只有少數 full-attention 層付 dequant 成本；所以前綴拉長，成本成長得比較慢。架構不同，同一份量化 cache 的結果就不同。",[111,572,573],{},"如果你要在自己的模型上開這個 cache，先測 2K。不是測「能不能生成」，是測「生成出來的東西對不對、會不會提前停」。作者給的退路很明確：超過約 1.5K，就退回 fp16。",[115,575,577],{"id":576},"看-benchmark-要問的四件事","看 benchmark 要問的四件事",[111,579,580],{},"這篇論文最有用的地方，不是「Apple Silicon 比較快」，是它把一張 benchmark 拆成幾個要問的問題。",[111,582,583,586],{},[276,584,585],{},"第一，平台。"," 這是 unified memory（CPU 與 GPU 共用記憶體）還是獨立 HBM（顯示卡自己的記憶體）？論文說：在獨立 HBM、dispatch 開銷高的 GPU 上，慣常的取捨會回復。也就是說，同樣的 kernel 搬過去，不一定還快。看到「量化加速」先問跑在哪。",[111,588,589,592],{},[276,590,591],{},"第二，kernel。"," 這是 fused 還是四步分開？如果 benchmark 沒寫，數字可能被 dispatch 稅主導。你可以直接問：一層一步幾次 dispatch？",[111,594,595,598],{},[276,596,597],{},"第三，前綴與架構。"," 256 前綴的 −8% 不是 4K 的 −3.2%；full-attention 堆疊和 sliding-window 的曲線也不一樣。要求對方給 256、1K、2K 三組數字，不要只看一張最好的圖。",[111,600,601,604],{},[276,602,603],{},"第四，品質。"," ΔPPL 的 baseline 與配方是什麼？+638.6 比 +7975 好，但仍然是 +638.6。per-token 和 per-channel+group 是兩種不同的東西，不能拿來互相比。",[194,606,607,620],{},[197,608,609],{},[200,610,611,614,617],{},[203,612,613],{},"問題",[203,615,616],{},"為什麼要問",[203,618,619],{},"怎麼查",[216,621,622,633,644,655],{},[200,623,624,627,630],{},[221,625,626],{},"平台是 unified memory 還是獨立 HBM？",[221,628,629],{},"取捨是否反轉的邊界",[221,631,632],{},"看硬體規格；要求分平台數字",[200,634,635,638,641],{},[221,636,637],{},"Kernel 是 fused 還是四步分開？",[221,639,640],{},"相對 SRHT 可能慢 12–17%",[221,642,643],{},"看 benchmark 是否寫「4 dispatches」或「fused」",[200,645,646,649,652],{},[221,647,648],{},"前綴多長、模型是 full-attention 還是 sliding-window？",[221,650,651],{},"長 context 可能失效",[221,653,654],{},"要求 256／1K／2K 三組；看架構",[200,656,657,660,663],{},[221,658,659],{},"ΔPPL 的 baseline 與量化配方是什麼？",[221,661,662],{},"+638.6 比 +7975 好，但仍是 +638.6",[221,664,665],{},"看 baseline、per-token 或 per-channel+group",[115,667,668],{"id":668},"別過度解讀",[670,671,672,676,679,682,685,688],"ul",{},[673,674,675],"li",{},"單一平台：只有 Apple M1；獨立 HBM 的 GPU 沒測。",[673,677,678],{},"模型都是小模型（135M–1.7B），沒有 7B 以上。",[673,680,681],{},"單人預印本，未同儕審查；我們沒有獨立複現。",[673,683,684],{},"Gemma 的 19.5× 不是同層比較；full-attention 層內約 3.2×。",[673,686,687],{},"Qwen 2K 會壞；作者自己寫了 1.5K 的退路。",[673,689,690],{},"這篇不是說「融合是唯一解」；它只處理四步拆開這一種成本。",[115,692,694],{"id":693},"結論問題不是量化快不快","結論：問題不是「量化快不快」",[111,696,697],{},"量化不是一個開關，是一條工程鏈：平台、kernel、量化配方、前綴長度、模型架構，任何一個換了，答案就換了。",[111,699,700],{},"這篇的兩本帳，可以這樣記：",[670,702,703,709],{},[673,704,705,708],{},[276,706,707],{},"實作稅"," 是同一份方法，eager 四步對融合兩顆，差距 11–21 個百分點。",[673,710,711,714],{},[276,712,713],{},"部署帳"," 是 int4 cache 對 fp16，短前綴最多 −8.2%，長前綴會縮到個位數百分比（Qwen 1024 是 −1.6%，Gemma 4096 是 −3.2%），而且可能提前 EOS。",[111,716,717],{},"明天可以先做一件事：用你的模型跑 256、1K、2K 三個前綴，記錄 ms\u002Ftok 和輸出品質；如果模型是 full-attention 堆疊，1.5K 以上先留 fp16 退路；2K 是測試點。",[111,719,720,721,725],{},"如果你要的是有人陪你把自己的模型跑過這三個前綴，我們的",[722,723,22],"a",{"href":724},"\u002Fservices","做的就是這件事。先判斷哪些工作現在就能交給 AI，哪些還不成熟。",[111,727,728,729,736,737,743],{},"量化系列走到這裡，三篇的弧線是：",[722,730,732],{"href":731},"\u002Fblog\u002Fstrata-125b-on-12gb",[733,734,735],"code",{},"strata"," 講取捨，",[722,738,740],{"href":739},"\u002Fblog\u002Fbroken-on-arrival-quantized-models",[733,741,742],{},"broken-on-arrival"," 講配方，這篇講實作。同一個問題，從「要不要量化」走到「怎麼寫那個 kernel」。",[745,746],"hr",{},[111,748,749],{},[750,751,752,753,759,760,763,764,767],"em",{},"論文：",[722,754,758],{"href":755,"rel":756},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2605.05699",[757],"nofollow","arXiv 2605.05699v1","（Mohamed Amine Bergach, Illumina）。前作：",[722,761,762],{"href":731},"125B 模型跑在 12GB 顯卡","、",[722,765,766],{"href":739},"你下載的量化模型可能是壞的","。",{"title":769,"searchDepth":770,"depth":770,"links":771},"",2,[772,773,774,775,776,777,778,779],{"id":117,"depth":770,"text":117},{"id":135,"depth":770,"text":136},{"id":185,"depth":770,"text":186},{"id":402,"depth":770,"text":403},{"id":504,"depth":770,"text":505},{"id":576,"depth":770,"text":577},{"id":668,"depth":770,"text":668},{"id":693,"depth":770,"text":694},"\u002Fimages\u002Fblog\u002Fquantization-kernel-dispatch.svg","標題「量化會不會加速，取決於你怎麼寫 kernel」的技術筆記封面","2026-11-05","同一份 int4 KV cache 量化，eager 版相對 SRHT 慢 12–17%，把四步融成兩顆 Metal kernel 後追平；端到端對 fp16，Gemma 短前綴快 8.2%，但 Qwen 2K 前綴只吐兩個 token。差別在四步拆開呼叫，還是融成兩顆——以及平台、前綴與架構的前提。",false,"md","zh-TW",{},"\u002Fimages\u002Fblog\u002Fquantization-kernel-dispatch.jpg","\u002Fblog\u002Fzh-tw\u002Fquantization-kernel-dispatch",{"title":102,"description":783},"quantization-kernel-dispatch","blog\u002Fzh-TW\u002Fquantization-kernel-dispatch",[794,795,796,797],"quantization","kv-cache","apple-silicon","inference",null,"nxxvpD8Ugm7w7mX3s1VM_GxUtkB58VVj0Hd2cm51owY",{"id":801,"title":802,"author":103,"body":803,"cover":1389,"coverAlt":1390,"date":782,"description":1391,"draft":784,"extension":785,"featured":784,"lang":1392,"meta":1393,"navigation":177,"ogImage":1394,"path":1395,"seo":1396,"slug":791,"stem":1397,"tags":1398,"translationKey":791,"updated":798,"__hash__":1399},"blog\u002Fblog\u002Fen\u002Fquantization-kernel-dispatch.md","Quantization Speed Is Often a Kernel Problem",{"type":105,"value":804,"toc":1379},[805,810,814,817,820,823,826,830,833,836,839,842,845,848,851,856,859,870,874,877,880,933,936,939,942,945,951,957,963,966,1045,1048,1051,1054,1058,1061,1064,1067,1122,1125,1128,1131,1134,1137,1140,1144,1147,1150,1194,1197,1200,1203,1206,1210,1213,1219,1225,1231,1237,1298,1302,1322,1326,1329,1332,1346,1349,1357,1364,1366],[108,806,807],{},[111,808,809],{},"Every number here comes from arXiv 2605.05699v1 (May 7, 2026). The author is Mohamed Amine Bergach at Illumina; this is a single-author preprint, not peer-reviewed. We did not rerun it on an Apple M1. Where I add a judgement, I say so.",[115,811,813],{"id":812},"a-scene","A scene",[111,815,816],{},"You are running a 1B model on a laptop; push the context from 256 to 4K and decoding slows as the KV cache (the K\u002FV state kept during decoding) grows with every token. This essay is about two ledgers hidden in the usual advice — \"quantize, but it slows you down\" — and when to keep an fp16 fallback.",[111,818,819],{},"The implementation tax: the same algorithm written as four eager dispatches or two fused kernels. The deployment bill: an int4 cache against fp16, end to end, including where it breaks. By the end you should know which number is being compared to what, and four questions to ask of any benchmark that claims quantization is faster.",[111,821,822],{},"The paper runs the same int4 KV-cache method both ways. The eager version is 17% slower on a 360M model, measured against SRHT (a sign-randomized Hadamard rotation used as the control). Fuse the four steps into two kernels and SmolLM comes out 1% faster; on a 1.7B model it is 5% faster. Qwen is still 1% slower. End to end against fp16, Gemma-3 1B at a 256-token prefix has 8.2% lower latency.",[111,824,825],{},"Then the warning. On Qwen2.5-1.5B at a 2K prefix, the int4 cache emits two tokens and stops. fp16 emits 64.",[115,827,829],{"id":828},"first-the-17","First, the 17%",[111,831,832],{},"The method is rotate-then-quantize. Each K and V vector in the KV cache is rotated, compressed to int4 (a 4-bit integer), then dequantized and rotated back when attention reads it.",[111,834,835],{},"The rotation here is SRFT (sign-randomized FFT): multiply by a fixed random ±1 pattern, then take an FFT. The point is to spread out outliers. In a K\u002FV vector, a few large values stretch the quantization scale and squash the rest toward zero. Rotate first and the coordinates look close to Gaussian; uniform quantization no longer has to survive a handful of spikes.",[111,837,838],{},"The problem is how the four steps are written. The eager version calls rotate, quantize, dequantize and inverse-rotate as four separate PyTorch\u002FMPS operations. Every layer, every decoding step, pays four kernel dispatches.",[111,840,841],{},"Dispatch is the queueing and data movement around a GPU kernel launch. The arithmetic may take microseconds; four launches and four round trips through memory can cost more than the arithmetic itself.",[111,843,844],{},"The paper's measured tax: 17% higher latency on SmolLM2-360M, 16% on 1.7B, 12% on Qwen2.5-1.5B. The baseline is SRHT (sign-randomized Hadamard transform), another rotation, not fp16.",[111,846,847],{},"Why SRHT? Because SRFT and SRHT produce quantization quality within noise of each other; the difference is whether the rotation is an FFT or a Hadamard transform. Using SRHT as the control isolates the cost of the implementation from the cost of the rotation.",[111,849,850],{},"The paper puts the mechanism in one line, which I paraphrase:",[108,852,853],{},[111,854,855],{},"The cost is dispatch, not compute. The underlying rfft and irfft calls take single-digit microseconds, but the four-step pipeline multiplies that overhead.",[111,857,858],{},"That is why the same algorithm turns from +17% into −1% — not because quantization improved, but because four launches became two (Figure 1).",[167,860,171,862,171,867],{"className":861},[170],[173,863],{"src":864,"poster":865,"autoPlay":177,"loop":177,"muted":177,"playsInline":177,"ariaLabel":866},"\u002Fimages\u002Fblog\u002Fen\u002Fquantization-kernel-dispatch\u002Fkernel-fusion.mp4","\u002Fimages\u002Fblog\u002Fen\u002Fquantization-kernel-dispatch\u002Fkernel-fusion-poster.jpg","Left: four eager kernel dispatches. Right: two fused kernels, forward and inverse.",[180,868,869],{},"Figure 1: The same quantization, eager with four dispatches per layer per step; fused into a forward and an inverse kernel.",[115,871,873],{"id":872},"fusing-four-steps-into-two-kernels-is-not-enough","Fusing four steps into two kernels is not enough",[111,875,876],{},"The fusion puts the first two steps and the last two into one Metal kernel each: forward does rotate plus quantize; inverse does dequantize plus inverse-rotate. Data stays in fast on-chip memory instead of round-tripping to device memory.",[111,878,879],{},"The same comparison flips for SmolLM, but Qwen stays 1% slower:",[194,881,882,898],{},[197,883,884],{},[200,885,886,889,892,895],{},[203,887,888],{},"Model",[203,890,891],{},"SRHT baseline",[203,893,894],{},"Eager Δ",[203,896,897],{},"Fused SRFT (Δ)",[216,899,900,911,922],{},[200,901,902,904,906,908],{},[221,903,223],{},[221,905,226],{},[221,907,229],{},[221,909,910],{},"37.0 ms\u002Ftok (−1%)",[200,912,913,915,917,919],{},[221,914,237],{},[221,916,240],{},[221,918,243],{},[221,920,921],{},"211.9 ms\u002Ftok (−5%)",[200,923,924,926,928,930],{},[221,925,251],{},[221,927,254],{},[221,929,257],{},[221,931,932],{},"85.7 ms\u002Ftok (+1%)",[111,934,935],{},"(Δ against SRHT; percentages rounded; 256-token prompt plus 128 generated tokens; Apple M1. Not against fp16.)",[111,937,938],{},"Why is the gap larger on 1.7B? The eager penalty is roughly the same (+17% vs +16%). The difference appears after fusion: 1.7B has a 2048-dim KV per layer, 360M only 320, so the fused kernel's fixed launch cost is spread across more work. That is why it moves from −1% to −5%.",[111,940,941],{},"At this point the story looks clean: four steps become two kernels, and 17% slower becomes 1% faster (on 360M). But if you stop here and conclude \"quantization is always faster\", you will miss the next section.",[111,943,944],{},"The end-to-end −8.2% against fp16 is not fusion alone. It needs three conditions.",[111,946,947,950],{},[276,948,949],{},"First, the quantizer."," With a single per-token scale (one scale per token), Qwen's 4-bit ΔPPL explodes to +7975. The author switches to per-channel λ plus per-group abs-max (one scale per channel, one abs-max per 32 values) and pulls the same number down to +638.6. That recipe runs inside the fused kernel.",[111,952,953,956],{},[276,954,955],{},"Second, a dequant-prefix cache."," The dequantized prefix is cached between steps instead of recomputed; the last 16 tokens stay in fp16 and the cache is invalidated when that window fills. Without it, Qwen at a 1024-token prefix is 2.4% slower than fp16; with it, it is 1.6% faster. The paper's text says −0.7%; its own ms\u002Ftok, 137.1 → 134.9, implies −1.6%. We use the table's numbers and note the discrepancy.",[111,958,959,962],{},[276,960,961],{},"Third, the model architecture."," Gemma-3 1B has sliding-window layers: most layers see a fixed window and stay in fp16; only a few full-attention layers (which see the whole prefix) pay the dequant cost. That is why its advantage survives to 4096 tokens. Qwen has 28 full-attention layers, so the cost grows linearly with prefix length.",[111,964,965],{},"Put those three back into the end-to-end table:",[194,967,968,983],{},[197,969,970],{},[200,971,972,974,977,980],{},[203,973,888],{},[203,975,976],{},"Prefix",[203,978,979],{},"fp16 → int4 (ms\u002Ftok)",[203,981,982],{},"Latency Δ",[216,984,985,995,1005,1015,1025,1035],{},[200,986,987,989,991,993],{},[221,988,318],{},[221,990,321],{},[221,992,324],{},[221,994,327],{},[200,996,997,999,1001,1003],{},[221,998,318],{},[221,1000,334],{},[221,1002,337],{},[221,1004,340],{},[200,1006,1007,1009,1011,1013],{},[221,1008,318],{},[221,1010,347],{},[221,1012,350],{},[221,1014,353],{},[200,1016,1017,1019,1021,1023],{},[221,1018,318],{},[221,1020,360],{},[221,1022,363],{},[221,1024,366],{},[200,1026,1027,1029,1031,1033],{},[221,1028,251],{},[221,1030,321],{},[221,1032,375],{},[221,1034,378],{},[200,1036,1037,1039,1041,1043],{},[221,1038,251],{},[221,1040,334],{},[221,1042,387],{},[221,1044,390],{},[111,1046,1047],{},"(Δ against fp16; Apple M1; 256-token prompt plus 128 generated tokens. Qwen at 2048 is not in the table; see the next section. Qwen 1024's −1.6% is the arithmetic from 137.1 → 134.9; the paper's text says −0.7%. We use the table's numbers.)",[111,1049,1050],{},"On memory: Qwen is 3.0–3.1×; Gemma's 19.5× is not a like-for-like comparison (an apples-to-apples comparison inside the full-attention layers is about 3.2×), so do not compare it directly with Qwen.",[111,1052,1053],{},"The −8.2% is the best case, at a 256-token prefix. The same model at 4096 is −3.2%; Qwen at 1024 is −1.6%. \"Quantization is 8% faster\" is not a constant; it is a curve that moves with prefix length.",[115,1055,1057],{"id":1056},"before-you-celebrate-check-the-quality-bill","Before you celebrate, check the quality bill",[111,1059,1060],{},"Speed is one ledger. Quality is the other.",[111,1062,1063],{},"Quantization quality is usually measured as perplexity (PPL). ΔPPL is the change from the fp16 baseline; lower is better.",[111,1065,1066],{},"The 4-bit per-token ΔPPL on Cosmopedia (8192 held-out tokens):",[194,1068,1069,1084],{},[197,1070,1071],{},[200,1072,1073,1075,1077,1080,1082],{},[203,1074,888],{},[203,1076,420],{},[203,1078,1079],{},"No rotation (identity)",[203,1081,426],{},[203,1083,429],{},[216,1085,1086,1098,1110],{},[200,1087,1088,1090,1092,1094,1096],{},[221,1089,436],{},[221,1091,439],{},[221,1093,442],{},[221,1095,445],{},[221,1097,448],{},[200,1099,1100,1102,1104,1106,1108],{},[221,1101,223],{},[221,1103,455],{},[221,1105,458],{},[221,1107,461],{},[221,1109,464],{},[200,1111,1112,1114,1116,1118,1120],{},[221,1113,237],{},[221,1115,471],{},[221,1117,474],{},[221,1119,477],{},[221,1121,477],{},[111,1123,1124],{},"(The first two rows are the mean ± 1σ over three seeds; 1.7B ran one seed because of compute cost. 6-bit and 8-bit are lossless in this test.)",[111,1126,1127],{},"Rotation is not optional. At 135M, no rotation is +1.96; with rotation it is +0.38, roughly a 5× reduction. That is also why \"quantization breaks\" and \"quantization is just slower\" are often two sides of the same problem: if the rotation is wrong, the 4-bit error lands directly in PPL.",[111,1129,1130],{},"6-bit and 8-bit are lossless; 4-bit is the trade-off. 3-bit is worse in absolute terms; the paper does not sell 3-bit, and I would not start there.",[111,1132,1133],{},"Qwen's 4-bit is a different story. The Python reference's per-token ΔPPL is +7975.3; the Metal per-token version is +7727.9. With per-channel λ plus per-group g32, the fused version lands at +638.6 — a 12.5× improvement, in the same range as the Python reference's +654.6.",[111,1135,1136],{},"But +638.6 is still +638.6. A 12.5× improvement is not the same as deployable. The paper calls this \"closing the catastrophe\"; I think it is closer to moving a fire from the whole house into one room. If you enable 4-bit on Qwen, validate with task metrics, not just the relative PPL gain.",[111,1138,1139],{},"Gemma looks better: per-token +18.7, fused +3.6. The Python reference is −5.2; the +8.8 gap comes from the kernel's fp32 reduction and rounding order, not the algorithm. The author rules that out with two ablations.",[115,1141,1143],{"id":1142},"it-really-breaks-qwen-at-2k-emits-two-tokens","It really breaks: Qwen at 2K emits two tokens",[111,1145,1146],{},"Back to the warning.",[111,1148,1149],{},"Qwen2.5-1.5B at a 2048-token prefix, int4, emits two tokens and hits EOS (end-of-sequence); fp16 emits 64.",[194,1151,1152,1163],{},[197,1153,1154],{},[200,1155,1156,1159,1161],{},[203,1157,1158],{},"Item",[203,1160,523],{},[203,1162,526],{},[216,1164,1165,1175,1184],{},[200,1166,1167,1170,1172],{},[221,1168,1169],{},"Generated tokens (2048 prefix)",[221,1171,536],{},[221,1173,1174],{},"2 (EOS)",[200,1176,1177,1180,1182],{},[221,1178,1179],{},"Peak memory",[221,1181,547],{},[221,1183,550],{},[200,1185,1186,1189,1191],{},[221,1187,1188],{},"Author's fallback",[221,1190,558],{},[221,1192,1193],{},"fp16 above about 1.5K",[111,1195,1196],{},"There is more than one cause. The dequant-prefix cache plus int4 storage pushes peak memory from 4447 to 5527 MB (+24%); combined with quantization noise across 28 full-attention layers, the logits tip into an early EOS.",[111,1198,1199],{},"I think the design that makes 1024 faster is also part of what makes 2048 break. The prefix cache turns dequantization from \"slower as the prefix grows\" into a fixed cost, but it also occupies memory; on a full-attention stack like Qwen, both happen at once.",[111,1201,1202],{},"Why is Gemma fine? Its sliding-window layers stay in fp16; only a few full-attention layers pay the dequant cost. Longer prefixes grow the cost more slowly. Same cache, different architecture, different curve.",[111,1204,1205],{},"If you want to enable this cache on your own model, test 2K first. Not \"does it generate\", but \"is the output still correct, and does it stop early\". The author's fallback is explicit: above about 1.5K, go back to fp16.",[115,1207,1209],{"id":1208},"four-questions-for-any-quantization-benchmark","Four questions for any quantization benchmark",[111,1211,1212],{},"The most useful part of the paper is not \"Apple Silicon is fast\". It is a way to split a benchmark into questions.",[111,1214,1215,1218],{},[276,1216,1217],{},"First, the platform."," Unified memory or separate HBM? The paper says the conventional trade-off returns on GPUs with separate HBM and high dispatch overhead. The same kernel does not necessarily stay faster when you move it. Ask where the number was measured.",[111,1220,1221,1224],{},[276,1222,1223],{},"Second, the kernel."," Fused, or four separate steps? If the benchmark does not say, the number may be dominated by dispatch tax. Ask: how many dispatches per layer per step?",[111,1226,1227,1230],{},[276,1228,1229],{},"Third, the prefix and the architecture."," The −8% at 256 tokens is not the −3.2% at 4K; full-attention and sliding-window stacks curve differently. Ask for 256, 1K and 2K, not just the best chart.",[111,1232,1233,1236],{},[276,1234,1235],{},"Fourth, the quality."," What is the ΔPPL baseline, and which quantizer? +638.6 is better than +7975, but it is still +638.6. Per-token and per-channel+group are different things; do not compare them as if they were the same.",[194,1238,1239,1252],{},[197,1240,1241],{},[200,1242,1243,1246,1249],{},[203,1244,1245],{},"Question",[203,1247,1248],{},"Why it matters",[203,1250,1251],{},"How to check",[216,1253,1254,1265,1276,1287],{},[200,1255,1256,1259,1262],{},[221,1257,1258],{},"Unified memory or separate HBM?",[221,1260,1261],{},"Where the trade-off flips",[221,1263,1264],{},"Hardware spec; ask for per-platform numbers",[200,1266,1267,1270,1273],{},[221,1268,1269],{},"Fused kernel or four dispatches?",[221,1271,1272],{},"Dispatch tax can be 12–17% vs SRHT",[221,1274,1275],{},"Does the benchmark say \"4 dispatches\" or \"fused\"?",[200,1277,1278,1281,1284],{},[221,1279,1280],{},"Prefix length and attention pattern?",[221,1282,1283],{},"Long context may break",[221,1285,1286],{},"Ask for 256 \u002F 1K \u002F 2K; check full-attention vs sliding-window",[200,1288,1289,1292,1295],{},[221,1290,1291],{},"ΔPPL baseline and quantizer?",[221,1293,1294],{},"+638.6 is better than +7975 but still +638.6",[221,1296,1297],{},"Check the baseline; per-token or per-channel+group",[115,1299,1301],{"id":1300},"what-this-does-not-prove","What this does not prove",[670,1303,1304,1307,1310,1313,1316,1319],{},[673,1305,1306],{},"One platform: Apple M1 only; no discrete-HBM GPU.",[673,1308,1309],{},"Small models: 135M–1.7B; nothing above 7B.",[673,1311,1312],{},"A single-author preprint, not peer-reviewed; we did not reproduce it.",[673,1314,1315],{},"Gemma's 19.5× is not like-for-like; an apples-to-apples comparison inside the full-attention layers is about 3.2×.",[673,1317,1318],{},"Qwen at 2K breaks; the author's own fallback is 1.5K.",[673,1320,1321],{},"This is not the only path. It only addresses the cost of writing the four steps separately.",[115,1323,1325],{"id":1324},"conclusion-the-question-is-not-is-quantization-fast","Conclusion: the question is not \"is quantization fast?\"",[111,1327,1328],{},"Quantization is not a switch. It is a stack: platform, kernel, quantizer, prefix length, architecture. Change one and the answer changes.",[111,1330,1331],{},"The two ledgers:",[670,1333,1334,1340],{},[673,1335,1336,1339],{},[276,1337,1338],{},"Implementation tax."," The same method, eager four steps vs two fused kernels: 11–21 percentage points.",[673,1341,1342,1345],{},[276,1343,1344],{},"Deployment bill."," int4 cache vs fp16: up to −8.2% at a short prefix, single digits at long prefixes (Qwen 1024 is −1.6%, Gemma 4096 is −3.2%), and it can stop early.",[111,1347,1348],{},"One thing to do tomorrow: run your model at 256, 1K and 2K, record ms\u002Ftok and output quality. If it is a full-attention stack, keep an fp16 fallback above 1.5K; 2K is the test point.",[111,1350,1351,1352,1356],{},"If you want help running those three prefixes on your own model, our ",[722,1353,1355],{"href":1354},"\u002Fen\u002Fservices","AI readiness audit"," is exactly that: what can be handed to AI now, and what should wait.",[111,1358,1359,1360,1363],{},"The series so far: ",[722,1361,742],{"href":1362},"\u002Fen\u002Fblog\u002Fbroken-on-arrival-quantized-models"," on the recipe, and this essay on the implementation. The earlier memory-trade-off essay is available in Chinese. The question moves from \"should we quantize?\" to \"how is that kernel written?\"",[745,1365],{},[111,1367,1368],{},[750,1369,1370,1371,1374,1375,1378],{},"Paper: ",[722,1372,758],{"href":755,"rel":1373},[757]," (Mohamed Amine Bergach, Illumina). Previous essay: ",[722,1376,1377],{"href":1362},"the quantized model may be broken",".",{"title":769,"searchDepth":770,"depth":770,"links":1380},[1381,1382,1383,1384,1385,1386,1387,1388],{"id":812,"depth":770,"text":813},{"id":828,"depth":770,"text":829},{"id":872,"depth":770,"text":873},{"id":1056,"depth":770,"text":1057},{"id":1142,"depth":770,"text":1143},{"id":1208,"depth":770,"text":1209},{"id":1300,"depth":770,"text":1301},{"id":1324,"depth":770,"text":1325},"\u002Fimages\u002Fblog\u002Fen\u002Fquantization-kernel-dispatch.svg","Cover for the English essay, Quantization Speed Is Often a Kernel Problem","Against SRHT, eager int4 KV-cache quantization ran 12–17% slower; fusing the four steps into Metal kernels cut most of the tax. The 8.2% win at 256 tokens has conditions—and Qwen breaks at 2K.","en",{},"\u002Fimages\u002Fblog\u002Fen\u002Fquantization-kernel-dispatch.jpg","\u002Fblog\u002Fen\u002Fquantization-kernel-dispatch",{"title":802,"description":1391},"blog\u002Fen\u002Fquantization-kernel-dispatch",[794,795,796,797],"VrMGy2tQ__ko07oo7IbI-0WNfg5nBCIyvoC2Snzani0",[1401,1870],{"id":1402,"title":1403,"author":103,"body":1404,"cover":1856,"coverAlt":1857,"date":1858,"description":1859,"draft":784,"extension":785,"featured":784,"lang":786,"meta":1860,"navigation":177,"ogImage":1861,"path":1862,"seo":1863,"slug":1864,"stem":1865,"tags":1866,"translationKey":1864,"updated":798,"__hash__":1869},"blog\u002Fblog\u002Fzh-TW\u002Fbroken-on-arrival-quantized-models.md","你下載的量化模型可能是壞的，而且 checksum 驗不出來",{"type":105,"value":1405,"toc":1845},[1406,1415,1417,1424,1427,1430,1437,1441,1444,1451,1457,1468,1471,1474,1481,1491,1498,1502,1505,1563,1566,1572,1576,1579,1586,1593,1596,1600,1603,1708,1718,1729,1732,1745,1755,1769,1771,1789,1792,1797,1803,1810,1812,1832],[108,1407,1408],{},[111,1409,1410,1411,1414],{},"數字來自 arXiv 2609.05881v3 與作者的 ",[733,1412,1413],{},"quantcheck"," 稽核資料；「今天還在不在」是我們自己查的，腳本在文末。",[115,1416,117],{"id":117},[111,1418,1419,1420,1423],{},"你在筆電上 ",[733,1421,1422],{},"ollama pull qwen2.5-coder:3b","，選了一個 2-bit 的版本，因為記憶體不夠。",[111,1425,1426],{},"模型跑起來了，有回應，速度也不錯。但你叫它寫一個排序函式，它寫出來的東西編譯不過；再試一題，還是不過。",[111,1428,1429],{},"你會先懷疑自己：prompt 寫錯？溫度太高？還是這顆模型本來就這麼弱？",[111,1431,1432,1433,1436],{},"有一篇論文把這個懷疑變成了普查：",[276,1434,1435],{},"Ollama 官方庫裡 305 個「能寫程式」的模型 tag，有 7 個在 164 題 HumanEval+ 上幾乎全錯。"," 其中六個一題都沒對。",[115,1438,1440],{"id":1439},"最反直覺的地方檔案是好的","最反直覺的地方：檔案是好的",[111,1442,1443],{},"如果模型壞了，你會想「是不是下載壞了、檔案損毀」。",[111,1445,1446,1447,1450],{},"作者把其中五個檔案從原始權重重新量化回來，",[276,1448,1449],{},"tensor 對 tensor 比對，完全一致","。Qwen2.5-Coder-3B 那四個 tag，434 個 tensor 全部相同。",[111,1452,1453,1454],{},"也就是說：",[276,1455,1456],{},"這些檔案是「正確的壞檔案」。",[111,1458,1459,1460,1463,1464,1467],{},"checksum 會過、檔案大小會對、",[733,1461,1462],{},"ollama show"," 看起來一切正常——因為檔案真的沒問題。壞的是",[276,1465,1466],{},"量化的配方","：同樣的權重、同樣的工具鏈、同樣的 tensor 組合，少了校準，就把模型壓成了不會寫程式的形狀。",[115,1469,1470],{"id":1470},"為什麼會這樣",[111,1472,1473],{},"量化是「把權重從 16 bit 壓到 2–4 bit」。壓的過程要不斷決定「這個誤差要丟給哪個權重」。",[111,1475,1476,1477,1480],{},"預設的做法是均勻地丟；比較好的做法是",[276,1478,1479],{},"看校準資料","，把誤差集中在不重要的權重上——llama.cpp 的 importance matrix（imatrix）就是做這件事。",[111,1482,1483,1484,1487,1488,767],{},"沒有 imatrix，小模型在低 bit 特別容易崩。這次的七個 suspects 全部落在 ",[276,1485,1486],{},"q2_K 到 q3_K_L、模型 ≤4B"," 這個範圍；同一個範圍有 48 個 tag，等於 ",[276,1489,1490],{},"14.6%",[111,1492,1493,1494,1497],{},"但要注意：",[276,1495,1496],{},"配方本身不能預測成敗","。同樣的無校準配方，Qwen2.5-Coder-1.5B 的 q2_K–q3_K_L 全部正常（13–15 題）。所以不是「低 bit 一定壞」，是「壞了你不會知道」。",[115,1499,1501],{"id":1500},"加回校準救回大部分","加回校準，救回大部分",[111,1503,1504],{},"作者固定權重、工具鏈與 tensor 組合，只補上 imatrix：",[194,1506,1507,1520],{},[197,1508,1509],{},[200,1510,1511,1514,1517],{},[203,1512,1513],{},"Tag",[203,1515,1516],{},"原本",[203,1518,1519],{},"補 imatrix 後（滿分 164）",[216,1521,1522,1533,1543,1553],{},[200,1523,1524,1527,1530],{},[221,1525,1526],{},"Qwen2.5-Coder-3B q2_K",[221,1528,1529],{},"0",[221,1531,1532],{},"106–113",[200,1534,1535,1538,1540],{},[221,1536,1537],{},"Qwen2.5-Coder-3B q3_K_M",[221,1539,1529],{},[221,1541,1542],{},"76–89",[200,1544,1545,1548,1550],{},[221,1546,1547],{},"Qwen2.5-Coder-3B q3_K_L",[221,1549,1529],{},[221,1551,1552],{},"80–86",[200,1554,1555,1558,1560],{},[221,1556,1557],{},"Qwen2.5-Coder-3B q3_K_S",[221,1559,1529],{},[221,1561,1562],{},"13–30（救不回）",[111,1564,1565],{},"（論文摘要寫「0 → 76–113」；上面是 v3.4 稽核資料的分級數字，我以資料為準。）",[111,1567,1568,1571],{},[276,1569,1570],{},"同一顆模型、同樣的位元數，只差一張校準表，從 0 題變成三位數。"," 這就是「量化不只是壓縮，是配方」的意思。",[115,1573,1575],{"id":1574},"另一種壞法少了-chat-template","另一種壞法：少了 chat template",[111,1577,1578],{},"七個 suspects 之外，作者還測了 22 個 Hugging Face 社群檔案，其中兩個（TheBloke 的 deepseek-coder-6.7B Q3_K_M／Q4_K_M）在普查時只對 5、6 題。",[111,1580,1581,1582,1585],{},"原因不是量化，是",[276,1583,1584],{},"檔案沒有附 chat template","。Ollama 找不到 template，就用預設的 ChatML 去套，跟模型原本的指令格式不合——30 個回答裡有 28 個自己接出幻覺的對話輪。",[111,1587,1588,1589,1592],{},"補上正確 template 之後：",[276,1590,1591],{},"98 與 101 題","（滿分 164）。",[111,1594,1595],{},"所以「跑得起來」跟「它能好好回答」之間，還隔著一個 template。",[115,1597,1599],{"id":1598},"我們今天查了這些-tag-還在","我們今天查了：這些 tag 還在",[111,1601,1602],{},"文章不能只講九月的事。我們拿稽核資料裡的 blob digest，去比對 Ollama registry 今天的 manifest：",[194,1604,1605,1620],{},[197,1606,1607],{},[200,1608,1609,1611,1614,1617],{},[203,1610,1513],{},[203,1612,1613],{},"稽核時的 digest",[203,1615,1616],{},"今天",[203,1618,1619],{},"狀態",[216,1621,1622,1636,1648,1660,1672,1684,1696],{},[200,1623,1624,1627,1630,1633],{},[221,1625,1626],{},"qwen2.5-coder:3b-instruct-q2_K",[221,1628,1629],{},"14c4247b8ce4…",[221,1631,1632],{},"相同",[221,1634,1635],{},"還在",[200,1637,1638,1641,1644,1646],{},[221,1639,1640],{},"qwen2.5-coder:3b-instruct-q3_K_S",[221,1642,1643],{},"81d167af7fa3…",[221,1645,1632],{},[221,1647,1635],{},[200,1649,1650,1653,1656,1658],{},[221,1651,1652],{},"qwen2.5-coder:3b-instruct-q3_K_M",[221,1654,1655],{},"9bb5d28b842a…",[221,1657,1632],{},[221,1659,1635],{},[200,1661,1662,1665,1668,1670],{},[221,1663,1664],{},"qwen2.5-coder:3b-instruct-q3_K_L",[221,1666,1667],{},"7a88dc11faae…",[221,1669,1632],{},[221,1671,1635],{},[200,1673,1674,1677,1680,1682],{},[221,1675,1676],{},"phi3.5:3.8b-mini-instruct-q2_K",[221,1678,1679],{},"54d47caa8bf3…",[221,1681,1632],{},[221,1683,1635],{},[200,1685,1686,1689,1692,1694],{},[221,1687,1688],{},"llama3.2:1b-instruct-q2_K",[221,1690,1691],{},"d1ede02a2111…",[221,1693,1632],{},[221,1695,1635],{},[200,1697,1698,1701,1704,1706],{},[221,1699,1700],{},"llama3.2:1b-instruct-q3_K_S",[221,1702,1703],{},"81881dcc12ec…",[221,1705,1632],{},[221,1707,1635],{},[111,1709,1710,1713,1714,1717],{},[276,1711,1712],{},"七個都還在，digest 跟當時一模一樣。"," 這不是歷史事件，是你今天 ",[733,1715,1716],{},"ollama pull"," 就可能拿到的檔案。",[111,1719,1720,1721,1724,1725,1728],{},"（查法：",[733,1722,1723],{},"https:\u002F\u002Fregistry.ollama.ai\u002Fv2\u002Flibrary\u002F\u003Cmodel>\u002Fmanifests\u002F\u003Ctag>","，比對 ",[733,1726,1727],{},"application\u002Fvnd.ollama.image.model"," 那層的 digest。腳本在文末。）",[115,1730,1731],{"id":1731},"你現在該做的三件事",[111,1733,1734,1737,1738,1740,1741,1744],{},[276,1735,1736],{},"一、篩選。"," 作者釋出了 ",[733,1739,1413],{},"：對每個 tag 跑 15 題螢幕測試。成本大約 ",[276,1742,1743],{},"30 秒生成","（4090、不含下載）。66 個通過螢幕測試的重測，沒有一個是壞的——所以它是個堪用的預檢。",[111,1746,1747,1750,1751,1754],{},[276,1748,1749],{},"二、如果你自己發佈量化模型","：補上 imatrix，然後",[276,1752,1753],{},"在發佈前跑一次功能測試","。registry 不會幫你測。",[111,1756,1757,1760,1761,1764,1765,1768],{},[276,1758,1759],{},"三、檢查 chat template。"," ",[733,1762,1763],{},"ollama show \u003Cmodel> --template","；如果印出 ",[733,1766,1767],{},"{{ .Prompt }}","，代表沒有 template，模型可能用錯格式回答。",[115,1770,668],{"id":668},[670,1772,1773,1776,1779,1786],{},[673,1774,1775],{},"2.3% 是「Ollama 官方庫、能寫程式、8GB 以下」這個子集的比例，不是所有量化模型",[673,1777,1778],{},"只測程式碼能力（HumanEval+）；perplexity 只能當預檢，不能當功能證明",[673,1780,1781,1782,1785],{},"七個是",[276,1783,1784],{},"下限","：deepseek-coder-1.3B 全系列（含 fp16）都只對 1–6 題，無法用群組基線判定，作者沒有把它們算進去",[673,1787,1788],{},"作者是獨立研究者，資料與工具都公開，但這是單一 registry、單一 runtime 版本的結果",[115,1790,1791],{"id":1791},"結論",[111,1793,1794],{},[276,1795,1796],{},"可重現，不等於可用。",[111,1798,1799,1800,1802],{},"檔案一致、checksum 通過、",[733,1801,1462],{}," 正常——這些都只證明「檔案是你以為的那個檔案」，不證明「模型會做事」。",[111,1804,1805,1806,1809],{},"所以問題不是「量化能不能用」，是它",[276,1807,1808],{},"有沒有被測過","。目前答案是：registries 沒有，所以你得自己測。",[745,1811],{},[111,1813,1814],{},[750,1815,1816,1817,763,1822,763,1827,767],{},"原始資料：",[722,1818,1821],{"href":1819,"rel":1820},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2609.05881",[757],"arXiv 2609.05881v3",[722,1823,1826],{"href":1824,"rel":1825},"https:\u002F\u002Fgithub.com\u002Faditi-p31\u002Fquantcheck",[757],"quantcheck（稽核資料與工具）",[722,1828,1831],{"href":1829,"rel":1830},"https:\u002F\u002Fgithub.com\u002Follama\u002Follama\u002Fissues\u002F18252",[757],"Ollama issue #18252",[111,1833,1834],{},[750,1835,1836,1837,1840,1841,1844],{},"我們比對 digest 的腳本：",[733,1838,1839],{},"scripts\u002Fcheck-ollama-tags.ts","。跑 ",[733,1842,1843],{},"npm run check:ollama -- --list"," 會列出這七個 tag 今天的 digest 與稽核值是否一致。",{"title":769,"searchDepth":770,"depth":770,"links":1846},[1847,1848,1849,1850,1851,1852,1853,1854,1855],{"id":117,"depth":770,"text":117},{"id":1439,"depth":770,"text":1440},{"id":1470,"depth":770,"text":1470},{"id":1500,"depth":770,"text":1501},{"id":1574,"depth":770,"text":1575},{"id":1598,"depth":770,"text":1599},{"id":1731,"depth":770,"text":1731},{"id":668,"depth":770,"text":668},{"id":1791,"depth":770,"text":1791},"\u002Fimages\u002Fblog\u002Fbroken-on-arrival-quantized-models.svg","量化模型檔案完整但功能失效的示意圖，checksum 打勾、程式碼全錯","2026-11-02","有人把 Ollama 官方庫 305 個程式碼模型全跑了一遍，發現 7 個低 bit 量化版幾乎全錯。但檔案沒有壞——重建回來 tensor 完全一致，所以 checksum 抓不到。這篇講它怎麼發生的、你現在能做的三件事。",{},"\u002Fimages\u002Fblog\u002Fbroken-on-arrival-quantized-models.jpg","\u002Fblog\u002Fzh-tw\u002Fbroken-on-arrival-quantized-models",{"title":1403,"description":1859},"broken-on-arrival-quantized-models","blog\u002Fzh-TW\u002Fbroken-on-arrival-quantized-models",[794,1867,1868],"local-llm","gguf","iM5l1m1nXRGY-pGG2MmAYggyxY9V2uAe132PQbEixw8",{"id":1871,"title":1872,"author":103,"body":1873,"cover":2355,"coverAlt":2356,"date":2357,"description":2358,"draft":784,"extension":785,"featured":784,"lang":786,"meta":2359,"navigation":177,"ogImage":2360,"path":2361,"seo":2362,"slug":2363,"stem":2364,"tags":2365,"translationKey":2363,"updated":798,"__hash__":2367},"blog\u002Fblog\u002Fzh-TW\u002Fstrata-125b-on-12gb.md","125B 模型，12GB 顯卡，94 tok\u002Fs",{"type":105,"value":1874,"toc":2343},[1875,1892,1895,1898,1901,1904,1910,1917,1921,1924,1939,1942,1946,1953,1956,1959,1963,1969,1972,2022,2025,2028,2034,2041,2045,2048,2051,2054,2057,2118,2125,2130,2137,2140,2146,2152,2158,2161,2164,2170,2177,2184,2188,2191,2198,2201,2249,2252,2259,2266,2269,2289,2292,2295,2298,2305,2317,2320,2340],[108,1876,1877],{},[111,1878,1879,1880,1885,1886,1891],{},"這篇沒有我們的實測。數據來自 Qwen 官方發布、",[722,1881,1884],{"href":1882,"rel":1883},"https:\u002F\u002Fgithub.com\u002FBlockYugaLabs\u002FStrata",[757],"Strata 開源專案"," 的說明，以及一位對岸開發者的",[722,1887,1890],{"href":1888,"rel":1889},"https:\u002F\u002Fwww.bilibili.com\u002Fvideo\u002FBV1WwJy6EEuN\u002F",[757],"公開實測影片","。文中會標明哪個數字是誰給的。",[115,1893,1894],{"id":1894},"三個數字",[111,1896,1897],{},"125B 的模型。12GB 的顯示卡。94 tok\u002Fs。",[111,1899,1900],{},"半年前，這三個數字放在一起是笑話。當時想在家用電腦跑 27B 的模型，量化後 20GB，加上 KV cache，你得有 24GB 的卡才勉強開到 128K 上下文。",[111,1902,1903],{},"現在 125B 跑在 12GB 上，而且不是「能跑」，是「比你讀字還快」。",[111,1905,1906,1907,767],{},"真正變的不是模型變聰明，是",[276,1908,1909],{},"模型有多大，和硬體要多好，已經開始脫鉤",[111,1911,1912,1913,1916],{},"先講清楚前提：94 tok\u002Fs 是 RTX 5070（12GB）搭配 ",[276,1914,1915],{},"64GB 系統記憶體","的成績。只有 32GB 記憶體的話，只能跑砍半的 Coder 版本。這不是「插一張卡就好」的故事。",[115,1918,1920],{"id":1919},"兩個限制被分別繞過","兩個限制，被分別繞過",[111,1922,1923],{},"大模型跑不動，卡在兩件事：",[1925,1926,1927,1933],"ol",{},[673,1928,1929,1932],{},[276,1930,1931],{},"裝不下","：權重是硬的。這顆模型加上 51B 的 n-gram 查表，量化後檔案還是 70–80GB。",[673,1934,1935,1938],{},[276,1936,1937],{},"算不動","：每個 token 都要讓所有參數參與計算。",[111,1940,1941],{},"Qwen3.8-Flash-Next 處理了第二個，Strata 處理了第一個。",[115,1943,1945],{"id":1944},"第一個限制不是每個專家都要上班","第一個限制：不是每個專家都要上班",[111,1947,1948,1949,1952],{},"Flash-Next 是 MoE（混合專家）架構。官方數字是 125B 總參數、",[276,1950,1951],{},"6B 激活","：每個 token 的計算量，大約等於一個 6B 模型。",[111,1954,1955],{},"打個比方：傳統稠密模型是「公司裡每個部門都得參與每一個案子」，MoE 是「只叫相關的幾個部門進來開會」。",[111,1957,1958],{},"所以「算不動」這件事，先解決了一半。",[115,1960,1962],{"id":1961},"第二個限制把整台電腦當成一台機器","第二個限制：把整台電腦當成一台機器",[111,1964,1965,1966,767],{},"但 6B 的計算量，不代表 6B 的記憶體——",[276,1967,1968],{},"125B 的權重還是得存在某個地方",[111,1970,1971],{},"Strata 的做法是不要全部塞進顯存，按「多常用」分三層放（見圖 1）：",[194,1973,1974,1987],{},[197,1975,1976],{},[200,1977,1978,1981,1984],{},[203,1979,1980],{},"層",[203,1982,1983],{},"放什麼",[203,1985,1986],{},"佔用",[216,1988,1989,2000,2011],{},[200,1990,1991,1994,1997],{},[221,1992,1993],{},"顯存 VRAM",[221,1995,1996],{},"最常命中的專家",[221,1998,1999],{},"12–24 GB",[200,2001,2002,2005,2008],{},[221,2003,2004],{},"系統記憶體",[221,2006,2007],{},"全部專家（開機時載入）",[221,2009,2010],{},"約 47 GB",[200,2012,2013,2016,2019],{},[221,2014,2015],{},"SSD",[221,2017,2018],{},"其餘權重與 n-gram 查表",[221,2020,2021],{},"檔案總量 70–80 GB",[111,2023,2024],{},"（47GB 是那位開發者在 IQ3_S 量化下的實測；官方建議 64GB 記憶體就是為了裝下這一層。）",[111,2026,2027],{},"跑起來的時候，GPU 算它手上的專家；沒命中的，要嘛走 PCIe 搬進顯存，要嘛直接讓 CPU 用記憶體裡的權重算。",[111,2029,2030,2031,767],{},"這就是為什麼 12GB 的卡能跑——",[276,2032,2033],{},"顯存不再是「裝不裝得下」的容器，而是「快取多大」的問題",[111,2035,2036],{},[2037,2038],"img",{"alt":2039,"src":2040},"如圖 1：三層記憶體階層——顯存放熱門專家、記憶體放全部專家、SSD 放冷門權重與 n-gram 查表","\u002Fimages\u002Fblog\u002Fstrata-125b-on-12gb\u002F01-memory-tiers.svg",[115,2042,2044],{"id":2043},"還有一個加速器猜然後驗證","還有一個加速器：猜，然後驗證",[111,2046,2047],{},"除了省記憶體，Strata 還用了投機解碼：小模型先猜接下來幾個 token，大模型一次驗證。",[111,2049,2050],{},"猜對就省下好幾步，官方數字大約快 1.6–1.8 倍。",[115,2052,2053],{"id":2053},"實際多快",[111,2055,2056],{},"官方在 RTX 5070（12GB 顯存、64GB RAM）上測到的：",[194,2058,2059,2072],{},[197,2060,2061],{},[200,2062,2063,2066,2069],{},[203,2064,2065],{},"量化",[203,2067,2068],{},"寫答案",[203,2070,2071],{},"讀 prompt（32K）",[216,2073,2074,2085,2096,2107],{},[200,2075,2076,2079,2082],{},[221,2077,2078],{},"Q2_0",[221,2080,2081],{},"94 tok\u002Fs",[221,2083,2084],{},"2,650 tok\u002Fs",[200,2086,2087,2090,2093],{},[221,2088,2089],{},"IQ2_XS",[221,2091,2092],{},"79 tok\u002Fs",[221,2094,2095],{},"2,090 tok\u002Fs",[200,2097,2098,2101,2104],{},[221,2099,2100],{},"IQ3_XXS",[221,2102,2103],{},"62 tok\u002Fs",[221,2105,2106],{},"1,750 tok\u002Fs",[200,2108,2109,2112,2115],{},[221,2110,2111],{},"IQ3_S",[221,2113,2114],{},"53 tok\u002Fs",[221,2116,2117],{},"1,620 tok\u002Fs",[111,2119,2120,2121,2124],{},"24GB 的卡更快。那位開發者在他的 4090 + 64GB RAM 上實測：",[276,2122,2123],{},"解碼約 140 tok\u002Fs、prefill 約 4,400 tok\u002Fs","，256K 上下文開滿也還跑得動。",[108,2126,2127],{},[111,2128,2129],{},"他第一次測到 203 tok\u002Fs，後來發現是 benchmark 沒把 reasoning 時間與快取命中算進去。修正後才是 140。這種「先懷疑自己的量測」比數字本身更有價值。",[111,2131,2132,2133,2136],{},"要注意上面那張表的「讀 prompt 2,650 tok\u002Fs」是快取熱起來之後的數字。",[276,2134,2135],{},"冷啟動第一次讀長文會慢很多","——官方文件寫的是每 30,000 tokens 約一分鐘，大約 500 tok\u002Fs。",[115,2138,2139],{"id":2139},"代價",[111,2141,2142,2145],{},[276,2143,2144],{},"一、量化到 2-bit。"," 94 tok\u002Fs 是 Q2_0 的數字，那是很兇的壓縮。要品質就得犧牲速度——IQ3_S 剩 53 tok\u002Fs。這是取捨，不是免費的。",[111,2147,2148,2151],{},[276,2149,2150],{},"二、記憶體與啟動。"," 開機要把 35–55GB 載進記憶體，電腦會卡 1–3 分鐘；整個程序佔用超過 50GB。32GB 的機器只能跑 Coder 變體（砍掉一半專家），官方也提醒它在中文本表現較弱。",[111,2153,2154,2157],{},[276,2155,2156],{},"三、一次一個請求。"," 預設不支援併發，第二個請求要排隊。開併發得調參數，12GB 的卡上會讓每個回答變慢。",[115,2159,2160],{"id":2160},"調參的迷思",[111,2162,2163],{},"Strata 有不少參數可調：PCIE_FRAC（GPU 與 CPU 怎麼分攤計算）、CPU Workers、MTP 深度、KVGROW。",[111,2165,2166,2167,767],{},"那位開發者花了大量時間做矩陣測試，他的結論是：",[276,2168,2169],{},"官方自動校準就夠了",[111,2171,2172,2173,2176],{},"他自己測出的「快 5%、快 9%」，多跑幾輪就消失，有時候還反過來。原因是速度受當下生成的內容、專家路由狀態、MTP 命中率、GPU 快取影響——這些變異比參數本身還大。這是",[276,2174,2175],{},"在他那台機器上","的結論，換硬體要重新驗證。",[111,2178,2179,2180,2183],{},"有一個參數值得單獨提：",[276,2181,2182],{},"KVGROW","。預設開 256K 上下文時，引擎會先把整段 KV cache 的顯存預留起來，結果短對話也沒位置放熱專家，速度掉到最慢檔。KVGROW 讓 cache 跟著實際用量長大：短對話快，長對話才讓位。",[115,2185,2187],{"id":2186},"所以該不該自架","所以，該不該自架？",[111,2189,2190],{},"先把興奮放一邊，算一算。匯率抓 1 美元 ≈ NT$31.5。",[111,2192,2193,2194,2197],{},"Qwen3.8-Flash 的 API 價格是 ",[276,2195,2196],{},"$0.16 \u002F 百萬 input token、$0.47 \u002F 百萬 output","——output 是 input 的三倍，這段先只算 input，方便比較。",[111,2199,2200],{},"一台跑得舒服的機器（4090 或同級 + 64GB RAM）大約台幣 8–10 萬。",[194,2202,2203,2216],{},[197,2204,2205],{},[200,2206,2207,2210,2213],{},[203,2208,2209],{},"每月 input 用量",[203,2211,2212],{},"API 成本",[203,2214,2215],{},"自架電費",[216,2217,2218,2229,2239],{},[200,2219,2220,2223,2226],{},[221,2221,2222],{},"1,000 萬 token",[221,2224,2225],{},"約 NT$50",[221,2227,2228],{},"NT$250–370",[200,2230,2231,2234,2237],{},[221,2232,2233],{},"1 億 token",[221,2235,2236],{},"約 NT$500",[221,2238,2228],{},[200,2240,2241,2244,2247],{},[221,2242,2243],{},"10 億 token",[221,2245,2246],{},"約 NT$5,000",[221,2248,2228],{},[111,2250,2251],{},"電費怎麼算的：推論時約 350W，每天用兩小時、其餘時間閒置 80W，一個月約 74 度；台電累進電價抓每度 NT$3–5，就是每個月 250–370 元。24 小時滿載的話是 250 度左右，大約 NT$750–1,250。",[111,2253,2254,2255,2258],{},"看表就知道結論：",[276,2256,2257],{},"每月十億 token 以下，API 連電費都比較便宜","，還沒算機器折舊。",[111,2260,2261,2262,2265],{},"粗算回本點：",[733,2263,2264],{},"10 萬 ÷ (5,000 − 370) ≈ 22 個月","——那是每月十億 token 的量才成立。對絕大多數中小企業，這個量級不會出現。",[111,2267,2268],{},"那自架還有什麼意義？三個：",[670,2270,2271,2277,2283],{},[673,2272,2273,2276],{},[276,2274,2275],{},"隱私","：資料不出門。這是最難用錢換的",[673,2278,2279,2282],{},[276,2280,2281],{},"離線","：工廠、船上、沒有網路的現場",[673,2284,2285,2288],{},[276,2286,2287],{},"不受限","：不用擔心費率調整、政策變更、API 下架",[111,2290,2291],{},"如果你不屬於這三種，用 API 就好，而且會用得比自架開心。",[115,2293,2294],{"id":2294},"真正的重點",[111,2296,2297],{},"這篇想說的不是「125B 好強」。",[111,2299,2300,2301,2304],{},"是",[276,2302,2303],{},"模型大小已經不能拿來判斷硬體需求了","。同樣 125B，你可以用 360GB 的伺服器跑，也可以用 12GB 的遊戲卡跑——差別只在於有沒有人幫你把記憶體階層調度好。",[111,2306,2307,2308,2311,2312,2316],{},"這也是我們做",[722,2309,2310],{"href":724},"地端部署","時真正在處理的問題：不是「這顆模型跑不跑得動」，而是「你的瓶頸在哪一層」。我們在",[722,2313,2315],{"href":2314},"\u002Fwork\u002Fembedded-vision-background","資源受限裝置","上累積的經驗，換到 LLM 上其實是同一件事。",[111,2318,2319],{},"如果你正在評估要不要自架，先問三個問題：",[1925,2321,2322,2328,2334],{},[673,2323,2324,2327],{},[276,2325,2326],{},"每月 token 量有沒有到億級？"," 沒有，API 幾乎一定更划算。",[673,2329,2330,2333],{},[276,2331,2332],{},"資料能不能出門？"," 不能，才輪到自架。",[673,2335,2336,2339],{},[276,2337,2338],{},"有沒有 64GB 記憶體與一張 12GB 以上的卡？"," 沒有，連門檻都不到。",[111,2341,2342],{},"三題有兩題是「否」，答案就是 API。",{"title":769,"searchDepth":770,"depth":770,"links":2344},[2345,2346,2347,2348,2349,2350,2351,2352,2353,2354],{"id":1894,"depth":770,"text":1894},{"id":1919,"depth":770,"text":1920},{"id":1944,"depth":770,"text":1945},{"id":1961,"depth":770,"text":1962},{"id":2043,"depth":770,"text":2044},{"id":2053,"depth":770,"text":2053},{"id":2139,"depth":770,"text":2139},{"id":2160,"depth":770,"text":2160},{"id":2186,"depth":770,"text":2187},{"id":2294,"depth":770,"text":2294},"\u002Fimages\u002Fblog\u002Fstrata-125b-on-12gb.svg","125B 模型跑在 12GB 顯卡上的示意圖","2026-10-24","Strata 讓 125B 的 Qwen3.8-Flash-Next 跑在 12GB 顯卡的遊戲機上，解碼 94 tok\u002Fs。拆解它靠什麼做到、代價在哪裡，以及中小企業該不該自架。",{},"\u002Fimages\u002Fblog\u002Fstrata-125b-on-12gb.jpg","\u002Fblog\u002Fzh-tw\u002Fstrata-125b-on-12gb",{"title":1872,"description":2358},"strata-125b-on-12gb","blog\u002Fzh-TW\u002Fstrata-125b-on-12gb",[1867,2366,797],"qwen","AzAFYoV9JKM6X1-qHOAhv64-PaZVkfvITS_J8_39_Oc",1793954704585]