[{"data":1,"prerenderedAt":10895},["ShallowReactive",2],{"site-info":3,"post-arxiv-2512-22471v2":99,"related-zh-TW-arxiv-2512-22471v2":5922},{"id":4,"company":5,"extension":16,"meta":17,"services":18,"social":96,"stem":97,"__hash__":98},"site\u002Fsite.yml",{"name":6,"nameEn":7,"email":8,"address":9,"addressEn":10,"lineUrl":11,"lineId":12,"duns":13,"youtube":14,"youtubeId":15},"銡策數位有限公司","Jecision Digital Co., Ltd.","contact@jecision.com","231 新北市新店區北新路二段 262 號 3 樓之 3","3F.-3, No. 262, Sec. 2, Beixin Rd., Xindian Dist., New Taipei City 231, Taiwan","https:\u002F\u002Fline.me\u002FR\u002Fti\u002Fp\u002F@374qnumi","@374qnumi","963905705","https:\u002F\u002Fwww.youtube.com\u002Fshorts\u002F6UE0WTmyjsg","6UE0WTmyjsg","yml",{},[19,43,69],{"key":20,"zh":21,"en":32},"audit",{"title":22,"tagline":23,"summary":24,"points":25,"deliverable":29,"detail":30,"duration":31},"企業 AI 導入健檢","先看清楚，再花錢","60 分鐘線上訪談，盤點最耗時、最依賴人力的營運環節；我們會告訴你哪些工作現在就能交給 AI、哪些還不成熟。",[26,27,28],"現況盤點：梳理日常營運中最耗時、最依賴人力的環節","痛點診斷：判斷哪些工作現在就能交給 AI，哪些建議先保留人力","可行性評估：依你的業務特性與預算，篩選適合的模型與工具","《企業 AI 導入建議報告書》（PDF）","含 1–2 個「開發成本最低、成效最明顯」的優先項目、技術棧建議、預期節省的人力時間與下一步","訪談 60 分鐘，報告書 3 個工作天內交付",{"title":33,"tagline":34,"summary":35,"points":36,"deliverable":40,"detail":41,"duration":42},"AI Readiness Audit","Look before you spend","A 60-minute remote audit of the workflows that eat the most time and people-power — and an honest answer on what AI can take over today, and what it cannot.",[37,38,39],"Audit the daily workflows that consume the most time and headcount","Triage which tasks AI can own today, and which should stay human","Shortlist the models and tools that fit your budget and constraints","AI Adoption Recommendation Report (PDF)","One or two highest-ROI entry points, a recommended tech stack, expected hours saved, and the concrete next step","60-minute call, report delivered within 3 business days",{"key":44,"zh":45,"en":56},"prompt",{"title":46,"tagline":47,"summary":48,"points":49,"deliverable":53,"detail":54,"duration":55},"提示詞工程","讓 AI 穩定產出，而不是碰運氣","針對你的實際任務設計提示詞與多步驟工作流，並做極限測試，確保換了資料、換了人操作也不會走鐘。",[50,51,52],"客製提示詞：ChatGPT、Claude、Gemini、Midjourney 等主流工具的專屬模板","除錯與優化：修掉幻覺、格式跑掉、時好時壞的問題","工作流建置：把複雜任務拆成多步驟提示詞與可重複的 SOP","提示詞文件或設定檔","附使用說明與範例，並提供 1 對 1 教學，確保你與團隊能自己維護","依任務複雜度報價，通常 3–7 個工作天",{"title":57,"tagline":58,"summary":59,"points":60,"deliverable":66,"detail":67,"duration":68},"Prompt Engineering","Make AI produce reliably, not by luck","Prompt sets and multi-step workflows designed for your actual tasks, stress-tested so they keep working when the data or the operator changes.",[61,62,64],"Custom prompts for ChatGPT, Claude, Gemini, Midjourney and friends",{"Debugging and tuning":63},"kill hallucinations, broken formats and flaky output",{"Workflow design":65},"break complex work into repeatable multi-step SOPs","Prompt library or exportable config","Ships with usage notes and examples, plus a 1:1 walkthrough so your team can maintain it","Quoted by complexity, typically 3–7 business days",{"key":70,"zh":71,"en":82},"poc",{"title":72,"tagline":73,"summary":74,"points":75,"deliverable":79,"detail":80,"duration":81},"極速 PoC","五天，看到能跑的東西","不談架構藍圖，直接做一個能動的雛形。五個工作天內交付線上可操作的智能體，用你真實的資料驗證。",[76,77,78],"Day 1：15 分鐘對焦，鎖定單一痛點與資料盤點","Day 2–4：模型串接、邏輯設計與雛形開發","Day 5：Demo 展示與 AI 擴展建議書","可線上操作的智能體雛形","交付範圍為概念驗證與展示，不含後續維運或大規模伺服器部署；要進階成完整系統會另行報價","5 個工作天",{"title":83,"tagline":84,"summary":85,"points":86,"deliverable":93,"detail":94,"duration":95},"Rapid PoC","Five days to something that runs","No architecture decks — a working prototype built in five business days, validated against your real data.",[87,89,91],{"Day 1":88},"15-minute scoping call, one pain point, data inventory",{"Day 2–4":90},"model integration, logic design, prototype build",{"Day 5":92},"live demo plus an AI expansion recommendation","A runnable online agent prototype","Proof of concept and demo scope only; ongoing operations and large-scale deployment are quoted separately","5 business days",[],"site","XTwARfVAOcrXNiKehsM6V6cdkd2HXy4d5yZVrSWrI3Q",{"zh":100,"en":5920},{"id":101,"title":102,"author":103,"body":104,"cover":5904,"coverAlt":102,"date":5905,"description":5906,"draft":5907,"extension":5908,"featured":5907,"lang":5909,"meta":5910,"navigation":5911,"path":5912,"seo":5913,"slug":5914,"stem":5915,"tags":5916,"translationKey":5914,"updated":5920,"__hash__":5921},"blog\u002Fblog\u002Fzh-TW\u002Farxiv-2512-22471v2.md","Transformer 注意力機制的貝葉斯幾何學：機制與理論深度分析報告","Jecision",{"type":105,"value":106,"toc":5868},"minimark",[107,115,119,122,125,131,138,141,144,331,337,344,431,434,691,694,725,728,1066,1069,1287,1345,1348,1751,1759,1765,1776,1782,1785,2241,2496,2499,2786,2792,2795,3081,3539,3542,3620,3750,3756,3759,4023,4029,4032,4038,4041,4267,4273,4276,4821,4827,4830,4854,4857,5174,5180,5186,5197,5203,5206,5226,5232,5235,5308,5314,5317,5337,5343,5346,5352,5355,5514,5520,5601,5607,5610,5616,5619,5639,5645,5651,5681,5687,5694,5712,5718,5721,5811,5817,5823,5838,5841,5844,5851],[108,109,111],"h2",{"id":110},"_1-執行摘要",[112,113,114],"strong",{},"1. 執行摘要",[116,117,118],"p",{},"在當前的人工智慧領域中，Transformer 架構已成為大型語言模型（LLMs）的基石。然而，關於其強大的上下文學習（In-Context Learning）能力背後的確切運作機制，學術界長期以來存在著激烈的爭論。核心問題在於：這些模型究竟是僅僅進行高維度的模式匹配與記憶，還是在演算法層面上真正實現了機率推理？由 Naman Agarwal、Siddhartha R. Dalal 和 Vishal Misra 於 2026 年發表的開創性研究《Transformer 注意力機制的貝葉斯幾何學》（The Bayesian Geometry of Transformer Attention）為此提供了具體的實證解答。",[116,120,121],{},"本報告旨在對該研究進行詳盡的深度分析與總結。研究團隊透過構建「貝葉斯風洞」（Bayesian Wind Tunnels）——一種具備解析解且無法通過記憶解決的受控實驗環境——證明了小型 Transformer 模型不僅能夠模擬貝葉斯推理，更能以機器級的精度（誤差低至10^{-3} 到 10^{-4} bits）追蹤確切的貝葉斯後驗分佈。與此形成鮮明對比的是，參數規模相當的多層感知機（MLP）在相同任務中表現出災難性的失敗，這確立了注意力機制在動態推理中的不可或缺性。",[116,123,124],{},"進一步的幾何診斷揭示了 Transformer 實現此一過程的三階段機制：首先在底層構建正交的假設框架（Hypothesis Frame），接著在中層透過注意力機制進行順序性的假設消除（Sequential Elimination），最後在頂層將後驗機率精確編碼於低維度的值流形（Value Manifold）上。這些發現不僅解釋了 Transformer 的成功原因，更為提示工程（Prompt Engineering）提供了理論依據——即提示應被視為消除假設空間的證據。",[108,126,128],{"id":127},"_2-理論背景與研究動機",[112,129,130],{},"2. 理論背景與研究動機",[132,133,135],"h3",{"id":134},"_21-大型語言模型的驗證危機",[112,136,137],{},"2.1 大型語言模型的驗證危機",[116,139,140],{},"隨著大型語言模型（如 GPT-4, Claude 等）展現出驚人的推理能力，一個根本性的科學難題隨之浮現：我們無法驗證這些模型是否真的在「推理」。自然語言數據本質上缺乏「基本真理」（Ground Truth）的後驗機率分佈。例如，在一段文本續寫任務中，下一個詞的真實機率分佈是未知的，且往往受到多種潛在因素的影響。此外，現代 LLM 的訓練數據量極其龐大，使得研究人員難以區分模型是在進行真正的即時機率計算，還是僅僅在提取海量訓練數據中記憶的片段。",[116,142,143],{},"這種「驗證危機」導致了對模型能力的解釋往往停留在定性描述或擬人化的比喻上，缺乏嚴謹的定量評估。為了突破這一局限，Agarwal 等人提出了「貝葉斯風洞」的概念。這是一種合成的實驗環境，設計上必須滿足三個嚴格條件：",[145,146,147,154,325],"ol",{},[148,149,150,153],"li",{},[112,151,152],{},"解析可解性（Analytic Tractability）："," 在序列的每一步，真實的貝葉斯後驗分佈必須是已知且可精確計算的閉式解（Closed-form）。",[148,155,156,159,160,257,258,324],{},[112,157,158],{},"抗記憶性（Memorization Resistance）："," 假設空間必須大到組合爆炸的程度（例如 ",[161,162,165,194],"span",{"className":163},[164],"katex",[161,166,169],{"className":167},[168],"katex-mathml",[170,171,173],"math",{"xmlns":172},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[174,175,176,189],"semantics",{},[177,178,179],"mrow",{},[180,181,182,186],"msup",{},[183,184,185],"mn",{},"10",[183,187,188],{},"18",[190,191,193],"annotation",{"encoding":192},"application\u002Fx-tex","10^{18}",[161,195,199],{"className":196,"ariaHidden":198},[197],"katex-html","true",[161,200,203,208,213],{"className":201},[202],"base",[161,204],{"className":205,"style":207},[206],"strut","height:0.8141em;",[161,209,212],{"className":210},[211],"mord","1",[161,214,216,220],{"className":215},[211],[161,217,219],{"className":218},[211],"0",[161,221,224],{"className":222},[223],"msupsub",[161,225,228],{"className":226},[227],"vlist-t",[161,229,232],{"className":230},[231],"vlist-r",[161,233,236],{"className":234,"style":207},[235],"vlist",[161,237,239,244],{"style":238},"top:-3.063em;margin-right:0.05em;",[161,240],{"className":241,"style":243},[242],"pstrut","height:2.7em;",[161,245,251],{"className":246},[247,248,249,250],"sizing","reset-size6","size3","mtight",[161,252,254],{"className":253},[211,250],[161,255,188],{"className":256},[211,250]," 或 ",[161,259,261,280],{"className":260},[164],[161,262,264],{"className":263},[168],[170,265,266],{"xmlns":172},[174,267,268,277],{},[177,269,270],{},[180,271,272,274],{},[183,273,185],{},[183,275,276],{},"40",[190,278,279],{"encoding":192},"10^{40}",[161,281,283],{"className":282,"ariaHidden":198},[197],[161,284,286,289,292],{"className":285},[202],[161,287],{"className":288,"style":207},[206],[161,290,212],{"className":291},[211],[161,293,295,298],{"className":294},[211],[161,296,219],{"className":297},[211],[161,299,301],{"className":300},[223],[161,302,304],{"className":303},[227],[161,305,307],{"className":306},[231],[161,308,310],{"className":309,"style":207},[235],[161,311,312,315],{"style":238},[161,313],{"className":314,"style":243},[242],[161,316,318],{"className":317},[247,248,249,250],[161,319,321],{"className":320},[211,250],[161,322,276],{"className":323},[211,250]," 種可能性），從而數學上排除模型通過背誦訓練樣本來解決任務的可能性。",[148,326,327,330],{},[112,328,329],{},"推理必要性（Inference Necessity）："," 任務必須要求模型根據上下文中的新證據動態更新信念，這正是貝葉斯推理的定義。",[132,332,334],{"id":333},"_22-交叉熵與貝葉斯推理的數學對偶性",[112,335,336],{},"2.2 交叉熵與貝葉斯推理的數學對偶性",[116,338,339,340,343],{},"本研究的理論基礎建立在深度學習訓練目標與機率論之間的深層聯繫上。研究團隊提出了",[112,341,342],{},"定理 1：交叉熵的群體最優解","（Population Optimum of Cross-Entropy）。",[116,345,346,347],{},"設想一個由潛在參數\n",[161,348,350,382],{"className":349},[164],[161,351,353],{"className":352},[168],[170,354,355],{"xmlns":172},[174,356,357,379],{},[177,358,359,363,367,370,374,376],{},[360,361,362],"mi",{},"θ",[364,365,366],"mo",{},"∼",[360,368,369],{},"π",[364,371,373],{"stretchy":372},"false","(",[360,375,362],{},[364,377,378],{"stretchy":372},")",[190,380,381],{"encoding":192},"\\theta \\sim \\pi(\\theta)",[161,383,385,409],{"className":384,"ariaHidden":198},[197],[161,386,388,392,397,402,406],{"className":387},[202],[161,389],{"className":390,"style":391},[206],"height:0.6944em;",[161,393,362],{"className":394,"style":396},[211,395],"mathnormal","margin-right:0.0278em;",[161,398],{"className":399,"style":401},[400],"mspace","margin-right:0.2778em;",[161,403,366],{"className":404},[405],"mrel",[161,407],{"className":408,"style":401},[400],[161,410,412,416,420,424,427],{"className":411},[202],[161,413],{"className":414,"style":415},[206],"height:1em;vertical-align:-0.25em;",[161,417,369],{"className":418,"style":419},[211,395],"margin-right:0.0359em;",[161,421,373],{"className":422},[423],"mopen",[161,425,362],{"className":426,"style":396},[211,395],[161,428,378],{"className":429},[430],"mclose",[116,432,433],{},"索引的任務族。對於每個任務，模型觀察到上下文",[116,435,436],{},[161,437,439,496],{"className":438},[164],[161,440,442],{"className":441},[168],[170,443,444],{"xmlns":172},[174,445,446,493],{},[177,447,448,451,454],{},[360,449,450],{},"c",[364,452,453],{},"=",[455,456,457,482,490],"msubsup",{},[177,458,459,461,470,473,480],{},[364,460,373],{"stretchy":372},[462,463,464,467],"msub",{},[360,465,466],{},"x",[360,468,469],{},"i",[364,471,472],{"separator":198},",",[462,474,475,478],{},[360,476,477],{},"y",[360,479,469],{},[364,481,378],{"stretchy":372},[177,483,484,486,488],{},[360,485,469],{},[364,487,453],{},[183,489,212],{},[360,491,492],{},"k",[190,494,495],{"encoding":192},"c={(x_i, y_i)}_{i=1}^k",[161,497,499,518],{"className":498,"ariaHidden":198},[197],[161,500,502,506,509,512,515],{"className":501},[202],[161,503],{"className":504,"style":505},[206],"height:0.4306em;",[161,507,450],{"className":508},[211,395],[161,510],{"className":511,"style":401},[400],[161,513,453],{"className":514},[405],[161,516],{"className":517,"style":401},[400],[161,519,521,525],{"className":520},[202],[161,522],{"className":523,"style":524},[206],"height:1.2887em;vertical-align:-0.2997em;",[161,526,528,632],{"className":527},[211],[161,529,531,534,580,584,588,629],{"className":530},[211],[161,532,373],{"className":533},[423],[161,535,537,540],{"className":536},[211],[161,538,466],{"className":539},[211,395],[161,541,543],{"className":542},[223],[161,544,547,571],{"className":545},[227,546],"vlist-t2",[161,548,550,566],{"className":549},[231],[161,551,554],{"className":552,"style":553},[235],"height:0.3117em;",[161,555,557,560],{"style":556},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[161,558],{"className":559,"style":243},[242],[161,561,563],{"className":562},[247,248,249,250],[161,564,469],{"className":565},[211,395,250],[161,567,570],{"className":568},[569],"vlist-s","​",[161,572,574],{"className":573},[231],[161,575,578],{"className":576,"style":577},[235],"height:0.15em;",[161,579],{},[161,581,472],{"className":582},[583],"mpunct",[161,585],{"className":586,"style":587},[400],"margin-right:0.1667em;",[161,589,591,594],{"className":590},[211],[161,592,477],{"className":593,"style":419},[211,395],[161,595,597],{"className":596},[223],[161,598,600,621],{"className":599},[227,546],[161,601,603,618],{"className":602},[231],[161,604,606],{"className":605,"style":553},[235],[161,607,609,612],{"style":608},"top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;",[161,610],{"className":611,"style":243},[242],[161,613,615],{"className":614},[247,248,249,250],[161,616,469],{"className":617},[211,395,250],[161,619,570],{"className":620},[569],[161,622,624],{"className":623},[231],[161,625,627],{"className":626,"style":577},[235],[161,628],{},[161,630,378],{"className":631},[430],[161,633,635],{"className":634},[223],[161,636,638,682],{"className":637},[227,546],[161,639,641,679],{"className":640},[231],[161,642,645,666],{"className":643,"style":644},[235],"height:0.989em;",[161,646,648,651],{"style":647},"top:-2.4003em;margin-right:0.05em;",[161,649],{"className":650,"style":243},[242],[161,652,654],{"className":653},[247,248,249,250],[161,655,657,660,663],{"className":656},[211,250],[161,658,469],{"className":659},[211,395,250],[161,661,453],{"className":662},[405,250],[161,664,212],{"className":665},[211,250],[161,667,669,672],{"style":668},"top:-3.2029em;margin-right:0.05em;",[161,670],{"className":671,"style":243},[242],[161,673,675],{"className":674},[247,248,249,250],[161,676,492],{"className":677,"style":678},[211,395,250],"margin-right:0.0315em;",[161,680,570],{"className":681},[569],[161,683,685],{"className":684},[231],[161,686,689],{"className":687,"style":688},[235],"height:0.2997em;",[161,690],{},[116,692,693],{},"並需要預測新的標籤",[116,695,696],{},[161,697,699,712],{"className":698},[164],[161,700,702],{"className":701},[168],[170,703,704],{"xmlns":172},[174,705,706,710],{},[177,707,708],{},[360,709,477],{},[190,711,477],{"encoding":192},[161,713,715],{"className":714,"ariaHidden":198},[197],[161,716,718,722],{"className":717},[202],[161,719],{"className":720,"style":721},[206],"height:0.625em;vertical-align:-0.1944em;",[161,723,477],{"className":724,"style":419},[211,395],[116,726,727],{},"訓練目標是最小化預期交叉熵損失：",[161,729,732],{"className":730},[731],"katex-display",[161,733,735,843],{"className":734},[164],[161,736,738],{"className":737},[168],[170,739,741],{"xmlns":172,"display":740},"block",[174,742,743,840],{},[177,744,745,749,751,754,756,758,772,809,812,815,818,821,823,825,827,829,831,833,835,837],{},[360,746,748],{"mathvariant":747},"script","L",[364,750,373],{"stretchy":372},[360,752,753],{},"q",[364,755,378],{"stretchy":372},[364,757,453],{},[462,759,760,764],{},[360,761,763],{"mathvariant":762},"double-struck","E",[177,765,766,768,770],{},[360,767,362],{},[364,769,366],{},[360,771,369],{},[462,773,774,776],{},[360,775,763],{"mathvariant":762},[177,777,778,780,782,784,786,788,790,792,794,796,798,801,805,807],{},[360,779,450],{},[364,781,472],{"separator":198},[364,783,373],{"stretchy":372},[360,785,466],{},[364,787,472],{"separator":198},[360,789,477],{},[364,791,378],{"stretchy":372},[364,793,366],{},[360,795,116],{},[364,797,373],{"stretchy":372},[364,799,800],{},"⋅",[360,802,804],{"mathvariant":803},"normal","∣",[360,806,362],{},[364,808,378],{"stretchy":372},[364,810,811],{"stretchy":372},"[",[364,813,814],{},"−",[360,816,817],{},"log",[364,819,820],{},"⁡",[360,822,753],{},[364,824,373],{"stretchy":372},[360,826,477],{},[360,828,804],{"mathvariant":803},[360,830,466],{},[364,832,472],{"separator":198},[360,834,450],{},[364,836,378],{"stretchy":372},[364,838,839],{"stretchy":372},"]",[190,841,842],{"encoding":192},"\\mathcal{L}(q)=\\mathbb{E}_{\\theta \\sim \\pi} \\mathbb{E}_{c, (x,y) \\sim p(\\cdot|\\theta)} [-\\log q(y|x,c)]",[161,844,846,874],{"className":845,"ariaHidden":198},[197],[161,847,849,852,856,859,862,865,868,871],{"className":848},[202],[161,850],{"className":851,"style":415},[206],[161,853,748],{"className":854},[211,855],"mathcal",[161,857,373],{"className":858},[423],[161,860,753],{"className":861,"style":419},[211,395],[161,863,378],{"className":864},[430],[161,866],{"className":867,"style":401},[400],[161,869,453],{"className":870},[405],[161,872],{"className":873,"style":401},[400],[161,875,877,881,932,1017,1020,1023,1026,1035,1038,1041,1044,1047,1050,1053,1056,1059,1062],{"className":876},[202],[161,878],{"className":879,"style":880},[206],"height:1.1052em;vertical-align:-0.3552em;",[161,882,884,888],{"className":883},[211],[161,885,763],{"className":886},[211,887],"mathbb",[161,889,891],{"className":890},[223],[161,892,894,924],{"className":893},[227,546],[161,895,897,921],{"className":896},[231],[161,898,901],{"className":899,"style":900},[235],"height:0.3361em;",[161,902,903,906],{"style":556},[161,904],{"className":905,"style":243},[242],[161,907,909],{"className":908},[247,248,249,250],[161,910,912,915,918],{"className":911},[211,250],[161,913,362],{"className":914,"style":396},[211,395,250],[161,916,366],{"className":917},[405,250],[161,919,369],{"className":920,"style":419},[211,395,250],[161,922,570],{"className":923},[569],[161,925,927],{"className":926},[231],[161,928,930],{"className":929,"style":577},[235],[161,931],{},[161,933,935,938],{"className":934},[211],[161,936,763],{"className":937},[211,887],[161,939,941],{"className":940},[223],[161,942,944,1008],{"className":943},[227,546],[161,945,947,1005],{"className":946},[231],[161,948,951],{"className":949,"style":950},[235],"height:0.3448em;",[161,952,954,957],{"style":953},"top:-2.5198em;margin-left:0em;margin-right:0.05em;",[161,955],{"className":956,"style":243},[242],[161,958,960],{"className":959},[247,248,249,250],[161,961,963,966,969,972,975,978,981,984,987,990,993,996,999,1002],{"className":962},[211,250],[161,964,450],{"className":965},[211,395,250],[161,967,472],{"className":968},[583,250],[161,970,373],{"className":971},[423,250],[161,973,466],{"className":974},[211,395,250],[161,976,472],{"className":977},[583,250],[161,979,477],{"className":980,"style":419},[211,395,250],[161,982,378],{"className":983},[430,250],[161,985,366],{"className":986},[405,250],[161,988,116],{"className":989},[211,395,250],[161,991,373],{"className":992},[423,250],[161,994,800],{"className":995},[211,250],[161,997,804],{"className":998},[211,250],[161,1000,362],{"className":1001,"style":396},[211,395,250],[161,1003,378],{"className":1004},[430,250],[161,1006,570],{"className":1007},[569],[161,1009,1011],{"className":1010},[231],[161,1012,1015],{"className":1013,"style":1014},[235],"height:0.3552em;",[161,1016],{},[161,1018,811],{"className":1019},[423],[161,1021,814],{"className":1022},[211],[161,1024],{"className":1025,"style":587},[400],[161,1027,1030,1031],{"className":1028},[1029],"mop","lo",[161,1032,1034],{"style":1033},"margin-right:0.0139em;","g",[161,1036],{"className":1037,"style":587},[400],[161,1039,753],{"className":1040,"style":419},[211,395],[161,1042,373],{"className":1043},[423],[161,1045,477],{"className":1046,"style":419},[211,395],[161,1048,804],{"className":1049},[211],[161,1051,466],{"className":1052},[211,395],[161,1054,472],{"className":1055},[583],[161,1057],{"className":1058,"style":587},[400],[161,1060,450],{"className":1061},[211,395],[161,1063,1065],{"className":1064},[430],")]",[116,1067,1068],{},"該定理證明，此目標函數的唯一全局最小值即為貝葉斯後驗預測分佈（Bayesian Posterior Predictive Distribution）：",[161,1070,1072],{"className":1071},[731],[161,1073,1075,1146],{"className":1074},[164],[161,1076,1078],{"className":1077},[168],[170,1079,1080],{"xmlns":172,"display":740},[174,1081,1082,1143],{},[177,1083,1084,1091,1093,1095,1097,1099,1101,1103,1105,1107,1110,1112,1114,1116,1118,1120,1122,1124,1126,1128,1130,1132,1134,1136,1138,1141],{},[180,1085,1086,1088],{},[360,1087,753],{},[364,1089,1090],{},"∗",[364,1092,373],{"stretchy":372},[360,1094,477],{},[360,1096,804],{"mathvariant":803},[360,1098,466],{},[364,1100,472],{"separator":198},[360,1102,450],{},[364,1104,378],{"stretchy":372},[364,1106,453],{},[364,1108,1109],{},"∫",[360,1111,116],{},[364,1113,373],{"stretchy":372},[360,1115,477],{},[360,1117,804],{"mathvariant":803},[360,1119,466],{},[364,1121,472],{"separator":198},[360,1123,362],{},[364,1125,378],{"stretchy":372},[360,1127,116],{},[364,1129,373],{"stretchy":372},[360,1131,362],{},[360,1133,804],{"mathvariant":803},[360,1135,450],{},[364,1137,378],{"stretchy":372},[360,1139,1140],{},"d",[360,1142,362],{},[190,1144,1145],{"encoding":192},"q^*(y|x,c)=\\int p(y|x,\\theta) p(\\theta|c) d\\theta",[161,1147,1149,1220],{"className":1148,"ariaHidden":198},[197],[161,1150,1152,1155,1187,1190,1193,1196,1199,1202,1205,1208,1211,1214,1217],{"className":1151},[202],[161,1153],{"className":1154,"style":415},[206],[161,1156,1158,1161],{"className":1157},[211],[161,1159,753],{"className":1160,"style":419},[211,395],[161,1162,1164],{"className":1163},[223],[161,1165,1167],{"className":1166},[227],[161,1168,1170],{"className":1169},[231],[161,1171,1174],{"className":1172,"style":1173},[235],"height:0.7387em;",[161,1175,1177,1180],{"style":1176},"top:-3.113em;margin-right:0.05em;",[161,1178],{"className":1179,"style":243},[242],[161,1181,1183],{"className":1182},[247,248,249,250],[161,1184,1090],{"className":1185},[1186,250],"mbin",[161,1188,373],{"className":1189},[423],[161,1191,477],{"className":1192,"style":419},[211,395],[161,1194,804],{"className":1195},[211],[161,1197,466],{"className":1198},[211,395],[161,1200,472],{"className":1201},[583],[161,1203],{"className":1204,"style":587},[400],[161,1206,450],{"className":1207},[211,395],[161,1209,378],{"className":1210},[430],[161,1212],{"className":1213,"style":401},[400],[161,1215,453],{"className":1216},[405],[161,1218],{"className":1219,"style":401},[400],[161,1221,1223,1227,1233,1236,1239,1242,1245,1248,1251,1254,1257,1260,1263,1266,1269,1272,1275,1278,1281,1284],{"className":1222},[202],[161,1224],{"className":1225,"style":1226},[206],"height:2.2222em;vertical-align:-0.8622em;",[161,1228,1109],{"className":1229,"style":1232},[1029,1230,1231],"op-symbol","large-op","margin-right:0.4445em;position:relative;top:-0.0011em;",[161,1234],{"className":1235,"style":587},[400],[161,1237,116],{"className":1238},[211,395],[161,1240,373],{"className":1241},[423],[161,1243,477],{"className":1244,"style":419},[211,395],[161,1246,804],{"className":1247},[211],[161,1249,466],{"className":1250},[211,395],[161,1252,472],{"className":1253},[583],[161,1255],{"className":1256,"style":587},[400],[161,1258,362],{"className":1259,"style":396},[211,395],[161,1261,378],{"className":1262},[430],[161,1264,116],{"className":1265},[211,395],[161,1267,373],{"className":1268},[423],[161,1270,362],{"className":1271,"style":396},[211,395],[161,1273,804],{"className":1274},[211],[161,1276,450],{"className":1277},[211,395],[161,1279,378],{"className":1280},[430],[161,1282,1140],{"className":1283},[211,395],[161,1285,362],{"className":1286,"style":396},[211,395],[116,1288,1289,1290,1344],{},"其中 ",[161,1291,1293,1317],{"className":1292},[164],[161,1294,1296],{"className":1295},[168],[170,1297,1298],{"xmlns":172},[174,1299,1300,1314],{},[177,1301,1302,1304,1306,1308,1310,1312],{},[360,1303,116],{},[364,1305,373],{"stretchy":372},[360,1307,362],{},[360,1309,804],{"mathvariant":803},[360,1311,450],{},[364,1313,378],{"stretchy":372},[190,1315,1316],{"encoding":192},"p(\\theta|c)",[161,1318,1320],{"className":1319,"ariaHidden":198},[197],[161,1321,1323,1326,1329,1332,1335,1338,1341],{"className":1322},[202],[161,1324],{"className":1325,"style":415},[206],[161,1327,116],{"className":1328},[211,395],[161,1330,373],{"className":1331},[423],[161,1333,362],{"className":1334,"style":396},[211,395],[161,1336,804],{"className":1337},[211],[161,1339,450],{"className":1340},[211,395],[161,1342,378],{"className":1343},[430],">",[116,1346,1347],{},"是根據貝葉斯法則計算的潛在參數後驗分佈：",[116,1349,1350],{},[161,1351,1353,1442],{"className":1352},[164],[161,1354,1356],{"className":1355},[168],[170,1357,1358],{"xmlns":172},[174,1359,1360,1439],{},[177,1361,1362,1364,1366,1368,1370,1372,1374,1377,1379,1381,1383,1385,1415,1417,1419,1425,1427,1433,1435,1437],{},[360,1363,116],{},[364,1365,373],{"stretchy":372},[360,1367,362],{},[360,1369,804],{"mathvariant":803},[360,1371,450],{},[364,1373,378],{"stretchy":372},[364,1375,1376],{},"∝",[360,1378,369],{},[364,1380,373],{"stretchy":372},[360,1382,362],{},[364,1384,378],{"stretchy":372},[462,1386,1387,1390],{},[364,1388,1389],{},"∏",[177,1391,1392,1394,1400,1402,1408,1410,1413],{},[364,1393,373],{"stretchy":372},[462,1395,1396,1398],{},[360,1397,466],{},[360,1399,469],{},[364,1401,472],{"separator":198},[462,1403,1404,1406],{},[360,1405,477],{},[360,1407,469],{},[364,1409,378],{"stretchy":372},[364,1411,1412],{},"∈",[360,1414,450],{},[360,1416,116],{},[364,1418,373],{"stretchy":372},[462,1420,1421,1423],{},[360,1422,477],{},[360,1424,469],{},[360,1426,804],{"mathvariant":803},[462,1428,1429,1431],{},[360,1430,466],{},[360,1432,469],{},[364,1434,472],{"separator":198},[360,1436,362],{},[364,1438,378],{"stretchy":372},[190,1440,1441],{"encoding":192},"p(\\theta|c) \\propto \\pi(\\theta) \\prod_{(x_i, y_i) \\in c} p(y_i|x_i, \\theta)",[161,1443,1445,1478],{"className":1444,"ariaHidden":198},[197],[161,1446,1448,1451,1454,1457,1460,1463,1466,1469,1472,1475],{"className":1447},[202],[161,1449],{"className":1450,"style":415},[206],[161,1452,116],{"className":1453},[211,395],[161,1455,373],{"className":1456},[423],[161,1458,362],{"className":1459,"style":396},[211,395],[161,1461,804],{"className":1462},[211],[161,1464,450],{"className":1465},[211,395],[161,1467,378],{"className":1468},[430],[161,1470],{"className":1471,"style":401},[400],[161,1473,1376],{"className":1474},[405],[161,1476],{"className":1477,"style":401},[400],[161,1479,1481,1485,1488,1491,1494,1497,1500,1647,1650,1653,1656,1696,1699,1739,1742,1745,1748],{"className":1480},[202],[161,1482],{"className":1483,"style":1484},[206],"height:1.2247em;vertical-align:-0.4747em;",[161,1486,369],{"className":1487,"style":419},[211,395],[161,1489,373],{"className":1490},[423],[161,1492,362],{"className":1493,"style":396},[211,395],[161,1495,378],{"className":1496},[430],[161,1498],{"className":1499,"style":587},[400],[161,1501,1503,1508],{"className":1502},[1029],[161,1504,1389],{"className":1505,"style":1507},[1029,1230,1506],"small-op","position:relative;top:0em;",[161,1509,1511],{"className":1510},[223],[161,1512,1514,1638],{"className":1513},[227,546],[161,1515,1517,1635],{"className":1516},[231],[161,1518,1521],{"className":1519,"style":1520},[235],"height:0.2253em;",[161,1522,1524,1527],{"style":1523},"top:-2.4003em;margin-left:0em;margin-right:0.05em;",[161,1525],{"className":1526,"style":243},[242],[161,1528,1530],{"className":1529},[247,248,249,250],[161,1531,1533,1536,1582,1585,1626,1629,1632],{"className":1532},[211,250],[161,1534,373],{"className":1535},[423,250],[161,1537,1539,1542],{"className":1538},[211,250],[161,1540,466],{"className":1541},[211,395,250],[161,1543,1545],{"className":1544},[223],[161,1546,1548,1573],{"className":1547},[227,546],[161,1549,1551,1570],{"className":1550},[231],[161,1552,1555],{"className":1553,"style":1554},[235],"height:0.3281em;",[161,1556,1558,1562],{"style":1557},"top:-2.357em;margin-left:0em;margin-right:0.0714em;",[161,1559],{"className":1560,"style":1561},[242],"height:2.5em;",[161,1563,1567],{"className":1564},[247,1565,1566,250],"reset-size3","size1",[161,1568,469],{"className":1569},[211,395,250],[161,1571,570],{"className":1572},[569],[161,1574,1576],{"className":1575},[231],[161,1577,1580],{"className":1578,"style":1579},[235],"height:0.143em;",[161,1581],{},[161,1583,472],{"className":1584},[583,250],[161,1586,1588,1591],{"className":1587},[211,250],[161,1589,477],{"className":1590,"style":419},[211,395,250],[161,1592,1594],{"className":1593},[223],[161,1595,1597,1618],{"className":1596},[227,546],[161,1598,1600,1615],{"className":1599},[231],[161,1601,1603],{"className":1602,"style":1554},[235],[161,1604,1606,1609],{"style":1605},"top:-2.357em;margin-left:-0.0359em;margin-right:0.0714em;",[161,1607],{"className":1608,"style":1561},[242],[161,1610,1612],{"className":1611},[247,1565,1566,250],[161,1613,469],{"className":1614},[211,395,250],[161,1616,570],{"className":1617},[569],[161,1619,1621],{"className":1620},[231],[161,1622,1624],{"className":1623,"style":1579},[235],[161,1625],{},[161,1627,378],{"className":1628},[430,250],[161,1630,1412],{"className":1631},[405,250],[161,1633,450],{"className":1634},[211,395,250],[161,1636,570],{"className":1637},[569],[161,1639,1641],{"className":1640},[231],[161,1642,1645],{"className":1643,"style":1644},[235],"height:0.4747em;",[161,1646],{},[161,1648],{"className":1649,"style":587},[400],[161,1651,116],{"className":1652},[211,395],[161,1654,373],{"className":1655},[423],[161,1657,1659,1662],{"className":1658},[211],[161,1660,477],{"className":1661,"style":419},[211,395],[161,1663,1665],{"className":1664},[223],[161,1666,1668,1688],{"className":1667},[227,546],[161,1669,1671,1685],{"className":1670},[231],[161,1672,1674],{"className":1673,"style":553},[235],[161,1675,1676,1679],{"style":608},[161,1677],{"className":1678,"style":243},[242],[161,1680,1682],{"className":1681},[247,248,249,250],[161,1683,469],{"className":1684},[211,395,250],[161,1686,570],{"className":1687},[569],[161,1689,1691],{"className":1690},[231],[161,1692,1694],{"className":1693,"style":577},[235],[161,1695],{},[161,1697,804],{"className":1698},[211],[161,1700,1702,1705],{"className":1701},[211],[161,1703,466],{"className":1704},[211,395],[161,1706,1708],{"className":1707},[223],[161,1709,1711,1731],{"className":1710},[227,546],[161,1712,1714,1728],{"className":1713},[231],[161,1715,1717],{"className":1716,"style":553},[235],[161,1718,1719,1722],{"style":556},[161,1720],{"className":1721,"style":243},[242],[161,1723,1725],{"className":1724},[247,248,249,250],[161,1726,469],{"className":1727},[211,395,250],[161,1729,570],{"className":1730},[569],[161,1732,1734],{"className":1733},[231],[161,1735,1737],{"className":1736,"style":577},[235],[161,1738],{},[161,1740,472],{"className":1741},[583],[161,1743],{"className":1744,"style":587},[400],[161,1746,362],{"className":1747,"style":396},[211,395],[161,1749,378],{"className":1750},[430],[116,1752,1753,1754,1758],{},"這一理論結果具有深遠的意義：它表明在無限數據和無限容量的極限下，通過交叉熵訓練的模型",[1755,1756,1757],"em",{},"應該","收斂為一個完美的貝葉斯推理機。然而，這僅僅定義了「目標函數」的性質。關鍵的實證問題在於：在有限的參數、有限的數據和特定的架構（Transformer vs. MLP）下，哪種模型能夠真正實現這一數學理想？這正是本研究試圖回答的核心問題。",[108,1760,1762],{"id":1761},"_3-實驗設計構建貝葉斯風洞",[112,1763,1764],{},"3. 實驗設計：構建貝葉斯風洞",[116,1766,1767,1768,1771,1772,1775],{},"為了全面評估模型的推理能力，研究團隊設計了兩個難度遞增的「風洞」任務：",[112,1769,1770],{},"雙射學習","（Bijection Learning）與",[112,1773,1774],{},"隱馬可夫模型狀態追蹤","（HMM State Tracking）。這兩個任務分別測試了離散假設消除和遞迴信念更新兩種根本不同的推理模式。",[132,1777,1779],{"id":1778},"_31-風洞一雙射學習bijection-learning",[112,1780,1781],{},"3.1 風洞一：雙射學習（Bijection Learning）",[116,1783,1784],{},"此任務模擬了一個嚴格的邏輯消除過程，類似於數獨或邏輯謎題。",[1786,1787,1788,2181,2187],"ul",{},[148,1789,1790,1793,1794,1823,1824,1983,1984,2037,2038,2109,2110,2180],{},[112,1791,1792],{},"任務定義："," 潛在參數 ",[161,1795,1797,1811],{"className":1796},[164],[161,1798,1800],{"className":1799},[168],[170,1801,1802],{"xmlns":172},[174,1803,1804,1808],{},[177,1805,1806],{},[360,1807,362],{},[190,1809,1810],{"encoding":192},"\\theta",[161,1812,1814],{"className":1813,"ariaHidden":198},[197],[161,1815,1817,1820],{"className":1816},[202],[161,1818],{"className":1819,"style":391},[206],[161,1821,362],{"className":1822,"style":396},[211,395]," 是一個雙射函數（一對一映射） ",[161,1825,1827,1879],{"className":1826},[164],[161,1828,1830],{"className":1829},[168],[170,1831,1832],{"xmlns":172},[174,1833,1834,1876],{},[177,1835,1836,1838,1841,1844,1846,1848,1851,1853,1856,1859,1862,1864,1866,1868,1870,1872,1874],{},[360,1837,369],{},[364,1839,1840],{},":",[364,1842,1843],{"stretchy":372},"{",[183,1845,212],{},[364,1847,472],{"separator":198},[364,1849,1850],{},"…",[364,1852,472],{"separator":198},[360,1854,1855],{},"V",[364,1857,1858],{"stretchy":372},"}",[364,1860,1861],{},"→",[364,1863,1843],{"stretchy":372},[183,1865,212],{},[364,1867,472],{"separator":198},[364,1869,1850],{},[364,1871,472],{"separator":198},[360,1873,1855],{},[364,1875,1858],{"stretchy":372},[190,1877,1878],{"encoding":192},"\\pi: \\{1, \\dots, V\\} \\to \\{1, \\dots, V\\}",[161,1880,1882,1900,1947],{"className":1881,"ariaHidden":198},[197],[161,1883,1885,1888,1891,1894,1897],{"className":1884},[202],[161,1886],{"className":1887,"style":505},[206],[161,1889,369],{"className":1890,"style":419},[211,395],[161,1892],{"className":1893,"style":401},[400],[161,1895,1840],{"className":1896},[405],[161,1898],{"className":1899,"style":401},[400],[161,1901,1903,1906,1909,1912,1915,1918,1922,1925,1928,1931,1935,1938,1941,1944],{"className":1902},[202],[161,1904],{"className":1905,"style":415},[206],[161,1907,1843],{"className":1908},[423],[161,1910,212],{"className":1911},[211],[161,1913,472],{"className":1914},[583],[161,1916],{"className":1917,"style":587},[400],[161,1919,1850],{"className":1920},[1921],"minner",[161,1923],{"className":1924,"style":587},[400],[161,1926,472],{"className":1927},[583],[161,1929],{"className":1930,"style":587},[400],[161,1932,1855],{"className":1933,"style":1934},[211,395],"margin-right:0.2222em;",[161,1936,1858],{"className":1937},[430],[161,1939],{"className":1940,"style":401},[400],[161,1942,1861],{"className":1943},[405],[161,1945],{"className":1946,"style":401},[400],[161,1948,1950,1953,1956,1959,1962,1965,1968,1971,1974,1977,1980],{"className":1949},[202],[161,1951],{"className":1952,"style":415},[206],[161,1954,1843],{"className":1955},[423],[161,1957,212],{"className":1958},[211],[161,1960,472],{"className":1961},[583],[161,1963],{"className":1964,"style":587},[400],[161,1966,1850],{"className":1967},[1921],[161,1969],{"className":1970,"style":587},[400],[161,1972,472],{"className":1973},[583],[161,1975],{"className":1976,"style":587},[400],[161,1978,1855],{"className":1979,"style":1934},[211,395],[161,1981,1858],{"className":1982},[430],"。模型在觀察到 ",[161,1985,1987,2005],{"className":1986},[164],[161,1988,1990],{"className":1989},[168],[170,1991,1992],{"xmlns":172},[174,1993,1994,2002],{},[177,1995,1996,1998,2000],{},[360,1997,492],{},[364,1999,814],{},[183,2001,212],{},[190,2003,2004],{"encoding":192},"k-1",[161,2006,2008,2027],{"className":2007,"ariaHidden":198},[197],[161,2009,2011,2015,2018,2021,2024],{"className":2010},[202],[161,2012],{"className":2013,"style":2014},[206],"height:0.7778em;vertical-align:-0.0833em;",[161,2016,492],{"className":2017,"style":678},[211,395],[161,2019],{"className":2020,"style":1934},[400],[161,2022,814],{"className":2023},[1186],[161,2025],{"className":2026,"style":1934},[400],[161,2028,2030,2034],{"className":2029},[202],[161,2031],{"className":2032,"style":2033},[206],"height:0.6444em;",[161,2035,212],{"className":2036},[211]," 個輸入-輸出對後，必須預測下一個新輸入 ",[161,2039,2041,2059],{"className":2040},[164],[161,2042,2044],{"className":2043},[168],[170,2045,2046],{"xmlns":172},[174,2047,2048,2056],{},[177,2049,2050],{},[462,2051,2052,2054],{},[360,2053,466],{},[360,2055,492],{},[190,2057,2058],{"encoding":192},"x_k",[161,2060,2062],{"className":2061,"ariaHidden":198},[197],[161,2063,2065,2069],{"className":2064},[202],[161,2066],{"className":2067,"style":2068},[206],"height:0.5806em;vertical-align:-0.15em;",[161,2070,2072,2075],{"className":2071},[211],[161,2073,466],{"className":2074},[211,395],[161,2076,2078],{"className":2077},[223],[161,2079,2081,2101],{"className":2080},[227,546],[161,2082,2084,2098],{"className":2083},[231],[161,2085,2087],{"className":2086,"style":900},[235],[161,2088,2089,2092],{"style":556},[161,2090],{"className":2091,"style":243},[242],[161,2093,2095],{"className":2094},[247,248,249,250],[161,2096,492],{"className":2097,"style":678},[211,395,250],[161,2099,570],{"className":2100},[569],[161,2102,2104],{"className":2103},[231],[161,2105,2107],{"className":2106,"style":577},[235],[161,2108],{}," 的對應輸出 ",[161,2111,2113,2131],{"className":2112},[164],[161,2114,2116],{"className":2115},[168],[170,2117,2118],{"xmlns":172},[174,2119,2120,2128],{},[177,2121,2122],{},[462,2123,2124,2126],{},[360,2125,477],{},[360,2127,492],{},[190,2129,2130],{"encoding":192},"y_k",[161,2132,2134],{"className":2133,"ariaHidden":198},[197],[161,2135,2137,2140],{"className":2136},[202],[161,2138],{"className":2139,"style":721},[206],[161,2141,2143,2146],{"className":2142},[211],[161,2144,477],{"className":2145,"style":419},[211,395],[161,2147,2149],{"className":2148},[223],[161,2150,2152,2172],{"className":2151},[227,546],[161,2153,2155,2169],{"className":2154},[231],[161,2156,2158],{"className":2157,"style":900},[235],[161,2159,2160,2163],{"style":608},[161,2161],{"className":2162,"style":243},[242],[161,2164,2166],{"className":2165},[247,248,249,250],[161,2167,492],{"className":2168,"style":678},[211,395,250],[161,2170,570],{"className":2171},[569],[161,2173,2175],{"className":2174},[231],[161,2176,2178],{"className":2177,"style":577},[235],[161,2179],{},"。",[148,2182,2183,2186],{},[112,2184,2185],{},"貝葉斯邏輯："," 由於映射是雙射的，每一個觀察到的輸出都排除了該輸出對應其他輸入的可能性。對於一個全新的輸入，其可能的輸出必須均勻分佈在尚未出現過的輸出集合中。",[148,2188,2189,2190,2240],{},"解析後驗： 若已觀察到 ",[161,2191,2193,2210],{"className":2192},[164],[161,2194,2196],{"className":2195},[168],[170,2197,2198],{"xmlns":172},[174,2199,2200,2208],{},[177,2201,2202,2204,2206],{},[360,2203,492],{},[364,2205,814],{},[183,2207,212],{},[190,2209,2004],{"encoding":192},[161,2211,2213,2231],{"className":2212,"ariaHidden":198},[197],[161,2214,2216,2219,2222,2225,2228],{"className":2215},[202],[161,2217],{"className":2218,"style":2014},[206],[161,2220,492],{"className":2221,"style":678},[211,395],[161,2223],{"className":2224,"style":1934},[400],[161,2226,814],{"className":2227},[1186],[161,2229],{"className":2230,"style":1934},[400],[161,2232,2234,2237],{"className":2233},[202],[161,2235],{"className":2236,"style":2033},[206],[161,2238,212],{"className":2239},[211]," 個不同的輸出，則下一個輸出的真實機率分佈為均勻分佈，其熵（Entropy）為：",[161,2242,2244],{"className":2243},[731],[161,2245,2247,2314],{"className":2246},[164],[161,2248,2250],{"className":2249},[168],[170,2251,2252],{"xmlns":172,"display":740},[174,2253,2254,2311],{},[177,2255,2256,2277,2279,2281,2283,2285,2296,2298,2300,2302,2304,2307,2309],{},[462,2257,2258,2261],{},[360,2259,2260],{},"H",[177,2262,2263,2266,2269,2271,2274],{},[360,2264,2265],{},"B",[360,2267,2268],{},"a",[360,2270,477],{},[360,2272,2273],{},"e",[360,2275,2276],{},"s",[364,2278,373],{"stretchy":372},[360,2280,492],{},[364,2282,378],{"stretchy":372},[364,2284,453],{},[462,2286,2287,2293],{},[177,2288,2289,2291],{},[360,2290,817],{},[364,2292,820],{},[183,2294,2295],{},"2",[364,2297,373],{"stretchy":372},[360,2299,1855],{},[364,2301,814],{},[360,2303,492],{},[364,2305,2306],{},"+",[183,2308,212],{},[364,2310,378],{"stretchy":372},[190,2312,2313],{"encoding":192},"H_{Bayes}(k)= \\log_2(V - k + 1)",[161,2315,2317,2400,2466,2484],{"className":2316,"ariaHidden":198},[197],[161,2318,2320,2324,2382,2385,2388,2391,2394,2397],{"className":2319},[202],[161,2321],{"className":2322,"style":2323},[206],"height:1.0361em;vertical-align:-0.2861em;",[161,2325,2327,2331],{"className":2326},[211],[161,2328,2260],{"className":2329,"style":2330},[211,395],"margin-right:0.0813em;",[161,2332,2334],{"className":2333},[223],[161,2335,2337,2373],{"className":2336},[227,546],[161,2338,2340,2370],{"className":2339},[231],[161,2341,2344],{"className":2342,"style":2343},[235],"height:0.3283em;",[161,2345,2347,2350],{"style":2346},"top:-2.55em;margin-left:-0.0813em;margin-right:0.05em;",[161,2348],{"className":2349,"style":243},[242],[161,2351,2353],{"className":2352},[247,248,249,250],[161,2354,2356,2360,2363,2366],{"className":2355},[211,250],[161,2357,2265],{"className":2358,"style":2359},[211,395,250],"margin-right:0.0502em;",[161,2361,2268],{"className":2362},[211,395,250],[161,2364,477],{"className":2365,"style":419},[211,395,250],[161,2367,2369],{"className":2368},[211,395,250],"es",[161,2371,570],{"className":2372},[569],[161,2374,2376],{"className":2375},[231],[161,2377,2380],{"className":2378,"style":2379},[235],"height:0.2861em;",[161,2381],{},[161,2383,373],{"className":2384},[423],[161,2386,492],{"className":2387,"style":678},[211,395],[161,2389,378],{"className":2390},[430],[161,2392],{"className":2393,"style":401},[400],[161,2395,453],{"className":2396},[405],[161,2398],{"className":2399,"style":401},[400],[161,2401,2403,2406,2451,2454,2457,2460,2463],{"className":2402},[202],[161,2404],{"className":2405,"style":415},[206],[161,2407,2409,2414],{"className":2408},[1029],[161,2410,1030,2412],{"className":2411},[1029],[161,2413,1034],{"style":1033},[161,2415,2417],{"className":2416},[223],[161,2418,2420,2442],{"className":2419},[227,546],[161,2421,2423,2439],{"className":2422},[231],[161,2424,2427],{"className":2425,"style":2426},[235],"height:0.207em;",[161,2428,2430,2433],{"style":2429},"top:-2.4559em;margin-right:0.05em;",[161,2431],{"className":2432,"style":243},[242],[161,2434,2436],{"className":2435},[247,248,249,250],[161,2437,2295],{"className":2438},[211,250],[161,2440,570],{"className":2441},[569],[161,2443,2445],{"className":2444},[231],[161,2446,2449],{"className":2447,"style":2448},[235],"height:0.2441em;",[161,2450],{},[161,2452,373],{"className":2453},[423],[161,2455,1855],{"className":2456,"style":1934},[211,395],[161,2458],{"className":2459,"style":1934},[400],[161,2461,814],{"className":2462},[1186],[161,2464],{"className":2465,"style":1934},[400],[161,2467,2469,2472,2475,2478,2481],{"className":2468},[202],[161,2470],{"className":2471,"style":2014},[206],[161,2473,492],{"className":2474,"style":678},[211,395],[161,2476],{"className":2477,"style":1934},[400],[161,2479,2306],{"className":2480},[1186],[161,2482],{"className":2483,"style":1934},[400],[161,2485,2487,2490,2493],{"className":2486},[202],[161,2488],{"className":2489,"style":415},[206],[161,2491,212],{"className":2492},[211],[161,2494,378],{"className":2495},[430],[116,2497,2498],{},"隨著序列推進，可能的假設數量減少，熵呈現階梯狀下降。",[1786,2500,2501],{},[148,2502,2503,2506,2507,2560,2561,2721,2722,2785],{},[112,2504,2505],{},"複雜度控制","： 設定詞彙表大小 ",[161,2508,2510,2529],{"className":2509},[164],[161,2511,2513],{"className":2512},[168],[170,2514,2515],{"xmlns":172},[174,2516,2517,2526],{},[177,2518,2519,2521,2523],{},[360,2520,1855],{},[364,2522,453],{},[183,2524,2525],{},"20",[190,2527,2528],{"encoding":192},"V=20",[161,2530,2532,2551],{"className":2531,"ariaHidden":198},[197],[161,2533,2535,2539,2542,2545,2548],{"className":2534},[202],[161,2536],{"className":2537,"style":2538},[206],"height:0.6833em;",[161,2540,1855],{"className":2541,"style":1934},[211,395],[161,2543],{"className":2544,"style":401},[400],[161,2546,453],{"className":2547},[405],[161,2549],{"className":2550,"style":401},[400],[161,2552,2554,2557],{"className":2553},[202],[161,2555],{"className":2556,"style":2033},[206],[161,2558,2525],{"className":2559},[211],"，可能的雙射函數總數為 ",[161,2562,2564,2622],{"className":2563},[164],[161,2565,2567],{"className":2566},[168],[170,2568,2569],{"xmlns":172},[174,2570,2571,2619],{},[177,2572,2573,2575,2579,2582,2584,2586,2588,2591,2594,2596,2599,2601,2604,2606,2609,2611,2613],{},[183,2574,2525],{},[2576,2577,2578],"mtext",{}," ⁣",[400,2580],{"linebreak":2581},"newline",[360,2583,2268],{},[360,2585,116],{},[360,2587,116],{},[360,2589,2590],{},"r",[360,2592,2593],{},"o",[360,2595,466],{},[183,2597,2598],{},"2.4",[400,2600],{"linebreak":2581},[360,2602,2603],{},"t",[360,2605,469],{},[360,2607,2608],{},"m",[360,2610,2273],{},[360,2612,2276],{},[180,2614,2615,2617],{},[183,2616,185],{},[183,2618,188],{},[190,2620,2621],{"encoding":192},"20\\! \\\\approx 2.4 \\\\times 10^{18}",[161,2623,2625,2638,2641,2667,2670],{"className":2624,"ariaHidden":198},[197],[161,2626,2628,2631,2634],{"className":2627},[202],[161,2629],{"className":2630,"style":2033},[206],[161,2632,2525],{"className":2633},[211],[161,2635],{"className":2636,"style":2637},[400],"margin-right:-0.1667em;",[161,2639],{"className":2640},[400,2581],[161,2642,2644,2648,2651,2655,2658,2661,2664],{"className":2643},[202],[161,2645],{"className":2646,"style":2647},[206],"height:0.8389em;vertical-align:-0.1944em;",[161,2649,2268],{"className":2650},[211,395],[161,2652,2654],{"className":2653},[211,395],"pp",[161,2656,2590],{"className":2657,"style":396},[211,395],[161,2659,2593],{"className":2660},[211,395],[161,2662,466],{"className":2663},[211,395],[161,2665,2598],{"className":2666},[211],[161,2668],{"className":2669},[400,2581],[161,2671,2673,2676,2679,2683,2686,2689],{"className":2672},[202],[161,2674],{"className":2675,"style":207},[206],[161,2677,2603],{"className":2678},[211,395],[161,2680,2682],{"className":2681},[211,395],"im",[161,2684,2369],{"className":2685},[211,395],[161,2687,212],{"className":2688},[211],[161,2690,2692,2695],{"className":2691},[211],[161,2693,219],{"className":2694},[211],[161,2696,2698],{"className":2697},[223],[161,2699,2701],{"className":2700},[227],[161,2702,2704],{"className":2703},[231],[161,2705,2707],{"className":2706,"style":207},[235],[161,2708,2709,2712],{"style":238},[161,2710],{"className":2711,"style":243},[242],[161,2713,2715],{"className":2714},[247,248,249,250],[161,2716,2718],{"className":2717},[211,250],[161,2719,188],{"className":2720},[211,250],"。由於訓練樣本僅有 ",[161,2723,2725,2744],{"className":2724},[164],[161,2726,2728],{"className":2727},[168],[170,2729,2730],{"xmlns":172},[174,2731,2732,2741],{},[177,2733,2734],{},[180,2735,2736,2738],{},[183,2737,185],{},[183,2739,2740],{},"5",[190,2742,2743],{"encoding":192},"10^5",[161,2745,2747],{"className":2746,"ariaHidden":198},[197],[161,2748,2750,2753,2756],{"className":2749},[202],[161,2751],{"className":2752,"style":207},[206],[161,2754,212],{"className":2755},[211],[161,2757,2759,2762],{"className":2758},[211],[161,2760,219],{"className":2761},[211],[161,2763,2765],{"className":2764},[223],[161,2766,2768],{"className":2767},[227],[161,2769,2771],{"className":2770},[231],[161,2772,2774],{"className":2773,"style":207},[235],[161,2775,2776,2779],{"style":238},[161,2777],{"className":2778,"style":243},[242],[161,2780,2782],{"className":2781},[247,248,249,250],[161,2783,2740],{"className":2784},[211,250]," 筆，模型在訓練中幾乎不可能見過測試集中的雙射函數。因此，模型無法「背誦」答案，必須學會「排除法」這一演算法。",[132,2787,2789],{"id":2788},"_32-風洞二隱馬可夫模型狀態追蹤hmm-state-tracking",[112,2790,2791],{},"3.2 風洞二：隱馬可夫模型狀態追蹤（HMM State Tracking）",[116,2793,2794],{},"此任務測試模型處理隨機性、噪聲以及遞迴狀態更新的能力，要求模型在內部激活中實現「前向演算法」（Forward Algorithm）。",[1786,2796,2797,3078],{},[148,2798,2799,2801,2802,2805,2806,2836,2837,2866,2867,3005,3006,3077],{},[112,2800,1792],{}," 每個序列由一個",[1755,2803,2804],{},"全新","的隱馬可夫模型生成。序列開頭包含一個「標頭」（Header），編碼了該特定 HMM 的轉移矩陣（",[161,2807,2809,2823],{"className":2808},[164],[161,2810,2812],{"className":2811},[168],[170,2813,2814],{"xmlns":172},[174,2815,2816,2821],{},[177,2817,2818],{},[360,2819,2820],{},"T",[190,2822,2820],{"encoding":192},[161,2824,2826],{"className":2825,"ariaHidden":198},[197],[161,2827,2829,2832],{"className":2828},[202],[161,2830],{"className":2831,"style":2538},[206],[161,2833,2820],{"className":2834,"style":2835},[211,395],"margin-right:0.1389em;","）和發射矩陣（",[161,2838,2840,2853],{"className":2839},[164],[161,2841,2843],{"className":2842},[168],[170,2844,2845],{"xmlns":172},[174,2846,2847,2851],{},[177,2848,2849],{},[360,2850,763],{},[190,2852,763],{"encoding":192},[161,2854,2856],{"className":2855,"ariaHidden":198},[197],[161,2857,2859,2862],{"className":2858},[202],[161,2860],{"className":2861,"style":2538},[206],[161,2863,763],{"className":2864,"style":2865},[211,395],"margin-right:0.0576em;","）。隨後，模型觀察一連串的觀測值 ",[161,2868,2870,2900],{"className":2869},[164],[161,2871,2873],{"className":2872},[168],[170,2874,2875],{"xmlns":172},[174,2876,2877,2897],{},[177,2878,2879,2885,2887,2893,2895],{},[462,2880,2881,2883],{},[360,2882,2593],{},[183,2884,212],{},[364,2886,472],{"separator":198},[462,2888,2889,2891],{},[360,2890,2593],{},[183,2892,2295],{},[364,2894,472],{"separator":198},[364,2896,1850],{},[190,2898,2899],{"encoding":192},"o_1, o_2, \\dots",[161,2901,2903],{"className":2902,"ariaHidden":198},[197],[161,2904,2906,2909,2950,2953,2956,2996,2999,3002],{"className":2905},[202],[161,2907],{"className":2908,"style":721},[206],[161,2910,2912,2915],{"className":2911},[211],[161,2913,2593],{"className":2914},[211,395],[161,2916,2918],{"className":2917},[223],[161,2919,2921,2942],{"className":2920},[227,546],[161,2922,2924,2939],{"className":2923},[231],[161,2925,2928],{"className":2926,"style":2927},[235],"height:0.3011em;",[161,2929,2930,2933],{"style":556},[161,2931],{"className":2932,"style":243},[242],[161,2934,2936],{"className":2935},[247,248,249,250],[161,2937,212],{"className":2938},[211,250],[161,2940,570],{"className":2941},[569],[161,2943,2945],{"className":2944},[231],[161,2946,2948],{"className":2947,"style":577},[235],[161,2949],{},[161,2951,472],{"className":2952},[583],[161,2954],{"className":2955,"style":587},[400],[161,2957,2959,2962],{"className":2958},[211],[161,2960,2593],{"className":2961},[211,395],[161,2963,2965],{"className":2964},[223],[161,2966,2968,2988],{"className":2967},[227,546],[161,2969,2971,2985],{"className":2970},[231],[161,2972,2974],{"className":2973,"style":2927},[235],[161,2975,2976,2979],{"style":556},[161,2977],{"className":2978,"style":243},[242],[161,2980,2982],{"className":2981},[247,248,249,250],[161,2983,2295],{"className":2984},[211,250],[161,2986,570],{"className":2987},[569],[161,2989,2991],{"className":2990},[231],[161,2992,2994],{"className":2993,"style":577},[235],[161,2995],{},[161,2997,472],{"className":2998},[583],[161,3000],{"className":3001,"style":587},[400],[161,3003,1850],{"className":3004},[1921],"，並需預測當前隱藏狀態 ",[161,3007,3009,3027],{"className":3008},[164],[161,3010,3012],{"className":3011},[168],[170,3013,3014],{"xmlns":172},[174,3015,3016,3024],{},[177,3017,3018],{},[462,3019,3020,3022],{},[360,3021,2276],{},[360,3023,2603],{},[190,3025,3026],{"encoding":192},"s_t",[161,3028,3030],{"className":3029,"ariaHidden":198},[197],[161,3031,3033,3036],{"className":3032},[202],[161,3034],{"className":3035,"style":2068},[206],[161,3037,3039,3042],{"className":3038},[211],[161,3040,2276],{"className":3041},[211,395],[161,3043,3045],{"className":3044},[223],[161,3046,3048,3069],{"className":3047},[227,546],[161,3049,3051,3066],{"className":3050},[231],[161,3052,3055],{"className":3053,"style":3054},[235],"height:0.2806em;",[161,3056,3057,3060],{"style":556},[161,3058],{"className":3059,"style":243},[242],[161,3061,3063],{"className":3062},[247,248,249,250],[161,3064,2603],{"className":3065},[211,395,250],[161,3067,570],{"className":3068},[569],[161,3070,3072],{"className":3071},[231],[161,3073,3075],{"className":3074,"style":577},[235],[161,3076],{}," 的後驗機率分佈。",[148,3079,3080],{},"貝葉斯邏輯： 真實的貝葉斯後驗由遞迴的前向演算法給出：",[161,3082,3084],{"className":3083},[731],[161,3085,3087,3182],{"className":3086},[164],[161,3088,3090],{"className":3089},[168],[170,3091,3092],{"xmlns":172,"display":740},[174,3093,3094,3179],{},[177,3095,3096,3103,3105,3107,3109,3111,3113,3115,3121,3123,3125,3127,3141,3143,3145,3147,3149,3155,3157,3169,3171,3177],{},[462,3097,3098,3101],{},[360,3099,3100],{},"α",[360,3102,2603],{},[364,3104,373],{"stretchy":372},[360,3106,2276],{},[364,3108,378],{"stretchy":372},[364,3110,1376],{},[360,3112,763],{},[364,3114,373],{"stretchy":372},[462,3116,3117,3119],{},[360,3118,2593],{},[360,3120,2603],{},[360,3122,804],{"mathvariant":803},[360,3124,2276],{},[364,3126,378],{"stretchy":372},[3128,3129,3130,3133],"munder",{},[364,3131,3132],{},"∑",[180,3134,3135,3137],{},[360,3136,2276],{},[364,3138,3140],{"mathvariant":803,"lspace":3139,"rspace":3139},"0em","′",[360,3142,2820],{},[364,3144,373],{"stretchy":372},[360,3146,2276],{},[360,3148,804],{"mathvariant":803},[180,3150,3151,3153],{},[360,3152,2276],{},[364,3154,3140],{"mathvariant":803,"lspace":3139,"rspace":3139},[364,3156,378],{"stretchy":372},[462,3158,3159,3161],{},[360,3160,3100],{},[177,3162,3163,3165,3167],{},[360,3164,2603],{},[364,3166,814],{},[183,3168,212],{},[364,3170,373],{"stretchy":372},[180,3172,3173,3175],{},[360,3174,2276],{},[364,3176,3140],{"mathvariant":803,"lspace":3139,"rspace":3139},[364,3178,378],{"stretchy":372},[190,3180,3181],{"encoding":192},"\\alpha_t(s) \\propto E(o_t|s) \\sum_{s'} T(s|s') \\alpha_{t-1}(s')",[161,3183,3185,3251],{"className":3184,"ariaHidden":198},[197],[161,3186,3188,3191,3233,3236,3239,3242,3245,3248],{"className":3187},[202],[161,3189],{"className":3190,"style":415},[206],[161,3192,3194,3198],{"className":3193},[211],[161,3195,3100],{"className":3196,"style":3197},[211,395],"margin-right:0.0037em;",[161,3199,3201],{"className":3200},[223],[161,3202,3204,3225],{"className":3203},[227,546],[161,3205,3207,3222],{"className":3206},[231],[161,3208,3210],{"className":3209,"style":3054},[235],[161,3211,3213,3216],{"style":3212},"top:-2.55em;margin-left:-0.0037em;margin-right:0.05em;",[161,3214],{"className":3215,"style":243},[242],[161,3217,3219],{"className":3218},[247,248,249,250],[161,3220,2603],{"className":3221},[211,395,250],[161,3223,570],{"className":3224},[569],[161,3226,3228],{"className":3227},[231],[161,3229,3231],{"className":3230,"style":577},[235],[161,3232],{},[161,3234,373],{"className":3235},[423],[161,3237,2276],{"className":3238},[211,395],[161,3240,378],{"className":3241},[430],[161,3243],{"className":3244,"style":401},[400],[161,3246,1376],{"className":3247},[405],[161,3249],{"className":3250,"style":401},[400],[161,3252,3254,3258,3261,3264,3304,3307,3310,3313,3316,3400,3403,3406,3409,3412,3415,3448,3451,3501,3504,3536],{"className":3253},[202],[161,3255],{"className":3256,"style":3257},[206],"height:2.344em;vertical-align:-1.294em;",[161,3259,763],{"className":3260,"style":2865},[211,395],[161,3262,373],{"className":3263},[423],[161,3265,3267,3270],{"className":3266},[211],[161,3268,2593],{"className":3269},[211,395],[161,3271,3273],{"className":3272},[223],[161,3274,3276,3296],{"className":3275},[227,546],[161,3277,3279,3293],{"className":3278},[231],[161,3280,3282],{"className":3281,"style":3054},[235],[161,3283,3284,3287],{"style":556},[161,3285],{"className":3286,"style":243},[242],[161,3288,3290],{"className":3289},[247,248,249,250],[161,3291,2603],{"className":3292},[211,395,250],[161,3294,570],{"className":3295},[569],[161,3297,3299],{"className":3298},[231],[161,3300,3302],{"className":3301,"style":577},[235],[161,3303],{},[161,3305,804],{"className":3306},[211],[161,3308,2276],{"className":3309},[211,395],[161,3311,378],{"className":3312},[430],[161,3314],{"className":3315,"style":587},[400],[161,3317,3320],{"className":3318},[1029,3319],"op-limits",[161,3321,3323,3391],{"className":3322},[227,546],[161,3324,3326,3388],{"className":3325},[231],[161,3327,3330,3377],{"className":3328,"style":3329},[235],"height:1.05em;",[161,3331,3333,3337],{"style":3332},"top:-1.856em;margin-left:0em;",[161,3334],{"className":3335,"style":3336},[242],"height:3.05em;",[161,3338,3340],{"className":3339},[247,248,249,250],[161,3341,3343],{"className":3342},[211,250],[161,3344,3346,3349],{"className":3345},[211,250],[161,3347,2276],{"className":3348},[211,395,250],[161,3350,3352],{"className":3351},[223],[161,3353,3355],{"className":3354},[227],[161,3356,3358],{"className":3357},[231],[161,3359,3362],{"className":3360,"style":3361},[235],"height:0.6828em;",[161,3363,3365,3368],{"style":3364},"top:-2.786em;margin-right:0.0714em;",[161,3366],{"className":3367,"style":1561},[242],[161,3369,3371],{"className":3370},[247,1565,1566,250],[161,3372,3374],{"className":3373},[211,250],[161,3375,3140],{"className":3376},[211,250],[161,3378,3380,3383],{"style":3379},"top:-3.05em;",[161,3381],{"className":3382,"style":3336},[242],[161,3384,3385],{},[161,3386,3132],{"className":3387},[1029,1230,1231],[161,3389,570],{"className":3390},[569],[161,3392,3394],{"className":3393},[231],[161,3395,3398],{"className":3396,"style":3397},[235],"height:1.294em;",[161,3399],{},[161,3401],{"className":3402,"style":587},[400],[161,3404,2820],{"className":3405,"style":2835},[211,395],[161,3407,373],{"className":3408},[423],[161,3410,2276],{"className":3411},[211,395],[161,3413,804],{"className":3414},[211],[161,3416,3418,3421],{"className":3417},[211],[161,3419,2276],{"className":3420},[211,395],[161,3422,3424],{"className":3423},[223],[161,3425,3427],{"className":3426},[227],[161,3428,3430],{"className":3429},[231],[161,3431,3434],{"className":3432,"style":3433},[235],"height:0.8019em;",[161,3435,3436,3439],{"style":1176},[161,3437],{"className":3438,"style":243},[242],[161,3440,3442],{"className":3441},[247,248,249,250],[161,3443,3445],{"className":3444},[211,250],[161,3446,3140],{"className":3447},[211,250],[161,3449,378],{"className":3450},[430],[161,3452,3454,3457],{"className":3453},[211],[161,3455,3100],{"className":3456,"style":3197},[211,395],[161,3458,3460],{"className":3459},[223],[161,3461,3463,3492],{"className":3462},[227,546],[161,3464,3466,3489],{"className":3465},[231],[161,3467,3469],{"className":3468,"style":2927},[235],[161,3470,3471,3474],{"style":3212},[161,3472],{"className":3473,"style":243},[242],[161,3475,3477],{"className":3476},[247,248,249,250],[161,3478,3480,3483,3486],{"className":3479},[211,250],[161,3481,2603],{"className":3482},[211,395,250],[161,3484,814],{"className":3485},[1186,250],[161,3487,212],{"className":3488},[211,250],[161,3490,570],{"className":3491},[569],[161,3493,3495],{"className":3494},[231],[161,3496,3499],{"className":3497,"style":3498},[235],"height:0.2083em;",[161,3500],{},[161,3502,373],{"className":3503},[423],[161,3505,3507,3510],{"className":3506},[211],[161,3508,2276],{"className":3509},[211,395],[161,3511,3513],{"className":3512},[223],[161,3514,3516],{"className":3515},[227],[161,3517,3519],{"className":3518},[231],[161,3520,3522],{"className":3521,"style":3433},[235],[161,3523,3524,3527],{"style":1176},[161,3525],{"className":3526,"style":243},[242],[161,3528,3530],{"className":3529},[247,248,249,250],[161,3531,3533],{"className":3532},[211,250],[161,3534,3140],{"className":3535},[211,250],[161,3537,378],{"className":3538},[430],[116,3540,3541],{},"這要求模型執行複雜的操作：",[145,3543,3544,3608,3614],{},[148,3545,3546,3549,3550,3578,3579,3607],{},[112,3547,3548],{},"解析上下文："," 從標頭中讀取並存儲 ",[161,3551,3553,3566],{"className":3552},[164],[161,3554,3556],{"className":3555},[168],[170,3557,3558],{"xmlns":172},[174,3559,3560,3564],{},[177,3561,3562],{},[360,3563,2820],{},[190,3565,2820],{"encoding":192},[161,3567,3569],{"className":3568,"ariaHidden":198},[197],[161,3570,3572,3575],{"className":3571},[202],[161,3573],{"className":3574,"style":2538},[206],[161,3576,2820],{"className":3577,"style":2835},[211,395]," 和 ",[161,3580,3582,3595],{"className":3581},[164],[161,3583,3585],{"className":3584},[168],[170,3586,3587],{"xmlns":172},[174,3588,3589,3593],{},[177,3590,3591],{},[360,3592,763],{},[190,3594,763],{"encoding":192},[161,3596,3598],{"className":3597,"ariaHidden":198},[197],[161,3599,3601,3604],{"className":3600},[202],[161,3602],{"className":3603,"style":2538},[206],[161,3605,763],{"className":3606,"style":2865},[211,395]," 矩陣。",[148,3609,3610,3613],{},[112,3611,3612],{},"狀態維護："," 在殘差流（Residual Stream）中維護一個信念狀態（Belief State）。",[148,3615,3616,3619],{},[112,3617,3618],{},"遞迴更新："," 每一步根據新的觀測值和轉移邏輯更新信念。",[1786,3621,3622],{},[148,3623,3624,3627,3628,3578,3656,3684,3685,3749],{},[112,3625,3626],{},"複雜度控制："," ",[161,3629,3631,3644],{"className":3630},[164],[161,3632,3634],{"className":3633},[168],[170,3635,3636],{"xmlns":172},[174,3637,3638,3642],{},[177,3639,3640],{},[360,3641,2820],{},[190,3643,2820],{"encoding":192},[161,3645,3647],{"className":3646,"ariaHidden":198},[197],[161,3648,3650,3653],{"className":3649},[202],[161,3651],{"className":3652,"style":2538},[206],[161,3654,2820],{"className":3655,"style":2835},[211,395],[161,3657,3659,3672],{"className":3658},[164],[161,3660,3662],{"className":3661},[168],[170,3663,3664],{"xmlns":172},[174,3665,3666,3670],{},[177,3667,3668],{},[360,3669,763],{},[190,3671,763],{"encoding":192},[161,3673,3675],{"className":3674,"ariaHidden":198},[197],[161,3676,3678,3681],{"className":3677},[202],[161,3679],{"className":3680,"style":2538},[206],[161,3682,763],{"className":3683,"style":2865},[211,395]," 的參數從 Dirichlet 分佈中採樣。可能的 HMM 空間在連續參數下是無限的，即使離散化後也超過 ",[161,3686,3688,3705],{"className":3687},[164],[161,3689,3691],{"className":3690},[168],[170,3692,3693],{"xmlns":172},[174,3694,3695,3703],{},[177,3696,3697],{},[180,3698,3699,3701],{},[183,3700,185],{},[183,3702,276],{},[190,3704,279],{"encoding":192},[161,3706,3708],{"className":3707,"ariaHidden":198},[197],[161,3709,3711,3714,3717],{"className":3710},[202],[161,3712],{"className":3713,"style":207},[206],[161,3715,212],{"className":3716},[211],[161,3718,3720,3723],{"className":3719},[211],[161,3721,219],{"className":3722},[211],[161,3724,3726],{"className":3725},[223],[161,3727,3729],{"className":3728},[227],[161,3730,3732],{"className":3731},[231],[161,3733,3735],{"className":3734,"style":207},[235],[161,3736,3737,3740],{"style":238},[161,3738],{"className":3739,"style":243},[242],[161,3741,3743],{"className":3742},[247,248,249,250],[161,3744,3746],{"className":3745},[211,250],[161,3747,276],{"className":3748},[211,250],"。這迫使模型必須成為一個「元學習者」（Meta-Learner），即學會執行前向演算法，而非記憶特定的 HMM。",[132,3751,3753],{"id":3752},"_33-模型架構與基準對照",[112,3754,3755],{},"3.3 模型架構與基準對照",[116,3757,3758],{},"為了隔離 Transformer 架構（特別是注意力機制）的貢獻，研究使用了嚴格控制的對照組。",[1786,3760,3761,4017],{},[148,3762,3763,3766,3767],{},[112,3764,3765],{},"Transformer 模型："," 採用小型但標準的架構（約 267-268 萬參數）。\n",[1786,3768,3769,3893,4014],{},[148,3770,3771,3772,2180],{},"雙射任務：6 層，6 個注意力頭，模型維度 ",[161,3773,3775,3809],{"className":3774},[164],[161,3776,3778],{"className":3777},[168],[170,3779,3780],{"xmlns":172},[174,3781,3782,3806],{},[177,3783,3784,3801,3803],{},[462,3785,3786,3788],{},[360,3787,1140],{},[177,3789,3790,3792,3794,3796,3798],{},[360,3791,2608],{},[360,3793,2593],{},[360,3795,1140],{},[360,3797,2273],{},[360,3799,3800],{},"l",[364,3802,453],{},[183,3804,3805],{},"192",[190,3807,3808],{"encoding":192},"d_{model}=192",[161,3810,3812,3884],{"className":3811,"ariaHidden":198},[197],[161,3813,3815,3819,3875,3878,3881],{"className":3814},[202],[161,3816],{"className":3817,"style":3818},[206],"height:0.8444em;vertical-align:-0.15em;",[161,3820,3822,3825],{"className":3821},[211],[161,3823,1140],{"className":3824},[211,395],[161,3826,3828],{"className":3827},[223],[161,3829,3831,3867],{"className":3830},[227,546],[161,3832,3834,3864],{"className":3833},[231],[161,3835,3837],{"className":3836,"style":900},[235],[161,3838,3839,3842],{"style":556},[161,3840],{"className":3841,"style":243},[242],[161,3843,3845],{"className":3844},[247,248,249,250],[161,3846,3848,3851,3854,3857,3860],{"className":3847},[211,250],[161,3849,2608],{"className":3850},[211,395,250],[161,3852,2593],{"className":3853},[211,395,250],[161,3855,1140],{"className":3856},[211,395,250],[161,3858,2273],{"className":3859},[211,395,250],[161,3861,3800],{"className":3862,"style":3863},[211,395,250],"margin-right:0.0197em;",[161,3865,570],{"className":3866},[569],[161,3868,3870],{"className":3869},[231],[161,3871,3873],{"className":3872,"style":577},[235],[161,3874],{},[161,3876],{"className":3877,"style":401},[400],[161,3879,453],{"className":3880},[405],[161,3882],{"className":3883,"style":401},[400],[161,3885,3887,3890],{"className":3886},[202],[161,3888],{"className":3889,"style":2033},[206],[161,3891,3805],{"className":3892},[211],[148,3894,3895,3896,2180],{},"HMM 任務：9 層，8 個注意力頭，模型維度 ",[161,3897,3899,3932],{"className":3898},[164],[161,3900,3902],{"className":3901},[168],[170,3903,3904],{"xmlns":172},[174,3905,3906,3929],{},[177,3907,3908,3924,3926],{},[462,3909,3910,3912],{},[360,3911,1140],{},[177,3913,3914,3916,3918,3920,3922],{},[360,3915,2608],{},[360,3917,2593],{},[360,3919,1140],{},[360,3921,2273],{},[360,3923,3800],{},[364,3925,453],{},[183,3927,3928],{},"256",[190,3930,3931],{"encoding":192},"d_{model}=256",[161,3933,3935,4005],{"className":3934,"ariaHidden":198},[197],[161,3936,3938,3941,3996,3999,4002],{"className":3937},[202],[161,3939],{"className":3940,"style":3818},[206],[161,3942,3944,3947],{"className":3943},[211],[161,3945,1140],{"className":3946},[211,395],[161,3948,3950],{"className":3949},[223],[161,3951,3953,3988],{"className":3952},[227,546],[161,3954,3956,3985],{"className":3955},[231],[161,3957,3959],{"className":3958,"style":900},[235],[161,3960,3961,3964],{"style":556},[161,3962],{"className":3963,"style":243},[242],[161,3965,3967],{"className":3966},[247,248,249,250],[161,3968,3970,3973,3976,3979,3982],{"className":3969},[211,250],[161,3971,2608],{"className":3972},[211,395,250],[161,3974,2593],{"className":3975},[211,395,250],[161,3977,1140],{"className":3978},[211,395,250],[161,3980,2273],{"className":3981},[211,395,250],[161,3983,3800],{"className":3984,"style":3863},[211,395,250],[161,3986,570],{"className":3987},[569],[161,3989,3991],{"className":3990},[231],[161,3992,3994],{"className":3993,"style":577},[235],[161,3995],{},[161,3997],{"className":3998,"style":401},[400],[161,4000,453],{"className":4001},[405],[161,4003],{"className":4004,"style":401},[400],[161,4006,4008,4011],{"className":4007},[202],[161,4009],{"className":4010,"style":2033},[206],[161,4012,3928],{"className":4013},[211],[148,4015,4016],{},"配置：包含學習的位置嵌入、Pre-norm 殘差塊和標準的多頭自注意力機制。",[148,4018,4019,4022],{},[112,4020,4021],{},"MLP 基準模型："," 為了測試注意力機制的必要性，研究訓練了容量匹配（Capacity-matched）的多層感知機（MLP）。這些 MLP 擁有與 Transformer 幾乎相同的參數數量（誤差在 1% 以內），採用 18-20 層的深層結構，並配備了殘差連接和層歸一化（LayerNorm）。這確保了任何性能差異都源於架構本身（是否存在注意力機制），而非參數規模或訓練技巧的差異。",[108,4024,4026],{"id":4025},"_4-實證結果行為層面的深度分析",[112,4027,4028],{},"4. 實證結果：行為層面的深度分析",[116,4030,4031],{},"實驗結果提供了決定性的證據，表明 Transformer 在演算法層面上與貝葉斯推理具有驚人的一致性，而傳統 MLP 則完全無法勝任此類任務。",[132,4033,4035],{"id":4034},"_41-雙射任務中的精確熵追蹤",[112,4036,4037],{},"4.1 雙射任務中的精確熵追蹤",[116,4039,4040],{},"在雙射學習任務中，Transformer 展現了近乎完美的假設消除能力。",[1786,4042,4043,4157,4261],{},[148,4044,4045,4048,4049,4156],{},[112,4046,4047],{},"階梯狀熵曲線："," 隨著模型接收到更多的輸入-輸出對，剩餘的可能性空間縮小，預測的不確定性（熵）應呈現離散的階梯狀下降。實驗數據顯示，Transformer 的預測熵曲線與理論上的 ",[161,4050,4052,4086],{"className":4051},[164],[161,4053,4055],{"className":4054},[168],[170,4056,4057],{"xmlns":172},[174,4058,4059,4083],{},[177,4060,4061,4077,4079,4081],{},[462,4062,4063,4065],{},[360,4064,2260],{},[177,4066,4067,4069,4071,4073,4075],{},[360,4068,2265],{},[360,4070,2268],{},[360,4072,477],{},[360,4074,2273],{},[360,4076,2276],{},[364,4078,373],{"stretchy":372},[360,4080,492],{},[364,4082,378],{"stretchy":372},[190,4084,4085],{"encoding":192},"H_{Bayes}(k)",[161,4087,4089],{"className":4088,"ariaHidden":198},[197],[161,4090,4092,4095,4147,4150,4153],{"className":4091},[202],[161,4093],{"className":4094,"style":2323},[206],[161,4096,4098,4101],{"className":4097},[211],[161,4099,2260],{"className":4100,"style":2330},[211,395],[161,4102,4104],{"className":4103},[223],[161,4105,4107,4139],{"className":4106},[227,546],[161,4108,4110,4136],{"className":4109},[231],[161,4111,4113],{"className":4112,"style":2343},[235],[161,4114,4115,4118],{"style":2346},[161,4116],{"className":4117,"style":243},[242],[161,4119,4121],{"className":4120},[247,248,249,250],[161,4122,4124,4127,4130,4133],{"className":4123},[211,250],[161,4125,2265],{"className":4126,"style":2359},[211,395,250],[161,4128,2268],{"className":4129},[211,395,250],[161,4131,477],{"className":4132,"style":419},[211,395,250],[161,4134,2369],{"className":4135},[211,395,250],[161,4137,570],{"className":4138},[569],[161,4140,4142],{"className":4141},[231],[161,4143,4145],{"className":4144,"style":2379},[235],[161,4146],{},[161,4148,373],{"className":4149},[423],[161,4151,492],{"className":4152,"style":678},[211,395],[161,4154,378],{"className":4155},[430]," 曲線幾乎完全重疊（參見原論文 Figure 1）。",[148,4158,4159,4162,4163,4260],{},[112,4160,4161],{},"微小的誤差："," 定量分析顯示，Transformer 的平均絕對熵誤差（MAE）僅為 ",[161,4164,4166,4194],{"className":4165},[164],[161,4167,4169],{"className":4168},[168],[170,4170,4171],{"xmlns":172},[174,4172,4173,4191],{},[177,4174,4175,4178,4181],{},[183,4176,4177],{},"3",[364,4179,4180],{},"×",[180,4182,4183,4185],{},[183,4184,185],{},[177,4186,4187,4189],{},[364,4188,814],{},[183,4190,4177],{},[190,4192,4193],{"encoding":192},"3 \\times 10^{-3}",[161,4195,4197,4216],{"className":4196,"ariaHidden":198},[197],[161,4198,4200,4204,4207,4210,4213],{"className":4199},[202],[161,4201],{"className":4202,"style":4203},[206],"height:0.7278em;vertical-align:-0.0833em;",[161,4205,4177],{"className":4206},[211],[161,4208],{"className":4209,"style":1934},[400],[161,4211,4180],{"className":4212},[1186],[161,4214],{"className":4215,"style":1934},[400],[161,4217,4219,4222,4225],{"className":4218},[202],[161,4220],{"className":4221,"style":207},[206],[161,4223,212],{"className":4224},[211],[161,4226,4228,4231],{"className":4227},[211],[161,4229,219],{"className":4230},[211],[161,4232,4234],{"className":4233},[223],[161,4235,4237],{"className":4236},[227],[161,4238,4240],{"className":4239},[231],[161,4241,4243],{"className":4242,"style":207},[235],[161,4244,4245,4248],{"style":238},[161,4246],{"className":4247,"style":243},[242],[161,4249,4251],{"className":4250},[247,248,249,250],[161,4252,4254,4257],{"className":4253},[211,250],[161,4255,814],{"className":4256},[211,250],[161,4258,4177],{"className":4259},[211,250]," bits。這一誤差級別極低，幾乎等同於單精度浮點數運算引入的數值噪聲，意味著模型已經達到了機器精度的極限。",[148,4262,4263,4266],{},[112,4264,4265],{},"序列級別的動態性："," 更重要的是，模型並非僅僅擬合了平均趨勢。在單個序列的層面上（原論文 Figure 2），當模型遇到一個已知的輸入重複出現時，其預測熵會瞬間坍縮至零；而當遇到新輸入時，則準確回到消除曲線。這種「鋸齒狀」的動態行為證明了模型正在進行逐個序列的即時推理，而非簡單的統計平均。",[132,4268,4270],{"id":4269},"_42-hmm-任務中的遞迴演算法學習",[112,4271,4272],{},"4.2 HMM 任務中的遞迴演算法學習",[116,4274,4275],{},"HMM 任務的結果進一步證實了模型處理隨機性和遞迴計算的能力。",[1786,4277,4278,4434,4763],{},[148,4279,4280,4283,4284,4337,4338,4433],{},[112,4281,4282],{},"前向演算法的模擬："," 在訓練長度（",[161,4285,4287,4306],{"className":4286},[164],[161,4288,4290],{"className":4289},[168],[170,4291,4292],{"xmlns":172},[174,4293,4294,4303],{},[177,4295,4296,4299,4301],{},[360,4297,4298],{},"K",[364,4300,453],{},[183,4302,2525],{},[190,4304,4305],{"encoding":192},"K=20",[161,4307,4309,4328],{"className":4308,"ariaHidden":198},[197],[161,4310,4312,4315,4319,4322,4325],{"className":4311},[202],[161,4313],{"className":4314,"style":2538},[206],[161,4316,4298],{"className":4317,"style":4318},[211,395],"margin-right:0.0715em;",[161,4320],{"className":4321,"style":401},[400],[161,4323,453],{"className":4324},[405],[161,4326],{"className":4327,"style":401},[400],[161,4329,4331,4334],{"className":4330},[202],[161,4332],{"className":4333,"style":2033},[206],[161,4335,2525],{"className":4336},[211],"）內，Transformer 的熵誤差低至 ",[161,4339,4341,4368],{"className":4340},[164],[161,4342,4344],{"className":4343},[168],[170,4345,4346],{"xmlns":172},[174,4347,4348,4365],{},[177,4349,4350,4353,4355],{},[183,4351,4352],{},"7.5",[364,4354,4180],{},[180,4356,4357,4359],{},[183,4358,185],{},[177,4360,4361,4363],{},[364,4362,814],{},[183,4364,2740],{},[190,4366,4367],{"encoding":192},"7.5 \\times 10^{-5}",[161,4369,4371,4389],{"className":4370,"ariaHidden":198},[197],[161,4372,4374,4377,4380,4383,4386],{"className":4373},[202],[161,4375],{"className":4376,"style":4203},[206],[161,4378,4352],{"className":4379},[211],[161,4381],{"className":4382,"style":1934},[400],[161,4384,4180],{"className":4385},[1186],[161,4387],{"className":4388,"style":1934},[400],[161,4390,4392,4395,4398],{"className":4391},[202],[161,4393],{"className":4394,"style":207},[206],[161,4396,212],{"className":4397},[211],[161,4399,4401,4404],{"className":4400},[211],[161,4402,219],{"className":4403},[211],[161,4405,4407],{"className":4406},[223],[161,4408,4410],{"className":4409},[227],[161,4411,4413],{"className":4412},[231],[161,4414,4416],{"className":4415,"style":207},[235],[161,4417,4418,4421],{"style":238},[161,4419],{"className":4420,"style":243},[242],[161,4422,4424],{"className":4423},[247,248,249,250],[161,4425,4427,4430],{"className":4426},[211,250],[161,4428,814],{"className":4429},[211,250],[161,4431,2740],{"className":4432},[211,250]," bits。這意味著模型內部實現的計算過程與數學上的前向演算法在功能上是等價的。",[148,4435,4436,4439,4440,3578,4492,4544,4545,4597,4598,4648,4649,4711,4712,4762],{},[112,4437,4438],{},"長度泛化（Length Generalization）："," 這是區分「推理」與「記憶」的黃金標準。研究人員將測試序列長度延伸至 ",[161,4441,4443,4462],{"className":4442},[164],[161,4444,4446],{"className":4445},[168],[170,4447,4448],{"xmlns":172},[174,4449,4450,4459],{},[177,4451,4452,4454,4456],{},[360,4453,4298],{},[364,4455,453],{},[183,4457,4458],{},"30",[190,4460,4461],{"encoding":192},"K=30",[161,4463,4465,4483],{"className":4464,"ariaHidden":198},[197],[161,4466,4468,4471,4474,4477,4480],{"className":4467},[202],[161,4469],{"className":4470,"style":2538},[206],[161,4472,4298],{"className":4473,"style":4318},[211,395],[161,4475],{"className":4476,"style":401},[400],[161,4478,453],{"className":4479},[405],[161,4481],{"className":4482,"style":401},[400],[161,4484,4486,4489],{"className":4485},[202],[161,4487],{"className":4488,"style":2033},[206],[161,4490,4458],{"className":4491},[211],[161,4493,4495,4514],{"className":4494},[164],[161,4496,4498],{"className":4497},[168],[170,4499,4500],{"xmlns":172},[174,4501,4502,4511],{},[177,4503,4504,4506,4508],{},[360,4505,4298],{},[364,4507,453],{},[183,4509,4510],{},"50",[190,4512,4513],{"encoding":192},"K=50",[161,4515,4517,4535],{"className":4516,"ariaHidden":198},[197],[161,4518,4520,4523,4526,4529,4532],{"className":4519},[202],[161,4521],{"className":4522,"style":2538},[206],[161,4524,4298],{"className":4525,"style":4318},[211,395],[161,4527],{"className":4528,"style":401},[400],[161,4530,453],{"className":4531},[405],[161,4533],{"className":4534,"style":401},[400],[161,4536,4538,4541],{"className":4537},[202],[161,4539],{"className":4540,"style":2033},[206],[161,4542,4510],{"className":4543},[211],"（即訓練長度的 1.5 倍和 2.5 倍）。結果顯示，Transformer 在超過訓練邊界（",[161,4546,4548,4566],{"className":4547},[164],[161,4549,4551],{"className":4550},[168],[170,4552,4553],{"xmlns":172},[174,4554,4555,4563],{},[177,4556,4557,4559,4561],{},[360,4558,2603],{},[364,4560,453],{},[183,4562,2525],{},[190,4564,4565],{"encoding":192},"t=20",[161,4567,4569,4588],{"className":4568,"ariaHidden":198},[197],[161,4570,4572,4576,4579,4582,4585],{"className":4571},[202],[161,4573],{"className":4574,"style":4575},[206],"height:0.6151em;",[161,4577,2603],{"className":4578},[211,395],[161,4580],{"className":4581,"style":401},[400],[161,4583,453],{"className":4584},[405],[161,4586],{"className":4587,"style":401},[400],[161,4589,4591,4594],{"className":4590},[202],[161,4592],{"className":4593,"style":2033},[206],[161,4595,2525],{"className":4596},[211],"）後，誤差僅呈現平滑且緩慢的增長（",[161,4599,4601,4618],{"className":4600},[164],[161,4602,4604],{"className":4603},[168],[170,4605,4606],{"xmlns":172},[174,4607,4608,4616],{},[177,4609,4610,4612,4614],{},[360,4611,4298],{},[364,4613,453],{},[183,4615,4510],{},[190,4617,4513],{"encoding":192},[161,4619,4621,4639],{"className":4620,"ariaHidden":198},[197],[161,4622,4624,4627,4630,4633,4636],{"className":4623},[202],[161,4625],{"className":4626,"style":2538},[206],[161,4628,4298],{"className":4629,"style":4318},[211,395],[161,4631],{"className":4632,"style":401},[400],[161,4634,453],{"className":4635},[405],[161,4637],{"className":4638,"style":401},[400],[161,4640,4642,4645],{"className":4641},[202],[161,4643],{"className":4644,"style":2033},[206],[161,4646,4510],{"className":4647},[211]," 時 MAE ",[161,4650,4652,4681],{"className":4651},[164],[161,4653,4655],{"className":4654},[168],[170,4656,4657],{"xmlns":172},[174,4658,4659,4678],{},[177,4660,4661,4663,4665,4667,4669,4671,4673,4675],{},[400,4662],{"linebreak":2581},[360,4664,2268],{},[360,4666,116],{},[360,4668,116],{},[360,4670,2590],{},[360,4672,2593],{},[360,4674,466],{},[183,4676,4677],{},"0.02",[190,4679,4680],{"encoding":192},"\\\\approx 0.02",[161,4682,4684,4687],{"className":4683,"ariaHidden":198},[197],[161,4685],{"className":4686},[400,2581],[161,4688,4690,4693,4696,4699,4702,4705,4708],{"className":4689},[202],[161,4691],{"className":4692,"style":2647},[206],[161,4694,2268],{"className":4695},[211,395],[161,4697,2654],{"className":4698},[211,395],[161,4700,2590],{"className":4701,"style":396},[211,395],[161,4703,2593],{"className":4704},[211,395],[161,4706,466],{"className":4707},[211,395],[161,4709,4677],{"className":4710},[211]," bits），且在 ",[161,4713,4715,4732],{"className":4714},[164],[161,4716,4718],{"className":4717},[168],[170,4719,4720],{"xmlns":172},[174,4721,4722,4730],{},[177,4723,4724,4726,4728],{},[360,4725,2603],{},[364,4727,453],{},[183,4729,2525],{},[190,4731,4565],{"encoding":192},[161,4733,4735,4753],{"className":4734,"ariaHidden":198},[197],[161,4736,4738,4741,4744,4747,4750],{"className":4737},[202],[161,4739],{"className":4740,"style":4575},[206],[161,4742,2603],{"className":4743},[211,395],[161,4745],{"className":4746,"style":401},[400],[161,4748,453],{"className":4749},[405],[161,4751],{"className":4752,"style":401},[400],[161,4754,4756,4759],{"className":4755},[202],[161,4757],{"className":4758,"style":2033},[206],[161,4760,2525],{"className":4761},[211]," 處沒有出現任何突變或斷裂。這證明模型學到的是一個與位置無關的通用遞迴算子（Recursive Operator），而非死記硬背了長度為 20 的特定模式。",[148,4764,4765,4768,4769,4820],{},[112,4766,4767],{},"語義不變性（Semantic Invariance）："," 研究人員進行了一項巧妙的測試：隨機置換 HMM 隱藏狀態的標籤（例如將「狀態 1」與「狀態 3」互換），並重新生成數據進行測試。結果顯示，模型的校準誤差在置換前後幾乎完全一致（分佈在 ",[161,4770,4772,4790],{"className":4771},[164],[161,4773,4775],{"className":4774},[168],[170,4776,4777],{"xmlns":172},[174,4778,4779,4787],{},[177,4780,4781,4783,4785],{},[360,4782,477],{},[364,4784,453],{},[360,4786,466],{},[190,4788,4789],{"encoding":192},"y=x",[161,4791,4793,4811],{"className":4792,"ariaHidden":198},[197],[161,4794,4796,4799,4802,4805,4808],{"className":4795},[202],[161,4797],{"className":4798,"style":721},[206],[161,4800,477],{"className":4801,"style":419},[211,395],[161,4803],{"className":4804,"style":401},[400],[161,4806,453],{"className":4807},[405],[161,4809],{"className":4810,"style":401},[400],[161,4812,4814,4817],{"className":4813},[202],[161,4815],{"className":4816,"style":505},[206],[161,4818,466],{"className":4819},[211,395]," 對角線上，如 Figure 8 所示）。這表明模型理解的是 HMM 的拓撲結構和轉移邏輯，而非依賴於特定的狀態 ID。",[132,4822,4824],{"id":4823},"_43-mlp-的架構性崩潰",[112,4825,4826],{},"4.3 MLP 的架構性崩潰",[116,4828,4829],{},"與 Transformer 的卓越表現形成強烈對比的是，容量匹配的 MLP 在兩個風洞任務中均遭遇了災難性的失敗。",[1786,4831,4832,4838,4844],{},[148,4833,4834,4837],{},[112,4835,4836],{},"無法進行假設消除："," 在雙射任務中，MLP 的 MAE 高達約 1.85 bits，比 Transformer 差了近三個數量級。其熵曲線幾乎保持平坦，顯示它只能學習到輸出的邊際分佈，完全無法根據上下文進行動態的假設剔除。",[148,4839,4840,4843],{},[112,4841,4842],{},"無法進行狀態追蹤："," 在 HMM 任務中，MLP 的誤差穩定在 0.4 bits 左右，且不隨序列長度變化。這表明它無法維護或更新信念狀態，只能進行靜態的猜測。",[148,4845,4846,4849,4850,4853],{},[112,4847,4848],{},"根本原因分析："," 這一失敗並非源於參數不足或訓練不當，而是源於架構缺陷。MLP 缺乏",[112,4851,4852],{},"內容定址（Content-Addressable）的路由機制","。在貝葉斯推理中，模型需要根據當前的證據（Query）去檢索相關的假設或參數（Key）。Transformer 的注意力機制天然支持這種操作，而 MLP 只能依賴固定的權重矩陣，無法動態地「查找」上下文中的信息。",[116,4855,4856],{},"為了直觀展示兩種架構的巨大差異，下表總結了關鍵的誤差數據：",[4858,4859,4860,4883],"table",{},[4861,4862,4863],"thead",{},[4864,4865,4866,4871,4874,4877,4880],"tr",{},[4867,4868,4870],"th",{"align":4869},"left","任務 (Task)",[4867,4872,4873],{"align":4869},"測試條件 (Condition)",[4867,4875,4876],{"align":4869},"Transformer MAE (bits)",[4867,4878,4879],{"align":4869},"MLP MAE (bits)",[4867,4881,4882],{"align":4869},"性能差距 (Degradation Factor)",[4884,4885,4886,4959,5031,5103],"tbody",{},[4864,4887,4888,4893,4946,4951,4954],{},[4889,4890,4891],"td",{"align":4869},[112,4892,1770],{},[4889,4894,4895,4896,378],{"align":4869},"訓練長度(",[161,4897,4899,4916],{"className":4898},[164],[161,4900,4902],{"className":4901},[168],[170,4903,4904],{"xmlns":172},[174,4905,4906,4914],{},[177,4907,4908,4910,4912],{},[360,4909,4298],{},[364,4911,453],{},[183,4913,2525],{},[190,4915,4305],{"encoding":192},[161,4917,4919,4937],{"className":4918,"ariaHidden":198},[197],[161,4920,4922,4925,4928,4931,4934],{"className":4921},[202],[161,4923],{"className":4924,"style":2538},[206],[161,4926,4298],{"className":4927,"style":4318},[211,395],[161,4929],{"className":4930,"style":401},[400],[161,4932,453],{"className":4933},[405],[161,4935],{"className":4936,"style":401},[400],[161,4938,4940,4943],{"className":4939},[202],[161,4941],{"className":4942,"style":2033},[206],[161,4944,2525],{"className":4945},[211],[4889,4947,4948],{"align":4869},[112,4949,4950],{},"0.0030",[4889,4952,4953],{"align":4869},"1.8540",[4889,4955,4956],{"align":4869},[112,4957,4958],{},"~618倍",[4864,4960,4961,4966,5018,5023,5026],{},[4889,4962,4963],{"align":4869},[112,4964,4965],{},"HMM 狀態追蹤",[4889,4967,4895,4968,378],{"align":4869},[161,4969,4971,4988],{"className":4970},[164],[161,4972,4974],{"className":4973},[168],[170,4975,4976],{"xmlns":172},[174,4977,4978,4986],{},[177,4979,4980,4982,4984],{},[360,4981,4298],{},[364,4983,453],{},[183,4985,2525],{},[190,4987,4305],{"encoding":192},[161,4989,4991,5009],{"className":4990,"ariaHidden":198},[197],[161,4992,4994,4997,5000,5003,5006],{"className":4993},[202],[161,4995],{"className":4996,"style":2538},[206],[161,4998,4298],{"className":4999,"style":4318},[211,395],[161,5001],{"className":5002,"style":401},[400],[161,5004,453],{"className":5005},[405],[161,5007],{"className":5008,"style":401},[400],[161,5010,5012,5015],{"className":5011},[202],[161,5013],{"className":5014,"style":2033},[206],[161,5016,2525],{"className":5017},[211],[4889,5019,5020],{"align":4869},[112,5021,5022],{},"0.000075",[4889,5024,5025],{"align":4869},"0.4090",[4889,5027,5028],{"align":4869},[112,5029,5030],{},"~5,467倍",[4864,5032,5033,5037,5090,5095,5098],{},[4889,5034,5035],{"align":4869},[112,5036,4965],{},[4889,5038,5039,5040,378],{"align":4869},"泛化長度(",[161,5041,5043,5060],{"className":5042},[164],[161,5044,5046],{"className":5045},[168],[170,5047,5048],{"xmlns":172},[174,5049,5050,5058],{},[177,5051,5052,5054,5056],{},[360,5053,4298],{},[364,5055,453],{},[183,5057,4458],{},[190,5059,4461],{"encoding":192},[161,5061,5063,5081],{"className":5062,"ariaHidden":198},[197],[161,5064,5066,5069,5072,5075,5078],{"className":5065},[202],[161,5067],{"className":5068,"style":2538},[206],[161,5070,4298],{"className":5071,"style":4318},[211,395],[161,5073],{"className":5074,"style":401},[400],[161,5076,453],{"className":5077},[405],[161,5079],{"className":5080,"style":401},[400],[161,5082,5084,5087],{"className":5083},[202],[161,5085],{"className":5086,"style":2033},[206],[161,5088,4458],{"className":5089},[211],[4889,5091,5092],{"align":4869},[112,5093,5094],{},"0.0125",[4889,5096,5097],{"align":4869},"0.4000",[4889,5099,5100],{"align":4869},[112,5101,5102],{},"32倍",[4864,5104,5105,5109,5161,5166,5169],{},[4889,5106,5107],{"align":4869},[112,5108,4965],{},[4889,5110,5039,5111,378],{"align":4869},[161,5112,5114,5131],{"className":5113},[164],[161,5115,5117],{"className":5116},[168],[170,5118,5119],{"xmlns":172},[174,5120,5121,5129],{},[177,5122,5123,5125,5127],{},[360,5124,4298],{},[364,5126,453],{},[183,5128,4510],{},[190,5130,4513],{"encoding":192},[161,5132,5134,5152],{"className":5133,"ariaHidden":198},[197],[161,5135,5137,5140,5143,5146,5149],{"className":5136},[202],[161,5138],{"className":5139,"style":2538},[206],[161,5141,4298],{"className":5142,"style":4318},[211,395],[161,5144],{"className":5145,"style":401},[400],[161,5147,453],{"className":5148},[405],[161,5150],{"className":5151,"style":401},[400],[161,5153,5155,5158],{"className":5154},[202],[161,5156],{"className":5157,"style":2033},[206],[161,5159,4510],{"className":5160},[211],[4889,5162,5163],{"align":4869},[112,5164,5165],{},"0.0288",[4889,5167,5168],{"align":4869},"0.4020",[4889,5170,5171],{"align":4869},[112,5172,5173],{},"14倍",[116,5175,5176,5177,2180],{},"這組數據強有力地證明：在需要複雜推理的任務中，注意力機制並非錦上添花，而是",[112,5178,5179],{},"幾何上的必要條件",[108,5181,5183],{"id":5182},"_5-機制解釋注意力機制的貝葉斯幾何學",[112,5184,5185],{},"5. 機制解釋：注意力機制的貝葉斯幾何學",[116,5187,5188,5189,5192,5193,5196],{},"既然證實了 Transformer ",[1755,5190,5191],{},"能夠","執行貝葉斯推理，接下來的關鍵問題是：它是",[1755,5194,5195],{},"如何","做到的？Agarwal 等人利用幾何診斷工具，分析了模型內部的向量表徵，揭示了一個跨越層級的三階段運作機制。這一機制將貝葉斯推理的抽象數學運算轉化為具體的幾何操作。",[132,5198,5200],{"id":5199},"_51-第一階段基礎綁定與假設框架layer-0",[112,5201,5202],{},"5.1 第一階段：基礎綁定與假設框架（Layer 0）",[116,5204,5205],{},"推理的起點在於構建一個能夠表示所有可能假設的幾何空間，這發生在網絡的最底層。",[1786,5207,5208,5214,5220],{},[148,5209,5210,5213],{},[112,5211,5212],{},"正交鍵基底（Orthogonal Key Basis）："," 在 Layer 0，模型學會將輸入 Token 的鍵向量（Keys）組織成一個近似正交的基底。幾何分析顯示，不同輸入 Token 對應的 Key 向量之間的餘弦相似度接近於零（參見 Figure 14）。這意味著每個輸入都被映射到高維空間中的一個獨立軸或子空間上，形成了一個坐標系。",[148,5215,5216,5219],{},[112,5217,5218],{},"假設頭（Hypothesis Head）的關鍵性："," 消融研究（Ablation Study）發現，Layer 0 中存在一個特定的注意力頭，其重要性遠超其他頭。移除這個「假設頭」會導致整體誤差激增。這個頭的功能是將原始輸入「綁定」到殘差流中的特定幾何位置，從而定義了後續推理的「假設空間」（Hypothesis Space）。如果沒有這個框架，後續層級就無法對假設進行操作。",[148,5221,5222,5225],{},[112,5223,5224],{},"早期穩定性："," 訓練動態顯示，這個幾何框架在訓練早期就已形成並保持穩定。模型首先學會「如何表示問題」，然後才學會「如何精確計算」。",[132,5227,5229],{"id":5228},"_52-第二階段順序消除與信息路由middle-layers",[112,5230,5231],{},"5.2 第二階段：順序消除與信息路由（Middle Layers）",[116,5233,5234],{},"一旦假設框架建立，中間層級開始執行核心的貝葉斯更新——即根據證據剔除不可能的假設。",[1786,5236,5237,5243,5249],{},[148,5238,5239,5242],{},[112,5240,5241],{},"注意力作為路由（Attention as Routing）："," 注意力機制在此充當了動態路由器的角色。當前位置的查詢向量（Query）會與上下文中的鍵向量（Key）進行比對。",[148,5244,5245,5248],{},[112,5246,5247],{},"漸進式銳化（Progressive Sharpening）："," 在淺層，注意力分佈可能較為分散。但隨著層數加深，Query 與 Key 之間的對齊變得越來越尖銳（Sharpened）。在雙射任務的 Layer 5 中，Query 向量幾乎只與那些與當前證據一致的「可行假設」的 Key 產生高相似度（參見 Figure 15）。",[148,5250,5251,5254,5255,5307],{},[112,5252,5253],{},"貝葉斯「與」運算（Bayesian AND-ing）："," 這種幾何上的聚焦效應對應於貝葉斯更新中的乘法性質。如果一個假設在第 ",[161,5256,5258,5276],{"className":5257},[164],[161,5259,5261],{"className":5260},[168],[170,5262,5263],{"xmlns":172},[174,5264,5265,5273],{},[177,5266,5267,5269,5271],{},[360,5268,2603],{},[364,5270,814],{},[183,5272,212],{},[190,5274,5275],{"encoding":192},"t-1",[161,5277,5279,5298],{"className":5278,"ariaHidden":198},[197],[161,5280,5282,5286,5289,5292,5295],{"className":5281},[202],[161,5283],{"className":5284,"style":5285},[206],"height:0.6984em;vertical-align:-0.0833em;",[161,5287,2603],{"className":5288},[211,395],[161,5290],{"className":5291,"style":1934},[400],[161,5293,814],{"className":5294},[1186],[161,5296],{"className":5297,"style":1934},[400],[161,5299,5301,5304],{"className":5300},[202],[161,5302],{"className":5303,"style":2033},[206],[161,5305,212],{"className":5306},[211]," 步被證據排除，後續層級的注意力機制就會停止從該假設路由信息。前饋網絡（FFN）則負責處理這些路由過來的信息，執行數值上的後驗計算，並更新殘差流中的信念狀態。這一過程展示了 Transformer 如何利用深度（Depth）來實現序列化的邏輯消除。",[132,5309,5311],{"id":5310},"_53-第三階段流形展開與精度精煉late-layers",[112,5312,5313],{},"5.3 第三階段：流形展開與精度精煉（Late Layers）",[116,5315,5316],{},"最後的階段涉及將粗略的信念狀態轉化為精確的機率輸出。",[1786,5318,5319,5325,5331],{},[148,5320,5321,5324],{},[112,5322,5323],{},"值流形（Value Manifold）："," 研究人員對模型末層的「值向量」（Value Vectors）進行主成分分析（PCA），發現了一個驚人的幾何結構。這些向量並非隨機分佈，而是排列在一個低維流形上（在雙射任務中是一條一維曲線）。",[148,5326,5327,5330],{},[112,5328,5329],{},"熵參數化："," 這條曲線被後驗熵完美地參數化。也就是說，模型將其「不確定性」直接編碼為流形上的位置。高熵（不確定）的狀態聚集在曲線的一端，低熵（確定）的狀態則位於另一端（參見 Figure 16）。",[148,5332,5333,5336],{},[112,5334,5335],{},"框架-精度分離（Frame-Precision Dissociation）："," 一個重要的發現是，「路由」（注意力圖）的穩定早於「流形」（數值精度）的完善。模型先學會了「去哪裡找信息」（幾何路由），然後在訓練後期才慢慢學會「如何精確表達機率」（流形展開）。這種分離解釋了為什麼模型在訓練後期能顯著降低誤差——它是在一個固定的幾何框架內微調數值精度。",[108,5338,5340],{"id":5339},"_6-深入數據分析與消融研究",[112,5341,5342],{},"6. 深入數據分析與消融研究",[116,5344,5345],{},"為了驗證上述機制的必要性與穩健性，報告提供了詳細的消融實驗數據。這些數據揭示了 Transformer 內部各組件的功能特異性。",[132,5347,5349],{"id":5348},"_61-層級與組件的消融影響",[112,5350,5351],{},"6.1 層級與組件的消融影響",[116,5353,5354],{},"研究人員逐一移除了模型的不同部分，觀察誤差（MAE）的變化。",[4858,5356,5357,5373],{},[4861,5358,5359],{},[4864,5360,5361,5364,5367,5370],{},[4867,5362,5363],{"align":4869},"消融類型 (Ablation Type)",[4867,5365,5366],{"align":4869},"被移除組件 (Component Removed)",[4867,5368,5369],{"align":4869},"對 MAE 的影響",[4867,5371,5372],{"align":4869},"洞察 (Insight)",[4884,5374,5375,5420,5438,5453],{},[4864,5376,5377,5382,5414,5417],{},[4889,5378,5379],{"align":4869},[112,5380,5381],{},"逐層消融",[4889,5383,5384,5385,378],{"align":4869},"任意單一層 (Layer ",[161,5386,5388,5401],{"className":5387},[164],[161,5389,5391],{"className":5390},[168],[170,5392,5393],{"xmlns":172},[174,5394,5395,5399],{},[177,5396,5397],{},[360,5398,469],{},[190,5400,469],{"encoding":192},[161,5402,5404],{"className":5403,"ariaHidden":198},[197],[161,5405,5407,5411],{"className":5406},[202],[161,5408],{"className":5409,"style":5410},[206],"height:0.6595em;",[161,5412,469],{"className":5413},[211,395],[4889,5415,5416],{"align":4869},"誤差增加 >10倍",[4889,5418,5419],{"align":4869},"證明計算是深度組合的（Compositional），沒有哪一層是冗餘的。每一層都執行了一個必要的消除步驟。",[4864,5421,5422,5427,5430,5435],{},[4889,5423,5424],{"align":4869},[112,5425,5426],{},"逐頭消融",[4889,5428,5429],{"align":4869},"Layer 0 「假設頭」",[4889,5431,5432],{"align":4869},[112,5433,5434],{},"系統性崩潰",[4889,5436,5437],{"align":4869},"此頭負責構建幾何基底。移除它等於摧毀了推理的坐標系，導致後續所有計算失效。",[4864,5439,5440,5444,5447,5450],{},[4889,5441,5442],{"align":4869},[112,5443,5426],{},[4889,5445,5446],{"align":4869},"後層注意力頭",[4889,5448,5449],{"align":4869},"中等影響",[4889,5451,5452],{"align":4869},"後層頭部主要負責精煉值流形，具有一定的並行冗餘性。",[4864,5454,5455,5460,5463,5507],{},[4889,5456,5457],{"align":4869},[112,5458,5459],{},"架構變體",[4889,5461,5462],{"align":4869},"頂層兩層的注意力",[4889,5464,5465,5466,5506],{"align":4869},"訓練長度內影響小，但在 ",[161,5467,5469,5488],{"className":5468},[164],[161,5470,5472],{"className":5471},[168],[170,5473,5474],{"xmlns":172},[174,5475,5476,5485],{},[177,5477,5478,5480,5483],{},[360,5479,4298],{},[2576,5481,5482],{}," ",[183,5484,2525],{},[190,5486,5487],{"encoding":192},"K\\>20",[161,5489,5491],{"className":5490,"ariaHidden":198},[197],[161,5492,5494,5497,5500,5503],{"className":5493},[202],[161,5495],{"className":5496,"style":2538},[206],[161,5498,4298],{"className":5499,"style":4318},[211,395],[161,5501],{"className":5502,"style":1934},[400],[161,5504,2525],{"className":5505},[211]," 時崩潰",[4889,5508,5509,5510,5513],{"align":4869},"頂層注意力對於",[112,5511,5512],{},"長度泛化","至關重要。它負責在長序列中維持穩定的信念更新，防止誤差累積。",[132,5515,5517],{"id":5516},"_62-多種子穩健性分析",[112,5518,5519],{},"6.2 多種子穩健性分析",[116,5521,5522,5523,5600],{},"為了確保這些發現不是隨機初始化的產物，研究在 5 個不同的隨機種子下重複了 HMM 實驗。結果顯示（參見原論文 Figure 11），不同種子之間的誤差曲線在 ",[161,5524,5526,5552],{"className":5525},[164],[161,5527,5529],{"className":5528},[168],[170,5530,5531],{"xmlns":172},[174,5532,5533,5549],{},[177,5534,5535,5537,5539,5541,5543,5545,5547],{},[360,5536,4298],{},[364,5538,453],{},[183,5540,2525],{},[364,5542,472],{"separator":198},[183,5544,4458],{},[364,5546,472],{"separator":198},[183,5548,4510],{},[190,5550,5551],{"encoding":192},"K=20, 30, 50",[161,5553,5555,5573],{"className":5554,"ariaHidden":198},[197],[161,5556,5558,5561,5564,5567,5570],{"className":5557},[202],[161,5559],{"className":5560,"style":2538},[206],[161,5562,4298],{"className":5563,"style":4318},[211,395],[161,5565],{"className":5566,"style":401},[400],[161,5568,453],{"className":5569},[405],[161,5571],{"className":5572,"style":401},[400],[161,5574,5576,5579,5582,5585,5588,5591,5594,5597],{"className":5575},[202],[161,5577],{"className":5578,"style":2647},[206],[161,5580,2525],{"className":5581},[211],[161,5583,472],{"className":5584},[583],[161,5586],{"className":5587,"style":587},[400],[161,5589,4458],{"className":5590},[211],[161,5592,472],{"className":5593},[583],[161,5595],{"className":5596,"style":587},[400],[161,5598,4510],{"className":5599},[211]," 時幾乎完全重合。種子間的變異性遠小於 Transformer 與 MLP 之間的性能差距，這證實了 Transformer 收斂到貝葉斯後驗是一個穩健的架構特性，而非運氣。",[108,5602,5604],{"id":5603},"_7-對大型語言模型與提示工程的啟示",[112,5605,5606],{},"7. 對大型語言模型與提示工程的啟示",[116,5608,5609],{},"雖然本研究是在小型受控環境中進行的，但其發現對於理解數千億參數的 LLM 及其應用（如提示工程）具有深遠的第二階與第三階啟示。",[132,5611,5613],{"id":5612},"_71-啟示一幻覺的幾何解釋",[112,5614,5615],{},"7.1 啟示一：「幻覺」的幾何解釋",[116,5617,5618],{},"本研究發現的「值流形」為 LLM 的幻覺（Hallucination）現象提供了一個嚴謹的幾何解釋。",[1786,5620,5621,5627,5633],{},[148,5622,5623,5626],{},[112,5624,5625],{},"機制："," 模型將其信心編碼為高維流形上的坐標。當模型處於高熵狀態（即不確定）時，其內部向量位於流形的「高不確定性」區域。",[148,5628,5629,5632],{},[112,5630,5631],{},"失效模式："," 然而，語言生成的最後一步是強制性的採樣（Sampling）。即使內部狀態準確地反映了「我不知道」（高熵），模型仍被迫選擇一個 Token 輸出。如果中間層的路由未能成功消除錯誤假設（即「不完全消除」，Incomplete Elimination），殘差流中的向量可能停留在流形的模糊地帶，導致模型自信地輸出錯誤答案。",[148,5634,5635,5638],{},[112,5636,5637],{},"結論："," 幻覺往往不是模型「發瘋」，而是模型在幾何上正確地表示了不確定性，但被解碼過程強制坍縮的結果。",[132,5640,5642],{"id":5641},"_72-啟示二提示工程即證據供給",[112,5643,5644],{},"7.2 啟示二：提示工程即「證據供給」",[116,5646,5647,5648,2180],{},"基於貝葉斯視角，提示工程（Prompt Engineering）的本質不再是「下指令」，而是",[112,5649,5650],{},"注入證據以切割假設空間",[1786,5652,5653,5659,5665,5675],{},[148,5654,5655,5658],{},[112,5656,5657],{},"假設空間初始化："," 當輸入一個提示時，Layer 0 立即建立了一個巨大的假設空間（例如：「這是一個代碼任務？還是詩歌創作？是翻譯？」）。",[148,5660,5661,5664],{},[112,5662,5663],{},"證據消除："," 提示中的每一個詞、每一個範例都在充當「證據」，用於消除與之不符的子空間。例如，提供代碼範例可以迅速消除「詩歌」和「翻譯」的假設子空間，引導注意力機制聚焦於「編程」軌道。",[148,5666,5667,5670,5671,5674],{},[112,5668,5669],{},"邊界範例的重要性："," 正如機器人交換（Robot Exchange）分析所指出的，提供位於",[112,5672,5673],{},"決策邊界","（Boundaries）的範例比提供典型範例更有效。邊界範例能更銳利地切割流形，幫助注意力機制區分相似的潛在假設。",[148,5676,5677,5680],{},[112,5678,5679],{},"領域干擾："," 如果提示中混合了多個領域（例如要求「用寫詩的風格寫代碼」），模型被迫在殘差流中同時維護多個互斥的假設軌道，這可能導致路由干擾（Interference），降低推理精度。清晰、單一領域的提示能讓幾何路由更乾淨地收斂。",[132,5682,5684],{"id":5683},"_73-啟示三深度即時間depth-as-time",[112,5685,5686],{},"7.3 啟示三：深度即時間（Depth as Time）",[116,5688,5689,5690,5693],{},"研究發現移除任何一層都會導致誤差激增，這強調了",[112,5691,5692],{},"深度","在推理中的不可替代性。",[1786,5695,5696,5702],{},[148,5697,5698,5701],{},[112,5699,5700],{},"序列化推理："," 貝葉斯推理本質上是序列化的：你必須先根據證據 A 更新信念，才能處理證據 B。在 Transformer 中，層數（Depth）代表了計算步驟或「時間」。",[148,5703,5704,5707,5708,5711],{},[112,5705,5706],{},"思維鏈（Chain of Thought）："," 這解釋了為什麼「思維鏈」技術如此有效。通過讓模型生成中間步驟，我們實際上是在利用",[112,5709,5710],{},"序列維度（Token位置）來擴展模型的虛擬深度","。這給予模型更多的幾何變換次數來執行必要的貝葉斯消除步驟，從而解決那些單次前向傳播（其物理深度有限）無法解決的複雜推理問題。",[108,5713,5715],{"id":5714},"_8-局限性與未來展望",[112,5716,5717],{},"8. 局限性與未來展望",[116,5719,5720],{},"儘管這項研究具有里程碑意義，但作者也誠實地指出了其局限性，這些局限性為未來的研究指明了方向。",[1786,5722,5723,5780,5786],{},[148,5724,5725,5728,5729,5779],{},[112,5726,5727],{},"規模限制："," 實驗使用的是小型模型（約 300 萬參數）和小詞彙表（",[161,5730,5732,5749],{"className":5731},[164],[161,5733,5735],{"className":5734},[168],[170,5736,5737],{"xmlns":172},[174,5738,5739,5747],{},[177,5740,5741,5743,5745],{},[360,5742,1855],{},[364,5744,453],{},[183,5746,2525],{},[190,5748,2528],{"encoding":192},[161,5750,5752,5770],{"className":5751,"ariaHidden":198},[197],[161,5753,5755,5758,5761,5764,5767],{"className":5754},[202],[161,5756],{"className":5757,"style":2538},[206],[161,5759,1855],{"className":5760,"style":1934},[211,395],[161,5762],{"className":5763,"style":401},[400],[161,5765,453],{"className":5766},[405],[161,5768],{"className":5769,"style":401},[400],[161,5771,5773,5776],{"className":5772},[202],[161,5774],{"className":5775,"style":2033},[206],[161,5777,2525],{"className":5778},[211],"）。雖然這足以證明原理，但真實語言模型的潛在空間維度要高得多，且假設空間的結構更為複雜（非結構化）。",[148,5781,5782,5785],{},[112,5783,5784],{},"任務簡化："," 風洞任務（雙射與 HMM）具有清晰的結構和邊界。真實世界的推理往往涉及模糊的定義和隱含的規則。",[148,5787,5788,5791],{},[112,5789,5790],{},"未來方向：",[145,5792,5793,5799,5805],{},[148,5794,5795,5798],{},[112,5796,5797],{},"擴展風洞："," 開發更複雜的潛在變量模型（如卡爾曼濾波、因果圖模型）的風洞，以測試模型處理連續變量和因果推理的能力。",[148,5800,5801,5804],{},[112,5802,5803],{},"大模型診斷："," 將本研究開發的幾何診斷工具（正交性檢測、流形 PCA）應用於預訓練的 LLM（如 LLaMA 或 GPT），檢查在自然語言訓練下是否湧現出類似的貝葉斯幾何結構。",[148,5806,5807,5810],{},[112,5808,5809],{},"架構探索："," 測試其他架構（如 Mamba、RWKV 等狀態空間模型）是否也能形成類似的貝葉斯流形，這將有助於理解不同架構的歸納偏置（Inductive Bias)。",[108,5812,5814],{"id":5813},"_9-結論",[112,5815,5816],{},"9. 結論",[116,5818,5819,5820],{},"《Transformer 注意力機制的貝葉斯幾何學》一文為我們理解現代 AI 提供了一個強大的理論與實證框架。它超越了以往模糊的「神經網絡是黑盒子」的認知，確立了一個具體的科學事實：",[112,5821,5822],{},"Transformer 是貝葉斯推理的幾何實現引擎。",[116,5824,5825,5826,5829,5830,5833,5834,5837],{},"Agarwal、Dalal 和 Misra 的研究證明，Transformer 之所以能從海量數據中湧現出推理能力，是因為其架構中的",[112,5827,5828],{},"殘差流","（作為信念載體）、",[112,5831,5832],{},"注意力機制","（作為內容定址的路由器）和",[112,5835,5836],{},"前饋網絡","（作為數值更新器）恰好對應了貝葉斯推理的三個核心步驟。它們共同作用，在模型的高維表示空間中雕刻出一個精確的幾何流形，用於整合證據並量化不確定性。",[116,5839,5840],{},"與之相對，MLP 的失敗提醒我們，架構設計至關重要。沒有正確的幾何歸納偏置（如注意力路由），單靠堆砌參數和數據無法產生真正的動態推理能力。",[116,5842,5843],{},"對於 AI 研究者與工程師而言，這份報告不僅證實了 Transformer 的優越性，更提供了一種全新的思維模型：我們不只是在訓練模型預測下一個詞，我們是在構建一個能夠容納並操作機率分佈的幾何機器。理解這一幾何學，將是我們從「模仿智能」邁向「真實推理」的關鍵一步。",[5845,5846,5848],"h4",{"id":5847},"works-cited",[112,5849,5850],{},"Works cited",[145,5852,5853,5861],{},[148,5854,5855,5856],{},"How to Prompt LLM Coding Agents as the Bayesian Inference Engines they are, accessed January 15, 2026, ",[2268,5857,5858],{"href":5858,"rel":5859},"https:\u002F\u002Fwww.robotexchange.io\u002Ft\u002Fhow-to-prompt-llm-coding-agents-as-the-bayesian-inference-engines-they-are\u002F4177",[5860],"nofollow",[148,5862,5863,5864],{},"The Bayesian Geometry of Transformer Attention - arXiv, accessed January 15, 2026, ",[2268,5865,5866],{"href":5866,"rel":5867},"https:\u002F\u002Farxiv.org\u002Fhtml\u002F2512.22471v2",[5860],{"title":5869,"searchDepth":5870,"depth":5870,"links":5871},"",2,[5872,5873,5878,5883,5888,5893,5897,5902,5903],{"id":110,"depth":5870,"text":114},{"id":127,"depth":5870,"text":130,"children":5874},[5875,5877],{"id":134,"depth":5876,"text":137},3,{"id":333,"depth":5876,"text":336},{"id":1761,"depth":5870,"text":1764,"children":5879},[5880,5881,5882],{"id":1778,"depth":5876,"text":1781},{"id":2788,"depth":5876,"text":2791},{"id":3752,"depth":5876,"text":3755},{"id":4025,"depth":5870,"text":4028,"children":5884},[5885,5886,5887],{"id":4034,"depth":5876,"text":4037},{"id":4269,"depth":5876,"text":4272},{"id":4823,"depth":5876,"text":4826},{"id":5182,"depth":5870,"text":5185,"children":5889},[5890,5891,5892],{"id":5199,"depth":5876,"text":5202},{"id":5228,"depth":5876,"text":5231},{"id":5310,"depth":5876,"text":5313},{"id":5339,"depth":5870,"text":5342,"children":5894},[5895,5896],{"id":5348,"depth":5876,"text":5351},{"id":5516,"depth":5876,"text":5519},{"id":5603,"depth":5870,"text":5606,"children":5898},[5899,5900,5901],{"id":5612,"depth":5876,"text":5615},{"id":5641,"depth":5876,"text":5644},{"id":5683,"depth":5876,"text":5686},{"id":5714,"depth":5870,"text":5717},{"id":5813,"depth":5870,"text":5816},"\u002Fimages\u002Fblog\u002F2512.22471v2.webp","2026-01-15","Naman Agarwal等人2026年開創性研究證實Transformer透過注意力機制精確實現貝葉斯推理，揭示其上下文學習的幾何本質並為提示工程奠定理論基礎。",false,"md","zh-TW",{},true,"\u002Fblog\u002Fzh-tw\u002Farxiv-2512-22471v2",{"title":102,"description":5906},"arxiv-2512-22471v2","blog\u002Fzh-TW\u002Farxiv-2512-22471v2",[5917,5918,5919],"paper","transformer","bayesian",null,"G_DGUjWKK_hCDisM4vLGWf13eighqDB2SOshjvUv17w",[5923,10510],{"id":5924,"title":5925,"author":103,"body":5926,"cover":10496,"coverAlt":10497,"date":10498,"description":10499,"draft":5907,"extension":5908,"featured":5907,"lang":5909,"meta":10500,"navigation":5911,"path":10501,"seo":10502,"slug":10503,"stem":10504,"tags":10505,"translationKey":10503,"updated":10508,"__hash__":10509},"blog\u002Fblog\u002Fzh-TW\u002Fattention-as-kernel.md","Attention 其實是一個核平滑器",{"type":105,"value":5927,"toc":10488},[5928,5931,5934,5937,5940,6283,6621,6653,7574,7793,8009,8552,8555,8558,8561,8567,8751,9391,9510,9884,9887,9890,9896,10322,10325,10328,10436,10468,10471,10474,10481,10484],[108,5929,5930],{"id":5930},"一句話讓我想了很久",[116,5932,5933],{},"「Attention 就是一個核平滑器。」",[116,5935,5936],{},"第一次看到這句話的時候我以為是某種比喻，畢竟「核方法」聽起來像是 SVM 那個年代的事，跟 Transformer 沒什麼關係。後來自己把公式展開一次，才發現它不是在打比方。",[108,5938,5939],{"id":5939},"把公式拆開",[116,5941,5942,5943,6044,6045,6142,6143,6282],{},"給定查詢矩陣 ",[161,5944,5946,5977],{"className":5945},[164],[161,5947,5949],{"className":5948},[168],[170,5950,5951],{"xmlns":172},[174,5952,5953,5974],{},[177,5954,5955,5958,5960],{},[360,5956,5957],{},"Q",[364,5959,1412],{},[180,5961,5962,5965],{},[360,5963,5964],{"mathvariant":762},"R",[177,5966,5967,5970,5972],{},[360,5968,5969],{},"n",[364,5971,4180],{},[360,5973,1140],{},[190,5975,5976],{"encoding":192},"Q \\in \\mathbb{R}^{n \\times d}",[161,5978,5980,5999],{"className":5979,"ariaHidden":198},[197],[161,5981,5983,5987,5990,5993,5996],{"className":5982},[202],[161,5984],{"className":5985,"style":5986},[206],"height:0.8778em;vertical-align:-0.1944em;",[161,5988,5957],{"className":5989},[211,395],[161,5991],{"className":5992,"style":401},[400],[161,5994,1412],{"className":5995},[405],[161,5997],{"className":5998,"style":401},[400],[161,6000,6002,6006],{"className":6001},[202],[161,6003],{"className":6004,"style":6005},[206],"height:0.8491em;",[161,6007,6009,6012],{"className":6008},[211],[161,6010,5964],{"className":6011},[211,887],[161,6013,6015],{"className":6014},[223],[161,6016,6018],{"className":6017},[227],[161,6019,6021],{"className":6020},[231],[161,6022,6024],{"className":6023,"style":6005},[235],[161,6025,6026,6029],{"style":238},[161,6027],{"className":6028,"style":243},[242],[161,6030,6032],{"className":6031},[247,248,249,250],[161,6033,6035,6038,6041],{"className":6034},[211,250],[161,6036,5969],{"className":6037},[211,395,250],[161,6039,4180],{"className":6040},[1186,250],[161,6042,1140],{"className":6043},[211,395,250],"、鍵矩陣 ",[161,6046,6048,6076],{"className":6047},[164],[161,6049,6051],{"className":6050},[168],[170,6052,6053],{"xmlns":172},[174,6054,6055,6073],{},[177,6056,6057,6059,6061],{},[360,6058,4298],{},[364,6060,1412],{},[180,6062,6063,6065],{},[360,6064,5964],{"mathvariant":762},[177,6066,6067,6069,6071],{},[360,6068,2608],{},[364,6070,4180],{},[360,6072,1140],{},[190,6074,6075],{"encoding":192},"K \\in \\mathbb{R}^{m \\times d}",[161,6077,6079,6098],{"className":6078,"ariaHidden":198},[197],[161,6080,6082,6086,6089,6092,6095],{"className":6081},[202],[161,6083],{"className":6084,"style":6085},[206],"height:0.7224em;vertical-align:-0.0391em;",[161,6087,4298],{"className":6088,"style":4318},[211,395],[161,6090],{"className":6091,"style":401},[400],[161,6093,1412],{"className":6094},[405],[161,6096],{"className":6097,"style":401},[400],[161,6099,6101,6104],{"className":6100},[202],[161,6102],{"className":6103,"style":6005},[206],[161,6105,6107,6110],{"className":6106},[211],[161,6108,5964],{"className":6109},[211,887],[161,6111,6113],{"className":6112},[223],[161,6114,6116],{"className":6115},[227],[161,6117,6119],{"className":6118},[231],[161,6120,6122],{"className":6121,"style":6005},[235],[161,6123,6124,6127],{"style":238},[161,6125],{"className":6126,"style":243},[242],[161,6128,6130],{"className":6129},[247,248,249,250],[161,6131,6133,6136,6139],{"className":6132},[211,250],[161,6134,2608],{"className":6135},[211,395,250],[161,6137,4180],{"className":6138},[1186,250],[161,6140,1140],{"className":6141},[211,395,250]," 與值矩陣 ",[161,6144,6146,6179],{"className":6145},[164],[161,6147,6149],{"className":6148},[168],[170,6150,6151],{"xmlns":172},[174,6152,6153,6176],{},[177,6154,6155,6157,6159],{},[360,6156,1855],{},[364,6158,1412],{},[180,6160,6161,6163],{},[360,6162,5964],{"mathvariant":762},[177,6164,6165,6167,6169],{},[360,6166,2608],{},[364,6168,4180],{},[462,6170,6171,6173],{},[360,6172,1140],{},[360,6174,6175],{},"v",[190,6177,6178],{"encoding":192},"V \\in \\mathbb{R}^{m \\times d_v}",[161,6180,6182,6200],{"className":6181,"ariaHidden":198},[197],[161,6183,6185,6188,6191,6194,6197],{"className":6184},[202],[161,6186],{"className":6187,"style":6085},[206],[161,6189,1855],{"className":6190,"style":1934},[211,395],[161,6192],{"className":6193,"style":401},[400],[161,6195,1412],{"className":6196},[405],[161,6198],{"className":6199,"style":401},[400],[161,6201,6203,6206],{"className":6202},[202],[161,6204],{"className":6205,"style":6005},[206],[161,6207,6209,6212],{"className":6208},[211],[161,6210,5964],{"className":6211},[211,887],[161,6213,6215],{"className":6214},[223],[161,6216,6218],{"className":6217},[227],[161,6219,6221],{"className":6220},[231],[161,6222,6224],{"className":6223,"style":6005},[235],[161,6225,6226,6229],{"style":238},[161,6227],{"className":6228,"style":243},[242],[161,6230,6232],{"className":6231},[247,248,249,250],[161,6233,6235,6238,6241],{"className":6234},[211,250],[161,6236,2608],{"className":6237},[211,395,250],[161,6239,4180],{"className":6240},[1186,250],[161,6242,6244,6247],{"className":6243},[211,250],[161,6245,1140],{"className":6246},[211,395,250],[161,6248,6250],{"className":6249},[223],[161,6251,6253,6274],{"className":6252},[227,546],[161,6254,6256,6271],{"className":6255},[231],[161,6257,6260],{"className":6258,"style":6259},[235],"height:0.1645em;",[161,6261,6262,6265],{"style":1557},[161,6263],{"className":6264,"style":1561},[242],[161,6266,6268],{"className":6267},[247,1565,1566,250],[161,6269,6175],{"className":6270,"style":419},[211,395,250],[161,6272,570],{"className":6273},[569],[161,6275,6277],{"className":6276},[231],[161,6278,6280],{"className":6279,"style":1579},[235],[161,6281],{},"，scaled dot-product attention 是",[161,6284,6286],{"className":6285},[731],[161,6287,6289,6359],{"className":6288},[164],[161,6290,6292],{"className":6291},[168],[170,6293,6294],{"xmlns":172,"display":740},[174,6295,6296,6356],{},[177,6297,6298,6301,6303,6305,6307,6309,6311,6313,6315,6317,6319,6322,6324,6326,6351,6353],{},[360,6299,6300],{"mathvariant":803},"Attn",[364,6302,820],{},[364,6304,373],{"stretchy":372},[360,6306,5957],{},[364,6308,472],{"separator":198},[360,6310,4298],{},[364,6312,472],{"separator":198},[360,6314,1855],{},[364,6316,378],{"stretchy":372},[364,6318,453],{},[360,6320,6321],{"mathvariant":803},"softmax",[364,6323,820],{},[2576,6325,2578],{},[177,6327,6328,6330,6349],{},[364,6329,373],{"fence":198},[6331,6332,6333,6344],"mfrac",{},[177,6334,6335,6337],{},[360,6336,5957],{},[180,6338,6339,6341],{},[360,6340,4298],{},[360,6342,6343],{"mathvariant":803},"⊤",[6345,6346,6347],"msqrt",{},[360,6348,1140],{},[364,6350,378],{"fence":198},[360,6352,1855],{},[360,6354,6355],{"mathvariant":803},".",[190,6357,6358],{"encoding":192},"\\operatorname{Attn}(Q, K, V) = \\operatorname{softmax}\\!\\left(\\frac{QK^{\\top}}{\\sqrt{d}}\\right) V .",[161,6360,6362,6411],{"className":6361,"ariaHidden":198},[197],[161,6363,6365,6368,6375,6378,6381,6384,6387,6390,6393,6396,6399,6402,6405,6408],{"className":6364},[202],[161,6366],{"className":6367,"style":415},[206],[161,6369,6371],{"className":6370},[1029],[161,6372,6300],{"className":6373},[211,6374],"mathrm",[161,6376,373],{"className":6377},[423],[161,6379,5957],{"className":6380},[211,395],[161,6382,472],{"className":6383},[583],[161,6385],{"className":6386,"style":587},[400],[161,6388,4298],{"className":6389,"style":4318},[211,395],[161,6391,472],{"className":6392},[583],[161,6394],{"className":6395,"style":587},[400],[161,6397,1855],{"className":6398,"style":1934},[211,395],[161,6400,378],{"className":6401},[430],[161,6403],{"className":6404,"style":401},[400],[161,6406,453],{"className":6407},[405],[161,6409],{"className":6410,"style":401},[400],[161,6412,6414,6418,6424,6427,6430,6612,6615,6618],{"className":6413},[202],[161,6415],{"className":6416,"style":6417},[206],"height:2.4761em;vertical-align:-0.95em;",[161,6419,6421],{"className":6420},[1029],[161,6422,6321],{"className":6423},[211,6374],[161,6425],{"className":6426,"style":2637},[400],[161,6428],{"className":6429,"style":587},[400],[161,6431,6433,6442,6606],{"className":6432},[1921],[161,6434,6438],{"className":6435,"style":6437},[423,6436],"delimcenter","top:0em;",[161,6439,373],{"className":6440},[6441,249],"delimsizing",[161,6443,6445,6449,6603],{"className":6444},[211],[161,6446],{"className":6447},[423,6448],"nulldelimiter",[161,6450,6452],{"className":6451},[6331],[161,6453,6455,6594],{"className":6454},[227,546],[161,6456,6458,6591],{"className":6457},[231],[161,6459,6462,6536,6547],{"className":6460,"style":6461},[235],"height:1.5261em;",[161,6463,6465,6469],{"style":6464},"top:-2.1778em;",[161,6466],{"className":6467,"style":6468},[242],"height:3em;",[161,6470,6472],{"className":6471},[211],[161,6473,6476],{"className":6474},[211,6475],"sqrt",[161,6477,6479,6527],{"className":6478},[227,546],[161,6480,6482,6524],{"className":6481},[231],[161,6483,6486,6501],{"className":6484,"style":6485},[235],"height:0.9322em;",[161,6487,6491,6494],{"className":6488,"style":6490},[6489],"svg-align","top:-3em;",[161,6492],{"className":6493,"style":6468},[242],[161,6495,6498],{"className":6496,"style":6497},[211],"padding-left:0.833em;",[161,6499,1140],{"className":6500},[211,395],[161,6502,6504,6507],{"style":6503},"top:-2.8922em;",[161,6505],{"className":6506,"style":6468},[242],[161,6508,6512],{"className":6509,"style":6511},[6510],"hide-tail","min-width:0.853em;height:1.08em;",[6513,6514,6520],"svg",{"xmlns":6515,"width":6516,"height":6517,"viewBox":6518,"preserveAspectRatio":6519},"http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","400em","1.08em","0 0 400000 1080","xMinYMin slice",[6521,6522],"path",{"d":6523},"M95,702\nc-2.7,0,-7.17,-2.7,-13.5,-8c-5.8,-5.3,-9.5,-10,-9.5,-14\nc0,-2,0.3,-3.3,1,-4c1.3,-2.7,23.83,-20.7,67.5,-54\nc44.2,-33.3,65.8,-50.3,66.5,-51c1.3,-1.3,3,-2,5,-2c4.7,0,8.7,3.3,12,10\ns173,378,173,378c0.7,0,35.3,-71,104,-213c68.7,-142,137.5,-285,206.5,-429\nc69,-144,104.5,-217.7,106.5,-221\nl0 -0\nc5.3,-9.3,12,-14,20,-14\nH400000v40H845.2724\ns-225.272,467,-225.272,467s-235,486,-235,486c-2.7,4.7,-9,7,-19,7\nc-6,0,-10,-1,-12,-3s-194,-422,-194,-422s-65,47,-65,47z\nM834 80h400000v40h-400000z",[161,6525,570],{"className":6526},[569],[161,6528,6530],{"className":6529},[231],[161,6531,6534],{"className":6532,"style":6533},[235],"height:0.1078em;",[161,6535],{},[161,6537,6539,6542],{"style":6538},"top:-3.23em;",[161,6540],{"className":6541,"style":6468},[242],[161,6543],{"className":6544,"style":6546},[6545],"frac-line","border-bottom-width:0.04em;",[161,6548,6550,6553],{"style":6549},"top:-3.677em;",[161,6551],{"className":6552,"style":6468},[242],[161,6554,6556,6559],{"className":6555},[211],[161,6557,5957],{"className":6558},[211,395],[161,6560,6562,6565],{"className":6561},[211],[161,6563,4298],{"className":6564,"style":4318},[211,395],[161,6566,6568],{"className":6567},[223],[161,6569,6571],{"className":6570},[227],[161,6572,6574],{"className":6573},[231],[161,6575,6577],{"className":6576,"style":6005},[235],[161,6578,6579,6582],{"style":238},[161,6580],{"className":6581,"style":243},[242],[161,6583,6585],{"className":6584},[247,248,249,250],[161,6586,6588],{"className":6587},[211,250],[161,6589,6343],{"className":6590},[211,250],[161,6592,570],{"className":6593},[569],[161,6595,6597],{"className":6596},[231],[161,6598,6601],{"className":6599,"style":6600},[235],"height:0.93em;",[161,6602],{},[161,6604],{"className":6605},[430,6448],[161,6607,6609],{"className":6608,"style":6437},[430,6436],[161,6610,378],{"className":6611},[6441,249],[161,6613],{"className":6614,"style":587},[400],[161,6616,1855],{"className":6617,"style":1934},[211,395],[161,6619,6355],{"className":6620},[211],[116,6622,6623,6624,6652],{},"看整體沒感覺，把它逐項展開。第 ",[161,6625,6627,6640],{"className":6626},[164],[161,6628,6630],{"className":6629},[168],[170,6631,6632],{"xmlns":172},[174,6633,6634,6638],{},[177,6635,6636],{},[360,6637,469],{},[190,6639,469],{"encoding":192},[161,6641,6643],{"className":6642,"ariaHidden":198},[197],[161,6644,6646,6649],{"className":6645},[202],[161,6647],{"className":6648,"style":5410},[206],[161,6650,469],{"className":6651},[211,395]," 個輸出向量是",[161,6654,6656],{"className":6655},[731],[161,6657,6659,6792],{"className":6658},[164],[161,6660,6662],{"className":6661},[168],[170,6663,6664],{"xmlns":172,"display":740},[174,6665,6666,6789],{},[177,6667,6668,6674,6676,6787],{},[462,6669,6670,6672],{},[360,6671,2593],{},[360,6673,469],{},[364,6675,453],{},[6331,6677,6678,6737],{},[177,6679,6680,6696,6699,6701,6703,6731],{},[6681,6682,6683,6685,6694],"munderover",{},[364,6684,3132],{},[177,6686,6687,6690,6692],{},[360,6688,6689],{},"j",[364,6691,453],{},[183,6693,212],{},[360,6695,2608],{},[360,6697,6698],{},"exp",[364,6700,820],{},[2576,6702,2578],{},[177,6704,6705,6707,6729],{},[364,6706,373],{"fence":198},[6331,6708,6709,6725],{},[177,6710,6711,6719],{},[455,6712,6713,6715,6717],{},[360,6714,753],{},[360,6716,469],{},[360,6718,6343],{"mathvariant":803},[462,6720,6721,6723],{},[360,6722,492],{},[360,6724,6689],{},[6345,6726,6727],{},[360,6728,1140],{},[364,6730,378],{"fence":198},[462,6732,6733,6735],{},[360,6734,6175],{},[360,6736,6689],{},[177,6738,6739,6753,6755,6757,6759],{},[6681,6740,6741,6743,6751],{},[364,6742,3132],{},[177,6744,6745,6747,6749],{},[360,6746,6689],{},[364,6748,453],{},[183,6750,212],{},[360,6752,2608],{},[360,6754,6698],{},[364,6756,820],{},[2576,6758,2578],{},[177,6760,6761,6763,6785],{},[364,6762,373],{"fence":198},[6331,6764,6765,6781],{},[177,6766,6767,6775],{},[455,6768,6769,6771,6773],{},[360,6770,753],{},[360,6772,469],{},[360,6774,6343],{"mathvariant":803},[462,6776,6777,6779],{},[360,6778,492],{},[360,6780,6689],{},[6345,6782,6783],{},[360,6784,1140],{},[364,6786,378],{"fence":198},[360,6788,6355],{"mathvariant":803},[190,6790,6791],{"encoding":192},"o_i = \\frac{\\sum_{j=1}^{m} \\exp\\!\\left(\\frac{q_i^{\\top} k_j}{\\sqrt{d}}\\right) v_j}{\\sum_{j=1}^{m} \\exp\\!\\left(\\frac{q_i^{\\top} k_j}{\\sqrt{d}}\\right)} .",[161,6793,6795,6850],{"className":6794,"ariaHidden":198},[197],[161,6796,6798,6801,6841,6844,6847],{"className":6797},[202],[161,6799],{"className":6800,"style":2068},[206],[161,6802,6804,6807],{"className":6803},[211],[161,6805,2593],{"className":6806},[211,395],[161,6808,6810],{"className":6809},[223],[161,6811,6813,6833],{"className":6812},[227,546],[161,6814,6816,6830],{"className":6815},[231],[161,6817,6819],{"className":6818,"style":553},[235],[161,6820,6821,6824],{"style":556},[161,6822],{"className":6823,"style":243},[242],[161,6825,6827],{"className":6826},[247,248,249,250],[161,6828,469],{"className":6829},[211,395,250],[161,6831,570],{"className":6832},[569],[161,6834,6836],{"className":6835},[231],[161,6837,6839],{"className":6838,"style":577},[235],[161,6840],{},[161,6842],{"className":6843,"style":401},[400],[161,6845,453],{"className":6846},[405],[161,6848],{"className":6849,"style":401},[400],[161,6851,6853,6857,7571],{"className":6852},[202],[161,6854],{"className":6855,"style":6856},[206],"height:3.8892em;vertical-align:-1.69em;",[161,6858,6860,6863,7568],{"className":6859},[211],[161,6861],{"className":6862},[423,6448],[161,6864,6866],{"className":6865},[6331],[161,6867,6869,7559],{"className":6868},[227,546],[161,6870,6872,7556],{"className":6871},[231],[161,6873,6876,7196,7205],{"className":6874,"style":6875},[235],"height:2.1991em;",[161,6877,6879,6883],{"style":6878},"top:-2.1191em;",[161,6880],{"className":6881,"style":6882},[242],"height:3.1591em;",[161,6884,6886,6952,6955,6958,6961,6964],{"className":6885},[211],[161,6887,6889,6892],{"className":6888},[1029],[161,6890,3132],{"className":6891,"style":1507},[1029,1230,1506],[161,6893,6895],{"className":6894},[223],[161,6896,6898,6943],{"className":6897},[227,546],[161,6899,6901,6940],{"className":6900},[231],[161,6902,6905,6926],{"className":6903,"style":6904},[235],"height:0.8043em;",[161,6906,6907,6910],{"style":1523},[161,6908],{"className":6909,"style":243},[242],[161,6911,6913],{"className":6912},[247,248,249,250],[161,6914,6916,6920,6923],{"className":6915},[211,250],[161,6917,6689],{"className":6918,"style":6919},[211,395,250],"margin-right:0.0572em;",[161,6921,453],{"className":6922},[405,250],[161,6924,212],{"className":6925},[211,250],[161,6927,6928,6931],{"style":668},[161,6929],{"className":6930,"style":243},[242],[161,6932,6934],{"className":6933},[247,248,249,250],[161,6935,6937],{"className":6936},[211,250],[161,6938,2608],{"className":6939},[211,395,250],[161,6941,570],{"className":6942},[569],[161,6944,6946],{"className":6945},[231],[161,6947,6950],{"className":6948,"style":6949},[235],"height:0.4358em;",[161,6951],{},[161,6953],{"className":6954,"style":587},[400],[161,6956,6698],{"className":6957},[1029],[161,6959],{"className":6960,"style":2637},[400],[161,6962],{"className":6963,"style":587},[400],[161,6965,6967,6974,7190],{"className":6966},[1921],[161,6968,6970],{"className":6969,"style":6437},[423,6436],[161,6971,373],{"className":6972},[6441,6973],"size2",[161,6975,6977,6980,7187],{"className":6976},[211],[161,6978],{"className":6979},[423,6448],[161,6981,6983],{"className":6982},[6331],[161,6984,6986,7178],{"className":6985},[227,546],[161,6987,6989,7175],{"className":6988},[231],[161,6990,6993,7055,7063],{"className":6991,"style":6992},[235],"height:1.1242em;",[161,6994,6996,6999],{"style":6995},"top:-2.5335em;",[161,6997],{"className":6998,"style":6468},[242],[161,7000,7002],{"className":7001},[247,248,249,250],[161,7003,7005],{"className":7004},[211,250],[161,7006,7008],{"className":7007},[211,6475,250],[161,7009,7011,7046],{"className":7010},[227,546],[161,7012,7014,7043],{"className":7013},[231],[161,7015,7018,7030],{"className":7016,"style":7017},[235],"height:0.9378em;",[161,7019,7021,7024],{"className":7020,"style":6490},[6489],[161,7022],{"className":7023,"style":6468},[242],[161,7025,7027],{"className":7026,"style":6497},[211,250],[161,7028,1140],{"className":7029},[211,395,250],[161,7031,7033,7036],{"style":7032},"top:-2.8978em;",[161,7034],{"className":7035,"style":6468},[242],[161,7037,7039],{"className":7038,"style":6511},[6510,250],[6513,7040,7041],{"xmlns":6515,"width":6516,"height":6517,"viewBox":6518,"preserveAspectRatio":6519},[6521,7042],{"d":6523},[161,7044,570],{"className":7045},[569],[161,7047,7049],{"className":7048},[231],[161,7050,7053],{"className":7051,"style":7052},[235],"height:0.1022em;",[161,7054],{},[161,7056,7057,7060],{"style":6538},[161,7058],{"className":7059,"style":6468},[242],[161,7061],{"className":7062,"style":6546},[6545],[161,7064,7066,7069],{"style":7065},"top:-3.5356em;",[161,7067],{"className":7068,"style":6468},[242],[161,7070,7072],{"className":7071},[247,248,249,250],[161,7073,7075,7133],{"className":7074},[211,250],[161,7076,7078,7081],{"className":7077},[211,250],[161,7079,753],{"className":7080,"style":419},[211,395,250],[161,7082,7084],{"className":7083},[223],[161,7085,7087,7124],{"className":7086},[227,546],[161,7088,7090,7121],{"className":7089},[231],[161,7091,7094,7106],{"className":7092,"style":7093},[235],"height:0.8408em;",[161,7095,7097,7100],{"style":7096},"top:-2.1777em;margin-left:-0.0359em;margin-right:0.0714em;",[161,7098],{"className":7099,"style":1561},[242],[161,7101,7103],{"className":7102},[247,1565,1566,250],[161,7104,469],{"className":7105},[211,395,250],[161,7107,7109,7112],{"style":7108},"top:-2.8448em;margin-right:0.0714em;",[161,7110],{"className":7111,"style":1561},[242],[161,7113,7115],{"className":7114},[247,1565,1566,250],[161,7116,7118],{"className":7117},[211,250],[161,7119,6343],{"className":7120},[211,250],[161,7122,570],{"className":7123},[569],[161,7125,7127],{"className":7126},[231],[161,7128,7131],{"className":7129,"style":7130},[235],"height:0.3223em;",[161,7132],{},[161,7134,7136,7139],{"className":7135},[211,250],[161,7137,492],{"className":7138,"style":678},[211,395,250],[161,7140,7142],{"className":7141},[223],[161,7143,7145,7166],{"className":7144},[227,546],[161,7146,7148,7163],{"className":7147},[231],[161,7149,7151],{"className":7150,"style":1554},[235],[161,7152,7154,7157],{"style":7153},"top:-2.357em;margin-left:-0.0315em;margin-right:0.0714em;",[161,7155],{"className":7156,"style":1561},[242],[161,7158,7160],{"className":7159},[247,1565,1566,250],[161,7161,6689],{"className":7162,"style":6919},[211,395,250],[161,7164,570],{"className":7165},[569],[161,7167,7169],{"className":7168},[231],[161,7170,7173],{"className":7171,"style":7172},[235],"height:0.2819em;",[161,7174],{},[161,7176,570],{"className":7177},[569],[161,7179,7181],{"className":7180},[231],[161,7182,7185],{"className":7183,"style":7184},[235],"height:0.538em;",[161,7186],{},[161,7188],{"className":7189},[430,6448],[161,7191,7193],{"className":7192,"style":6437},[430,6436],[161,7194,378],{"className":7195},[6441,6973],[161,7197,7199,7202],{"style":7198},"top:-3.3891em;",[161,7200],{"className":7201,"style":6882},[242],[161,7203],{"className":7204,"style":6546},[6545],[161,7206,7208,7211],{"style":7207},"top:-4.1991em;",[161,7209],{"className":7210,"style":6882},[242],[161,7212,7214,7277,7280,7283,7286,7289,7513,7516],{"className":7213},[211],[161,7215,7217,7220],{"className":7216},[1029],[161,7218,3132],{"className":7219,"style":1507},[1029,1230,1506],[161,7221,7223],{"className":7222},[223],[161,7224,7226,7269],{"className":7225},[227,546],[161,7227,7229,7266],{"className":7228},[231],[161,7230,7232,7252],{"className":7231,"style":6904},[235],[161,7233,7234,7237],{"style":1523},[161,7235],{"className":7236,"style":243},[242],[161,7238,7240],{"className":7239},[247,248,249,250],[161,7241,7243,7246,7249],{"className":7242},[211,250],[161,7244,6689],{"className":7245,"style":6919},[211,395,250],[161,7247,453],{"className":7248},[405,250],[161,7250,212],{"className":7251},[211,250],[161,7253,7254,7257],{"style":668},[161,7255],{"className":7256,"style":243},[242],[161,7258,7260],{"className":7259},[247,248,249,250],[161,7261,7263],{"className":7262},[211,250],[161,7264,2608],{"className":7265},[211,395,250],[161,7267,570],{"className":7268},[569],[161,7270,7272],{"className":7271},[231],[161,7273,7275],{"className":7274,"style":6949},[235],[161,7276],{},[161,7278],{"className":7279,"style":587},[400],[161,7281,6698],{"className":7282},[1029],[161,7284],{"className":7285,"style":2637},[400],[161,7287],{"className":7288,"style":587},[400],[161,7290,7292,7298,7507],{"className":7291},[1921],[161,7293,7295],{"className":7294,"style":6437},[423,6436],[161,7296,373],{"className":7297},[6441,6973],[161,7299,7301,7304,7504],{"className":7300},[211],[161,7302],{"className":7303},[423,6448],[161,7305,7307],{"className":7306},[6331],[161,7308,7310,7496],{"className":7309},[227,546],[161,7311,7313,7493],{"className":7312},[231],[161,7314,7317,7375,7383],{"className":7315,"style":7316},[235],"height:1.1591em;",[161,7318,7319,7322],{"style":6995},[161,7320],{"className":7321,"style":6468},[242],[161,7323,7325],{"className":7324},[247,248,249,250],[161,7326,7328],{"className":7327},[211,250],[161,7329,7331],{"className":7330},[211,6475,250],[161,7332,7334,7367],{"className":7333},[227,546],[161,7335,7337,7364],{"className":7336},[231],[161,7338,7340,7352],{"className":7339,"style":7017},[235],[161,7341,7343,7346],{"className":7342,"style":6490},[6489],[161,7344],{"className":7345,"style":6468},[242],[161,7347,7349],{"className":7348,"style":6497},[211,250],[161,7350,1140],{"className":7351},[211,395,250],[161,7353,7354,7357],{"style":7032},[161,7355],{"className":7356,"style":6468},[242],[161,7358,7360],{"className":7359,"style":6511},[6510,250],[6513,7361,7362],{"xmlns":6515,"width":6516,"height":6517,"viewBox":6518,"preserveAspectRatio":6519},[6521,7363],{"d":6523},[161,7365,570],{"className":7366},[569],[161,7368,7370],{"className":7369},[231],[161,7371,7373],{"className":7372,"style":7052},[235],[161,7374],{},[161,7376,7377,7380],{"style":6538},[161,7378],{"className":7379,"style":6468},[242],[161,7381],{"className":7382,"style":6546},[6545],[161,7384,7386,7389],{"style":7385},"top:-3.5102em;",[161,7387],{"className":7388,"style":6468},[242],[161,7390,7392],{"className":7391},[247,248,249,250],[161,7393,7395,7453],{"className":7394},[211,250],[161,7396,7398,7401],{"className":7397},[211,250],[161,7399,753],{"className":7400,"style":419},[211,395,250],[161,7402,7404],{"className":7403},[223],[161,7405,7407,7444],{"className":7406},[227,546],[161,7408,7410,7441],{"className":7409},[231],[161,7411,7414,7426],{"className":7412,"style":7413},[235],"height:0.927em;",[161,7415,7417,7420],{"style":7416},"top:-2.214em;margin-left:-0.0359em;margin-right:0.0714em;",[161,7418],{"className":7419,"style":1561},[242],[161,7421,7423],{"className":7422},[247,1565,1566,250],[161,7424,469],{"className":7425},[211,395,250],[161,7427,7429,7432],{"style":7428},"top:-2.931em;margin-right:0.0714em;",[161,7430],{"className":7431,"style":1561},[242],[161,7433,7435],{"className":7434},[247,1565,1566,250],[161,7436,7438],{"className":7437},[211,250],[161,7439,6343],{"className":7440},[211,250],[161,7442,570],{"className":7443},[569],[161,7445,7447],{"className":7446},[231],[161,7448,7451],{"className":7449,"style":7450},[235],"height:0.286em;",[161,7452],{},[161,7454,7456,7459],{"className":7455},[211,250],[161,7457,492],{"className":7458,"style":678},[211,395,250],[161,7460,7462],{"className":7461},[223],[161,7463,7465,7485],{"className":7464},[227,546],[161,7466,7468,7482],{"className":7467},[231],[161,7469,7471],{"className":7470,"style":1554},[235],[161,7472,7473,7476],{"style":7153},[161,7474],{"className":7475,"style":1561},[242],[161,7477,7479],{"className":7478},[247,1565,1566,250],[161,7480,6689],{"className":7481,"style":6919},[211,395,250],[161,7483,570],{"className":7484},[569],[161,7486,7488],{"className":7487},[231],[161,7489,7491],{"className":7490,"style":7172},[235],[161,7492],{},[161,7494,570],{"className":7495},[569],[161,7497,7499],{"className":7498},[231],[161,7500,7502],{"className":7501,"style":7184},[235],[161,7503],{},[161,7505],{"className":7506},[430,6448],[161,7508,7510],{"className":7509,"style":6437},[430,6436],[161,7511,378],{"className":7512},[6441,6973],[161,7514],{"className":7515,"style":587},[400],[161,7517,7519,7522],{"className":7518},[211],[161,7520,6175],{"className":7521,"style":419},[211,395],[161,7523,7525],{"className":7524},[223],[161,7526,7528,7548],{"className":7527},[227,546],[161,7529,7531,7545],{"className":7530},[231],[161,7532,7534],{"className":7533,"style":553},[235],[161,7535,7536,7539],{"style":608},[161,7537],{"className":7538,"style":243},[242],[161,7540,7542],{"className":7541},[247,248,249,250],[161,7543,6689],{"className":7544,"style":6919},[211,395,250],[161,7546,570],{"className":7547},[569],[161,7549,7551],{"className":7550},[231],[161,7552,7554],{"className":7553,"style":2379},[235],[161,7555],{},[161,7557,570],{"className":7558},[569],[161,7560,7562],{"className":7561},[231],[161,7563,7566],{"className":7564,"style":7565},[235],"height:1.69em;",[161,7567],{},[161,7569],{"className":7570},[430,6448],[161,7572,6355],{"className":7573},[211],[116,7575,7576,7577,7648,7649,7719,7720,7792],{},"左邊的分子是一堆 ",[161,7578,7580,7598],{"className":7579},[164],[161,7581,7583],{"className":7582},[168],[170,7584,7585],{"xmlns":172},[174,7586,7587,7595],{},[177,7588,7589],{},[462,7590,7591,7593],{},[360,7592,6175],{},[360,7594,6689],{},[190,7596,7597],{"encoding":192},"v_j",[161,7599,7601],{"className":7600,"ariaHidden":198},[197],[161,7602,7604,7608],{"className":7603},[202],[161,7605],{"className":7606,"style":7607},[206],"height:0.7167em;vertical-align:-0.2861em;",[161,7609,7611,7614],{"className":7610},[211],[161,7612,6175],{"className":7613,"style":419},[211,395],[161,7615,7617],{"className":7616},[223],[161,7618,7620,7640],{"className":7619},[227,546],[161,7621,7623,7637],{"className":7622},[231],[161,7624,7626],{"className":7625,"style":553},[235],[161,7627,7628,7631],{"style":608},[161,7629],{"className":7630,"style":243},[242],[161,7632,7634],{"className":7633},[247,248,249,250],[161,7635,6689],{"className":7636,"style":6919},[211,395,250],[161,7638,570],{"className":7639},[569],[161,7641,7643],{"className":7642},[231],[161,7644,7646],{"className":7645,"style":2379},[235],[161,7647],{}," 的加權和，權重是 ",[161,7650,7652,7670],{"className":7651},[164],[161,7653,7655],{"className":7654},[168],[170,7656,7657],{"xmlns":172},[174,7658,7659,7667],{},[177,7660,7661],{},[462,7662,7663,7665],{},[360,7664,753],{},[360,7666,469],{},[190,7668,7669],{"encoding":192},"q_i",[161,7671,7673],{"className":7672,"ariaHidden":198},[197],[161,7674,7676,7679],{"className":7675},[202],[161,7677],{"className":7678,"style":721},[206],[161,7680,7682,7685],{"className":7681},[211],[161,7683,753],{"className":7684,"style":419},[211,395],[161,7686,7688],{"className":7687},[223],[161,7689,7691,7711],{"className":7690},[227,546],[161,7692,7694,7708],{"className":7693},[231],[161,7695,7697],{"className":7696,"style":553},[235],[161,7698,7699,7702],{"style":608},[161,7700],{"className":7701,"style":243},[242],[161,7703,7705],{"className":7704},[247,248,249,250],[161,7706,469],{"className":7707},[211,395,250],[161,7709,570],{"className":7710},[569],[161,7712,7714],{"className":7713},[231],[161,7715,7717],{"className":7716,"style":577},[235],[161,7718],{}," 與 ",[161,7721,7723,7741],{"className":7722},[164],[161,7724,7726],{"className":7725},[168],[170,7727,7728],{"xmlns":172},[174,7729,7730,7738],{},[177,7731,7732],{},[462,7733,7734,7736],{},[360,7735,492],{},[360,7737,6689],{},[190,7739,7740],{"encoding":192},"k_j",[161,7742,7744],{"className":7743,"ariaHidden":198},[197],[161,7745,7747,7751],{"className":7746},[202],[161,7748],{"className":7749,"style":7750},[206],"height:0.9805em;vertical-align:-0.2861em;",[161,7752,7754,7757],{"className":7753},[211],[161,7755,492],{"className":7756,"style":678},[211,395],[161,7758,7760],{"className":7759},[223],[161,7761,7763,7784],{"className":7762},[227,546],[161,7764,7766,7781],{"className":7765},[231],[161,7767,7769],{"className":7768,"style":553},[235],[161,7770,7772,7775],{"style":7771},"top:-2.55em;margin-left:-0.0315em;margin-right:0.05em;",[161,7773],{"className":7774,"style":243},[242],[161,7776,7778],{"className":7777},[247,248,249,250],[161,7779,6689],{"className":7780,"style":6919},[211,395,250],[161,7782,570],{"className":7783},[569],[161,7785,7787],{"className":7786},[231],[161,7788,7790],{"className":7789,"style":2379},[235],[161,7791],{}," 的相似度；左邊的分母只是把權重歸一化，讓它們加起來等於 1。",[116,7794,7795,7796,8008],{},"如果把 ",[161,7797,7799,7853],{"className":7798},[164],[161,7800,7802],{"className":7801},[168],[170,7803,7804],{"xmlns":172},[174,7805,7806,7850],{},[177,7807,7808,7811,7813,7815,7817,7819,7821,7823,7825,7827,7829],{},[360,7809,7810],{},"κ",[364,7812,373],{"stretchy":372},[360,7814,753],{},[364,7816,472],{"separator":198},[360,7818,492],{},[364,7820,378],{"stretchy":372},[364,7822,453],{},[360,7824,6698],{},[364,7826,820],{},[2576,7828,2578],{},[177,7830,7831,7833,7839,7841,7844,7848],{},[364,7832,373],{"fence":198},[180,7834,7835,7837],{},[360,7836,753],{},[360,7838,6343],{"mathvariant":803},[360,7840,492],{},[360,7842,7843],{"mathvariant":803},"\u002F",[6345,7845,7846],{},[360,7847,1140],{},[364,7849,378],{"fence":198},[190,7851,7852],{"encoding":192},"\\kappa(q, k) = \\exp\\!\\left(q^{\\top}k \u002F \\sqrt{d}\\right)",[161,7854,7856,7892],{"className":7855,"ariaHidden":198},[197],[161,7857,7859,7862,7865,7868,7871,7874,7877,7880,7883,7886,7889],{"className":7858},[202],[161,7860],{"className":7861,"style":415},[206],[161,7863,7810],{"className":7864},[211,395],[161,7866,373],{"className":7867},[423],[161,7869,753],{"className":7870,"style":419},[211,395],[161,7872,472],{"className":7873},[583],[161,7875],{"className":7876,"style":587},[400],[161,7878,492],{"className":7879,"style":678},[211,395],[161,7881,378],{"className":7882},[430],[161,7884],{"className":7885,"style":401},[400],[161,7887,453],{"className":7888},[405],[161,7890],{"className":7891,"style":401},[400],[161,7893,7895,7899,7902,7905,7908],{"className":7894},[202],[161,7896],{"className":7897,"style":7898},[206],"height:1.8em;vertical-align:-0.65em;",[161,7900,6698],{"className":7901},[1029],[161,7903],{"className":7904,"style":2637},[400],[161,7906],{"className":7907,"style":587},[400],[161,7909,7911,7917,7949,7952,7955,8002],{"className":7910},[1921],[161,7912,7914],{"className":7913,"style":6437},[423,6436],[161,7915,373],{"className":7916},[6441,6973],[161,7918,7920,7923],{"className":7919},[211],[161,7921,753],{"className":7922,"style":419},[211,395],[161,7924,7926],{"className":7925},[223],[161,7927,7929],{"className":7928},[227],[161,7930,7932],{"className":7931},[231],[161,7933,7935],{"className":7934,"style":6005},[235],[161,7936,7937,7940],{"style":238},[161,7938],{"className":7939,"style":243},[242],[161,7941,7943],{"className":7942},[247,248,249,250],[161,7944,7946],{"className":7945},[211,250],[161,7947,6343],{"className":7948},[211,250],[161,7950,492],{"className":7951,"style":678},[211,395],[161,7953,7843],{"className":7954},[211],[161,7956,7958],{"className":7957},[211,6475],[161,7959,7961,7994],{"className":7960},[227,546],[161,7962,7964,7991],{"className":7963},[231],[161,7965,7967,7979],{"className":7966,"style":6485},[235],[161,7968,7970,7973],{"className":7969,"style":6490},[6489],[161,7971],{"className":7972,"style":6468},[242],[161,7974,7976],{"className":7975,"style":6497},[211],[161,7977,1140],{"className":7978},[211,395],[161,7980,7981,7984],{"style":6503},[161,7982],{"className":7983,"style":6468},[242],[161,7985,7987],{"className":7986,"style":6511},[6510],[6513,7988,7989],{"xmlns":6515,"width":6516,"height":6517,"viewBox":6518,"preserveAspectRatio":6519},[6521,7990],{"d":6523},[161,7992,570],{"className":7993},[569],[161,7995,7997],{"className":7996},[231],[161,7998,8000],{"className":7999,"style":6533},[235],[161,8001],{},[161,8003,8005],{"className":8004,"style":6437},[430,6436],[161,8006,378],{"className":8007},[6441,6973]," 當成一個核函數，上面這一坨就是",[161,8010,8012],{"className":8011},[731],[161,8013,8015,8104],{"className":8014},[164],[161,8016,8018],{"className":8017},[168],[170,8019,8020],{"xmlns":172,"display":740},[174,8021,8022,8101],{},[177,8023,8024,8030,8032,8099],{},[462,8025,8026,8028],{},[360,8027,2593],{},[360,8029,469],{},[364,8031,453],{},[6331,8033,8034,8071],{},[177,8035,8036,8042,8044,8046,8052,8054,8060,8062,8065],{},[3128,8037,8038,8040],{},[364,8039,3132],{},[360,8041,6689],{},[360,8043,7810],{},[364,8045,373],{"stretchy":372},[462,8047,8048,8050],{},[360,8049,753],{},[360,8051,469],{},[364,8053,472],{"separator":198},[462,8055,8056,8058],{},[360,8057,492],{},[360,8059,6689],{},[364,8061,378],{"stretchy":372},[2576,8063,8064],{}," ",[462,8066,8067,8069],{},[360,8068,6175],{},[360,8070,6689],{},[177,8072,8073,8079,8081,8083,8089,8091,8097],{},[3128,8074,8075,8077],{},[364,8076,3132],{},[360,8078,6689],{},[360,8080,7810],{},[364,8082,373],{"stretchy":372},[462,8084,8085,8087],{},[360,8086,753],{},[360,8088,469],{},[364,8090,472],{"separator":198},[462,8092,8093,8095],{},[360,8094,492],{},[360,8096,6689],{},[364,8098,378],{"stretchy":372},[364,8100,472],{"separator":198},[190,8102,8103],{"encoding":192},"o_i = \\frac{\\sum_j \\kappa(q_i, k_j)\\, v_j}{\\sum_j \\kappa(q_i, k_j)} ,",[161,8105,8107,8162],{"className":8106,"ariaHidden":198},[197],[161,8108,8110,8113,8153,8156,8159],{"className":8109},[202],[161,8111],{"className":8112,"style":2068},[206],[161,8114,8116,8119],{"className":8115},[211],[161,8117,2593],{"className":8118},[211,395],[161,8120,8122],{"className":8121},[223],[161,8123,8125,8145],{"className":8124},[227,546],[161,8126,8128,8142],{"className":8127},[231],[161,8129,8131],{"className":8130,"style":553},[235],[161,8132,8133,8136],{"style":556},[161,8134],{"className":8135,"style":243},[242],[161,8137,8139],{"className":8138},[247,248,249,250],[161,8140,469],{"className":8141},[211,395,250],[161,8143,570],{"className":8144},[569],[161,8146,8148],{"className":8147},[231],[161,8149,8151],{"className":8150,"style":577},[235],[161,8152],{},[161,8154],{"className":8155,"style":401},[400],[161,8157,453],{"className":8158},[405],[161,8160],{"className":8161,"style":401},[400],[161,8163,8165,8169,8549],{"className":8164},[202],[161,8166],{"className":8167,"style":8168},[206],"height:2.6976em;vertical-align:-1.1218em;",[161,8170,8172,8175,8546],{"className":8171},[211],[161,8173],{"className":8174},[423,6448],[161,8176,8178],{"className":8177},[6331],[161,8179,8181,8537],{"className":8180},[227,546],[161,8182,8184,8534],{"className":8183},[231],[161,8185,8188,8336,8344],{"className":8186,"style":8187},[235],"height:1.5758em;",[161,8189,8191,8194],{"style":8190},"top:-2.314em;",[161,8192],{"className":8193,"style":6468},[242],[161,8195,8197,8238,8241,8244,8247,8287,8290,8293,8333],{"className":8196},[211],[161,8198,8200,8203],{"className":8199},[1029],[161,8201,3132],{"className":8202,"style":1507},[1029,1230,1506],[161,8204,8206],{"className":8205},[223],[161,8207,8209,8230],{"className":8208},[227,546],[161,8210,8212,8227],{"className":8211},[231],[161,8213,8216],{"className":8214,"style":8215},[235],"height:0.162em;",[161,8217,8218,8221],{"style":1523},[161,8219],{"className":8220,"style":243},[242],[161,8222,8224],{"className":8223},[247,248,249,250],[161,8225,6689],{"className":8226,"style":6919},[211,395,250],[161,8228,570],{"className":8229},[569],[161,8231,8233],{"className":8232},[231],[161,8234,8236],{"className":8235,"style":6949},[235],[161,8237],{},[161,8239],{"className":8240,"style":587},[400],[161,8242,7810],{"className":8243},[211,395],[161,8245,373],{"className":8246},[423],[161,8248,8250,8253],{"className":8249},[211],[161,8251,753],{"className":8252,"style":419},[211,395],[161,8254,8256],{"className":8255},[223],[161,8257,8259,8279],{"className":8258},[227,546],[161,8260,8262,8276],{"className":8261},[231],[161,8263,8265],{"className":8264,"style":553},[235],[161,8266,8267,8270],{"style":608},[161,8268],{"className":8269,"style":243},[242],[161,8271,8273],{"className":8272},[247,248,249,250],[161,8274,469],{"className":8275},[211,395,250],[161,8277,570],{"className":8278},[569],[161,8280,8282],{"className":8281},[231],[161,8283,8285],{"className":8284,"style":577},[235],[161,8286],{},[161,8288,472],{"className":8289},[583],[161,8291],{"className":8292,"style":587},[400],[161,8294,8296,8299],{"className":8295},[211],[161,8297,492],{"className":8298,"style":678},[211,395],[161,8300,8302],{"className":8301},[223],[161,8303,8305,8325],{"className":8304},[227,546],[161,8306,8308,8322],{"className":8307},[231],[161,8309,8311],{"className":8310,"style":553},[235],[161,8312,8313,8316],{"style":7771},[161,8314],{"className":8315,"style":243},[242],[161,8317,8319],{"className":8318},[247,248,249,250],[161,8320,6689],{"className":8321,"style":6919},[211,395,250],[161,8323,570],{"className":8324},[569],[161,8326,8328],{"className":8327},[231],[161,8329,8331],{"className":8330,"style":2379},[235],[161,8332],{},[161,8334,378],{"className":8335},[430],[161,8337,8338,8341],{"style":6538},[161,8339],{"className":8340,"style":6468},[242],[161,8342],{"className":8343,"style":6546},[6545],[161,8345,8347,8350],{"style":8346},"top:-3.8258em;",[161,8348],{"className":8349,"style":6468},[242],[161,8351,8353,8393,8396,8399,8402,8442,8445,8448,8488,8491,8494],{"className":8352},[211],[161,8354,8356,8359],{"className":8355},[1029],[161,8357,3132],{"className":8358,"style":1507},[1029,1230,1506],[161,8360,8362],{"className":8361},[223],[161,8363,8365,8385],{"className":8364},[227,546],[161,8366,8368,8382],{"className":8367},[231],[161,8369,8371],{"className":8370,"style":8215},[235],[161,8372,8373,8376],{"style":1523},[161,8374],{"className":8375,"style":243},[242],[161,8377,8379],{"className":8378},[247,248,249,250],[161,8380,6689],{"className":8381,"style":6919},[211,395,250],[161,8383,570],{"className":8384},[569],[161,8386,8388],{"className":8387},[231],[161,8389,8391],{"className":8390,"style":6949},[235],[161,8392],{},[161,8394],{"className":8395,"style":587},[400],[161,8397,7810],{"className":8398},[211,395],[161,8400,373],{"className":8401},[423],[161,8403,8405,8408],{"className":8404},[211],[161,8406,753],{"className":8407,"style":419},[211,395],[161,8409,8411],{"className":8410},[223],[161,8412,8414,8434],{"className":8413},[227,546],[161,8415,8417,8431],{"className":8416},[231],[161,8418,8420],{"className":8419,"style":553},[235],[161,8421,8422,8425],{"style":608},[161,8423],{"className":8424,"style":243},[242],[161,8426,8428],{"className":8427},[247,248,249,250],[161,8429,469],{"className":8430},[211,395,250],[161,8432,570],{"className":8433},[569],[161,8435,8437],{"className":8436},[231],[161,8438,8440],{"className":8439,"style":577},[235],[161,8441],{},[161,8443,472],{"className":8444},[583],[161,8446],{"className":8447,"style":587},[400],[161,8449,8451,8454],{"className":8450},[211],[161,8452,492],{"className":8453,"style":678},[211,395],[161,8455,8457],{"className":8456},[223],[161,8458,8460,8480],{"className":8459},[227,546],[161,8461,8463,8477],{"className":8462},[231],[161,8464,8466],{"className":8465,"style":553},[235],[161,8467,8468,8471],{"style":7771},[161,8469],{"className":8470,"style":243},[242],[161,8472,8474],{"className":8473},[247,248,249,250],[161,8475,6689],{"className":8476,"style":6919},[211,395,250],[161,8478,570],{"className":8479},[569],[161,8481,8483],{"className":8482},[231],[161,8484,8486],{"className":8485,"style":2379},[235],[161,8487],{},[161,8489,378],{"className":8490},[430],[161,8492],{"className":8493,"style":587},[400],[161,8495,8497,8500],{"className":8496},[211],[161,8498,6175],{"className":8499,"style":419},[211,395],[161,8501,8503],{"className":8502},[223],[161,8504,8506,8526],{"className":8505},[227,546],[161,8507,8509,8523],{"className":8508},[231],[161,8510,8512],{"className":8511,"style":553},[235],[161,8513,8514,8517],{"style":608},[161,8515],{"className":8516,"style":243},[242],[161,8518,8520],{"className":8519},[247,248,249,250],[161,8521,6689],{"className":8522,"style":6919},[211,395,250],[161,8524,570],{"className":8525},[569],[161,8527,8529],{"className":8528},[231],[161,8530,8532],{"className":8531,"style":2379},[235],[161,8533],{},[161,8535,570],{"className":8536},[569],[161,8538,8540],{"className":8539},[231],[161,8541,8544],{"className":8542,"style":8543},[235],"height:1.1218em;",[161,8545],{},[161,8547],{"className":8548},[430,6448],[161,8550,472],{"className":8551},[583],[116,8553,8554],{},"——非參數回歸裡的 Nadaraya–Watson 估計。一模一樣。",[116,8556,8557],{},"差別只在於：這裡的「樣本」是模型自己學出來的，不是給定的資料集。",[108,8559,8560],{"id":8560},"知道這件事有什麼用",[116,8562,8563,8564,2180],{},"不是為了數學上的優雅。是因為它會告訴你",[112,8565,8566],{},"哪裡可以砍",[116,8568,8569,8570,8599,8600,8750],{},"如果 ",[161,8571,8573,8587],{"className":8572},[164],[161,8574,8576],{"className":8575},[168],[170,8577,8578],{"xmlns":172},[174,8579,8580,8584],{},[177,8581,8582],{},[360,8583,7810],{},[190,8585,8586],{"encoding":192},"\\kappa",[161,8588,8590],{"className":8589,"ariaHidden":198},[197],[161,8591,8593,8596],{"className":8592},[202],[161,8594],{"className":8595,"style":505},[206],[161,8597,7810],{"className":8598},[211,395]," 可以寫成有限維特徵映射的內積，也就是 ",[161,8601,8603,8651],{"className":8602},[164],[161,8604,8606],{"className":8605},[168],[170,8607,8608],{"xmlns":172},[174,8609,8610,8648],{},[177,8611,8612,8614,8616,8618,8620,8622,8624,8627,8630,8632,8634,8640,8642,8644,8646],{},[360,8613,7810],{},[364,8615,373],{"stretchy":372},[360,8617,753],{},[364,8619,472],{"separator":198},[360,8621,492],{},[364,8623,378],{"stretchy":372},[364,8625,8626],{},"≈",[360,8628,8629],{},"ϕ",[364,8631,373],{"stretchy":372},[360,8633,753],{},[180,8635,8636,8638],{},[364,8637,378],{"stretchy":372},[360,8639,6343],{"mathvariant":803},[360,8641,8629],{},[364,8643,373],{"stretchy":372},[360,8645,492],{},[364,8647,378],{"stretchy":372},[190,8649,8650],{"encoding":192},"\\kappa(q,k) \\approx \\phi(q)^{\\top}\\phi(k)",[161,8652,8654,8690],{"className":8653,"ariaHidden":198},[197],[161,8655,8657,8660,8663,8666,8669,8672,8675,8678,8681,8684,8687],{"className":8656},[202],[161,8658],{"className":8659,"style":415},[206],[161,8661,7810],{"className":8662},[211,395],[161,8664,373],{"className":8665},[423],[161,8667,753],{"className":8668,"style":419},[211,395],[161,8670,472],{"className":8671},[583],[161,8673],{"className":8674,"style":587},[400],[161,8676,492],{"className":8677,"style":678},[211,395],[161,8679,378],{"className":8680},[430],[161,8682],{"className":8683,"style":401},[400],[161,8685,8626],{"className":8686},[405],[161,8688],{"className":8689,"style":401},[400],[161,8691,8693,8697,8700,8703,8706,8738,8741,8744,8747],{"className":8692},[202],[161,8694],{"className":8695,"style":8696},[206],"height:1.0991em;vertical-align:-0.25em;",[161,8698,8629],{"className":8699},[211,395],[161,8701,373],{"className":8702},[423],[161,8704,753],{"className":8705,"style":419},[211,395],[161,8707,8709,8712],{"className":8708},[430],[161,8710,378],{"className":8711},[430],[161,8713,8715],{"className":8714},[223],[161,8716,8718],{"className":8717},[227],[161,8719,8721],{"className":8720},[231],[161,8722,8724],{"className":8723,"style":6005},[235],[161,8725,8726,8729],{"style":238},[161,8727],{"className":8728,"style":243},[242],[161,8730,8732],{"className":8731},[247,248,249,250],[161,8733,8735],{"className":8734},[211,250],[161,8736,6343],{"className":8737},[211,250],[161,8739,8629],{"className":8740},[211,395],[161,8742,373],{"className":8743},[423],[161,8745,492],{"className":8746,"style":678},[211,395],[161,8748,378],{"className":8749},[430],"，那麼",[161,8752,8754],{"className":8753},[731],[161,8755,8757,8861],{"className":8756},[164],[161,8758,8760],{"className":8759},[168],[170,8761,8762],{"xmlns":172,"display":740},[174,8763,8764,8858],{},[177,8765,8766,8772,8774,8856],{},[462,8767,8768,8770],{},[360,8769,2593],{},[360,8771,469],{},[364,8773,8626],{},[6331,8775,8776,8820],{},[177,8777,8778,8780,8782,8788,8794,8800,8802,8804,8810,8812],{},[360,8779,8629],{},[364,8781,373],{"stretchy":372},[462,8783,8784,8786],{},[360,8785,753],{},[360,8787,469],{},[180,8789,8790,8792],{},[364,8791,378],{"stretchy":372},[360,8793,6343],{"mathvariant":803},[3128,8795,8796,8798],{},[364,8797,3132],{},[360,8799,6689],{},[360,8801,8629],{},[364,8803,373],{"stretchy":372},[462,8805,8806,8808],{},[360,8807,492],{},[360,8809,6689],{},[364,8811,378],{"stretchy":372},[455,8813,8814,8816,8818],{},[360,8815,6175],{},[360,8817,6689],{},[360,8819,6343],{"mathvariant":803},[177,8821,8822,8824,8826,8832,8838,8844,8846,8848,8854],{},[360,8823,8629],{},[364,8825,373],{"stretchy":372},[462,8827,8828,8830],{},[360,8829,753],{},[360,8831,469],{},[180,8833,8834,8836],{},[364,8835,378],{"stretchy":372},[360,8837,6343],{"mathvariant":803},[3128,8839,8840,8842],{},[364,8841,3132],{},[360,8843,6689],{},[360,8845,8629],{},[364,8847,373],{"stretchy":372},[462,8849,8850,8852],{},[360,8851,492],{},[360,8853,6689],{},[364,8855,378],{"stretchy":372},[360,8857,6355],{"mathvariant":803},[190,8859,8860],{"encoding":192},"o_i \\approx \\frac{\\phi(q_i)^{\\top} \\sum_j \\phi(k_j) v_j^{\\top}}{\\phi(q_i)^{\\top} \\sum_j \\phi(k_j)} .",[161,8862,8864,8920],{"className":8863,"ariaHidden":198},[197],[161,8865,8867,8871,8911,8914,8917],{"className":8866},[202],[161,8868],{"className":8869,"style":8870},[206],"height:0.6331em;vertical-align:-0.15em;",[161,8872,8874,8877],{"className":8873},[211],[161,8875,2593],{"className":8876},[211,395],[161,8878,8880],{"className":8879},[223],[161,8881,8883,8903],{"className":8882},[227,546],[161,8884,8886,8900],{"className":8885},[231],[161,8887,8889],{"className":8888,"style":553},[235],[161,8890,8891,8894],{"style":556},[161,8892],{"className":8893,"style":243},[242],[161,8895,8897],{"className":8896},[247,248,249,250],[161,8898,469],{"className":8899},[211,395,250],[161,8901,570],{"className":8902},[569],[161,8904,8906],{"className":8905},[231],[161,8907,8909],{"className":8908,"style":577},[235],[161,8910],{},[161,8912],{"className":8913,"style":401},[400],[161,8915,8626],{"className":8916},[405],[161,8918],{"className":8919,"style":401},[400],[161,8921,8923,8927,9388],{"className":8922},[202],[161,8924],{"className":8925,"style":8926},[206],"height:2.7967em;vertical-align:-1.1218em;",[161,8928,8930,8933,9385],{"className":8929},[211],[161,8931],{"className":8932},[423,6448],[161,8934,8936],{"className":8935},[6331],[161,8937,8939,9377],{"className":8938},[227,546],[161,8940,8942,9374],{"className":8941},[231],[161,8943,8946,9129,9137],{"className":8944,"style":8945},[235],"height:1.6749em;",[161,8947,8948,8951],{"style":8190},[161,8949],{"className":8950,"style":6468},[242],[161,8952,8954,8957,8960,9000,9034,9037,9077,9080,9083,9086,9126],{"className":8953},[211],[161,8955,8629],{"className":8956},[211,395],[161,8958,373],{"className":8959},[423],[161,8961,8963,8966],{"className":8962},[211],[161,8964,753],{"className":8965,"style":419},[211,395],[161,8967,8969],{"className":8968},[223],[161,8970,8972,8992],{"className":8971},[227,546],[161,8973,8975,8989],{"className":8974},[231],[161,8976,8978],{"className":8977,"style":553},[235],[161,8979,8980,8983],{"style":608},[161,8981],{"className":8982,"style":243},[242],[161,8984,8986],{"className":8985},[247,248,249,250],[161,8987,469],{"className":8988},[211,395,250],[161,8990,570],{"className":8991},[569],[161,8993,8995],{"className":8994},[231],[161,8996,8998],{"className":8997,"style":577},[235],[161,8999],{},[161,9001,9003,9006],{"className":9002},[430],[161,9004,378],{"className":9005},[430],[161,9007,9009],{"className":9008},[223],[161,9010,9012],{"className":9011},[227],[161,9013,9015],{"className":9014},[231],[161,9016,9019],{"className":9017,"style":9018},[235],"height:0.7751em;",[161,9020,9022,9025],{"style":9021},"top:-2.989em;margin-right:0.05em;",[161,9023],{"className":9024,"style":243},[242],[161,9026,9028],{"className":9027},[247,248,249,250],[161,9029,9031],{"className":9030},[211,250],[161,9032,6343],{"className":9033},[211,250],[161,9035],{"className":9036,"style":587},[400],[161,9038,9040,9043],{"className":9039},[1029],[161,9041,3132],{"className":9042,"style":1507},[1029,1230,1506],[161,9044,9046],{"className":9045},[223],[161,9047,9049,9069],{"className":9048},[227,546],[161,9050,9052,9066],{"className":9051},[231],[161,9053,9055],{"className":9054,"style":8215},[235],[161,9056,9057,9060],{"style":1523},[161,9058],{"className":9059,"style":243},[242],[161,9061,9063],{"className":9062},[247,248,249,250],[161,9064,6689],{"className":9065,"style":6919},[211,395,250],[161,9067,570],{"className":9068},[569],[161,9070,9072],{"className":9071},[231],[161,9073,9075],{"className":9074,"style":6949},[235],[161,9076],{},[161,9078],{"className":9079,"style":587},[400],[161,9081,8629],{"className":9082},[211,395],[161,9084,373],{"className":9085},[423],[161,9087,9089,9092],{"className":9088},[211],[161,9090,492],{"className":9091,"style":678},[211,395],[161,9093,9095],{"className":9094},[223],[161,9096,9098,9118],{"className":9097},[227,546],[161,9099,9101,9115],{"className":9100},[231],[161,9102,9104],{"className":9103,"style":553},[235],[161,9105,9106,9109],{"style":7771},[161,9107],{"className":9108,"style":243},[242],[161,9110,9112],{"className":9111},[247,248,249,250],[161,9113,6689],{"className":9114,"style":6919},[211,395,250],[161,9116,570],{"className":9117},[569],[161,9119,9121],{"className":9120},[231],[161,9122,9124],{"className":9123,"style":2379},[235],[161,9125],{},[161,9127,378],{"className":9128},[430],[161,9130,9131,9134],{"style":6538},[161,9132],{"className":9133,"style":6468},[242],[161,9135],{"className":9136,"style":6546},[6545],[161,9138,9139,9142],{"style":8346},[161,9140],{"className":9141,"style":6468},[242],[161,9143,9145,9148,9151,9191,9223,9226,9266,9269,9272,9275,9315,9318],{"className":9144},[211],[161,9146,8629],{"className":9147},[211,395],[161,9149,373],{"className":9150},[423],[161,9152,9154,9157],{"className":9153},[211],[161,9155,753],{"className":9156,"style":419},[211,395],[161,9158,9160],{"className":9159},[223],[161,9161,9163,9183],{"className":9162},[227,546],[161,9164,9166,9180],{"className":9165},[231],[161,9167,9169],{"className":9168,"style":553},[235],[161,9170,9171,9174],{"style":608},[161,9172],{"className":9173,"style":243},[242],[161,9175,9177],{"className":9176},[247,248,249,250],[161,9178,469],{"className":9179},[211,395,250],[161,9181,570],{"className":9182},[569],[161,9184,9186],{"className":9185},[231],[161,9187,9189],{"className":9188,"style":577},[235],[161,9190],{},[161,9192,9194,9197],{"className":9193},[430],[161,9195,378],{"className":9196},[430],[161,9198,9200],{"className":9199},[223],[161,9201,9203],{"className":9202},[227],[161,9204,9206],{"className":9205},[231],[161,9207,9209],{"className":9208,"style":6005},[235],[161,9210,9211,9214],{"style":238},[161,9212],{"className":9213,"style":243},[242],[161,9215,9217],{"className":9216},[247,248,249,250],[161,9218,9220],{"className":9219},[211,250],[161,9221,6343],{"className":9222},[211,250],[161,9224],{"className":9225,"style":587},[400],[161,9227,9229,9232],{"className":9228},[1029],[161,9230,3132],{"className":9231,"style":1507},[1029,1230,1506],[161,9233,9235],{"className":9234},[223],[161,9236,9238,9258],{"className":9237},[227,546],[161,9239,9241,9255],{"className":9240},[231],[161,9242,9244],{"className":9243,"style":8215},[235],[161,9245,9246,9249],{"style":1523},[161,9247],{"className":9248,"style":243},[242],[161,9250,9252],{"className":9251},[247,248,249,250],[161,9253,6689],{"className":9254,"style":6919},[211,395,250],[161,9256,570],{"className":9257},[569],[161,9259,9261],{"className":9260},[231],[161,9262,9264],{"className":9263,"style":6949},[235],[161,9265],{},[161,9267],{"className":9268,"style":587},[400],[161,9270,8629],{"className":9271},[211,395],[161,9273,373],{"className":9274},[423],[161,9276,9278,9281],{"className":9277},[211],[161,9279,492],{"className":9280,"style":678},[211,395],[161,9282,9284],{"className":9283},[223],[161,9285,9287,9307],{"className":9286},[227,546],[161,9288,9290,9304],{"className":9289},[231],[161,9291,9293],{"className":9292,"style":553},[235],[161,9294,9295,9298],{"style":7771},[161,9296],{"className":9297,"style":243},[242],[161,9299,9301],{"className":9300},[247,248,249,250],[161,9302,6689],{"className":9303,"style":6919},[211,395,250],[161,9305,570],{"className":9306},[569],[161,9308,9310],{"className":9309},[231],[161,9311,9313],{"className":9312,"style":2379},[235],[161,9314],{},[161,9316,378],{"className":9317},[430],[161,9319,9321,9324],{"className":9320},[211],[161,9322,6175],{"className":9323,"style":419},[211,395],[161,9325,9327],{"className":9326},[223],[161,9328,9330,9365],{"className":9329},[227,546],[161,9331,9333,9362],{"className":9332},[231],[161,9334,9336,9348],{"className":9335,"style":6005},[235],[161,9337,9339,9342],{"style":9338},"top:-2.4413em;margin-left:-0.0359em;margin-right:0.05em;",[161,9340],{"className":9341,"style":243},[242],[161,9343,9345],{"className":9344},[247,248,249,250],[161,9346,6689],{"className":9347,"style":6919},[211,395,250],[161,9349,9350,9353],{"style":238},[161,9351],{"className":9352,"style":243},[242],[161,9354,9356],{"className":9355},[247,248,249,250],[161,9357,9359],{"className":9358},[211,250],[161,9360,6343],{"className":9361},[211,250],[161,9363,570],{"className":9364},[569],[161,9366,9368],{"className":9367},[231],[161,9369,9372],{"className":9370,"style":9371},[235],"height:0.3948em;",[161,9373],{},[161,9375,570],{"className":9376},[569],[161,9378,9380],{"className":9379},[231],[161,9381,9383],{"className":9382,"style":8543},[235],[161,9384],{},[161,9386],{"className":9387},[430,6448],[161,9389,6355],{"className":9390},[211],[116,9392,9393,9394,9442,9443,9509],{},"右邊的兩個累加量可以事先算好，每個查詢只要查表。複雜度從 ",[161,9395,9397,9420],{"className":9396},[164],[161,9398,9400],{"className":9399},[168],[170,9401,9402],{"xmlns":172},[174,9403,9404,9417],{},[177,9405,9406,9409,9411,9413,9415],{},[360,9407,9408],{},"O",[364,9410,373],{"stretchy":372},[360,9412,5969],{},[360,9414,2608],{},[364,9416,378],{"stretchy":372},[190,9418,9419],{"encoding":192},"O(nm)",[161,9421,9423],{"className":9422,"ariaHidden":198},[197],[161,9424,9426,9429,9432,9435,9439],{"className":9425},[202],[161,9427],{"className":9428,"style":415},[206],[161,9430,9408],{"className":9431,"style":396},[211,395],[161,9433,373],{"className":9434},[423],[161,9436,9438],{"className":9437},[211,395],"nm",[161,9440,378],{"className":9441},[430]," 掉到 ",[161,9444,9446,9470],{"className":9445},[164],[161,9447,9449],{"className":9448},[168],[170,9450,9451],{"xmlns":172},[174,9452,9453,9467],{},[177,9454,9455,9457,9459,9461,9463,9465],{},[360,9456,9408],{},[364,9458,373],{"stretchy":372},[360,9460,5969],{},[364,9462,2306],{},[360,9464,2608],{},[364,9466,378],{"stretchy":372},[190,9468,9469],{"encoding":192},"O(n + m)",[161,9471,9473,9497],{"className":9472,"ariaHidden":198},[197],[161,9474,9476,9479,9482,9485,9488,9491,9494],{"className":9475},[202],[161,9477],{"className":9478,"style":415},[206],[161,9480,9408],{"className":9481,"style":396},[211,395],[161,9483,373],{"className":9484},[423],[161,9486,5969],{"className":9487},[211,395],[161,9489],{"className":9490,"style":1934},[400],[161,9492,2306],{"className":9493},[1186],[161,9495],{"className":9496,"style":1934},[400],[161,9498,9500,9503,9506],{"className":9499},[202],[161,9501],{"className":9502,"style":415},[206],[161,9504,2608],{"className":9505},[211,395],[161,9507,378],{"className":9508},[430],"：",[161,9511,9513],{"className":9512},[731],[161,9514,9516,9617],{"className":9515},[164],[161,9517,9519],{"className":9518},[168],[170,9520,9521],{"xmlns":172,"display":740},[174,9522,9523,9614],{},[9524,9525,9528,9565],"mtable",{"rowspacing":9526,"columnalign":9527,"columnspacing":3139},"0.25em","right left",[9529,9530,9531,9540],"mtr",{},[9532,9533,9534],"mtd",{},[9535,9536,9537],"mstyle",{"scriptlevel":219,"displaystyle":198},[2576,9538,9539],{},"標準 attention",[9532,9541,9542],{},[9535,9543,9544],{"scriptlevel":219,"displaystyle":198},[177,9545,9546,9548,9550,9553,9555,9557,9559,9561,9563],{},[177,9547],{},[364,9549,1840],{},[400,9551],{"width":9552},"1em",[360,9554,9408],{},[364,9556,373],{"stretchy":372},[360,9558,5969],{},[360,9560,2608],{},[360,9562,1140],{},[364,9564,378],{"stretchy":372},[9529,9566,9567,9574],{},[9532,9568,9569],{},[9535,9570,9571],{"scriptlevel":219,"displaystyle":198},[2576,9572,9573],{},"線性 attention",[9532,9575,9576],{},[9535,9577,9578],{"scriptlevel":219,"displaystyle":198},[177,9579,9580,9582,9584,9586,9588,9590],{},[177,9581],{},[364,9583,1840],{},[400,9585],{"width":9552},[360,9587,9408],{},[2576,9589,2578],{},[177,9591,9592,9594,9596,9602,9604,9606,9612],{},[364,9593,373],{"fence":198},[360,9595,5969],{},[180,9597,9598,9600],{},[360,9599,1140],{},[183,9601,2295],{},[364,9603,2306],{},[360,9605,2608],{},[180,9607,9608,9610],{},[360,9609,1140],{},[183,9611,2295],{},[364,9613,378],{"fence":198},[190,9615,9616],{"encoding":192},"\\begin{aligned}\n\\text{標準 attention} &: \\quad O(n m d) \\\\\n\\text{線性 attention} &: \\quad O\\!\\left(n d^2 + m d^2\\right)\n\\end{aligned}",[161,9618,9620],{"className":9619,"ariaHidden":198},[197],[161,9621,9623,9627],{"className":9622},[202],[161,9624],{"className":9625,"style":9626},[206],"height:2.7241em;vertical-align:-1.1121em;",[161,9628,9630],{"className":9629},[211],[161,9631,9633,9700],{"className":9632},[9524],[161,9634,9637],{"className":9635},[9636],"col-align-r",[161,9638,9640,9691],{"className":9639},[227,546],[161,9641,9643,9688],{"className":9642},[231],[161,9644,9647,9669],{"className":9645,"style":9646},[235],"height:1.6121em;",[161,9648,9650,9653],{"style":9649},"top:-3.7721em;",[161,9651],{"className":9652,"style":6468},[242],[161,9654,9656],{"className":9655},[211],[161,9657,9660,9665],{"className":9658},[211,9659],"text",[161,9661,9664],{"className":9662},[211,9663],"cjk_fallback","標準",[161,9666,9668],{"className":9667},[211]," attention",[161,9670,9672,9675],{"style":9671},"top:-2.2479em;",[161,9673],{"className":9674,"style":6468},[242],[161,9676,9678],{"className":9677},[211],[161,9679,9681,9685],{"className":9680},[211,9659],[161,9682,9684],{"className":9683},[211,9663],"線性",[161,9686,9668],{"className":9687},[211],[161,9689,570],{"className":9690},[569],[161,9692,9694],{"className":9693},[231],[161,9695,9698],{"className":9696,"style":9697},[235],"height:1.1121em;",[161,9699],{},[161,9701,9704],{"className":9702},[9703],"col-align-l",[161,9705,9707,9876],{"className":9706},[227,546],[161,9708,9710,9873],{"className":9709},[231],[161,9711,9713,9752],{"className":9712,"style":9646},[235],[161,9714,9715,9718],{"style":9649},[161,9716],{"className":9717,"style":6468},[242],[161,9719,9721,9724,9727,9730,9734,9737,9740,9743,9746,9749],{"className":9720},[211],[161,9722],{"className":9723},[211],[161,9725],{"className":9726,"style":401},[400],[161,9728,1840],{"className":9729},[405],[161,9731],{"className":9732,"style":9733},[400],"margin-right:1em;",[161,9735],{"className":9736,"style":401},[400],[161,9738,9408],{"className":9739,"style":396},[211,395],[161,9741,373],{"className":9742},[423],[161,9744,9438],{"className":9745},[211,395],[161,9747,1140],{"className":9748},[211,395],[161,9750,378],{"className":9751},[430],[161,9753,9754,9757],{"style":9671},[161,9755],{"className":9756,"style":6468},[242],[161,9758,9760,9763,9766,9769,9772,9775,9778,9781,9784],{"className":9759},[211],[161,9761],{"className":9762},[211],[161,9764],{"className":9765,"style":401},[400],[161,9767,1840],{"className":9768},[405],[161,9770],{"className":9771,"style":9733},[400],[161,9773],{"className":9774,"style":401},[400],[161,9776,9408],{"className":9777,"style":396},[211,395],[161,9779],{"className":9780,"style":2637},[400],[161,9782],{"className":9783,"style":587},[400],[161,9785,9787,9793,9796,9826,9829,9832,9835,9838,9867],{"className":9786},[1921],[161,9788,9790],{"className":9789,"style":6437},[423,6436],[161,9791,373],{"className":9792},[6441,1566],[161,9794,5969],{"className":9795},[211,395],[161,9797,9799,9802],{"className":9798},[211],[161,9800,1140],{"className":9801},[211,395],[161,9803,9805],{"className":9804},[223],[161,9806,9808],{"className":9807},[227],[161,9809,9811],{"className":9810},[231],[161,9812,9815],{"className":9813,"style":9814},[235],"height:0.8641em;",[161,9816,9817,9820],{"style":1176},[161,9818],{"className":9819,"style":243},[242],[161,9821,9823],{"className":9822},[247,248,249,250],[161,9824,2295],{"className":9825},[211,250],[161,9827],{"className":9828,"style":1934},[400],[161,9830,2306],{"className":9831},[1186],[161,9833],{"className":9834,"style":1934},[400],[161,9836,2608],{"className":9837},[211,395],[161,9839,9841,9844],{"className":9840},[211],[161,9842,1140],{"className":9843},[211,395],[161,9845,9847],{"className":9846},[223],[161,9848,9850],{"className":9849},[227],[161,9851,9853],{"className":9852},[231],[161,9854,9856],{"className":9855,"style":9814},[235],[161,9857,9858,9861],{"style":1176},[161,9859],{"className":9860,"style":243},[242],[161,9862,9864],{"className":9863},[247,248,249,250],[161,9865,2295],{"className":9866},[211,250],[161,9868,9870],{"className":9869,"style":6437},[430,6436],[161,9871,378],{"className":9872},[6441,1566],[161,9874,570],{"className":9875},[569],[161,9877,9879],{"className":9878},[231],[161,9880,9882],{"className":9881,"style":9697},[235],[161,9883],{},[116,9885,9886],{},"這就是 linear attention 那一整家族在做的事。",[108,9888,9889],{"id":9889},"代價在分母",[116,9891,9892,9893],{},"這裡有個很容易被忽略的地方：",[112,9894,9895],{},"誤差最大的地方不是分子，是分母。",[116,9897,9898,9899,10131,10132,10321],{},"分子是「加權和」，近似得差一點，輸出只是稍微偏掉；分母是「歸一化常數」，它一錯，整個輸出會跟著縮放。序列越長，",[161,9900,9902,9948],{"className":9901},[164],[161,9903,9905],{"className":9904},[168],[170,9906,9907],{"xmlns":172},[174,9908,9909,9945],{},[177,9910,9911,9913,9915,9921,9927,9933,9935,9937,9943],{},[360,9912,8629],{},[364,9914,373],{"stretchy":372},[462,9916,9917,9919],{},[360,9918,753],{},[360,9920,469],{},[180,9922,9923,9925],{},[364,9924,378],{"stretchy":372},[360,9926,6343],{"mathvariant":803},[462,9928,9929,9931],{},[364,9930,3132],{},[360,9932,6689],{},[360,9934,8629],{},[364,9936,373],{"stretchy":372},[462,9938,9939,9941],{},[360,9940,492],{},[360,9942,6689],{},[364,9944,378],{"stretchy":372},[190,9946,9947],{"encoding":192},"\\phi(q_i)^{\\top}\\sum_j \\phi(k_j)",[161,9949,9951],{"className":9950,"ariaHidden":198},[197],[161,9952,9954,9958,9961,9964,10004,10036,10039,10079,10082,10085,10088,10128],{"className":9953},[202],[161,9955],{"className":9956,"style":9957},[206],"height:1.2849em;vertical-align:-0.4358em;",[161,9959,8629],{"className":9960},[211,395],[161,9962,373],{"className":9963},[423],[161,9965,9967,9970],{"className":9966},[211],[161,9968,753],{"className":9969,"style":419},[211,395],[161,9971,9973],{"className":9972},[223],[161,9974,9976,9996],{"className":9975},[227,546],[161,9977,9979,9993],{"className":9978},[231],[161,9980,9982],{"className":9981,"style":553},[235],[161,9983,9984,9987],{"style":608},[161,9985],{"className":9986,"style":243},[242],[161,9988,9990],{"className":9989},[247,248,249,250],[161,9991,469],{"className":9992},[211,395,250],[161,9994,570],{"className":9995},[569],[161,9997,9999],{"className":9998},[231],[161,10000,10002],{"className":10001,"style":577},[235],[161,10003],{},[161,10005,10007,10010],{"className":10006},[430],[161,10008,378],{"className":10009},[430],[161,10011,10013],{"className":10012},[223],[161,10014,10016],{"className":10015},[227],[161,10017,10019],{"className":10018},[231],[161,10020,10022],{"className":10021,"style":6005},[235],[161,10023,10024,10027],{"style":238},[161,10025],{"className":10026,"style":243},[242],[161,10028,10030],{"className":10029},[247,248,249,250],[161,10031,10033],{"className":10032},[211,250],[161,10034,6343],{"className":10035},[211,250],[161,10037],{"className":10038,"style":587},[400],[161,10040,10042,10045],{"className":10041},[1029],[161,10043,3132],{"className":10044,"style":1507},[1029,1230,1506],[161,10046,10048],{"className":10047},[223],[161,10049,10051,10071],{"className":10050},[227,546],[161,10052,10054,10068],{"className":10053},[231],[161,10055,10057],{"className":10056,"style":8215},[235],[161,10058,10059,10062],{"style":1523},[161,10060],{"className":10061,"style":243},[242],[161,10063,10065],{"className":10064},[247,248,249,250],[161,10066,6689],{"className":10067,"style":6919},[211,395,250],[161,10069,570],{"className":10070},[569],[161,10072,10074],{"className":10073},[231],[161,10075,10077],{"className":10076,"style":6949},[235],[161,10078],{},[161,10080],{"className":10081,"style":587},[400],[161,10083,8629],{"className":10084},[211,395],[161,10086,373],{"className":10087},[423],[161,10089,10091,10094],{"className":10090},[211],[161,10092,492],{"className":10093,"style":678},[211,395],[161,10095,10097],{"className":10096},[223],[161,10098,10100,10120],{"className":10099},[227,546],[161,10101,10103,10117],{"className":10102},[231],[161,10104,10106],{"className":10105,"style":553},[235],[161,10107,10108,10111],{"style":7771},[161,10109],{"className":10110,"style":243},[242],[161,10112,10114],{"className":10113},[247,248,249,250],[161,10115,6689],{"className":10116,"style":6919},[211,395,250],[161,10118,570],{"className":10119},[569],[161,10121,10123],{"className":10122},[231],[161,10124,10126],{"className":10125,"style":2379},[235],[161,10127],{},[161,10129,378],{"className":10130},[430]," 和真正的 ",[161,10133,10135,10173],{"className":10134},[164],[161,10136,10138],{"className":10137},[168],[170,10139,10140],{"xmlns":172},[174,10141,10142,10170],{},[177,10143,10144,10150,10152,10154,10160,10162,10168],{},[462,10145,10146,10148],{},[364,10147,3132],{},[360,10149,6689],{},[360,10151,7810],{},[364,10153,373],{"stretchy":372},[462,10155,10156,10158],{},[360,10157,753],{},[360,10159,469],{},[364,10161,472],{"separator":198},[462,10163,10164,10166],{},[360,10165,492],{},[360,10167,6689],{},[364,10169,378],{"stretchy":372},[190,10171,10172],{"encoding":192},"\\sum_j \\kappa(q_i,k_j)",[161,10174,10176],{"className":10175,"ariaHidden":198},[197],[161,10177,10179,10183,10223,10226,10229,10232,10272,10275,10278,10318],{"className":10178},[202],[161,10180],{"className":10181,"style":10182},[206],"height:1.1858em;vertical-align:-0.4358em;",[161,10184,10186,10189],{"className":10185},[1029],[161,10187,3132],{"className":10188,"style":1507},[1029,1230,1506],[161,10190,10192],{"className":10191},[223],[161,10193,10195,10215],{"className":10194},[227,546],[161,10196,10198,10212],{"className":10197},[231],[161,10199,10201],{"className":10200,"style":8215},[235],[161,10202,10203,10206],{"style":1523},[161,10204],{"className":10205,"style":243},[242],[161,10207,10209],{"className":10208},[247,248,249,250],[161,10210,6689],{"className":10211,"style":6919},[211,395,250],[161,10213,570],{"className":10214},[569],[161,10216,10218],{"className":10217},[231],[161,10219,10221],{"className":10220,"style":6949},[235],[161,10222],{},[161,10224],{"className":10225,"style":587},[400],[161,10227,7810],{"className":10228},[211,395],[161,10230,373],{"className":10231},[423],[161,10233,10235,10238],{"className":10234},[211],[161,10236,753],{"className":10237,"style":419},[211,395],[161,10239,10241],{"className":10240},[223],[161,10242,10244,10264],{"className":10243},[227,546],[161,10245,10247,10261],{"className":10246},[231],[161,10248,10250],{"className":10249,"style":553},[235],[161,10251,10252,10255],{"style":608},[161,10253],{"className":10254,"style":243},[242],[161,10256,10258],{"className":10257},[247,248,249,250],[161,10259,469],{"className":10260},[211,395,250],[161,10262,570],{"className":10263},[569],[161,10265,10267],{"className":10266},[231],[161,10268,10270],{"className":10269,"style":577},[235],[161,10271],{},[161,10273,472],{"className":10274},[583],[161,10276],{"className":10277,"style":587},[400],[161,10279,10281,10284],{"className":10280},[211],[161,10282,492],{"className":10283,"style":678},[211,395],[161,10285,10287],{"className":10286},[223],[161,10288,10290,10310],{"className":10289},[227,546],[161,10291,10293,10307],{"className":10292},[231],[161,10294,10296],{"className":10295,"style":553},[235],[161,10297,10298,10301],{"style":7771},[161,10299],{"className":10300,"style":243},[242],[161,10302,10304],{"className":10303},[247,248,249,250],[161,10305,6689],{"className":10306,"style":6919},[211,395,250],[161,10308,570],{"className":10309},[569],[161,10311,10313],{"className":10312},[231],[161,10314,10316],{"className":10315,"style":2379},[235],[161,10317],{},[161,10319,378],{"className":10320},[430]," 差得越多，輸出就越不可信。",[116,10323,10324],{},"所以我們在評估這類方法時，看的第一個數字不是整體 perplexity，而是分母的相對誤差。",[108,10326,10327],{"id":10327},"實務上的三個檢查點",[4858,10329,10330,10340],{},[4861,10331,10332],{},[4864,10333,10334,10337],{},[4867,10335,10336],{},"檢查",[4867,10338,10339],{},"為什麼",[4884,10341,10342,10350,10428],{},[4864,10343,10344,10347],{},[4889,10345,10346],{},"分母的相對誤差",[4889,10348,10349],{},"比分子更致命，因為它會整體縮放輸出",[4864,10351,10352,10425],{},[4889,10353,10354,10355],{},"特徵維度 ",[161,10356,10358,10376],{"className":10357},[164],[161,10359,10361],{"className":10360},[168],[170,10362,10363],{"xmlns":172},[174,10364,10365,10373],{},[177,10366,10367],{},[462,10368,10369,10371],{},[360,10370,1140],{},[360,10372,8629],{},[190,10374,10375],{"encoding":192},"d_\\phi",[161,10377,10379],{"className":10378,"ariaHidden":198},[197],[161,10380,10382,10385],{"className":10381},[202],[161,10383],{"className":10384,"style":7750},[206],[161,10386,10388,10391],{"className":10387},[211],[161,10389,1140],{"className":10390},[211,395],[161,10392,10394],{"className":10393},[223],[161,10395,10397,10417],{"className":10396},[227,546],[161,10398,10400,10414],{"className":10399},[231],[161,10401,10403],{"className":10402,"style":900},[235],[161,10404,10405,10408],{"style":556},[161,10406],{"className":10407,"style":243},[242],[161,10409,10411],{"className":10410},[247,248,249,250],[161,10412,8629],{"className":10413},[211,395,250],[161,10415,570],{"className":10416},[569],[161,10418,10420],{"className":10419},[231],[161,10421,10423],{"className":10422,"style":2379},[235],[161,10424],{},[4889,10426,10427],{},"太少則近似粗糙，太多則省不下計算",[4864,10429,10430,10433],{},[4889,10431,10432],{},"序列長度分佈",[4889,10434,10435],{},"短序列幾乎沒有收益，長序列才值得換",[10437,10438,10442],"pre",{"className":10439,"code":10440,"language":10441,"meta":5869,"style":5869},"language-python shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","# 概念示意：先把 K、V 的累加量算好\nkv = einsum(\"m d, m e -> d e\", phi(K), V)   # O(m d^2)\nz  = phi(K).sum(dim=0)                       # O(m d)\nout = einsum(\"n d, d e -> n e\", phi(Q), kv) \u002F einsum(\"n d, d -> n\", phi(Q), z)\n","python",[10443,10444,10445,10452,10457,10462],"code",{"__ignoreMap":5869},[161,10446,10449],{"class":10447,"line":10448},"line",1,[161,10450,10451],{},"# 概念示意：先把 K、V 的累加量算好\n",[161,10453,10454],{"class":10447,"line":5870},[161,10455,10456],{},"kv = einsum(\"m d, m e -> d e\", phi(K), V)   # O(m d^2)\n",[161,10458,10459],{"class":10447,"line":5876},[161,10460,10461],{},"z  = phi(K).sum(dim=0)                       # O(m d)\n",[161,10463,10465],{"class":10447,"line":10464},4,[161,10466,10467],{},"out = einsum(\"n d, d e -> n e\", phi(Q), kv) \u002F einsum(\"n d, d -> n\", phi(Q), z)\n",[108,10469,10470],{"id":10470},"給下一個要調這個模型的人",[116,10472,10473],{},"如果你正在為長序列煩惱，我會建議先把 attention 當成核平滑器看一遍，再決定要不要換模型。",[116,10475,10476,10477,10480],{},"因為換成 linear attention 不是「更快的 attention」，而是",[112,10478,10479],{},"用特徵映射換掉精確的歸一化","——你付出的是準確度，不是免費的午餐。",[116,10482,10483],{},"知道自己在付什麼，比知道自己在用什麼重要。",[10485,10486,10487],"style",{},"html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":5869,"searchDepth":5870,"depth":5870,"links":10489},[10490,10491,10492,10493,10494,10495],{"id":5930,"depth":5870,"text":5930},{"id":5939,"depth":5870,"text":5939},{"id":8560,"depth":5870,"text":8560},{"id":9889,"depth":5870,"text":9889},{"id":10327,"depth":5870,"text":10327},{"id":10470,"depth":5870,"text":10470},"\u002Fimages\u002Fblog\u002Fattention-kernel.svg","Attention 與核函數對應關係的示意圖","2026-10-20","第一次看到「attention 是核方法」這個說法時，我以為那只是類比。自己推過一次才發現不是——而這件事直接決定了我們該在哪裡砍計算量。",{},"\u002Fblog\u002Fzh-tw\u002Fattention-as-kernel",{"title":5925,"description":10499},"attention-as-kernel","blog\u002Fzh-TW\u002Fattention-as-kernel",[5918,10506,10507],"數學","推論","2026-10-22","ipCLaHTj0DVQiZoXlJ6DPzX7hl68udhNuMH4wa4gke8",{"id":10511,"title":10512,"author":103,"body":10513,"cover":10883,"coverAlt":10512,"date":10884,"description":10885,"draft":5907,"extension":5908,"featured":5907,"lang":5909,"meta":10886,"navigation":5911,"path":10887,"seo":10888,"slug":10889,"stem":10890,"tags":10891,"translationKey":10889,"updated":5920,"__hash__":10894},"blog\u002Fblog\u002Fzh-TW\u002Farxiv-2510-27688v1.md","連續自迴歸語言模型 CALM 技術解析",{"type":105,"value":10514,"toc":10873},[10515,10525,10531,10537,10543,10547,10550,10553,10567,10573,10577,10583,10589,10593,10649,10658,10662,10713,10733,10740,10754,10758,10761,10793,10797,10823,10827,10850,10865],[116,10516,10517,10520,10521,10524],{},[112,10518,10519],{},"研究單位","：騰訊微信 AI × 清華大學\n",[112,10522,10523],{},"核心命題","：傳統大模型逐字（Token-by-token）生成的效率已觸及物理天花板。CALM 將離散 Token 轉換為連續向量（Continuous Vectors），將生成步數縮減至原來的 1\u002FK 為大語言模型的擴充開闢了全新的效率路徑。",[116,10526,10527],{},[2268,10528,10529],{"href":10529,"rel":10530},"https:\u002F\u002Farxiv.org\u002Fhtml\u002F2510.27688v1",[5860],[116,10532,10533],{},[2268,10534,10535],{"href":10535,"rel":10536},"https:\u002F\u002Fshaochenze.github.io\u002Fblog\u002F2025\u002FCALM\u002F",[5860],[116,10538,10539],{},[2268,10540,10541],{"href":10541,"rel":10542},"https:\u002F\u002Fgithub.com\u002Fmahmoodlab\u002Fclam",[5860],[108,10544,10546],{"id":10545},"一背景與動機離散生成的效率瓶頸","一、背景與動機：離散生成的效率瓶頸",[116,10548,10549],{},"現有的大型語言模型（LLMs）都依賴「逐字預測」的舊範式。每個 Token 都需要進行一次完整的前向計算，當序列變長時，計算成本呈線性甚至平方級增長。",[116,10551,10552],{},"儘管文本表徵已經從字符集進化到子詞集（如 BPE），但這種「離散表示」已觸及物理極限：",[1786,10554,10555,10561],{},[148,10556,10557,10560],{},[112,10558,10559],{},"詞表膨脹的詛咒","：為了涵蓋更多語義，詞表規模從 32,000 膨脹至 256,000，導致最後一層的計算與記憶體成本飆升。",[148,10562,10563,10566],{},[112,10564,10565],{},"資訊密度極低","：每個離散詞元僅攜帶約 15–18 bit 的資訊，潛力已然耗盡。模型擁有龐大的算力，卻被困在一次只能預測一個低資訊量 Token 的勞力密集工作中。",[116,10568,10569,10572],{},[112,10570,10571],{},"結論","：必須轉向高維連續路線，從根本上改變生成機制。",[108,10574,10576],{"id":10575},"二calm-核心思路與原理","二、CALM 核心思路與原理",[116,10578,10579,10582],{},[112,10580,10581],{},"基本機制","：利用自編碼器(Autoencoder)將 K 個離散 Token 壓縮為「一個」高維連續向量。這使得模型的序列長度從 T 降至 T\u002FK，生成步數直接縮減為原來的 K 分之一。",[116,10584,10585,10588],{},[112,10586,10587],{},"本質","：以更高的「語義頻寬」(Semantic Bandwidth)換取更少的生成步驟，實現算力與速度的跨越式突破。",[132,10590,10592],{"id":10591},"傳統範式-vs-calm-連續模型","傳統範式 vs. CALM 連續模型",[4858,10594,10595,10608],{},[4861,10596,10597],{},[4864,10598,10599,10602,10605],{},[4867,10600,10601],{},"維度",[4867,10603,10604],{},"傳統離散模型 (Discrete)",[4867,10606,10607],{},"CALM 連續模型 (Continuous)",[4884,10609,10610,10623,10636],{},[4864,10611,10612,10617,10620],{},[4889,10613,10614],{},[112,10615,10616],{},"擴容方式",[4889,10618,10619],{},"詞表指數級膨脹，算力爆炸",[4889,10621,10622],{},"增加向量維度，算力平滑增長",[4864,10624,10625,10630,10633],{},[4889,10626,10627],{},[112,10628,10629],{},"計算目標",[4889,10631,10632],{},"從數十萬詞中選一個機率",[4889,10634,10635],{},"在連續空間中直接預測精確座標",[4864,10637,10638,10643,10646],{},[4889,10639,10640],{},[112,10641,10642],{},"序列長度",[4889,10644,10645],{},"極長序列 T",[4889,10647,10648],{},"短序列 T\u002FK，步數銳減",[10650,10651,10652],"blockquote",{},[116,10653,10654,10657],{},[112,10655,10656],{},"通俗比喻","：傳統生成就像在「擁有數十萬個抽屜的倉庫中盲目尋寶」，而 CALM 則是「直接繪製高維座標」，徹底擺脫了詞表膨脹的物理限制。",[108,10659,10661],{"id":10660},"三calm-的四大技術支柱","三、CALM 的四大技術支柱",[145,10663,10664,10699,10708],{},[148,10665,10666,10669,10670,10698],{},[112,10667,10668],{},"高保真自編碼器 (Autoencoder)","\n將 ",[161,10671,10673,10686],{"className":10672},[164],[161,10674,10676],{"className":10675},[168],[170,10677,10678],{"xmlns":172},[174,10679,10680,10684],{},[177,10681,10682],{},[360,10683,4298],{},[190,10685,4298],{"encoding":192},[161,10687,10689],{"className":10688,"ariaHidden":198},[197],[161,10690,10692,10695],{"className":10691},[202],[161,10693],{"className":10694,"style":2538},[206],[161,10696,4298],{"className":10697,"style":4318},[211,395]," 個離散詞元壓縮為 128 或更高維度的連續向量。在約 7500 萬參數的輕量級開銷下，重構準確率高達 > 99.9%。它是實現離散與連續間無損轉換的基石。",[148,10700,10701,10704,10705,2180],{},[112,10702,10703],{},"能量生成頭與無似然框架","\n摒棄了傳統的 Softmax 機率計算，在連續空間中直接透過迴歸（Regression）預測下一個目標向量，無需像擴散模型（Diffusion）那樣進行數十上百次的去噪迭代，實現",[112,10706,10707],{},"單步生成",[148,10709,10710],{},[112,10711,10712],{},"連續空間魯棒性三大魔法",[1786,10714,10715,10721,10727],{},[148,10716,10717,10720],{},[112,10718,10719],{},"變分正則化 (Variational Regularization)","：熨平潛空間（Latent space）中的褶皺與斷層。",[148,10722,10723,10726],{},[112,10724,10725],{},"KL 裁減 (KL Clipping)","：防止潛空間維度嚴重塌陷。",[148,10728,10729,10732],{},[112,10730,10731],{},"雙重 Dropout","：提供額外的抗噪能力。\n這三者共同構建了平滑且富有彈性的高維地形，確保微小的生成偏差不會導致亂碼。",[145,10734,10735],{"start":10464},[148,10736,10737],{},[112,10738,10739],{},"無機率評估體系 BrierLM & 溫度控制",[1786,10741,10742,10748],{},[148,10743,10744,10747],{},[112,10745,10746],{},"Brier 評估 (Brier Score)","：傳統的困惑度 (Perplexity) 在連續模型中失效。CALM 引入了無似然的 Brier 分數（準確率 − 碰撞率）作為專屬標尺，與交叉熵損失呈極強負相關。",[148,10749,10750,10753],{},[112,10751,10752],{},"黑盒溫度採樣","：因為沒有傳統機率分母可調，CALM 以「批量大小 N」(Batch Size) 作為新型的溫度旋鈕。N 小則低溫高確定性，N 大則高溫高多樣性。搭配拒絕採樣機制，效果與傳統溫度調節高度一致。",[108,10755,10757],{"id":10756},"四流程總覽","四、流程總覽",[116,10759,10760],{},"整個運作閉環如下，速度遠超傳統的逐字生成：",[145,10762,10763,10769,10775,10781,10787],{},[148,10764,10765,10768],{},[112,10766,10767],{},"輸入","：離散 Token 序列。",[148,10770,10771,10774],{},[112,10772,10773],{},"壓縮","：編碼器將每 K 個 Token 壓縮為 1 個連續向量。",[148,10776,10777,10780],{},[112,10778,10779],{},"處理","：Transformer 骨幹網絡在極短的連續向量序列上進行深層注意力計算。",[148,10782,10783,10786],{},[112,10784,10785],{},"預測","：生成頭直接單步預測出下一個目標連續向量。",[148,10788,10789,10792],{},[112,10790,10791],{},"還原","：解碼器將預測出的連續向量無損還原為 K 個離散 Token。",[108,10794,10796],{"id":10795},"五實驗結果與性能優勢","五、實驗結果與性能優勢",[1786,10798,10799,10805,10811,10817],{},[148,10800,10801,10804],{},[112,10802,10803],{},"極致的算力節省","：在 K=4 的設置下，訓練算力降低 44%，推理算力降低 34%，生成步數從 T 驟降至 T\u002F4。",[148,10806,10807,10810],{},[112,10808,10809],{},"打破「更大必定更耗算力」的認知","：CALM 參數達 3.71 億（略大於對照組），卻能在打平甚至超越基線性能的情況下，消耗更少的整體算力。",[148,10812,10813,10816],{},[112,10814,10815],{},"全新縮放維度「資訊密度」","：引入了全新的 Scaling 維度。當語義頻寬提高時，CALM 能以更低的算力達到更高的 Brier 分數，擊穿了原有的算力-性能邊界。",[148,10818,10819,10822],{},[112,10820,10821],{},"「後積薄發」的訓練動態","：呈現特殊的訓練曲線。模型在 20 萬步後性能會迎來急劇上升並超越傳統模型，顯示掌握高維連續規律需要前期積累，但後續爆發力極強。",[108,10824,10826],{"id":10825},"六總結與展望","六、總結與展望",[116,10828,10829,10830,10833,10834,10837,10838,10841,10842,10845,10846,10849],{},"CALM 成功構建了從離散到連續的完整基礎設施，其組件協同形成了一個強大的生態閉環：\n",[112,10831,10832],{},"魯棒自編碼器"," (高保真壓縮) → ",[112,10835,10836],{},"頻寬倍增"," (步數縮短至 1\u002FK) → ",[112,10839,10840],{},"單步生成引擎"," (極速推理) → ",[112,10843,10844],{},"拒絕採樣"," (多樣性可調) → ",[112,10847,10848],{},"BrierLM"," (公正尺標)。",[116,10851,10852,10853,10856,10857,10860,10861,10864],{},"這項研究證明了語言模型的未來可能不在離散的像素與詞表中，而是在連續的波瀾裡。這為未來的",[112,10854,10855],{},"多模態連續生成","、",[112,10858,10859],{},"強化學習整合","以及",[112,10862,10863],{},"終端設備部署","開啟了全新的紀元。",[10650,10866,10867],{},[116,10868,10869,10872],{},[112,10870,10871],{},"開放資源","：相關論文與程式碼已在 GitHub 開源，標誌著大語言模型從離散向連續遷徙的開端。",{"title":5869,"searchDepth":5870,"depth":5870,"links":10874},[10875,10876,10879,10880,10881,10882],{"id":10545,"depth":5870,"text":10546},{"id":10575,"depth":5870,"text":10576,"children":10877},[10878],{"id":10591,"depth":5876,"text":10592},{"id":10660,"depth":5870,"text":10661},{"id":10756,"depth":5870,"text":10757},{"id":10795,"depth":5870,"text":10796},{"id":10825,"depth":5870,"text":10826},"\u002Fimages\u002Fblog\u002F2510.27688v1.webp","2026-08-06","傳統大模型逐字（Token-by-token）生成的效率已觸及物理天花板。CALM 將離散 Token 轉換為連續向量（Continuous Vectors），將生成步數縮減至原來的 1\u002FK，為大語言模型的擴充開闢了全新的效率路徑。",{},"\u002Fblog\u002Fzh-tw\u002Farxiv-2510-27688v1",{"title":10512,"description":10885},"arxiv-2510-27688v1","blog\u002Fzh-TW\u002Farxiv-2510-27688v1",[5917,10892,10893],"llm","efficiency","YXovptigvOJLPbjjtVgdCae3S6_ZcSZxJw3mH-BUBnc",1792775651438]