[{"data":1,"prerenderedAt":563},["ShallowReactive",2],{"article-other\u002Ftransformer_base":3},{"_path":4,"_dir":5,"_draft":6,"_partial":6,"_locale":7,"title":8,"description":9,"date":10,"tags":11,"body":13,"_type":557,"_id":558,"_source":559,"_file":560,"_stem":561,"_extension":562},"\u002Farticles\u002Fother\u002Ftransformer_base","other",false,"","Transformer 基础：从自注意力到大语言模型与 Agent","梳理 Transformer 的三种架构、Token 与位置编码、Self-Attention 和多头注意力，并说明它如何支撑大语言模型，以及与 Agent 编排层的边界。","2026-08-20",[12],"随笔",{"type":14,"children":15,"toc":541},"root",[16,24,34,40,72,78,98,103,145,150,164,169,187,192,199,204,219,233,239,244,262,276,282,287,305,319,325,343,349,354,367,372,390,396,402,407,412,451,456,461,466,471,476,481,504,535],{"type":17,"tag":18,"props":19,"children":21},"element","h2",{"id":20},"问题",[22],{"type":23,"value":20},"text",{"type":17,"tag":25,"props":26,"children":27},"ol",{},[28],{"type":17,"tag":29,"props":30,"children":31},"li",{},[32],{"type":23,"value":33},"模型都无到有的过程",{"type":17,"tag":18,"props":35,"children":37},{"id":36},"两个重要组成",[38],{"type":23,"value":39},"两个重要组成：",{"type":17,"tag":41,"props":42,"children":43},"ul",{},[44,67],{"type":17,"tag":29,"props":45,"children":46},{},[47,49],{"type":23,"value":48},"编码器(Encoder)",{"type":17,"tag":25,"props":50,"children":51},{},[52,57,62],{"type":17,"tag":29,"props":53,"children":54},{},[55],{"type":23,"value":56},"每个 Token 可以同时看到前后文",{"type":17,"tag":29,"props":58,"children":59},{},[60],{"type":23,"value":61},"擅长理解文本",{"type":17,"tag":29,"props":63,"children":64},{},[65],{"type":23,"value":66},"常用于文本分类、实体识别、Embedding",{"type":17,"tag":29,"props":68,"children":69},{},[70],{"type":23,"value":71},"解码器（Decoder）\n现在主流大模型都是基于transformer架构，本质上就是：根据前面所有的token，预测下一个token的概率分布。模型能够写代码、调用工具、进行对话，都是在这个基础能力上涌现出来的。",{"type":17,"tag":73,"props":74,"children":75},"p",{},[76],{"type":23,"value":77},"假设输入：",{"type":17,"tag":79,"props":80,"children":83},"pre",{"className":81,"code":82,"language":23,"meta":7,"style":7},"language-text shiki shiki-themes github-dark","中国的首都是\n",[84],{"type":17,"tag":85,"props":86,"children":87},"code",{"__ignoreMap":7},[88],{"type":17,"tag":89,"props":90,"children":93},"span",{"class":91,"line":92},"line",1,[94],{"type":17,"tag":89,"props":95,"children":96},{},[97],{"type":23,"value":82},{"type":17,"tag":73,"props":99,"children":100},{},[101],{"type":23,"value":102},"模型计算下一个 Token 的概率：",{"type":17,"tag":79,"props":104,"children":106},{"className":81,"code":105,"language":23,"meta":7,"style":7},"北京：0.91\n上海：0.03\n南京：0.02\n其他：0.04\n",[107],{"type":17,"tag":85,"props":108,"children":109},{"__ignoreMap":7},[110,118,127,136],{"type":17,"tag":89,"props":111,"children":112},{"class":91,"line":92},[113],{"type":17,"tag":89,"props":114,"children":115},{},[116],{"type":23,"value":117},"北京：0.91\n",{"type":17,"tag":89,"props":119,"children":121},{"class":91,"line":120},2,[122],{"type":17,"tag":89,"props":123,"children":124},{},[125],{"type":23,"value":126},"上海：0.03\n",{"type":17,"tag":89,"props":128,"children":130},{"class":91,"line":129},3,[131],{"type":17,"tag":89,"props":132,"children":133},{},[134],{"type":23,"value":135},"南京：0.02\n",{"type":17,"tag":89,"props":137,"children":139},{"class":91,"line":138},4,[140],{"type":17,"tag":89,"props":141,"children":142},{},[143],{"type":23,"value":144},"其他：0.04\n",{"type":17,"tag":73,"props":146,"children":147},{},[148],{"type":23,"value":149},"选择“北京”后，把它追加到上下文：",{"type":17,"tag":79,"props":151,"children":153},{"className":81,"code":152,"language":23,"meta":7,"style":7},"中国的首都是北京\n",[154],{"type":17,"tag":85,"props":155,"children":156},{"__ignoreMap":7},[157],{"type":17,"tag":89,"props":158,"children":159},{"class":91,"line":92},[160],{"type":17,"tag":89,"props":161,"children":162},{},[163],{"type":23,"value":152},{"type":17,"tag":73,"props":165,"children":166},{},[167],{"type":23,"value":168},"然后继续预测下一个 Token。如此循环，直到：",{"type":17,"tag":41,"props":170,"children":171},{},[172,177,182],{"type":17,"tag":29,"props":173,"children":174},{},[175],{"type":23,"value":176},"生成结束 Token",{"type":17,"tag":29,"props":178,"children":179},{},[180],{"type":23,"value":181},"达到最大长度",{"type":17,"tag":29,"props":183,"children":184},{},[185],{"type":23,"value":186},"被系统停止",{"type":17,"tag":18,"props":188,"children":190},{"id":189},"三种常见架构",[191],{"type":23,"value":189},{"type":17,"tag":193,"props":194,"children":196},"h3",{"id":195},"encoder-only",[197],{"type":23,"value":198},"Encoder-only",{"type":17,"tag":73,"props":200,"children":201},{},[202],{"type":23,"value":203},"代表模型：BERT。\n特点：",{"type":17,"tag":41,"props":205,"children":206},{},[207,211,215],{"type":17,"tag":29,"props":208,"children":209},{},[210],{"type":23,"value":56},{"type":17,"tag":29,"props":212,"children":213},{},[214],{"type":23,"value":61},{"type":17,"tag":29,"props":216,"children":217},{},[218],{"type":23,"value":66},{"type":17,"tag":79,"props":220,"children":222},{"className":81,"code":221,"language":23,"meta":7,"style":7},"输入文本 → Encoder → 文本表示\n",[223],{"type":17,"tag":85,"props":224,"children":225},{"__ignoreMap":7},[226],{"type":17,"tag":89,"props":227,"children":228},{"class":91,"line":92},[229],{"type":17,"tag":89,"props":230,"children":231},{},[232],{"type":23,"value":221},{"type":17,"tag":193,"props":234,"children":236},{"id":235},"decoder-only",[237],{"type":23,"value":238},"Decoder-only",{"type":17,"tag":73,"props":240,"children":241},{},[242],{"type":23,"value":243},"代表模型：GPT、Llama、Qwen 等。\n特点：",{"type":17,"tag":41,"props":245,"children":246},{},[247,252,257],{"type":17,"tag":29,"props":248,"children":249},{},[250],{"type":23,"value":251},"只能看到当前位置之前的 Token",{"type":17,"tag":29,"props":253,"children":254},{},[255],{"type":23,"value":256},"根据已有内容预测下一个 Token",{"type":17,"tag":29,"props":258,"children":259},{},[260],{"type":23,"value":261},"当前大语言模型最常见的架构",{"type":17,"tag":79,"props":263,"children":265},{"className":81,"code":264,"language":23,"meta":7,"style":7},"输入 Token → Decoder → 预测下一个 Token\n",[266],{"type":17,"tag":85,"props":267,"children":268},{"__ignoreMap":7},[269],{"type":17,"tag":89,"props":270,"children":271},{"class":91,"line":92},[272],{"type":17,"tag":89,"props":273,"children":274},{},[275],{"type":23,"value":264},{"type":17,"tag":193,"props":277,"children":279},{"id":278},"encoder-decoder",[280],{"type":23,"value":281},"Encoder-Decoder",{"type":17,"tag":73,"props":283,"children":284},{},[285],{"type":23,"value":286},"代表模型：Transformer、T5、M6 等。\n特点：",{"type":17,"tag":41,"props":288,"children":289},{},[290,295,300],{"type":17,"tag":29,"props":291,"children":292},{},[293],{"type":23,"value":294},"Encoder 负责理解输入",{"type":17,"tag":29,"props":296,"children":297},{},[298],{"type":23,"value":299},"Decoder 根据 Encoder 的结果生成输出",{"type":17,"tag":29,"props":301,"children":302},{},[303],{"type":23,"value":304},"常用于翻译、摘要等任务",{"type":17,"tag":79,"props":306,"children":308},{"className":81,"code":307,"language":23,"meta":7,"style":7},"输入 → Encoder → 中间表示 → Decoder → 输出\n",[309],{"type":17,"tag":85,"props":310,"children":311},{"__ignoreMap":7},[312],{"type":17,"tag":89,"props":313,"children":314},{"class":91,"line":92},[315],{"type":17,"tag":89,"props":316,"children":317},{},[318],{"type":23,"value":307},{"type":17,"tag":18,"props":320,"children":322},{"id":321},"流程",[323],{"type":23,"value":324},"流程：",{"type":17,"tag":25,"props":326,"children":327},{},[328,333,338],{"type":17,"tag":29,"props":329,"children":330},{},[331],{"type":23,"value":332},"用分词工具，将句子拆分成最小的语义单位token",{"type":17,"tag":29,"props":334,"children":335},{},[336],{"type":23,"value":337},"每个token被编码为一个512维的词向量",{"type":17,"tag":29,"props":339,"children":340},{},[341],{"type":23,"value":342},"给每个词向量加一个位置编码，告诉模型这个token在句子中的位置",{"type":17,"tag":18,"props":344,"children":346},{"id":345},"self-attention自注意力机制",[347],{"type":23,"value":348},"Self-Attention(自注意力机制)",{"type":17,"tag":193,"props":350,"children":352},{"id":351},"核心思想",[353],{"type":23,"value":351},{"type":17,"tag":41,"props":355,"children":356},{},[357,362],{"type":17,"tag":29,"props":358,"children":359},{},[360],{"type":23,"value":361},"自己的理解：让每个词和其他词的关联关系，能用计算机的方式做出标记",{"type":17,"tag":29,"props":363,"children":364},{},[365],{"type":23,"value":366},"gpt的改正：Self-Attention 的核心思想，是让每个 Token 根据当前上下文，计算自己应该关注其他 Token 的程度，然后按 Attention 权重聚合相关信息，从而得到包含上下文语义的新表示。",{"type":17,"tag":73,"props":368,"children":369},{},[370],{"type":23,"value":371},"每个token会生成三个向量：",{"type":17,"tag":41,"props":373,"children":374},{},[375,380,385],{"type":17,"tag":29,"props":376,"children":377},{},[378],{"type":23,"value":379},"Query(查询向量)：我正在寻找什么",{"type":17,"tag":29,"props":381,"children":382},{},[383],{"type":23,"value":384},"Key(键向量)：我包含什么信息",{"type":17,"tag":29,"props":386,"children":387},{},[388],{"type":23,"value":389},"Value(值向量)：如果你关注我，我可以提供什么\n简称：Q、K、V",{"type":17,"tag":18,"props":391,"children":393},{"id":392},"简单理解先计算当前-token-应该关注谁再按关注程度汇总其他-token-的信息",[394],{"type":23,"value":395},"简单理解：先计算当前 Token 应该关注谁，再按关注程度汇总其他 Token 的信息。",{"type":17,"tag":18,"props":397,"children":399},{"id":398},"multi-head-attention多头注意力机制",[400],{"type":23,"value":401},"Multi-Head Attention(多头注意力机制)",{"type":17,"tag":73,"props":403,"children":404},{},[405],{"type":23,"value":406},"一个 Token 与其他 Token 可能存在多种关系。",{"type":17,"tag":73,"props":408,"children":409},{},[410],{"type":23,"value":411},"如：小明昨天在商店买了一个苹果。\n不同注意力头可能分别关注：",{"type":17,"tag":79,"props":413,"children":415},{"className":81,"code":414,"language":23,"meta":7,"style":7},"谁买的：小明\n什么时候买的：昨天\n在哪里买的：商店\n买了什么：苹果\n",[416],{"type":17,"tag":85,"props":417,"children":418},{"__ignoreMap":7},[419,427,435,443],{"type":17,"tag":89,"props":420,"children":421},{"class":91,"line":92},[422],{"type":17,"tag":89,"props":423,"children":424},{},[425],{"type":23,"value":426},"谁买的：小明\n",{"type":17,"tag":89,"props":428,"children":429},{"class":91,"line":120},[430],{"type":17,"tag":89,"props":431,"children":432},{},[433],{"type":23,"value":434},"什么时候买的：昨天\n",{"type":17,"tag":89,"props":436,"children":437},{"class":91,"line":129},[438],{"type":17,"tag":89,"props":439,"children":440},{},[441],{"type":23,"value":442},"在哪里买的：商店\n",{"type":17,"tag":89,"props":444,"children":445},{"class":91,"line":138},[446],{"type":17,"tag":89,"props":447,"children":448},{},[449],{"type":23,"value":450},"买了什么：苹果\n",{"type":17,"tag":18,"props":452,"children":454},{"id":453},"自己的理解",[455],{"type":23,"value":453},{"type":17,"tag":73,"props":457,"children":458},{},[459],{"type":23,"value":460},"我对 Transformer 的理解主要分为三个层面。",{"type":17,"tag":73,"props":462,"children":463},{},[464],{"type":23,"value":465},"首先，Transformer 是目前大语言模型的核心网络架构。它相比传统 RNN，不需要严格按照顺序逐个处理 Token，训练时可以并 行计算，并且更擅长 建模长距离的上下文关系。",{"type":17,"tag":73,"props":467,"children":468},{},[469],{"type":23,"value":470},"其次，它的核心是 Self-Attention。输入文本会先经过 Tokenization 进行分词将句子拆分为最小的语义单位token 和 Embedding 转化为向量，并加入位置信息。然后每个 Token 生成 Query、Key、Value，通过 Q 和 K 计算当前 Token 应该关注上下文中的哪些内容，再对对应的 Value 加权求和。Multi-Head Attention 则允许模型从不同角度学习语义、语法和上下文关系。Attention 之后还会经过前馈网络，并配合残差连接和 Layer Normalization，这些模块会堆叠很多层。",{"type":17,"tag":73,"props":472,"children":473},{},[474],{"type":23,"value":475},"在大语言模型中，GPT、DeepSeek 这类模型主要使用 Decoder-only Transformer，并通过 Causal Mask 保证生成当前 Token 时只能看到前面的内容。模型本质上是在根据已有上下文不断预测下一个 Token。",{"type":17,"tag":73,"props":477,"children":478},{},[479],{"type":23,"value":480},"放到 Agent 开发中，Transformer 是 LLM 内部负责理解和生成的核心；工具调用、状态管理、记忆和循环执行则属于模型外部的 Agent 编排层。模型只负责生成 tool call，真正执行工具的是 Agent 框架。",{"type":17,"tag":25,"props":482,"children":483},{},[484,489,494,499],{"type":17,"tag":29,"props":485,"children":486},{},[487],{"type":23,"value":488},"Q、K、V 是什么？\nQ 表示当前 Token 想寻找什么，K 表示其他 Token 可以用什么特征被匹配，V 是匹配成功后真正获取的信息。 视频中的解释，Q(当前次想要关注什么)，K(能为其他词提供什么，类似于标签或者索引)，V(该词实际包含的信息内容)",{"type":17,"tag":29,"props":490,"children":491},{},[492],{"type":23,"value":493},"为什么需要位置编码？\nAttention 本身不天然理解词序，需要额外的位置信息区分“我打你”和“你打我”。",{"type":17,"tag":29,"props":495,"children":496},{},[497],{"type":23,"value":498},"Transformer 有什么缺点？\n标准 Attention 的时间和显存开销通常随序列长度呈平方增长，因此长上下文成本较高。",{"type":17,"tag":29,"props":500,"children":501},{},[502],{"type":23,"value":503},"Transformer 和 RAG、Agent 是什么关系？\nTransformer 是模型内部结构；RAG 给模型补充外部知识；Agent 给模型增加工具、记忆和执行循环。",{"type":17,"tag":79,"props":505,"children":507},{"className":81,"code":506,"language":23,"meta":7,"style":7},"Transformer →  模型内部结构\nLLM         →  基于 Transformer 训练出的语言模型\nAgent       →  LLM + 工具 + 记忆 + 执行循环(LLM + Herness)\n",[508],{"type":17,"tag":85,"props":509,"children":510},{"__ignoreMap":7},[511,519,527],{"type":17,"tag":89,"props":512,"children":513},{"class":91,"line":92},[514],{"type":17,"tag":89,"props":515,"children":516},{},[517],{"type":23,"value":518},"Transformer →  模型内部结构\n",{"type":17,"tag":89,"props":520,"children":521},{"class":91,"line":120},[522],{"type":17,"tag":89,"props":523,"children":524},{},[525],{"type":23,"value":526},"LLM         →  基于 Transformer 训练出的语言模型\n",{"type":17,"tag":89,"props":528,"children":529},{"class":91,"line":129},[530],{"type":17,"tag":89,"props":531,"children":532},{},[533],{"type":23,"value":534},"Agent       →  LLM + 工具 + 记忆 + 执行循环(LLM + Herness)\n",{"type":17,"tag":536,"props":537,"children":538},"style",{},[539],{"type":23,"value":540},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}",{"title":7,"searchDepth":120,"depth":120,"links":542},[543,544,545,550,551,554,555,556],{"id":20,"depth":120,"text":20},{"id":36,"depth":120,"text":39},{"id":189,"depth":120,"text":189,"children":546},[547,548,549],{"id":195,"depth":129,"text":198},{"id":235,"depth":129,"text":238},{"id":278,"depth":129,"text":281},{"id":321,"depth":120,"text":324},{"id":345,"depth":120,"text":348,"children":552},[553],{"id":351,"depth":129,"text":351},{"id":392,"depth":120,"text":395},{"id":398,"depth":120,"text":401},{"id":453,"depth":120,"text":453},"markdown","content:articles:other:transformer_base.md","content","articles\u002Fother\u002Ftransformer_base.md","articles\u002Fother\u002Ftransformer_base","md",1789465114557]