import torch import torch.nn as nn import math class SelfAttention(nn.Module): def __init__(self, d_model, n_heads): super().__init__() self.d_k = d_model // n_heads self.n_heads = n_heads self.W_q = nn.Linear(d_model, d_model) self.W_k = nn.Linear(d_model, d_model) self.W_v = nn.Linear(d_model, d_model) self.W_o = nn.Linear(d_model, d_model) def forward(self, x): B, L, D = x.shape # 生成 Q, K, V Q = self.W_q(x).view(B,L,self.n_heads,self.d_k) K = self.W_k(x).view(B,L,self.n_heads,self.d_k) V = self.W_v(x).view(B,L,self.n_heads,self.d_k) # 转置: (B, heads, L, d_k) Q, K, V = [t.transpose(1,2) for t in (Q,K,V)] # 注意力计算 scores = Q @ K.transpose(-2,-1) / math.sqrt(self.d_k) attn = torch.softmax(scores, dim=-1) out = (attn @ V).transpose(1,2).reshape(B,L,D) return self.W_o(out)

# 文本分类 (BERT) from transformers import pipeline classifier = pipeline("sentiment-analysis") result = classifier("I love transformers!") # [{'label':'POSITIVE','score':0.99}] # 文本生成 (GPT-2) generator = pipeline("text-generation", model="gpt2") text = generator("Transformers are", max_length=50) # 机器翻译 translator = pipeline("translation_en_to_zh", model="Helsinki-NLP/opus-mt-en-zh") result = translator("Hello world!") # 问答系统 qa = pipeline("question-answering") answer = qa( question="What is Transformer?", context="Transformer is a neural network architecture..." )