完整项目???????
本篇博客,主要介绍各个模型的模块定义,包括模型本身的定义以及模型对应的配置(超参数)的定义,二者在一个模块文件中。
目录
1. FastText
2. TextCNN
3. TextRNN
4. TextRCNN
5. TextRNN_Atten
6. DPCNN
7. Transformer
1. FastText
- 配置类
class Config(object):"""FastText配置参数"""def __init__(self, dataset, embedding):self.model_name = 'FastText'#训练集、验证集、测试集路径self.train_path = dataset + '/data/train.txt'self.dev_path = dataset + '/data/dev.txt'self.test_path = dataset + '/data/test.txt'#数据集的所有类别self.class_list = [x.strip() for x in open(dataset + '/data/class.txt').readlines()]#构建好的词/字典路径self.vocab_path = dataset + '/data/vocab.pkl'#训练好的模型参数保存路径self.save_path = dataset + '/saved_dict/' + self.model_name + '.ckpt'#模型日志保存路径self.log_path = dataset + '/log/' + self.model_name#如果词/字嵌入矩阵不随机初始化 则加载初始化好的词/字嵌入矩阵 类别为float32 并转换为tensor 否则为Noneself.embedding_pretrained = torch.tensor(np.load(dataset + '/data/' + embedding)["embeddings"].astype('float32'))\if embedding != 'random' else Noneself.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # 设备self.dropout = 0.5 # 随机失活 丢弃率self.require_improvement = 1000 # 若超过1000batch效果还没提升,则提前结束训练self.num_classes = len(self.class_list) # 类别数self.n_vocab = 0 # 词表大小,在运行时赋值self.num_epochs = 20 # epoch数self.batch_size = 128 # mini-batch大小self.pad_size = 32 # 每句话处理成的长度(短填长切)self.learning_rate = 1e-3 # 学习率self.embed = self.embedding_pretrained.size(1)\if self.embedding_pretrained is not None else 300 # 字向量维度self.hidden_size = 256 # 隐藏层大小self.n_gram_vocab = 250499 #n-gram词表大小
- 模型定义类
class Model(nn.Module):def __init__(self, config):super(Model, self).__init__()if config.embedding_pretrained is not None: #加载初始化好的预训练词/字嵌入矩阵 微调funetuningself.embedding = nn.Embedding.from_pretrained(config.embedding_pretrained, freeze=False)else: #否则随机初始化词/字嵌入矩阵 指定填充对应的索引self.embedding = nn.Embedding(config.n_vocab, config.embed, padding_idx=config.n_vocab - 1)#分别随机初始化 bi-gram tri-gram对应的词嵌入矩阵 self.embedding_ngram2 = nn.Embedding(config.n_gram_vocab, config.embed)self.embedding_ngram3 = nn.Embedding(config.n_gram_vocab, config.embed)#dropoutself.dropout = nn.Dropout(config.dropout)#隐层self.fc1 = nn.Linear(config.embed * 3, config.hidden_size)# self.dropout2 = nn.Dropout(config.dropout)#输出层self.fc2 = nn.Linear(config.hidden_size, config.num_classes)def forward(self, x):#x (uni-gram,seq_len,bi-gram,tri-gram)#基于uni-gram、bi-gram、tri-gram对应的索引 在各自的词嵌入矩阵中查询 得到词嵌入#(batch,seq_len,embed)out_word = self.embedding(x[0]) out_bigram = self.embedding_ngram2(x[2])out_trigram = self.embedding_ngram3(x[3])#三种嵌入进行拼接 (batch,seq,embed*3)out = torch.cat((out_word, out_bigram, out_trigram), -1)#沿长度维 作平均 (batch,embed*3)out = out.mean(dim=1)#通过fropoutout = self.dropout(out)#通过隐层 (batch,hidden_size)out = self.fc1(out)out = F.relu(out)#输出层 (batch,classes)out = self.fc2(out)return out
2. TextCNN
- 配置类
class Config(object):"""TextCNN配置参数"""def __init__(self, dataset, embedding):self.model_name = 'TextCNN'# 训练集、验证集、测试集路径self.train_path = dataset + '/data/train.txt'self.dev_path = dataset + '/data/dev.txt'self.test_path = dataset + '/data/test.txt'# 数据集的所有类别self.class_list = [x.strip() for x in open(dataset + '/data/class.txt').readlines()]# 构建好的词/字典路径self.vocab_path = dataset + '/data/vocab.pkl'# 训练好的模型参数保存路径self.save_path = dataset + '/saved_dict/' + self.model_name + '.ckpt'# 模型日志保存路径self.log_path = dataset + '/log/' + self.model_name# 如果词/字嵌入矩阵不随机初始化 则加载初始化好的词/字嵌入矩阵 类别为float32 并转换为tensor 否则为Noneself.embedding_pretrained = torch.tensor(np.load(dataset + '/data/' + embedding)["embeddings"].astype('float32')) \if embedding != 'random' else Noneself.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # 设备self.dropout = 0.5 # 随机失活self.require_improvement = 1000 # 若超过1000batch效果还没提升,则提前结束训练self.num_classes = len(self.class_list) # 类别数self.n_vocab = 0 # 词表大小,在运行时赋值self.num_epochs = 20 # epoch数self.batch_size = 128 # mini-batch大小self.pad_size = 32 # 每句话处理成的长度(短填长切)self.learning_rate = 1e-3 # 学习率self.embed = self.embedding_pretrained.size(1)\if self.embedding_pretrained is not None else 300 # 字向量维度self.filter_sizes = (2, 3, 4) # 不同大小卷积核尺寸self.num_filters = 256 # 每种卷积核数量
- 模型定义类
class Model(nn.Module):def __init__(self, config):super(Model, self).__init__()if config.embedding_pretrained is not None: # 加载初始化好的预训练词/字嵌入矩阵 微调funetuningself.embedding = nn.Embedding.from_pretrained(config.embedding_pretrained, freeze=False)else: # 否则随机初始化词/字嵌入矩阵 指定填充对应的索引self.embedding = nn.Embedding(config.n_vocab, config.embed, padding_idx=config.n_vocab - 1)#不同大小卷积核对应的卷积操作self.convs = nn.ModuleList([nn.Conv2d(1, config.num_filters, (k, config.embed)) for k in config.filter_sizes])self.dropout = nn.Dropout(config.dropout)self.fc = nn.Linear(config.num_filters * len(config.filter_sizes), config.num_classes)def conv_and_pool(self, x, conv):x = F.relu(conv(x)).squeeze(3) #(batch,num_filters,height)x = F.max_pool1d(x, x.size(2)).squeeze(2) #(batch,num_filters) 全局最大池化return xdef forward(self, x):out = self.embedding(x[0]) #(batch,seq) -> (batch,seq,embed)out = out.unsqueeze(1) #添加通道维 (batch,1,seq,embed)#通过不同大小的卷积核提取特征 并对池化结果进行拼接 (batch,num_filters*len(filter_size))out = torch.cat([self.conv_and_pool(out, conv) for conv in self.convs], 1)out = self.dropout(out)out = self.fc(out) #(batch,classes)return out
3. TextRNN
- 配置类
class Config(object):"""配置参数"""def __init__(self, dataset, embedding):self.model_name = 'TextRNN'# 训练集、验证集、测试集路径self.train_path = dataset + '/data/train.txt'self.dev_path = dataset + '/data/dev.txt'self.test_path = dataset + '/data/test.txt'# 数据集的所有类别self.class_list = [x.strip() for x in open(dataset + '/data/class.txt').readlines()]# 构建好的词/字典路径self.vocab_path = dataset + '/data/vocab.pkl'# 训练好的模型参数保存路径self.save_path = dataset + '/saved_dict/' + self.model_name + '.ckpt'# 模型日志保存路径self.log_path = dataset + '/log/' + self.model_name# 如果词/字嵌入矩阵不随机初始化 则加载初始化好的词/字嵌入矩阵 类别为float32 并转换为tensor 否则为Noneself.embedding_pretrained = torch.tensor(np.load(dataset + '/data/' + embedding)["embeddings"].astype('float32')) \if embedding != 'random' else Noneself.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # 设备self.dropout = 0.5 # 随机失活self.require_improvement = 1000 # 若超过1000batch效果还没提升,则提前结束训练self.num_classes = len(self.class_list) # 类别数self.n_vocab = 0 # 词表大小,在运行时赋值self.num_epochs = 10 # epoch数self.batch_size = 128 # mini-batch大小self.pad_size = 32 # 每句话处理成的长度(短填长切)self.learning_rate = 1e-3 # 学习率self.embed = self.embedding_pretrained.size(1)\if self.embedding_pretrained is not None else 300 # 字向量维度, 若使用了预训练词向量,则维度统一self.hidden_size = 128 # lstm隐藏单元数self.num_layers = 2 # lstm层数
- 模型定义类
class Model(nn.Module):def __init__(self, config):super(Model, self).__init__()if config.embedding_pretrained is not None: # 加载初始化好的预训练词/字嵌入矩阵 微调funetuningself.embedding = nn.Embedding.from_pretrained(config.embedding_pretrained, freeze=False)else: # 否则随机初始化词/字嵌入矩阵 指定填充对应的索引self.embedding = nn.Embedding(config.n_vocab, config.embed, padding_idx=config.n_vocab - 1)#2层双向lstm batch_size为第一维度self.lstm = nn.LSTM(config.embed, config.hidden_size, config.num_layers,bidirectional=True, batch_first=True, dropout=config.dropout)#输出层self.fc = nn.Linear(config.hidden_size * 2, config.num_classes)def forward(self, x):x, _ = x #(batch,SEQ_LEN)out = self.embedding(x) # [batch_size, seq_len, embeding]=[128, 32, 300]out, _ = self.lstm(out) #(batch_size,seq_len,hidden_size*2)out = self.fc(out[:, -1, :]) # 句子最后时刻的 hidden state (batch,hidden_size*2)->(batch,classes)return out'''变长RNN,效果差不多,甚至还低了点...'''# def forward(self, x):# x, seq_len = x# out = self.embedding(x)# _, idx_sort = torch.sort(seq_len, dim=0, descending=True) # 长度从长到短排序(index)# _, idx_unsort = torch.sort(idx_sort) # 排序后,原序列的 index# out = torch.index_select(out, 0, idx_sort)# seq_len = list(seq_len[idx_sort])# out = nn.utils.rnn.pack_padded_sequence(out, seq_len, batch_first=True)# # [batche_size, seq_len, num_directions * hidden_size]# out, (hn, _) = self.lstm(out)# out = torch.cat((hn[2], hn[3]), -1)# # out, _ = nn.utils.rnn.pad_packed_sequence(out, batch_first=True)# out = out.index_select(0, idx_unsort)# out = self.fc(out)# return out
4. TextRCNN
class Config(object):"""配置参数"""def __init__(self, dataset, embedding):self.model_name = 'TextRCNN'# 训练集、验证集、测试集路径self.train_path = dataset + '/data/train.txt'self.dev_path = dataset + '/data/dev.txt'self.test_path = dataset + '/data/test.txt'# 数据集的所有类别self.class_list = [x.strip() for x in open(dataset + '/data/class.txt').readlines()]# 构建好的词/字典路径self.vocab_path = dataset + '/data/vocab.pkl'# 训练好的模型参数保存路径self.save_path = dataset + '/saved_dict/' + self.model_name + '.ckpt'# 模型日志保存路径self.log_path = dataset + '/log/' + self.model_name# 如果词/字嵌入矩阵不随机初始化 则加载初始化好的词/字嵌入矩阵 类别为float32 并转换为tensor 否则为Noneself.embedding_pretrained = torch.tensor(np.load(dataset + '/data/' + embedding)["embeddings"].astype('float32')) \if embedding != 'random' else Noneself.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # 设备self.dropout = 1.0 # 随机失活self.require_improvement = 1000 # 若超过1000batch效果还没提升,则提前结束训练self.num_classes = len(self.class_list) # 类别数self.n_vocab = 0 # 词表大小,在运行时赋值self.num_epochs = 10 # epoch数self.batch_size = 128 # mini-batch大小self.pad_size = 32 # 每句话处理成的长度(短填长切)self.learning_rate = 1e-3 # 学习率self.embed = self.embedding_pretrained.size(1)\if self.embedding_pretrained is not None else 300 # 字向量维度, 若使用了预训练词向量,则维度统一self.hidden_size = 256 # lstm隐藏单元数self.num_layers = 1 # lstm层数
- 模型定义类
class Model(nn.Module):def __init__(self, config):super(Model, self).__init__()if config.embedding_pretrained is not None: # 加载初始化好的预训练词/字嵌入矩阵 微调funetuningself.embedding = nn.Embedding.from_pretrained(config.embedding_pretrained, freeze=False)else: # 否则随机初始化词/字嵌入矩阵 指定填充对应的索引self.embedding = nn.Embedding(config.n_vocab, config.embed, padding_idx=config.n_vocab - 1)#单层双向lstm batch_size为第一维度self.lstm = nn.LSTM(config.embed, config.hidden_size, config.num_layers,bidirectional=True, batch_first=True, dropout=config.dropout)self.maxpool = nn.MaxPool1d(config.pad_size) #沿长度方向作全局最大池化#输出层self.fc = nn.Linear(config.hidden_size * 2 + config.embed, config.num_classes)def forward(self, x):x, _ = x #(batch,seq_len)embed = self.embedding(x) # [batch_size, seq_len, embeding]=[64, 32, 64]out, _ = self.lstm(embed) #(batch_size,seq_len,hidden_size*2)out = torch.cat((embed, out), 2) #把词嵌入和lstm输出进行拼接 (batch,seq_len.embed+hidden_size*2)out = F.relu(out)out = out.permute(0, 2, 1) #(batch,embed+hidden_size*2,seq_len)out = self.maxpool(out).squeeze() #(batch,embed+hidden_size*2)out = self.fc(out) #(batch,classes)return out
5. TextRNN_Atten
- 配置类
class Config(object):"""配置参数"""def __init__(self, dataset, embedding):self.model_name = 'TextRNN_Att'# 训练集、验证集、测试集路径self.train_path = dataset + '/data/train.txt'self.dev_path = dataset + '/data/dev.txt'self.test_path = dataset + '/data/test.txt'# 数据集的所有类别self.class_list = [x.strip() for x in open(dataset + '/data/class.txt').readlines()]# 构建好的词/字典路径self.vocab_path = dataset + '/data/vocab.pkl'# 训练好的模型参数保存路径self.save_path = dataset + '/saved_dict/' + self.model_name + '.ckpt'# 模型日志保存路径self.log_path = dataset + '/log/' + self.model_name# 如果词/字嵌入矩阵不随机初始化 则加载初始化好的词/字嵌入矩阵 类别为float32 并转换为tensor 否则为Noneself.embedding_pretrained = torch.tensor(np.load(dataset + '/data/' + embedding)["embeddings"].astype('float32')) \if embedding != 'random' else Noneself.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # 设备self.dropout = 0.5 # 随机失活self.require_improvement = 1000 # 若超过1000batch效果还没提升,则提前结束训练self.num_classes = len(self.class_list) # 类别数self.n_vocab = 0 # 词表大小,在运行时赋值self.num_epochs = 10 # epoch数self.batch_size = 128 # mini-batch大小self.pad_size = 32 # 每句话处理成的长度(短填长切)self.learning_rate = 1e-3 # 学习率self.embed = self.embedding_pretrained.size(1)\if self.embedding_pretrained is not None else 300 # 字向量维度, 若使用了预训练词向量,则维度统一self.hidden_size = 128 # lstm隐藏单元数self.num_layers = 2 # lstm层数self.hidden_size2 = 64 #全连接层隐藏单元数
- 模型定义类
class Model(nn.Module):def __init__(self, config):super(Model, self).__init__()if config.embedding_pretrained is not None: # 加载初始化好的预训练词/字嵌入矩阵 微调funetuningself.embedding = nn.Embedding.from_pretrained(config.embedding_pretrained, freeze=False)else: # 否则随机初始化词/字嵌入矩阵 指定填充对应的索引self.embedding = nn.Embedding(config.n_vocab, config.embed, padding_idx=config.n_vocab - 1)#2层双向 LSTM batch_size为第一维度self.lstm = nn.LSTM(config.embed, config.hidden_size, config.num_layers,bidirectional=True, batch_first=True, dropout=config.dropout)self.tanh1 = nn.Tanh()# self.u = nn.Parameter(torch.Tensor(config.hidden_size * 2, config.hidden_size * 2))#定义一个参数向量 作为Queryself.w = nn.Parameter(torch.Tensor(config.hidden_size * 2))self.tanh2 = nn.Tanh()#隐层self.fc1 = nn.Linear(config.hidden_size * 2, config.hidden_size2)#输出层self.fc = nn.Linear(config.hidden_size2, config.num_classes)def forward(self, x):x, _ = x #(batch,seq_len)emb = self.embedding(x) # [batch_size, seq_len, embeding]=[128, 32, 300]H, _ = self.lstm(emb) # [batch_size, seq_len, hidden_size * num_direction]=[128, 32, 256] 各时刻隐状态 作为valueM = self.tanh1(H) # [128, 32, 256] 各时刻隐状态通过tanh激活函数 作为Key# M = torch.tanh(torch.matmul(H, self.u))#Key和Query作运算 在通过softmax 得到每个时刻对应的权重alpha = F.softmax(torch.matmul(M, self.w), dim=1).unsqueeze(-1) # [128, 32, 1]#各时刻的权重和各时刻的隐状态Value对应相乘out = H * alpha # [128, 32, 256]#再相加out = torch.sum(out, 1) # [128, 256] (batch,hidden*2)out = F.relu(out)out = self.fc1(out) #(batch,hidden2)out = self.fc(out) # (batch,classes)return out
6. DPCNN
- 配置类
class Config(object):"""配置参数"""def __init__(self, dataset, embedding):self.model_name = 'DPCNN'# 训练集、验证集、测试集路径self.train_path = dataset + '/data/train.txt'self.dev_path = dataset + '/data/dev.txt'self.test_path = dataset + '/data/test.txt'# 数据集的所有类别self.class_list = [x.strip() for x in open(dataset + '/data/class.txt').readlines()]# 构建好的词/字典路径self.vocab_path = dataset + '/data/vocab.pkl'# 训练好的模型参数保存路径self.save_path = dataset + '/saved_dict/' + self.model_name + '.ckpt'# 模型日志保存路径self.log_path = dataset + '/log/' + self.model_name# 如果词/字嵌入矩阵不随机初始化 则加载初始化好的词/字嵌入矩阵 类别为float32 并转换为tensor 否则为Noneself.embedding_pretrained = torch.tensor(np.load(dataset + '/data/' + embedding)["embeddings"].astype('float32')) \if embedding != 'random' else Noneself.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # 设备self.dropout = 0.5 # 随机失活self.require_improvement = 1000 # 若超过1000batch效果还没提升,则提前结束训练self.num_classes = len(self.class_list) # 类别数self.n_vocab = 0 # 词表大小,在运行时赋值self.num_epochs = 20 # epoch数self.batch_size = 128 # mini-batch大小self.pad_size = 32 # 每句话处理成的长度(短填长切)self.learning_rate = 1e-3 # 学习率self.embed = self.embedding_pretrained.size(1)\if self.embedding_pretrained is not None else 300 # 字/词向量维度self.num_filters = 250 # 卷积核数量(channels数)
- 模型定义类
class Model(nn.Module):def __init__(self, config):super(Model, self).__init__()if config.embedding_pretrained is not None: # 加载初始化好的预训练词/字嵌入矩阵 微调funetuningself.embedding = nn.Embedding.from_pretrained(config.embedding_pretrained, freeze=False)else: # 否则随机初始化词/字嵌入矩阵 指定填充对应的索引self.embedding = nn.Embedding(config.n_vocab, config.embed, padding_idx=config.n_vocab - 1)#region embedding 类似于TextCNN中的卷积操作self.conv_region = nn.Conv2d(1, config.num_filters, (3, config.embed), stride=1)self.conv = nn.Conv2d(config.num_filters, config.num_filters, (3, 1), stride=1)self.max_pool = nn.MaxPool2d(kernel_size=(3, 1), stride=2)self.padding1 = nn.ZeroPad2d((0, 0, 1, 1)) # top bottom 上下各添加1个0self.padding2 = nn.ZeroPad2d((0, 0, 0, 1)) # bottom 下添加一个0self.relu = nn.ReLU()self.fc = nn.Linear(config.num_filters, config.num_classes)def forward(self, x):x = x[0] #(batch,seq_len)x = self.embedding(x) #(batch,seq_len,embed)x = x.unsqueeze(1) # 添加通道维 进行2d卷积 (batch,1,seq_len,embed)x = self.conv_region(x) # (batch,num_filters,seq_len-3+1,1)#先卷积 再填充 等价于等长卷积 序列长度不变x = self.padding1(x) # [batch_size, num_filters, seq_len, 1]x = self.relu(x)x = self.conv(x) # [batch_size, num_filters, seq_len-3+1, 1]x = self.padding1(x) # [batch_size, num_filters, seq_len, 1]x = self.relu(x)x = self.conv(x) # [batch_size, num_filters, seq_len-3+1, 1]while x.size()[2] > 2: x = self._block(x)x = x.squeeze() # [batch_size, num_filters]x = self.fc(x) #(batch,classes)return xdef _block(self, x):x = self.padding2(x) #[batch_size, num_filters, seq_len-1, 1]#长度减半px = self.max_pool(x) #[batch_size, num_filters, (seq_len-1)/2, 1]#等长卷积 长度不变x = self.padding1(px)x = F.relu(x)x = self.conv(x)# 等长卷积 长度不变x = self.padding1(x)x = F.relu(x)x = self.conv(x)# Short Cutx = x + pxreturn x
7. Transformer
- 配置类
class Config(object):"""配置参数"""def __init__(self, dataset, embedding):self.model_name = 'Transformer'# 训练集、验证集、测试集路径self.train_path = dataset + '/data/train.txt'self.dev_path = dataset + '/data/dev.txt'self.test_path = dataset + '/data/test.txt'# 数据集的所有类别self.class_list = [x.strip() for x in open(dataset + '/data/class.txt').readlines()]# 构建好的词/字典路径self.vocab_path = dataset + '/data/vocab.pkl'# 训练好的模型参数保存路径self.save_path = dataset + '/saved_dict/' + self.model_name + '.ckpt'# 模型日志保存路径self.log_path = dataset + '/log/' + self.model_name# 如果词/字嵌入矩阵不随机初始化 则加载初始化好的词/字嵌入矩阵 类别为float32 并转换为tensor 否则为Noneself.embedding_pretrained = torch.tensor(np.load(dataset + '/data/' + embedding)["embeddings"].astype('float32')) \if embedding != 'random' else Noneself.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # 设备self.dropout = 0.5 # 随机失活self.require_improvement = 2000 # 若超过2000batch效果还没提升,则提前结束训练self.num_classes = len(self.class_list) # 类别数self.n_vocab = 0 # 词表大小,在运行时赋值self.num_epochs = 20 # epoch数self.batch_size = 128 # mini-batch大小self.pad_size = 32 # 每句话处理成的长度(短填长切)self.learning_rate = 5e-4 # 学习率self.embed = self.embedding_pretrained.size(1)\if self.embedding_pretrained is not None else 300 # 字向量维度self.dim_model = 300self.hidden = 1024self.last_hidden = 512self.num_head = 5 #5头注意力机制self.num_encoder = 2 #两个transformer encoder block
- 模型定义类
class Model(nn.Module):def __init__(self, config):super(Model, self).__init__()#词/字嵌入if config.embedding_pretrained is not None: # 加载初始化好的预训练词/字嵌入矩阵 微调funetuningself.embedding = nn.Embedding.from_pretrained(config.embedding_pretrained, freeze=False)else: # 否则随机初始化词/字嵌入矩阵 指定填充对应的索引self.embedding = nn.Embedding(config.n_vocab, config.embed, padding_idx=config.n_vocab - 1)#位置编码self.postion_embedding = Positional_Encoding(config.embed, config.pad_size, config.dropout, config.device)#transformer encoder blockself.encoder = Encoder(config.dim_model, config.num_head, config.hidden, config.dropout)#多个transformer encoder blockself.encoders = nn.ModuleList([copy.deepcopy(self.encoder)# Encoder(config.dim_model, config.num_head, config.hidden, config.dropout)for _ in range(config.num_encoder)])#输出层self.fc1 = nn.Linear(config.pad_size * config.dim_model, config.num_classes)# self.fc2 = nn.Linear(config.last_hidden, config.num_classes)# self.fc1 = nn.Linear(config.dim_model, config.num_classes)def forward(self, x):out = self.embedding(x[0]) #(batch,seq_len) -> (batch,seq_len,embed)out = self.postion_embedding(out) # 添加位置编码 (batch,seq_len,embed)for encoder in self.encoders: #通过多个ender blockout = encoder(out) #(batch,seq_len,dim_model)out = out.view(out.size(0), -1) #(batch,seq_len*dim_model)# out = torch.mean(out, 1)out = self.fc1(out) #(batch,classes)return outclass Encoder(nn.Module):def __init__(self, dim_model, num_head, hidden, dropout):super(Encoder, self).__init__()#多头注意力机制self.attention = Multi_Head_Attention(dim_model, num_head, dropout)#两个全连接层self.feed_forward = Position_wise_Feed_Forward(dim_model, hidden, dropout)def forward(self, x): #x (batch,seq_len,embed_size) embed_size = dim_modelout = self.attention(x) #计算多头注意力结果 (batch,seq_len,dim_model)out = self.feed_forward(out) #通过两个全连接层增加 非线性转换能力 (batch,seq_len,dim_model)return outclass Positional_Encoding(nn.Module):#位置编码def __init__(self, embed, pad_size, dropout, device):super(Positional_Encoding, self).__init__()self.device = device#利用sin cos生成绝对位置编码self.pe = torch.tensor([[pos / (10000.0 ** (i // 2 * 2.0 / embed)) for i in range(embed)] for pos in range(pad_size)])self.pe[:, 0::2] = np.sin(self.pe[:, 0::2])self.pe[:, 1::2] = np.cos(self.pe[:, 1::2])self.dropout = nn.Dropout(dropout)def forward(self, x): #token embedding + 绝对位置编码out = x + nn.Parameter(self.pe, requires_grad=False).to(self.device)#再通过dropoutout = self.dropout(out)return outclass Scaled_Dot_Product_Attention(nn.Module):'''Scaled Dot-Product Attention '''def __init__(self):super(Scaled_Dot_Product_Attention, self).__init__()def forward(self, Q, K, V, scale=None):'''Args:Q: [batch_size, len_Q, dim_Q]K: [batch_size, len_K, dim_K]V: [batch_size, len_V, dim_V]scale: 缩放因子 论文为根号dim_KReturn:self-attention后的张量,以及attention张量'''#Q与K的第2、3维转置计算内积 (batch*num_head,seq_len,seq_len)attention = torch.matmul(Q, K.permute(0, 2, 1))if scale: #作缩放 减小结果的方差 attention = attention * scale# if mask: # TODO change this # attention = attention.masked_fill_(mask == 0, -1e9)attention = F.softmax(attention, dim=-1) #转换为权重context = torch.matmul(attention, V) #再与V运算 得到结果 (batch*num_head,seq_len,dim_head)return contextclass Multi_Head_Attention(nn.Module):#多头注意力机制 encoder block的第一部分def __init__(self, dim_model, num_head, dropout=0.0):super(Multi_Head_Attention, self).__init__()self.num_head = num_head #头数 assert dim_model % num_head == 0 #必须整除self.dim_head = dim_model // self.num_head #分别通过三个Dense层 生成Q、K、Vself.fc_Q = nn.Linear(dim_model, num_head * self.dim_head)self.fc_K = nn.Linear(dim_model, num_head * self.dim_head)self.fc_V = nn.Linear(dim_model, num_head * self.dim_head)#Attention计算self.attention = Scaled_Dot_Product_Attention()self.fc = nn.Linear(num_head * self.dim_head, dim_model)self.dropout = nn.Dropout(dropout)#层归一化self.layer_norm = nn.LayerNorm(dim_model)def forward(self, x): #(batch,seq_len,dim_model)batch_size = x.size(0) # Q,K,V维度 (batch,seq_len,dim_head*num_head)Q = self.fc_Q(x) K = self.fc_K(x)V = self.fc_V(x)#沿第三个维度进行切分 切分为num_head份 再沿第一个维度拼接 多个注意力头并行计算#Q,K,V维度 (batch*num_head,seq_len,dim_head)Q = Q.view(batch_size * self.num_head, -1, self.dim_head) K = K.view(batch_size * self.num_head, -1, self.dim_head)V = V.view(batch_size * self.num_head, -1, self.dim_head)# if mask: # TODO# mask = mask.repeat(self.num_head, 1, 1) # TODO change thisscale = K.size(-1) ** -0.5 # 缩放因子 dim_head的开放取倒数 对内积结果进行缩放 减小结果的方差 有利于训练#attention计算 多个注意力头并行计算(矩阵运算)context = self.attention(Q, K, V, scale)#多头注意力计算结果 沿第一个维度进行切分 再沿第三个维度拼接 转为原来的维度(batch,seq_len,dim_head*num_head)context = context.view(batch_size, -1, self.dim_head * self.num_head)out = self.fc(context)#(batch,seq_len,dim_model)out = self.dropout(out)out = out + x # 残差连接out = self.layer_norm(out)return outclass Position_wise_Feed_Forward(nn.Module):#encoder block的第二部分def __init__(self, dim_model, hidden, dropout=0.0):#定义两个全连接层 多头注意力的计算结果 通过两个全连接层 增加非线性 super(Position_wise_Feed_Forward, self).__init__()self.fc1 = nn.Linear(dim_model, hidden)self.fc2 = nn.Linear(hidden, dim_model)self.dropout = nn.Dropout(dropout)self.layer_norm = nn.LayerNorm(dim_model)def forward(self, x): #(batch,seq_len,dim_model)out = self.fc1(x) #(batch,seq_len,hidden)out = F.relu(out)out = self.fc2(out) #(batch,seq_len,dim_model)out = self.dropout(out)out = out + x # 残差连接out = self.layer_norm(out) #层归一化return out