完整项目???????
本篇博客主要介绍一下数据预处理,包括构建数据集和迭代器。由于预训练语言模型有相应的词表文件,所以不需要手动构建词典,不过在预处理阶段,需要指明输入序列的填充部分和非填充部分。
目录
1. 构建数据集
2. 构建迭代器
1. 构建数据集
PAD, CLS = '[PAD]', '[CLS]' # padding符号, bert中综合信息符号def build_dataset(config):def load_dataset(path, pad_size=32):contents = []with open(path, 'r', encoding='UTF-8') as f:for line in tqdm(f):#遍历每一行lin = line.strip()#去掉首尾空白符if not lin:#遇到空行 跳过continuecontent, label = lin.split('\t')#text label;每一行以\t为切分,拿到文本token = config.tokenizer.tokenize(content) #分字(汉语 character-level) bert内置的tokenizertoken = [CLS] + token #头部加入 [CLS] tokenseq_len = len(token) #文本实际长度(填充或截断之前)mask = [] #区分填充部分和非填充部分token_ids = config.tokenizer.convert_tokens_to_ids(token) #把tokenizer转换为索引(基于下载的词表文件)if pad_size:#长截短填if len(token) < pad_size: #序列长度小于 填充长度mask = [1] * len(token_ids) + [0] * (pad_size - len(token))#mask 填充部分对应0 非填充部分为1token_ids += ([0] * (pad_size - len(token))) #用0作填充else: #此时没有填充 序列长度大于填充长度mask = [1] * pad_size #全部都是非填充token_ids = token_ids[:pad_size] #截断seq_len = pad_size #实际长度为填充长度contents.append((token_ids, int(label), seq_len, mask)) #[([...],label,seq_len,[...])]return contents# 分别对训练集、验证集、测试集进行处理train = load_dataset(config.train_path, config.pad_size)dev = load_dataset(config.dev_path, config.pad_size)test = load_dataset(config.test_path, config.pad_size)# 返回预处理好的训练集、验证集、测试集return train, dev, test
2. 构建迭代器
class DatasetIterater(object):#自定义数据集迭代器def __init__(self, batches, batch_size, device):self.batch_size = batch_sizeself.batches = batches#数据集self.n_batches = len(batches) // batch_size #得到batch数量self.residue = False # 记录batch数量是否为整数if len(batches) % self.n_batches != 0:#不能整除self.residue = Trueself.index = 0self.device = devicedef _to_tensor(self, datas):# 转换为tensor 并 to(device)x = torch.LongTensor([_[0] for _ in datas]).to(self.device) #输入序列y = torch.LongTensor([_[1] for _ in datas]).to(self.device) #标签# seq_len为文本的实际长度(不包含填充的长度) 转换为tensor 并 to(device)seq_len = torch.LongTensor([_[2] for _ in datas]).to(self.device)#maskmask = torch.LongTensor([_[3] for _ in datas]).to(self.device)return (x, seq_len, mask), ydef __next__(self):if self.residue and self.index == self.n_batches:#当数据集大小 不整除 batch_size时,构建最后一个batchbatches = self.batches[self.index * self.batch_size: len(self.batches)]self.index += 1batches = self._to_tensor(batches)#把最后一个batch转换为tensor 并 to(device)return batcheselif self.index > self.n_batches:self.index = 0raise StopIterationelse:#构建每一个batchbatches = self.batches[self.index * self.batch_size: (self.index + 1) * self.batch_size]self.index += 1batches = self._to_tensor(batches)#把当前batch转换为tensor 并 to(device)return batchesdef __iter__(self):return selfdef __len__(self):if self.residue:return self.n_batches + 1 #不整除 batch数加1else:return self.n_batchesdef build_iterator(dataset, config):#构建数据集迭代器iter = DatasetIterater(dataset, config.batch_size, config.device)return iter