CAIL2021-阅读理解任务-top3-数据预处理模块
代码地址：https://github.com/davidfan1224/CAIL2021_Multi-span_MRC
class SquadExample(object):
"""
A single training/test example for the Squad dataset.
For examples without an answer, the start and end position are -1.
"""
def __init__(self,
qas_id,
question_text,
doc_tokens,
orig_answer_text=None,
start_position=None,
end_position=None,
is_impossible=None,
is_yes=None,
is_no=None,
labels=None):
self.qas_id = qas_id
self.question_text = question_text
self.doc_tokens = doc_tokens
self.orig_answer_text = orig_answer_text
self.start_position = start_position
self.end_position = end_position
self.is_impossible = is_impossible
self.is_yes = is_yes
self.is_no = is_no
self.labels = labels
def __str__(self):
return self.__repr__()
# def __repr__(self):
# s = ""
# s += "qas_id: %s" % (self.qas_id) + '\n'
# s += ", question_text: %s" % (
# self.question_text) + '\n'
# s += ", doc_tokens: [%s]" % (" ".join(self.doc_tokens)) + '\n'
# if self.start_position:
# s += ", start_position: %s" % (str(self.start_position)) + '\n'
# if self.end_position:
# s += ", end_position: %s" % (str(self.end_position)) + '\n'
# if self.is_impossible:
# s += ", is_impossible: %r" % (self.is_impossible) + '\n'
# return s
def __repr__(self):
string = ""
for key, value in self.__dict__.items():
string += f"{key}: {value}\n"
return f"<{string}>"
def read_squad_examples(input_file, is_training, version_2_with_negative):
"""Read a SQuAD json file into a list of SquadExample."""
with open(input_file, "r", encoding='utf-8') as reader:
input_data = json.load(reader)["data"]
def is_whitespace(c):
if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F:
return True
return False
examples = []
change_ques_num = 0
for entry in tqdm(input_data, desc='convert examples:'):
for paragraph in entry["paragraphs"]:
paragraph_text = paragraph["context"]
doc_tokens = []
char_to_word_offset = []
# 这里直接是按照字符进行划分的
for c in paragraph_text:
doc_tokens.append(c)
char_to_word_offset.append(len(doc_tokens) - 1)
for qa in paragraph["qas"]:
qas_id = qa["id"]
question_text = qa["question"]
if question_text != qa['question']:
change_ques_num += 1
start_position = None
end_position = None
orig_answer_text = None
# 多答案：
start_positions = []
end_positions = []
orig_answer_texts = []
is_impossible = False
is_yes = False
is_no = False
labels = ['O'] * len(doc_tokens)
if is_training:
if version_2_with_negative:
if qa['is_impossible'] == 'false':
is_impossible = False
else:
is_impossible = True
# 下面这一行将不坑回答的问题进行过滤掉了
if (len(qa["answers"]) != 1) and (not is_impossible):
continue
if not is_impossible:
all_answers = qa["answers"]
if len(all_answers) == 0:
answers = []
else:
answers = all_answers[0]
if type(answers) == dict:
answers = [answers]
for answer in answers:
orig_answer_text = answer["text"]
answer_offset = answer["answer_start"]
answer_length = len(orig_answer_text)
start_position = char_to_word_offset[answer_offset]
end_position = char_to_word_offset[answer_offset + answer_length - 1]
# Only add answers where the text can be exactly recovered from the
# document. If this CAN'T happen it's likely due to weird Unicode
# stuff so we will just skip the example.
#
# Note that this means for training mode, every example is NOT
# guaranteed to be preserved.
actual_text = "".join(doc_tokens[start_position:(end_position + 1)])
cleaned_answer_text = " ".join(
whitespace_tokenize(orig_answer_text))
if actual_text.find(cleaned_answer_text) == -1:
if cleaned_answer_text == 'YES':
is_yes = True
orig_answer_text = 'YES'
start_position = -1
end_position = -1
labels = ['O'] * len(doc_tokens)
start_positions.append(start_position)
end_positions.append(end_position)
orig_answer_texts.append(orig_answer_text)
elif cleaned_answer_text == 'NO':
is_no = True
start_position = -1
end_position = -1
orig_answer_text = 'NO'
labels = ['O'] * len(doc_tokens)
start_positions.append(start_position)
end_positions.append(end_position)
orig_answer_texts.append(orig_answer_text)
else:
logger.warning("Could not find answer: '%s' vs. '%s'",
actual_text, cleaned_answer_text)
continue
else:
start_positions.append(start_position)
end_positions.append(end_position)
orig_answer_texts.append(orig_answer_text)
start_index = answer['answer_start']
end_index = start_index + len(answer['text'])
labels[start_index: end_index] = ['I'] * (len(answer['text'])) # 是答案的用I标记，否则用O标记
else:
start_position = -1
end_position = -1
orig_answer_text = ""
start_positions.append(start_position)
end_positions.append(end_position)
orig_answer_texts.append(orig_answer_text)
labels = ['O'] * len(doc_tokens)
example = SquadExample(
qas_id=qas_id,
question_text=question_text,
doc_tokens=doc_tokens,
orig_answer_text=orig_answer_texts,
start_position=start_positions,
end_position=end_positions,
is_impossible=is_impossible,
is_yes=is_yes,
is_no=is_no,
labels=labels)
examples.append(example)
return examples
if __name__ == '__main__':
data_file = 'data_sample/cail2021_mrc_small.json'
examples = read_squad_examples(data_file, is_training=True, version_2_with_negative=True)
print(examples[0])
这里需要注意的有两点：
- 这里最初的切分是以字符进行切分的。
- 有一个labels，这里和文本的长度是一致的，以序列标注的IO格式进行标注，如果是YES和NO类的，那么labels里面就全部是O。
接着就是将每一个example转换为feature了：
class InputFeatures(object):
"""A single set of features of data."""
def __init__(self,
unique_id,
example_index,
doc_span_index,
tokens,
token_to_orig_map,
token_is_max_context,
input_ids,
input_mask,
segment_ids,
paragraph_len,
label_id=None,
start_position=None,
end_position=None,
is_impossible=None,
unk_mask=None,
yes_mask=None,
no_mask=None,
answer_mask=None,
answer_num=None
):
self.unique_id = unique_id
self.example_index = example_index
self.doc_span_index = doc_span_index
self.tokens = tokens
self.token_to_orig_map = token_to_orig_map
self.token_is_max_context = token_is_max_context
self.input_ids = input_ids
self.input_mask = input_mask
self.segment_ids = segment_ids
self.paragraph_len = paragraph_len
self.label_id = label_id
self.start_position = start_position
self.end_position = end_position
self.is_impossible = is_impossible
self.unk_mask = unk_mask
self.yes_mask = yes_mask
self.no_mask = no_mask
self.answer_mask = answer_mask
self.answer_num = answer_num
def __repr__(self):
string = ""
for key, value in self.__dict__.items():
string += f"{key}: {value}\n"
return f"<{string}>"
def convert_examples_to_features(examples, tokenizer, max_seq_length,
doc_stride, max_query_length, is_training, max_n_answers=1):
"""Loads a data file into a list of `InputBatch`s."""
unique_id = 1000000000
features = []
unk_tokens = {}
label_map = {'O': 0, 'I': 1}
convert_token_list = {
'“': '"', "”": '"', '…': '...', '﹤': '<', '﹥': '>', '‘': "'", '’': "'",
'﹪': '%', 'Ⅹ': 'x', '―': '-', '—': '-', '﹟': '#', '㈠': '一'
}
example_index = 0
for example in tqdm(examples, desc='convert_examples_to_features:'):
# 对问题进行分词
query_tokens = tokenizer.tokenize(example.question_text)
# 如果问题的最大长度超过了设置的则截断
if len(query_tokens) > max_query_length:
query_tokens = query_tokens[0:max_query_length]
labellist = example.labels
tok_to_orig_index = []
orig_to_tok_index = []
all_doc_tokens = []
for (i, token) in enumerate(example.doc_tokens):
orig_to_tok_index.append(len(all_doc_tokens))
sub_tokens = tokenizer.tokenize(token)
if "[UNK]" in sub_tokens:
if token in unk_tokens:
unk_tokens[token] += 1
else:
unk_tokens[token] = 1
for sub_token in sub_tokens:
tok_to_orig_index.append(i)
all_doc_tokens.append(sub_token)
tok_start_position = None
tok_end_position = None
# multi answers
tok_start_positions = []
tok_end_positions = []
if is_training and example.is_impossible:
tok_start_position = -1
tok_end_position = -1
tok_start_positions.append(-1)
tok_end_positions.append(-1)
if is_training and not example.is_impossible:
for orig_answer_text, start_position, end_position in zip(example.orig_answer_text, example.start_position, example.end_position):
tok_start_position = orig_to_tok_index[start_position]
if end_position < len(example.doc_tokens) - 1:
tok_end_position = orig_to_tok_index[end_position + 1] - 1
else:
tok_end_position = len(all_doc_tokens) - 1
(tok_start_position, tok_end_position) = _improve_answer_span(
all_doc_tokens, tok_start_position, tok_end_position, tokenizer,
orig_answer_text)
tok_start_positions.append(tok_start_position)
tok_end_positions.append(tok_end_position)
# The -3 accounts for [CLS], [SEP] and [SEP]
max_tokens_for_doc = max_seq_length - len(query_tokens) - 3
# We can have documents that are longer than the maximum sequence length.
# To deal with this we do a sliding window approach, where we take chunks
# of the up to our max length with a stride of `doc_stride`.
_DocSpan = collections.namedtuple( # pylint: disable=invalid-name
"DocSpan", ["start", "length"])
doc_spans = []
start_offset = 0
while start_offset < len(all_doc_tokens):
length = len(all_doc_tokens) - start_offset
if length > max_tokens_for_doc:
length = max_tokens_for_doc
doc_spans.append(_DocSpan(start=start_offset, length=length))
if start_offset + length == len(all_doc_tokens):
break
start_offset += min(length, doc_stride)
while start_offset < len(all_doc_tokens) and all_doc_tokens[start_offset - 1] != "," and all_doc_tokens[start_offset - 1] != "。": # 滑窗 保留完整一句话 此处是英文的，
start_offset += 1
for (doc_span_index, doc_span) in enumerate(doc_spans):
tokens = []
token_to_orig_map = {}
token_is_max_context = {}
segment_ids = []
doc_span_labels = []
label_ids = []
tokens.append("[CLS]")
segment_ids.append(0)
label_ids.append(label_map["O"])
for token in query_tokens:
tokens.append(token)
segment_ids.append(0)
label_ids.append(label_map["O"])
tokens.append("[SEP]")
segment_ids.append(0)
label_ids.append(label_map["O"])
for i in range(doc_span.length):
split_token_index = doc_span.start + i
doc_span_labels.append(labellist[split_token_index])
for i in range(doc_span.length):
split_token_index = doc_span.start + i
token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index]
is_max_context = _check_is_max_context(doc_spans, doc_span_index,
split_token_index)
token_is_max_context[len(tokens)] = is_max_context
tokens.append(all_doc_tokens[split_token_index])
segment_ids.append(1)
label_ids.append(label_map[doc_span_labels[i]])
paragraph_len = doc_span.length
tokens.append("[SEP]")
segment_ids.append(1)
label_ids.append(label_map["O"])
input_ids = tokenizer.convert_tokens_to_ids(tokens)
# The mask has 1 for real tokens and 0 for padding tokens. Only real
# tokens are attended to.
input_mask = [1] * len(input_ids)
# Zero-pad up to the sequence length.
while len(input_ids) < max_seq_length:
input_ids.append(0)
input_mask.append(0)
segment_ids.append(0)
label_ids.append(label_map["O"])
assert len(input_ids) == max_seq_length
assert len(input_mask) == max_seq_length
assert len(segment_ids) == max_seq_length
span_is_impossible = example.is_impossible
start_position = None
end_position = None
# multi answers
start_positions = []
end_positions = []
if is_training and not example.is_impossible:
doc_start = doc_span.start
doc_end = doc_span.start + doc_span.length - 1
for tok_start_position, tok_end_position in zip(tok_start_positions, tok_end_positions): # 多个答案 计算各自起始位置
out_of_span = False
if not (tok_start_position >= doc_start and
tok_end_position <= doc_end):
out_of_span = True
if out_of_span:
start_position = max_seq_length
end_position = max_seq_length
else:
doc_offset = len(query_tokens) + 2
start_position = tok_start_position - doc_start + doc_offset
end_position = tok_end_position - doc_start + doc_offset
start_positions.append(start_position)
end_positions.append(end_position)
unk_mask, yes_mask, no_mask = [0], [0], [0]
if is_training and example.is_impossible: # no answer
start_position = max_seq_length
end_position = max_seq_length
unk_mask = [1]
if start_positions:
start_positions.clear()
end_positions.clear()
start_positions.append(start_position)
end_positions.append(end_position)
else:
start_positions.append(start_position)
end_positions.append(end_position)
elif is_training and example.is_yes: # YES
start_position = max_seq_length+1
end_position = max_seq_length+1
yes_mask = [1]
if start_positions:
start_positions.clear()
end_positions.clear()
start_positions.append(start_position)
end_positions.append(end_position)
else:
start_positions.append(start_position)
end_positions.append(end_position)
elif is_training and example.is_no: # NO
start_position = max_seq_length+2
end_position = max_seq_length+2
no_mask = [1]
if start_positions:
start_positions.clear()
end_positions.clear()
start_positions.append(start_position)
end_positions.append(end_position)
else:
start_positions.append(start_position)
end_positions.append(end_position)
# 如果答案列表长度 大于设置的阈值最大答案数量 则随机选择
if len(start_positions) > max_n_answers:
idxs = np.random.choice(len(start_positions), max_n_answers, replace=False)
st = []
en = []
for idx in idxs:
st.append(start_positions[idx])
en.append(end_positions[idx])
start_positions = st
end_positions = en
answer_num = len(start_positions) if not example.is_impossible else 0
answer_mask = [1 for _ in range(len(start_positions))]
for _ in range(max_n_answers - len(start_positions)):
start_positions.append(0)
end_positions.append(0)
answer_mask.append(0)
features.append(
InputFeatures(
unique_id=unique_id,
example_index=example_index,
doc_span_index=doc_span_index,
tokens=tokens,
token_to_orig_map=token_to_orig_map,
token_is_max_context=token_is_max_context,
input_ids=input_ids,
input_mask=input_mask,
segment_ids=segment_ids,
paragraph_len=paragraph_len,
start_position=start_positions,
end_position=end_positions,
is_impossible=example.is_impossible,
unk_mask=unk_mask,
yes_mask=yes_mask,
no_mask=no_mask,
answer_mask=answer_mask,
answer_num=answer_num,
label_id=label_ids
))
unique_id += 1
example_index += 1
if is_training:
with open("unk_tokens_clean", "w", encoding="utf-8") as fh:
for key, value in unk_tokens.items():
fh.write(key+" " + str(value)+"\n")
return features
tokenizer = BertTokenizer.from_pretrained('model_hub/chinese-bert-wwm-ext/')
features, output_examples = convert_examples_to_features(examples, tokenizer, 512, 128, 64, True, max_n_answers=3)
for i, (example, feature) in enumerate(zip(output_examples, features)):
if i < 10:
print(example)
print(feature)
print("="*100)
结果：
====================================================================================================
====================================================================================================
====================================================================================================
====================================================================================================
可以看到，会对太长的句子进行切分。
我们看看一些特殊的情况：
对于不能够回答的问题，会将start_position和end_position里设置为512，并且answer_mask设置为[1,0,0]，answer_num为0。
对于是YES/NO类型的问题：
start_position: [513, 0, 0]
end_position: [513, 0, 0]
is_impossible: False
unk_mask: [0]
yes_mask: [1]
no_mask: [0]
answer_mask: [1, 0, 0]
answer_num: 1
>
对应的是start_position、end_position里513或者514。