|
import re |
|
|
|
|
|
def clean_corpus(chat_export_file): |
|
"""Prepare a WhatsApp chat export for training with chatterbot.""" |
|
message_corpus = remove_chat_metadata(chat_export_file) |
|
cleaned_corpus = remove_non_message_text(message_corpus) |
|
return cleaned_corpus |
|
|
|
|
|
def remove_chat_metadata(chat_export_file): |
|
"""Remove WhatsApp chat metadata. |
|
|
|
WhatsApp chat exports come with metadata about each message: |
|
|
|
date time username message |
|
--------------------------------------- |
|
8/26/22, 17:47 - Jane Doe: Message text |
|
|
|
This function removes all the metadata up to the text of each message. |
|
|
|
Args: |
|
chat_export_file (str): The name of the chat export file |
|
|
|
Returns: |
|
tuple: The text of each message in the conversation |
|
""" |
|
date_time = r"(\d+\/\d+\/\d+,\s\d+:\d+)" |
|
dash_whitespace = r"\s-\s" |
|
username = r"([\w\s]+)" |
|
metadata_end = r":\s" |
|
pattern = date_time + dash_whitespace + username + metadata_end |
|
|
|
with open(chat_export_file, "r") as corpus_file: |
|
content = corpus_file.read() |
|
cleaned_corpus = re.sub(pattern, "", content) |
|
return tuple(cleaned_corpus.split("\n")) |
|
|
|
|
|
def remove_non_message_text(export_text_lines): |
|
"""Remove conversation-irrelevant text from chat export. |
|
|
|
WhatsApp chat exports come with a standardized intro line, |
|
and an empty line at the end of the file. |
|
Text exports also replace media messages with text that isn't |
|
relevant for the conversation. This function removes all that. |
|
|
|
Args: |
|
export_text_lines (tuple): All lines from the export file |
|
|
|
Returns: |
|
tuple: Messages that are a relevant part of the conversation |
|
""" |
|
messages = export_text_lines[1:-1] |
|
|
|
filter_out_msgs = ("<Media omitted>",) |
|
return tuple((msg for msg in messages if msg not in filter_out_msgs)) |
|
|
