From afc292e58d768d96245272ebe7d985f72b03da73 Mon Sep 17 00:00:00 2001 From: Augustine Theodore Date: Mon, 26 Jun 2023 00:38:43 +0530 Subject: [PATCH] Fix WhatsAppChatLoader : Enable parsing additional formats (#6663) - Description: Updated regex to support a new format that was observed when whatsapp chat was exported. - Issue: #6654 - Dependencies: No new dependencies - Tag maintainer: @rlancemartin, @eyurtsev --- langchain/document_loaders/whatsapp_chat.py | 6 ++++-- .../document_loaders/test_whatsapp_chat.py | 2 ++ tests/integration_tests/examples/whatsapp_chat.txt | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/langchain/document_loaders/whatsapp_chat.py b/langchain/document_loaders/whatsapp_chat.py index d98a16fdbf6..c2ffc46ee52 100644 --- a/langchain/document_loaders/whatsapp_chat.py +++ b/langchain/document_loaders/whatsapp_chat.py @@ -40,7 +40,7 @@ class WhatsAppChatLoader(BaseLoader): (?: :\d{2} )? - (?:[ _](?:AM|PM))? + (?:[\s_](?:AM|PM))? ) \]? [\s-]* @@ -50,7 +50,9 @@ class WhatsAppChatLoader(BaseLoader): (.+) """ for line in lines: - result = re.match(message_line_regex, line.strip(), flags=re.VERBOSE) + result = re.match( + message_line_regex, line.strip(), flags=re.VERBOSE | re.IGNORECASE + ) if result: date, sender, text = result.groups() text_content += concatenate_rows(date, sender, text) diff --git a/tests/integration_tests/document_loaders/test_whatsapp_chat.py b/tests/integration_tests/document_loaders/test_whatsapp_chat.py index 671925be617..be59d4f2f22 100644 --- a/tests/integration_tests/document_loaders/test_whatsapp_chat.py +++ b/tests/integration_tests/document_loaders/test_whatsapp_chat.py @@ -18,4 +18,6 @@ def test_whatsapp_chat_loader() -> None: "User 1 on 1/23/23, 3:22_AM: And let me know if anything changes\n\n" "~ User name 2 on 1/24/21, 12:41:03 PM: Of course!\n\n" "~ User 2 on 2023/5/4, 16:13:23: See you!\n\n" + "User 1 on 7/19/22, 11:32 PM: Hello\n\n" + "User 2 on 7/20/22, 11:32 am: Goodbye\n\n" ) diff --git a/tests/integration_tests/examples/whatsapp_chat.txt b/tests/integration_tests/examples/whatsapp_chat.txt index f64cea183ec..bdd4d63fd05 100644 --- a/tests/integration_tests/examples/whatsapp_chat.txt +++ b/tests/integration_tests/examples/whatsapp_chat.txt @@ -3,4 +3,6 @@ 1/23/23, 3:19 AM - User 2: Bye! 1/23/23, 3:22_AM - User 1: And let me know if anything changes [1/24/21, 12:41:03 PM] ~ User name 2: Of course! -[2023/5/4, 16:13:23] ~ User 2: See you! \ No newline at end of file +[2023/5/4, 16:13:23] ~ User 2: See you! +7/19/22, 11:32 PM - User 1: Hello +7/20/22, 11:32 am - User 2: Goodbye