From ce6748dbfeeeec92014189a2ec08e6e6d957e4a6 Mon Sep 17 00:00:00 2001
From: Bagatur <22008038+baskaryan@users.noreply.github.com>
Date: Thu, 19 Dec 2024 13:23:30 -0800
Subject: [PATCH] xfail openai image token count test (#28828)

---
 .../tests/integration_tests/chat_models/test_base.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/libs/partners/openai/tests/integration_tests/chat_models/test_base.py b/libs/partners/openai/tests/integration_tests/chat_models/test_base.py
index f2490f89501..427a030779e 100644
--- a/libs/partners/openai/tests/integration_tests/chat_models/test_base.py
+++ b/libs/partners/openai/tests/integration_tests/chat_models/test_base.py
@@ -706,6 +706,12 @@ async def test_openai_response_headers_async() -> None:
     assert "content-type" in headers
 
 
+@pytest.mark.xfail(
+    reason=(
+        "As of 12.19.24 OpenAI API returns 1151 instead of 1118. Not clear yet if "
+        "this is an undocumented API change or a bug on their end."
+    )
+)
 def test_image_token_counting_jpeg() -> None:
     model = ChatOpenAI(model="gpt-4o", temperature=0)
     image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
@@ -738,6 +744,12 @@ def test_image_token_counting_jpeg() -> None:
     assert expected == actual
 
 
+@pytest.mark.xfail(
+    reason=(
+        "As of 12.19.24 OpenAI API returns 871 instead of 779. Not clear yet if "
+        "this is an undocumented API change or a bug on their end."
+    )
+)
 def test_image_token_counting_png() -> None:
     model = ChatOpenAI(model="gpt-4o", temperature=0)
     image_url = "https://upload.wikimedia.org/wikipedia/commons/4/47/PNG_transparency_demonstration_1.png"