github/gpt4all
1
0
Fork 0
mirror of https://github.com/nomic-ai/gpt4all.git synced 2025-09-09 12:29:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add tests for error codes with local API server (#3131)

Browse Source 
Signed-off-by: Adam Treat <treat.adam@gmail.com>
Co-authored-by: Jared Van Bortel <jared@nomic.ai>
This commit is contained in:
AT
2024-10-30 10:15:19 -04:00
committed by GitHub
parent 861453c4d7
commit 62cab695eb
5 changed files with 251 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

251
gpt4all-chat/tests/python/test_server_api.py
View File

@@ -1,9 +1,11 @@
import os
import shutil
import signal
import subprocess
import sys
import tempfile
import textwrap
from contextlib import contextmanager
from pathlib import Path
from subprocess import CalledProcessError
from typing import Any, Iterator
@@ -20,59 +22,82 @@ class Requestor:
self.session = requests.Session()
self.http_adapter = self.session.adapters['http://']
def get(self, path: str, *, wait: bool = False) -> Any:
return self._request('GET', path, wait)
def get(self, path: str, *, raise_for_status: bool = True, wait: bool = False) -> Any:
return self._request('GET', path, raise_for_status=raise_for_status, wait=wait)
def _request(self, method: str, path: str, wait: bool) -> Any:
def post(self, path: str, data: dict[str, Any] | None, *, raise_for_status: bool = True, wait: bool = False) -> Any:
return self._request('POST', path, data, raise_for_status=raise_for_status, wait=wait)
def _request(
self, method: str, path: str, data: dict[str, Any] | None = None, *, raise_for_status: bool, wait: bool,
) -> Any:
if wait:
retry = Retry(total=None, connect=10, read=False, status=0, other=0, backoff_factor=.01)
else:
retry = Retry(total=False)
self.http_adapter.max_retries = retry # type: ignore[attr-defined]
resp = self.session.request(method, f'http://localhost:4891/v1/{path}')
resp.raise_for_status()
return resp.json()
resp = self.session.request(method, f'http://localhost:4891/v1/{path}', json=data)
if raise_for_status:
resp.raise_for_status()
return resp.json()
try:
json_data = resp.json()
except ValueError:
json_data = None
return resp.status_code, json_data
request = Requestor()
@pytest.fixture
def chat_server_config() -> Iterator[dict[str, str]]:
def create_chat_server_config(tmpdir: Path, model_copied: bool = False) -> dict[str, str]:
xdg_confdir = tmpdir / 'config'
app_confdir = xdg_confdir / 'nomic.ai'
app_confdir.mkdir(parents=True)
with open(app_confdir / 'GPT4All.ini', 'w') as conf:
conf.write(textwrap.dedent(f"""\
[General]
serverChat=true
[download]
lastVersionStarted={config.APP_VERSION}
[network]
isActive=false
usageStatsActive=false
"""))
if model_copied:
app_data_dir = tmpdir / 'share' / 'nomic.ai' / 'GPT4All'
app_data_dir.mkdir(parents=True)
local_env_file_path = Path(os.environ['TEST_MODEL_PATH'])
shutil.copy(local_env_file_path, app_data_dir / local_env_file_path.name)
return dict(
os.environ,
XDG_CACHE_HOME=str(tmpdir / 'cache'),
XDG_DATA_HOME=str(tmpdir / 'share'),
XDG_CONFIG_HOME=str(xdg_confdir),
APPIMAGE=str(tmpdir), # hack to bypass SingleApplication
)
@contextmanager
def prepare_chat_server(model_copied: bool = False) -> Iterator[dict[str, str]]:
if os.name != 'posix' or sys.platform == 'darwin':
pytest.skip('Need non-Apple Unix to use alternate config path')
with tempfile.TemporaryDirectory(prefix='gpt4all-test') as td:
tmpdir = Path(td)
xdg_confdir = tmpdir / 'config'
app_confdir = xdg_confdir / 'nomic.ai'
app_confdir.mkdir(parents=True)
with open(app_confdir / 'GPT4All.ini', 'w') as conf:
conf.write(textwrap.dedent(f"""\
[General]
serverChat=true
[download]
lastVersionStarted={config.APP_VERSION}
[network]
isActive=false
usageStatsActive=false
"""))
yield dict(
os.environ,
XDG_CACHE_HOME=str(tmpdir / 'cache'),
XDG_DATA_HOME=str(tmpdir / 'share'),
XDG_CONFIG_HOME=str(xdg_confdir),
APPIMAGE=str(tmpdir), # hack to bypass SingleApplication
)
config = create_chat_server_config(tmpdir, model_copied=model_copied)
yield config
@pytest.fixture
def chat_server(chat_server_config: dict[str, str]) -> Iterator[None]:
def start_chat_server(config: dict[str, str]) -> Iterator[None]:
chat_executable = Path(os.environ['CHAT_EXECUTABLE']).absolute()
with subprocess.Popen(chat_executable, env=chat_server_config) as process:
with subprocess.Popen(chat_executable, env=config) as process:
try:
yield
except:
@@ -83,5 +108,161 @@ def chat_server(chat_server_config: dict[str, str]) -> Iterator[None]:
raise CalledProcessError(retcode, process.args)
def test_list_models_empty(chat_server: None) -> None:
assert request.get('models', wait=True) == {'object': 'list', 'data': []}
@pytest.fixture
def chat_server() -> Iterator[None]:
with prepare_chat_server(model_copied=False) as config:
yield from start_chat_server(config)
@pytest.fixture
def chat_server_with_model() -> Iterator[None]:
with prepare_chat_server(model_copied=True) as config:
yield from start_chat_server(config)
def test_with_models_empty(chat_server: None) -> None:
# non-sense endpoint
status_code, response = request.get('foobarbaz', wait=True, raise_for_status=False)
assert status_code == 404
assert response is None
# empty model list
response = request.get('models')
assert response == {'object': 'list', 'data': []}
# empty model info
response = request.get('models/foo')
assert response == {}
# POST for model list
status_code, response = request.post('models', data=None, raise_for_status=False)
assert status_code == 405
assert response == {'error': {
'code': None,
'message': 'Not allowed to POST on /v1/models. (HINT: Perhaps you meant to use a different HTTP method?)',
'param': None,
'type': 'invalid_request_error',
}}
# POST for model info
status_code, response = request.post('models/foo', data=None, raise_for_status=False)
assert status_code == 405
assert response == {'error': {
'code': None,
'message': 'Not allowed to POST on /v1/models/*. (HINT: Perhaps you meant to use a different HTTP method?)',
'param': None,
'type': 'invalid_request_error',
}}
# GET for completions
status_code, response = request.get('completions', raise_for_status=False)
assert status_code == 405
assert response == {'error': {
'code': 'method_not_supported',
'message': 'Only POST requests are accepted.',
'param': None,
'type': 'invalid_request_error',
}}
# GET for chat completions
status_code, response = request.get('chat/completions', raise_for_status=False)
assert status_code == 405
assert response == {'error': {
'code': 'method_not_supported',
'message': 'Only POST requests are accepted.',
'param': None,
'type': 'invalid_request_error',
}}
EXPECTED_MODEL_INFO = {
'created': 0,
'id': 'Llama 3.2 1B Instruct',
'object': 'model',
'owned_by': 'humanity',
'parent': None,
'permissions': [
{
'allow_create_engine': False,
'allow_fine_tuning': False,
'allow_logprobs': False,
'allow_sampling': False,
'allow_search_indices': False,
'allow_view': True,
'created': 0,
'group': None,
'id': 'placeholder',
'is_blocking': False,
'object': 'model_permission',
'organization': '*',
},
],
'root': 'Llama 3.2 1B Instruct',
}
EXPECTED_COMPLETIONS_RESPONSE = {
'choices': [
{
'finish_reason': 'stop',
'index': 0,
'logprobs': None,
'references': None,
'text': ' jumps over the lazy dog.',
},
],
'id': 'placeholder',
'model': 'Llama 3.2 1B Instruct',
'object': 'text_completion',
'usage': {
'completion_tokens': 6,
'prompt_tokens': 5,
'total_tokens': 11,
},
}
def test_with_models(chat_server_with_model: None) -> None:
response = request.get('models', wait=True)
assert response == {
'data': [EXPECTED_MODEL_INFO],
'object': 'list',
}
# Test the specific model endpoint
response = request.get('models/Llama 3.2 1B Instruct')
assert response == EXPECTED_MODEL_INFO
# Test the completions endpoint
status_code, response = request.post('completions', data=None, raise_for_status=False)
assert status_code == 400
assert response == {'error': {
'code': None,
'message': 'error parsing request JSON: illegal value',
'param': None,
'type': 'invalid_request_error',
}}
data = {
'model': 'Llama 3.2 1B Instruct',
'prompt': 'The quick brown fox',
'temperature': 0,
}
response = request.post('completions', data=data)
assert len(response['choices']) == 1
assert response['choices'][0].keys() == {'text', 'index', 'logprobs', 'references', 'finish_reason'}
assert response['choices'][0]['text'] == ' jumps over the lazy dog.'
assert 'created' in response
response.pop('created') # Remove the dynamic field for comparison
assert response == EXPECTED_COMPLETIONS_RESPONSE
@pytest.mark.xfail(reason='Assertion failure in GPT4All. See nomic-ai/gpt4all#3133')
def test_with_models_temperature(chat_server_with_model: None) -> None:
data = {
'model': 'Llama 3.2 1B Instruct',
'prompt': 'The quick brown fox',
'temperature': 0.5,
}
request.post('completions', data=data, wait=True, raise_for_status=True)