From 98d3304108de3e55c18f2af8a66a501541ec658b Mon Sep 17 00:00:00 2001 From: Heiner Lohaus Date: Tue, 22 Aug 2023 23:27:34 +0200 Subject: Improve providers with tests --- testing/test_providers.py | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) (limited to 'testing/test_providers.py') diff --git a/testing/test_providers.py b/testing/test_providers.py index a5c6f87b..fee79e20 100644 --- a/testing/test_providers.py +++ b/testing/test_providers.py @@ -3,50 +3,51 @@ from pathlib import Path sys.path.append(str(Path(__file__).parent.parent)) -from g4f import BaseProvider, models, provider +from g4f import BaseProvider, models, Provider +logging = False def main(): providers = get_providers() - results: list[list[str | bool]] = [] + failed_providers = [] for _provider in providers: - print("start", _provider.__name__) - actual_working = judge(_provider) - expected_working = _provider.working - match = actual_working == expected_working + if _provider.needs_auth: + continue + print("Provider:", _provider.__name__) + result = judge(_provider) + print("Result:", result) + if _provider.working and not result: + failed_providers.append([_provider, result]) - results.append([_provider.__name__, expected_working, actual_working, match]) - - print("failed provider list") - for result in results: - if not result[3]: - print(result) + print("Failed providers:") + for _provider, result in failed_providers: + print(f"{_provider.__name__}: {result}") def get_providers() -> list[type[BaseProvider]]: - provider_names = dir(provider) + provider_names = dir(Provider) ignore_names = [ "base_provider", - "BaseProvider", + "BaseProvider" ] provider_names = [ provider_name for provider_name in provider_names if not provider_name.startswith("__") and provider_name not in ignore_names ] - return [getattr(provider, provider_name) for provider_name in provider_names] + return [getattr(Provider, provider_name) for provider_name in provider_names] def create_response(_provider: type[BaseProvider]) -> str: model = ( models.gpt_35_turbo.name - if _provider is not provider.H2o - else models.falcon_7b.name + if _provider.supports_gpt_35_turbo + else _provider.model ) response = _provider.create_completion( model=model, - messages=[{"role": "user", "content": "Hello world!, plz yourself"}], + messages=[{"role": "user", "content": "Hello world!"}], stream=False, ) return "".join(response) @@ -59,9 +60,10 @@ def judge(_provider: type[BaseProvider]) -> bool: try: response = create_response(_provider) assert type(response) is str - return len(response) > 1 + return response except Exception as e: - print(e) + if logging: + print(e) return False -- cgit v1.2.3 From 69ca98ac85743edd76ac6ce49feb233cd5365099 Mon Sep 17 00:00:00 2001 From: Heiner Lohaus Date: Thu, 24 Aug 2023 21:32:22 +0200 Subject: Improve provider list --- testing/test_providers.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) (limited to 'testing/test_providers.py') diff --git a/testing/test_providers.py b/testing/test_providers.py index fee79e20..6d3b62d8 100644 --- a/testing/test_providers.py +++ b/testing/test_providers.py @@ -1,5 +1,6 @@ import sys from pathlib import Path +from colorama import Fore sys.path.append(str(Path(__file__).parent.parent)) @@ -20,9 +21,14 @@ def main(): if _provider.working and not result: failed_providers.append([_provider, result]) - print("Failed providers:") - for _provider, result in failed_providers: - print(f"{_provider.__name__}: {result}") + print() + + if failed_providers: + print(f"{Fore.RED}Failed providers:\n") + for _provider, result in failed_providers: + print(f"{Fore.RED}{_provider.__name__}") + else: + print(f"{Fore.GREEN}All providers are working") def get_providers() -> list[type[BaseProvider]]: @@ -36,18 +42,21 @@ def get_providers() -> list[type[BaseProvider]]: for provider_name in provider_names if not provider_name.startswith("__") and provider_name not in ignore_names ] - return [getattr(Provider, provider_name) for provider_name in provider_names] + return [getattr(Provider, provider_name) for provider_name in sorted(provider_names)] def create_response(_provider: type[BaseProvider]) -> str: - model = ( - models.gpt_35_turbo.name - if _provider.supports_gpt_35_turbo - else _provider.model - ) + if _provider.supports_gpt_35_turbo: + model = models.gpt_35_turbo.name + elif _provider.supports_gpt_4: + model = models.gpt_4 + elif hasattr(_provider, "model"): + model = _provider.model + else: + model = None response = _provider.create_completion( model=model, - messages=[{"role": "user", "content": "Hello world!"}], + messages=[{"role": "user", "content": "Hello"}], stream=False, ) return "".join(response) @@ -57,9 +66,13 @@ def judge(_provider: type[BaseProvider]) -> bool: if _provider.needs_auth: return _provider.working + return test(_provider) + +def test(_provider: type[BaseProvider]) -> bool: try: response = create_response(_provider) assert type(response) is str + assert len(response) > 0 return response except Exception as e: if logging: -- cgit v1.2.3 From 1f56c792220427b86ec7780a7a298d14f205b466 Mon Sep 17 00:00:00 2001 From: Heiner Lohaus Date: Fri, 25 Aug 2023 00:32:13 +0200 Subject: Remove judge function --- testing/test_providers.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) (limited to 'testing/test_providers.py') diff --git a/testing/test_providers.py b/testing/test_providers.py index 6d3b62d8..c4fcbc0c 100644 --- a/testing/test_providers.py +++ b/testing/test_providers.py @@ -16,16 +16,16 @@ def main(): if _provider.needs_auth: continue print("Provider:", _provider.__name__) - result = judge(_provider) + result = test(_provider) print("Result:", result) if _provider.working and not result: - failed_providers.append([_provider, result]) + failed_providers.append(_provider) print() if failed_providers: print(f"{Fore.RED}Failed providers:\n") - for _provider, result in failed_providers: + for _provider in failed_providers: print(f"{Fore.RED}{_provider.__name__}") else: print(f"{Fore.GREEN}All providers are working") @@ -61,12 +61,6 @@ def create_response(_provider: type[BaseProvider]) -> str: ) return "".join(response) - -def judge(_provider: type[BaseProvider]) -> bool: - if _provider.needs_auth: - return _provider.working - - return test(_provider) def test(_provider: type[BaseProvider]) -> bool: try: -- cgit v1.2.3