From f6ef3cb2237d8c336e915ef77ddbe6f37934c4fd Mon Sep 17 00:00:00 2001 From: MIDORIBIN Date: Fri, 28 Jul 2023 19:07:17 +0900 Subject: refactor: refactor provider --- g4f/Provider/Vercel.py | 337 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 337 insertions(+) create mode 100644 g4f/Provider/Vercel.py (limited to 'g4f/Provider/Vercel.py') diff --git a/g4f/Provider/Vercel.py b/g4f/Provider/Vercel.py new file mode 100644 index 00000000..186662c4 --- /dev/null +++ b/g4f/Provider/Vercel.py @@ -0,0 +1,337 @@ +import base64 +import json +import uuid + +import quickjs +from curl_cffi import requests + +from ..typing import Any, CreateResult, TypedDict +from .base_provider import BaseProvider + + +class Vercel(BaseProvider): + url = "https://play.vercel.ai" + working = True + supports_gpt_35_turbo = True + + @staticmethod + def create_completion( + model: str, + messages: list[dict[str, str]], + stream: bool, + **kwargs: Any, + ) -> CreateResult: + if model in ["gpt-3.5-turbo", "gpt-4"]: + model = "openai:" + model + yield _chat(model_id=model, messages=messages) + + +def _chat(model_id: str, messages: list[dict[str, str]]) -> str: + session = requests.Session(impersonate="chrome107") + + url = "https://sdk.vercel.ai/api/generate" + header = _create_header(session) + payload = _create_payload(model_id, messages) + + response = session.post(url=url, headers=header, json=payload) + response.raise_for_status() + return response.text + + +def _create_payload(model_id: str, messages: list[dict[str, str]]) -> dict[str, Any]: + default_params = model_info[model_id]["default_params"] + return { + "messages": messages, + "playgroundId": str(uuid.uuid4()), + "chatIndex": 0, + "model": model_id, + } | default_params + + +def _create_header(session: requests.Session): + custom_encoding = _get_custom_encoding(session) + return {"custom-encoding": custom_encoding} + + +# based on https://github.com/ading2210/vercel-llm-api +def _get_custom_encoding(session: requests.Session): + url = "https://sdk.vercel.ai/openai.jpeg" + response = session.get(url=url) + + data = json.loads(base64.b64decode(response.text, validate=True)) + script = """ + String.prototype.fontcolor = function() {{ + return `${{this}}` + }} + var globalThis = {{marker: "mark"}}; + ({script})({key}) + """.format( + script=data["c"], key=data["a"] + ) + + context = quickjs.Context() # type: ignore + token_data = json.loads(context.eval(script).json()) # type: ignore + token_data[2] = "mark" + token = {"r": token_data, "t": data["t"]} + token_str = json.dumps(token, separators=(",", ":")).encode("utf-16le") + return base64.b64encode(token_str).decode() + + +class ModelInfo(TypedDict): + id: str + default_params: dict[str, Any] + + +model_info: dict[str, ModelInfo] = { + "anthropic:claude-instant-v1": { + "id": "anthropic:claude-instant-v1", + "default_params": { + "temperature": 1, + "maxTokens": 200, + "topP": 1, + "topK": 1, + "presencePenalty": 1, + "frequencyPenalty": 1, + "stopSequences": ["\n\nHuman:"], + }, + }, + "anthropic:claude-v1": { + "id": "anthropic:claude-v1", + "default_params": { + "temperature": 1, + "maxTokens": 200, + "topP": 1, + "topK": 1, + "presencePenalty": 1, + "frequencyPenalty": 1, + "stopSequences": ["\n\nHuman:"], + }, + }, + "anthropic:claude-v2": { + "id": "anthropic:claude-v2", + "default_params": { + "temperature": 1, + "maxTokens": 200, + "topP": 1, + "topK": 1, + "presencePenalty": 1, + "frequencyPenalty": 1, + "stopSequences": ["\n\nHuman:"], + }, + }, + "replicate:a16z-infra/llama7b-v2-chat": { + "id": "replicate:a16z-infra/llama7b-v2-chat", + "default_params": { + "temperature": 0.75, + "maxTokens": 500, + "topP": 1, + "repetitionPenalty": 1, + }, + }, + "replicate:a16z-infra/llama13b-v2-chat": { + "id": "replicate:a16z-infra/llama13b-v2-chat", + "default_params": { + "temperature": 0.75, + "maxTokens": 500, + "topP": 1, + "repetitionPenalty": 1, + }, + }, + "huggingface:bigscience/bloom": { + "id": "huggingface:bigscience/bloom", + "default_params": { + "temperature": 0.5, + "maxTokens": 200, + "topP": 0.95, + "topK": 4, + "repetitionPenalty": 1.03, + }, + }, + "huggingface:google/flan-t5-xxl": { + "id": "huggingface:google/flan-t5-xxl", + "default_params": { + "temperature": 0.5, + "maxTokens": 200, + "topP": 0.95, + "topK": 4, + "repetitionPenalty": 1.03, + }, + }, + "huggingface:EleutherAI/gpt-neox-20b": { + "id": "huggingface:EleutherAI/gpt-neox-20b", + "default_params": { + "temperature": 0.5, + "maxTokens": 200, + "topP": 0.95, + "topK": 4, + "repetitionPenalty": 1.03, + "stopSequences": [], + }, + }, + "huggingface:OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5": { + "id": "huggingface:OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5", + "default_params": {"maxTokens": 200, "typicalP": 0.2, "repetitionPenalty": 1}, + }, + "huggingface:OpenAssistant/oasst-sft-1-pythia-12b": { + "id": "huggingface:OpenAssistant/oasst-sft-1-pythia-12b", + "default_params": {"maxTokens": 200, "typicalP": 0.2, "repetitionPenalty": 1}, + }, + "huggingface:bigcode/santacoder": { + "id": "huggingface:bigcode/santacoder", + "default_params": { + "temperature": 0.5, + "maxTokens": 200, + "topP": 0.95, + "topK": 4, + "repetitionPenalty": 1.03, + }, + }, + "cohere:command-light-nightly": { + "id": "cohere:command-light-nightly", + "default_params": { + "temperature": 0.9, + "maxTokens": 200, + "topP": 1, + "topK": 0, + "presencePenalty": 0, + "frequencyPenalty": 0, + "stopSequences": [], + }, + }, + "cohere:command-nightly": { + "id": "cohere:command-nightly", + "default_params": { + "temperature": 0.9, + "maxTokens": 200, + "topP": 1, + "topK": 0, + "presencePenalty": 0, + "frequencyPenalty": 0, + "stopSequences": [], + }, + }, + "openai:gpt-4": { + "id": "openai:gpt-4", + "default_params": { + "temperature": 0.7, + "maxTokens": 500, + "topP": 1, + "presencePenalty": 0, + "frequencyPenalty": 0, + "stopSequences": [], + }, + }, + "openai:gpt-4-0613": { + "id": "openai:gpt-4-0613", + "default_params": { + "temperature": 0.7, + "maxTokens": 500, + "topP": 1, + "presencePenalty": 0, + "frequencyPenalty": 0, + "stopSequences": [], + }, + }, + "openai:code-davinci-002": { + "id": "openai:code-davinci-002", + "default_params": { + "temperature": 0.5, + "maxTokens": 200, + "topP": 1, + "presencePenalty": 0, + "frequencyPenalty": 0, + "stopSequences": [], + }, + }, + "openai:gpt-3.5-turbo": { + "id": "openai:gpt-3.5-turbo", + "default_params": { + "temperature": 0.7, + "maxTokens": 500, + "topP": 1, + "topK": 1, + "presencePenalty": 1, + "frequencyPenalty": 1, + "stopSequences": [], + }, + }, + "openai:gpt-3.5-turbo-16k": { + "id": "openai:gpt-3.5-turbo-16k", + "default_params": { + "temperature": 0.7, + "maxTokens": 500, + "topP": 1, + "topK": 1, + "presencePenalty": 1, + "frequencyPenalty": 1, + "stopSequences": [], + }, + }, + "openai:gpt-3.5-turbo-16k-0613": { + "id": "openai:gpt-3.5-turbo-16k-0613", + "default_params": { + "temperature": 0.7, + "maxTokens": 500, + "topP": 1, + "topK": 1, + "presencePenalty": 1, + "frequencyPenalty": 1, + "stopSequences": [], + }, + }, + "openai:text-ada-001": { + "id": "openai:text-ada-001", + "default_params": { + "temperature": 0.5, + "maxTokens": 200, + "topP": 1, + "presencePenalty": 0, + "frequencyPenalty": 0, + "stopSequences": [], + }, + }, + "openai:text-babbage-001": { + "id": "openai:text-babbage-001", + "default_params": { + "temperature": 0.5, + "maxTokens": 200, + "topP": 1, + "presencePenalty": 0, + "frequencyPenalty": 0, + "stopSequences": [], + }, + }, + "openai:text-curie-001": { + "id": "openai:text-curie-001", + "default_params": { + "temperature": 0.5, + "maxTokens": 200, + "topP": 1, + "presencePenalty": 0, + "frequencyPenalty": 0, + "stopSequences": [], + }, + }, + "openai:text-davinci-002": { + "id": "openai:text-davinci-002", + "default_params": { + "temperature": 0.5, + "maxTokens": 200, + "topP": 1, + "presencePenalty": 0, + "frequencyPenalty": 0, + "stopSequences": [], + }, + }, + "openai:text-davinci-003": { + "id": "openai:text-davinci-003", + "default_params": { + "temperature": 0.5, + "maxTokens": 200, + "topP": 1, + "presencePenalty": 0, + "frequencyPenalty": 0, + "stopSequences": [], + }, + }, +} -- cgit v1.2.3