from __future__ import annotations
import asyncio
import json
import uuid
import cloudscraper
from typing import AsyncGenerator
from ..typing import AsyncResult, Messages
from .base_provider import AsyncGeneratorProvider, ProviderModelMixin
from .helper import format_prompt
class Cloudflare(AsyncGeneratorProvider, ProviderModelMixin):
url = "https://playground.ai.cloudflare.com"
api_endpoint = "https://playground.ai.cloudflare.com/api/inference"
working = True
supports_stream = True
supports_system_message = True
supports_message_history = True
default_model = '@cf/meta/llama-3.1-8b-instruct'
models = [
'@cf/deepseek-ai/deepseek-math-7b-instruct', # Specific answer
'@cf/thebloke/discolm-german-7b-v1-awq',
'@cf/tiiuae/falcon-7b-instruct', # Specific answer
'@hf/google/gemma-7b-it',
'@cf/meta/llama-2-7b-chat-fp16',
'@cf/meta/llama-2-7b-chat-int8',
'@cf/meta/llama-3-8b-instruct',
'@cf/meta/llama-3-8b-instruct-awq',
default_model,
'@hf/meta-llama/meta-llama-3-8b-instruct',
'@cf/meta/llama-3.1-8b-instruct-awq',
'@cf/meta/llama-3.1-8b-instruct-fp8',
'@cf/meta/llama-3.2-11b-vision-instruct',
'@cf/meta/llama-3.2-1b-instruct',
'@cf/meta/llama-3.2-3b-instruct',
'@cf/mistral/mistral-7b-instruct-v0.1',
'@hf/mistral/mistral-7b-instruct-v0.2',
'@cf/openchat/openchat-3.5-0106',
'@cf/microsoft/phi-2',
'@cf/qwen/qwen1.5-0.5b-chat',
'@cf/qwen/qwen1.5-1.8b-chat',
'@cf/qwen/qwen1.5-14b-chat-awq',
'@cf/qwen/qwen1.5-7b-chat-awq',
'@cf/defog/sqlcoder-7b-2', # Specific answer
'@cf/tinyllama/tinyllama-1.1b-chat-v1.0',
'@cf/fblgit/una-cybertron-7b-v2-bf16',
]
model_aliases = {
"german-7b-v1": "@cf/thebloke/discolm-german-7b-v1-awq",
"gemma-7b": "@hf/google/gemma-7b-it",
"llama-2-7b": "@cf/meta/llama-2-7b-chat-fp16",
"llama-2-7b": "@cf/meta/llama-2-7b-chat-int8",
"llama-3-8b": "@cf/meta/llama-3-8b-instruct",
"llama-3-8b": "@cf/meta/llama-3-8b-instruct-awq",
"llama-3-8b": "@cf/meta/llama-3.1-8b-instruct",
"llama-3-8b": "@hf/meta-llama/meta-llama-3-8b-instruct",
"llama-3.1-8b": "@cf/meta/llama-3.1-8b-instruct-awq",
"llama-3.1-8b": "@cf/meta/llama-3.1-8b-instruct-fp8",
"llama-3.1-8b": "@cf/meta/llama-3.1-8b-instruct-fp8",
"llama-3.2-11b": "@cf/meta/llama-3.2-11b-vision-instruct",
"llama-3.2-1b": "@cf/meta/llama-3.2-1b-instruct",
"llama-3.2-3b": "@cf/meta/llama-3.2-3b-instruct",
"mistral-7b": "@cf/mistral/mistral-7b-instruct-v0.1",
"mistral-7b": "@hf/mistral/mistral-7b-instruct-v0.2",
"openchat-3.5": "@cf/openchat/openchat-3.5-0106",
"phi-2": "@cf/microsoft/phi-2",
"qwen-1.5-0.5b": "@cf/qwen/qwen1.5-0.5b-chat",
"qwen-1.5-1.8b": "@cf/qwen/qwen1.5-1.8b-chat",
"qwen-1.5-14b": "@cf/qwen/qwen1.5-14b-chat-awq",
"qwen-1.5-7b": "@cf/qwen/qwen1.5-7b-chat-awq",
"tinyllama-1.1b": "@cf/tinyllama/tinyllama-1.1b-chat-v1.0",
"cybertron-7b": "@cf/fblgit/una-cybertron-7b-v2-bf16",
}
@classmethod
def get_model(cls, model: str) -> str:
if model in cls.models:
return model
elif model in cls.model_aliases:
return cls.model_aliases[model]
else:
return cls.default_model
@classmethod
async def create_async_generator(
cls,
model: str,
messages: Messages,
proxy: str = None,
max_tokens: str = 2048,
stream: bool = True,
**kwargs
) -> AsyncResult:
model = cls.get_model(model)
headers = {
'Accept': 'text/event-stream',
'Accept-Language': 'en-US,en;q=0.9',
'Cache-Control': 'no-cache',
'Content-Type': 'application/json',
'Origin': cls.url,
'Pragma': 'no-cache',
'Referer': f'{cls.url}/',
'Sec-Ch-Ua': '"Chromium";v="129", "Not=A?Brand";v="8"',
'Sec-Ch-Ua-Mobile': '?0',
'Sec-Ch-Ua-Platform': '"Linux"',
'Sec-Fetch-Dest': 'empty',
'Sec-Fetch-Mode': 'cors',
'Sec-Fetch-Site': 'same-origin',
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36',
}
cookies = {
'__cf_bm': uuid.uuid4().hex,
}
scraper = cloudscraper.create_scraper()
prompt = format_prompt(messages)
data = {
"messages": [
{"role": "system", "content": "You are a helpful assistant"},
{"role": "user", "content": prompt}
],
"lora": None,
"model": model,
"max_tokens": max_tokens,
"stream": stream
}
max_retries = 3
for attempt in range(max_retries):
try:
response = scraper.post(
cls.api_endpoint,
headers=headers,
cookies=cookies,
json=data,
stream=True,
proxies={'http': proxy, 'https': proxy} if proxy else None
)
if response.status_code == 403:
await asyncio.sleep(2 ** attempt)
continue
response.raise_for_status()
for line in response.iter_lines():
if line.startswith(b'data: '):
if line == b'data: [DONE]':
break
try:
content = json.loads(line[6:].decode('utf-8'))['response']
yield content
except Exception:
continue
break
except Exception as e:
if attempt == max_retries - 1:
raise
@classmethod
async def create_async(
cls,
model: str,
messages: Messages,
proxy: str = None,
**kwargs
) -> str:
full_response = ""
async for response in cls.create_async_generator(model, messages, proxy, **kwargs):
full_response += response
return full_response
