diff options
Diffstat (limited to '')
| -rw-r--r-- | docker/background.png (renamed from g4f.png) | bin | 155456 -> 155456 bytes | |||
| -rw-r--r-- | g4f/Provider/MyShell.py | 18 | ||||
| -rw-r--r-- | g4f/Provider/unfinished/AiChatting.py | 66 | ||||
| -rw-r--r-- | g4f/Provider/unfinished/__init__.py | 3 | ||||
| -rw-r--r-- | g4f/__init__.py | 57 | ||||
| -rw-r--r-- | g4f/debug.py | 3 | ||||
| -rw-r--r-- | g4f/gui/client/css/style.css | 3 | ||||
| -rw-r--r-- | g4f/gui/client/html/index.html | 4 | ||||
| -rw-r--r-- | g4f/gui/client/js/chat.v1.js | 15 | ||||
| -rw-r--r-- | g4f/gui/server/backend.py | 24 | ||||
| -rw-r--r-- | g4f/gui/server/internet.py | 187 | ||||
| -rw-r--r-- | g4f/requests.py | 28 | ||||
| -rw-r--r-- | g4f/webdriver.py | 48 |
13 files changed, 333 insertions, 123 deletions
diff --git a/g4f.png b/docker/background.png Binary files differindex 41bf9e6b..41bf9e6b 100644 --- a/g4f.png +++ b/docker/background.png diff --git a/g4f/Provider/MyShell.py b/g4f/Provider/MyShell.py index 2ee94bb6..b0a01016 100644 --- a/g4f/Provider/MyShell.py +++ b/g4f/Provider/MyShell.py @@ -5,7 +5,7 @@ import time, json from ..typing import CreateResult, Messages from .base_provider import BaseProvider from .helper import format_prompt -from ..webdriver import WebDriver, WebDriverSession +from ..webdriver import WebDriver, WebDriverSession, bypass_cloudflare class MyShell(BaseProvider): url = "https://app.myshell.ai/chat" @@ -25,16 +25,8 @@ class MyShell(BaseProvider): **kwargs ) -> CreateResult: with WebDriverSession(webdriver, "", proxy=proxy) as driver: - from selenium.webdriver.common.by import By - from selenium.webdriver.support.ui import WebDriverWait - from selenium.webdriver.support import expected_conditions as EC - - driver.get(cls.url) - - # Wait for page load and cloudflare validation - WebDriverWait(driver, timeout).until( - EC.presence_of_element_located((By.CSS_SELECTOR, "body:not(.no-js)")) - ) + bypass_cloudflare(driver, cls.url, timeout) + # Send request with message data = { "botId": "4738", @@ -58,11 +50,11 @@ window._reader = response.body.pipeThrough(new TextDecoderStream()).getReader(); driver.execute_script(script.replace("{body}", json.dumps(data))) script = """ chunk = await window._reader.read(); -if (chunk['done']) { +if (chunk.done) { return null; } content = ''; -chunk['value'].split('\\n').forEach((line, index) => { +chunk.value.split('\\n').forEach((line, index) => { if (line.startsWith('data: ')) { try { const data = JSON.parse(line.substring('data: '.length)); diff --git a/g4f/Provider/unfinished/AiChatting.py b/g4f/Provider/unfinished/AiChatting.py new file mode 100644 index 00000000..a66921c1 --- /dev/null +++ b/g4f/Provider/unfinished/AiChatting.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from urllib.parse import unquote + +from ...typing import AsyncResult, Messages +from ..base_provider import BaseProvider +from ...webdriver import WebDriver +from ...requests import Session, get_session_from_browser + +class AiChatting(BaseProvider): + url = "https://www.aichatting.net" + supports_gpt_35_turbo = True + _session: Session = None + + @classmethod + def create_completion( + cls, + model: str, + messages: Messages, + stream: bool, + proxy: str = None, + timeout: int = 120, + webdriver: WebDriver = None, + **kwargs + ) -> AsyncResult: + if not cls._session: + cls._session = get_session_from_browser(cls.url, webdriver, proxy, timeout) + visitorId = unquote(cls._session.cookies.get("aichatting.website.visitorId")) + + headers = { + "accept": "application/json, text/plain, */*", + "lang": "en", + "source": "web" + } + data = { + "roleId": 0, + } + try: + response = cls._session.post("https://aga-api.aichatting.net/aigc/chat/record/conversation/create", json=data, headers=headers) + response.raise_for_status() + conversation_id = response.json()["data"]["conversationId"] + except Exception as e: + cls.reset() + raise e + headers = { + "authority": "aga-api.aichatting.net", + "accept": "text/event-stream,application/json, text/event-stream", + "lang": "en", + "source": "web", + "vtoken": visitorId, + } + data = { + "spaceHandle": True, + "roleId": 0, + "messages": messages, + "conversationId": conversation_id, + } + response = cls._session.post("https://aga-api.aichatting.net/aigc/chat/v2/stream", json=data, headers=headers, stream=True) + response.raise_for_status() + for chunk in response.iter_lines(): + if chunk.startswith(b"data:"): + yield chunk[5:].decode().replace("-=- --", " ").replace("-=-n--", "\n").replace("--@DONE@--", "") + + @classmethod + def reset(cls): + cls._session = None
\ No newline at end of file diff --git a/g4f/Provider/unfinished/__init__.py b/g4f/Provider/unfinished/__init__.py index 22e021be..eb5e8825 100644 --- a/g4f/Provider/unfinished/__init__.py +++ b/g4f/Provider/unfinished/__init__.py @@ -1,3 +1,4 @@ from .MikuChat import MikuChat from .Komo import Komo -from .ChatAiGpt import ChatAiGpt
\ No newline at end of file +from .ChatAiGpt import ChatAiGpt +from .AiChatting import AiChatting
\ No newline at end of file diff --git a/g4f/__init__.py b/g4f/__init__.py index 8bfed8e3..92bce194 100644 --- a/g4f/__init__.py +++ b/g4f/__init__.py @@ -1,25 +1,44 @@ from __future__ import annotations -from requests import get -from .models import Model, ModelUtils, _all_models -from .Provider import BaseProvider, AsyncGeneratorProvider, RetryProvider -from .typing import Messages, CreateResult, AsyncResult, Union, List -from . import debug -version = '0.1.9.2' -version_check = True +import os +from requests import get +from importlib.metadata import version as get_package_version, PackageNotFoundError +from subprocess import check_output, CalledProcessError, PIPE -def check_pypi_version() -> None: - try: - response = get("https://pypi.org/pypi/g4f/json").json() - latest_version = response["info"]["version"] +from .models import Model, ModelUtils, _all_models +from .Provider import BaseProvider, AsyncGeneratorProvider, RetryProvider +from .typing import Messages, CreateResult, AsyncResult, Union, List +from . import debug - if version != latest_version: - print(f'New pypi version: {latest_version} (current: {version}) | pip install -U g4f') - return False - return True +def get_version() -> str: + # Read from package manager + try: + return get_package_version("g4f") + except PackageNotFoundError: + pass + # Read from docker environment + current_version = os.environ.get("G4F_VERSION") + if current_version: + return current_version + # Read from git repository + try: + command = ["git", "describe", "--tags", "--abbrev=0"] + return check_output(command, text=True, stderr=PIPE).strip() + except CalledProcessError: + pass + +def get_lastet_version() -> str: + response = get("https://pypi.org/pypi/g4f/json").json() + return response["info"]["version"] +def check_pypi_version() -> None: + try: + version = get_version() + latest_version = get_lastet_version() except Exception as e: print(f'Failed to check g4f pypi version: {e}') + if version != latest_version: + print(f'New pypi version: {latest_version} (current: {version}) | pip install -U g4f') def get_model_and_provider(model : Union[Model, str], provider : Union[type[BaseProvider], None], @@ -27,6 +46,9 @@ def get_model_and_provider(model : Union[Model, str], ignored : List[str] = None, ignore_working: bool = False, ignore_stream: bool = False) -> tuple[Model, type[BaseProvider]]: + if debug.version_check: + check_pypi_version() + debug.version_check = False if isinstance(model, str): if model in ModelUtils.convert: @@ -118,7 +140,4 @@ class Completion: result = provider.create_completion(model.name, [{"role": "user", "content": prompt}], stream, **kwargs) - return result if stream else ''.join(result) - -if version_check: - check_pypi_version()
\ No newline at end of file + return result if stream else ''.join(result)
\ No newline at end of file diff --git a/g4f/debug.py b/g4f/debug.py index 558a2428..984d973a 100644 --- a/g4f/debug.py +++ b/g4f/debug.py @@ -1 +1,2 @@ -logging = False
\ No newline at end of file +logging = False +version_check = True
\ No newline at end of file diff --git a/g4f/gui/client/css/style.css b/g4f/gui/client/css/style.css index 2700de8b..254a4b15 100644 --- a/g4f/gui/client/css/style.css +++ b/g4f/gui/client/css/style.css @@ -211,6 +211,9 @@ body { .convo-title { color: var(--colour-3); font-size: 14px; + text-overflow: ellipsis; + overflow: hidden; + white-space: nowrap; } .message { diff --git a/g4f/gui/client/html/index.html b/g4f/gui/client/html/index.html index 66534a51..53c028d7 100644 --- a/g4f/gui/client/html/index.html +++ b/g4f/gui/client/html/index.html @@ -88,6 +88,10 @@ <span class="convo-title">github ~ <a href="https://github.com/xtekky/gpt4free">@gpt4free</a> </span> </div> + <div class="info"> + <i class="fa-solid fa-star"></i> + <span id="version_text" class="convo-title"></span> + </div> </div> </div> <div class="conversation"> diff --git a/g4f/gui/client/js/chat.v1.js b/g4f/gui/client/js/chat.v1.js index 2844b73e..2a1bdd73 100644 --- a/g4f/gui/client/js/chat.v1.js +++ b/g4f/gui/client/js/chat.v1.js @@ -628,4 +628,19 @@ observer.observe(message_input, { attributes: true }); option.value = option.text = provider; select.appendChild(option); } +})(); + +(async () => { + response = await fetch('/backend-api/v2/version') + versions = await response.json() + + document.title = 'g4f - gui - ' + versions["version"]; + text = "version ~ " + if (versions["version"] != versions["lastet_version"]) { + release_url = 'https://github.com/xtekky/gpt4free/releases/tag/' + versions["lastet_version"]; + text += '<a href="' + release_url +'" target="_blank" title="New version: ' + versions["lastet_version"] +'">' + versions["version"] + ' 🆕</a>'; + } else { + text += versions["version"]; + } + document.getElementById("version_text").innerHTML = text })();
\ No newline at end of file diff --git a/g4f/gui/server/backend.py b/g4f/gui/server/backend.py index e1abb764..082e31b2 100644 --- a/g4f/gui/server/backend.py +++ b/g4f/gui/server/backend.py @@ -1,8 +1,7 @@ import g4f from flask import request -from .internet import search -from .config import special_instructions +from .internet import get_search_message g4f.debug.logging = True @@ -18,6 +17,10 @@ class Backend_Api: 'function': self.providers, 'methods' : ['GET'] }, + '/backend-api/v2/version': { + 'function': self.version, + 'methods' : ['GET'] + }, '/backend-api/v2/conversation': { 'function': self._conversation, 'methods': ['POST'] @@ -45,6 +48,12 @@ class Backend_Api: provider.__name__ for provider in g4f.Provider.__providers__ if provider.working and provider is not g4f.Provider.RetryProvider ] + + def version(self): + return { + "version": g4f.get_version(), + "lastet_version": g4f.get_lastet_version(), + } def _gen_title(self): return { @@ -53,14 +62,15 @@ class Backend_Api: def _conversation(self): try: - #jailbreak = request.json['jailbreak'] - #internet_access = request.json['meta']['content']['internet_access'] - #conversation = request.json['meta']['content']['conversation'] + #jailbreak = request.json['jailbreak'] + web_search = request.json['meta']['content']['internet_access'] messages = request.json['meta']['content']['parts'] + if web_search: + messages[-1]["content"] = get_search_message(messages[-1]["content"]) model = request.json.get('model') model = model if model else g4f.models.default - provider = request.json.get('provider', 'Auto').replace('g4f.Provider.', '') - provider = provider if provider != "Auto" else None + provider = request.json.get('provider').replace('g4f.Provider.', '') + provider = provider if provider and provider != "Auto" else None if provider != None: provider = g4f.Provider.ProviderUtils.convert.get(provider) diff --git a/g4f/gui/server/internet.py b/g4f/gui/server/internet.py index 220a6e7c..9a14e25f 100644 --- a/g4f/gui/server/internet.py +++ b/g4f/gui/server/internet.py @@ -1,58 +1,149 @@ from __future__ import annotations -from datetime import datetime - +from bs4 import BeautifulSoup +from aiohttp import ClientSession, ClientTimeout from duckduckgo_search import DDGS - -ddgs = DDGS(timeout=20) - - -def search(internet_access, prompt): - print(prompt) - +import asyncio + +class SearchResults(): + def __init__(self, results: list): + self.results = results + + def __iter__(self): + yield from self.results + + def __str__(self): + search = "" + for idx, result in enumerate(self.results): + if search: + search += "\n\n\n" + search += f"Title: {result.title}\n\n" + if result.text: + search += result.text + else: + search += result.snippet + search += f"\n\nSource: [[{idx}]]({result.url})" + return search + +class SearchResultEntry(): + def __init__(self, title: str, url: str, snippet: str, text: str = None): + self.title = title + self.url = url + self.snippet = snippet + self.text = text + + def set_text(self, text: str): + self.text = text + +def scrape_text(html: str, max_words: int = None) -> str: + soup = BeautifulSoup(html, "html.parser") + for exclude in soup(["script", "style"]): + exclude.extract() + for selector in [ + "main", + ".main-content-wrapper", + ".main-content", + ".emt-container-inner", + ".content-wrapper", + "#content", + "#mainContent", + ]: + select = soup.select_one(selector) + if select: + soup = select + break + # Zdnet + for remove in [".c-globalDisclosure"]: + select = soup.select_one(remove) + if select: + select.extract() + clean_text = "" + for paragraph in soup.select("p"): + text = paragraph.get_text() + for line in text.splitlines(): + words = [] + for word in line.replace("\t", " ").split(" "): + if word: + words.append(word) + count = len(words) + if not count: + continue + if max_words: + max_words -= count + if max_words <= 0: + break + if clean_text: + clean_text += "\n" + clean_text += " ".join(words) + + return clean_text + +async def fetch_and_scrape(session: ClientSession, url: str, max_words: int = None) -> str: try: - if not internet_access: - return [] - - results = duckduckgo_search(q=prompt) - - if not search: - return [] + async with session.get(url) as response: + if response.status == 200: + html = await response.text() + return scrape_text(html, max_words) + except: + return + +async def search(query: str, n_results: int = 5, max_words: int = 2500, add_text: bool = True) -> SearchResults: + with DDGS() as ddgs: + results = [] + for result in ddgs.text( + query, + region="wt-wt", + safesearch="moderate", + timelimit="y", + ): + results.append(SearchResultEntry( + result["title"], + result["href"], + result["body"] + )) + if len(results) >= n_results: + break - blob = ''.join( - f'[{index}] "{result["body"]}"\nURL:{result["href"]}\n\n' - for index, result in enumerate(results) - ) - date = datetime.now().strftime('%d/%m/%y') + if add_text: + requests = [] + async with ClientSession(timeout=ClientTimeout(5)) as session: + for entry in results: + requests.append(fetch_and_scrape(session, entry.url, int(max_words / (n_results - 1)))) + texts = await asyncio.gather(*requests) + + formatted_results = [] + left_words = max_words; + for i, entry in enumerate(results): + if add_text: + entry.text = texts[i] + if left_words: + left_words -= entry.title.count(" ") + 5 + if entry.text: + left_words -= entry.text.count(" ") + else: + left_words -= entry.snippet.count(" ") + if 0 > left_words: + break + formatted_results.append(entry) + + return SearchResults(formatted_results) + + +def get_search_message(prompt) -> str: + try: + search_results = asyncio.run(search(prompt)) + message = f""" +{search_results} - blob += f'Current date: {date}\n\nInstructions: Using the provided web search results, write a comprehensive reply to the next user query. Make sure to cite results using [[number](URL)] notation after the reference. If the provided search results refer to multiple subjects with the same name, write separate answers for each subject. Ignore your previous response if any.' - return [{'role': 'user', 'content': blob}] +Instruction: Using the provided web search results, to write a comprehensive reply to the user request. +Make sure to add the sources of cites using [[Number]](Url) notation after the reference. Example: [[0]](http://google.com) +If the provided search results refer to multiple subjects with the same name, write separate answers for each subject. +User request: +{prompt} +""" + return message except Exception as e: print("Couldn't search DuckDuckGo:", e) - print(e.__traceback__.tb_next) - return [] - - -def duckduckgo_search(q: str, max_results: int = 3, safesearch: str = "moderate", region: str = "us-en") -> list | None: - if region is None: - region = "us-en" - - if safesearch is None: - safesearch = "moderate" - - if q is None: - return None - - results = [] - - try: - for r in ddgs.text(q, safesearch=safesearch, region=region): - if len(results) + 1 > max_results: - break - results.append(r) - except Exception as e: - print(e) - - return results + return prompt diff --git a/g4f/requests.py b/g4f/requests.py index 8cf70ac9..00ab9488 100644 --- a/g4f/requests.py +++ b/g4f/requests.py @@ -6,10 +6,7 @@ from functools import partialmethod from typing import AsyncGenerator from urllib.parse import urlparse from curl_cffi.requests import AsyncSession, Session, Response -from .webdriver import WebDriver, WebDriverSession -from selenium.webdriver.common.by import By -from selenium.webdriver.support.ui import WebDriverWait -from selenium.webdriver.support import expected_conditions as EC +from .webdriver import WebDriver, WebDriverSession, bypass_cloudflare class StreamResponse: def __init__(self, inner: Response) -> None: @@ -58,28 +55,7 @@ class StreamSession(AsyncSession): def get_session_from_browser(url: str, webdriver: WebDriver = None, proxy: str = None, timeout: int = 120): with WebDriverSession(webdriver, "", proxy=proxy, virtual_display=True) as driver: - driver.get(url) - - # Is cloudflare protection - if driver.find_element(By.TAG_NAME, "body").get_attribute("class") == "no-js": - try: - # Click button in iframe - WebDriverWait(driver, 5).until( - EC.presence_of_element_located((By.CSS_SELECTOR, "#turnstile-wrapper iframe")) - ) - driver.switch_to.frame(driver.find_element(By.CSS_SELECTOR, "#turnstile-wrapper iframe")) - WebDriverWait(driver, 5).until( - EC.presence_of_element_located((By.CSS_SELECTOR, "#challenge-stage input")) - ) - driver.find_element(By.CSS_SELECTOR, "#challenge-stage input").click() - except: - pass - finally: - driver.switch_to.default_content() - # No cloudflare protection - WebDriverWait(driver, timeout).until( - EC.presence_of_element_located((By.CSS_SELECTOR, "body:not(.no-js)")) - ) + bypass_cloudflare(driver, url, timeout) cookies = dict([(cookie["name"], cookie["value"]) for cookie in driver.get_cookies()]) user_agent = driver.execute_script("return navigator.userAgent") diff --git a/g4f/webdriver.py b/g4f/webdriver.py index f0fa1fba..d274c619 100644 --- a/g4f/webdriver.py +++ b/g4f/webdriver.py @@ -1,10 +1,12 @@ from __future__ import annotations -import time from platformdirs import user_config_dir from selenium.webdriver.remote.webdriver import WebDriver from undetected_chromedriver import Chrome, ChromeOptions -import os.path +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from os import path from . import debug try: @@ -21,16 +23,47 @@ def get_browser( ) -> WebDriver: if user_data_dir == None: user_data_dir = user_config_dir("g4f") - if debug.logging: - print("Open browser with config dir:", user_data_dir) + if user_data_dir and debug.logging: + print("Open browser with config dir:", user_data_dir) if not options: options = ChromeOptions() if proxy: options.add_argument(f'--proxy-server={proxy}') driver = '/usr/bin/chromedriver' - if not os.path.isfile(driver): + if not path.isfile(driver): driver = None - return Chrome(options=options, user_data_dir=user_data_dir, driver_executable_path=driver, headless=headless) + return Chrome( + options=options, + user_data_dir=user_data_dir, + driver_executable_path=driver, + headless=headless + ) + +def bypass_cloudflare(driver: WebDriver, url: str, timeout: int) -> None: + # Open website + driver.get(url) + # Is cloudflare protection + if driver.find_element(By.TAG_NAME, "body").get_attribute("class") == "no-js": + if debug.logging: + print("Cloudflare protection detected:", url) + try: + # Click button in iframe + WebDriverWait(driver, 5).until( + EC.presence_of_element_located((By.CSS_SELECTOR, "#turnstile-wrapper iframe")) + ) + driver.switch_to.frame(driver.find_element(By.CSS_SELECTOR, "#turnstile-wrapper iframe")) + WebDriverWait(driver, 5).until( + EC.presence_of_element_located((By.CSS_SELECTOR, "#challenge-stage input")) + ) + driver.find_element(By.CSS_SELECTOR, "#challenge-stage input").click() + except: + pass + finally: + driver.switch_to.default_content() + # No cloudflare protection + WebDriverWait(driver, timeout).until( + EC.presence_of_element_located((By.CSS_SELECTOR, "body:not(.no-js)")) + ) class WebDriverSession(): def __init__( @@ -47,7 +80,7 @@ class WebDriverSession(): self.headless = headless self.virtual_display = None if has_pyvirtualdisplay and virtual_display: - self.virtual_display = Display(size=(1920,1080)) + self.virtual_display = Display(size=(1920, 1080)) self.proxy = proxy self.options = options self.default_driver = None @@ -82,7 +115,6 @@ class WebDriverSession(): self.default_driver.close() except: pass - time.sleep(0.1) self.default_driver.quit() if self.virtual_display: self.virtual_display.stop()
\ No newline at end of file |
