From 60a8ec295b0f32f5e48a4366b741af3dfbbeeae9 Mon Sep 17 00:00:00 2001 From: KimigaiiWuyi <444835641@qq.com> Date: Thu, 12 Oct 2023 02:00:58 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20=E6=96=B0=E5=A2=9E`utils.download?= =?UTF-8?q?=5Fresource`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gsuid_core/bot.py | 31 +++- gsuid_core/logger.py | 2 +- gsuid_core/segment.py | 3 +- .../utils/download_resource/download_core.py | 140 ++++++++++++++++++ .../utils/download_resource/download_file.py | 28 ++++ 5 files changed, 195 insertions(+), 9 deletions(-) create mode 100644 gsuid_core/utils/download_resource/download_core.py create mode 100644 gsuid_core/utils/download_resource/download_file.py diff --git a/gsuid_core/bot.py b/gsuid_core/bot.py index a77d978..c11a112 100644 --- a/gsuid_core/bot.py +++ b/gsuid_core/bot.py @@ -103,7 +103,9 @@ class Bot: reply: Optional[ Union[Message, List[Message], List[str], str, bytes] ] = None, - option_list: Optional[List[Union[str, Button]]] = None, + option_list: Optional[ + Union[List[str], List[Button], List[List[str]], List[List[Button]]] + ] = None, unsuported_platform: bool = False, ): return await self.receive_resp( @@ -115,7 +117,9 @@ class Bot: reply: Optional[ Union[Message, List[Message], List[str], str, bytes] ] = None, - option_list: Optional[List[Union[str, Button]]] = None, + option_list: Optional[ + Union[List[str], List[Button], List[List[str]], List[List[Button]]] + ] = None, unsuported_platform: bool = False, is_recive: bool = True, timeout: float = 60, @@ -128,20 +132,33 @@ class Bot: if self.ev.real_bot_id in ['qqgroup']: _reply_str = await to_markdown(_reply) - _buttons: List[Button] = [] + _buttons = [] for option in option_list: - if isinstance(option, Button): + if isinstance(option, List): + _button_row: List[Button] = [] + for op in option: + if isinstance(op, Button): + _button_row.append(op) + else: + _button_row.append(Button(op, op, op)) + _buttons.append(_button_row) + elif isinstance(option, Button): _buttons.append(option) else: _buttons.append(Button(option, option, option)) - logger.debug(_reply_str) - logger.debug(_buttons) + await self.send(MessageSegment.markdown(_reply_str, _buttons)) else: if unsuported_platform: _options: List[str] = [] for option in option_list: - if isinstance(option, Button): + if isinstance(option, List): + for op in option: + if isinstance(op, Button): + _options.append(op.data) + else: + _options.append(op) + elif isinstance(option, Button): _options.append(option.data) else: _options.append(option) diff --git a/gsuid_core/logger.py b/gsuid_core/logger.py index 4fe68c2..bb82fea 100644 --- a/gsuid_core/logger.py +++ b/gsuid_core/logger.py @@ -113,6 +113,6 @@ logger.add( format=format_event, rotation=datetime.time(), level=LEVEL, - # diagnose=False, + diagnose=False, # backtrace=False, ) diff --git a/gsuid_core/segment.py b/gsuid_core/segment.py index b750826..25737d7 100644 --- a/gsuid_core/segment.py +++ b/gsuid_core/segment.py @@ -63,7 +63,8 @@ class MessageSegment: @staticmethod def markdown( - content: str, buttons: Optional[List[Button]] = None + content: str, + buttons: Optional[Union[List[Button], List[List[Button]]]] = None, ) -> List[Message]: data = [Message(type='markdown', data=content)] if buttons: diff --git a/gsuid_core/utils/download_resource/download_core.py b/gsuid_core/utils/download_resource/download_core.py new file mode 100644 index 0000000..4bdcf01 --- /dev/null +++ b/gsuid_core/utils/download_resource/download_core.py @@ -0,0 +1,140 @@ +import os +import time +import asyncio +from typing import Dict +from pathlib import Path + +import aiohttp +from bs4 import BeautifulSoup +from aiohttp import TCPConnector +from aiohttp.client import ClientSession, ClientTimeout + +from gsuid_core.logger import logger + +from .download_file import download + +TAG: str = '[HKFRP]' +BASE_URL = 'http://hk-1.5gbps-2.lcf.icu:10200/' + + +async def check_url(tag: str, url: str): + async with aiohttp.ClientSession() as session: + try: + start_time = time.time() + async with session.get(url) as response: + elapsed_time = time.time() - start_time + if response.status == 200: + logger.info(f'{tag} {url} 延时: {elapsed_time}') + return tag, url, elapsed_time + else: + logger.info(f'{tag} {url} 超时...') + return tag, url, float('inf') + except aiohttp.ClientError: + logger.info(f'{tag} {url} 超时...') + return tag, url, float('inf') + + +async def find_fastest_url(urls: Dict[str, str]): + tasks = [] + for tag in urls: + tasks.append(asyncio.create_task(check_url(tag, urls[tag]))) + + results = await asyncio.gather(*tasks, return_exceptions=True) + fastest_tag = '' + fastest_url = None + fastest_time = float('inf') + + for result in results: + if isinstance(result, Exception): + continue + tag, url, elapsed_time = result + if elapsed_time < fastest_time: + fastest_url = url + fastest_time = elapsed_time + fastest_tag = tag + + return fastest_tag, fastest_url + + +async def check_speed(): + logger.info('[GsCore资源下载]测速中...') + + URL_LIB = { + '[JPFRP]': 'http://jp-2.lcf.icu:13643', + '[qxqx]': 'https://kr-arm.qxqx.me', + '[HKFRP]': 'http://hk-1.5gbps-2.lcf.icu:10200', + } + + global TAG + global BASE_URL + TAG, BASE_URL = await find_fastest_url(URL_LIB) + logger.info(f"最快资源站: {TAG} {BASE_URL}") + + +async def _get_url(url: str, sess: ClientSession): + req = await sess.get(url=url) + return await req.read() + + +async def download_all_file(plugin_name: str, EPATH_MAP: Dict[str, Path]): + await check_speed() + + PLUGIN_RES = f'{BASE_URL}/{plugin_name}' + + TASKS = [] + async with ClientSession( + connector=TCPConnector(verify_ssl=False), + timeout=ClientTimeout(total=None, sock_connect=20, sock_read=200), + ) as sess: + for endpoint in EPATH_MAP: + url = f'{PLUGIN_RES}/{endpoint}/' + path = EPATH_MAP[endpoint] + + base_data = await _get_url(url, sess) + content_bs = BeautifulSoup(base_data, 'lxml') + pre_data = content_bs.find_all('pre')[0] + data_list = pre_data.find_all('a') + size_list = [i for i in content_bs.strings] + logger.info(f'{TAG} 数据库 {endpoint} 中存在 {len(data_list)} 个内容!') + + temp_num = 0 + for index, data in enumerate(data_list): + if data['href'] == '../': + continue + file_url = f'{url}{data["href"]}' + name: str = data.text + size = size_list[index * 2 + 6].split(' ')[-1] + size = size.replace('\r\n', '') + file_path = path / name + if file_path.exists(): + is_diff = size == str(os.stat(file_path).st_size) + else: + is_diff = True + if ( + not file_path.exists() + or not os.stat(file_path).st_size + or not is_diff + ): + logger.info( + f'{TAG} {plugin_name} 开始下载 {endpoint}/{name} ...' + ) + temp_num += 1 + TASKS.append( + asyncio.wait_for( + download(file_url, path, name, sess, TAG), + timeout=600, + ) + ) + if len(TASKS) >= 10: + await asyncio.gather(*TASKS) + TASKS.clear() + else: + await asyncio.gather(*TASKS) + TASKS.clear() + + if temp_num == 0: + im = f'{TAG} 数据库 {endpoint} 无需下载!' + else: + im = f'{TAG}数据库 {endpoint} 已下载{temp_num}个内容!' + temp_num = 0 + logger.info(im) diff --git a/gsuid_core/utils/download_resource/download_file.py b/gsuid_core/utils/download_resource/download_file.py new file mode 100644 index 0000000..22d47da --- /dev/null +++ b/gsuid_core/utils/download_resource/download_file.py @@ -0,0 +1,28 @@ +from pathlib import Path +from typing import Optional + +import aiofiles +from aiohttp.client import ClientSession +from aiohttp.client_exceptions import ClientConnectorError + +from gsuid_core.logger import logger + + +async def download( + url: str, + path: Path, + name: str, + sess: Optional[ClientSession] = None, + tag: str = '', +): + if sess is None: + sess = ClientSession() + + try: + async with sess.get(url) as res: + content = await res.read() + async with aiofiles.open(path / name, "wb") as f: + await f.write(content) + logger.info(f'{tag} {name} 下载完成！') + except ClientConnectorError: + logger.warning(f"{tag} {name} 下载失败！")