update

message/message_coinmarketcap.py

@@ -13,93 +13,64 @@ from base.base_load_config import load_config, get_base_path
 config_json = load_config()
 base_project = get_base_path()
 
-DB_USER = config_json.get('DB_USER')
-DB_PASSWORD = config_json.get('DB_PASSWORD')
-DB_IP = config_json.get('DB_IP')
-DB_PORT = config_json.get('DB_PORT')
-MONGO_LINK = f'mongodb://{DB_USER}:{DB_PASSWORD}@{DB_IP}:{DB_PORT}/'
-
-from utils.utils_mongo_handle import MongoHandle
-
 from message_check_base import *
 from utils.utils_send_gotify import *
-from utils.utils_send_serverchan import *
 
 
 class CheckCoinmarketcap:
     def __init__(self):
-        self.db = 'Message'
-        self.collection = 'coin_price'
-        self.client = MongoHandle(self.db, self.collection)
         self.currency = '¥'
         self.url_list = [
-            {'BTC': 'https://www.coinmarketcap.com/zh/currencies/bitcoin/'},
-            {'ETH': 'https://www.coinmarketcap.com/zh/currencies/ethereum/'},
-            {'DOGE': 'https://coinmarketcap.com/zh/currencies/dogecoin/'},
-            {'ARB': 'https://coinmarketcap.com/zh/currencies/arbitrum/'},
-            {'ATH': 'https://coinmarketcap.com/zh/currencies/aethir/'},
+            {'BTC': 'https://www.coinmarketcap.com/currencies/bitcoin/'},
+            {'ETH': 'https://www.coinmarketcap.com/currencies/ethereum/'},
+            {'DOGE': 'https://coinmarketcap.com/currencies/dogecoin/'},
+            {'ARB': 'https://coinmarketcap.com/currencies/arbitrum/'},
+            {'ATH': 'https://coinmarketcap.com/currencies/aethir/'},
+            {'SUI': 'https://coinmarketcap.com/currencies/sui/'},
         ]
 
-        self.selectors = ['#section-coin-overview > div.sc-65e7f566-0.czwNaM.flexStart.alignBaseline > span']
-
-    def processed_data(self, all_data):
-        # TODO 写入并读取当前数据和上次数据作对比
-        ts = str(int(time.time()))
-        data = {ts: {}}
-        for i in all_data:
-            for key, value in i[0].items():
-                data[ts].update({key: float(value.replace(self.currency, '').replace(',', ''))})
-
-        self.client.write_data(data)
+        # self.selectors = ['#section-coin-overview > div.sc-65e7f566-0.czwNaM.flexStart.alignBaseline > span']
+        self.selectors = ['div#section-coin-overview > div:nth-of-type(2)']
 
-        load_data = self.client.load_data()
-        if len(load_data) < 2:
-            return {}
+    def process_data(self, all_data):
+        result = {}
+        for data in all_data:
+            for key, value in data[0].items():
+                value_list = value.split('\xa0')
+                value_list = [item for item in value_list if item]
+                if key not in result:
+                    result[key] = value_list
+        return result
 
-        current_data = [value for value in load_data[-1].values()][0]
-        last_data = [value for value in load_data[-2].values()][0]
-
-        result_data = {}
+    def send_data(self, all_data):
+        # 打印结果，只包含货币名称和价格
+        context = ''
 
-        for current_key, current_value in current_data.items():
-            for last_key, last_value in last_data.items():
-                if current_key == last_key:
-                    change = ((current_value - last_value) / last_value) * 100
-                    if change != 0:
-                        change = round(change, 3)
-                    else:
-                        change = 0
-                    result_data[current_key] = [f'{self.currency}{current_value}', f'{change}%']
+        for key, value in all_data.items():
+            context += f'{key}: {value[0]}     {value[1]}    {value[2]}\n'
 
-        return result_data
+        if context:
+            context += '\n{}'.format(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
+            print(context)
+            # 推送到 message
+            GotifyNotifier('实时coin价格', context).send_message()
+        else:
+            print('no data!')
 
     def main(self):
         # 运行异步函数
         all_data = None
         crawler = CryptoCrawler(self.url_list, self.selectors, headless=True)
-        all_data = crawler.main()  # 返回的数据格式: [[{'BTC': '¥411,060.52'}], [{'ETH': '¥16,829.59'}], [{'DOGE': '¥0.726'}], [{'ARB': '¥3.69'}], [{'ATH': '¥0.3642'}]])
+        all_data = crawler.main()  # 返回的数据格式: [[{'BTC': '¥483,784.76\xa0\xa01.02%\xa0(1天)'}], ...
+
         if not all_data:
             print('获取数据失败')
             return
 
-        # TODO 加一个 mongo 存取历史数据, 并且对比上次检测时的增减
-        processed_data = self.processed_data(all_data)
-
-        # 打印结果，只包含货币名称和价格
-        context = ''
-
-        for key, value in processed_data.items():
-            context += f'{key}: {value[0]}     {value[1]}\n'
-
-        if context:
-            context += '\n{}'.format(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
-            # 推送到 message
-            GotifyNotifier('实时coin价格', context).send_message()
+        # 清理一下数据
+        all_data = self.process_data(all_data)  # {'BTC': ['¥483,482.98', '0.95%', '(1天)'], ...
 
-            # 推送到 serverchan
-            ServerChanNotifier('实时coin价格', context.replace('\n', '\n\n')).send_message()
-        else:
-            print('no data!')
+        self.send_data(all_data)
 
 
 if __name__ == '__main__':

message/message_dlt.py

@@ -9,7 +9,6 @@ import os
 sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'))
 
 from utils.utils_send_gotify import *
-from utils.utils_send_serverchan import *
 from utils.utils_send_email import *
 
 
@@ -147,9 +146,6 @@ class CheckDlt:
         # 推送到 message
         GotifyNotifier(title, text).send_message()
 
-        # 推送到 serverchan
-        ServerChanNotifier(title, text.replace('\n', '\n\n')).send_message()
-
         # 发送到邮件
         SendEmail(title, title, text).send()
 

message/message_get_one_week_weather.py

@@ -12,7 +12,6 @@ sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'
 
 from utils.utils_logs_handle import LogsHandle
 from utils.utils_send_gotify import *
-from utils.utils_send_serverchan import *
 
 
 class Weather():
@@ -61,9 +60,6 @@ class Weather():
         # 推送到 message
         GotifyNotifier('天气预报数', text).send_message()
 
-        # 推送到 serverchan
-        # ServerChanNotifier('天气预报数', text.replace('\n', '\n\n')).send_message()
-
         self.logs_handle.logs_write('Weather forecast', '天气预报数据已获取', 'done', False)
 
 

message/message_maizi_gas.py

@@ -7,10 +7,8 @@ import sys
 
 sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'))
 
-import httpx
 
 from utils.utils_send_gotify import *
-from utils.utils_send_serverchan import *
 
 url = "https://a5.maiziqianbao.net/api/v1/chains/EVM/1/gas_price"
 
@@ -51,19 +49,13 @@ try:
 
     print(result)
 
-    if int(fastest) > 20 or int(fast) > 20 or int(standard) > 20 or int(low) > 20 or int(base) > 20:
+    if int(fastest) > 30 or int(fast) > 30 or int(standard) > 30 or int(low) > 30 or int(base) > 30:
         # 推送到 message
-        GotifyNotifier('gas 费大于 20', result).send_message()
-
-        # 推送到 serverchan
-        ServerChanNotifier('gas 费大于 20', result.replace('\n', '\n\n')).send_message()
+        GotifyNotifier('gas 费大于 30', result).send_message()
     elif int(fastest) < 5 or int(fast) < 5 or int(standard) < 5 or int(low) < 5 or int(base) < 5:
         # 推送到 message
         GotifyNotifier('gas 费小于 5', result).send_message()
 
-        # 推送到 serverchan
-        ServerChanNotifier('gas 费小于 5', result.replace('\n', '\n\n')).send_message()
-
 
 except Exception as e:
     print(e)

message/message_rss_data_handel.py

@@ -9,7 +9,7 @@ from psycopg2 import Error
 
 class FreshRSSDatabase:
     def __init__(self):
-        self.hostname = '192.168.100.122'
+        self.hostname = 'erhe.top'
         self.port = 20788
         self.database = 'freshrss'
         self.user = 'freshrss'
@@ -17,7 +17,7 @@ class FreshRSSDatabase:
         self.conn = None
         self.keys = [
             {'web3新闻': 'web3|btc|eth|区块链|NFT|数字币|数字资产|Dapp|DeFi|NFT|稳定币|元宇宙|GameFi|跨链|以太坊'},
-            {'购物类新闻': '硬盘|鼠标|纸巾'}
+            {'购物类新闻': '大疆|无人机|硬盘|鼠标|纸巾'}
         ]
         self.ellipsis = 300
 
@@ -40,7 +40,7 @@ class FreshRSSDatabase:
     def execute_query(self, key):
         sql = f"""
         SELECT *
-        FROM freshrsstoor_entry
+        FROM freshrss_toor_entry
         WHERE title LIKE %s 
         OR content LIKE %s
         AND "date" > EXTRACT(EPOCH FROM NOW() - INTERVAL '1 day')

message/message_vix.py

@@ -11,7 +11,6 @@ sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'
 
 from message_check_base import *
 from utils.utils_send_gotify import *
-from utils.utils_send_serverchan import *
 
 
 class CheckVIX:
@@ -67,9 +66,6 @@ class CheckVIX:
             context += '\n{}'.format(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
             # 推送到 message
             GotifyNotifier('vix恐慌指数', context).send_message()
-
-            # 推送到 serverchan
-            ServerChanNotifier('vix恐慌指数', context.replace('\n', '\n\n')).send_message()
         else:
             print(f"VIX恐慌指数小于{self.check_value}，不发送消息\n{context}")
 

remind/auto_remind.py

@@ -9,7 +9,6 @@ sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'
 from datetime import datetime
 
 from utils.utils_send_gotify import GotifyNotifier
-from utils.utils_send_serverchan import ServerChanNotifier
 
 from base.base_load_config import load_config, get_base_path
 
@@ -50,9 +49,6 @@ class AutoRemind:
                     # 推送到 message
                     GotifyNotifier(title, context).send_message()
 
-                    # 推送到 serverchan
-                    ServerChanNotifier(title, context.replace('\n', '\n\n')).send_message()
-
                     # 发送后 retry - 1
                     task_data = {
                         'title': task_data['title'],

utils/utils_proxy.py

@@ -1,44 +0,0 @@
-# -*- coding: utf-8 -*-
-import subprocess
-import os
-import platform
-from base.base_load_config import load_config
-
-comfig_json = load_config()
-
-use_proxy = comfig_json.get('USE_PROXY')
-proxy_host = comfig_json.get('PROXY_HOST')
-proxy_port = comfig_json.get('PROXY_PORT')
-
-if use_proxy:
-    system = platform.system()
-
-    if system == 'Windows':
-        env = os.environ.copy()
-        command = ['set',
-                   'http_proxy=http://{}:{}'.format(proxy_host, proxy_port),
-                   '&',
-                   'set',
-                   'https_proxy=http://{}:{}'.format(proxy_host, proxy_port)
-                   ]
-    elif system == 'Darwin':
-        env = os.environ.copy()
-        command = ['export',
-                   'https_proxy=http://{}:{}'.format(proxy_host, proxy_port),
-                   'http_proxy=http://{}:{}'.format(proxy_host, proxy_port),
-                   'all_proxy=socks5://{}:{}'.format(proxy_host, proxy_port)
-                   ]
-    elif system == 'Linux':
-        env = os.environ.copy()
-        command = ['export',
-                   'https_proxy=http://{}:{}'.format(proxy_host, proxy_port),
-                   'http_proxy=http://{}:{}'.format(proxy_host, proxy_port),
-                   'all_proxy=socks5://{}:{}'.format(proxy_host, proxy_port)
-                   ]
-    else:
-        print("未知操作系统")
-        exit(0)
-
-    result = subprocess.run(command, text=True, capture_output=True)
-
-    print(result)

utils/utils_ql_create_tasks.py

@@ -4,7 +4,7 @@ import os
 import requests
 
 # 青龙面板的地址
-url = "https://ql-message.erhe.top"
+url = "https://auto.erhe.top"
 
 
 # 登录青龙面板
@@ -74,9 +74,7 @@ def main():
                 print("Tasks list is empty")
 
             project_path = '/ql/data/scripts/auto/'
-            spider_common_path = os.path.join(project_path, 'spider_common')
-            spider_news_path = os.path.join(project_path, 'spider_news')
-            spider_rsshub_path = os.path.join(project_path, 'spider_rsshub')
+            spider_path = os.path.join(project_path, 'spider')
             utils_path = os.path.join(project_path, 'utils')
             message_path = os.path.join(project_path, 'message')
             rss_data_path = os.path.join(project_path, 'rss_data_handel')
@@ -107,58 +105,48 @@ def main():
                         "labels": ["base"]
                     }
                 ],
-                'spider_common_tasks': [
-                    {
-                        "name": "自动执行dlt爬虫",
-                        "command": "python3 {}/spider_get_and_check_dlt.py".format(spider_common_path),
-                        "schedule": "30 22 * * 1,3,6",
-                        "labels": ["spider_common"]
-                    }
-                ],
-                'spider_news': [
+                'spider': [
                     {
                         "name": "自动执行反斗限免爬虫",
-                        "command": "python3 {}/news_get_apprcn.py".format(spider_news_path),
+                        "command": "python3 {}/news_get_apprcn.py".format(spider_path),
                         "schedule": "0 0 3,6,9,12,15,18,21 * * *",
                         "labels": ["spider-common"]
                     },
                     {
                         "name": "自动执行chiphell爬虫",
-                        "command": "python3 {}/news_get_chiphell.py".format(spider_news_path),
+                        "command": "python3 {}/news_get_chiphell.py".format(spider_path),
                         "schedule": "0 0 3,6,9,12,15,18,21 * * *",
                         "labels": ["spider-common"]
                     },
                     {
                         "name": "自动执行hello_github爬虫",
-                        "command": "python3 {}/news_get_hello_github.py".format(spider_news_path),
+                        "command": "python3 {}/news_get_hello_github.py".format(spider_path),
                         "schedule": "0 0 3,6,9,12,15,18,21 * * *",
                         "labels": ["spider-common"]
                     },
                     {
                         "name": "自动执行Anyknew爬虫",
-                        "command": "python3 {}/news_get_news.py".format(spider_news_path),
+                        "command": "python3 {}/news_get_news.py".format(spider_path),
                         "schedule": "0 0 3,6,9,12,15,18,21 * * *",
                         "labels": ["spider-common"]
                     },
                     {
                         "name": "自动执行币界网文章爬虫",
-                        "command": "python3 {}/spider_web3_coin_world.py".format(spider_news_path),
+                        "command": "python3 {}/spider_web3_coin_world.py".format(spider_path),
                         "schedule": "0 0 3,6,9,12,15,18,21 * * *",
                         "labels": ["spider-common"]
                     },
                     {
                         "name": "获取 web3 新闻",
-                        "command": "python3 {}/spider_web3_news.py".format(spider_news_path),
+                        "command": "python3 {}/spider_web3_news.py".format(spider_path),
                         "schedule": "0 0 3,6,9,12,15,18,21 * * *",
                         "labels": ["spider-common"]
                     },
-                ],
-                'spider_rsshub_tasks': [
                     {
-                        "name": "自动执行 rsshub 爬虫",
-                        "command": "python3 {}/rsshub_spider.py".format(spider_rsshub_path),
-                        "schedule": "0 0 3,6,9,12,15,18,21 * * *",
-                        "labels": ["spider-rsshub"]
+                        "name": "自动执行dlt爬虫",
+                        "command": "python3 {}/spider_get_and_check_dlt.py".format(spider_path),
+                        "schedule": "30 22 * * 1,3,6",
+                        "labels": ["spider_common"]
                     }
                 ],
                 'message_tasks': [
@@ -185,16 +173,14 @@ def main():
                         "command": "python3 {}/message_get_one_week_weather.py".format(message_path),
                         "schedule": "0 0 6,22 * * *",
                         "labels": ["message"]
-                    }
-                ],
-                'rss_data': [
+                    },
                     {
                         "name": "从 freshrss-psql 数据库中读取数据并发送",
-                        "command": "python3 {}/rss_data_handel_main.py".format(rss_data_path),
+                        "command": "python3 {}/message_rss_data_handel.py".format(rss_data_path),
                         "schedule": "30 6,9,12,18,22 * * *",
                         "labels": ["message"]
                     },
-                ]
+                ],
             }]
 
             for task_template in tasks_template:

utils/utils_send_gotify.py

@@ -6,7 +6,7 @@ import httpx
 class GotifyNotifier:
     def __init__(self, title, message):
         self.gotify_url = 'https://gotify.erhe.top'
-        self.app_token = 'AT2QGp_vyCX4akW'
+        self.app_token = 'A9KF--mx_12PjSu'
         self.title = title
         self.message = message