update

base/base_daily_logs_generate.py

@@ -1,87 +1,223 @@
 # -*- coding: utf-8 -*-
-'''
-设置每天 00:00:00 新建一个日志记录
-'''
+"""
+用于青龙面板的日志初始化脚本
+每天 00:00:00 执行，创建当天的日志记录
+"""
 import os
 import sys
-
-sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'))
-
 import time
+import logging
 from datetime import datetime
-import pymongo
-from base.base_load_config import load_config, get_base_path
-
-config_json = load_config()
-base_project = get_base_path()
-
-PROJECT_NAME = config_json.get('PROJECT_NAME')
-DB_USER = config_json.get('DB_USER')
-DB_PASSWORD = config_json.get('DB_PASSWORD')
-DB_IP = config_json.get('DB_IP')
-DB_PORT = config_json.get('DB_PORT')
-MONGO_LINK = f'mongodb://{DB_USER}:{DB_PASSWORD}@{DB_IP}:{DB_PORT}/'
-MAIL_HOST = config_json.get('MAIL_HOST')
-MAIL_USER = config_json.get('MAIL_USER')
-MAIL_PASS = config_json.get('MAIL_PASS')
-MAIL_SENDER = config_json.get('MAIL_SENDER')
-MAIL_RECEIVERS = config_json.get('MAIL_RECEIVERS')
+from typing import Dict, Any
+
+# 添加项目路径
+sys.path.append(os.path.join(os.path.abspath(__file__).split('AutoInfo')[0] + 'AutoInfo'))
+
+from utils.utils import *
+
+# 配置日志
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+
+class Config:
+    """配置管理类"""
+    _instance = None
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+            cls._instance._load_config()
+        return cls._instance
+
+    def _load_config(self):
+        """加载配置"""
+        try:
+            config_json = LoadConfig().load_config()
+            self.PROJECT_NAME = config_json.get('PROJECT_NAME', 'AutoInfo')
+            self.DB_USER = config_json.get('DB_USER', '')
+            self.DB_PASSWORD = config_json.get('DB_PASSWORD', '')
+            self.DB_IP = config_json.get('DB_IP', 'localhost')
+            self.DB_PORT = config_json.get('DB_PORT', 27017)
+            self.MAIL_HOST = config_json.get('MAIL_HOST', '')
+            self.MAIL_USER = config_json.get('MAIL_USER', '')
+            self.MAIL_PASS = config_json.get('MAIL_PASS', '')
+            self.MAIL_SENDER = config_json.get('MAIL_SENDER', '')
+            self.MAIL_RECEIVERS = config_json.get('MAIL_RECEIVERS', [])
+
+            # 构建MongoDB连接字符串
+            if self.DB_USER and self.DB_PASSWORD:
+                self.MONGO_LINK = f'mongodb://{self.DB_USER}:{self.DB_PASSWORD}@{self.DB_IP}:{self.DB_PORT}/?authSource=admin'
+            else:
+                self.MONGO_LINK = f'mongodb://{self.DB_IP}:{self.DB_PORT}/'
+
+        except Exception as e:
+            logger.error(f"加载配置失败: {e}")
+            raise
+
+
+class MongoHandle:
+    """MongoDB操作处理类"""
+
+    def __init__(self, db: str, collection: str, del_db: bool = False,
+                 del_collection: bool = False, auto_remove: int = 0):
+        """
+        初始化MongoDB连接
+
+        Args:
+            db: 数据库名
+            collection: 集合名
+            del_db: 是否删除数据库
+            del_collection: 是否删除集合
+            auto_remove: 自动删除数据的天数阈值
+        """
+        self.config = Config()
+        logger.info(f"连接数据库: {self.config.MONGO_LINK}")
+
+        try:
+            self.client = pymongo.MongoClient(
+                self.config.MONGO_LINK,
+                serverSelectionTimeoutMS=5000  # 5秒超时
+            )
+            # 测试连接
+            self.client.admin.command('ismaster')
+
+            self.db_name = db
+            self.collection_name = collection
+
+            self._setup_database(del_db, del_collection)
+
+            if auto_remove > 0:
+                self.auto_remove_data(auto_remove)
+
+        except pymongo.errors.ServerSelectionTimeoutError:
+            logger.error("无法连接到MongoDB服务器")
+            raise
+        except Exception as e:
+            logger.error(f"数据库初始化失败: {e}")
+            raise
+
+    def _setup_database(self, del_db: bool, del_collection: bool):
+        """设置数据库和集合"""
+        if del_db and self.db_name:
+            if self.db_name in self.client.list_database_names():
+                self.client.drop_database(self.db_name)
+                logger.info(f"已删除数据库: {self.db_name}")
+
+        self.db = self.client[self.db_name]
+
+        if del_collection and self.collection_name:
+            if self.collection_name in self.db.list_collection_names():
+                self.db.drop_collection(self.collection_name)
+                logger.info(f"已删除集合: {self.collection_name}")
+
+        self.collection = self.db[self.collection_name]
+
+    def write_data(self, data: Dict[str, Any]) -> bool:
+        """写入数据"""
+        try:
+            result = self.collection.insert_one(data)
+            logger.debug(f"数据插入成功, ID: {result.inserted_id}")
+            return True
+        except Exception as e:
+            logger.error(f"数据插入失败: {e}")
+            return False
+
+    def auto_remove_data(self, days: int):
+        """自动删除指定天数前的数据"""
+        try:
+            cutoff_time = int(time.time()) - days * 24 * 60 * 60
+            result = self.collection.delete_many({
+                'create_time': {'$lt': cutoff_time}
+            })
+            if result.deleted_count > 0:
+                logger.info(f"已删除 {result.deleted_count} 条过期数据")
+        except Exception as e:
+            logger.error(f"自动删除数据失败: {e}")
+
+    def close(self):
+        """关闭数据库连接"""
+        if hasattr(self, 'client'):
+            self.client.close()
+            logger.info("数据库连接已关闭")
+
+
+class LogsHandler:
+    """日志处理类"""
 
-now_day = time.strftime('%Y-%m-%d', time.localtime())
-rss_base_url = 'http://home.erhe.link:20002/xmlfile/'
-
-
-class LogsHandle(object):
     def __init__(self):
-        self.now_day = time.strftime('%Y-%m-%d', time.localtime())
-        db = 'logs'
-        collection = 'logs_' + self.now_day
-        self.mongo = MongoHandle(db=db, collection=collection, del_db=False, del_collection=False, auto_remove=0)
-
-    def logs_generate(self):
+        self.config = Config()
+        self.now_day = datetime.now().strftime('%Y-%m-%d')
+        self.mongo = None
+        self._setup_mongo()
+
+    def _setup_mongo(self):
+        """设置MongoDB连接"""
+        try:
+            self.mongo = MongoHandle(
+                db='logs',
+                collection=f'logs_{self.now_day}',
+                del_db=False,
+                del_collection=False,
+                auto_remove=0  # 不自动删除，由其他机制处理
+            )
+        except Exception as e:
+            logger.error(f"初始化MongoDB连接失败: {e}")
+            raise
+
+    def logs_generate(self) -> bool:
+        """生成当天的日志记录"""
         data_to_insert = {
-            "title": "logs",
-            "context": 'generate message logs',
-            "state": "create",
+            "title": "daily_log_init",
+            "context": f"Daily log collection created for {self.now_day}",
+            "state": "created",
             "create_time": int(time.time()),
-            "create_datetime": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+            "create_datetime": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+            "project": self.config.PROJECT_NAME
         }
 
-        self.mongo.collection.insert_one(data_to_insert)
-
-
-class MongoHandle(object):
-    def __init__(self, db, collection, del_db=False, del_collection=False, auto_remove=0):
-        self.client = pymongo.MongoClient(MONGO_LINK)
-        self.db = db
-        self.collection = collection
-
-        if del_db and db:
-            # 检查数据库是否存在
-            if db in self.client.list_database_names():
-                # 删除数据库
-                self.client.drop_database(db)
-        self.db = self.client[db]
-
-        if del_collection and self.collection:
-            # 检查集合是否存在
-            if self.collection in self.db.list_collection_names():
-                # 删除集合
-                self.db.drop_collection(collection)
-        self.collection = self.db[collection]
-
-        if auto_remove:
-            self.auto_remove_data(auto_remove)
-
-    def write_data(self, data):
-        self.collection.insert_one(data)
-
-    def auto_remove_data(self, day):
-        for data in self.collection.find({'create_time': {'$lt': int(time.time()) - day * 24 * 60 * 60}}):
-            self.collection.delete_one({'_id': data['_id']})
+        try:
+            success = self.mongo.write_data(data_to_insert)
+            if success:
+                logger.info(f"成功创建当天日志记录: {self.now_day}")
+            return success
+        except Exception as e:
+            logger.error(f"创建日志记录失败: {e}")
+            return False
+
+    def cleanup(self):
+        """清理资源"""
+        if self.mongo:
+            self.mongo.close()
+
+
+def main():
+    """主函数"""
+    logger.info("开始创建当天日志记录...")
+
+    logs_handler = None
+    try:
+        logs_handler = LogsHandler()
+        success = logs_handler.logs_generate()
+
+        if success:
+            logger.info("当天日志记录创建成功")
+            return 0
+        else:
+            logger.error("当天日志记录创建失败")
+            return 1
+
+    except Exception as e:
+        logger.error(f"执行过程中发生错误: {e}")
+        return 1
+    finally:
+        if logs_handler:
+            logs_handler.cleanup()
 
 
 if __name__ == '__main__':
-    print("新建当天日志记录...")
-    LogsHandle().logs_generate()
-    print("当天日志记录已创建...")
+    exit_code = main()
+    sys.exit(exit_code)

base/base_daily_logs_send.py

@@ -2,21 +2,17 @@
 '''
 设置每天 23:59 执行, 读取当天数据库中, 所有日志, 发送到指定邮箱
 '''
-import time
 import os
 import sys
+import time
 import httpx
-
-sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'))
-
 import pymongo
-import smtplib
-from email.mime.text import MIMEText
-from email.header import Header
-from base.base_load_config import load_config, get_base_path
 
-config_json = load_config()
-base_project = get_base_path()
+sys.path.append(os.path.join(os.path.abspath(__file__).split('AutoInfo')[0] + 'AutoInfo'))
+from utils.utils import LoadConfig
+
+config_json = LoadConfig().load_config()
+base_project = LoadConfig().get_base_path()
 
 PROJECT_NAME = config_json.get('PROJECT_NAME')
 DB_USER = config_json.get('DB_USER')
@@ -41,8 +37,7 @@ class LogsHandle(object):
         self.mongo = MongoHandle(db=db, collection=collection, del_db=False, del_collection=False, auto_remove=0)
 
     def logs_send(self):
-        subject = 'auto message logs'
-        title = 'auto message - daily logs: {}'.format(self.now_day)
+        title = 'AutoInfo message - daily logs: {}'.format(self.now_day)
         text = ''
 
         # TODO
@@ -60,10 +55,7 @@ class LogsHandle(object):
             )
 
         if text:
-            S = SendEmail(subject=subject, title=title, text=text)
-            S.send()
-
-            G = GotifyNotifier(title=title, message=text, token='base')
+            G = GotifyNotifier(title=title, message=text, token_name='base')
             G.send_message()
         else:
             print("No error logs found for today.")
@@ -100,67 +92,27 @@ class MongoHandle(object):
             self.collection.delete_one({'_id': data['_id']})
 
 
-class SendEmail(object):
-    def __init__(self, subject='auto subject', title='auto title', text='auto text') -> None:
-        # 第三方 SMTP 服务
-        self.mail_host = MAIL_HOST  # 设置服务器
-        self.mail_user = MAIL_USER  # 用户名
-        self.mail_pass = MAIL_PASS  # 口令
-
-        self.sender = MAIL_SENDER
-        self.receivers = [MAIL_RECEIVERS]
-
-        self.subject = subject
-        self.title = title
-        self.text = text
-
-    def send(self):
-        message = MIMEText(self.text, 'plain', 'utf-8')
-        message['From'] = Header(self.title, 'utf-8')
-        message['To'] = Header("auto", 'utf-8')
-        message['Subject'] = Header(self.subject, 'utf-8')
-
-        try:
-            smtpObj = smtplib.SMTP_SSL(self.mail_host)
-            smtpObj.login(self.mail_user, self.mail_pass)
-            smtpObj.sendmail(self.sender, self.receivers, message.as_string())
-            print("邮件发送成功")
-        except smtplib.SMTPException as e:
-            print("Error: 无法发送邮件", e)
-
-
 class GotifyNotifier:
-    def __init__(self, title, message, token='A8EVb0Cmxnb2vfk'):
+    def __init__(self, title, message, token_name='A52cfQ1UZ2e.Z0B'):
         self.gotify_url = 'https://gotify.erhe.top'
-        self.app_token = token
+        self.app_token = token_name
         self.title = title
         self.message = message
 
     def send_message(self):
-        # 构建POST请求的headers
-        headers = {
-            'Content-Type': 'application/json'
-        }
-
-        # 构建POST请求的body
-        body = {
-            'title': self.title,
-            'message': self.message
-        }
-
         # 发送POST请求
         with httpx.Client() as client:
             response = client.post(
                 url=f"{self.gotify_url}/message?token={self.app_token}",
-                headers=headers,
-                json=body
+                headers={'Content-Type': 'application/json'},
+                json={'title': self.title, 'message': self.message}
             )
 
-        # 检查响应状态码
-        if response.status_code == 200:
-            print('Gotify Message sent successfully!')
-        else:
-            print('Failed to send message:', response.text)
+            # 检查响应状态码
+            if response.status_code == 200:
+                print('Gotify Message sent successfully!')
+            else:
+                print('Failed to send message:', response.text)
 
 
 if __name__ == '__main__':

base/base_load_config.py

@@ -1,34 +0,0 @@
-# -*- coding: utf-8 -*-
-'''
-用于读取config.json
-无需定时
-'''
-
-import json
-import os
-import sys
-
-sys.path.append(os.path.join(os.path.abspath(__file__).split('AutoInfo')[0] + 'AutoInfo'))
-
-def load_config():
-    try:
-        sys.path.append(os.path.join(os.getcwd().split('AutoInfo')[0], 'AutoInfo'))
-        base_project = os.path.join(os.getcwd().split('AutoInfo')[0], 'AutoInfo')
-
-        config_path = os.path.join(base_project, 'config.json')
-        config_json = {}
-        with open(config_path, 'r') as f:
-            config_json = json.load(f)
-
-        if not config_json:
-            print('No config file found')
-            exit(0)
-    except Exception as e:
-        print(e)
-        exit(0)
-
-    return config_json
-
-
-def get_base_path():
-    return os.path.join(os.getcwd().split('auto')[0], 'auto')

base/base_news_data_collation.py

@@ -0,0 +1,238 @@
+'''
+每日从 mongo 数据库, 做新闻汇总,发送到邮箱
+'''
+import os
+import sys
+
+from utils.utils import LoadConfig
+
+sys.path.append(os.path.join(os.path.abspath(__file__).split('AutoInfo')[0] + 'AutoInfo'))
+
+from pymongo import MongoClient
+from datetime import datetime, timedelta
+import re
+import smtplib
+from email.mime.text import MIMEText
+from email.header import Header
+from utils.utils import *
+
+config_json = LoadConfig().load_config()
+base_project = LoadConfig().get_base_path()
+
+PROJECT_NAME = config_json.get('PROJECT_NAME')
+DB_USER = config_json.get('DB_USER')
+DB_PASSWORD = config_json.get('DB_PASSWORD')
+DB_IP = config_json.get('DB_IP')
+DB_PORT = config_json.get('DB_PORT')
+MAIL_HOST = config_json.get('MAIL_HOST')
+MAIL_USER = config_json.get('MAIL_USER')
+MAIL_PASS = config_json.get('MAIL_PASS')
+MAIL_SENDER = config_json.get('MAIL_SENDER')
+MAIL_RECEIVERS = config_json.get('MAIL_RECEIVERS')
+DB_NAME = config_json.get('DB_NAME')  # 确保配置文件中有这个键
+MONGO_LINK = f'mongodb://{DB_USER}:{DB_PASSWORD}@{DB_IP}:{DB_PORT}/'.format(**config_json)
+now_day = datetime.now().strftime('%Y-%m-%d')  # 获取今天的日期
+filter_days = config_json.get('FILTER_DAYS')
+filter_keys = config_json.get('FILTER_KEYS')
+filter_switch = True
+
+
+class NewsDataCollation(object):
+    def __init__(self):
+        # 第三方 SMTP 服务
+        self.mail_host = MAIL_HOST  # 设置服务器
+        self.mail_user = MAIL_USER  # 用户名
+        self.mail_pass = MAIL_PASS  # 口令
+
+        self.sender = MAIL_SENDER
+        self.receivers = [MAIL_RECEIVERS]
+
+        self.processed_data = []
+
+    def load_data(self):
+        processed_data = []
+        # 读取数据
+        print('程序正在读取数据')
+        client = MongoClient(MONGO_LINK)
+        db = client['NEWS']
+
+        # 根据 self.days 获取日期范围
+        start_date = (datetime.now() - timedelta(days=filter_days - 1)).strftime('%Y-%m-%d')
+        end_date = datetime.now().strftime('%Y-%m-%d')
+
+        # 构造查询条件，匹配日期范围内的日期
+        query = {
+            "create_datetime": {
+                "$regex": f"^{start_date}|{end_date}",
+                "$options": "i"  # 使用不区分大小写的匹配
+            }
+        }
+
+        # 遍历数据库中的所有集合
+        for collection_name in db.list_collection_names():
+            print(collection_name)
+            collection = db[collection_name]
+            cursor = collection.find(query)
+            for document in cursor:
+                if not document.get('title'):
+                    continue
+
+                # 检查 'repush_times' 字段是否存在，如果不存在则默认为 5
+                repush_times = document.get('repush_times', 5)
+
+                # 减少 repush_times 的值
+                new_repush_times = repush_times - 1
+
+                # 更新数据库中的 repush_times 字段
+                collection.update_one(
+                    {"_id": document['_id']},  # 假设文档中有 _id 字段作为唯一标识
+                    {"$set": {"repush_times": new_repush_times}}
+                )
+
+                data = self.process_data(document)
+                if data:
+                    processed_data.append(data)
+
+        # 关闭MongoDB连接
+        client.close()
+        return processed_data
+
+    def process_data(self, document):
+        # 处理数据
+        data = {
+            "title": document.get('title') or '',
+            "context": document.get('context') or '',
+            "source_url": document.get('source_url') or '',
+            'link': document.get('link') or '',
+            "article_type": document.get('article_type') or '',
+            "article_source": document.get('article_source') or '',
+            "img_url": document.get('img_url') or '',
+            'keyword': document.get('keyword') or '',
+            "posted_date": document.get('posted_date') or '',
+            "create_time": document.get('create_time') or '',
+            "create_datetime": document.get('create_datetime') or '',
+            "repush_times": document.get('repush_times', 5) - 1
+        }
+
+        data['title'] = self.clean_string(data['title'], 'title')
+        data['context'] = self.clean_string(data['context'], 'context')
+
+        return data
+
+    def clean_string(self, input_string, text_type):
+        # 清除 title 和 context 中的换行符和制表符
+        if not isinstance(input_string, str):
+            return ''
+
+        # 清除所有空白字符（包括空格、制表符、换行符等）
+        cleaned_string = re.sub(r'\s+', '', input_string)
+
+        if len(cleaned_string) > 100:
+            cleaned_string = cleaned_string[:100] + '...'
+
+        if text_type == 'context':
+            pass
+
+        return cleaned_string
+
+    def send_email(self, processed_data):
+        # 发送邮件
+        print('准备发送邮件')
+        subject = '新闻汇总sub'
+        title = '新闻汇总title'
+        text = '********************************************************\n'
+        for data in processed_data:
+            text += '标题: {}\n'.format(data['title'])
+            text += '正文: {}\n'.format(data['context'])
+            text += '文章地址: {}\n'.format(data['link'])
+            text += '类型: {}\n'.format(data['article_type'])
+            text += '板块: {}\n'.format(data['article_source'])
+            text += '文章时间: {}\n'.format(data['posted_date'])
+            text += '获取时间: {}\n'.format(data['create_datetime'])
+            text += '********************************************************\n\n'
+
+        message = MIMEText(text, 'plain', 'utf-8')
+        message['From'] = Header(title, 'utf-8')
+        message['To'] = Header("auto", 'utf-8')
+        message['Subject'] = Header(subject, 'utf-8')
+
+        try:
+            smtpObj = smtplib.SMTP_SSL(self.mail_host)
+            smtpObj.login(self.mail_user, self.mail_pass)
+            smtpObj.sendmail(self.sender, self.receivers, message.as_string())
+            print("邮件发送成功")
+        except smtplib.SMTPException as e:
+            print("Error: 无法发送邮件", e)
+
+    def send_email_with_keyword(self, series, keys, processed_data):
+        process_send_data = {}
+        keys = keys.split('|')
+        have_data_keys = []
+        for key in keys:
+            # print(f'通过关键字: {key} 过滤')  # 用来调试 key 是否正确
+            for data in processed_data:
+                if key in data['title'] or key in data['context']:
+                    # 如果数据里面无 keyword, 用当前 key 替换一下
+                    if not data.get('keyword'):
+                        data['keyword'] = key
+
+                    if series not in process_send_data:
+                        process_send_data[series] = [data]
+                    else:
+                        process_send_data[series].append(data)
+
+                    # 储存一下有数据的 key, 输出用
+                    have_data_keys.append(key)
+
+        if process_send_data:
+            print('{}系列, 以下关键字有数据\n{}'.format(series, list(set(have_data_keys))))
+            # 发送邮件
+            print('程序正在准备发送邮件的数据')
+            for key in process_send_data:
+                subject = '新闻汇总sub - {}'.format(series)
+                title = '新闻汇总title - {}'.format(series)
+                text = '********************************************************\n'
+                for data in process_send_data[key]:
+                    text += '标题: {}\n'.format(data['title'])
+                    text += '正文: {}\n'.format(data['context'])
+                    text += '文章地址: {}\n'.format(data['link'])
+                    text += '类型: {}\n'.format(data['article_type'])
+                    text += '板块: {}\n'.format(data['article_source'])
+                    text += '关键词: {}\n'.format(key)
+                    text += '文章时间: {}\n'.format(data['posted_date'])
+                    text += '获取时间: {}\n'.format(data['create_datetime'])
+                    text += '********************************************************\n\n'
+
+                message = MIMEText(text, 'plain', 'utf-8')
+                message['From'] = Header(title, 'utf-8')
+                message['To'] = Header("auto", 'utf-8')
+                message['Subject'] = Header(subject, 'utf-8')
+
+                try:
+                    smtpObj = smtplib.SMTP_SSL(self.mail_host)
+                    smtpObj.login(self.mail_user, self.mail_pass)
+                    smtpObj.sendmail(self.sender, self.receivers, message.as_string())
+                    print("关键字: {} 的邮件发送成功".format(series))
+                except smtplib.SMTPException as e:
+                    print("Error: 无法发送邮件", e)
+
+    def main(self):
+        # 加载指定天数的所有数据
+        processed_data = self.load_data()
+
+        # 如果无数据, 则退出
+        if not processed_data:
+            print("没有找到任何数据")
+            exit(0)
+
+        # 发送一次所有数据的邮件
+        # self.send_email(processed_data)
+
+        # # 这里是通过关键词过滤然后再发送邮件
+        if filter_switch and filter_keys:
+            for series, keys in filter_keys.items():
+                self.send_email_with_keyword(series, keys, processed_data)
+
+
+if __name__ == '__main__':
+    NewsDataCollation().main()

base/base_timing_remove_data.py

@@ -8,21 +8,14 @@ import sys
 import os
 from datetime import datetime
 import pymongo
-import smtplib
-from email.mime.text import MIMEText
-from email.header import Header
 
-sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'))
-base_project = os.path.join(os.getcwd().split('auto')[0], 'auto')
-import json
+sys.path.append(os.path.join(os.path.abspath(__file__).split('AutoInfo')[0] + 'AutoInfo'))
+base_project = os.path.join(os.getcwd().split('AutoInfo')[0], 'AutoInfo')
 
-config_path = os.path.join(base_project, 'config.json')
-with open(config_path, 'r') as f:
-    config_json = json.load(f)
+from utils.utils import LoadConfig, GotifyNotifier
 
-if not config_json:
-    print('No config file found')
-    exit(0)
+config_json = LoadConfig().load_config()
+base_project = LoadConfig().get_base_path()
 
 PROJECT_NAME = config_json.get('PROJECT_NAME')
 DB_USER = config_json.get('DB_USER')
@@ -37,8 +30,6 @@ MAIL_SENDER = config_json.get('MAIL_SENDER')
 MAIL_RECEIVERS = config_json.get('MAIL_RECEIVERS')
 
 now_day = time.strftime('%Y-%m-%d', time.localtime())
-rss_base_url = 'http://home.erhe.link:20002/xmlfile/'
-base_project = os.path.join(os.getcwd().split(PROJECT_NAME)[0], PROJECT_NAME)
 
 
 class MongoHandle(object):
@@ -73,7 +64,7 @@ class MongoHandle(object):
 
 
 class SendEmail(object):
-    def __init__(self, subject='auto subject', title='auto title', text='auto text') -> None:
+    def __init__(self, subject='AutoInfo subject', title='AutoInfo title', text='AutoInfo text') -> None:
         # 第三方 SMTP 服务
         self.mail_host = MAIL_HOST  # 设置服务器
         self.mail_user = MAIL_USER  # 用户名
@@ -87,18 +78,11 @@ class SendEmail(object):
         self.text = text
 
     def send(self):
-        message = MIMEText(self.text, 'plain', 'utf-8')
-        message['From'] = Header(self.title, 'utf-8')
-        message['To'] = Header("auto", 'utf-8')
-        message['Subject'] = Header(self.subject, 'utf-8')
-
-        try:
-            smtpObj = smtplib.SMTP_SSL(self.mail_host)
-            smtpObj.login(self.mail_user, self.mail_pass)
-            smtpObj.sendmail(self.sender, self.receivers, message.as_string())
-            print("邮件发送成功")
-        except smtplib.SMTPException as e:
-            print("Error: 无法发送邮件", e)
+        if self.title:
+            G = GotifyNotifier(title=self.title, message=self.subject)
+            G.send_message()
+        else:
+            print("No error logs found for today.")
 
 
 class LogsHandle(object):
@@ -127,7 +111,7 @@ class LogsHandle(object):
                 data_to_insert.setdefault('content'),
                 data_to_insert.setdefault('state'),
                 data_to_insert.setdefault('create_datetime'),
-                )
+            )
 
             Send = SendEmail(subject=subject, title=title, text=text)
             Send.send()

config.json

@@ -1,5 +1,5 @@
 {
-  "PROJECT_NAME": "auto",
+  "PROJECT_NAME": "AutoInfo",
   "MAIL_HOST": "smtp.163.com",
   "MAIL_USER": "pushmessagebot@163.com",
   "MAIL_PASS": "WSMSRKBKXIHIQWTU",
@@ -7,8 +7,9 @@
   "MAIL_RECEIVERS": "pushmessagebot@163.com",
   "DB_USER": "root",
   "DB_PASSWORD": "aaaAAA111!!!",
-  "DB_IP": "192.168.31.177",
-  "DB_PORT": "38000",
+  "DB_IP": "localhost",
+  "DB_PORT": "27017",
+  "GOTIFY_URL": "https://gotify.erhe.top",
   "FILTER_DAYS": 1,
   "FILTER_KEYS": {
     "新闻汇总": "经济|金融|失业率",

spider/news_get_news.py → message-email/anyknew.py

@@ -1,21 +1,15 @@
 # -*- coding: utf-8 -*-
-import time
-import httpx
-from datetime import datetime
 import os
 import sys
 
-sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'))
+sys.path.append(os.path.join(os.path.abspath(__file__).split('AutoInfo')[0] + 'AutoInfo'))
 
-from utils.utils_mongo_handle import MongoHandle
-from utils.utils_logs_handle import LogsHandle
-from utils.utils_send_email import SendEmail
+from utils.utils import *
 
-from base.base_load_config import load_config
-
-config_json = load_config()
+config_json = LoadConfig().load_config()
 DEFAULT_RE_PUSH_TIMES = config_json['DEFAULT_RE_PUSH_TIMES']
 
+
 class HotNews():
     def __init__(self):
         self.base_url = 'https://www.anyknew.com/go/'
@@ -34,8 +28,7 @@ class HotNews():
             'life': 'https://www.anyknew.com/api/v1/cats/life',
             'binary': 'https://www.anyknew.com/api/v1/cats/binary'
         }
-        self.send_email_datas = []
-        self.send_email_now = 0
+        self.temp_datas = []
 
     def main(self):
         self.logs_handle.logs_write('聚合新闻', '任务开始', 'start', False)
@@ -45,12 +38,11 @@ class HotNews():
         if resp_data:
             self.save_to_mongo(resp_data)
 
-            if self.send_email_now:
-                if self.send_email_datas:
-                    print('准备发送邮件')
-                    self.send_to_email()
-                else:
-                    print('无新数据')
+            if self.temp_datas:
+                print('准备发送消息')
+                self.send_to_gotify()
+            else:
+                print('无新数据')
 
         else:
             self.logs_handle.logs_write('聚合新闻', '获取数据为空', 'error', False)
@@ -130,7 +122,7 @@ class HotNews():
                 if count == 0:
                     # 如果没有找到匹配的文档，插入新文档
                     result = mongo.collection.insert_one(data_to_insert)
-                    self.send_email_datas.append(data_to_insert)
+                    self.temp_datas.append(data_to_insert)
 
             except TypeError as te:
                 print('\n%s' % te)
@@ -138,9 +130,9 @@ class HotNews():
                 return 0
         print(f'Anyknew数据处理')
 
-    def send_to_email(self):
-        text = '********************************************************\n'
-        for data in self.send_email_datas:
+    def send_to_gotify(self):
+        text = '****************************************\n'
+        for data in self.temp_datas:
             text += '标题: {}\n'.format(data['title'])
             text += '正文: {}\n'.format(data['context'])
             text += '文章地址: {}\n'.format(data['link'])
@@ -148,11 +140,13 @@ class HotNews():
             text += '板块: {}\n'.format(data['article_source'])
             text += '文章时间: {}\n'.format(data['posted_date'])
             text += '获取时间: {}\n'.format(data['create_datetime'])
-            text += '********************************************************\n\n'
+            text += '***********************************\n\n'
+
+        title = 'Anyknew新闻 - ' + str(datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
+        sub = 'Anyknew新闻'
 
-        send_email = SendEmail(subject='Anyknew', title='Anyknew_info', text=text)
-        send_email.send()
-        print('邮件已发送')
+        SendEmail(subject=sub, title=title, text=text).send()
+        # GotifyNotifier(title=title, message=text, token_name='news').send_message()
 
 
 if __name__ == '__main__':

spider/news_get_apprcn.py → message-email/apprcn.py

@@ -6,21 +6,14 @@
 3, 发送到指定邮件
 '''
 import re
-import time
-from datetime import datetime
-import httpx
 import sys
 import os
 
-sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'))
+sys.path.append(os.path.join(os.path.abspath(__file__).split('AutoInfo')[0] + 'AutoInfo'))
 
-from utils.utils_mongo_handle import MongoHandle
-from utils.utils_logs_handle import LogsHandle
-from utils.utils_send_email import SendEmail
+from utils.utils import *
 
-from base.base_load_config import load_config
-
-config_json = load_config()
+config_json = LoadConfig().load_config()
 DEFAULT_RE_PUSH_TIMES = config_json['DEFAULT_RE_PUSH_TIMES']
 
 
@@ -33,10 +26,9 @@ class APPRCN(object):
             'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8'
         }
         db = 'NEWS'
-        collection = 'apprcn_info'
+        collection = 'apprcn-info'
         self.mongo = MongoHandle(db=db, collection=collection, del_db=False, del_collection=False, auto_remove=0)
-        self.send_email_datas = []
-        self.send_email_now = 0
+        self.temp_datas = []
 
     def main(self):
         self.logs_handle.logs_write('apprcn', '开始获取反斗限免数据', 'start', False)
@@ -46,8 +38,7 @@ class APPRCN(object):
         if response_data:
             self.save_to_mongo(response_data)
 
-            if self.send_email_now:
-                self.send_to_email()
+            self.send_to_gotify()
 
             self.logs_handle.logs_write('apprcn', '反斗限免数据获取完成', 'done', False)
             print('done')
@@ -113,7 +104,7 @@ class APPRCN(object):
                 if count == 0:
                     # 如果没有找到匹配的文档，插入新文档
                     result = self.mongo.collection.insert_one(data_to_insert)
-                    self.send_email_datas.append(data_to_insert)
+                    self.temp_datas.append(data_to_insert)
 
             except TypeError as te:
                 print('\n%s' % te)
@@ -121,19 +112,23 @@ class APPRCN(object):
                 return 0
         print('储存数据完成', datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
 
-    def send_to_email(self):
-        if self.send_email_datas:
+    def send_to_gotify(self):
+        if self.temp_datas:
             text = ''
-            for data in self.send_email_datas:
+            for data in self.temp_datas:
                 text += '标题: %s\n内容: %s\n时间: %s\n链接: %s\n\n' % (
                     data['title'], data['context'], data['posted_date'], data['source_url'])
-            send_email = SendEmail(subject='反斗限免', title='反斗限免', text=text)
-            send_email.send()
 
-            self.logs_handle.logs_write('apprcn', '发送邮件完成', 'done', False)
+            title = '反斗限免 - ' + str(datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
+            sub = '反斗限免'
+            SendEmail(subject=sub, title=title, text=text).send()
+            # GotifyNotifier(title=title, message=text, token_name='news').send_message()
+
+            self.logs_handle.logs_write('apprcn', '发送消息完成', 'done', False)
         else:
             self.logs_handle.logs_write('apprcn', '没有新数据, 不发送邮件', 'done', False)
 
 
+
 if __name__ == "__main__":
     APPRCN().main()

message-email/chaincatcher.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+'''
+网络爬虫，抓取链捕手新闻(data-v-***** 此参数会失效, 定期更换)
+'''
+
+import sys
+import os
+from playwright.sync_api import sync_playwright
+from bs4 import BeautifulSoup
+
+sys.path.append(os.path.join(os.path.abspath(__file__).split('AutoInfo')[0] + 'AutoInfo'))
+from utils.utils import *
+
+
+def chaincatcher_news():
+    url = "https://www.chaincatcher.com/news"
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        page = browser.new_page()
+        try:
+            page.goto(url)
+
+            time.sleep(2)
+            start_time = time.time()
+            while time.time() - start_time < 10:
+                page.mouse.wheel(0, 100)
+                time.sleep(0.1)
+            page_content = page.content()
+            browser.close()
+            soup = BeautifulSoup(page_content, 'html.parser')
+            contents = [span.get_text(strip=True) for span in soup.find_all('span', class_='text', attrs={'data-v-6560eea9': True}) if "微信扫码" not in span]
+            result = '\n'.join(contents)
+            if result:
+                result += f'\n推送时间: {datetime.now().strftime("%Y年%m月%d日 %H时%M分%S秒")}'
+
+                title = 'ChainCatcher' + str(datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
+                sub = 'ChainCatcher News'
+                SendEmail(subject=sub, title=title, text=result).send()
+                # GotifyNotifier(title='ChainCatcher News', message=result, token_name='news').send_message()
+            else:
+                print("No news found.")
+        except Exception as e:
+            raise e
+        finally:
+            browser.close()
+
+
+for retry in range(5):
+    try:
+        chaincatcher_news()
+        break
+    except Exception as e:
+        sleep_time = 20
+        print(f"Error occurred: {e}. Retrying... {retry + 1} \t sleep time: {sleep_time}")
+        time.sleep(sleep_time)

spider/news_get_chiphell.py → message-email/chiphell.py

@@ -7,19 +7,12 @@ import random
 import sys
 import threading
 import re
-import time
-from datetime import datetime
-import httpx
 
-sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'))
+sys.path.append(os.path.join(os.path.abspath(__file__).split('AutoInfo')[0] + 'AutoInfo'))
 
-from utils.utils_mongo_handle import MongoHandle
-from utils.utils_logs_handle import LogsHandle
-from utils.utils_send_email import SendEmail
+from utils.utils import *
 
-from base.base_load_config import load_config
-
-config_json = load_config()
+config_json = LoadConfig().load_config()
 DEFAULT_RE_PUSH_TIMES = config_json['DEFAULT_RE_PUSH_TIMES']
 
 
@@ -34,8 +27,7 @@ class CHIPHELL(object):
         self.headers = {
             'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8'
         }
-        self.send_email_datas = []
-        self.send_email_now = 0
+        self.temp_datas = []
 
     def req(self, source, target):
         print(f'正在获取 {source} 数据')
@@ -108,7 +100,7 @@ class CHIPHELL(object):
                     result = mongo.collection.insert_one(data_to_insert)
 
                     # 准备发送邮件的数据
-                    self.send_email_datas.append(data_to_insert)
+                    self.temp_datas.append(data_to_insert)
 
             except TypeError as te:
                 print('\n%s' % te)
@@ -117,10 +109,8 @@ class CHIPHELL(object):
         print(f'处理 chiphell - {collection}数据完成')
 
     def send_to_email(self):
-        title = 'chiphell - info'
-        subject = 'chiphell - info'
         text = '********************************************************\n'
-        for data in self.send_email_datas:
+        for data in self.temp_datas:
             text += '标题: {}\n'.format(data['title'])
             text += '正文: {}\n'.format(data['context'])
             text += '板块: {}\n'.format(data['article_source'])
@@ -130,8 +120,10 @@ class CHIPHELL(object):
             text += '获取时间: {}\n'.format(data['create_datetime'])
             text += '********************************************************\n\n'
 
-        send_email = SendEmail(subject=subject, title=title, text=text)
-        send_email.send()
+        title = 'chiphell - info - ' + str(datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
+        sub = 'chiphell - info'
+        SendEmail(subject=sub, title=title, text=text).send()
+        # GotifyNotifier(title=title, message=text, token_name='news').send_message()
 
         self.logs_handle.logs_write('chiphell', f'{title}-发送邮件完成', 'done', False)
 
@@ -231,14 +223,13 @@ class CHIPHELL(object):
 
             for thread in threads:
                 thread.join()
-        else:
-            self.logs_handle.logs_write('chiphell', '获取数据为空', 'error', False)
-            return False
 
-        # 如果 self.send_email_datas 中有数据, 则发送邮件
-        if self.send_email_now:
-            if self.send_email_datas:
+            if self.temp_datas:
                 self.send_to_email()
+            return None
+        else:
+            self.logs_handle.logs_write('chiphell - info', '获取数据为空', 'error', False)
+            return False
 
 
 if __name__ == '__main__':

spider/spider_get_and_check_dlt.py → message-email/dlt.py

@@ -7,7 +7,7 @@ from datetime import datetime
 import time
 import httpx
 
-sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'))
+sys.path.append(os.path.join(os.path.abspath(__file__).split('AutoInfo')[0] + 'AutoInfo'))
 
 from utils.utils_mongo_handle import MongoHandle
 from utils.utils_logs_handle import LogsHandle

spider/news_get_hello_github.py → message-email/hello_github.py

@@ -8,16 +8,12 @@ import time
 from datetime import datetime
 import httpx
 
-sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'))
+sys.path.append(os.path.join(os.path.abspath(__file__).split('AutoInfo')[0] + 'AutoInfo'))
 
-from utils.utils_mongo_handle import MongoHandle
-from utils.utils_logs_handle import LogsHandle
-from utils.utils_send_email import SendEmail
+from utils.utils import *
 
-from base.base_load_config import load_config
-
-config_json = load_config()
-DEFAULT_RE_PUSH_TIMES = config_json['DEFAULT_RE_PUSH_TIMES']
+config_json = LoadConfig().load_config()
+DEFAULT_RE_PUSH_TIMES = config_json.get('DEFAULT_RE_PUSH_TIMES')
 
 
 class HelloGithub(object):

message/message_coin_detail.py → message-gotify/coin_detail.py

@@ -2,11 +2,11 @@
 import os
 import sys
 
-sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'))
+sys.path.append(os.path.join(os.path.abspath(__file__).split('AutoInfo')[0] + 'AutoInfo'))
 
 import httpx
 from datetime import datetime
-from utils.utils_gotify import *
+from utils.utils_send_gotify import *
 
 retry_count = 5
 

spider/spider_web3_coin_world.py → message-gotify/coin_world.py

@@ -10,7 +10,7 @@ import re
 import time
 from datetime import datetime
 
-sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'))
+sys.path.append(os.path.join(os.path.abspath(__file__).split('AutoInfo')[0] + 'AutoInfo'))
 
 from utils.utils_mongo_handle import MongoHandle
 from utils.utils_logs_handle import LogsHandle

message/message_get_one_week_weather.py → message-gotify/weather7d.py

@@ -9,12 +9,9 @@ sys.path.append(os.path.join(os.path.abspath(__file__).split('AutoInfo')[0] + 'A
 SUB_PROJECT_NAME = "获取天气预报"
 PROJECT_PATH = os.path.join(os.path.abspath(__file__).split('AutoInfo')[0] + 'AutoInfo')
 
-import time
-import json
 from bs4 import BeautifulSoup
-import httpx
 
-from utils.utils_gotify import GotifyNotifier
+from utils.utils import *
 
 
 
@@ -60,5 +57,10 @@ class Weather():
 
 
 if __name__ == "__main__":
-    W = Weather().main()
+    try:
+        W = Weather().main()
+    except Exception as e:
+        print(e)
+        L = LogsHandle()
+        L.logs_write('Weather forecast', str(e), 'error')
 

spider/spider_web3_news.py → message-gotify/web3_news.py

@@ -10,7 +10,7 @@ import time
 
 import httpx
 
-sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'))
+sys.path.append(os.path.join(os.path.abspath(__file__).split('AutoInfo')[0] + 'AutoInfo'))
 from html import unescape
 from datetime import datetime
 import re

message/message_chaincatcher.py

@@ -1,39 +0,0 @@
-# -*- coding: utf-8 -*-
-import sys
-import os
-from playwright.sync_api import sync_playwright
-from bs4 import BeautifulSoup
-import time
-from datetime import datetime
-
-sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'))
-from utils.utils_gotify import *
-
-
-def chaincatcher_news():
-    url = "https://www.chaincatcher.com/news"
-    with sync_playwright() as p:
-        browser = p.chromium.launch(headless=True)
-        page = browser.new_page()
-        page.goto(url)
-        time.sleep(2)
-        start_time = time.time()
-        while time.time() - start_time < 10:
-            page.mouse.wheel(0, 100)
-            time.sleep(0.1)
-        page_content = page.content()
-        browser.close()
-        soup = BeautifulSoup(page_content, 'html.parser')
-        contents = [span.get_text(strip=True) for span in soup.find_all('span', class_='text', attrs={'data-v-aea07cf0': True}) if "微信扫码" not in span]
-        result = '\n'.join(contents)
-        print(result)
-        if result:
-            result += f'\n推送时间: {datetime.now().strftime("%Y年%m月%d日 %H时%M分%S秒")}'
-            gotify_notifier = GotifyNotifier(title='ChainCatcher News', message=result, token_name='news')
-            gotify_notifier.send_message()
-            print(result)
-        else:
-            print("No news found.")
-
-
-chaincatcher_news()

message/message_dlt.py

@@ -1,177 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-获取超级大乐透结果, 并匹配自己购买的号码, 配合定时执行使用
-"""
-
-import sys
-import os
-import asyncio
-import httpx
-
-sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'))
-
-from utils.utils_gotify import GotifyNotifier
-
-
-class CheckDlt:
-    def __init__(self):
-        self.url = 'https://webapi.sporttery.cn/gateway/lottery/getHistoryPageListV1.qry?gameNo=85&provinceId=0&pageSize=1&isVerify=1&pageNo=1'
-        self.headers = {
-            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
-            "Accept-Encoding": "gzip, deflate, br, zstd",
-            "Accept-Language": "zh-CN,zh;q=0.9",
-            "Cache-Control": "max-age=0",
-            "Priority": "u=0, i",
-            "Sec-CH-UA": '"Not;A=Brand";v="99", "Google Chrome";v="139", "Chromium";v="139"',
-            "Sec-CH-UA-Mobile": "?0",
-            "Sec-CH-UA-Platform": '"macOS"',
-            "Sec-Fetch-Dest": "document",
-            "Sec-Fetch-Mode": "navigate",
-            "Sec-Fetch-Site": "none",
-            "Sec-Fetch-User": "?1",
-            "Upgrade-Insecure-Requests": "1",
-            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36"
-        }
-        self.my_dlt = [
-            ['10', '11', '16', '17', '18', '11', '12'],
-            ['02', '03', '11', '12', '23', '05', '06'],
-            ['07', '09', '15', '17', '22', '09', '11'],
-            ['05', '06', '07', '34', '35', '02', '09'],
-            ['09', '10', '11', '21', '22', '04', '05']
-        ]
-
-    async def req(self):
-        async with httpx.AsyncClient() as client:
-            try:
-                resp = await client.get(self.url, timeout=5)
-                if resp.status_code != 200:
-                    print('state code: {}'.format(resp.status_code))
-                    log_detail = '访问失败, 状态码:{},url:{}'.format(resp.status_code, self.url)
-                    print(log_detail)
-                    return None
-            except Exception as e:
-                print(f'请求失败 {e}')
-                return None
-
-            return resp.json()
-
-    def data_handle(self, data):
-        if not data:
-            print('获取数据为空')
-            return None
-
-        value = data.get('value')
-        data_list = value.get('list')
-
-        if not data_list:
-            print('获取数据为空')
-            return None
-
-        result_data = []
-
-        for d in data_list:
-            numbers = d.get('lotteryUnsortDrawresult')
-            try:
-                if len(numbers.split(' ')) < 7:
-                    continue
-            except Exception as e:
-                print('numbers: {}, err: {}'.format(numbers, e))
-                continue
-
-            red_list = numbers.split(' ')[:5]
-            blue_list = numbers.split(' ')[5:]
-
-            red_list.sort()
-            blue_list.sort()
-
-            try:
-                # 切开红球,蓝球数组
-                red1 = red_list[0]
-                red2 = red_list[1]
-                red3 = red_list[2]
-                red4 = red_list[3]
-                red5 = red_list[4]
-                blue1 = blue_list[0]
-                blue2 = blue_list[1]
-            except Exception as e:
-                print('红球或蓝球数据丢失')
-                continue
-
-            result_data.append({
-                'serial': d.get('lotteryDrawNum'),
-                'red1': red1 or '',
-                'red2': red2 or '',
-                'red3': red3 or '',
-                'red4': red4 or '',
-                'red5': red5 or '',
-                'blue1': blue1 or '',
-                'blue2': blue2 or '',
-                'drawPdfUrl': d.get('drawPdfUrl'),
-                'date': d.get('lotteryDrawTime'),
-                'pool': d.get('poolBalanceAfterdraw')
-            })
-
-        if result_data:
-            return result_data
-        else:
-            print('返回的数据为空, 获取数据失败')
-            return None
-
-    def data_compare(self, all_data):
-        if not all_data:
-            return '', ''
-
-        data = all_data[0]
-
-        red_list = [data['red1'], data['red2'], data['red3'], data['red4'], data['red5']]
-        blue_list = [data['blue1'], data['blue2']]
-
-        # 期号
-        subject = '{}'.format(data['serial'])
-
-        # 组成每期数据的text
-        serial_text = 'serial: {}\t\tlottery draw date: {}\nbonus pool: {} RMB\n{}\nlottery draw num: {} + {}\n\n'.format(
-            data['serial'], data['date'], data['pool'], '*' * 90,
-            red_list, blue_list)
-
-        for my_num in self.my_dlt:
-            my_red_list = my_num[:5]
-            my_blue_list = my_num[5:]
-
-            # 使用列表推导式找出两个列表中都存在的元素
-            red_common_elements = [element for element in red_list if element in my_red_list]
-            blue_common_elements = [element for element in blue_list if element in my_blue_list]
-
-            # 计算相等元素的数量
-            red_equal_count = len(red_common_elements)
-            blue_equal_count = len(blue_common_elements)
-
-            serial_text += 'my nums: {} + {}\nred hit: {}\nblue hit: {}\n\n'.format(my_red_list, my_blue_list,
-                                                                                    red_equal_count,
-                                                                                    blue_equal_count)
-
-        serial_text += '{}\n\n\n\n'.format('*' * 90)
-
-        return serial_text, subject
-
-    def send_message(self, text, subject):
-        if not text:
-            return
-
-        title = f'dlt {subject}'
-
-        # 推送到 message
-        GotifyNotifier(title, text, 'dlt').send_message()
-
-    async def main(self):
-        data = await self.req()
-        result_data = self.data_handle(data)
-        if not result_data:
-            return
-
-        text, subject = self.data_compare(result_data)
-        self.send_message(text, subject)
-
-
-if __name__ == '__main__':
-    asyncio.run(CheckDlt().main())

spider/spider_get_and_check_ssq.py

@@ -1,94 +0,0 @@
-# -*-coding: utf-8 -*-
-import datetime
-import os
-import sqlite3
-from selenium import webdriver
-import httpx
-
-
-def get_cookies(url):
-    chrome_options = webdriver.ChromeOptions()
-    args = ['--headless', '--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage']
-    for arg in args:
-        chrome_options.add_argument(arg)
-    driver = webdriver.Chrome(options=chrome_options)
-    driver.get(url)
-
-    result_cookie = driver.get_cookies()
-    if result_cookie:
-        return result_cookie
-    else:
-        pass
-
-
-def req(url, cookies):
-    with httpx.Client() as client:
-        headers = {
-            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
-            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
-            "Connection": "keep-alive",
-            "Cookie": cookies,
-            "Host": "www.cwl.gov.cn",
-            "User-Agent": "Mozilla/5.0"
-        }
-        res = client.get(url, headers=headers, follow_redirects=True)
-
-        if res.status_code != 200:
-            print(res.status_code)
-            log_file_path = os.path.join(get_path.get_logs_path(), str(datetime.date.today()) + '.log')
-            with open(log_file_path, 'a') as f:
-                f.write("\n spider_dlt: %s")
-            return
-
-        res_json = res.json()
-        data_handle(res_json['result'])
-
-
-def data_handle(source_data):
-    ssq_db_path = os.path.join(utils_get_path.get_db_path(), 'ssq.db')
-    conn = sqlite3.connect(ssq_db_path)
-
-    c = conn.cursor()
-
-    c.execute('drop table if exists data;')
-
-    c.execute(
-        'create table if not exists `ssq` (id INT PRIMARY KEY NOT NULL, `code` varchar(10),`red1` varchar(2),`red2` varchar(2),`red3` varchar(2),`red4` varchar(2),`red5` varchar(2),`red6` varchar(2),`blue` varchar(2),`date` varchar(12),`sales` varchar(15),`poolmoney` varchar(15),`content` varchar(255));')
-
-    id = 1
-    for data in source_data:
-        insert_sql = "INSERT INTO ssq ('id', 'code', 'red1', 'red2', 'red3', 'red4', 'red5', 'red6', 'blue', 'date', 'sales', 'poolmoney', 'content') VALUES ({0}, '{1}', '{2}', '{3}', '{4}', '{5}', '{6}', '{7}', '{8}', '{9}', '{10}', '{11}', '{12}')".format(
-            id,
-            data.setdefault('code'),
-            data.setdefault('red').split(',')[0],
-            data.setdefault('red').split(',')[1],
-            data.setdefault('red').split(',')[2],
-            data.setdefault('red').split(',')[3],
-            data.setdefault('red').split(',')[4],
-            data.setdefault('red').split(',')[5],
-            data.setdefault('blue'),
-            data.setdefault('date'),
-            data.setdefault('sales'),
-            data.setdefault('poolmoney'),
-            data.setdefault('content')
-        )
-        c.execute(insert_sql)
-        conn.commit()
-        id += 1
-
-    conn.close()
-
-
-if __name__ == "__main__":
-    url = 'http://www.cwl.gov.cn/cwl_admin/front/cwlkj/search/kjxx/findDrawNotice?name=ssq&issueCount=&issueStart=&issueEnd=&dayStart=&dayEnd=&pageNo=1&pageSize=10&week=&systemType=PC'
-
-    # result_cookie = util_get_cookies.get_cookies(url)
-    #
-    # cookies = '{}={}'.format(result_cookie[0].setdefault('name'), result_cookie[0].setdefault('value'))
-    #
-    # print(cookies)
-
-    # 测试时使用的 cookies
-    cookies = "HMF_CI=1b2fd73192f2054a429b2bfa4f58c3ff98119441420133cc8a04ca9c95aa2266eaec5bb7cf1d37df5f9864b8629ba407bacc9c58cadf26e2d726582df3870b0969"
-
-    req(url, cookies)

base/base_utils_ql_create_tasks.py → tools/tools_ql_create_tasks.py

@@ -4,7 +4,7 @@ import os
 import requests
 
 # 青龙面板的地址
-url = "https://auto.erhe.top"
+url = "https://qinglong.erhe.link"
 
 
 # 登录青龙面板
@@ -73,12 +73,10 @@ def main():
             else:
                 print("Tasks list is empty")
 
-            project_path = '/ql/data/scripts/auto/'
+            project_path = '/ql/data/scripts/AutoInfo/'
             base_path = os.path.join(project_path, 'base')
-            spider_path = os.path.join(project_path, 'spider')
             message_path = os.path.join(project_path, 'message')
             manual_path = os.path.join(project_path, 'manual')
-            daily_path = os.path.join(project_path, 'daily')
             tasks_template = [{
                 'base_tasks': [
                     {
@@ -104,56 +102,6 @@ def main():
                         "command": "python3 {}/base_news_data_collation.py".format(base_path),
                         "schedule": "0 10 6,12,18 * * *",
                         "labels": ["base"]
-                    },
-                    {
-                        "name": "定时刷新 freshrss 订阅源",
-                        "command": "python3 {}/update_feed.py".format(base_path),
-                        "schedule": "0 6,22 * * *",
-                        "labels": ["base"]
-                    }
-                ],
-                'spider': [
-                    {
-                        "name": "自动执行反斗限免爬虫",
-                        "command": "python3 {}/news_get_apprcn.py".format(spider_path),
-                        "schedule": "0 0 3,6,9,12,15,18,21 * * *",
-                        "labels": ["spider"]
-                    },
-                    {
-                        "name": "自动执行chiphell爬虫",
-                        "command": "python3 {}/news_get_chiphell.py".format(spider_path),
-                        "schedule": "0 0 3,6,9,12,15,18,21 * * *",
-                        "labels": ["spider"]
-                    },
-                    {
-                        "name": "自动执行hello_github爬虫",
-                        "command": "python3 {}/news_get_hello_github.py".format(spider_path),
-                        "schedule": "0 0 3,6,9,12,15,18,21 * * *",
-                        "labels": ["spider"]
-                    },
-                    {
-                        "name": "自动执行Anyknew爬虫",
-                        "command": "python3 {}/news_get_news.py".format(spider_path),
-                        "schedule": "0 0 3,6,9,12,15,18,21 * * *",
-                        "labels": ["spider"]
-                    },
-                    {
-                        "name": "自动执行币界网文章爬虫",
-                        "command": "python3 {}/spider_web3_coin_world.py".format(spider_path),
-                        "schedule": "0 0 3,6,9,12,15,18,21 * * *",
-                        "labels": ["spider"]
-                    },
-                    {
-                        "name": "获取 web3 新闻",
-                        "command": "python3 {}/spider_web3_news.py".format(spider_path),
-                        "schedule": "0 0 3,6,9,12,15,18,21 * * *",
-                        "labels": ["spider"]
-                    },
-                    {
-                        "name": "自动执行dlt爬虫",
-                        "command": "python3 {}/spider_get_and_check_dlt.py".format(spider_path),
-                        "schedule": "30 22 * * 1,3,6",
-                        "labels": ["spider"]
                     }
                 ],
                 'message_tasks': [
@@ -195,21 +143,8 @@ def main():
                     }
                 ],
                 'manual': [
-                    {
-                        "name": "手动读取rss订阅新闻",
-                        "command": "python3 {}/read_news.py".format(manual_path),
-                        "schedule": "0 0 1 1 *",
-                        "labels": ["manual"]
-                    }
-                ],
-                'daily': [
-                    {
-                        "name": "3dos自动签到",
-                        "command": "python3 {}/daily_3dos.py".format(daily_path),
-                        "schedule": "*/5 * * * *",
-                        "labels": ["daily"]
-                    }
-                ],
+
+                ]
             }]
 
             for task_template in tasks_template:

utils/__init__.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

utils/utils.py

@@ -1,9 +1,12 @@
 # -*- coding: utf-8 -*-
 import json
+import smtplib
 import time
 from datetime import datetime
 import os
 import sys
+from email.header import Header
+from email.mime.text import MIMEText
 
 import httpx
 import pymongo
@@ -11,19 +14,6 @@ import pymongo
 sys.path.append(os.path.join(os.path.abspath(__file__).split('AutoInfo')[0] + 'AutoInfo'))
 PROJECT_PATH = os.path.join(os.path.abspath(__file__).split('AutoInfo')[0] + 'AutoInfo')
 
-from utils.utils_mongo_handle import MongoHandle
-from base.base_load_config import load_config, get_base_path
-
-config_json = load_config()
-base_project = get_base_path()
-
-
-DB_USER = config_json.get('DB_USER')
-DB_PASSWORD = config_json.get('DB_PASSWORD')
-DB_IP = config_json.get('DB_IP')
-DB_PORT = config_json.get('DB_PORT')
-MONGO_LINK = f'mongodb://{DB_USER}:{DB_PASSWORD}@{DB_IP}:{DB_PORT}/'
-
 
 class LogsHandle(object):
     def __init__(self):
@@ -44,8 +34,7 @@ class LogsHandle(object):
         self.mongo.collection.insert_one(data_to_insert)
 
     def logs_send(self):
-        subject = 'auto collection logs'
-        title = 'auto collection - logs: {}'.format(self.now_day)
+        title = 'autoinfo - logs: {}'.format(self.now_day)
         text = ''
 
         # TODO
@@ -88,7 +77,14 @@ class LogsHandle(object):
 
 class MongoHandle(object):
     def __init__(self, db, collection, del_db=False, del_collection=False, auto_remove=0):
-        self.client = pymongo.MongoClient(MONGO_LINK)
+        config_json = LoadConfig().load_config()
+        base_project = LoadConfig().get_base_path()
+        db_user = config_json.get('DB_USER')
+        db_password = config_json.get('DB_PASSWORD')
+        db_ip = config_json.get('DB_IP')
+        db_port = config_json.get('DB_PORT')
+        mongo_link = f'mongodb://{db_user}:{db_password}@{db_ip}:{db_port}/'
+        self.client = pymongo.MongoClient(mongo_link)
         self.db = db
         self.collection = collection
 
@@ -121,17 +117,53 @@ class MongoHandle(object):
             self.collection.delete_one({'_id': data['_id']})
 
 
+class SendEmail(object):
+    def __init__(self, subject='AutoInfo subject', title='AutoInfo title', text='auto text') -> None:
+        config_json = LoadConfig().load_config()
+        mail_host = config_json.get('MAIL_HOST')
+        mail_user = config_json.get('MAIL_USER')
+        mail_pass = config_json.get('MAIL_PASS')
+        mail_sender = config_json.get('MAIL_SENDER')
+        mail_receivers = config_json.get('MAIL_RECEIVERS')
+        # 第三方 SMTP 服务
+        self.mail_host = mail_host  # 设置服务器
+        self.mail_user = mail_user  # 用户名
+        self.mail_pass = mail_pass  # 口令
+
+        self.sender = mail_sender
+        self.receivers = [mail_receivers]
+
+        self.subject = subject
+        self.title = title
+        self.text = text
+
+    def send(self):
+        message = MIMEText(self.text, 'plain', 'utf-8')
+        message['From'] = Header(self.title, 'utf-8')
+        message['To'] = Header("auto", 'utf-8')
+        message['Subject'] = Header(self.subject, 'utf-8')
+
+        try:
+            smtpObj = smtplib.SMTP_SSL(self.mail_host)
+            smtpObj.login(self.mail_user, self.mail_pass)
+            smtpObj.sendmail(self.sender, self.receivers, message.as_string())
+            print("邮件发送成功")
+        except smtplib.SMTPException as e:
+            print("Error: 无法发送邮件", e)
+
+
 class GotifyNotifier:
     def __init__(self, title, message, token_name=''):
-        self.gotify_url = 'https://gotify.erhe.top'
-        self.app_token = self.match_token_name(token_name)
+        config_json = LoadConfig().load_config()
+        self.gotify_url = config_json.get('GOTIFY_URL', 'https://gotify.erhe.top')
+        self.app_token = self.match_token_name(token_name or 'test')
         self.title = title
         self.message = message
 
     def match_token_name(self, name):
         token_name_dict = {}
         # 读取项目根目录下的 gotify_config.json 文件
-        gotify_config_path = os.path.join(PROJECT_PATH, 'gotify_config.json')
+        gotify_config_path = os.path.join(str(PROJECT_PATH), 'gotify_config.json')
         with open(gotify_config_path, 'r') as f:
             token_name_dict = json.load(f)
         token = token_name_dict.get(name)
@@ -141,30 +173,41 @@ class GotifyNotifier:
             return token_name_dict['base']
 
     def send_message(self):
-        # 构建POST请求的headers
-        headers = {
-            'Content-Type': 'application/json'
-        }
-
-        # 构建POST请求的body
-        body = {
-            'title': self.title,
-            'message': self.message
-        }
-
         # 发送POST请求
         with httpx.Client() as client:
             response = client.post(
                 url=f"{self.gotify_url}/message?token={self.app_token}",
-                headers=headers,
-                json=body
+                headers={'Content-Type': 'application/json'},
+                json={'title': self.title, 'message': self.message}
             )
 
         # 或者可以使用 curl
-        # curl -k "https://gotify.erhe.top/message?token=A0Xg6ZE5946iBYg" -F "title=测试发送信息" -F "message=假装有信息,测试发送" -F "priority=5"
+        # curl -k "https://gotify.erhe.top/message?token=A0Xg6ZE5946iBYg" -F "title=测试发送信息" -F "message=测试信息,测试发送" -F "priority=5"
 
         # 检查响应状态码
         if response.status_code == 200:
+            print(self.title)
             print('Gotify Message sent successfully!')
         else:
             print('Failed to send message:', response.text)
+
+
+class LoadConfig:
+    def load_config(self):
+        try:
+            config_path = os.path.join(PROJECT_PATH, 'config.json')
+            config_json = {}
+            with open(config_path, 'r') as f:
+                config_json = json.load(f)
+
+            if not config_json:
+                print('No config file found')
+                exit(0)
+        except Exception as e:
+            print(e)
+            exit(0)
+
+        return config_json
+
+    def get_base_path(self):
+        return os.path.join(os.getcwd().split('AutoInfo')[0], 'AutoInfo')