update

utils/utils_call_free_al.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+
+from openai import OpenAI
+
+
+class FREEAI(object):
+    def call_ai(self, message):
+        try:
+            client = OpenAI(
+                api_key="sk-rM32T5VuyyCFyZGyEe006aEdFe6e4301A7627f7a3973Df17",
+                base_url="https://knox.chat/v1",
+            )
+
+            completion = client.chat.completions.create(
+                model="deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free",
+                messages=[{"role": "user", "content": f"{message}"}],
+                temperature=0.3,
+            )
+
+            result = completion.choices[0].message.content
+
+            return result
+        except Exception as e:
+            print(e)
+
+# free = FREEAI()
+# chat = free.call_kimi('你好')
+# print(chat)

utils/utils_call_kimi.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+
+from openai import OpenAI
+
+
+class KIMI(object):
+    def call_kimi(self, message):
+        try:
+            print('call kimi')
+            client = OpenAI(
+                api_key="sk-Fz9tRF8naXReN2H7zcB1AEtnpOmhonFPJgxlVvQHpql54Ymu",
+                base_url="https://api.moonshot.cn/v1",
+            )
+
+            completion = client.chat.completions.create(
+                model="moonshot-v1-8k",
+                messages=[{"role": "user", "content": f"{message}"}],
+                temperature=0.3,
+            )
+
+            result = completion.choices[0].message.content
+
+            return result
+        except Exception as e:
+            print(e)
+
+    def call_ai(self, message):
+        pass

utils/utils_call_ollama.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+import time
+from ollama import Client as oClient
+
+
+class OllamaChat(object):
+    def call_ollama(self, host, role, text, prompt_words, model='llava:13b', temperature=0.4):
+        message = text + '\n\n' + prompt_words
+        print(f'use model: {model}')
+        try:
+            response_iter = oClient(host=host).chat(model=model,
+                                                    messages=[
+                                                        {'role': 'system', 'content': role},
+                                                        {'role': 'user', 'content': message}
+                                                    ],
+                                                    options={"temperature": temperature},
+                                                    stream=False)
+            return response_iter['message']['content']
+        except Exception as e:
+            print(f"\n发生错误: {e}")
+
+
+# if __name__ == "__main__":
+#     C = OllamaChat()
+#     start_time = time.time()
+#
+#     role = '你是一个聊天机器人'
+#
+#     text = 'hello'
+#
+#     prompt_words = '你好呀'
+#
+#     response_context = C.call_ollama('http://192.168.31.28:11434', role, text, prompt_words, model='llava:13b')
+#     print(response_context)
+#
+#     end_time = time.time()
+#     run_time = end_time - start_time
+#     print(f"程序运行时间：{run_time} 秒\n")

utils/utils_check_base.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+"""
+消息模块基础, 用于打开浏览器等相关操作
+"""
+import random
+
+from playwright.sync_api import sync_playwright
+import sys
+import os
+import time
+
+sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'))
+
+from utils.utils_logs_handle import LogsHandle
+
+
+class CryptoCrawler:
+    def __init__(self, url_list, selectors, check_difference=False, headless=True, proxy=False):
+        self.url_list = url_list
+        self.selectors = selectors
+        self.check_difference = check_difference  # 用于检测数据是否发生变化 (开关)
+        self.data_difference = False  # 用于检测数据是否发生变化 (结果) (默认 否)
+        self.logs_handle = LogsHandle()  # 记录日志
+        self.db = 'CHECK'
+        self.collection = 'check'
+        self.headless = headless
+        self.proxy = proxy
+
+    def main(self):
+        with sync_playwright() as playwright:
+            if self.proxy:
+                browser = playwright.webkit.launch(headless=self.headless, proxy={'server': '127.0.0.1:7890'})
+            else:
+                browser = playwright.webkit.launch(headless=self.headless)
+            context = browser.new_context(viewport={'width': 1920, 'height': 1080})
+            page = context.new_page()
+
+            all_data = []
+
+            for url_info in self.url_list:
+                for key, url in url_info.items():
+                    result_list = []
+                    try:
+                        page.goto(url)
+                        page.wait_for_load_state('load')
+                        time.sleep(5)  # 确保页面完全加载
+
+                        for selector in self.selectors:
+                            element = page.query_selector(selector)
+                            if element:
+                                res = element.text_content().strip()
+                                result_list.append({key: res})
+                    except Exception as e:
+                        err_str = f"Error fetching {url}: {e}"
+                        self.logs_handle.logs_write(self.collection, err_str, 'error', False)
+                        continue
+
+                    if result_list:
+                        all_data.append(result_list)
+
+                    time.sleep(random.randint(1, 3))
+
+            browser.close()
+
+            if all_data:
+                return all_data
+            else:
+                return None

utils/utils_create_rss_record.py

@@ -0,0 +1,98 @@
+import time
+
+import psycopg2
+from psycopg2 import Error
+
+
+class CreateRssRecord(object):
+    def __init__(self):
+        self.hostname = 'erhe.top'
+        self.port = 20788
+        self.database = 'freshrss'
+        self.user = 'freshrss'
+        self.password = 'freshrss'
+        self.conn = None
+
+    def connect(self):
+        """连接到 PostgreSQL 数据库"""
+        try:
+            self.conn = psycopg2.connect(
+                dbname=self.database,
+                user=self.user,
+                password=self.password,
+                host=self.hostname,
+                port=self.port
+            )
+        except Error as e:
+            print(f"Error connecting to the database: {e}")
+        else:
+            print("Connected to the database successfully.")
+
+        if not self.conn:
+            raise Exception("Database connection failed")
+
+    def check_and_insert(self, data):
+        """检查 URL 是否存在，如果不存在则插入整个数据字典"""
+        try:
+            with self.conn.cursor() as cursor:
+                # 检查 URL 是否存在
+                select_sql = "SELECT COUNT(*) FROM freshrsstoor_feed WHERE url = %s"
+                cursor.execute(select_sql, (data['url'],))
+                result = cursor.fetchone()
+                if result[0] == 0:
+                    # URL 不存在，插入新记录
+                    columns = ', '.join(data.keys())
+                    values = ', '.join(['%s'] * len(data))
+                    insert_sql = """INSERT INTO freshrsstoor_feed 
+                    (url, kind, category, "name", website, description, "lastUpdate", priority, "pathEntries", "httpAuth", "error", "ttl", attributes, "cache_nbEntries", "cache_nbUnreads") 
+                    VALUES ('{}', {}, {}, '{}', '{}', '{}', {}, {}, '{}', '{}', {}, {}, '{}', {}, {});""".format(
+                        data['url'],
+                        data['kind'],
+                        data['category'],
+                        data['name'],
+                        data['website'],
+                        data['description'],
+                        data['lastUpdate'],
+                        data['priority'],
+                        data['pathEntries'],
+                        data['httpAuth'],
+                        data['error'],
+                        data['ttl'],
+                        data['attributes'],
+                        data['cache_nbEntries'],
+                        data['cache_nbUnreads']
+                    )
+                    cursor.execute(insert_sql, tuple(data.values()))
+                    self.conn.commit()
+                    print("Data inserted successfully.")
+                else:
+                    print("URL already exists.")
+        except Error as e:
+            print(f"Error: {e}")
+        finally:
+            if self.conn is not None:
+                self.conn.close()
+
+
+# 使用示例
+if __name__ == "__main__":
+    cr = CreateRssRecord()
+    cr.connect()
+    insert_data = {
+        'url': 'https://rsshub.app/jike/topic/556688fae4b00c57d9dd46ee',
+        'category': 7,
+        'name': '今日份的摄影 - 即刻圈子',
+        'website': 'http://finance.sina.com.cn/china/',
+        'description': '爱摄影的即友都在这里~分享原创摄影作品，感受照片背后的共鸣吧！ - Powered by RSSHub',
+        'kind': 0,
+        'lastUpdate': int(time.time()),
+        'priority': 10,
+        'pathEntries': '',
+        'httpAuth': '',
+        'error': 0,
+        'ttl': 0,
+        'attributes': '{"curl_params":null,"ssl_verify":null,"timeout":null}',
+        'cache_nbEntries': 0,
+        'cache_nbUnreads': 0
+    }
+    cr.check_and_insert(insert_data)

utils/utils_daily_task_scheduler.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+import os
+import time
+from datetime import datetime
+"""
+TaskScheduler 类用于管理定时任务的执行时间。
+
+该类通过在指定的文件中存储时间戳来记录任务的上次执行时间和下次执行时间。
+它提供了两个主要方法：load_start_time 和 save_start_time。
+load_start_time 方法用于检查是否到达任务的执行时间，如果未到达则返回 0，否则返回上次执行时间。
+save_start_time 方法用于更新下次执行时间，默认为当前时间加上指定的时间间隔（默认为 24 小时）。
+
+Attributes:
+    file_path (str): 存储时间戳的文件路径。
+    time_span (int): 任务执行的时间间隔，默认为 86400 秒（24 小时）。
+
+Methods:
+    load_start_time(): 检查是否到达任务的执行时间。
+    save_start_time(): 更新下次执行时间。
+
+Example:
+    >>> scheduler = TaskScheduler("task_time.txt")
+    >>> last_start_time = scheduler.load_start_time()
+    >>> if last_start_time != 0:
+    >>>     # 执行任务
+    >>> scheduler.save_start_time()
+
+Note:
+    该类假设文件路径是有效的，并且有足够的权限进行读写操作。
+    如果文件不存在，将自动创建并写入当前时间戳。
+"""
+
+class TaskScheduler:
+    def __init__(self, file_path, time_span=86400):
+        self.file_path = file_path
+        self.time_span = time_span
+
+    def load_start_time(self):
+        timestamp_now = int(time.time())
+
+        start_time = 0
+        if not os.path.exists(self.file_path):
+            with open(self.file_path, "w") as file:
+                file.write(str(timestamp_now))
+            print('创建执行时间文件 {}'.format(datetime.fromtimestamp(timestamp_now).strftime('%Y-%m-%d %H:%M:%S')))
+            return timestamp_now
+        else:
+            with open(self.file_path, "r") as file:
+                start_time = int(file.read())
+            if start_time > timestamp_now:
+                print('任务未到执行时间, 下次执行时间: {}'.format(datetime.fromtimestamp(start_time).strftime('%Y-%m-%d %H:%M:%S')))
+                return 0
+            print('开始执行定时任务')
+            return start_time
+
+    def save_start_time(self):
+        timestamp_now = int(time.time())
+        date_now = datetime.fromtimestamp(timestamp_now).strftime('%Y-%m-%d %H:%M:%S')
+
+        # 时间戳 +24 小时
+        timestamp_now += self.time_span
+        print('写入下次执行时间: {}'.format(datetime.fromtimestamp(timestamp_now).strftime('%Y-%m-%d %H:%M:%S')))
+        with open(self.file_path, "w") as file:
+            file.write(str(timestamp_now))

utils/utils_get_current_ip.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+import aiohttp
+import asyncio
+
+
+async def fetch(url):
+    async with aiohttp.ClientSession() as session:
+        async with session.get(url) as response:
+            return await response.text()
+
+
+async def main():
+    url = 'http://api.ip.cc'
+    content = await fetch(url)
+    print(content)
+
+
+asyncio.run(main())
+
+'''
+Proxy server: 175.99.17.215
+port: 6011
+username: lumi-jiege0210
+password: aaaAAA111
+'''

utils/utils_get_public_ip.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+import httpx
+
+
+def get_public_ip():
+    try:
+        # 使用 httpx 发起请求
+        response = httpx.get("https://httpbin.org/ip", timeout=10)
+        response.raise_for_status()  # 检查请求是否成功
+        ip_data = response.json()
+        return ip_data["origin"]
+    except httpx.RequestError as e:
+        print(f"An error occurred while obtaining the public IP address.：{e}")
+        exit(1)
+

utils/utils_logs_handle.py

@@ -0,0 +1,82 @@
+# -*- coding: UTF-8 -*-
+'''
+获取每天日期,新建 logs 文件
+存到 logs 文件夹中
+'''
+import time
+from datetime import datetime
+import os
+import sys
+
+sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'))
+
+from utils.utils_mongo_handle import MongoHandle
+from utils.utils_send_email import SendEmail
+from base.base_load_config import load_config, get_base_path
+
+config_json = load_config()
+base_project = get_base_path()
+
+
+class LogsHandle(object):
+    def __init__(self):
+        self.now_day = time.strftime('%Y-%m-%d', time.localtime())
+        db = 'logs'
+        collection = 'logs_' + self.now_day
+        self.mongo = MongoHandle(db=db, collection=collection, del_db=False, del_collection=False, auto_remove=0)
+
+    def logs_generate(self):
+        data_to_insert = {
+            "title": "logs",
+            "context": 'generate logs',
+            "state": "create",
+            "create_time": int(time.time()),
+            "create_datetime": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        }
+
+        self.mongo.collection.insert_one(data_to_insert)
+
+    def logs_send(self):
+        subject = 'auto collection logs'
+        title = 'auto collection - logs: {}'.format(self.now_day)
+        text = ''
+
+        # TODO
+        # 从 mongodb 读取日志, 拼接 text, 发送邮件
+        # 查询所有文档
+        cursor = self.mongo.collection.find()
+        # 遍历结果集
+        for record in cursor:
+            text += "logs_source: {}, logs_detail: {}, state: {} logs_create_time: {}\n\n".format(
+                record.setdefault('title'),
+                record.setdefault('content'),
+                record.setdefault('state'),
+                record.setdefault('create_datetime'),
+            )
+
+        S = SendEmail(subject=subject, title=title, text=text)
+        S.send()
+
+    def logs_write(self, title_source=None, content=None, state=None, send_now=False):
+        data_to_insert = {
+            "title": title_source,
+            "context": content,
+            "state": state,
+            "create_time": int(time.time()),
+            "create_datetime": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        }
+
+        self.mongo.collection.insert_one(data_to_insert)
+
+        if send_now:
+            subject = 'auto collection'
+            title = 'auto collection - running logs: {}'.format(self.now_day)
+            text = 'logs_source: {}, logs_detail: {}, state: {} logs_create_time: {}'.format(
+                data_to_insert.setdefault('title'),
+                data_to_insert.setdefault('content'),
+                data_to_insert.setdefault('state'),
+                data_to_insert.setdefault('create_datetime'),
+            )
+
+            Send = SendEmail(subject=subject, title=title, text=text)
+            Send.send()

utils/utils_mongo_handle.py

@@ -0,0 +1,62 @@
+# -*-coding: utf-8 -*-
+import pymongo
+from pymongo import errors
+import time
+import sys
+import os
+
+sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'))
+
+from base.base_load_config import load_config, get_base_path
+
+config_json = load_config()
+base_project = get_base_path()
+
+DB_USER = config_json.get('DB_USER')
+DB_PASSWORD = config_json.get('DB_PASSWORD')
+DB_IP = config_json.get('DB_IP')
+DB_PORT = config_json.get('DB_PORT')
+MONGO_LINK = f'mongodb://{DB_USER}:{DB_PASSWORD}@{DB_IP}:{DB_PORT}/'
+
+
+class MongoHandle(object):
+    def __init__(self, db, collection, del_db=False, del_collection=False, auto_remove=0):
+        self.client = pymongo.MongoClient(MONGO_LINK)
+        self.db = db
+        self.collection = collection
+
+        if del_db and db:
+            # 检查数据库是否存在
+            if db in self.client.list_database_names():
+                # 删除数据库
+                self.client.drop_database(db)
+        self.db = self.client[db]
+
+        if del_collection and self.collection:
+            # 检查集合是否存在
+            if self.collection in self.db.list_collection_names():
+                # 删除集合
+                self.db.drop_collection(collection)
+        self.collection = self.db[collection]
+
+        if auto_remove:
+            self.auto_remove_data(auto_remove)
+
+    def write_data(self, data):
+        self.collection.insert_one(data)
+
+    def load_data(self):
+        # MongoDB 会在第一次写入时自动创建数据库和集合
+        return list(self.collection.find({}, {'_id': False}))
+
+    def auto_remove_data(self, day):
+        for data in self.collection.find({'create_time': {'$lt': int(time.time()) - day * 24 * 60 * 60}}):
+            self.collection.delete_one({'_id': data['_id']})
+
+# if __name__ == '__main__':
+#     mongo = MongoHandle('test_db', 'test_collection', False, False, 0)
+#     mongo.collection.insert_one({'name': 'test'})
+#     mongo.collection.insert_many([{'name': 'test1'}, {'name': 'test2'}])
+#     print(mongo.collection.find_one())
+#     print(mongo.collection.find())
+#     print('done!')

utils/utils_ql_create_tasks.py

@@ -0,0 +1,236 @@
+# -*- coding: utf-8 -*-
+import os
+
+import requests
+
+# 青龙面板的地址
+url = "https://auto.erhe.top"
+
+
+# 登录青龙面板
+def login():
+    response = requests.post(f"{url}/api/user/login", json={"username": "toor", "password": "!QAZ2wsx+0913"})
+    print(response)
+    if response.status_code != 200:
+        print(response.status_code)
+        print(response.text)
+        exit(0)
+    return response.json()['data']['token']
+
+
+# 获取任务列表
+def get_tasks(token):
+    response = requests.get(f"{url}/api/crons", headers={"Authorization": f"Bearer {token}"})
+    return response.json()['data']['data']
+
+
+# 创建任务
+def create_task(task_template, token):
+    payload = {
+        "name": task_template["name"],
+        "command": task_template["command"],
+        "schedule": task_template["schedule"],
+        "labels": task_template["labels"]
+    }
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Content-Type": "application/json"
+    }
+    response = requests.post(f"{url}/api/crons", headers=headers, json=payload)
+    return response.json()
+
+
+# 创建视图分类
+def create_view_type(token):
+    view_type_list = ['base', 'spider_common']
+    for view_type in view_type_list:
+        payload = {
+            "name": view_type,
+            "filters": {
+                'property': 'labels',
+                'operation': 'Reg',
+                'value': view_type
+            },
+            'filterRelation': 'and'
+        }
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json"
+        }
+        response = requests.post(f"{url}/api/crons", headers=headers, json=payload)
+
+
+# 主逻辑
+def main():
+    while True:
+        try:
+            token = login()
+            print(f"已连接到 {url}")
+            tasks = get_tasks(token)
+            tasks_names = [task['name'] for task in tasks]
+            if tasks:
+                print("Current tasks name: \n{}, \ntotal: {}".format('\n'.join(tasks_names), str(len(tasks_names))))
+            else:
+                print("Tasks list is empty")
+
+            project_path = '/ql/data/scripts/auto/'
+            base_path = os.path.join(project_path, 'base')
+            spider_path = os.path.join(project_path, 'spider')
+            message_path = os.path.join(project_path, 'message')
+            manual_path = os.path.join(project_path, 'manual')
+            daily_path = os.path.join(project_path, 'daily')
+            tasks_template = [{
+                'base_tasks': [
+                    {
+                        "name": "每天开始自动创建日志",
+                        "command": "python3 {}/base_daily_logs_generate.py".format(base_path),
+                        "schedule": "0 0 * * *",
+                        "labels": ["base"]
+                    },
+                    {
+                        "name": "每天结束自动发送日志",
+                        "command": "python3 {}/base_daily_logs_send.py".format(base_path),
+                        "schedule": "58 23 * * *",
+                        "labels": ["base"]
+                    },
+                    {
+                        "name": "每天自动删除旧数据",
+                        "command": "python3 {}/base_timing_remove_data.py".format(base_path),
+                        "schedule": "1 0 * * *",
+                        "labels": ["base"]
+                    },
+                    {
+                        "name": "每天新闻汇总,发送到邮箱",
+                        "command": "python3 {}/base_news_data_collation.py".format(base_path),
+                        "schedule": "0 10 6,12,18 * * *",
+                        "labels": ["base"]
+                    },
+                    {
+                        "name": "定时刷新 freshrss 订阅源",
+                        "command": "python3 {}/update_feed.py".format(base_path),
+                        "schedule": "0 6,22 * * *",
+                        "labels": ["base"]
+                    }
+                ],
+                'spider': [
+                    {
+                        "name": "自动执行反斗限免爬虫",
+                        "command": "python3 {}/news_get_apprcn.py".format(spider_path),
+                        "schedule": "0 0 3,6,9,12,15,18,21 * * *",
+                        "labels": ["spider"]
+                    },
+                    {
+                        "name": "自动执行chiphell爬虫",
+                        "command": "python3 {}/news_get_chiphell.py".format(spider_path),
+                        "schedule": "0 0 3,6,9,12,15,18,21 * * *",
+                        "labels": ["spider"]
+                    },
+                    {
+                        "name": "自动执行hello_github爬虫",
+                        "command": "python3 {}/news_get_hello_github.py".format(spider_path),
+                        "schedule": "0 0 3,6,9,12,15,18,21 * * *",
+                        "labels": ["spider"]
+                    },
+                    {
+                        "name": "自动执行Anyknew爬虫",
+                        "command": "python3 {}/news_get_news.py".format(spider_path),
+                        "schedule": "0 0 3,6,9,12,15,18,21 * * *",
+                        "labels": ["spider"]
+                    },
+                    {
+                        "name": "自动执行币界网文章爬虫",
+                        "command": "python3 {}/spider_web3_coin_world.py".format(spider_path),
+                        "schedule": "0 0 3,6,9,12,15,18,21 * * *",
+                        "labels": ["spider"]
+                    },
+                    {
+                        "name": "获取 web3 新闻",
+                        "command": "python3 {}/spider_web3_news.py".format(spider_path),
+                        "schedule": "0 0 3,6,9,12,15,18,21 * * *",
+                        "labels": ["spider"]
+                    },
+                    {
+                        "name": "自动执行dlt爬虫",
+                        "command": "python3 {}/spider_get_and_check_dlt.py".format(spider_path),
+                        "schedule": "30 22 * * 1,3,6",
+                        "labels": ["spider"]
+                    }
+                ],
+                'message_tasks': [
+                    {
+                        "name": "获取coin实时数据",
+                        "command": "python3 {}/message_coin_detail.py".format(message_path),
+                        "schedule": "0 * * * *",
+                        "labels": ["message"]
+                    },
+                    {
+                        "name": "对比大乐透最新一期数据,匹配已购买号码,发送消息",
+                        "command": "python3 {}/message_dlt.py".format(message_path),
+                        "schedule": "30 22 * * 1,3,6",
+                        "labels": ["message"]
+                    },
+                    {
+                        "name": "获取未来 7 天的天气预报",
+                        "command": "python3 {}/message_get_one_week_weather.py".format(message_path),
+                        "schedule": "0 0 6,22 * * *",
+                        "labels": ["message"]
+                    },
+                    {
+                        "name": "从 freshrss-psql 数据库中读取数据并发送",
+                        "command": "python3 {}/message_rss_data_handel.py".format(message_path),
+                        "schedule": "30 6,9,12,18,22 * * *",
+                        "labels": ["message"]
+                    },
+                    {
+                        "name": "空投任务消息",
+                        "command": "python3 {}/message_airdrop_tasks.py".format(message_path),
+                        "schedule": "0 8,20 * * *",
+                        "labels": ["message"]
+                    },
+                    {
+                        "name": "链捕手快讯消息推送",
+                        "command": "python3 {}/message_chaincatcher.py".format(message_path),
+                        "schedule": "0 */2 * * *",
+                        "labels": ["message"]
+                    }
+                ],
+                'manual': [
+                    {
+                        "name": "手动读取rss订阅新闻",
+                        "command": "python3 {}/read_news.py".format(manual_path),
+                        "schedule": "0 0 1 1 *",
+                        "labels": ["manual"]
+                    }
+                ],
+                'daily': [
+                    {
+                        "name": "3dos自动签到",
+                        "command": "python3 {}/daily_3dos.py".format(daily_path),
+                        "schedule": "*/5 * * * *",
+                        "labels": ["daily"]
+                    }
+                ],
+            }]
+
+            for task_template in tasks_template:
+                for task_type, task_list in task_template.items():
+                    for task in task_list:
+                        task_name = task["name"]
+                        if task_name in tasks_names:
+                            print("Task {} already exists.".format(task_name))
+                        else:
+                            result = create_task(task, token)
+                            print("Task creation result:", result)
+
+            # 创建所有任务之后, 创建视图分类
+            # create_view_type(token)
+            break  # 正常执行完成后退出循环
+
+        except Exception as e:
+            print("An error occurred: ", e)
+            print("Retrying...")
+
+
+if __name__ == "__main__":
+    main()
+    print('done!')

utils/utils_send_email.py

@@ -0,0 +1,58 @@
+# -*- coding: UTF-8 -*-
+import smtplib
+from email.mime.text import MIMEText
+from email.header import Header
+import os
+import sys
+
+sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'))
+
+from base.base_load_config import load_config, get_base_path
+
+config_json = load_config()
+base_project = get_base_path()
+
+PROJECT_NAME = config_json.get('PROJECT_NAME')
+DB_USER = config_json.get('DB_USER')
+DB_PASSWORD = config_json.get('DB_PASSWORD')
+DB_IP = config_json.get('DB_IP')
+DB_PORT = config_json.get('DB_PORT')
+MONGO_LINK = f'mongodb://{DB_USER}:{DB_PASSWORD}@{DB_IP}:{DB_PORT}/'
+MAIL_HOST = config_json.get('MAIL_HOST')
+MAIL_USER = config_json.get('MAIL_USER')
+MAIL_PASS = config_json.get('MAIL_PASS')
+MAIL_SENDER = config_json.get('MAIL_SENDER')
+MAIL_RECEIVERS = config_json.get('MAIL_RECEIVERS')
+
+
+class SendEmail(object):
+    def __init__(self, subject='auto subject', title='auto title', text='auto text') -> None:
+        # 第三方 SMTP 服务
+        self.mail_host = MAIL_HOST  # 设置服务器
+        self.mail_user = MAIL_USER  # 用户名
+        self.mail_pass = MAIL_PASS  # 口令
+
+        self.sender = MAIL_SENDER
+        self.receivers = [MAIL_RECEIVERS]
+
+        self.subject = subject
+        self.title = title
+        self.text = text
+
+    def send(self):
+        message = MIMEText(self.text, 'plain', 'utf-8')
+        message['From'] = Header(self.title, 'utf-8')
+        message['To'] = Header("auto", 'utf-8')
+        message['Subject'] = Header(self.subject, 'utf-8')
+
+        try:
+            smtpObj = smtplib.SMTP_SSL(self.mail_host)
+            smtpObj.login(self.mail_user, self.mail_pass)
+            smtpObj.sendmail(self.sender, self.receivers, message.as_string())
+            print("邮件发送成功")
+        except smtplib.SMTPException as e:
+            print("Error: 无法发送邮件", e)
+
+# if __name__ == '__main__':
+#     email = SendEmail(subject="测试邮件", title="测试邮件", text="这是一封测试邮件。")
+#     email.send()

utils/utils_send_gotify.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+
+import httpx
+
+
+class GotifyNotifier:
+    def __init__(self, title, message, token_name=''):
+        self.gotify_url = 'https://gotify.erhe.top'
+        self.app_token = self.match_token_name(token_name)
+        self.title = title
+        self.message = message
+
+    def match_token_name(self, name):
+        token_name_dict = {
+            'base': 'A8EVb0Cmxnb2vfk',
+            'coin': 'AgfOJESqDKftBTQ',
+            'dlt': 'A3bqt9Dlbs.fPUb',
+            'AirdropTasksNews': 'Aoe0VKt-kkZnm8d',
+            'weather': 'A9KF--mx_12PjSu',
+            'news': 'AT2QGp_vyCX4akW',
+            'CheckAndRemind': 'Aw7XKE2Ppk7Dgwk',
+            'test': 'A0Xg6ZE5946iBYg',
+        }
+
+        token = token_name_dict.get(name)
+        if token:
+            return token
+        else:
+            return token_name_dict['base']
+
+    def send_message(self):
+        # 构建POST请求的headers
+        headers = {
+            'Content-Type': 'application/json'
+        }
+
+        # 构建POST请求的body
+        body = {
+            'title': self.title,
+            'message': self.message
+        }
+
+        # 发送POST请求
+        with httpx.Client() as client:
+            response = client.post(
+                url=f"{self.gotify_url}/message?token={self.app_token}",
+                headers=headers,
+                json=body
+            )
+
+        # 或者可以使用 curl
+        # curl -k "https://gotify.erhe.top/message?token=A0Xg6ZE5946iBYg" -F "title=测试发送信息" -F "message=假装有信息,测试发送" -F "priority=5"
+
+        # 检查响应状态码
+        if response.status_code == 200:
+            print('Gotify Message sent successfully!')
+        else:
+            print('Failed to send message:', response.text)

utils/utils_send_matrix.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+# pip install matrix_client
+
+from matrix_client.client import MatrixClient
+from matrix_client.api import MatrixHttpApi
+
+class MatrixBot:
+    def __init__(self, user, password):
+        self.base_url = "https://matrix.erhe.top"
+        self.user = user
+        self.password = password
+        self.client = MatrixClient("https://matrix.erhe.top")
+        self.token = self.login()
+        self.to = "!CgWvWEnLbKYvhXLvil:chat.abeginner.cn"
+
+    def login(self):
+        self.token = self.client.login(username=self.user, password=self.password)
+        return self.token
+
+    def send_message(self, message):
+        if self.token:
+            try:
+                api = MatrixHttpApi(self.base_url, token=self.token)
+                api.send_message(self.to, message)
+            except Exception as e:
+                print(e)
+                api = MatrixHttpApi(self.base_url, token=self.token)
+                api.send_message(self.to, str(e))
+
+        else:
+            print("Bot is not logged in. Please login first.")
+
+if __name__ == '__main__':
+    # 测试调用
+    user = "message-bot"
+    pw = "aaaAAA111!!!"
+    message = "123987456"
+
+    bot = MatrixBot(user, pw)
+    bot.send_message(message)

utils/utils_send_serverchan.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+
+import httpx
+
+
+class ServerChanNotifier:
+    def __init__(self, title, message):
+        self.serverchan_url = 'https://sctapi.ftqq.com/SCT158272TRDcSniEdGR0TKhqhUl66LPHb.send'
+        self.title = title
+        self.message = message
+
+    def send_message(self):
+        # 构建POST请求的body
+        body = {
+            'text': self.title,
+            'desp': self.message
+        }
+
+        # 发送POST请求
+        with httpx.Client() as client:
+            response = client.post(
+                url=self.serverchan_url,
+                headers={'Content-Type': 'application/x-www-form-urlencoded'},
+                data=body
+            )
+
+        # 检查响应状态码
+        if response.status_code == 200:
+            print('ServerChan Message sent successfully!')
+        else:
+            print('Failed to send message:', response.text)

utils/utils_sync_psql.py

@@ -0,0 +1,219 @@
+# -*- coding: utf-8 -*-
+# 内部数据库同步数据脚本, 用于同一台服务器, 将mongodb数据同步到pgsql
+from datetime import datetime
+from pymongo import MongoClient
+import psycopg2
+from psycopg2 import sql
+
+MONGOLINK = 'mongodb://root:aaaAAA111!!!@erhe.top:38000/'
+PGSQLPARAMS = {
+    # 'database': 'postgres',  # 默认连接到postgres数据库，超级用户数据库
+    'user': 'toor',
+    'password': 'aaaAAA111',  # 替换为你的密码
+    'host': '192.168.31.177',
+    'port': 5432
+}
+
+
+def mongo():
+    # mongodb
+    client = MongoClient(MONGOLINK)
+
+    # 指定数据库名称
+    db_name = 'NEWS'  # 替换为你的数据库名称
+
+    # 选择数据库
+    db = client[db_name]
+
+    # 列出数据库中的所有集合
+    collections = db.list_collection_names()
+
+    all_data = []
+
+    for collection_name in collections:
+        # 选择集合
+        collection = db[collection_name]
+
+        # 读取集合中的所有数据
+        for document in collection.find({}, {'_id': 0}):
+            all_data.append(document)
+
+    sorted_data = []
+
+    if all_data:
+        sorted_data = sorted(all_data, key=lambda x: x['create_time'], reverse=True)
+
+    return sorted_data
+
+
+def pg(sorted_data):
+    table_name = 'auto'
+    PGSQLPARAMS.update({'database': table_name})
+    conn = psycopg2.connect(**PGSQLPARAMS)
+
+    # 记录相同数据, 如果查过指定数量, 则退出程序, 避免浪费资源
+    same_data = 500
+    for doc in sorted_data:
+        if same_data > 0:
+            try:
+                cur = conn.cursor()
+
+                create_time_dt = None
+                if doc.get('create_time'):
+                    create_time_dt = datetime.utcfromtimestamp(doc['create_time'])
+
+                values = {
+                    'name': doc.get('title'),
+                    'context': doc.get('context') or '',
+                    'source_url': doc.get('source_url') or '',
+                    'link': doc.get('line') or '',
+                    'article_type': doc.get('article_type') or '',
+                    'article_source': doc.get('article_source') or '',
+                    'img_url': doc.get('img_url') or '',
+                    'keyword': doc.get('keyword') or '',
+                    'posted_date': doc.get('posted_date') or '',
+                    'create_time_ts': doc.get('create_time') or '',
+                    'create_time': create_time_dt,
+                    'create_datetime': datetime.strptime(doc['create_datetime'], '%Y-%m-%d %H:%M:%S') if doc.get(
+                        'create_datetime') else None
+                }
+
+                # 将create_time转换为适合数据库的时间戳格式
+                create_time_dt = datetime.utcfromtimestamp(values['create_time_ts']) if values[
+                    'create_time_ts'] else None
+                values['create_time'] = create_time_dt
+
+                # 将create_datetime转换为适合数据库的时间戳格式
+                create_datetime_str = doc.get('create_datetime')
+                values['create_datetime'] = datetime.strptime(create_datetime_str,
+                                                              '%Y-%m-%d %H:%M:%S') if create_datetime_str else None
+
+                # 检查数据库中是否已存在相同title的记录
+                check_query = "SELECT id FROM news_info WHERE name = %s;"
+                cur.execute(check_query, (values['name'],))
+
+                # 如果没有找到记录，则插入新记录
+                if not cur.fetchone():
+                    insert_query = """
+                        INSERT INTO news_info (name, context, source_url, link, article_type, article_source, img_url, keyword, posted_date, create_time, create_datetime)
+                        VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
+                        """
+                    cur.execute(insert_query, (
+                        values['name'],
+                        values['context'],
+                        values['source_url'],
+                        values['link'],
+                        values['article_type'],
+                        values['article_source'],
+                        values['img_url'],
+                        values['keyword'],
+                        values['posted_date'],
+                        values['create_time'],
+                        values['create_datetime']
+                    ))
+                    conn.commit()
+                    print(f'已保存{values}')
+                else:
+                    same_data -= 1
+            except Exception as e:
+                print("Error during search: ", e)
+            finally:
+                # 关闭游标和连接
+                if 'cur' in locals():
+                    cur.close()
+        else:
+            print('相同数据计数已到最大值, 退出程序')
+            break
+    conn.close()
+
+
+def create_db():
+    db_name = 'postgres'
+
+    PGSQLPARAMS.update({'database': db_name})
+    # 连接到数据库
+    conn = psycopg2.connect(**PGSQLPARAMS)
+    conn.autocommit = True  # 确保创建数据库的命令可以立即执行
+
+    # 创建一个cursor对象
+    cur = conn.cursor()
+
+    # 要创建的数据库名
+    new_db = 'auto'
+    # 数据库所有者
+    owner = 'toor'
+
+    try:
+        # 检查数据库是否存在
+        cur.execute(sql.SQL("SELECT 1 FROM pg_catalog.pg_database WHERE datname = %s"), (new_db,))
+        exists = cur.fetchone()
+        if not exists:
+            # 如果数据库不存在，创建它
+            cur.execute(sql.SQL("CREATE DATABASE {} WITH OWNER {} ENCODING 'UTF8'").format(
+                sql.Identifier(new_db),
+                sql.Identifier(owner)
+            ))
+            print(f"Database '{new_db}' created successfully.")
+        else:
+            print(f"Database '{new_db}' already exists.")
+    except Exception as e:
+        print(f"An error occurred: {e}")
+    finally:
+        # 关闭cursor和连接
+        cur.close()
+        conn.close()
+
+
+def create_table():
+    db_name = 'auto'
+
+    PGSQLPARAMS.update({'database': db_name})
+
+    # 创建数据库连接
+    conn = psycopg2.connect(**PGSQLPARAMS)
+
+    # 创建一个 cursor 对象
+    cur = conn.cursor()
+
+    # 定义表结构的 SQL 语句
+    create_table_sql = """
+    CREATE TABLE IF NOT EXISTS news_info (
+        id SERIAL PRIMARY KEY,
+        name VARCHAR(255) NOT NULL,
+        context TEXT,
+        source_url VARCHAR(255),
+        link VARCHAR(255),
+        article_type VARCHAR(50),
+        article_source VARCHAR(50),
+        img_url VARCHAR(255),
+        keyword VARCHAR(255),
+        posted_date VARCHAR,
+        create_time_ts BIGINT,
+        create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+        create_datetime TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP
+    );
+    """
+
+    try:
+        # 执行 SQL 语句来创建表
+        cur.execute(create_table_sql)
+        conn.commit()  # 提交事务
+        print("Table 'auto' created successfully.")
+    except Exception as e:
+        print(f"An error occurred while creating table: {e}")
+        conn.rollback()  # 发生错误时回滚事务
+    finally:
+        # 关闭 cursor 和连接
+        cur.close()
+        conn.close()
+
+
+if __name__ == '__main__':
+    create_db()
+    create_table()
+
+    sorted_data = mongo()
+    if sorted_data:
+        pg(sorted_data)
+
+    print("Done!")