update

.gitignore

@@ -0,0 +1,65 @@
+.DS_Store
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+.idea/*
+xml_files/
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+other/split_clash_config/split_config
+ai_news/save_data
+daily/*.txt

backend/app/api/categories.py

@@ -0,0 +1,38 @@
+# backend/app/api/categories.py
+from fastapi import APIRouter, Depends, HTTPException, status
+from sqlalchemy.orm import Session
+from typing import List
+
+from ..database import get_db
+from ..models import Category
+from ..schemas import CategoryCreate, CategoryUpdate, Category, ListResponse
+
+router = APIRouter(prefix="/api/categories", tags=["categories"])
+
+@router.post("/", response_model=Category)
+def create_category(category: CategoryCreate, db: Session = Depends(get_db)):
+    # 检查名称是否已存在
+    db_category = db.query(Category).filter(Category.name == category.name).first()
+    if db_category:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="分类名称已存在"
+        )
+    
+    db_category = Category(**category.dict())
+    db.add(db_category)
+    db.commit()
+    db.refresh(db_category)
+    return db_category
+
+@router.get("/", response_model=ListResponse)
+def get_categories(db: Session = Depends(get_db)):
+    categories = db.query(Category).all()
+    # 为每个分类计算脚本数量
+    for category in categories:
+        from ..models import Spider
+        category.spider_count = db.query(Spider).filter(Spider.category_id == category.id).count()
+    
+    return ListResponse(total=len(categories), items=categories)
+
+# ... 其他分类接口（更新、删除等）

backend/app/core/config.py

@@ -0,0 +1,27 @@
+# backend/app/core/config.py
+from pydantic_settings import BaseSettings
+import os
+
+class Settings(BaseSettings):
+    PROJECT_NAME: str = "爬虫任务调度系统"
+    VERSION: str = "1.0.0"
+    API_V1_STR: str = "/api"
+    
+    # 数据库配置
+    DATABASE_URL: str = "sqlite:///./spider_scheduler.db"
+    
+    # 文件上传配置
+    UPLOAD_DIR: str = "uploaded_scripts"
+    MAX_FILE_SIZE: int = 10 * 1024 * 1024  # 10MB
+    
+    # Celery配置
+    CELERY_BROKER_URL: str = "redis://localhost:6379/0"
+    CELERY_RESULT_BACKEND: str = "redis://localhost:6379/0"
+    
+    class Config:
+        case_sensitive = True
+
+settings = Settings()
+
+# 创建上传目录
+os.makedirs(settings.UPLOAD_DIR, exist_ok=True)

backend/app/database.py

@@ -0,0 +1,20 @@
+# backend/app/database.py
+from sqlalchemy import create_engine
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker
+from .core.config import settings
+
+engine = create_engine(
+    settings.DATABASE_URL, 
+    connect_args={"check_same_thread": False}
+)
+
+SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+Base = declarative_base()
+
+def get_db():
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()

backend/app/main.py

@@ -0,0 +1,49 @@
+# backend/app/main.py
+import sys
+import os
+
+# 添加当前目录到 Python 路径
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+
+# 现在使用绝对导入
+from core.config import settings
+from database import engine, Base
+# 注意：暂时注释掉这些导入，我们先让基础部分运行起来
+# from api import categories, spiders, executions
+
+# 创建数据库表
+Base.metadata.create_all(bind=engine)
+
+app = FastAPI(
+    title=settings.PROJECT_NAME,
+    version=settings.VERSION
+)
+
+# CORS配置
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# 暂时注释掉路由注册
+# app.include_router(categories.router)
+# app.include_router(spiders.router)
+# app.include_router(executions.router)
+
+@app.get("/")
+def read_root():
+    return {"message": "爬虫任务调度系统 API"}
+
+@app.get("/health")
+def health_check():
+    return {"status": "healthy", "service": "spider-scheduler"}
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000, reload=True)

backend/app/models.py

@@ -0,0 +1,48 @@
+# backend/app/models.py
+from sqlalchemy import Column, Integer, String, Text, Boolean, DateTime, ForeignKey
+from sqlalchemy.orm import relationship
+from sqlalchemy.sql import func
+from .database import Base
+
+class Category(Base):
+    __tablename__ = "categories"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    name = Column(String(100), unique=True, nullable=False, index=True)
+    description = Column(Text, nullable=True)
+    color = Column(String(20), nullable=True, default="#3498DB")
+    icon = Column(String(50), nullable=True, default="el-icon-DataBoard")
+    created_at = Column(DateTime, default=func.now())
+    
+    spiders = relationship("Spider", back_populates="category", cascade="all, delete-orphan")
+
+class Spider(Base):
+    __tablename__ = "spiders"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    name = Column(String(100), nullable=False, index=True)
+    description = Column(Text, nullable=True)
+    filename = Column(String(255), nullable=False)
+    file_path = Column(String(500), nullable=False)
+    code_content = Column(Text, nullable=True)
+    cron_expression = Column(String(50), nullable=False, default="0 0 * * *")
+    enabled = Column(Boolean, default=True)
+    timeout = Column(Integer, default=300)
+    created_at = Column(DateTime, default=func.now())
+    category_id = Column(Integer, ForeignKey("categories.id"), nullable=False)
+    
+    category = relationship("Category", back_populates="spiders")
+    executions = relationship("TaskExecutionLog", back_populates="spider")
+
+class TaskExecutionLog(Base):
+    __tablename__ = "task_execution_logs"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    spider_id = Column(Integer, ForeignKey("spiders.id"), nullable=False)
+    status = Column(String(20), nullable=False)
+    started_at = Column(DateTime, default=func.now())
+    finished_at = Column(DateTime, nullable=True)
+    log_content = Column(Text, nullable=True)
+    trigger_method = Column(String(20), nullable=False)
+    
+    spider = relationship("Spider", back_populates="executions")

backend/app/schemas.py

@@ -0,0 +1,80 @@
+# backend/app/schemas.py
+from pydantic import BaseModel
+from typing import Optional, List
+from datetime import datetime
+
+class CategoryBase(BaseModel):
+    name: str
+    description: Optional[str] = None
+    color: Optional[str] = "#3498DB"
+    icon: Optional[str] = "el-icon-DataBoard"
+
+class CategoryCreate(CategoryBase):
+    pass
+
+class CategoryUpdate(CategoryBase):
+    name: Optional[str] = None
+
+class Category(CategoryBase):
+    id: int
+    created_at: datetime
+    spider_count: Optional[int] = 0
+    
+    class Config:
+        from_attributes = True
+
+class SpiderBase(BaseModel):
+    name: str
+    description: Optional[str] = None
+    cron_expression: str = "0 0 * * *"
+    enabled: bool = True
+    timeout: int = 300
+    category_id: int
+
+class SpiderCreate(SpiderBase):
+    code_content: Optional[str] = None
+
+class SpiderUpdate(BaseModel):
+    name: Optional[str] = None
+    description: Optional[str] = None
+    cron_expression: Optional[str] = None
+    enabled: Optional[bool] = None
+    timeout: Optional[int] = None
+    category_id: Optional[int] = None
+    code_content: Optional[str] = None
+
+class Spider(SpiderBase):
+    id: int
+    filename: str
+    file_path: str
+    created_at: datetime
+    category: Optional[Category] = None
+    
+    class Config:
+        from_attributes = True
+
+class TaskExecutionLogBase(BaseModel):
+    status: str
+    trigger_method: str
+
+class TaskExecutionLogCreate(TaskExecutionLogBase):
+    spider_id: int
+
+class TaskExecutionLog(TaskExecutionLogBase):
+    id: int
+    spider_id: int
+    started_at: datetime
+    finished_at: Optional[datetime]
+    log_content: Optional[str]
+    spider: Optional[Spider] = None
+    
+    class Config:
+        from_attributes = True
+
+class ListResponse(BaseModel):
+    total: int
+    items: List
+
+class SuccessResponse(BaseModel):
+    success: bool
+    message: str

backend/app/services/file_service.py

@@ -0,0 +1,46 @@
+# backend/app/services/file_service.py
+import os
+import shutil
+import uuid
+from fastapi import UploadFile, HTTPException, status
+
+class FileService:
+    def __init__(self, upload_dir: str):
+        self.upload_dir = upload_dir
+        os.makedirs(upload_dir, exist_ok=True)
+    
+    def save_uploaded_file(self, file: UploadFile) -> tuple[str, str]:
+        """保存上传的文件"""
+        if not file.filename.endswith('.py'):
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail="只支持.py文件"
+            )
+        
+        filename = f"{uuid.uuid4().hex}_{file.filename}"
+        file_path = os.path.join(self.upload_dir, filename)
+        
+        with open(file_path, "wb") as buffer:
+            shutil.copyfileobj(file.file, buffer)
+        
+        return filename, file_path
+    
+    def save_code_content(self, code_content: str, name: str) -> tuple[str, str]:
+        """保存代码内容为文件"""
+        filename = f"{uuid.uuid4().hex}_{name.replace(' ', '_')}.py"
+        file_path = os.path.join(self.upload_dir, filename)
+        
+        with open(file_path, 'w', encoding='utf-8') as f:
+            f.write(code_content)
+        
+        return filename, file_path
+    
+    def read_file_content(self, file_path: str) -> str:
+        """读取文件内容"""
+        with open(file_path, 'r', encoding='utf-8') as f:
+            return f.read()
+    
+    def delete_file(self, file_path: str):
+        """删除文件"""
+        if os.path.exists(file_path):
+            os.remove(file_path)

backend/requirements.txt

@@ -0,0 +1,10 @@
+fastapi==0.104.1
+uvicorn==0.24.0
+sqlalchemy==2.0.23
+alembic==1.12.1
+pydantic==2.5.0
+pydantic-settings==2.1.0
+python-multipart==0.0.6
+celery==5.3.4
+redis==5.0.1
+apscheduler==3.10.4

mk.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+mkdir -p backend/app/{crud,api,core,services,celery_app}
+mkdir -p backend/{uploaded_scripts,alembic/versions}
+mkdir -p frontend/{public,src/{router,stores,components/{Layout,Common,Dialogs},views,api,utils}}
+mkdir -p nginx/conf.d docker scripts logs
+
+touch backend/app/__init__.py
+touch backend/app/main.py
+touch backend/app/models.py
+touch backend/app/schemas.py
+touch backend/app/database.py
+touch backend/app/crud/__init__.py
+touch backend/app/crud/categories.py
+touch backend/app/crud/spiders.py
+touch backend/app/crud/executions.py
+touch backend/app/api/__init__.py
+touch backend/app/api/categories.py
+touch backend/app/api/spiders.py
+touch backend/app/api/executions.py
+touch backend/app/core/__init__.py
+touch backend/app/core/config.py
+touch backend/app/core/security.py
+touch backend/app/services/__init__.py
+touch backend/app/services/spider_service.py
+touch backend/app/services/scheduler_service.py
+touch backend/app/services/file_service.py
+touch backend/app/celery_app/__init__.py
+touch backend/app/celery_app/celery.py
+touch backend/app/celery_app/tasks.py
+touch backend/app/celery_app/scheduler.py
+touch backend/uploaded_scripts/.gitkeep
+touch backend/requirements.txt
+touch backend/alembic/env.py
+touch backend/alembic/alembic.ini
+touch frontend/public/index.html
+touch frontend/public/favicon.ico
+touch frontend/src/main.js
+touch frontend/src/App.vue
+touch frontend/src/router/index.js
+touch frontend/src/stores/index.js
+touch frontend/src/stores/categories.js
+touch frontend/src/stores/spiders.js
+touch frontend/src/stores/executions.js
+touch frontend/src/components/Layout/Sidebar.vue
+touch frontend/src/components/Layout/Header.vue
+touch frontend/src/components/Common/CronEditor.vue
+touch frontend/src/components/Common/CodeEditor.vue
+touch frontend/src/components/Common/LogViewer.vue
+touch frontend/src/components/Dialogs/CategoryDialog.vue
+touch frontend/src/components/Dialogs/SpiderDialog.vue
+touch frontend/src/views/Dashboard.vue
+touch frontend/src/views/Categories.vue
+touch frontend/src/views/Spiders.vue
+touch frontend/src/views/Executions.vue
+touch frontend/src/api/index.js
+touch frontend/src/api/categories.js
+touch frontend/src/api/spiders.js
+touch frontend/src/api/executions.js
+touch frontend/src/utils/request.js
+touch frontend/src/utils/helpers.js
+touch frontend/package.json
+touch frontend/vite.config.js
+touch frontend/index.html
+touch nginx/nginx.conf
+touch nginx/conf.d/spider-scheduler.conf
+touch docker/Dockerfile.backend
+touch docker/Dockerfile.frontend
+touch docker/docker-compose.yml
+touch scripts/start.sh
+touch scripts/init_db.py
+touch scripts/backup.sh
+touch logs/app.log
+touch logs/celery.log
+touch logs/scheduler.log
+touch .env.example
+touch .gitignore
+touch README.md
+touch requirements.txt
+
+echo "✅ 当前目录骨架创建完成！"