ehsan-dashboards/dashboards/VPN_chanel/backend/app/scheduler.py

63 lines
2.4 KiB
Python

import logging
import asyncio
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from sqlalchemy import select, desc
from app.database import AsyncSessionLocal
from app.models import Channel, RawPost, ContentDraft
from app.agents.agent_a_scraper import run_channel_ingestion
from app.agents.agent_b_curator import generate_draft_for_post
from app.agents.agent_c_publisher import dispatch_pending_queue
from app.config import settings
logger = logging.getLogger("pipeline_scheduler")
scheduler = AsyncIOScheduler()
async def scheduled_scrape_and_curate():
logger.info("Executing periodic scrape and curate cycle...")
async with AsyncSessionLocal() as session:
try:
# 1. Scrape active channels
channels = (await session.execute(select(Channel).where(Channel.is_active == True))).scalars().all()
for ch in channels:
await run_channel_ingestion(session, ch, limit=10)
# 2. Curate pending raw posts
stmt_unprocessed = (
select(RawPost)
.outerjoin(ContentDraft, RawPost.id == ContentDraft.raw_post_id)
.where(ContentDraft.id == None)
.order_by(desc(RawPost.views), desc(RawPost.posted_at))
.limit(10)
)
unprocessed_posts = (await session.execute(stmt_unprocessed)).scalars().all()
for p in unprocessed_posts:
await generate_draft_for_post(session, p)
logger.info(f"Scrape & Curate completed. Curated {len(unprocessed_posts)} posts.")
except Exception as e:
logger.error(f"Scheduled scrape/curate error: {e}")
async def scheduled_publisher():
async with AsyncSessionLocal() as session:
try:
dispatched = await dispatch_pending_queue(session, batch_size=5)
if dispatched > 0:
logger.info(f"Dispatched {dispatched} posts to Telegram channel.")
except Exception as e:
logger.error(f"Scheduled publisher error: {e}")
def start_scheduler():
scheduler.add_job(
scheduled_scrape_and_curate,
"interval",
hours=settings.SCRAPE_INTERVAL_HOURS,
id="scrape_and_curate_job"
)
scheduler.add_job(
scheduled_publisher,
"interval",
seconds=settings.PUBLISH_CHECK_INTERVAL_SECONDS,
id="publisher_job"
)
scheduler.start()
logger.info("Continuous pipeline scheduler started.")