import logging import asyncio from apscheduler.schedulers.asyncio import AsyncIOScheduler from sqlalchemy import select, desc from app.database import AsyncSessionLocal from app.models import Channel, RawPost, ContentDraft from app.agents.agent_a_scraper import run_channel_ingestion from app.agents.agent_b_curator import generate_draft_for_post from app.agents.agent_c_publisher import dispatch_pending_queue from app.config import settings logger = logging.getLogger("pipeline_scheduler") scheduler = AsyncIOScheduler() async def scheduled_scrape_and_curate(): logger.info("Executing periodic scrape and curate cycle...") async with AsyncSessionLocal() as session: try: # 1. Scrape active channels channels = (await session.execute(select(Channel).where(Channel.is_active == True))).scalars().all() for ch in channels: await run_channel_ingestion(session, ch, limit=10) # 2. Curate pending raw posts stmt_unprocessed = ( select(RawPost) .outerjoin(ContentDraft, RawPost.id == ContentDraft.raw_post_id) .where(ContentDraft.id == None) .order_by(desc(RawPost.views), desc(RawPost.posted_at)) .limit(10) ) unprocessed_posts = (await session.execute(stmt_unprocessed)).scalars().all() for p in unprocessed_posts: await generate_draft_for_post(session, p) logger.info(f"Scrape & Curate completed. Curated {len(unprocessed_posts)} posts.") except Exception as e: logger.error(f"Scheduled scrape/curate error: {e}") async def scheduled_publisher(): async with AsyncSessionLocal() as session: try: dispatched = await dispatch_pending_queue(session, batch_size=5) if dispatched > 0: logger.info(f"Dispatched {dispatched} posts to Telegram channel.") except Exception as e: logger.error(f"Scheduled publisher error: {e}") def start_scheduler(): scheduler.add_job( scheduled_scrape_and_curate, "interval", hours=settings.SCRAPE_INTERVAL_HOURS, id="scrape_and_curate_job" ) scheduler.add_job( scheduled_publisher, "interval", seconds=settings.PUBLISH_CHECK_INTERVAL_SECONDS, id="publisher_job" ) scheduler.start() logger.info("Continuous pipeline scheduler started.")