63 lines
2.4 KiB
Python
63 lines
2.4 KiB
Python
import logging
|
|
import asyncio
|
|
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
|
from sqlalchemy import select, desc
|
|
from app.database import AsyncSessionLocal
|
|
from app.models import Channel, RawPost, ContentDraft
|
|
from app.agents.agent_a_scraper import run_channel_ingestion
|
|
from app.agents.agent_b_curator import generate_draft_for_post
|
|
from app.agents.agent_c_publisher import dispatch_pending_queue
|
|
from app.config import settings
|
|
|
|
logger = logging.getLogger("pipeline_scheduler")
|
|
scheduler = AsyncIOScheduler()
|
|
|
|
async def scheduled_scrape_and_curate():
|
|
logger.info("Executing periodic scrape and curate cycle...")
|
|
async with AsyncSessionLocal() as session:
|
|
try:
|
|
# 1. Scrape active channels
|
|
channels = (await session.execute(select(Channel).where(Channel.is_active == True))).scalars().all()
|
|
for ch in channels:
|
|
await run_channel_ingestion(session, ch, limit=10)
|
|
|
|
# 2. Curate pending raw posts
|
|
stmt_unprocessed = (
|
|
select(RawPost)
|
|
.outerjoin(ContentDraft, RawPost.id == ContentDraft.raw_post_id)
|
|
.where(ContentDraft.id == None)
|
|
.order_by(desc(RawPost.views), desc(RawPost.posted_at))
|
|
.limit(10)
|
|
)
|
|
unprocessed_posts = (await session.execute(stmt_unprocessed)).scalars().all()
|
|
for p in unprocessed_posts:
|
|
await generate_draft_for_post(session, p)
|
|
|
|
logger.info(f"Scrape & Curate completed. Curated {len(unprocessed_posts)} posts.")
|
|
except Exception as e:
|
|
logger.error(f"Scheduled scrape/curate error: {e}")
|
|
|
|
async def scheduled_publisher():
|
|
async with AsyncSessionLocal() as session:
|
|
try:
|
|
dispatched = await dispatch_pending_queue(session, batch_size=5)
|
|
if dispatched > 0:
|
|
logger.info(f"Dispatched {dispatched} posts to Telegram channel.")
|
|
except Exception as e:
|
|
logger.error(f"Scheduled publisher error: {e}")
|
|
|
|
def start_scheduler():
|
|
scheduler.add_job(
|
|
scheduled_scrape_and_curate,
|
|
"interval",
|
|
hours=settings.SCRAPE_INTERVAL_HOURS,
|
|
id="scrape_and_curate_job"
|
|
)
|
|
scheduler.add_job(
|
|
scheduled_publisher,
|
|
"interval",
|
|
seconds=settings.PUBLISH_CHECK_INTERVAL_SECONDS,
|
|
id="publisher_job"
|
|
)
|
|
scheduler.start()
|
|
logger.info("Continuous pipeline scheduler started.")
|