import asyncio import os import re import sqlite3 from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker from sqlalchemy import select, desc from app.models import Base, Channel, RawPost, ContentDraft, PublishQueue, FeedbackLog from app.agents.agent_a_scraper import seed_default_channels, run_channel_ingestion from app.agents.agent_b_curator import generate_draft_for_post DB_PATH = "/workspace/personal/VPN_chanel/backend/vpn_pipeline.db" DATABASE_URL = f"sqlite+aiosqlite:///{DB_PATH}" def calculate_quality_score(text: str) -> int: """ Score a post based on its educational, analytical, and technical value. Penalizes raw config dumps and spam. """ if not text or len(text.strip()) < 30: return 0 score = 10 t_low = text.lower() # Educational & Tutorial Keywords (+30) edu_words = ["آموزش", "تنظیمات", "راهنما", "کلاینت", "نحوه", "چگونه", "حل مشکل", "ارور", "رفع خطا", "ستاپ"] for w in edu_words: if w in t_low: score += 25 break # Software & Apps (+20) app_words = ["hiddify", "v2rayng", "sing-box", "singbox", "mihomo", "clash", "outline", "nekoray", "shadowrocket", "تور", "tor"] for w in app_words: if w in t_low: score += 20 break # Censorship, Disruptions & Datacenter News (+25) news_words = ["اختلال", "فیلترینگ", "قطعی", "کلودفلر", "پورت", "دیتاسنتر", "همراه اول", "ایرانسل", "مخابرات", "رادار", "زیرساخت", "سراسری"] for w in news_words: if w in t_low: score += 25 break # Security & Privacy (+25) sec_words = ["امنیت", "حریم خصوصی", "بدافزار", "جاسوسی", "شنود", "لو رفتن", "رمزنگاری", "نشت", "dns", "فیشینگ"] for w in sec_words: if w in t_low: score += 25 break # Technical Optimization (+20) tech_words = ["پینگ", "کاهش پینگ", "سرعت", "فرگمنت", "fragment", "reality", "tls", "ech", "doh", "پروتکل"] for w in tech_words: if w in t_low: score += 20 break # Bonus for well-structured long-form content if 200 <= len(text) <= 2500: score += 25 # Heavy Penalty for pure config dumps without educational text (-40) raw_configs = ["vless://", "vmess://", "trojan://", "ss://"] config_matches = sum(1 for c in raw_configs if c in t_low) if config_matches > 0: # If it's mostly a dump of configs lines = [l for l in text.splitlines() if any(c in l for c in raw_configs)] if len(lines) >= 3 or (len(text) < 300 and config_matches >= 1): score -= 50 return score async def main(): print("1. Cleaning old database entries...") if os.path.exists(DB_PATH): os.remove(DB_PATH) print("Removed old vpn_pipeline.db") engine = create_async_engine(DATABASE_URL) async with engine.begin() as conn: await conn.run_sync(Base.metadata.create_all) print("Created clean SQLite schema") session_maker = async_sessionmaker(engine, expire_on_commit=False) async with session_maker() as session: print("\n2. Seeding TOP 20 high-value channels...") await seed_default_channels(session) channels = (await session.execute(select(Channel))).scalars().all() print(f"Seeded {len(channels)} channels:") for idx, ch in enumerate(channels, 1): print(f" {idx:2d}. @{ch.telegram_handle:18} | {ch.title[:35]}") print("\n3. Ingesting freshest posts from all 20 channels...") for ch in channels: try: new_count = await run_channel_ingestion(session, ch, limit=20) print(f"Scraped @{ch.telegram_handle}: {new_count} posts") except Exception as e: print(f"Error scraping @{ch.telegram_handle}: {e}") # Fetch all ingested posts stmt = select(RawPost) all_posts = (await session.execute(stmt)).scalars().all() print(f"\nTotal raw posts ingested across 20 channels: {len(all_posts)}") # Score and rank posts by educational / tech-news quality scored_posts = [] for p in all_posts: s = calculate_quality_score(p.original_text or "") if s >= 40: # Only high quality analytical/educational/news posts scored_posts.append((s, p)) scored_posts.sort(key=lambda x: (x[0], x[1].posted_at or x[1].created_at), reverse=True) print(f"Total high-quality educational/news candidates (score >= 40): {len(scored_posts)}") # Print top 10 candidates print("\nTop 10 High-Yield Educational/News Posts:") for rank, (score, post) in enumerate(scored_posts[:10], 1): first_l = post.original_text.splitlines()[0] if post.original_text else "" print(f" #{rank:02d} [Score: {score:3d}] Channel: @{post.channel.telegram_handle if post.channel else 'N/A'}") print(f" Title: {first_l[:70]}") print("\n4. Curating top 5 educational/news posts with AI Model 01 and generating customized banners...") top_curate = [p for _, p in scored_posts[:5]] for idx, post in enumerate(top_curate, 1): ch_name = post.channel.telegram_handle if post.channel else 'unknown' print(f"\n--- Curating Educational Post {idx}/{len(top_curate)} from @{ch_name} ---") try: draft = await generate_draft_for_post(session, post) print(f"Draft ID: {draft.id}") print(f"Banner URL: {draft.banner_image_url}") print(f"AI Generated Text Preview:\n{draft.current_text[:200]}...\n") except Exception as e: print(f"Error curating post {post.id}: {e}") await engine.dispose() print("\nRefresh and educational curation completed successfully!") if __name__ == "__main__": asyncio.run(main())