ehsan-dashboards/dashboards/VPN_chanel/backend/refresh_and_curate.py

144 lines
6 KiB
Python

import asyncio
import os
import re
import sqlite3
from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker
from sqlalchemy import select, desc
from app.models import Base, Channel, RawPost, ContentDraft, PublishQueue, FeedbackLog
from app.agents.agent_a_scraper import seed_default_channels, run_channel_ingestion
from app.agents.agent_b_curator import generate_draft_for_post
DB_PATH = "/workspace/personal/VPN_chanel/backend/vpn_pipeline.db"
DATABASE_URL = f"sqlite+aiosqlite:///{DB_PATH}"
def calculate_quality_score(text: str) -> int:
"""
Score a post based on its educational, analytical, and technical value.
Penalizes raw config dumps and spam.
"""
if not text or len(text.strip()) < 30:
return 0
score = 10
t_low = text.lower()
# Educational & Tutorial Keywords (+30)
edu_words = ["آموزش", "تنظیمات", "راهنما", "کلاینت", "نحوه", "چگونه", "حل مشکل", "ارور", "رفع خطا", "ستاپ"]
for w in edu_words:
if w in t_low:
score += 25
break
# Software & Apps (+20)
app_words = ["hiddify", "v2rayng", "sing-box", "singbox", "mihomo", "clash", "outline", "nekoray", "shadowrocket", "تور", "tor"]
for w in app_words:
if w in t_low:
score += 20
break
# Censorship, Disruptions & Datacenter News (+25)
news_words = ["اختلال", "فیلترینگ", "قطعی", "کلودفلر", "پورت", "دیتاسنتر", "همراه اول", "ایرانسل", "مخابرات", "رادار", "زیرساخت", "سراسری"]
for w in news_words:
if w in t_low:
score += 25
break
# Security & Privacy (+25)
sec_words = ["امنیت", "حریم خصوصی", "بدافزار", "جاسوسی", "شنود", "لو رفتن", "رمزنگاری", "نشت", "dns", "فیشینگ"]
for w in sec_words:
if w in t_low:
score += 25
break
# Technical Optimization (+20)
tech_words = ["پینگ", "کاهش پینگ", "سرعت", "فرگمنت", "fragment", "reality", "tls", "ech", "doh", "پروتکل"]
for w in tech_words:
if w in t_low:
score += 20
break
# Bonus for well-structured long-form content
if 200 <= len(text) <= 2500:
score += 25
# Heavy Penalty for pure config dumps without educational text (-40)
raw_configs = ["vless://", "vmess://", "trojan://", "ss://"]
config_matches = sum(1 for c in raw_configs if c in t_low)
if config_matches > 0:
# If it's mostly a dump of configs
lines = [l for l in text.splitlines() if any(c in l for c in raw_configs)]
if len(lines) >= 3 or (len(text) < 300 and config_matches >= 1):
score -= 50
return score
async def main():
print("1. Cleaning old database entries...")
if os.path.exists(DB_PATH):
os.remove(DB_PATH)
print("Removed old vpn_pipeline.db")
engine = create_async_engine(DATABASE_URL)
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
print("Created clean SQLite schema")
session_maker = async_sessionmaker(engine, expire_on_commit=False)
async with session_maker() as session:
print("\n2. Seeding TOP 20 high-value channels...")
await seed_default_channels(session)
channels = (await session.execute(select(Channel))).scalars().all()
print(f"Seeded {len(channels)} channels:")
for idx, ch in enumerate(channels, 1):
print(f" {idx:2d}. @{ch.telegram_handle:18} | {ch.title[:35]}")
print("\n3. Ingesting freshest posts from all 20 channels...")
for ch in channels:
try:
new_count = await run_channel_ingestion(session, ch, limit=20)
print(f"Scraped @{ch.telegram_handle}: {new_count} posts")
except Exception as e:
print(f"Error scraping @{ch.telegram_handle}: {e}")
# Fetch all ingested posts
stmt = select(RawPost)
all_posts = (await session.execute(stmt)).scalars().all()
print(f"\nTotal raw posts ingested across 20 channels: {len(all_posts)}")
# Score and rank posts by educational / tech-news quality
scored_posts = []
for p in all_posts:
s = calculate_quality_score(p.original_text or "")
if s >= 40: # Only high quality analytical/educational/news posts
scored_posts.append((s, p))
scored_posts.sort(key=lambda x: (x[0], x[1].posted_at or x[1].created_at), reverse=True)
print(f"Total high-quality educational/news candidates (score >= 40): {len(scored_posts)}")
# Print top 10 candidates
print("\nTop 10 High-Yield Educational/News Posts:")
for rank, (score, post) in enumerate(scored_posts[:10], 1):
first_l = post.original_text.splitlines()[0] if post.original_text else ""
print(f" #{rank:02d} [Score: {score:3d}] Channel: @{post.channel.telegram_handle if post.channel else 'N/A'}")
print(f" Title: {first_l[:70]}")
print("\n4. Curating top 5 educational/news posts with AI Model 01 and generating customized banners...")
top_curate = [p for _, p in scored_posts[:5]]
for idx, post in enumerate(top_curate, 1):
ch_name = post.channel.telegram_handle if post.channel else 'unknown'
print(f"\n--- Curating Educational Post {idx}/{len(top_curate)} from @{ch_name} ---")
try:
draft = await generate_draft_for_post(session, post)
print(f"Draft ID: {draft.id}")
print(f"Banner URL: {draft.banner_image_url}")
print(f"AI Generated Text Preview:\n{draft.current_text[:200]}...\n")
except Exception as e:
print(f"Error curating post {post.id}: {e}")
await engine.dispose()
print("\nRefresh and educational curation completed successfully!")
if __name__ == "__main__":
asyncio.run(main())