144 lines
6 KiB
Python
144 lines
6 KiB
Python
import asyncio
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker
|
|
from sqlalchemy import select, desc
|
|
from app.models import Base, Channel, RawPost, ContentDraft, PublishQueue, FeedbackLog
|
|
from app.agents.agent_a_scraper import seed_default_channels, run_channel_ingestion
|
|
from app.agents.agent_b_curator import generate_draft_for_post
|
|
|
|
DB_PATH = "/workspace/personal/VPN_chanel/backend/vpn_pipeline.db"
|
|
DATABASE_URL = f"sqlite+aiosqlite:///{DB_PATH}"
|
|
|
|
def calculate_quality_score(text: str) -> int:
|
|
"""
|
|
Score a post based on its educational, analytical, and technical value.
|
|
Penalizes raw config dumps and spam.
|
|
"""
|
|
if not text or len(text.strip()) < 30:
|
|
return 0
|
|
|
|
score = 10
|
|
t_low = text.lower()
|
|
|
|
# Educational & Tutorial Keywords (+30)
|
|
edu_words = ["آموزش", "تنظیمات", "راهنما", "کلاینت", "نحوه", "چگونه", "حل مشکل", "ارور", "رفع خطا", "ستاپ"]
|
|
for w in edu_words:
|
|
if w in t_low:
|
|
score += 25
|
|
break
|
|
|
|
# Software & Apps (+20)
|
|
app_words = ["hiddify", "v2rayng", "sing-box", "singbox", "mihomo", "clash", "outline", "nekoray", "shadowrocket", "تور", "tor"]
|
|
for w in app_words:
|
|
if w in t_low:
|
|
score += 20
|
|
break
|
|
|
|
# Censorship, Disruptions & Datacenter News (+25)
|
|
news_words = ["اختلال", "فیلترینگ", "قطعی", "کلودفلر", "پورت", "دیتاسنتر", "همراه اول", "ایرانسل", "مخابرات", "رادار", "زیرساخت", "سراسری"]
|
|
for w in news_words:
|
|
if w in t_low:
|
|
score += 25
|
|
break
|
|
|
|
# Security & Privacy (+25)
|
|
sec_words = ["امنیت", "حریم خصوصی", "بدافزار", "جاسوسی", "شنود", "لو رفتن", "رمزنگاری", "نشت", "dns", "فیشینگ"]
|
|
for w in sec_words:
|
|
if w in t_low:
|
|
score += 25
|
|
break
|
|
|
|
# Technical Optimization (+20)
|
|
tech_words = ["پینگ", "کاهش پینگ", "سرعت", "فرگمنت", "fragment", "reality", "tls", "ech", "doh", "پروتکل"]
|
|
for w in tech_words:
|
|
if w in t_low:
|
|
score += 20
|
|
break
|
|
|
|
# Bonus for well-structured long-form content
|
|
if 200 <= len(text) <= 2500:
|
|
score += 25
|
|
|
|
# Heavy Penalty for pure config dumps without educational text (-40)
|
|
raw_configs = ["vless://", "vmess://", "trojan://", "ss://"]
|
|
config_matches = sum(1 for c in raw_configs if c in t_low)
|
|
if config_matches > 0:
|
|
# If it's mostly a dump of configs
|
|
lines = [l for l in text.splitlines() if any(c in l for c in raw_configs)]
|
|
if len(lines) >= 3 or (len(text) < 300 and config_matches >= 1):
|
|
score -= 50
|
|
|
|
return score
|
|
|
|
async def main():
|
|
print("1. Cleaning old database entries...")
|
|
if os.path.exists(DB_PATH):
|
|
os.remove(DB_PATH)
|
|
print("Removed old vpn_pipeline.db")
|
|
|
|
engine = create_async_engine(DATABASE_URL)
|
|
async with engine.begin() as conn:
|
|
await conn.run_sync(Base.metadata.create_all)
|
|
print("Created clean SQLite schema")
|
|
|
|
session_maker = async_sessionmaker(engine, expire_on_commit=False)
|
|
async with session_maker() as session:
|
|
print("\n2. Seeding TOP 20 high-value channels...")
|
|
await seed_default_channels(session)
|
|
|
|
channels = (await session.execute(select(Channel))).scalars().all()
|
|
print(f"Seeded {len(channels)} channels:")
|
|
for idx, ch in enumerate(channels, 1):
|
|
print(f" {idx:2d}. @{ch.telegram_handle:18} | {ch.title[:35]}")
|
|
|
|
print("\n3. Ingesting freshest posts from all 20 channels...")
|
|
for ch in channels:
|
|
try:
|
|
new_count = await run_channel_ingestion(session, ch, limit=20)
|
|
print(f"Scraped @{ch.telegram_handle}: {new_count} posts")
|
|
except Exception as e:
|
|
print(f"Error scraping @{ch.telegram_handle}: {e}")
|
|
|
|
# Fetch all ingested posts
|
|
stmt = select(RawPost)
|
|
all_posts = (await session.execute(stmt)).scalars().all()
|
|
print(f"\nTotal raw posts ingested across 20 channels: {len(all_posts)}")
|
|
|
|
# Score and rank posts by educational / tech-news quality
|
|
scored_posts = []
|
|
for p in all_posts:
|
|
s = calculate_quality_score(p.original_text or "")
|
|
if s >= 40: # Only high quality analytical/educational/news posts
|
|
scored_posts.append((s, p))
|
|
|
|
scored_posts.sort(key=lambda x: (x[0], x[1].posted_at or x[1].created_at), reverse=True)
|
|
print(f"Total high-quality educational/news candidates (score >= 40): {len(scored_posts)}")
|
|
|
|
# Print top 10 candidates
|
|
print("\nTop 10 High-Yield Educational/News Posts:")
|
|
for rank, (score, post) in enumerate(scored_posts[:10], 1):
|
|
first_l = post.original_text.splitlines()[0] if post.original_text else ""
|
|
print(f" #{rank:02d} [Score: {score:3d}] Channel: @{post.channel.telegram_handle if post.channel else 'N/A'}")
|
|
print(f" Title: {first_l[:70]}")
|
|
|
|
print("\n4. Curating top 5 educational/news posts with AI Model 01 and generating customized banners...")
|
|
top_curate = [p for _, p in scored_posts[:5]]
|
|
|
|
for idx, post in enumerate(top_curate, 1):
|
|
ch_name = post.channel.telegram_handle if post.channel else 'unknown'
|
|
print(f"\n--- Curating Educational Post {idx}/{len(top_curate)} from @{ch_name} ---")
|
|
try:
|
|
draft = await generate_draft_for_post(session, post)
|
|
print(f"Draft ID: {draft.id}")
|
|
print(f"Banner URL: {draft.banner_image_url}")
|
|
print(f"AI Generated Text Preview:\n{draft.current_text[:200]}...\n")
|
|
except Exception as e:
|
|
print(f"Error curating post {post.id}: {e}")
|
|
|
|
await engine.dispose()
|
|
print("\nRefresh and educational curation completed successfully!")
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|