import re import logging from datetime import datetime, timezone from typing import List, Dict, Any, Optional import httpx from bs4 import BeautifulSoup from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy import select, update from app.models import Channel, RawPost logger = logging.getLogger("agent_a_scraper") HEADERS = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.9,fa;q=0.8", } DEFAULT_SEED_CHANNELS = [ {"handle": "NetVibe_VPN", "title": "کانال رسمی NetVibe (الگوی مرجع لحن و هویت)", "niche": "کانال اختصاصی کسب‌وکار", "subscribers": 5000}, {"handle": "FilterShekanha", "title": "فیلترشکن‌ها و اینترنت آزاد", "niche": "آموزش امنیت و ضدسانسور", "subscribers": 61300}, {"handle": "hiddify", "title": "اکوسیستم رسمی Hiddify", "niche": "کلاینت و ابزارهای نوین دور زدن", "subscribers": 121000}, {"handle": "TechTube", "title": "تک‌توب آموزش شبکه و اینترنت", "niche": "آموزش تخصصی شبکه و فرگمنت", "subscribers": 75300}, {"handle": "MiladNouri", "title": "دیده‌بان اینترنت میلاد نوری", "niche": "اخبار اختلالات و سانسور اینترنت", "subscribers": 27200}, {"handle": "NarimanGharib", "title": "امنیت سایبری نریمان غریب", "niche": "تحلیل اختلالات و امنیت دیجیتال", "subscribers": 16700}, {"handle": "vpn_tehran", "title": "مرکز تخصصی شبکه و V2Ray", "niche": "آموزش و بهینه‌سازی کلاینت‌ها", "subscribers": 21700}, {"handle": "ClashMeta", "title": "هسته Mihomo و Clash Meta", "niche": "تنظیمات هسته و رول‌های هوشمند", "subscribers": 17000}, {"handle": "TorProject", "title": "پروژه رسمی Tor و بریج‌ها", "niche": "حریم خصوصی و ارتباطات ناشناس", "subscribers": 34000}, {"handle": "v2fly", "title": "هسته رسمی V2Fly", "niche": "توسعه پروتکل‌های V2Ray", "subscribers": 6300}, {"handle": "Outline_Vpn", "title": "Outline Vpn Official", "niche": "پروتکل شادوساکس و زیرساخت", "subscribers": 680000}, {"handle": "AzadNet", "title": "آزادنت آموزش و اخبار", "niche": "راهنمای اتصال و ابزارهای امن", "subscribers": 249000}, {"handle": "V2Ray_Alpha", "title": "آلفا V2Ray تحلیل پروتکل", "niche": "بررسی فنی پروتکل‌های ارتباطی", "subscribers": 244000}, {"handle": "V2RayNG_VPN", "title": "راهنمای کلاینت v2rayNG", "niche": "آموزش و آپدیت‌های اندروید", "subscribers": 7750}, {"handle": "v2ray_collector", "title": "پایشگر شبکه V2RayCollector", "niche": "تست و مانیتورینگ اختلالات", "subscribers": 4750}, {"handle": "v2ray_free_conf", "title": "اخبار زیرساخت سرورها", "niche": "وضعیت سرورها و پینگ دیتاسنترها", "subscribers": 15200}, {"handle": "V2rayProxy", "title": "پایش پروکسی‌ها و پورت‌ها", "niche": "تحلیل پورت‌ها و پروتکل‌ها", "subscribers": 7180}, {"handle": "shadowrocket_vpn", "title": "راهنمای Shadowrocket و iOS", "niche": "تنظیمات کلاینت‌های آیفون", "subscribers": 1710}, {"handle": "Digiato", "title": "دیجیاتو اخبار فناوری و اینترنت", "niche": "اخبار رسمی اینترنت و زیرساخت", "subscribers": 115000}, {"handle": "ProxyMTProto", "title": "اطلاعیه‌ها و پروکسی‌های امن", "niche": "تحلیل پروکسی و ترافیک تلگرام", "subscribers": 11900000}, {"handle": "v2rayngvpn", "title": "جامعه کاربری v2rayNG", "niche": "تجربیات اتصال و ترفندهای کاربران", "subscribers": 1030000}, ] async def seed_default_channels(session: AsyncSession): for seed in DEFAULT_SEED_CHANNELS: clean_handle = seed["handle"].replace("@", "").strip() stmt = select(Channel).where(Channel.telegram_handle == clean_handle) res = await session.execute(stmt) existing = res.scalar_one_or_none() if not existing: ch = Channel( telegram_handle=clean_handle, title=seed["title"], niche=seed["niche"], subscriber_count=seed.get("subscribers", 10000), is_active=True ) session.add(ch) else: existing.title = seed["title"] existing.niche = seed["niche"] if seed.get("subscribers"): existing.subscriber_count = seed["subscribers"] await session.commit() def parse_views_count(view_text: str) -> int: if not view_text: return 0 clean = view_text.strip().upper() try: if "M" in clean: return int(float(clean.replace("M", "")) * 1_000_000) if "K" in clean: return int(float(clean.replace("K", "")) * 1_000) return int(clean) except ValueError: return 0 async def scrape_public_telegram_channel(handle: str, limit: int = 20) -> List[Dict[str, Any]]: clean_handle = handle.replace("@", "").strip() url = f"https://t.me/s/{clean_handle}" async with httpx.AsyncClient(headers=HEADERS, timeout=20.0, follow_redirects=True) as client: try: res = await client.get(url) if res.status_code != 200: logger.warning(f"Channel {clean_handle} returned HTTP {res.status_code}") return [] except Exception as e: logger.error(f"Error fetching channel {clean_handle}: {e}") return [] soup = BeautifulSoup(res.text, "html.parser") msg_divs = soup.find_all("div", class_="tgme_widget_message") results: List[Dict[str, Any]] = [] for msg in msg_divs[-limit:]: post_attr = msg.get("data-post") if not post_attr or "/" not in post_attr: continue parts = post_attr.split("/") try: msg_id = int(parts[1]) except ValueError: continue post_url = f"https://t.me/{post_attr}" # Text text_elem = msg.find("div", class_="tgme_widget_message_text") original_text = text_elem.get_text(separator="\n").strip() if text_elem else "" # Media Extraction media_type = "none" media_url = None photo_elem = msg.find("a", class_="tgme_widget_message_photo_wrap") video_elem = msg.find("video") doc_elem = msg.find("div", class_="tgme_widget_message_document") if photo_elem: media_type = "photo" style = photo_elem.get("style", "") bg_match = re.search(r"url\('([^']+)'\)", style) if bg_match: media_url = bg_match.group(1) elif video_elem: media_type = "video" media_url = video_elem.get("src") elif doc_elem: media_type = "document" # Views views_elem = msg.find("span", class_="tgme_widget_message_views") views = parse_views_count(views_elem.text if views_elem else "0") # Date & Recency check time_elem = msg.find("time") posted_at = datetime.now(timezone.utc) if time_elem and time_elem.get("datetime"): try: dt_str = time_elem["datetime"].replace("Z", "+00:00") posted_at = datetime.fromisoformat(dt_str) except Exception: pass # STRICT RECENCY FILTER: Discard posts older than 7 days now_utc = datetime.now(timezone.utc) days_old = (now_utc - posted_at).total_seconds() / 86400.0 if days_old > 7.0: logger.debug(f"Skipping stale post {post_attr} ({days_old:.1f} days old)") continue # OUTDATED SEASONAL KEYWORDS FILTER (No outdated Nowruz, past holiday greetings) stale_keywords = [ "نوروز", "سال نو", "عید نوروز", "تعطیلات عید", "فروردین", "تبریک سال نو", "1402", "۱۴۰۲", "سال ۱۴۰۲", "نوروز مبارک", "سال پیش" ] text_lower = (original_text or "").lower() if any(sk in text_lower for sk in stale_keywords): logger.info(f"Skipping outdated seasonal post {post_attr} containing stale holiday keywords") continue if original_text or media_url: results.append({ "message_id": msg_id, "original_text": original_text, "post_url": post_url, "media_type": media_type, "media_url": media_url, "views": views, "reactions": 0, "posted_at": posted_at }) return results async def run_channel_ingestion(session: AsyncSession, channel: Channel, limit: int = 15) -> int: posts = await scrape_public_telegram_channel(channel.telegram_handle, limit=limit) new_count = 0 for post_data in posts: # Check uniqueness by channel_id and message_id stmt = select(RawPost).where( RawPost.channel_id == channel.id, RawPost.message_id == post_data["message_id"] ) existing = (await session.execute(stmt)).scalar_one_or_none() if not existing: raw_post = RawPost( channel_id=channel.id, message_id=post_data["message_id"], original_text=post_data["original_text"], post_url=post_data["post_url"], media_type=post_data["media_type"], media_url=post_data["media_url"], views=post_data["views"], reactions=post_data["reactions"], posted_at=post_data["posted_at"] ) session.add(raw_post) new_count += 1 await session.execute( update(Channel) .where(Channel.id == channel.id) .values(last_scraped_at=datetime.now(timezone.utc)) ) await session.commit() return new_count