ehsan-dashboards/dashboards/VPN_chanel/backend/app/agents/agent_a_scraper.py

210 lines
11 KiB
Python

import re
import logging
from datetime import datetime, timezone
from typing import List, Dict, Any, Optional
import httpx
from bs4 import BeautifulSoup
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, update
from app.models import Channel, RawPost
logger = logging.getLogger("agent_a_scraper")
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.9,fa;q=0.8",
}
DEFAULT_SEED_CHANNELS = [
{"handle": "NetVibe_VPN", "title": "کانال رسمی NetVibe (الگوی مرجع لحن و هویت)", "niche": "کانال اختصاصی کسب‌وکار", "subscribers": 5000},
{"handle": "FilterShekanha", "title": "فیلترشکن‌ها و اینترنت آزاد", "niche": "آموزش امنیت و ضدسانسور", "subscribers": 61300},
{"handle": "hiddify", "title": "اکوسیستم رسمی Hiddify", "niche": "کلاینت و ابزارهای نوین دور زدن", "subscribers": 121000},
{"handle": "TechTube", "title": "تک‌توب آموزش شبکه و اینترنت", "niche": "آموزش تخصصی شبکه و فرگمنت", "subscribers": 75300},
{"handle": "MiladNouri", "title": "دیده‌بان اینترنت میلاد نوری", "niche": "اخبار اختلالات و سانسور اینترنت", "subscribers": 27200},
{"handle": "NarimanGharib", "title": "امنیت سایبری نریمان غریب", "niche": "تحلیل اختلالات و امنیت دیجیتال", "subscribers": 16700},
{"handle": "vpn_tehran", "title": "مرکز تخصصی شبکه و V2Ray", "niche": "آموزش و بهینه‌سازی کلاینت‌ها", "subscribers": 21700},
{"handle": "ClashMeta", "title": "هسته Mihomo و Clash Meta", "niche": "تنظیمات هسته و رول‌های هوشمند", "subscribers": 17000},
{"handle": "TorProject", "title": "پروژه رسمی Tor و بریج‌ها", "niche": "حریم خصوصی و ارتباطات ناشناس", "subscribers": 34000},
{"handle": "v2fly", "title": "هسته رسمی V2Fly", "niche": "توسعه پروتکل‌های V2Ray", "subscribers": 6300},
{"handle": "Outline_Vpn", "title": "Outline Vpn Official", "niche": "پروتکل شادوساکس و زیرساخت", "subscribers": 680000},
{"handle": "AzadNet", "title": "آزادنت آموزش و اخبار", "niche": "راهنمای اتصال و ابزارهای امن", "subscribers": 249000},
{"handle": "V2Ray_Alpha", "title": "آلفا V2Ray تحلیل پروتکل", "niche": "بررسی فنی پروتکل‌های ارتباطی", "subscribers": 244000},
{"handle": "V2RayNG_VPN", "title": "راهنمای کلاینت v2rayNG", "niche": "آموزش و آپدیت‌های اندروید", "subscribers": 7750},
{"handle": "v2ray_collector", "title": "پایشگر شبکه V2RayCollector", "niche": "تست و مانیتورینگ اختلالات", "subscribers": 4750},
{"handle": "v2ray_free_conf", "title": "اخبار زیرساخت سرورها", "niche": "وضعیت سرورها و پینگ دیتاسنترها", "subscribers": 15200},
{"handle": "V2rayProxy", "title": "پایش پروکسی‌ها و پورت‌ها", "niche": "تحلیل پورت‌ها و پروتکل‌ها", "subscribers": 7180},
{"handle": "shadowrocket_vpn", "title": "راهنمای Shadowrocket و iOS", "niche": "تنظیمات کلاینت‌های آیفون", "subscribers": 1710},
{"handle": "Digiato", "title": "دیجیاتو اخبار فناوری و اینترنت", "niche": "اخبار رسمی اینترنت و زیرساخت", "subscribers": 115000},
{"handle": "ProxyMTProto", "title": "اطلاعیه‌ها و پروکسی‌های امن", "niche": "تحلیل پروکسی و ترافیک تلگرام", "subscribers": 11900000},
{"handle": "v2rayngvpn", "title": "جامعه کاربری v2rayNG", "niche": "تجربیات اتصال و ترفندهای کاربران", "subscribers": 1030000},
]
async def seed_default_channels(session: AsyncSession):
for seed in DEFAULT_SEED_CHANNELS:
clean_handle = seed["handle"].replace("@", "").strip()
stmt = select(Channel).where(Channel.telegram_handle == clean_handle)
res = await session.execute(stmt)
existing = res.scalar_one_or_none()
if not existing:
ch = Channel(
telegram_handle=clean_handle,
title=seed["title"],
niche=seed["niche"],
subscriber_count=seed.get("subscribers", 10000),
is_active=True
)
session.add(ch)
else:
existing.title = seed["title"]
existing.niche = seed["niche"]
if seed.get("subscribers"):
existing.subscriber_count = seed["subscribers"]
await session.commit()
def parse_views_count(view_text: str) -> int:
if not view_text:
return 0
clean = view_text.strip().upper()
try:
if "M" in clean:
return int(float(clean.replace("M", "")) * 1_000_000)
if "K" in clean:
return int(float(clean.replace("K", "")) * 1_000)
return int(clean)
except ValueError:
return 0
async def scrape_public_telegram_channel(handle: str, limit: int = 20) -> List[Dict[str, Any]]:
clean_handle = handle.replace("@", "").strip()
url = f"https://t.me/s/{clean_handle}"
async with httpx.AsyncClient(headers=HEADERS, timeout=20.0, follow_redirects=True) as client:
try:
res = await client.get(url)
if res.status_code != 200:
logger.warning(f"Channel {clean_handle} returned HTTP {res.status_code}")
return []
except Exception as e:
logger.error(f"Error fetching channel {clean_handle}: {e}")
return []
soup = BeautifulSoup(res.text, "html.parser")
msg_divs = soup.find_all("div", class_="tgme_widget_message")
results: List[Dict[str, Any]] = []
for msg in msg_divs[-limit:]:
post_attr = msg.get("data-post")
if not post_attr or "/" not in post_attr:
continue
parts = post_attr.split("/")
try:
msg_id = int(parts[1])
except ValueError:
continue
post_url = f"https://t.me/{post_attr}"
# Text
text_elem = msg.find("div", class_="tgme_widget_message_text")
original_text = text_elem.get_text(separator="\n").strip() if text_elem else ""
# Media Extraction
media_type = "none"
media_url = None
photo_elem = msg.find("a", class_="tgme_widget_message_photo_wrap")
video_elem = msg.find("video")
doc_elem = msg.find("div", class_="tgme_widget_message_document")
if photo_elem:
media_type = "photo"
style = photo_elem.get("style", "")
bg_match = re.search(r"url\('([^']+)'\)", style)
if bg_match:
media_url = bg_match.group(1)
elif video_elem:
media_type = "video"
media_url = video_elem.get("src")
elif doc_elem:
media_type = "document"
# Views
views_elem = msg.find("span", class_="tgme_widget_message_views")
views = parse_views_count(views_elem.text if views_elem else "0")
# Date & Recency check
time_elem = msg.find("time")
posted_at = datetime.now(timezone.utc)
if time_elem and time_elem.get("datetime"):
try:
dt_str = time_elem["datetime"].replace("Z", "+00:00")
posted_at = datetime.fromisoformat(dt_str)
except Exception:
pass
# STRICT RECENCY FILTER: Discard posts older than 7 days
now_utc = datetime.now(timezone.utc)
days_old = (now_utc - posted_at).total_seconds() / 86400.0
if days_old > 7.0:
logger.debug(f"Skipping stale post {post_attr} ({days_old:.1f} days old)")
continue
# OUTDATED SEASONAL KEYWORDS FILTER (No outdated Nowruz, past holiday greetings)
stale_keywords = [
"نوروز", "سال نو", "عید نوروز", "تعطیلات عید", "فروردین", "تبریک سال نو",
"1402", "۱۴۰۲", "سال ۱۴۰۲", "نوروز مبارک", "سال پیش"
]
text_lower = (original_text or "").lower()
if any(sk in text_lower for sk in stale_keywords):
logger.info(f"Skipping outdated seasonal post {post_attr} containing stale holiday keywords")
continue
if original_text or media_url:
results.append({
"message_id": msg_id,
"original_text": original_text,
"post_url": post_url,
"media_type": media_type,
"media_url": media_url,
"views": views,
"reactions": 0,
"posted_at": posted_at
})
return results
async def run_channel_ingestion(session: AsyncSession, channel: Channel, limit: int = 15) -> int:
posts = await scrape_public_telegram_channel(channel.telegram_handle, limit=limit)
new_count = 0
for post_data in posts:
# Check uniqueness by channel_id and message_id
stmt = select(RawPost).where(
RawPost.channel_id == channel.id,
RawPost.message_id == post_data["message_id"]
)
existing = (await session.execute(stmt)).scalar_one_or_none()
if not existing:
raw_post = RawPost(
channel_id=channel.id,
message_id=post_data["message_id"],
original_text=post_data["original_text"],
post_url=post_data["post_url"],
media_type=post_data["media_type"],
media_url=post_data["media_url"],
views=post_data["views"],
reactions=post_data["reactions"],
posted_at=post_data["posted_at"]
)
session.add(raw_post)
new_count += 1
await session.execute(
update(Channel)
.where(Channel.id == channel.id)
.values(last_scraped_at=datetime.now(timezone.utc))
)
await session.commit()
return new_count