210 lines
11 KiB
Python
210 lines
11 KiB
Python
import re
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
from typing import List, Dict, Any, Optional
|
|
import httpx
|
|
from bs4 import BeautifulSoup
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
from sqlalchemy import select, update
|
|
from app.models import Channel, RawPost
|
|
|
|
logger = logging.getLogger("agent_a_scraper")
|
|
|
|
HEADERS = {
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
"Accept-Language": "en-US,en;q=0.9,fa;q=0.8",
|
|
}
|
|
|
|
DEFAULT_SEED_CHANNELS = [
|
|
{"handle": "NetVibe_VPN", "title": "کانال رسمی NetVibe (الگوی مرجع لحن و هویت)", "niche": "کانال اختصاصی کسبوکار", "subscribers": 5000},
|
|
{"handle": "FilterShekanha", "title": "فیلترشکنها و اینترنت آزاد", "niche": "آموزش امنیت و ضدسانسور", "subscribers": 61300},
|
|
{"handle": "hiddify", "title": "اکوسیستم رسمی Hiddify", "niche": "کلاینت و ابزارهای نوین دور زدن", "subscribers": 121000},
|
|
{"handle": "TechTube", "title": "تکتوب آموزش شبکه و اینترنت", "niche": "آموزش تخصصی شبکه و فرگمنت", "subscribers": 75300},
|
|
{"handle": "MiladNouri", "title": "دیدهبان اینترنت میلاد نوری", "niche": "اخبار اختلالات و سانسور اینترنت", "subscribers": 27200},
|
|
{"handle": "NarimanGharib", "title": "امنیت سایبری نریمان غریب", "niche": "تحلیل اختلالات و امنیت دیجیتال", "subscribers": 16700},
|
|
{"handle": "vpn_tehran", "title": "مرکز تخصصی شبکه و V2Ray", "niche": "آموزش و بهینهسازی کلاینتها", "subscribers": 21700},
|
|
{"handle": "ClashMeta", "title": "هسته Mihomo و Clash Meta", "niche": "تنظیمات هسته و رولهای هوشمند", "subscribers": 17000},
|
|
{"handle": "TorProject", "title": "پروژه رسمی Tor و بریجها", "niche": "حریم خصوصی و ارتباطات ناشناس", "subscribers": 34000},
|
|
{"handle": "v2fly", "title": "هسته رسمی V2Fly", "niche": "توسعه پروتکلهای V2Ray", "subscribers": 6300},
|
|
{"handle": "Outline_Vpn", "title": "Outline Vpn Official", "niche": "پروتکل شادوساکس و زیرساخت", "subscribers": 680000},
|
|
{"handle": "AzadNet", "title": "آزادنت آموزش و اخبار", "niche": "راهنمای اتصال و ابزارهای امن", "subscribers": 249000},
|
|
{"handle": "V2Ray_Alpha", "title": "آلفا V2Ray تحلیل پروتکل", "niche": "بررسی فنی پروتکلهای ارتباطی", "subscribers": 244000},
|
|
{"handle": "V2RayNG_VPN", "title": "راهنمای کلاینت v2rayNG", "niche": "آموزش و آپدیتهای اندروید", "subscribers": 7750},
|
|
{"handle": "v2ray_collector", "title": "پایشگر شبکه V2RayCollector", "niche": "تست و مانیتورینگ اختلالات", "subscribers": 4750},
|
|
{"handle": "v2ray_free_conf", "title": "اخبار زیرساخت سرورها", "niche": "وضعیت سرورها و پینگ دیتاسنترها", "subscribers": 15200},
|
|
{"handle": "V2rayProxy", "title": "پایش پروکسیها و پورتها", "niche": "تحلیل پورتها و پروتکلها", "subscribers": 7180},
|
|
{"handle": "shadowrocket_vpn", "title": "راهنمای Shadowrocket و iOS", "niche": "تنظیمات کلاینتهای آیفون", "subscribers": 1710},
|
|
{"handle": "Digiato", "title": "دیجیاتو اخبار فناوری و اینترنت", "niche": "اخبار رسمی اینترنت و زیرساخت", "subscribers": 115000},
|
|
{"handle": "ProxyMTProto", "title": "اطلاعیهها و پروکسیهای امن", "niche": "تحلیل پروکسی و ترافیک تلگرام", "subscribers": 11900000},
|
|
{"handle": "v2rayngvpn", "title": "جامعه کاربری v2rayNG", "niche": "تجربیات اتصال و ترفندهای کاربران", "subscribers": 1030000},
|
|
]
|
|
|
|
async def seed_default_channels(session: AsyncSession):
|
|
for seed in DEFAULT_SEED_CHANNELS:
|
|
clean_handle = seed["handle"].replace("@", "").strip()
|
|
stmt = select(Channel).where(Channel.telegram_handle == clean_handle)
|
|
res = await session.execute(stmt)
|
|
existing = res.scalar_one_or_none()
|
|
if not existing:
|
|
ch = Channel(
|
|
telegram_handle=clean_handle,
|
|
title=seed["title"],
|
|
niche=seed["niche"],
|
|
subscriber_count=seed.get("subscribers", 10000),
|
|
is_active=True
|
|
)
|
|
session.add(ch)
|
|
else:
|
|
existing.title = seed["title"]
|
|
existing.niche = seed["niche"]
|
|
if seed.get("subscribers"):
|
|
existing.subscriber_count = seed["subscribers"]
|
|
await session.commit()
|
|
|
|
def parse_views_count(view_text: str) -> int:
|
|
if not view_text:
|
|
return 0
|
|
clean = view_text.strip().upper()
|
|
try:
|
|
if "M" in clean:
|
|
return int(float(clean.replace("M", "")) * 1_000_000)
|
|
if "K" in clean:
|
|
return int(float(clean.replace("K", "")) * 1_000)
|
|
return int(clean)
|
|
except ValueError:
|
|
return 0
|
|
|
|
async def scrape_public_telegram_channel(handle: str, limit: int = 20) -> List[Dict[str, Any]]:
|
|
clean_handle = handle.replace("@", "").strip()
|
|
url = f"https://t.me/s/{clean_handle}"
|
|
|
|
async with httpx.AsyncClient(headers=HEADERS, timeout=20.0, follow_redirects=True) as client:
|
|
try:
|
|
res = await client.get(url)
|
|
if res.status_code != 200:
|
|
logger.warning(f"Channel {clean_handle} returned HTTP {res.status_code}")
|
|
return []
|
|
except Exception as e:
|
|
logger.error(f"Error fetching channel {clean_handle}: {e}")
|
|
return []
|
|
|
|
soup = BeautifulSoup(res.text, "html.parser")
|
|
msg_divs = soup.find_all("div", class_="tgme_widget_message")
|
|
|
|
results: List[Dict[str, Any]] = []
|
|
for msg in msg_divs[-limit:]:
|
|
post_attr = msg.get("data-post")
|
|
if not post_attr or "/" not in post_attr:
|
|
continue
|
|
|
|
parts = post_attr.split("/")
|
|
try:
|
|
msg_id = int(parts[1])
|
|
except ValueError:
|
|
continue
|
|
|
|
post_url = f"https://t.me/{post_attr}"
|
|
|
|
# Text
|
|
text_elem = msg.find("div", class_="tgme_widget_message_text")
|
|
original_text = text_elem.get_text(separator="\n").strip() if text_elem else ""
|
|
|
|
# Media Extraction
|
|
media_type = "none"
|
|
media_url = None
|
|
|
|
photo_elem = msg.find("a", class_="tgme_widget_message_photo_wrap")
|
|
video_elem = msg.find("video")
|
|
doc_elem = msg.find("div", class_="tgme_widget_message_document")
|
|
|
|
if photo_elem:
|
|
media_type = "photo"
|
|
style = photo_elem.get("style", "")
|
|
bg_match = re.search(r"url\('([^']+)'\)", style)
|
|
if bg_match:
|
|
media_url = bg_match.group(1)
|
|
elif video_elem:
|
|
media_type = "video"
|
|
media_url = video_elem.get("src")
|
|
elif doc_elem:
|
|
media_type = "document"
|
|
|
|
# Views
|
|
views_elem = msg.find("span", class_="tgme_widget_message_views")
|
|
views = parse_views_count(views_elem.text if views_elem else "0")
|
|
|
|
# Date & Recency check
|
|
time_elem = msg.find("time")
|
|
posted_at = datetime.now(timezone.utc)
|
|
if time_elem and time_elem.get("datetime"):
|
|
try:
|
|
dt_str = time_elem["datetime"].replace("Z", "+00:00")
|
|
posted_at = datetime.fromisoformat(dt_str)
|
|
except Exception:
|
|
pass
|
|
|
|
# STRICT RECENCY FILTER: Discard posts older than 7 days
|
|
now_utc = datetime.now(timezone.utc)
|
|
days_old = (now_utc - posted_at).total_seconds() / 86400.0
|
|
if days_old > 7.0:
|
|
logger.debug(f"Skipping stale post {post_attr} ({days_old:.1f} days old)")
|
|
continue
|
|
|
|
# OUTDATED SEASONAL KEYWORDS FILTER (No outdated Nowruz, past holiday greetings)
|
|
stale_keywords = [
|
|
"نوروز", "سال نو", "عید نوروز", "تعطیلات عید", "فروردین", "تبریک سال نو",
|
|
"1402", "۱۴۰۲", "سال ۱۴۰۲", "نوروز مبارک", "سال پیش"
|
|
]
|
|
text_lower = (original_text or "").lower()
|
|
if any(sk in text_lower for sk in stale_keywords):
|
|
logger.info(f"Skipping outdated seasonal post {post_attr} containing stale holiday keywords")
|
|
continue
|
|
|
|
if original_text or media_url:
|
|
results.append({
|
|
"message_id": msg_id,
|
|
"original_text": original_text,
|
|
"post_url": post_url,
|
|
"media_type": media_type,
|
|
"media_url": media_url,
|
|
"views": views,
|
|
"reactions": 0,
|
|
"posted_at": posted_at
|
|
})
|
|
|
|
return results
|
|
|
|
async def run_channel_ingestion(session: AsyncSession, channel: Channel, limit: int = 15) -> int:
|
|
posts = await scrape_public_telegram_channel(channel.telegram_handle, limit=limit)
|
|
new_count = 0
|
|
|
|
for post_data in posts:
|
|
# Check uniqueness by channel_id and message_id
|
|
stmt = select(RawPost).where(
|
|
RawPost.channel_id == channel.id,
|
|
RawPost.message_id == post_data["message_id"]
|
|
)
|
|
existing = (await session.execute(stmt)).scalar_one_or_none()
|
|
if not existing:
|
|
raw_post = RawPost(
|
|
channel_id=channel.id,
|
|
message_id=post_data["message_id"],
|
|
original_text=post_data["original_text"],
|
|
post_url=post_data["post_url"],
|
|
media_type=post_data["media_type"],
|
|
media_url=post_data["media_url"],
|
|
views=post_data["views"],
|
|
reactions=post_data["reactions"],
|
|
posted_at=post_data["posted_at"]
|
|
)
|
|
session.add(raw_post)
|
|
new_count += 1
|
|
|
|
await session.execute(
|
|
update(Channel)
|
|
.where(Channel.id == channel.id)
|
|
.values(last_scraped_at=datetime.now(timezone.utc))
|
|
)
|
|
await session.commit()
|
|
return new_count
|