import sqlite3, requests, time from concurrent.futures import ThreadPoolExecutor, as_completed DB_PATH = '/workspace/ecommerce_brands_research.db' headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36' } def check_url(row): row_id, url = row status = 0 try: # First try HEAD resp = requests.head(url, headers=headers, timeout=6, allow_redirects=True) status = resp.status_code if status in [403, 405, 404]: # Retry with GET in case server blocks HEAD resp = requests.get(url, headers=headers, timeout=6, stream=True) status = resp.status_code except Exception as e: status = 599 # Network error return row_id, status def run_validation(): conn = sqlite3.connect(DB_PATH) c = conn.cursor() c.execute('SELECT id, url FROM sources WHERE http_status IS NULL OR http_status = 200') rows = c.fetchall() conn.close() print(f"Checking HTTP health for {len(rows)} URLs with ThreadPoolExecutor...") results = [] with ThreadPoolExecutor(max_workers=25) as executor: futures = {executor.submit(check_url, r): r for r in rows} done = 0 for f in as_completed(futures): results.append(f.result()) done += 1 if done % 100 == 0 or done == len(rows): print(f"Validated {done}/{len(rows)} URLs...") # Update DB in batch conn = sqlite3.connect(DB_PATH) c = conn.cursor() c.executemany('UPDATE sources SET http_status = ? WHERE id = ?', [(status, row_id) for row_id, status in results]) conn.commit() c.execute('SELECT http_status, count(*) FROM sources GROUP BY http_status') stats = c.fetchall() print("\nHTTP Status Summary:") for code, count in stats: print(f" HTTP {code}: {count}") conn.close() if __name__ == '__main__': run_validation()