hamed-dashboards/dashboards/ecommerce/validate_links.py

58 lines
2 KiB
Python
Executable file

import sqlite3, requests, time
from concurrent.futures import ThreadPoolExecutor, as_completed
DB_PATH = '/workspace/ecommerce_brands_research.db'
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36'
}
def check_url(row):
row_id, url = row
status = 0
try:
# First try HEAD
resp = requests.head(url, headers=headers, timeout=6, allow_redirects=True)
status = resp.status_code
if status in [403, 405, 404]:
# Retry with GET in case server blocks HEAD
resp = requests.get(url, headers=headers, timeout=6, stream=True)
status = resp.status_code
except Exception as e:
status = 599 # Network error
return row_id, status
def run_validation():
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute('SELECT id, url FROM sources WHERE http_status IS NULL OR http_status = 200')
rows = c.fetchall()
conn.close()
print(f"Checking HTTP health for {len(rows)} URLs with ThreadPoolExecutor...")
results = []
with ThreadPoolExecutor(max_workers=25) as executor:
futures = {executor.submit(check_url, r): r for r in rows}
done = 0
for f in as_completed(futures):
results.append(f.result())
done += 1
if done % 100 == 0 or done == len(rows):
print(f"Validated {done}/{len(rows)} URLs...")
# Update DB in batch
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.executemany('UPDATE sources SET http_status = ? WHERE id = ?', [(status, row_id) for row_id, status in results])
conn.commit()
c.execute('SELECT http_status, count(*) FROM sources GROUP BY http_status')
stats = c.fetchall()
print("\nHTTP Status Summary:")
for code, count in stats:
print(f" HTTP {code}: {count}")
conn.close()
if __name__ == '__main__':
run_validation()