58 lines
2 KiB
Python
Executable file
58 lines
2 KiB
Python
Executable file
import sqlite3, requests, time
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
DB_PATH = '/workspace/ecommerce_brands_research.db'
|
|
|
|
headers = {
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36'
|
|
}
|
|
|
|
def check_url(row):
|
|
row_id, url = row
|
|
status = 0
|
|
try:
|
|
# First try HEAD
|
|
resp = requests.head(url, headers=headers, timeout=6, allow_redirects=True)
|
|
status = resp.status_code
|
|
if status in [403, 405, 404]:
|
|
# Retry with GET in case server blocks HEAD
|
|
resp = requests.get(url, headers=headers, timeout=6, stream=True)
|
|
status = resp.status_code
|
|
except Exception as e:
|
|
status = 599 # Network error
|
|
return row_id, status
|
|
|
|
def run_validation():
|
|
conn = sqlite3.connect(DB_PATH)
|
|
c = conn.cursor()
|
|
c.execute('SELECT id, url FROM sources WHERE http_status IS NULL OR http_status = 200')
|
|
rows = c.fetchall()
|
|
conn.close()
|
|
|
|
print(f"Checking HTTP health for {len(rows)} URLs with ThreadPoolExecutor...")
|
|
|
|
results = []
|
|
with ThreadPoolExecutor(max_workers=25) as executor:
|
|
futures = {executor.submit(check_url, r): r for r in rows}
|
|
done = 0
|
|
for f in as_completed(futures):
|
|
results.append(f.result())
|
|
done += 1
|
|
if done % 100 == 0 or done == len(rows):
|
|
print(f"Validated {done}/{len(rows)} URLs...")
|
|
|
|
# Update DB in batch
|
|
conn = sqlite3.connect(DB_PATH)
|
|
c = conn.cursor()
|
|
c.executemany('UPDATE sources SET http_status = ? WHERE id = ?', [(status, row_id) for row_id, status in results])
|
|
conn.commit()
|
|
|
|
c.execute('SELECT http_status, count(*) FROM sources GROUP BY http_status')
|
|
stats = c.fetchall()
|
|
print("\nHTTP Status Summary:")
|
|
for code, count in stats:
|
|
print(f" HTTP {code}: {count}")
|
|
conn.close()
|
|
|
|
if __name__ == '__main__':
|
|
run_validation()
|