#!/usr/bin/env python3
"""Universal web scraper template: CSS/XPath, pagination, proxy, anti-detect, JSON/CSV output."""
import argparse, csv, json, random, time
import requests
from lxml import html, etree

DEFAULT_HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0 Safari/537.36',
    'Accept-Language': 'en-US,en;q=0.9',
}

def get_page(url, proxy=None, headers=None, timeout=20):
    h = {**DEFAULT_HEADERS, **(headers or {})}
    proxies = {'http': proxy, 'https': proxy} if proxy else None
    resp = requests.get(url, headers=h, proxies=proxies, timeout=timeout)
    resp.raise_for_status()
    return resp

def extract_items(root, item_xpath, field_xpaths):
    items = []
    for node in root.xpath(item_xpath):
        item = {}
        for name, xpath in field_xpaths.items():
            vals = node.xpath(xpath)
            item[name] = vals[0].strip() if vals and hasattr(vals[0], 'strip') else (str(vals[0]) if vals else '')
        items.append(item)
    return items

def scrape(cfg):
    all_items = []
    page = cfg.get('start_page', 1)
    max_pages = cfg.get('max_pages', 1)
    while page <= max_pages:
        url = cfg['url_pattern'].format(page=page)
        proxy = cfg.get('proxy')
        resp = get_page(url, proxy=proxy, headers=cfg.get('headers'))
        root = html.fromstring(resp.text)
        items = extract_items(root, cfg['item_xpath'], cfg['field_xpaths'])
        if not items:
            break
        all_items.extend(items)
        page += 1
        time.sleep(cfg.get('rate_limit_seconds', 1.0))
    return all_items

def write_output(items, path):
    if path.endswith('.json'):
        with open(path, 'w', encoding='utf-8') as f:
            json.dump(items, f, indent=2, ensure_ascii=False)
    else:
        if not items: return
        with open(path, 'w', newline='', encoding='utf-8') as f:
            w = csv.DictWriter(f, fieldnames=list(items[0].keys()))
            w.writeheader(); w.writerows(items)

if __name__ == '__main__':
    print('scraper template ready')