Building Resilient Web Scrapers: Retry, Circuit Breaker, and Fallback Patterns

import time
import random
import alterlab

client = alterlab.Client("YOUR_API_KEY")

def scrape_with_retry(url, max_attempts=5):
    attempt = 0
    while attempt < max_attempts:
        try:
            response = client.scrape(url)
            if response.status_code < 500:
                return response
        except alterlab.APIError as exc:
            if exc.status_code >= 500:
                pass  # treat as retryable
            else:
                raise  # client errors (4xx) are not retried
        attempt += 1
        delay = (2 ** attempt) + random.uniform(0, 1)  # exponential backoff + jitter
        time.sleep(delay)
    raise RuntimeError(f"Failed to scrape {url} after {max_attempts} attempts")

# usage
data = scrape_with_retry("https://example.com/public-data")
print(data.json)

curl -X POST https://api.alterlab.io/v1/scrape \
  -H "X-API-Key: YOUR_KEY" \
  -H "Retry-After: 1" \
  -d '{"url": "https://example.com/public-data", "max_retries": 4}'

import time
from enum import Enum

class State(Enum):
    CLOSED = 0
    OPEN = 1
    HALF_OPEN = 2

class CircuitBreaker:
    def __init__(self, fail_threshold=5, timeout=30):
        self.fail_threshold = fail_threshold
        self.timeout = timeout
        self.fail_count = 0
        self.state = State.CLOSED
        self.last_failure_time = None

    def call(self, func, *args, **kwargs):
        if self.state == State.OPEN:
            if time.time() - self.last_failure_time > self.timeout:
                self.state = State.HALF_OPEN
            else:
                raise RuntimeError("Circuit breaker is OPEN")
        try:
            result = func(*args, **kwargs)
            self.on_success()
            return result
        except Exception as exc:
            self.on_failure()
            raise exc

    def on_success(self):
        self.fail_count = 0
        self.state = State.CLOSED

    def on_failure(self):
        self.fail_count += 1
        self.last_failure_time = time.time()
        if self.fail_count >= self.fail_threshold:
            self.state = State.OPEN

breaker = CircuitBreaker(fail_threshold=4, timeout=20)

def safe_scrape(url):
    return breaker.call(scrape_with_retry, url)

# usage
try:
    data = safe_scrape("https://example.com/public-data")
except RuntimeError as err:
    print("Scrape unavailable:", err)

import json
import os
from datetime import datetime, timedelta

CACHE_DIR = "./scrape_cache"
CACHE_TTL = timedelta(hours=6)

def get_cached(url):
    path = os.path.join(CACHE_DIR, hash(url) + ".json")
    if not os.path.exists(path):
        return None
    with open(path) as f:
        entry = json.load(f)
    if datetime.fromisoentry["fetched"] + CACHE_TTL < datetime.utcnow():
        return None
    return entry["data"]

def save_cache(url, data):
    os.m