Web Crawling

curl -X POST https://api.alterlab.io/api/v1/crawl \
  -H "X-API-Key: your_api_key" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://example.com",
    "max_pages": 100,
    "max_depth": 3,
    "include_patterns": ["/blog/*", "/docs/*"],
    "exclude_patterns": ["/admin/*", "*.pdf"],
    "formats": ["text", "markdown"],
    "webhook_url": "https://your-server.com/webhook"
  }'

{
  "url": "https://example.com",
  "include_patterns": ["/blog/*", "/products/*"],
  "exclude_patterns": ["/admin/*", "/internal/*", "*.pdf"]
}

curl https://api.alterlab.io/api/v1/crawl/c9a1b2d3-...?include_results=true \
  -H "X-API-Key: your_api_key"

{
  "crawl_id": "c9a1b2d3-...",
  "status": "scraping",
  "total": 47,
  "completed": 32,
  "failed": 2,
  "in_progress": 13,
  "credits_debited": 47000,
  "created_at": "2026-03-24T10:00:00Z",
  "current_depth": 2,
  "max_depth": 3,
  "total_discovered": 47,
  "billing": {
    "credits_debited": 47000,
    "credits_used": 34000,
    "credits_refunded": 0,
    "estimated_cost_usd": "$0.47",
    "actual_cost_usd": "$0.34",
    "is_byop": false,
    "tier_breakdown": { "1": 28, "3": 4 }
  },
  "pages": [
    {
      "job_id": "a1b2c3d4-...",
      "url": "https://example.com/blog/post-1",
      "status": "succeeded",
      "result": { "text": "...", "metadata": { ... } }
    }
  ]
}

curl -X DELETE https://api.alterlab.io/api/v1/crawl/c9a1b2d3-... \
  -H "X-API-Key: your_api_key"

import alterlab
import time

client = alterlab.AlterLab(api_key="your_api_key")

# Start a crawl
crawl = client.crawl(
    url="https://example.com",
    max_pages=100,
    max_depth=3,
    include_patterns=["/blog/*", "/docs/*"],
    exclude_patterns=["/admin/*"],
    formats=["text", "markdown"],
    webhook_url="https://your-server.com/webhook",
)

print(f"Crawl ID: {crawl['crawl_id']}")
print(f"Pages discovered: {crawl['estimated_pages']}")
print(f"Estimated credits: {crawl['estimated_credits']}")

# Poll until complete
while True:
    status = client.get_crawl_status(crawl["crawl_id"], include_results=True)
    print(f"Status: {status['status']} -- depth {status['current_depth']}/{status['max_depth']}")
    print(f"  {status['completed']}/{status['total']} pages done, {status['failed']} failed")
    if status["status"] not in ("discovering", "scraping"):
        break
    time.sleep(3)

# Process results
for page in status.get("pages", []):
    if page["status"] == "succeeded":
        text = page["result"].get("text", "")
        print(f"  {page['url']}: {len(text)} chars")
    else:
        print(f"  {page['url']}: FAILED -- {page.get('error', 'unknown')}")

# Check billing
billing = status.get("billing", {})
print(f"Cost: {billing.get('actual_cost_usd', 'N/A')} (estimated {billing.get('estimated_cost_usd', 'N/A')})")
print(f"Refunded: {billing.get('credits_refunded', 0)} credits")

import AlterLab from "@alterlab/sdk";

const client = new AlterLab({ apiKey: "your_api_key" });

// Start a crawl
const crawl = await client.crawl({
  url: "https://example.com",
  maxPages: 100,
  maxDepth: 3,
  includePatterns: ["/blog/*", "/docs/*"],
  excludePatterns: ["/admin/*"],
  formats: ["text", "markdown"],
  webhookUrl: "https://your-server.com/webhook",
});

console.log(`Crawl ID: ${crawl.crawlId}`);
console.log(`Pages discovered: ${crawl.estimatedPages}`);

// Poll until complete
let status;
do {
  await new Promise((r) => setTimeout(r, 3000));
  status = await client.getCrawlStatus(crawl.crawlId, { includeResults: true });
  console.log(`${status.completed}/${status.total} done (depth ${status.currentDepth}/${status.maxDepth})`);
} while (["discovering", "scraping"].includes(status.status));

// Process results
for (const page of status.pages ?? []) {
  if (page.status === "succeeded") {
    console.log(`  ${page.url}: ${page.result?.text?.length ?? 0} chars`);
  } else {
    console.log(`  ${page.url}: FAILED -- ${page.error}`);
  }
}

// Cancel a crawl (if needed)
// const cancelled = await client.cancelCrawl(crawl.crawlId);
// console.log(`Refunded: ${cancelled.creditsRefunded} credits`);