LangChain

pip install alterlab langchain langchain-community langchain-openai chromadb

from langchain.schema import Document
from alterlab import AlterLab

client = AlterLab(api_key="your_api_key")

def load_url(url: str, mode: str = "auto") -> Document:
    """Load a URL as a LangChain Document via AlterLab."""
    result = client.scrape(url, formats=["markdown"], mode=mode)
    return Document(
        page_content=result.get("markdown", result.get("text", "")),
        metadata={
            "source": url,
            "title": result.get("metadata", {}).get("title", ""),
            "status_code": result.get("metadata", {}).get("status_code"),
            "credits_used": result.get("cost", {}).get("credits_charged"),
        },
    )

# Load a single page
doc = load_url("https://example.com/blog/ai-trends")
print(f"Loaded {len(doc.page_content)} chars from {doc.metadata['source']}")

# JavaScript-heavy SPA — use JS rendering
doc = load_url("https://app.example.com/dashboard", mode="js")

# Get multiple formats at once
result = client.scrape(
    "https://example.com",
    formats=["markdown", "text", "json"],
)

# Use cost controls for budget-conscious pipelines
result = client.scrape(
    "https://example.com",
    formats=["markdown"],
    cost_controls={"max_tier": "2", "fail_fast": True},
)

from langchain.text_splitter import RecursiveCharacterTextSplitter

# Load multiple pages
urls = [
    "https://docs.example.com/getting-started",
    "https://docs.example.com/api-reference",
    "https://docs.example.com/tutorials",
]

documents = [load_url(url) for url in urls]
print(f"Loaded {len(documents)} documents")

# Split into chunks for embedding
splitter = RecursiveCharacterTextSplitter(
    chunk_size=1000,
    chunk_overlap=200,
    separators=["\n## ", "\n### ", "\n\n", "\n", " "],
)

chunks = splitter.split_documents(documents)
print(f"Split into {len(chunks)} chunks")

from langchain_openai import OpenAIEmbeddings
from langchain_community.vectorstores import Chroma

# Create embeddings and store in ChromaDB
embeddings = OpenAIEmbeddings()
vectorstore = Chroma.from_documents(
    documents=chunks,
    embedding=embeddings,
    collection_name="web_docs",
    persist_directory="./chroma_db",
)

print(f"Stored {len(chunks)} chunks in vector store")

from langchain_openai import ChatOpenAI
from langchain.chains import RetrievalQA

llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)

qa_chain = RetrievalQA.from_chain_type(
    llm=llm,
    chain_type="stuff",
    retriever=vectorstore.as_retriever(search_kwargs={"k": 4}),
)

answer = qa_chain.invoke("How do I authenticate with the API?")
print(answer["result"])

from alterlab import AlterLab
from langchain.schema import Document
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_openai import OpenAIEmbeddings, ChatOpenAI
from langchain_community.vectorstores import Chroma
from langchain.chains import RetrievalQA

# 1. Configure clients
scraper = AlterLab(api_key="your_alterlab_key")
llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)

# 2. Scrape pages into Documents
urls = [
    "https://docs.stripe.com/api/charges",
    "https://docs.stripe.com/api/customers",
    "https://docs.stripe.com/api/refunds",
]

documents = []
for url in urls:
    result = scraper.scrape(url, formats=["markdown"])
    documents.append(
        Document(
            page_content=result.get("markdown", ""),
            metadata={"source": url},
        )
    )

# 3. Chunk
splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
chunks = splitter.split_documents(documents)

# 4. Embed and store
vectorstore = Chroma.from_documents(
    documents=chunks,
    embedding=OpenAIEmbeddings(),
)

# 5. Query
qa = RetrievalQA.from_chain_type(
    llm=llm,
    retriever=vectorstore.as_retriever(search_kwargs={"k": 4}),
)

questions = [
    "How do I create a charge?",
    "What parameters does the refund endpoint accept?",
    "How do I list all customers?",
]

for q in questions:
    answer = qa.invoke(q)
    print(f"Q: {q}")
    print(f"A: {answer['result']}\n")

import time
from alterlab import AlterLab
from langchain.schema import Document

client = AlterLab(api_key="your_api_key")

urls = [
    "https://example.com/page-1",
    "https://example.com/page-2",
    "https://example.com/page-3",
    # ... up to 100 URLs per batch
]

# Submit batch
batch = client.batch_scrape(
    urls=[{"url": u, "formats": ["markdown"]} for u in urls],
)

# Poll until done
while True:
    status = client.get_batch_status(batch["batch_id"])
    if status["status"] != "processing":
        break
    time.sleep(2)

# Convert to LangChain Documents
documents = []
for item in status["items"]:
    if item["status"] == "succeeded":
        documents.append(
            Document(
                page_content=item["result"].get("markdown", ""),
                metadata={"source": item["url"]},
            )
        )

print(f"Loaded {len(documents)} documents from batch")

# Use AlterLab's built-in extraction profiles
result = client.scrape(
    "https://example.com/product",
    formats=["json"],
    extraction_profile="product",
)

# The JSON output contains structured product data
product = result.get("json", {})
print(f"Product: {product.get('name')}")
print(f"Price: {product.get('price')}")

# Or use a custom schema
result = client.scrape(
    "https://example.com/article",
    formats=["json"],
    extraction_schema={
        "type": "object",
        "properties": {
            "title": {"type": "string"},
            "author": {"type": "string"},
            "published_date": {"type": "string"},
            "summary": {"type": "string"},
        },
    },
)