Rendering, Waits, and Proxies¶

Browser-style controls available through the unified scrape endpoint: JS rendering, wait conditions, resource blocking, cookies, sessions, and proxy selection.

JavaScript rendering¶

Render JavaScript-heavy pages (SPAs, React/Angular/Vue):

curl -X POST "https://scrape.toolkitapi.io/v1/scrape" \
  -H "X-API-Key: YOUR_KEY" \
  -H "Content-Type: application/json" \
  -d '{"url": "https://quotes.toscrape.com/js/", "render_js": true, "formats": ["html"]}'

import requests

resp = requests.post(
    "https://scrape.toolkitapi.io/v1/scrape",
    headers={"X-API-Key": "YOUR_KEY"},
    json={"url": "https://quotes.toscrape.com/js/", "render_js": True, "formats": ["html"]},
)
data = resp.json()
print(f"JS rendered: {data.get('js_rendered', False)}")

const resp = await fetch("https://scrape.toolkitapi.io/v1/scrape", {
  method: "POST",
  headers: { "X-API-Key": "YOUR_KEY", "Content-Type": "application/json" },
  body: JSON.stringify({ url: "https://quotes.toscrape.com/js/", render_js: true, formats: ["html"] }),
});
const data = await resp.json();
console.log(`JS rendered: ${data.js_rendered}`);

Wait modes¶

Control when the browser considers the page "ready":

from toolkitapi import Scrape

with Scrape(api_key="tk_...") as scrape:
    result = scrape.render_page(
        url="https://toolkitapi.io/dashboard",
        wait_until="load",
        output="clean",
    )

Wait for a DOM element¶

curl -X POST "https://scrape.toolkitapi.io/v1/scrape" \
  -H "X-API-Key: YOUR_KEY" \
  -H "Content-Type: application/json" \
  -d '{"url": "https://toolkitapi.io/search", "render_js": true, "wait_for": "#results", "wait_timeout": 20000, "formats": ["text"]}'

from toolkitapi import Scrape

with Scrape(api_key="tk_...") as scrape:
    result = scrape.render_page(
        url="https://toolkitapi.io/search",
        wait_for="#results",
        wait_timeout=20000,
        output="text",
    )

const resp = await fetch("https://scrape.toolkitapi.io/v1/scrape", {
  method: "POST",
  headers: { "X-API-Key": "YOUR_KEY", "Content-Type": "application/json" },
  body: JSON.stringify({
    url: "https://toolkitapi.io/search",
    render_js: true,
    wait_for: "#results",
    wait_timeout: 20000,
    formats: ["text"],
  }),
});

Infinite scroll pages¶

curl -X POST "https://scrape.toolkitapi.io/v1/scrape" \
  -H "X-API-Key: YOUR_KEY" \
  -H "Content-Type: application/json" \
  -d '{"url": "https://toolkitapi.io/feed", "render_js": true, "scroll": true, "scroll_count": 5, "formats": ["markdown"]}'

from toolkitapi import Scrape

with Scrape(api_key="tk_...") as scrape:
    result = scrape.render_page(
        url="https://toolkitapi.io/feed",
        scroll=True,
        scroll_count=5,
        wait_until="networkidle",
        output="markdown",
    )

Block expensive resources¶

Speed up scraping by blocking images, fonts, and media:

curl -X POST "https://scrape.toolkitapi.io/v1/scrape" \
  -H "X-API-Key: YOUR_KEY" \
  -H "Content-Type: application/json" \
  -d '{"url": "https://toolkitapi.io", "render_js": true, "block_resources": ["image", "font", "media"], "formats": ["text"]}'

from toolkitapi import Scrape

with Scrape(api_key="tk_...") as scrape:
    result = scrape.render_page(
        url="https://toolkitapi.io",
        block_resources=["image", "font", "media"],
        output="text",
    )

Custom headers and cookies¶

curl -X POST "https://scrape.toolkitapi.io/v1/scrape" \
  -H "X-API-Key: YOUR_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://httpbin.org/headers",
    "headers": {"Accept-Language": "en-GB,en;q=0.9", "X-Debug": "true"},
    "cookies": [{"name": "sessionid", "value": "abc123", "domain": ".toolkitapi.io"}],
    "formats": ["html"]
  }'

from toolkitapi import Scrape

with Scrape(api_key="tk_...") as scrape:
    result = scrape.fetch(
        url="https://httpbin.org/headers",
        headers={"Accept-Language": "en-GB,en;q=0.9", "X-Debug": "true"},
        output="html",
    )

Browser session reuse¶

# Request 1 — log in
curl -X POST "https://scrape.toolkitapi.io/v1/scrape" \
  -H "X-API-Key: YOUR_KEY" \
  -H "Content-Type: application/json" \
  -d '{"url": "https://toolkitapi.io/login", "render_js": true, "session_name": "saved-session"}'

# Request 2 — reuse session
curl -X POST "https://scrape.toolkitapi.io/v1/scrape" \
  -H "X-API-Key: YOUR_KEY" \
  -H "Content-Type: application/json" \
  -d '{"url": "https://toolkitapi.io/account", "render_js": true, "session_name": "saved-session", "formats": ["text"]}'

from toolkitapi import Scrape

with Scrape(api_key="tk_...") as scrape:
    scrape.render_page(url="https://toolkitapi.io/login", session_name="saved-session")
    page = scrape.render_page(
        url="https://toolkitapi.io/account",
        session_name="saved-session",
        output="text",
    )
    print(page["content"])

Stealth mode¶

curl -X POST "https://scrape.toolkitapi.io/v1/scrape" \
  -H "X-API-Key: YOUR_KEY" \
  -H "Content-Type: application/json" \
  -d '{"url": "https://toolkitapi.io/protected", "render_js": true, "stealth": true, "formats": ["html"]}'

from toolkitapi import Scrape

with Scrape(api_key="tk_...") as scrape:
    result = scrape.render_page(
        url="https://toolkitapi.io/protected",
        stealth=True,
        output="html",
    )

Proxy options¶

curl -X POST "https://scrape.toolkitapi.io/v1/scrape" \
  -H "X-API-Key: YOUR_KEY" \
  -H "Content-Type: application/json" \
  -d '{"url": "https://toolkitapi.io", "proxy": "US", "formats": ["html"]}'

from toolkitapi import Scrape

with Scrape(api_key="tk_...") as scrape:
    us_page = scrape.fetch(url="https://toolkitapi.io", proxy="US", output="html")
    dc_page = scrape.fetch(url="https://toolkitapi.io", proxy="datacenter", output="html")

Rendering parameters¶