{ "openapi": "3.0.1", "info": { "title": "Web Scraper & Crawler for RAG (Markdown)", "description": "Turn any website into clean Markdown or structured JSON. Read pages, explore whole sites, discover every URL, pull AI fields, search the web and read PDFs. Also runs as a fast HTTP API in standby mode.", "version": "1.0", "x-build-id": "DORaulZzIKMjbiGNm" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/byteblisk~sitereader/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-byteblisk-sitereader", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/byteblisk~sitereader/runs": { "post": { "operationId": "runs-sync-byteblisk-sitereader", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/byteblisk~sitereader/run-sync": { "post": { "operationId": "run-sync-byteblisk-sitereader", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "mode" ], "properties": { "mode": { "title": "What to do", "enum": [ "read", "explore", "discover", "fields", "find", "lookup", "feed", "watch" ], "type": "string", "description": "Read: each URL. Explore: follow links within each site. Discover: list a site's URLs. Fields: AI pulls the fields you ask for across the pages. Find: web search, optionally reading each result. Feed: a Greenhouse, Lever or Ashby job board, a Shopify store, a WordPress site or an RSS feed, as clean items. Watch: compare each page with the previous run and output what is new, changed or removed (run it on an Apify schedule). Find all: a table of every item matching your query across search results, each row with its sources.", "default": "read" }, "urls": { "title": "URLs", "type": "array", "description": "Pages or sites. For AI fields, end a URL with /* to let SiteReader pick the most relevant pages of that site.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "query": { "title": "Search query", "type": "string", "description": "Find mode only." }, "intent": { "title": "Find: rank by intent (AI)", "type": "string", "description": "What you are really after; results are scored 0-1 and sorted by it." }, "columns": { "title": "Find all: columns", "type": "object", "description": "{\"company\": \"company name\", \"role\": \"job title\", \"city\": \"location\"} - the first column identifies a row." }, "pages": { "title": "Find all: pages to read", "minimum": 1, "maximum": 20, "type": "integer", "description": "Lookup: how many search result pages to read for the table.", "default": 8 }, "feed_kind": { "title": "Feed: source type", "enum": [ "", "shopify", "wordpress", "rss", "google_play", "google_play_developer", "google_play_reviews", "google_play_search", "instagram_business", "meta_ads", "youtube_search", "youtube_channel", "youtube_comments", "app_store", "app_store_search", "app_store_developer", "app_store_reviews", "app_store_top" ], "type": "string", "description": "Job boards and Google Play app or developer URLs are recognised by themselves; choose the type for Shopify, WordPress, RSS or your own app's reviews.", "default": "" }, "google_access_token": { "title": "Google Play reviews: your Google access token", "type": "string", "description": "Only for reading your own app's reviews through Google's official API (last 7 days). Used for the call, never stored." }, "meta_access_token": { "title": "Meta access token (Instagram / Ad Library)", "type": "string", "description": "Your Meta token: for Instagram Business Discovery (with your Instagram business account) or the Ad Library. Never stored." }, "youtube_api_key": { "title": "YouTube Data API key", "type": "string", "description": "Your own key from Google Cloud. Never stored." }, "feed_options": { "title": "Feed options", "type": "object", "description": "Instagram: {\"ig_user_id\": \"\"}. Ad Library: {\"countries\": [\"US\"], \"page_ids\": []}. Any: {\"limit\": 50}. For these feeds put usernames, search terms, @handles or video URLs in URLs (or Search query)." }, "watch_name": { "title": "Watch: name", "type": "string", "description": "Keeps this watch's memory apart from your other watches. Defaults to one built from the URLs." }, "only_changes": { "title": "Watch: only new, changed and removed pages", "type": "boolean", "description": "Watch: only new, changed and removed pages.", "default": true }, "include_content": { "title": "Watch: include the page Markdown", "type": "boolean", "description": "Watch: include the page Markdown.", "default": false }, "output": { "title": "Output", "type": "array", "description": "What to return for each page.", "items": { "type": "string", "enum": [ "markdown", "text", "html", "raw_html", "links", "images", "screenshot", "summary", "fields", "structured" ], "enumTitles": [ "Markdown", "Plain text", "Clean HTML", "Original HTML", "Links", "Images", "Screenshot", "AI summary", "AI fields", "Structured data (schema.org, tables)" ] }, "default": [ "markdown" ] }, "content": { "title": "Content", "enum": [ "main", "full" ], "type": "string", "description": "Main: drop menus, footers and other boilerplate. Full: the whole page.", "default": "main" }, "keep": { "title": "Keep only these parts (CSS selectors)", "type": "array", "description": "For example article, .job-list, #main", "items": { "type": "string" } }, "drop": { "title": "Remove these parts (CSS selectors)", "type": "array", "description": "For example nav, footer, .cookie-banner", "items": { "type": "string" } }, "clean": { "title": "Remove cookie and consent banners", "type": "boolean", "description": "Drops cookie / consent banners and overlays before the page is turned into text.", "default": true }, "expand": { "title": "Open collapsed sections (browser)", "type": "boolean", "description": "Opens accordions, 'read more' toggles and
before reading. Uses the browser.", "default": false }, "engine": { "title": "Engine", "enum": [ "auto", "browser" ], "type": "string", "description": "Auto: a fast fetch, with a browser only when the page needs JavaScript. Browser: always render (slower, priced as a browser page).", "default": "auto" }, "wait_ms": { "title": "Wait after load (ms)", "minimum": 0, "maximum": 30000, "type": "integer", "description": "Extra time for JavaScript widgets to appear. Uses the browser.", "default": 0 }, "steps": { "title": "Browser steps", "type": "array", "description": "Steps before the page is read: wait, click, type, key, scroll, script. Example: [{\"do\": \"click\", \"selector\": \"#load-more\"}, {\"do\": \"wait\", \"ms\": 1000}]" }, "headers": { "title": "Custom HTTP headers", "type": "object", "description": "Your own headers, e.g. for your own site: {\"Authorization\": \"Bearer ...\"}. Sent only to the URLs in this run." }, "cookies": { "title": "Cookies", "type": "array", "description": "Your own cookies for pages you are allowed to read: [{\"name\": \"session\", \"value\": \"...\", \"domain\": \"example.com\"}]. Pages read with cookies are never cached or shared." }, "limit": { "title": "Max pages / URLs / results", "minimum": 1, "type": "integer", "description": "Explore: pages per site. Discover: URLs per site. Find: results.", "default": 50 }, "depth": { "title": "Explore depth", "minimum": 0, "type": "integer", "description": "How many path levels below the site's root to follow.", "default": 3 }, "only_paths": { "title": "Explore only paths matching (regex)", "type": "array", "description": "For example ^/careers", "items": { "type": "string" } }, "skip_paths": { "title": "Skip paths matching (regex)", "type": "array", "description": "For example ^/blog", "items": { "type": "string" } }, "ignore_query": { "title": "Treat query-string variants as one page", "type": "boolean", "description": "Explore: URLs that differ only after the ? (sorting, filters, sessions) count as one page. Tracking parameters like utm_ are always ignored.", "default": false }, "time_budget_secs": { "title": "Time budget (seconds)", "minimum": 10, "type": "integer", "description": "Explore: stop after this many seconds and keep the pages read so far. Pages are saved as they are read." }, "from_sitemap": { "title": "Start exploring from the sitemap too", "type": "boolean", "description": "Start exploring from the sitemap too.", "default": true }, "filter": { "title": "Discover: keep URLs containing", "type": "string", "description": "Words to look for in the URLs, best matches first." }, "read_results": { "title": "Find: read each result page", "type": "boolean", "description": "Find: read each result page.", "default": false }, "ai_instructions": { "title": "AI: what to extract", "type": "string", "description": "For example: every open job with its title, location and link." }, "ai_schema": { "title": "AI: JSON schema of the result (optional)", "type": "object", "description": "The fields you want back. Without it, the AI chooses a structure that fits the instructions." }, "ai_model": { "title": "AI model", "enum": [ "haiku", "opus" ], "type": "string", "description": "Haiku: fast and cheap, right for most pages. Opus: the strongest model, priced higher.", "default": "haiku" }, "anthropic_api_key": { "title": "Your own Anthropic API key (optional)", "type": "string", "description": "Use your own Claude key and pay Anthropic directly; AI pages are then not charged here." }, "region": { "title": "Region: country code", "type": "string", "description": "See the site as a visitor in this country, for example IL, US, DE. Uses Apify Proxy; priced per page." }, "language": { "title": "Page language", "type": "string", "description": "Ask sites for this language, e.g. he, de, pt-BR. Works without a country; with a region, the region's languages are used." }, "proxy_type": { "title": "Region: proxy type", "enum": [ "none", "datacenter", "residential" ], "type": "string", "description": "Region: proxy type.", "default": "none" }, "robots": { "title": "robots.txt", "enum": [ "ignore", "respect" ], "type": "string", "description": "Ignore (default) or respect each site's robots.txt rules.", "default": "ignore" }, "fallback": { "title": "When a site blocks the request", "enum": [ "none", "archive" ], "type": "string", "description": "Archive: return the site's latest Internet Archive copy, marked with its date.", "default": "none" }, "delay_secs": { "title": "Seconds between requests to one site", "minimum": 0, "type": "number", "description": "Seconds between requests to one site.", "default": 2 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }