{ "openapi": "3.0.1", "info": { "title": "SiteVerdict — Web Page to Clean Markdown for AI (LLM-ready)", "description": "Turn public web pages, sitemaps or whole sites into clean Markdown, text and RAG chunks with title, author, dates, canonical URL, outline, links, tables and JSON-LD. Respects robots.txt, paywalls and AI opt-outs. Optional llms.txt.", "version": "2.0", "x-build-id": "W1dOBJTn1zfeYx1Ed" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/siteverdict~siteverdict-web-to-markdown/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-siteverdict-siteverdict-web-to-markdown", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/siteverdict~siteverdict-web-to-markdown/runs": { "post": { "operationId": "runs-sync-siteverdict-siteverdict-web-to-markdown", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/siteverdict~siteverdict-web-to-markdown/run-sync": { "post": { "operationId": "run-sync-siteverdict-siteverdict-web-to-markdown", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "URLs", "type": "array", "description": "Pages to convert (mode 'urls'), sites or sitemap.xml files (mode 'sitemap'), or crawl seeds (mode 'crawl'). Bare domains like example.com are fine.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "mode": { "title": "What to convert", "enum": [ "urls", "sitemap", "crawl" ], "type": "string", "description": "'urls' converts exactly the URLs you list. 'sitemap' reads each site's sitemap (from robots.txt or /sitemap.xml; or give a sitemap URL directly). 'crawl' follows links within the same site up to Max depth.", "default": "urls" }, "maxPages": { "title": "Max pages", "minimum": 1, "maximum": 20000, "type": "integer", "description": "Hard cap on pages fetched in this run (all statuses). You are only charged for pages successfully converted.", "default": 50 }, "maxDepth": { "title": "Max crawl depth", "minimum": 0, "maximum": 20, "type": "integer", "description": "Crawl mode only. 0 = start URLs only, 1 = plus pages they link to, and so on.", "default": 2 }, "includeGlobs": { "title": "Include only URLs matching (globs)", "type": "array", "description": "Optional. Globs starting with '/' match the path (e.g. /docs/**); others match the full URL (e.g. https://example.com/blog/*). Start URLs are always processed.", "default": [], "items": { "type": "string" } }, "excludeGlobs": { "title": "Exclude URLs matching (globs)", "type": "array", "description": "Optional. e.g. /tag/**, /search*, **/*?page=*", "default": [], "items": { "type": "string" } }, "allowSubdomains": { "title": "Include subdomains when crawling", "type": "boolean", "description": "If on, crawling example.com also follows docs.example.com. Off = same host only (www. is ignored).", "default": false }, "renderingMode": { "title": "Rendering", "enum": [ "http", "auto", "browser" ], "type": "string", "description": "'http' is fast and cheap (no JavaScript). 'auto' uses a headless browser only when a page looks like a JavaScript app shell (charged as page-converted-browser). 'browser' renders every page. Bot challenges are never 'solved' in any mode.", "default": "http" }, "extractionMode": { "title": "Main-content extraction", "enum": [ "auto", "readability", "main-element", "full-page" ], "type": "string", "description": "'auto' (v2 engine) runs Mozilla Readability, then maps its picks back onto the page to recover paragraphs it dropped (after images, forum post headers) and trims related-content rails. Highest measured F1 in BENCHMARK.md. 'full-page' keeps everything except navigation, header, footer and sidebars.", "default": "auto" }, "contentSelector": { "title": "Content CSS selector (optional)", "type": "string", "description": "If set and found on the page, only these elements are converted, e.g. 'article .post-body' or '#docs-content'." }, "removeSelectors": { "title": "Remove elements (CSS selectors)", "type": "array", "description": "Extra elements to strip before extraction, e.g. '.newsletter-signup', '#related'. Scripts, cookie banners, dialogs and hidden elements are always removed.", "default": [], "items": { "type": "string" } }, "removeReferenceLists": { "title": "Strip citation noise (Wikipedia [1] markers, reference lists, navboxes)", "type": "boolean", "description": "Removes MediaWiki-style footnote markers, reference lists, navboxes, hatnotes and maintenance banners. Saves tokens on wiki pages; turn off if you need the bibliography.", "default": true }, "includeLinksInMarkdown": { "title": "Keep hyperlinks in Markdown", "type": "boolean", "description": "Off = link text only (fewer tokens). Links are always listed separately in 'links'.", "default": true }, "includeImagesInMarkdown": { "title": "Keep images in Markdown", "type": "boolean", "description": "Off = no ![alt](src) lines. Decorative images (empty alt), tracking pixels and repeated gallery images are always left out of the Markdown; every image is still listed in 'images'.", "default": true }, "includePlainText": { "title": "Also output plain text", "type": "boolean", "description": "Adds a 'text' field (Markdown syntax stripped).", "default": true }, "includeJsonLd": { "title": "Include JSON-LD structured data", "type": "boolean", "description": "Adds parsed schema.org JSON-LD blocks (@graph flattened).", "default": true }, "convertDocuments": { "title": "Convert PDF and DOCX", "type": "boolean", "description": "PDF (text layer, per-page citations) and Word .docx URLs are converted to Markdown and charged as 'document-converted'. Scanned PDFs without text are reported, not charged (no OCR). Max 25 MB / 500 pages.", "default": true }, "followDocumentLinks": { "title": "Follow links to PDF / DOCX while crawling", "type": "boolean", "description": "Crawl and sitemap modes: also queue in-scope .pdf / .docx links (they count toward Max pages).", "default": false }, "minWords": { "title": "Minimum words", "minimum": 0, "maximum": 5000, "type": "integer", "description": "Pages with fewer extracted words get status 'too-short' and are not charged (content is still included).", "default": 20 }, "dedupe": { "title": "De-duplicate", "enum": [ "canonical-and-hash", "hash", "none" ], "type": "string", "description": "Skip pages whose canonical URL or normalised text hash was already converted in this run (status 'duplicate', not charged).", "default": "canonical-and-hash" }, "enableChunking": { "title": "Split into RAG chunks", "type": "boolean", "description": "Heading-aware chunks with source URL, heading path and deep-link anchor. Code blocks and tables are kept whole when they fit; oversized tables repeat their header row.", "default": true }, "chunkPreset": { "title": "Embedding preset", "enum": [ "custom", "openai-text-embedding-3", "openai-large-context", "cohere-embed-v3", "voyage-claude-rag", "small-open-model" ], "type": "string", "description": "Sets chunk size, overlap and token counter for your embedding model. OpenAI presets count tokens EXACTLY (tiktoken cl100k_base). Cohere / Claude-RAG / 512-token open models use cl100k with a safety margin. 'custom' uses the fields below.", "default": "custom" }, "chunkSize": { "title": "Chunk size (tokens)", "minimum": 64, "maximum": 8192, "type": "integer", "description": "Target maximum tokens per chunk (custom preset), counted with the token counter below.", "default": 512 }, "chunkOverlap": { "title": "Chunk overlap (tokens)", "minimum": 0, "maximum": 1024, "type": "integer", "description": "Tokens repeated from the end of the previous chunk (whole sentences) when a section is split. Not applied across headings. Max half the chunk size.", "default": 64 }, "tokenizer": { "title": "Token counter (custom preset)", "enum": [ "cl100k", "o200k", "estimate" ], "type": "string", "description": "cl100k = exact OpenAI text-embedding-3 / GPT-4 counts; o200k = GPT-4o family; estimate = fast heuristic.", "default": "cl100k" }, "prependHeadingPath": { "title": "Prefix chunks with page title > heading path", "type": "boolean", "description": "Adds a breadcrumb line like 'Guide > Install > Linux' to each chunk's text. Improves retrieval; the path is also in 'headingPath'.", "default": true }, "generateLlmsTxt": { "title": "Generate llms.txt", "type": "boolean", "description": "Builds an llms.txt (llmstxt.org format) and llms-full.txt from the converted pages and saves them to the key-value store. Best with mode 'sitemap' or 'crawl'.", "default": false }, "monitoringMode": { "title": "Monitoring: only output new / changed pages", "type": "boolean", "description": "Remembers every page between runs (named store per Monitor ID). Next runs send If-None-Match / If-Modified-Since, skip sitemap URLs whose lastmod did not move, and output only new and changed pages with a section-level change summary. Unchanged pages are NEVER charged. Schedule the Actor to keep a RAG index fresh.", "default": false }, "monitorId": { "title": "Monitor ID", "type": "string", "description": "Name of this monitor's memory (e.g. 'docs-site'). Use a different ID per site or URL list.", "default": "default" }, "outputUnchanged": { "title": "Also output unchanged pages", "type": "boolean", "description": "Adds a free row with changeStatus 'unchanged' (no content).", "default": false }, "useSitemapLastmod": { "title": "Trust sitemap (sitemap delta)", "type": "boolean", "description": "Sitemap mode: a known URL whose is identical to last run is not downloaded at all.", "default": true }, "reportRemoved": { "title": "Report removed pages", "type": "boolean", "description": "URLs and sitemap mode: pages known from earlier runs that are gone get a free 'removed' row (only when this run covered the full list).", "default": true }, "webhookUrl": { "title": "Change alert webhook (HTTPS)", "type": "string", "description": "Optional. POSTs a JSON summary with a Slack-compatible 'text' field when pages are new / changed / removed. HTTPS only, public hosts only." }, "webhookOnNoChanges": { "title": "Send webhook even when nothing changed", "type": "boolean", "description": "Heartbeat mode.", "default": false }, "aiEnrichment": { "title": "AI summary / Q&A per page (your own key)", "enum": [ "none", "summary", "qa", "summary+qa" ], "type": "string", "description": "Off by default. Uses YOUR Anthropic or OpenAI key to add a summary, key points and grounded question/answer pairs to each converted page (field 'ai'). Charged as 'ai-enrichment' only when the model returns valid output; your provider bills its tokens to you.", "default": "none" }, "aiProvider": { "title": "AI provider", "enum": [ "anthropic", "openai" ], "type": "string", "description": "Which API your key belongs to.", "default": "anthropic" }, "aiApiKey": { "title": "Your AI API key", "type": "string", "description": "Stored encrypted by Apify as a secret input. Sent only to the chosen provider; never logged or written to the dataset." }, "aiModel": { "title": "Model (optional)", "type": "string", "description": "Default: claude-sonnet-5-5 (Anthropic) or gpt-4o-mini (OpenAI). For cheaper bulk runs choose e.g. claude-haiku-5-5." }, "aiQaPairs": { "title": "Q&A pairs per page", "minimum": 1, "maximum": 20, "type": "integer", "description": "Upper limit; short pages may yield fewer.", "default": 5 }, "aiMaxInputChars": { "title": "Max page characters sent to the model", "minimum": 2000, "maximum": 400000, "type": "integer", "description": "Longer pages are truncated and flagged ai.inputTruncated.", "default": 60000 }, "aiLanguage": { "title": "Output language (optional)", "type": "string", "description": "Default: the page's language." }, "robotsUserAgent": { "title": "robots.txt user-agent token", "type": "string", "description": "We always obey robots.txt rules for 'SiteVerdictBot' (and '*'). You may add your own bot name: its rules then apply too, so it can only add restrictions. Other crawlers' names (Googlebot, GPTBot, ...) are refused.", "default": "SiteVerdictBot" }, "perHostDelayMs": { "title": "Delay between requests to the same site (ms)", "minimum": 250, "maximum": 60000, "type": "integer", "description": "Minimum spacing per host. A longer robots.txt Crawl-delay (up to 30 s) is honoured automatically.", "default": 1000 }, "maxConcurrency": { "title": "Max concurrency", "minimum": 1, "maximum": 50, "type": "integer", "description": "Pages processed in parallel across all sites (browser mode is capped at 8).", "default": 5 }, "requestTimeoutSecs": { "title": "Request timeout (seconds)", "minimum": 5, "maximum": 120, "type": "integer", "description": "Per request / page render.", "default": 30 }, "maxRetries": { "title": "Retries", "minimum": 0, "maximum": 5, "type": "integer", "description": "Retries for network errors, 429 and 5xx (with backoff and Retry-After). 401/403 are never retried.", "default": 2 }, "acceptLanguage": { "title": "Accept-Language header (optional)", "type": "string", "description": "e.g. 'en-GB,en;q=0.9' to request a language version where sites support it." }, "outputSkippedPages": { "title": "Output blocked / failed pages too", "type": "boolean", "description": "Adds a row with status and reason (robots-txt, bot-protection, paywalled, login-required, ai-opt-out, duplicate, error...) and no content. Never charged.", "default": true } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }