{ "openapi": "3.0.1", "info": { "title": "Website to AI / RAG Data Extractor", "description": "Turns any website into a clean, embedding-ready corpus. Strips navigation, footers and cookie banners, converts the real content to markdown, and splits it into overlapping chunks that each carry their own URL, title and metadata. Crawl by URL list, sitemap or link graph.", "version": "0.1", "x-build-id": "lBkS3IKx0cu5Hahqf" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/scrapyx~website-rag-extractor/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-scrapyx-website-rag-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/scrapyx~website-rag-extractor/runs": { "post": { "operationId": "runs-sync-scrapyx-website-rag-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/scrapyx~website-rag-extractor/run-sync": { "post": { "operationId": "run-sync-scrapyx-website-rag-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "Where to start. Must be absolute http(s) URLs — bare domains like 'example.com' are dropped during normalisation. In 'urls' mode these are the only pages fetched; in 'sitemap' mode their sitemaps are read; in 'crawl' mode they seed the link graph.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "discoveryMode": { "title": "Discovery mode", "enum": [ "urls", "sitemap", "crawl" ], "type": "string", "description": "How to decide which pages to fetch. 'urls' fetches exactly the start URLs and follows nothing. 'sitemap' reads the Sitemap: lines from robots.txt (falling back to /sitemap.xml) and follows nested sitemap indexes. 'crawl' walks internal links breadth-first up to Max crawl depth.", "default": "urls" }, "outputGranularity": { "title": "Output granularity", "enum": [ "chunk", "page", "both" ], "type": "string", "description": "What a dataset row is. 'chunk' emits RAG-ready chunks, each carrying its own URL, title and metadata so it can go straight into a vector store. 'page' emits one row per page with the full cleaned markdown. 'both' emits pages and chunks, which roughly doubles dataset size.", "default": "chunk" }, "maxPages": { "title": "Max pages", "minimum": 1, "maximum": 20000, "type": "integer", "description": "Hard cap on how many pages are fetched across the whole run. This is the main cost control — a sitemap or crawl can otherwise expand to tens of thousands of URLs.", "default": 50 }, "maxDepth": { "title": "Max crawl depth", "minimum": 0, "maximum": 10, "type": "integer", "description": "How many link hops from a start URL to follow. 0 means the start URLs only, 1 adds the pages they link to, and so on. Only used in 'crawl' mode.", "default": 2 }, "sameDomainOnly": { "title": "Stay on the same domain", "type": "boolean", "description": "In 'crawl' mode, only follow links whose host matches a start URL's host (ignoring a leading www.). Turning this off lets the crawl wander onto any site the pages link to, which is almost never what a RAG corpus wants.", "default": true }, "includeUrlPatterns": { "title": "Include URL patterns (regex)", "type": "array", "description": "If set, a URL is only fetched when at least one of these regular expressions matches it. Useful for pulling just the docs section out of a sitemap, e.g. '/docs/' or '^https://example\\.com/blog/'.", "items": { "type": "string" } }, "excludeUrlPatterns": { "title": "Exclude URL patterns (regex)", "type": "array", "description": "URLs matching any of these regular expressions are skipped. Applied before the include patterns. Good for dropping tag pages, paginated archives or login flows, e.g. '/tag/' or '\\?page='.", "items": { "type": "string" } }, "chunkSize": { "title": "Chunk size (characters)", "minimum": 200, "maximum": 20000, "type": "integer", "description": "Target chunk length in CHARACTERS, not tokens — a token count would depend on your tokenizer and be wrong for non-English text. Each chunk also reports an estimatedTokens figure. Chunks break at a paragraph, line or sentence boundary near the target rather than cutting mid-sentence.", "default": 2000 }, "chunkOverlap": { "title": "Chunk overlap (characters)", "minimum": 0, "maximum": 5000, "type": "integer", "description": "How many characters each chunk repeats from the end of the previous one, so a fact spanning a boundary is retrievable from either side. Must be smaller than the chunk size. Set to 0 to disable.", "default": 200 }, "includeRawHtml": { "title": "Include raw HTML", "type": "boolean", "description": "Attach each page's raw HTML to its PAGE row. Off by default because it multiplies dataset size several times over and a RAG pipeline consumes the cleaned text, not the markup.", "default": false }, "respectRobotsTxt": { "title": "Respect robots.txt", "type": "boolean", "description": "Check each host's robots.txt before fetching and skip anything it disallows, reporting the skip in the summary row. Leave this on unless you own the site or have written authorisation to crawl it. An unreachable robots.txt is treated as 'no policy published', which allows the fetch.", "default": true }, "maxConcurrency": { "title": "Max concurrent requests", "minimum": 1, "maximum": 30, "type": "integer", "description": "Upper bound on pages fetched at the same time. Lower it for small or slow sites — this actor points at hosts nobody profiled, and hammering one is both rude and a fast route to a block.", "default": 5 }, "minRequestInterval": { "title": "Min seconds between requests", "type": "string", "description": "Minimum delay between the START of consecutive requests. This, not concurrency, is the honest speed control: once the rate cap binds, extra concurrency buys nothing. Raise it if a site starts returning 429s.", "default": "0.2" }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "description": "Proxy used for every request. Datacenter Apify Proxy is the default here rather than Residential: most documentation sites, blogs and company websites have no bot mitigation, so residential rates would be wasted. Switch to Residential for hosts that block datacenter traffic.", "default": { "useApifyProxy": true } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } } }