{ "openapi": "3.0.1", "info": { "title": "Website to Markdown for RAG: Crawl & Changed Pages", "description": "Turn a website or docs section into clean Markdown, one row per page, for RAG and LLM pipelines. Stays inside your start path, respects robots.txt and maxPages exactly. Repeat runs can return only new and changed pages. No login.", "version": "0.1", "x-build-id": "B3s4EW9dLFUvwRYbj" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/cliqtomedia~website-to-markdown/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-cliqtomedia-website-to-markdown", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/cliqtomedia~website-to-markdown/runs": { "post": { "operationId": "runs-sync-cliqtomedia-website-to-markdown", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/cliqtomedia~website-to-markdown/run-sync": { "post": { "operationId": "run-sync-cliqtomedia-website-to-markdown", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "Public pages to start from (up to 50). By default only pages under the same path are crawled: https://docs.example.com/guide crawls /guide/… but not /blog/. Pages behind a login, private or internal addresses are not crawled.", "items": { "type": "string" } }, "maxPages": { "title": "Max pages", "minimum": 1, "maximum": 2000, "type": "integer", "description": "Stop after this many HTML pages (1–2000). The run never returns more pages than this.", "default": 50 }, "maxDepth": { "title": "Max link depth", "minimum": 0, "maximum": 20, "type": "integer", "description": "How many clicks away from a start URL to follow. 0 = only the start URLs.", "default": 5 }, "crawlScope": { "title": "Crawl scope", "enum": [ "path", "host", "domain" ], "type": "string", "description": "Which links to follow: under the start URL path (recommended for docs), anywhere on the same host, or on any subdomain of the same domain.", "default": "path" }, "useSitemap": { "title": "Use sitemap.xml", "type": "boolean", "description": "Also add in-scope URLs from the site's sitemap (from robots.txt or /sitemap.xml). Finds pages that are not linked. Sitemap dates fill lastModified.", "default": false }, "includeGlobs": { "title": "Include URL patterns", "type": "array", "description": "Only crawl URLs matching one of these patterns (full URL). * matches inside one path segment, ** matches anything. Example: https://docs.example.com/**/api/**. When set, replaces the path rule but stays on the start hosts. Pages in between (hubs) must match too, or the pages linked only from them are not found.", "items": { "type": "string" } }, "excludeGlobs": { "title": "Exclude URL patterns", "type": "array", "description": "Never crawl URLs matching these patterns. Example: **/changelog/** or **/*?page=*.", "items": { "type": "string" } }, "onlyChanged": { "title": "Only changed pages (delta)", "type": "boolean", "description": "Remember each page's content between runs and return only new and changed pages. Unchanged pages are not saved to the dataset and cost less. Pages from the previous run that now answer HTTP 404 or 410 are listed as 'removed' (free); pages the crawl no longer reaches by links are re-checked directly. The first run returns every page as 'new'.", "default": false }, "stateKey": { "title": "Delta state name", "type": "string", "description": "Optional name for the saved delta state (letters, digits, -_.). By default it is derived from the start URLs and patterns, so the same input always continues the same state. Use different names to keep separate histories." }, "removeSelectors": { "title": "Extra elements to remove", "type": "string", "description": "CSS selectors to remove before conversion, e.g. .ad, .related-posts. Navigation, headers, footers, sidebars and cookie banners are always removed." }, "includeLinks": { "title": "Include page links", "type": "boolean", "description": "Add the list of links found on each page, including links to other sites (up to 1000).", "default": false }, "maxConcurrency": { "title": "Parallel requests", "minimum": 1, "maximum": 10, "type": "integer", "description": "How many pages to load at the same time (1–10). The site's robots.txt Crawl-delay is always respected.", "default": 5 }, "proxyConfiguration": { "title": "Proxy", "type": "object", "description": "Usually not needed. Use a proxy only if the site blocks the default connection. Apify Proxy with automatic or datacenter groups, or your own proxy URLs. Residential and SERP proxy groups are not supported (the run fails with an input error).", "default": { "useApifyProxy": false } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }