{
"openapi": "3.0.1",
"info": {
"title": "Website Content Crawler for AI & RAG: Markdown, $0.40/1K",
"description": "Crawl any website into clean text and Markdown for LLMs, RAG and search. Same input and output fields as Apify Website Content Crawler. Respects robots.txt. No browser, so fast and cheap: $0.40 per 1,000 pages.",
"version": "0.0",
"x-build-id": "HiDFJ6rBZrbCiF4mn"
},
"servers": [
{
"url": "https://api.apify.com/v2"
}
],
"paths": {
"/acts/leadgenguy~ai-website-content-crawler/run-sync-get-dataset-items": {
"post": {
"operationId": "run-sync-get-dataset-items-leadgenguy-ai-website-content-crawler",
"x-openai-isConsequential": false,
"summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
"tags": [
"Run Actor"
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/inputSchema"
}
}
}
},
"parameters": [
{
"name": "token",
"in": "query",
"required": true,
"schema": {
"type": "string"
},
"description": "Enter your Apify token here"
}
],
"responses": {
"200": {
"description": "OK"
}
}
}
},
"/acts/leadgenguy~ai-website-content-crawler/runs": {
"post": {
"operationId": "runs-sync-leadgenguy-ai-website-content-crawler",
"x-openai-isConsequential": false,
"summary": "Executes an Actor and returns information about the initiated run in response.",
"tags": [
"Run Actor"
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/inputSchema"
}
}
}
},
"parameters": [
{
"name": "token",
"in": "query",
"required": true,
"schema": {
"type": "string"
},
"description": "Enter your Apify token here"
}
],
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/runsResponseSchema"
}
}
}
}
}
}
},
"/acts/leadgenguy~ai-website-content-crawler/run-sync": {
"post": {
"operationId": "run-sync-leadgenguy-ai-website-content-crawler",
"x-openai-isConsequential": false,
"summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
"tags": [
"Run Actor"
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/inputSchema"
}
}
}
},
"parameters": [
{
"name": "token",
"in": "query",
"required": true,
"schema": {
"type": "string"
},
"description": "Enter your Apify token here"
}
],
"responses": {
"200": {
"description": "OK"
}
}
}
}
},
"components": {
"schemas": {
"inputSchema": {
"type": "object",
"required": [
"startUrls"
],
"properties": {
"startUrls": {
"title": "Start URLs",
"type": "array",
"description": "Web addresses to start from, e.g. https://docs.example.com. Links on the same site are followed.",
"items": {
"type": "object",
"required": [
"url"
],
"properties": {
"url": {
"type": "string",
"title": "URL of a web page",
"format": "uri"
}
}
}
},
"maxCrawlPages": {
"title": "Max pages",
"minimum": 1,
"maximum": 1000000,
"type": "integer",
"description": "Stop after saving this many pages. Free Apify plans: 25 per run.",
"default": 10
},
"maxCrawlDepth": {
"title": "Max crawling depth",
"minimum": 0,
"maximum": 100,
"type": "integer",
"description": "How many link-hops from the start URLs. 0 saves only the start URLs.",
"default": 20
},
"includeUrlGlobs": {
"title": "Include URLs (globs)",
"type": "array",
"description": "Only follow links matching these patterns, e.g. https://example.com/docs/**. Empty means the whole site.",
"default": [],
"items": {
"type": "object",
"required": [
"glob"
],
"properties": {
"glob": {
"type": "string",
"title": "Glob of a web page"
}
}
}
},
"excludeUrlGlobs": {
"title": "Exclude URLs (globs)",
"type": "array",
"description": "Never follow links matching these patterns, e.g. https://example.com/**/archive/**.",
"default": [],
"items": {
"type": "object",
"required": [
"glob"
],
"properties": {
"glob": {
"type": "string",
"title": "Glob of a web page"
}
}
}
},
"crawlerType": {
"title": "Crawler type",
"enum": [
"cheerio"
],
"type": "string",
"description": "This crawler reads the HTML the server sends (fast and cheap). Pages that only build their content with JavaScript are not supported.",
"default": "cheerio"
},
"removeElementsCssSelector": {
"title": "Remove elements (CSS selector)",
"type": "string",
"description": "Parts of the page to drop before extracting text. Leave empty for a sensible default (navigation, footers, cookie banners, ads, forms)."
},
"saveMarkdown": {
"title": "Save Markdown",
"type": "boolean",
"description": "Add the page content as Markdown (headings, lists, links, code and tables kept).",
"default": true
},
"saveHtml": {
"title": "Save cleaned HTML",
"type": "boolean",
"description": "Also add the cleaned main-content HTML.",
"default": false
},
"useSitemaps": {
"title": "Use sitemaps",
"type": "boolean",
"description": "Also discover pages from the site's sitemap.",
"default": false
},
"respectRobotsTxtFile": {
"title": "Respect robots.txt",
"type": "boolean",
"description": "Skip pages the site asks crawlers not to visit. Recommended.",
"default": true
},
"maxConcurrency": {
"title": "Max concurrency",
"minimum": 1,
"maximum": 32,
"type": "integer",
"description": "Pages fetched at the same time. Lower it to be gentler on small sites.",
"default": 8
},
"proxyConfiguration": {
"title": "Proxy configuration",
"type": "object",
"description": "Off by default. Turn on Apify Proxy for sites that block direct requests.",
"default": {
"useApifyProxy": false
}
}
}
},
"runsResponseSchema": {
"type": "object",
"properties": {
"data": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"actId": {
"type": "string"
},
"userId": {
"type": "string"
},
"startedAt": {
"type": "string",
"format": "date-time",
"example": "2025-01-08T00:00:00.000Z"
},
"finishedAt": {
"type": "string",
"format": "date-time",
"example": "2025-01-08T00:00:00.000Z"
},
"status": {
"type": "string",
"example": "READY"
},
"meta": {
"type": "object",
"properties": {
"origin": {
"type": "string",
"example": "API"
},
"userAgent": {
"type": "string"
}
}
},
"stats": {
"type": "object",
"properties": {
"inputBodyLen": {
"type": "integer",
"example": 2000
},
"rebootCount": {
"type": "integer",
"example": 0
},
"restartCount": {
"type": "integer",
"example": 0
},
"resurrectCount": {
"type": "integer",
"example": 0
},
"computeUnits": {
"type": "integer",
"example": 0
}
}
},
"options": {
"type": "object",
"properties": {
"build": {
"type": "string",
"example": "latest"
},
"timeoutSecs": {
"type": "integer",
"example": 300
},
"memoryMbytes": {
"type": "integer",
"example": 1024
},
"diskMbytes": {
"type": "integer",
"example": 2048
}
}
},
"buildId": {
"type": "string"
},
"defaultKeyValueStoreId": {
"type": "string"
},
"defaultDatasetId": {
"type": "string"
},
"defaultRequestQueueId": {
"type": "string"
},
"buildNumber": {
"type": "string",
"example": "1.0.0"
},
"containerUrl": {
"type": "string"
},
"usage": {
"type": "object",
"properties": {
"ACTOR_COMPUTE_UNITS": {
"type": "integer",
"example": 0
},
"DATASET_READS": {
"type": "integer",
"example": 0
},
"DATASET_WRITES": {
"type": "integer",
"example": 0
},
"KEY_VALUE_STORE_READS": {
"type": "integer",
"example": 0
},
"KEY_VALUE_STORE_WRITES": {
"type": "integer",
"example": 1
},
"KEY_VALUE_STORE_LISTS": {
"type": "integer",
"example": 0
},
"REQUEST_QUEUE_READS": {
"type": "integer",
"example": 0
},
"REQUEST_QUEUE_WRITES": {
"type": "integer",
"example": 0
},
"DATA_TRANSFER_INTERNAL_GBYTES": {
"type": "integer",
"example": 0
},
"DATA_TRANSFER_EXTERNAL_GBYTES": {
"type": "integer",
"example": 0
},
"PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
"type": "integer",
"example": 0
},
"PROXY_SERPS": {
"type": "integer",
"example": 0
},
"PROXY_UNBLOCKER_UNITS": {
"type": "integer",
"example": 0
}
}
},
"usageTotalUsd": {
"type": "number",
"example": 0.00005
},
"usageUsd": {
"type": "object",
"properties": {
"ACTOR_COMPUTE_UNITS": {
"type": "integer",
"example": 0
},
"DATASET_READS": {
"type": "integer",
"example": 0
},
"DATASET_WRITES": {
"type": "integer",
"example": 0
},
"KEY_VALUE_STORE_READS": {
"type": "integer",
"example": 0
},
"KEY_VALUE_STORE_WRITES": {
"type": "number",
"example": 0.00005
},
"KEY_VALUE_STORE_LISTS": {
"type": "integer",
"example": 0
},
"REQUEST_QUEUE_READS": {
"type": "integer",
"example": 0
},
"REQUEST_QUEUE_WRITES": {
"type": "integer",
"example": 0
},
"DATA_TRANSFER_INTERNAL_GBYTES": {
"type": "integer",
"example": 0
},
"DATA_TRANSFER_EXTERNAL_GBYTES": {
"type": "integer",
"example": 0
},
"PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
"type": "integer",
"example": 0
},
"PROXY_SERPS": {
"type": "integer",
"example": 0
},
"PROXY_UNBLOCKER_UNITS": {
"type": "integer",
"example": 0
}
}
}
}
}
}
}
}
}
}