{
  "openapi": "3.0.1",
  "info": {
    "title": "Website Content Crawler for AI - Markdown & RAG Chunks",
    "description": "Crawl a website to clean Markdown, text and token-counted RAG chunks for LLMs, vector DBs, n8n and Make. Sitemaps, robots.txt, JS pages, linked PDF/DOCX, llms.txt, change-only re-crawls. Flat price per page.",
    "version": "1.0",
    "x-build-id": "wm6n4GWBewOsDllA3"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/kantolabs~website-content-crawler-ai/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-kantolabs-website-content-crawler-ai",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/kantolabs~website-content-crawler-ai/runs": {
      "post": {
        "operationId": "runs-sync-kantolabs-website-content-crawler-ai",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/kantolabs~website-content-crawler-ai/run-sync": {
      "post": {
        "operationId": "run-sync-kantolabs-website-content-crawler-ai",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "required": [
          "startUrls"
        ],
        "properties": {
          "startUrls": {
            "title": "Start URLs",
            "type": "array",
            "description": "Where the crawl starts, usually the top of a docs site, blog or help centre. Pages under the same path are followed (https://example.com/blog/ crawls /blog/...). A start URL that is a PDF/DOCX/PPTX/XLSX file is converted as a document.",
            "default": [
              {
                "url": "https://www.python-httpx.org/"
              }
            ],
            "items": {
              "type": "object",
              "required": [
                "url"
              ],
              "properties": {
                "url": {
                  "type": "string",
                  "title": "URL of a web page",
                  "format": "uri"
                }
              }
            }
          },
          "crawlerType": {
            "title": "Crawler type",
            "enum": [
              "auto",
              "http",
              "browser"
            ],
            "type": "string",
            "description": "auto (default): fast plain HTTP, and only pages that need JavaScript are re-rendered in headless Chromium (billed as Browser page). http: never use the browser (cheapest). browser: render every page in headless Chromium.",
            "default": "auto"
          },
          "maxPages": {
            "title": "Max pages",
            "minimum": 1,
            "maximum": 1000000,
            "type": "integer",
            "description": "Most web pages fetched in this run (start URLs, linked and sitemap pages). Linked documents have their own limit.",
            "default": 100
          },
          "maxCrawlDepth": {
            "title": "Max crawl depth",
            "minimum": 0,
            "maximum": 100,
            "type": "integer",
            "description": "How many links away from a start URL to go. 0 = only the start URLs; sitemap pages count as depth 1.",
            "default": 20
          },
          "crawlScope": {
            "title": "Crawl scope",
            "enum": [
              "startUrlPath",
              "sameDomain",
              "sameDomainAndSubdomains"
            ],
            "type": "string",
            "description": "Which links to follow when no include globs are set. startUrlPath (default): same host and under the start URL's folder. sameDomain: the whole host. sameDomainAndSubdomains: also docs./blog./www. subdomains.",
            "default": "startUrlPath"
          },
          "includeUrlGlobs": {
            "title": "Include URLs (globs)",
            "type": "array",
            "description": "If set, only links matching one of these globs are followed (replaces the crawl scope). ** = anything, * = anything except /, {a,b} = either. Example: https://example.com/docs/**. Write \\? for a literal ?.",
            "default": [],
            "items": {
              "type": "string"
            }
          },
          "excludeUrlGlobs": {
            "title": "Exclude URLs (globs)",
            "type": "array",
            "description": "Links matching any of these globs are never crawled (start URLs always are). Example: https://example.com/**/changelog/** or https://example.com/**\\?page=*.",
            "default": [],
            "items": {
              "type": "string"
            }
          },
          "useSitemaps": {
            "title": "Use sitemaps",
            "type": "boolean",
            "description": "Read the site's sitemaps (from robots.txt, /sitemap.xml, /sitemap_index.xml) to find in-scope pages that are not linked.",
            "default": true
          },
          "respectRobotsTxt": {
            "title": "Respect robots.txt",
            "type": "boolean",
            "description": "On by default: pages that robots.txt disallows for crawlers are skipped, and Crawl-delay is honoured (max 10 s). Turn off ONLY for a website you own or have written permission to crawl.",
            "default": true
          },
          "chunkSize": {
            "title": "RAG chunk size (tokens)",
            "minimum": 0,
            "maximum": 8000,
            "type": "integer",
            "description": "Split each page's Markdown into heading-aware chunks of at most this many tokens (OpenAI cl100k_base), each with URL and heading path. 0 = no chunks.",
            "default": 512
          },
          "chunkOverlap": {
            "title": "Chunk overlap (tokens)",
            "minimum": 0,
            "maximum": 4000,
            "type": "integer",
            "description": "Tokens of the previous chunk repeated at the start of the next one (cut at a sentence boundary when possible). At most half the chunk size.",
            "default": 64
          },
          "convertLinkedDocuments": {
            "title": "Convert linked documents (PDF, DOCX, PPTX, XLSX)",
            "type": "boolean",
            "description": "Also download PDF, Word, PowerPoint and Excel files linked from crawled pages (on any host, exclude globs still apply) and convert them to Markdown + chunks. Billed per document. Scanned PDFs without a text layer give a free error row (no OCR).",
            "default": false
          },
          "maxDocuments": {
            "title": "Max linked documents",
            "minimum": 0,
            "maximum": 10000,
            "type": "integer",
            "description": "Most linked documents converted in this run.",
            "default": 50
          },
          "maxDocumentSizeMb": {
            "title": "Max document size (MB)",
            "minimum": 1,
            "maximum": 200,
            "type": "integer",
            "description": "Larger linked files are skipped with a free error row.",
            "default": 25
          },
          "previousDatasetId": {
            "title": "Only changed pages since this dataset (re-crawl)",
            "type": "string",
            "description": "Pick the dataset of an earlier run of this crawler. Pages whose content hash is the same (or that answer 304 Not Modified) become free 'unchanged' rows; you pay only for new and changed pages."
          },
          "removeDuplicates": {
            "title": "Skip duplicate and near-identical pages",
            "type": "boolean",
            "description": "Pages with the same canonical URL, identical text or near-identical text (SimHash, e.g. print views, tag/sort variants) become free 'duplicate' rows pointing to the first copy.",
            "default": true
          },
          "saveLlmsTxt": {
            "title": "Save llms.txt + full-site Markdown",
            "type": "boolean",
            "description": "Write llms.txt (page index), llms-full.txt (every page's Markdown in one file) and chunks.jsonl (all chunks, one JSON per line) to the run's key-value store. Links are in the OUTPUT record.",
            "default": true
          },
          "saveMarkdown": {
            "title": "Include Markdown",
            "type": "boolean",
            "description": "Put the page's main content as Markdown in each row.",
            "default": true
          },
          "saveText": {
            "title": "Include plain text",
            "type": "boolean",
            "description": "Put the main content as plain text (no Markdown syntax) in each row.",
            "default": true
          },
          "keepLinks": {
            "title": "Keep links in Markdown",
            "type": "boolean",
            "description": "Keep [text](url) links in the Markdown. Turn off for leaner LLM input (link text is kept).",
            "default": true
          },
          "keepImages": {
            "title": "Keep images in Markdown",
            "type": "boolean",
            "description": "Keep ![alt](url) image references in the Markdown.",
            "default": false
          },
          "removeElementsCssSelector": {
            "title": "Extra elements to remove (CSS selector)",
            "type": "string",
            "description": "Optional. Elements removed on top of the built-in cleaning (navigation, headers, footers, sidebars, cookie banners, ads, share widgets). Example: .promo, #newsletter-box"
          },
          "contentCssSelector": {
            "title": "Main content (CSS selector)",
            "type": "string",
            "description": "Optional. Keep only the elements matching this selector (e.g. article.post-body). Empty = automatic main-content detection."
          },
          "maxConcurrency": {
            "title": "Max concurrency",
            "minimum": 1,
            "maximum": 50,
            "type": "integer",
            "description": "Pages fetched in parallel. Browser pages run fewer at a time (about one per 700 MB of memory).",
            "default": 10
          },
          "requestTimeoutSecs": {
            "title": "Page timeout (s)",
            "minimum": 5,
            "maximum": 180,
            "type": "integer",
            "description": "Per-request timeout.",
            "default": 30
          },
          "maxRetries": {
            "title": "Retries",
            "minimum": 0,
            "maximum": 5,
            "type": "integer",
            "description": "Retries on network errors, 5xx and 429 answers. Blocked pages (bot checks, 401/403) are not retried.",
            "default": 2
          },
          "browserWaitSecs": {
            "title": "Browser: wait for content (s)",
            "minimum": 0,
            "maximum": 30,
            "type": "integer",
            "description": "Headless browser only: how long to wait for network quiet after the page loads.",
            "default": 3
          },
          "maxPageSizeMb": {
            "title": "Max page size (MB)",
            "minimum": 1,
            "maximum": 50,
            "type": "integer",
            "description": "HTML pages larger than this are skipped with a free error row.",
            "default": 10
          },
          "acceptLanguage": {
            "title": "Accept-Language header",
            "type": "string",
            "description": "Language the crawler asks for on multilingual sites, e.g. de-DE,de;q=0.9.",
            "default": "en-US,en;q=0.8,*;q=0.5"
          },
          "proxyConfiguration": {
            "title": "Proxy",
            "type": "object",
            "description": "Optional. No proxy by default. Apify datacenter proxy can be used; residential proxy groups are not used.",
            "default": {
              "useApifyProxy": false
            }
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}