{
  "openapi": "3.0.1",
  "info": {
    "title": "Website to Markdown Crawler - Content Crawler for LLM & RAG",
    "description": "Website to Markdown crawler for RAG and LLMs: crawl a website or docs site into clean Markdown (URL to Markdown, HTML to Markdown) with RAG chunks. Sitemaps, JavaScript pages, linked PDFs and Word. $1/1k pages.",
    "version": "0.1",
    "x-build-id": "SSKQ85If75HoCA0R4"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/tidytools~website-markdown-crawler/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-tidytools-website-markdown-crawler",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/tidytools~website-markdown-crawler/runs": {
      "post": {
        "operationId": "runs-sync-tidytools-website-markdown-crawler",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/tidytools~website-markdown-crawler/run-sync": {
      "post": {
        "operationId": "run-sync-tidytools-website-markdown-crawler",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "properties": {
          "urls": {
            "title": "Start URLs",
            "type": "array",
            "description": "Main input (fill this, or `startUrls`). Where to start crawling, one URL per line, e.g. the home page of a docs site. Bare domains (example.com) are fine; blank or invalid lines are skipped and listed in the results (not charged).",
            "items": {
              "type": "string"
            }
          },
          "crawlScope": {
            "title": "Crawl scope",
            "enum": [
              "path",
              "domain",
              "page"
            ],
            "type": "string",
            "description": "Which discovered links to follow. \"Same folder\" keeps the crawl under the start URL's path (best for docs); when the start page links to nothing in its own folder (docs sites that give every page its own folder, e.g. /en/getting-started/), the folder one level up is used for it, never the whole site. Values: path = Same folder as the start URL; domain = Whole website (incl. subdomains); page = Only the start URLs (no crawling).",
            "default": "path"
          },
          "maxPages": {
            "title": "Max pages",
            "minimum": 1,
            "maximum": 10000,
            "type": "integer",
            "description": "Stop after this many pages in total (all start URLs together). You pay only for pages converted successfully.",
            "default": 50
          },
          "maxPagesPerStartUrl": {
            "title": "Max pages per start URL",
            "minimum": 0,
            "maximum": 10000,
            "type": "integer",
            "description": "With several start URLs: stop crawling one site after this many pages, so every site gets its share of Max pages. 0 = no per-site limit.",
            "default": 0
          },
          "maxDepth": {
            "title": "Max link depth",
            "minimum": 0,
            "maximum": 20,
            "type": "integer",
            "description": "How many clicks away from a start URL to go. 0 = only the start URLs.",
            "default": 5
          },
          "useSitemap": {
            "title": "Use the site's sitemap",
            "enum": [
              "off",
              "seed",
              "only"
            ],
            "type": "string",
            "description": "Choose \"off\", \"seed\" (default) or \"only\", e.g. {\"useSitemap\":\"seed\"}. Off follows links only; seed queues sitemap.xml URLs alongside discovered links; only queues sitemap URLs without following links. Sitemap locations come from robots.txt or common paths. Ignored when crawl scope is \"Only the start URLs\". Content links are crawled before menu and sitemap URLs.",
            "default": "seed"
          },
          "respectRobotsTxt": {
            "title": "Respect robots.txt",
            "type": "boolean",
            "description": "Skip pages the site's robots.txt disallows for crawlers. Skipped pages are not output, not charged and don't count toward Max pages (see disallowedByRobotsTxt in the SUMMARY record).",
            "default": true
          },
          "mode": {
            "title": "Page fetching (HTTP or browser)",
            "enum": [
              "auto",
              "fast",
              "browser"
            ],
            "type": "string",
            "description": "\"Auto\" uses the cheap fast mode and switches to a real browser only when a page blocks it or needs JavaScript. \"Fast\" never uses a browser; \"Browser\" always does. Values: auto = Auto (recommended); fast = Fast (HTTP only); browser = Browser (JavaScript rendering).",
            "default": "auto"
          },
          "cssSelector": {
            "title": "Content selector (CSS)",
            "type": "string",
            "description": "Optional. Convert only this part of each page, e.g. \"main\" or \"article\", to drop menus and footers."
          },
          "includeUrlPatterns": {
            "title": "Include URL patterns",
            "type": "array",
            "description": "Optional glob patterns; only matching URLs are crawled. * matches within a path segment, ** matches anything. Example: https://example.com/blog/**",
            "items": {
              "type": "string"
            }
          },
          "excludeUrlPatterns": {
            "title": "Exclude URL patterns",
            "type": "array",
            "description": "Optional glob patterns to skip, e.g. **/tag/** or **/login*",
            "items": {
              "type": "string"
            }
          },
          "includeDocuments": {
            "title": "Convert linked documents",
            "type": "boolean",
            "description": "Also convert linked PDF, Word, Excel and CSV files to Markdown.",
            "default": true
          },
          "outputFormats": {
            "title": "Output formats",
            "type": "array",
            "description": "Markdown keeps headings, links and tables. Plain text strips all formatting (good for NLP and word counts). Chunks are always made from the Markdown.",
            "items": {
              "type": "string",
              "enum": [
                "markdown",
                "text"
              ],
              "enumTitles": [
                "Markdown",
                "Plain text"
              ]
            },
            "default": [
              "markdown"
            ]
          },
          "chunkSize": {
            "title": "RAG chunk size (characters)",
            "minimum": 0,
            "maximum": 20000,
            "type": "integer",
            "description": "If set (e.g. 2000), each page also gets a \"chunks\" array split at paragraph boundaries, ready for embeddings. 0 = off.",
            "default": 0
          },
          "chunkOverlap": {
            "title": "Chunk overlap (characters)",
            "minimum": 0,
            "maximum": 5000,
            "type": "integer",
            "description": "Characters repeated between consecutive chunks to keep context.",
            "default": 200
          },
          "combinedFile": {
            "title": "Also save one combined Markdown file",
            "type": "boolean",
            "description": "Saves every converted page in crawl order as one Markdown file (llms-full.txt style), each page under a \"# Title\" header with a \"Source:\" line, as OUTPUT.md in the run's key-value store. Its link is in the SUMMARY record (combinedFileUrl). Very large crawls are split into 8 MB parts (OUTPUT-2.md, ...). Free: no extra charge. Set to false to skip it.",
            "default": true
          },
          "maxConcurrency": {
            "title": "Parallel pages",
            "minimum": 1,
            "maximum": 30,
            "type": "integer",
            "description": "How many pages are processed at the same time.",
            "default": 10
          },
          "httpVia": {
            "title": "Plain HTTP requests from",
            "enum": [
              "auto",
              "backend",
              "direct",
              "vm"
            ],
            "type": "string",
            "description": "Some sites block requests from data centers. Auto retries from a second network before falling back to a real browser. If both are refused, Auto also tries our second server (Oracle, different IP) before a real browser.",
            "default": "auto"
          },
          "requestHeaders": {
            "title": "Custom request headers (your own site only)",
            "type": "object",
            "description": "Optional JSON object of HTTP header names and values for a site you own or may access, e.g. {\"requestHeaders\":{\"Authorization\":\"Bearer <token>\"}}. Headers go only to start-URL hosts over a direct connection and are dropped on cross-host redirects. Values are encrypted and not logged. Protected pages use fast mode without browser fallback; this does not log in to third-party sites."
          },
          "proxyConfiguration": {
            "title": "Proxy (optional)",
            "type": "object",
            "description": "Only used for requests sent from Apify's network. Apify proxy usage is billed to your Apify account."
          },
          "removeBoilerplate": {
            "title": "Remove navigation, headers and footers",
            "type": "boolean",
            "description": "Recommended (on by default, also for API calls). Drops nav bars, site headers and footers, sidebars, breadcrumbs, cookie notices, \"Skip to main content\" links and table-of-contents widgets from the Markdown before it is saved. Links for crawling are still taken from the whole page. Set to false to keep the full page.",
            "default": true
          },
          "removeSelectors": {
            "title": "Also remove these elements (CSS selectors)",
            "type": "string",
            "description": "Optional comma-separated CSS selectors removed from the Markdown, e.g. \".related-posts, #comments, .ad\". Works with or without the option above."
          },
          "tidyMarkdown": {
            "title": "Tidy the Markdown",
            "type": "boolean",
            "description": "Removes leftover <!doctype> lines, empty headings, 1×1 tracking-pixel images, schema.org JSON-LD code blocks and exact repeated sections (e.g. a page rendered twice for desktop and mobile). Set to false to keep the converter output unchanged.",
            "default": true
          },
          "startUrls": {
            "title": "Start URLs (request list, optional)",
            "type": "array",
            "description": "Same as the list above, in Apify's request-list format (also accepts a link to a text file of URLs). Kept for older inputs; the plain list above is recommended, especially for API calls.",
            "items": {
              "type": "object",
              "required": [
                "url"
              ],
              "properties": {
                "url": {
                  "type": "string",
                  "title": "URL of a web page",
                  "format": "uri"
                }
              }
            }
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}