{
  "openapi": "3.0.1",
  "info": {
    "title": "Website Content Crawler to Markdown for AI & LLMs",
    "description": "Website content crawler for AI, RAG and LLMs: give start URLs or a sitemap, get every page's clean Markdown and text with title, description, headings, author, dates, canonical URL and links. Fast HTTP crawling, URL globs, depth and page limits, sitemaps, llms.txt and PDFs. No login.",
    "version": "0.1",
    "x-build-id": "hWEuHyZqlMESCmgS0"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/deepmine~website-content-crawler/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-deepmine-website-content-crawler",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/deepmine~website-content-crawler/runs": {
      "post": {
        "operationId": "runs-sync-deepmine-website-content-crawler",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/deepmine~website-content-crawler/run-sync": {
      "post": {
        "operationId": "run-sync-deepmine-website-content-crawler",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "required": [
          "startUrls"
        ],
        "properties": {
          "startUrls": {
            "title": "🌐 Start URLs",
            "type": "array",
            "description": "Pages to start from. The crawler follows links to their sub-pages (same path), e.g. https://docs.example.com/guide. A sitemap.xml or llms.txt URL crawls every page it lists.",
            "items": {
              "type": "object",
              "required": [
                "url"
              ],
              "properties": {
                "url": {
                  "type": "string",
                  "title": "URL of a web page",
                  "format": "uri"
                }
              }
            }
          },
          "maxResults": {
            "title": "💯 Max pages",
            "minimum": 1,
            "maximum": 9999999,
            "type": "integer",
            "description": "Most pages to save (each one is charged once). Empty = the whole site."
          },
          "useSitemaps": {
            "title": "🗺️ Also crawl the sitemap",
            "type": "boolean",
            "description": "Add the pages listed in each site's sitemap (from robots.txt or /sitemap.xml) that fall under your start URLs, to find pages no link points to.",
            "default": false
          },
          "maxCrawlDepth": {
            "title": "🔗 Max link depth",
            "minimum": 0,
            "maximum": 1000,
            "type": "integer",
            "description": "How many links away from a start URL to go: 0 = only the start URLs, 1 = pages they link to, and so on.",
            "default": 20
          },
          "includeUrlGlobs": {
            "title": "Include URL patterns (globs)",
            "type": "array",
            "description": "Also crawl links that match, e.g. https://example.com/blog/** (** = anything, * = anything but /). Empty = only sub-pages of the start URLs.",
            "default": [],
            "items": {
              "type": "object",
              "required": [
                "glob"
              ],
              "properties": {
                "glob": {
                  "type": "string",
                  "title": "Glob of a web page"
                }
              }
            }
          },
          "excludeUrlGlobs": {
            "title": "Exclude URL patterns (globs)",
            "type": "array",
            "description": "Never crawl links that match, e.g. https://example.com/**/tag/** or **?page=*. Start URLs are always crawled.",
            "default": [],
            "items": {
              "type": "object",
              "required": [
                "glob"
              ],
              "properties": {
                "glob": {
                  "type": "string",
                  "title": "Glob of a web page"
                }
              }
            }
          },
          "maxCrawlPages": {
            "title": "Max URLs to fetch",
            "minimum": 1,
            "maximum": 9999999,
            "type": "integer",
            "description": "Most URLs to request, including ones that fail or are skipped, as a safety stop on huge sites. Empty = no limit."
          },
          "useLlmsTxt": {
            "title": "Crawl /llms.txt and its Markdown pages",
            "type": "boolean",
            "description": "Also read each site's /llms.txt (an index for AI tools) and the pages it links to.",
            "default": false
          },
          "onlyCrawlPagesModifiedSince": {
            "title": "Only pages changed since",
            "type": "string",
            "description": "Save only pages updated after this date (from the sitemap, the page's own dates or the server), for refresh runs, e.g. 7 days. Empty = every page."
          },
          "respectRobotsTxtFile": {
            "title": "Respect robots.txt",
            "type": "boolean",
            "description": "Skip URLs the site's robots.txt disallows for all crawlers (listed as free robotsTxtDisallowed rows in the errors dataset).",
            "default": false
          },
          "includeFiles": {
            "title": "Include files (PDF, Word, Excel, CSV)",
            "type": "boolean",
            "description": "Also save linked documents: PDFs as rows with their text; Word, Excel, PowerPoint, CSV, RTF and EPUB files stored and linked in the fileUrl column (text files with their text). Off = files are skipped.",
            "default": false
          },
          "saveContentTypes": {
            "title": "File types to save",
            "type": "string",
            "description": "Save linked files of these content types, comma separated, wildcards allowed (image/*, text/csv). One row per file with its fileUrl, even when the file is outside your start URLs. Empty = Include files' documents."
          },
          "renderingMode": {
            "title": "Rendering",
            "enum": [
              "auto",
              "http",
              "browser"
            ],
            "type": "string",
            "description": "Auto renders only JavaScript app shells in headless Chrome (renderedInBrowser). Browser compares each site's first pages with their HTML and reads the site over HTTP when the text is the same; screenshots render every page.",
            "default": "auto"
          },
          "saveScreenshots": {
            "title": "Save screenshots",
            "type": "boolean",
            "description": "Saves a screenshot (JPEG, top of the page) of each page rendered in the browser, linked in the screenshotUrl column. With Browser for every page, that's every page.",
            "default": false
          },
          "saveMarkdown": {
            "title": "Markdown",
            "type": "boolean",
            "description": "The page's main content as Markdown (headings, lists, tables, code, links) in the markdown column.",
            "default": true
          },
          "saveHtml": {
            "title": "Cleaned HTML in the dataset",
            "type": "boolean",
            "description": "Adds the html column: the main content's HTML after cleaning. Makes rows much larger.",
            "default": false
          },
          "saveHtmlAsFile": {
            "title": "Cleaned HTML as files",
            "type": "boolean",
            "description": "Saves each page's cleaned HTML to the run's key-value store and links it in the htmlUrl column.",
            "default": false
          },
          "keepUrlFragments": {
            "title": "URL #fragments are separate pages",
            "type": "boolean",
            "description": "Treat example.com/page#a and example.com/page#b as two pages (single-page apps with # routes). Off = one page.",
            "default": false
          },
          "ignoreCanonicalUrl": {
            "title": "Ignore canonical URLs",
            "type": "boolean",
            "description": "Off: pages that name the same canonical URL are saved once. On: every URL is its own page (for sites with a wrong canonical tag).",
            "default": false
          },
          "htmlTransformer": {
            "title": "Main content",
            "enum": [
              "readableText",
              "readableTextIfPossible",
              "none"
            ],
            "type": "string",
            "description": "How to pick the content to keep after the elements below are removed.",
            "default": "readableText"
          },
          "removeElementsCssSelector": {
            "title": "Remove HTML elements (CSS selector)",
            "type": "string",
            "description": "Elements to drop before extracting, e.g. .ads, #comments. The default removes navigation, footers, scripts, banners and dialogs.",
            "default": "nav, footer, script, style, noscript, svg, img[src^='data:'],\n[role=\"alert\"],\n[role=\"banner\"],\n[role=\"dialog\"],\n[role=\"alertdialog\"],\n[role=\"region\"][aria-label*=\"skip\" i],\n[aria-modal=\"true\"]"
          },
          "keepElementsCssSelector": {
            "title": "Keep only these elements (CSS selector)",
            "type": "string",
            "description": "Extract only matching elements, e.g. article, .post-body. Empty = the main content."
          },
          "proxyConfiguration": {
            "title": "Proxy",
            "type": "object",
            "description": "Leave as is: Apify datacenter proxy, fresh IPs on refusals. Pick residential only for sites that block datacenter IPs.",
            "default": {
              "useApifyProxy": true
            }
          },
          "crawlerType": {
            "title": "Crawler type",
            "type": "string",
            "description": "Kept for inputs saved from other crawlers: cheerio / jsdom = plain HTTP only; browser types work like Rendering: Auto."
          },
          "maxConcurrency": {
            "title": "Max concurrency",
            "minimum": 1,
            "maximum": 200,
            "type": "integer",
            "description": "Most requests at once (default: set from the run's memory)."
          },
          "requestTimeoutSecs": {
            "title": "Request timeout",
            "minimum": 5,
            "maximum": 300,
            "type": "integer",
            "description": "Seconds per request.",
            "default": 30
          },
          "maxRequestRetries": {
            "title": "Retries",
            "minimum": 0,
            "maximum": 10,
            "type": "integer",
            "description": "Tries on new IPs after a refusal or network error.",
            "default": 3
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}