{
  "openapi": "3.0.1",
  "info": {
    "title": "Website Content to Markdown Crawler for LLMs",
    "description": "Website content crawler: crawl a site and convert pages to clean Markdown for LLMs, RAG and AI training. HTTP only, no browser. Sitemap-first discovery, main content only, tables and code kept. Adds title, language, word count, token estimate, RAG chunks and llms.txt files.",
    "version": "0.1",
    "x-build-id": "0eCdhMKnJhyYMqIYM"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/maliksaid~website-to-markdown/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-maliksaid-website-to-markdown",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/maliksaid~website-to-markdown/runs": {
      "post": {
        "operationId": "runs-sync-maliksaid-website-to-markdown",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/maliksaid~website-to-markdown/run-sync": {
      "post": {
        "operationId": "run-sync-maliksaid-website-to-markdown",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "properties": {
          "startUrls": {
            "title": "Start URLs",
            "type": "array",
            "description": "Pages where the crawl starts. By default the crawl stays inside the folder of each start URL. For example, https://example.com/docs/intro crawls pages under https://example.com/docs/.",
            "default": [
              {
                "url": "https://crawlee.dev/js/docs/quick-start"
              }
            ],
            "items": {
              "type": "object",
              "required": [
                "url"
              ],
              "properties": {
                "url": {
                  "type": "string",
                  "title": "URL of a web page",
                  "format": "uri"
                }
              }
            }
          },
          "maxPages": {
            "title": "Maximum pages",
            "minimum": 0,
            "type": "integer",
            "description": "Stop after converting this many pages. The default keeps test runs short and cheap. Set 0 for no limit.",
            "default": 30
          },
          "scope": {
            "title": "Crawl scope",
            "enum": [
              "path",
              "domain",
              "subdomains"
            ],
            "type": "string",
            "description": "Which links the crawl follows. 'Same folder' stays under the start URL's folder. 'Same domain' covers the whole host (www. is ignored). 'Domain and subdomains' also covers blog.example.com and similar.",
            "default": "path"
          },
          "includeGlobs": {
            "title": "Include URL patterns",
            "type": "array",
            "description": "Optional. Crawl only URLs that match one of these patterns. They replace the crawl scope above. * matches any text except /, ** matches any text. Example: https://example.com/docs/**",
            "default": [],
            "items": {
              "type": "object",
              "required": [
                "glob"
              ],
              "properties": {
                "glob": {
                  "type": "string",
                  "title": "Glob of a web page"
                }
              }
            }
          },
          "excludeGlobs": {
            "title": "Exclude URL patterns",
            "type": "array",
            "description": "Optional. Skip URLs that match one of these patterns. Example: https://example.com/**/tag/**",
            "default": [],
            "items": {
              "type": "object",
              "required": [
                "glob"
              ],
              "properties": {
                "glob": {
                  "type": "string",
                  "title": "Glob of a web page"
                }
              }
            }
          },
          "useSitemaps": {
            "title": "Read sitemaps first",
            "type": "boolean",
            "description": "Read the site's XML sitemaps (from robots.txt, or /sitemap.xml) and queue their in-scope pages before following links. This finds pages faster and finds pages that no link points to.",
            "default": true
          },
          "useLlmsTxt": {
            "title": "Use the site's llms.txt",
            "type": "boolean",
            "description": "Also read the site's own /llms.txt and crawl the in-scope pages it links to. robots.txt still applies.",
            "default": false
          },
          "followLinks": {
            "title": "Follow links",
            "type": "boolean",
            "description": "Follow in-scope links found on each page. Turn off to convert only the start URLs and sitemap pages.",
            "default": true
          },
          "maxCrawlDepth": {
            "title": "Maximum link depth",
            "minimum": 0,
            "type": "integer",
            "description": "How many links away from a start URL the crawl may go. Start URLs are depth 0. Sitemap pages are depth 1.",
            "default": 20
          },
          "mainContentOnly": {
            "title": "Main content only",
            "type": "boolean",
            "description": "Keep only the main article text (Mozilla Readability). Menus, headers, footers and sidebars are removed. Turn off to convert the whole page body without header, footer and navigation.",
            "default": true
          },
          "contentSelector": {
            "title": "Content selector",
            "type": "string",
            "description": "Optional. Convert only the elements that match this CSS selector, for example article.post or #content. Pages where it matches nothing use the normal extraction. Leave empty to let the crawler find the main content.",
            "default": ""
          },
          "removeSelectors": {
            "title": "Remove elements",
            "type": "string",
            "description": "Optional. Drop elements that match these CSS selectors before conversion, for example .ads, #comments, .related-posts. Separate selectors with commas.",
            "default": ""
          },
          "includeLinks": {
            "title": "Keep links",
            "type": "boolean",
            "description": "Keep links as Markdown links with absolute URLs. Turn off to keep only the link text.",
            "default": true
          },
          "includeImages": {
            "title": "Keep images",
            "type": "boolean",
            "description": "Keep images as Markdown image links. Off by default, because image links rarely help an LLM.",
            "default": false
          },
          "includePdfs": {
            "title": "Include linked PDF files",
            "type": "boolean",
            "description": "Also convert in-scope PDF files linked from crawled pages to Markdown (text layer only, no OCR). Each PDF counts as one page. Files over 20 MB, encrypted files and files robots.txt disallows are skipped free. Up to 300 pages of each PDF are read.",
            "default": false
          },
          "chunking": {
            "title": "Split into chunks for RAG",
            "type": "boolean",
            "description": "Add a list of chunks to each page, ready for a vector database. Each chunk has its heading path and a token estimate. A new chunk starts at each H1 to H3 heading and when the size limit is reached.",
            "default": true
          },
          "chunkSize": {
            "title": "Chunk size (tokens)",
            "minimum": 50,
            "maximum": 8000,
            "type": "integer",
            "description": "Largest chunk, in estimated tokens. The estimate is about 4 characters per token for English.",
            "default": 500
          },
          "chunkOverlap": {
            "title": "Chunk overlap (tokens)",
            "minimum": 0,
            "maximum": 2000,
            "type": "integer",
            "description": "Estimated tokens repeated from the end of the previous chunk. At most half the chunk size.",
            "default": 0
          },
          "saveLlmsTxt": {
            "title": "Save llms.txt and llms-full.txt",
            "type": "boolean",
            "description": "Save an llms.txt index of all converted pages and an llms-full.txt file with all their Markdown to the run's key-value store. llms-full.txt stops at 20 MB.",
            "default": true
          },
          "changeOnly": {
            "title": "Change-only mode",
            "type": "boolean",
            "description": "Save only pages that are new or whose Markdown changed since the last change-only run with the same state key. Unchanged pages are charged at a lower price. Schedule the Actor with this on to keep a knowledge base fresh.",
            "default": false
          },
          "modifiedSince": {
            "title": "Modified since",
            "type": "string",
            "description": "Only load pages changed on or after this date, for example 2026-10-01 or 30 days. Uses the sitemap's last-modified date and the If-Modified-Since header. Start URLs and pages with no date are always loaded. Skipped pages cost nothing."
          },
          "stateKey": {
            "title": "State key",
            "type": "string",
            "description": "Name of the memory that change-only mode uses. Use a different key for each site you watch.",
            "default": "default"
          },
          "maxConcurrency": {
            "title": "Parallel requests",
            "minimum": 1,
            "maximum": 20,
            "type": "integer",
            "description": "Most pages loaded at the same time. A robots.txt Crawl-delay lowers this to 1.",
            "default": 3
          },
          "maxRequestsPerMinute": {
            "title": "Requests per minute",
            "minimum": 1,
            "maximum": 600,
            "type": "integer",
            "description": "Most page requests per minute across the run. A robots.txt Crawl-delay lowers this. Keep it polite on sites you do not own.",
            "default": 120
          },
          "proxyConfiguration": {
            "title": "Proxy",
            "type": "object",
            "description": "Optional. Most sites need no proxy. A proxy does not get past bot checks: pages behind a challenge are skipped.",
            "default": {
              "useApifyProxy": false
            }
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}