{ "openapi": "3.0.1", "info": { "title": "Website to Markdown Crawler: Sitemap, RAG Chunks, llms.txt", "description": "Turn any website into clean Markdown for RAG using its sitemap. Incremental mode returns only new and changed pages and lists removed URLs. Heading-aware token chunks and llms.txt in the same run at no extra charge. Link-crawl fallback, contentHash. No browser. $0.50 per 1,000 pages.", "version": "0.1", "x-build-id": "z1oDoHxB45m88qDdi" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/chimey_forearm~sitemap-to-markdown/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-chimey_forearm-sitemap-to-markdown", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/chimey_forearm~sitemap-to-markdown/runs": { "post": { "operationId": "runs-sync-chimey_forearm-sitemap-to-markdown", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/chimey_forearm~sitemap-to-markdown/run-sync": { "post": { "operationId": "run-sync-chimey_forearm-sitemap-to-markdown", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Website or sitemap URLs", "type": "array", "description": "One per line. A website (https://docs.example.com) auto-discovers its sitemap from robots.txt, /sitemap.xml, /sitemap_index.xml, /wp-sitemap.xml and .xml.gz, following nested sitemap indexes. A section URL (https://example.com/blog) keeps only pages under that path. A sitemap URL (…/sitemap.xml, .xml.gz or a plain-text list) is used directly.", "items": { "type": "string" } }, "maxPages": { "title": "Max pages per start URL", "minimum": 0, "type": "integer", "description": "Stop after this many pages (or URLs in sitemap-only mode) for each start URL, after filters. 0 = no limit (the run's spending limit still applies).", "default": 100 }, "sitemapOnly": { "title": "Sitemap only (list URLs, don't fetch pages)", "type": "boolean", "description": "Just list every URL from the sitemaps with lastmod, changefreq and priority. Fast and cheap: use it to audit a site or plan a crawl, then run again with filters.", "default": false }, "includePatterns": { "title": "Include URL patterns", "type": "array", "description": "Keep only URLs matching ANY of these. Glob (\"/docs/**\", \"**/blog/*\") or regex in slashes (\"/\\/20(24|25)\\//\"). Overrides the automatic section-path filter.", "items": { "type": "string" } }, "excludePatterns": { "title": "Exclude URL patterns", "type": "array", "description": "Drop URLs matching any of these, e.g. \"**/tag/**\", \"**/author/**\", \"/\\.pdf$/\".", "items": { "type": "string" } }, "lastmodAfter": { "title": "Changed after (lastmod)", "type": "string", "description": "Keep only URLs whose sitemap is on/after this date. Absolute (2025-01-31) or relative (\"30 days\", \"2 weeks\"). URLs without lastmod are dropped when this is set. Great for incremental RAG refreshes." }, "incremental": { "title": "Incremental mode (only new or changed pages)", "type": "boolean", "description": "Remember every page between runs (in the named key-value store \"sitemap-to-markdown-state\"). Pages whose sitemap hasn't changed are skipped without being fetched; fetched pages whose content hash is the same as last time are left out of the dataset. You're only charged for new and changed pages. Each page gets changeStatus (new/changed), and the CHANGES record lists new, changed, unchanged and removed URLs, so you can update or delete vectors. The first run saves a baseline.", "default": false }, "incrementalStateKey": { "title": "Incremental state name", "type": "string", "description": "Optional. Runs with the same start URLs share state automatically. Set a name to keep separate histories for the same site (letters, digits, - _ . only)." }, "outputUnchanged": { "title": "Also output unchanged pages", "type": "boolean", "description": "In incremental mode, also save pages that were fetched but whose content didn't change (changeStatus \"unchanged\"; these count as results). Pages skipped by lastmod are never fetched or saved.", "default": false }, "sortBy": { "title": "Order", "enum": [ "sitemap", "lastmodDesc" ], "type": "string", "description": "Sitemap order, or newest lastmod first (reads all sitemaps first, then takes the newest pages).", "default": "sitemap" }, "linkCrawl": { "title": "Link crawl (no sitemap?)", "enum": [ "fallback", "always", "never" ], "type": "string", "description": "What to do when a website has no sitemap, or its sitemap lists nothing under your start URL. \"fallback\" (default): crawl same-site links from the start URL instead, up to the depth below. \"always\": use the sitemap AND follow links, to catch pages the sitemap misses. \"never\": sitemap only. Not used for direct sitemap URLs or in sitemap-only mode.", "default": "fallback" }, "maxCrawlDepth": { "title": "Max link-crawl depth", "minimum": 0, "maximum": 10, "type": "integer", "description": "How many clicks away from the start URL the link crawl goes (0 = start page only). Pages still stop at Max pages.", "default": 2 }, "chunkSize": { "title": "Chunk size (0 = no chunking)", "minimum": 0, "type": "integer", "description": "Split each page's Markdown into RAG-ready chunks of about this size, on heading and paragraph boundaries (code blocks kept whole when they fit). Each chunk carries its heading path. Chunks are an array inside the page record, so they don't cost extra.", "default": 0 }, "chunkUnit": { "title": "Chunk size unit", "enum": [ "tokens", "chars" ], "type": "string", "description": "Tokens are estimated as characters ÷ 4 (close to OpenAI/Anthropic tokenizers for English).", "default": "tokens" }, "chunkOverlap": { "title": "Chunk overlap", "minimum": 0, "type": "integer", "description": "Repeat this much of the end of the previous chunk at the start of the next (same unit as chunk size).", "default": 0 }, "createLlmsTxt": { "title": "Save llms.txt and llms-full.txt", "type": "boolean", "description": "Also save an llms.txt index (title, link, description per page) and llms-full.txt (every page's Markdown in one file) to the run's key-value store.", "default": true }, "contentSelector": { "title": "Main content CSS selector (optional)", "type": "string", "description": "Override automatic main-content detection, e.g. \"article.post\" or \".markdown\". Leave empty for auto (semantic
/
first, Mozilla Readability fallback)." }, "removeSelectors": { "title": "Extra CSS selectors to remove", "type": "array", "description": "Elements to strip before conversion, e.g. \".newsletter-signup\", \"#related\". Nav, header, footer, sidebars, scripts and cookie banners are removed automatically.", "items": { "type": "string" } }, "includeLinksInMarkdown": { "title": "Keep links in Markdown", "type": "boolean", "description": "Keep [text](url) links (absolute URLs). Turn off for plain text-like Markdown with fewer tokens.", "default": true }, "includeImages": { "title": "Keep images in Markdown", "type": "boolean", "description": "Keep ![alt](src) image references. Off by default to save tokens.", "default": false }, "addFrontMatter": { "title": "Add YAML front matter", "type": "boolean", "description": "Start each page's Markdown with a YAML front matter block: title, url, canonical_url, lastmod (from the sitemap), language, word_count, thin_content, scraped_at. The same values are also in the frontMatter field.", "default": true }, "minWords": { "title": "Thin page threshold (words)", "minimum": 0, "type": "integer", "description": "Pages whose main content has fewer words than this are flagged as thin (type \"thin_page\", thinContent: true) and left out of llms.txt and llms-full.txt. Set 0 to turn off.", "default": 50 }, "saveThinPages": { "title": "Save thin pages", "type": "boolean", "description": "Keep flagged thin pages in the dataset (they count as results). Turn off to skip them entirely, so you aren't charged for them.", "default": true }, "skipDuplicates": { "title": "Skip duplicate pages", "type": "boolean", "description": "Leave out pages whose Markdown is identical to a page already saved in this run (same contentHash), or that share a canonical URL with a saved page and have near-identical length. Skipped duplicates aren't charged.", "default": true }, "respectRobotsTxt": { "title": "Respect robots.txt", "type": "boolean", "description": "Skip pages disallowed for all user agents (User-agent: *) and honor Crawl-delay (up to 10 s).", "default": true }, "sameDomainOnly": { "title": "Same domain only", "type": "boolean", "description": "Ignore sitemap URLs on other domains (www. is treated as the same domain).", "default": true }, "restrictToStartPath": { "title": "Restrict to start URL path", "type": "boolean", "description": "For a start URL with a path (https://example.com/docs), keep only pages under /docs. Ignored when include patterns are set.", "default": true }, "maxConcurrency": { "title": "Max concurrent requests", "minimum": 1, "maximum": 20, "type": "integer", "description": "Parallel page downloads. Keep it low to be polite to small sites.", "default": 5 }, "saveErrorRecords": { "title": "Save error records", "type": "boolean", "description": "Add a record for every input or page that failed (NO_SITEMAP_FOUND, FETCH_FAILED, NOT_HTML, INVALID_URL). Error records count as results.", "default": true } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }