{ "openapi": "3.0.1", "info": { "title": "Website to Markdown Crawler for AI & RAG — Only Changed Pages", "description": "Crawl a website over plain HTTP and get clean Markdown for AI and RAG. Sitemap first, no browser, no login, robots.txt respected. Run it again and get only the pages whose main text changed since the last run, so a RAG index stays fresh for cents. $1 per 1,000 pages.", "version": "0.1", "x-build-id": "edGPMExpD4irAgkF7" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/northbell~website-markdown-crawler/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-northbell-website-markdown-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/northbell~website-markdown-crawler/runs": { "post": { "operationId": "runs-sync-northbell-website-markdown-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/northbell~website-markdown-crawler/run-sync": { "post": { "operationId": "run-sync-northbell-website-markdown-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "One web address per line, for example https://docs.apify.com/academy. The crawl starts here, reads the site's sitemap and follows links to other pages of the same site (see Crawl scope). Each start URL gets its own summary row.", "items": { "type": "string" } }, "maxPages": { "title": "Maximum pages per start URL", "minimum": 1, "maximum": 5000, "type": "integer", "description": "The most pages fetched for each start URL. Pages are fetched in the order they are found: the start page first, then the sitemap, then links. Short pages that are left out and notes about pages that need JavaScript count towards this limit, but they are never charged.", "default": 50 }, "maxDepth": { "title": "Maximum link depth", "minimum": 0, "maximum": 20, "type": "integer", "description": "How many clicks away from the start page the crawl may go. 0 reads only the start page itself. Pages taken from the sitemap count as depth 1. Raise it for sites without a sitemap.", "default": 3 }, "scope": { "title": "Crawl scope", "enum": [ "path", "domain" ], "type": "string", "description": "Which pages count as part of the site. \"Start URL folder\" keeps to the start URL's path (a start URL of https://example.com/docs reads /docs and everything under it). \"Whole site\" reads any page on the same host (www.example.com and example.com are the same site). Other websites are never followed.", "default": "path" }, "useSitemap": { "title": "Use the sitemap first", "type": "boolean", "description": "Read robots.txt and sitemap.xml (including sitemap indexes and .gz sitemaps) to find pages before following links. Fast and complete on sites that publish a sitemap. Turn off to follow links only.", "default": true }, "includeGlobs": { "title": "Only these URLs (globs)", "type": "array", "description": "Optional. Only pages whose address matches one of these patterns are crawled (start URLs are always crawled). * matches anything except /, ** matches anything. Example: https://example.com/blog/**. A pattern that starts with / is matched against the path only, for example /blog/**.", "items": { "type": "string" } }, "excludeGlobs": { "title": "Skip these URLs (globs)", "type": "array", "description": "Optional. Pages whose address matches one of these patterns are skipped. Same pattern rules as above. Example: **/tag/** or /login/**.", "items": { "type": "string" } }, "changedOnly": { "title": "Return only new and changed pages", "type": "boolean", "description": "When on, pages whose main text is the same as in the previous run are not returned and not charged. The first run returns every page. Leave off to get every page every time (each row still says whether it is new, changed or unchanged).", "default": false }, "minTextLength": { "title": "Minimum text length", "minimum": 0, "maximum": 100000, "type": "integer", "description": "Pages with fewer characters of main text than this are left out and reported in a free note row. Pages that only show their text after JavaScript runs are reported as \"needs-javascript\"; other short pages as \"too-little-text\". 0 keeps any page that has at least one character.", "default": 200 }, "includeHtml": { "title": "Also return cleaned HTML", "type": "boolean", "description": "Add an html field with the main content as cleaned HTML (no scripts, styles, navigation or footer; links and image addresses made absolute).", "default": false }, "includeImages": { "title": "Keep images in the Markdown", "type": "boolean", "description": "Write images as ![alt](address). Off by default because most images on a page are logos and icons that add noise for AI.", "default": false } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }