{ "openapi": "3.0.1", "info": { "title": "Website Content Crawler – Markdown for AI, Emails & Contacts", "description": "Website content crawler that turns a whole site into clean Markdown for LLM and RAG pipelines, with the role emails, social profiles and WhatsApp numbers found on each page. Seeded from the sitemap, honours robots.txt, plain HTTP with no browser.", "version": "0.1", "x-build-id": "RRak6R6qaVBN3XG0M" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/locomint~website-crawler-content-contacts/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-locomint-website-crawler-content-contacts", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/locomint~website-crawler-content-contacts/runs": { "post": { "operationId": "runs-sync-locomint-website-crawler-content-contacts", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/locomint~website-crawler-content-contacts/run-sync": { "post": { "operationId": "run-sync-locomint-website-crawler-content-contacts", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Websites to crawl", "type": "array", "description": "One or more site homepages (full URLs with https://). Each site is crawled separately, up to 50 sites per run.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "maxPagesPerSite": { "title": "Max pages per site", "minimum": 1, "maximum": 2000, "type": "integer", "description": "Stop after this many pages on a site. You pay per page delivered.", "default": 50 }, "maxDepth": { "title": "Max link depth", "minimum": 0, "maximum": 10, "type": "integer", "description": "How far to follow links: 0 = follow none, 1 = pages linked from the start page, and so on. Sitemap pages are crawled whatever this is set to; turn off 'Use the sitemap' for a start-page-only run.", "default": 3 }, "includeSubdomains": { "title": "Include subdomains", "type": "boolean", "description": "Also follow links to blog.example.com, shop.example.com, etc.", "default": false }, "includeUrlPatterns": { "title": "Only crawl URLs containing", "type": "array", "description": "Optional. Keep only URLs containing one of these texts, e.g. [\"/docs/\"]. The start page is always included.", "items": { "type": "string" } }, "excludeUrlPatterns": { "title": "Skip URLs containing", "type": "array", "description": "Optional. Skip URLs containing any of these texts, e.g. [\"/tag/\", \"?page=\"].", "items": { "type": "string" } }, "useSitemap": { "title": "Use the sitemap", "type": "boolean", "description": "Seed the crawl from sitemap.xml when the site has one - finds pages that are not linked from the homepage.", "default": true }, "respectRobots": { "title": "Respect robots.txt", "type": "boolean", "description": "Filter pages the site's robots.txt disallows out of the queue before they are fetched. A disallowed page is never fetched or charged either way: every page is checked against robots.txt right before its fetch.", "default": true }, "outputFormat": { "title": "Content format", "enum": [ "markdown", "text" ], "type": "string", "description": "Markdown keeps headings, lists, links and code; plain text is just the words.", "default": "markdown" }, "maxChars": { "title": "Max characters per page", "minimum": 500, "maximum": 500000, "type": "integer", "description": "Longer content is cut here and marked truncated.", "default": 50000 }, "includeContacts": { "title": "Extract business contacts", "type": "boolean", "description": "Email addresses (company mailboxes first; named staff addresses are also listed in named_emails), social profiles, staff profile links (named_profiles), WhatsApp, contact page URL, tech stack - from every page, no extra cost.", "default": true }, "includeLinks": { "title": "Include outgoing links", "type": "boolean", "description": "Add the list of links found on each page (up to 500).", "default": false }, "concurrency": { "title": "Parallel fetches", "minimum": 1, "maximum": 10, "type": "integer", "description": "Pages fetched at the same time per site.", "default": 5 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }