{ "openapi": "3.0.1", "info": { "title": "Technical SEO Site Crawler – Broken Links & Audit", "description": "Crawl a website within robots.txt and get one row per page: status code, redirects, title, meta description, H1, canonical, noindex, broken internal links, duplicates, orphan pages, sitemap gaps and a 0–100 SEO score.", "version": "1.0", "x-build-id": "evRQaQnDV5LNJsf0u" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/benelux-signals~technical-seo-site-crawler/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-benelux-signals-technical-seo-site-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/benelux-signals~technical-seo-site-crawler/runs": { "post": { "operationId": "runs-sync-benelux-signals-technical-seo-site-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/benelux-signals~technical-seo-site-crawler/run-sync": { "post": { "operationId": "run-sync-benelux-signals-technical-seo-site-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Website to crawl", "type": "array", "description": "Homepage or section URL to start from. The crawler follows internal links on the same host (www and non-www count as the same site). Add several URLs of the same site to crawl several sections; for different websites start separate runs.", "default": [ "https://crawlee.dev/" ], "items": { "type": "string" } }, "maxPages": { "title": "Max pages", "minimum": 1, "maximum": 10000, "type": "integer", "description": "Stop after this many pages (HTML pages, redirects and error pages each count as one). You pay per page, so this is your cost cap.", "default": 100 }, "maxDepth": { "title": "Max click depth", "minimum": 0, "maximum": 100, "type": "integer", "description": "How many clicks away from the start URL the crawler goes. 0 = only the start URL(s) (orphan pages from the sitemap are then not crawled either).", "default": 20 }, "useSitemap": { "title": "Check XML sitemap", "type": "boolean", "description": "Read robots.txt sitemaps (or /sitemap.xml) to flag indexable pages missing from the sitemap, sitemap URLs that are not indexable, and orphan pages (in the sitemap but not linked). Orphan pages are crawled once the link crawl is done and pages are left within Max pages.", "default": true }, "includePatterns": { "title": "Only crawl URLs containing", "type": "array", "description": "Optional. Only follow URLs that contain one of these texts (e.g. /blog/). Use * as wildcard for a full-URL pattern (e.g. https://example.com/nl/*). Start URLs are always crawled.", "default": [], "items": { "type": "string" } }, "excludePatterns": { "title": "Skip URLs containing", "type": "array", "description": "Optional. Do not crawl URLs that contain one of these texts, e.g. /tag/, ?sort=, /wp-admin/. * is a wildcard.", "default": [], "items": { "type": "string" } }, "ignoreQueryStrings": { "title": "Ignore query strings", "type": "boolean", "description": "Treat /page?a=1 and /page?a=2 as the same URL. Turn on for shops and sites with filter or tracking parameters to avoid crawling thousands of variants.", "default": false }, "includeSubdomains": { "title": "Include subdomains", "type": "boolean", "description": "Also crawl subdomains such as blog.example.com and shop.example.com.", "default": false }, "followNofollowLinks": { "title": "Follow nofollow links", "type": "boolean", "description": "By default links with rel=nofollow (and pages with meta robots nofollow) are not followed, like search engines.", "default": false }, "onlyPagesWithIssues": { "title": "Only save pages with errors or warnings", "type": "boolean", "description": "Skip pages without errors or warnings in the dataset (they are not charged). The SITE_REPORT still covers all crawled pages.", "default": false }, "minDelayMs": { "title": "Delay between requests (ms)", "minimum": 200, "maximum": 30000, "type": "integer", "description": "Minimum time between two requests to the site. A Crawl-delay in robots.txt is respected when it is longer (up to 30 s). Keep this polite for small sites.", "default": 500 }, "maxConcurrency": { "title": "Parallel requests", "minimum": 1, "maximum": 10, "type": "integer", "description": "Requests in flight at the same time. The delay above still spaces them out.", "default": 3 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }