{ "openapi": "3.0.1", "info": { "title": "Robots.txt, Sitemap & Schema Validator + AI Crawler Check", "description": "Validate robots.txt, XML sitemaps and JSON-LD schema markup in one run and see which AI crawlers (GPTBot, ClaudeBot, Googlebot) are allowed. Scored JSON, from $0.02.", "version": "0.1", "x-build-id": "Rtq94VlmGxNRaVJQk" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/appropriate_songbird~structured-data-sitemap-validator/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-appropriate_songbird-structured-data-sitemap-validator", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/appropriate_songbird~structured-data-sitemap-validator/runs": { "post": { "operationId": "runs-sync-appropriate_songbird-structured-data-sitemap-validator", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/appropriate_songbird~structured-data-sitemap-validator/run-sync": { "post": { "operationId": "run-sync-appropriate_songbird-structured-data-sitemap-validator", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "siteUrls" ], "properties": { "siteUrls": { "title": "Site URLs", "type": "array", "description": "Home page of each site you own or have permission to test (1 to 20). Only that host is contacted. robots.txt is always obeyed for page fetches.", "items": { "type": "string" } }, "pageUrls": { "title": "Extra page URLs (optional)", "type": "array", "description": "Specific pages to validate, for example one URL per page template (product, article, event). Each must be on the same host as one of your site URLs. Up to 100.", "items": { "type": "string" } }, "checkRobots": { "title": "Check robots.txt", "type": "boolean", "description": "Validate syntax and rules, and report which AI and search crawlers (GPTBot, ClaudeBot, CCBot, Googlebot and others) the file allows or blocks.", "default": true }, "checkSitemaps": { "title": "Check XML sitemaps", "type": "boolean", "description": "Find sitemaps via robots.txt (or /sitemap.xml), follow sitemap indexes and validate protocol limits, URLs, lastmod, duplicates and off-host entries.", "default": true }, "checkStructuredData": { "title": "Validate structured data", "type": "boolean", "description": "Fetch sampled pages and check JSON-LD (plus microdata/RDFa types) for parse errors and missing required or recommended properties.", "default": true }, "pagesPerSite": { "title": "Pages to sample per site", "minimum": 0, "maximum": 200, "type": "integer", "description": "How many pages to fetch per site: the home page plus an even spread of sitemap URLs (0 to 200). Set 0 to check only robots.txt and sitemaps.", "default": 10 }, "maxSitemapsPerSite": { "title": "Max sitemap files per site", "minimum": 1, "maximum": 100, "type": "integer", "description": "Cap on sitemap files read per site, including those listed in sitemap indexes (1 to 100).", "default": 25 }, "requestDelayMs": { "title": "Delay between page requests (ms)", "minimum": 500, "maximum": 10000, "type": "integer", "description": "Pause between requests to the same host (500 to 10000). A higher Crawl-delay in robots.txt wins.", "default": 500 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }