{ "openapi": "3.0.1", "info": { "title": "Robots.txt Generator", "description": "Generate deployable robots.txt files from presets, custom bot rules, sitemap URLs, and host directives. Create one file or batch files for multiple sites, then export raw text plus validation data.", "version": "0.1", "x-build-id": "3GwztC4I9IzwwwSxI" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/maximedupre~robots-txt-generator/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-maximedupre-robots-txt-generator", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/maximedupre~robots-txt-generator/runs": { "post": { "operationId": "runs-sync-maximedupre-robots-txt-generator", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/maximedupre~robots-txt-generator/run-sync": { "post": { "operationId": "run-sync-maximedupre-robots-txt-generator", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "preset": { "title": "Preset", "enum": [ "none", "allow-all", "block-all", "block-ai-crawlers", "seo-friendly", "wordpress-seo", "ecommerce-seo", "staging-private", "ai-training-opt-out" ], "type": "string", "description": "Start with a common robots.txt policy, then add custom rules below.", "default": "seo-friendly" }, "sitemaps": { "title": "Sitemap URLs", "uniqueItems": true, "type": "array", "description": "Add full sitemap URLs to include as Sitemap directives.", "items": { "type": "string", "minLength": 1 } }, "host": { "title": "Host directive", "type": "string", "description": "Add the canonical host when you want a Host directive in the file." }, "rules": { "title": "Custom bot rules", "type": "array", "description": "Add per-bot Allow, Disallow, Crawl-delay, and comment rules.", "items": { "type": "object", "required": [ "userAgents" ], "properties": { "userAgents": { "title": "Bot names", "description": "Crawler user-agent names for this rule block.", "type": "array", "items": { "type": "string", "minLength": 1 }, "minItems": 1 }, "allow": { "title": "Allow paths", "description": "Paths these bots can crawl.", "type": "array", "items": { "type": "string" } }, "disallow": { "title": "Disallow paths", "description": "Paths these bots should skip.", "type": "array", "items": { "type": "string" } }, "crawlDelay": { "title": "Crawl delay", "description": "Seconds these bots should wait between requests.", "type": "number", "minimum": 0 }, "comment": { "title": "Rule comment", "description": "Comment written above this rule block.", "type": "string" } } } }, "presetConflictMode": { "title": "Preset conflict handling", "enum": [ "append", "override" ], "type": "string", "description": "Choose how custom rules behave when they use the same bot name as a preset rule.", "default": "append" }, "sites": { "title": "Batch site configs", "type": "array", "description": "Generate several robots.txt files in one run. Each site can set its own label, preset, rules, sitemaps, and host.", "items": { "type": "object", "properties": { "label": { "title": "Site label", "description": "Short name shown with this generated file.", "type": "string" }, "preset": { "title": "Site preset", "description": "Preset for this site.", "type": "string" }, "sitemaps": { "title": "Site sitemap URLs", "description": "Sitemap URLs for this site.", "type": "array", "items": { "type": "string" } }, "host": { "title": "Site host", "description": "Host directive for this site.", "type": "string" }, "rules": { "title": "Site custom rules", "description": "Custom bot rules for this site.", "type": "array", "items": { "type": "object" } }, "fileName": { "title": "File name", "description": "Output file name for this site.", "type": "string" } } } }, "validateRules": { "title": "Validate rules", "type": "boolean", "description": "Return warnings for duplicate bots, invalid paths, conflicting directives, and missing wildcard rules.", "default": true }, "includeTimestamp": { "title": "Include generated timestamp", "type": "boolean", "description": "Add a generated-at comment to the robots.txt content.", "default": true }, "includeGeneratorComment": { "title": "Include generator comment", "type": "boolean", "description": "Add a short comment that identifies the file as generated by this Actor.", "default": true } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } } }