{
"openapi": "3.0.1",
"info": {
"title": "AI Crawler Policy Auditor",
"description": "Audit robots and AI crawler policy signals for AI crawler access, blocking, and site-readiness.",
"version": "1.0",
"x-build-id": "80vNFOXNuAAgowY5A"
},
"servers": [
{
"url": "https://api.apify.com/v2"
}
],
"paths": {
"/acts/junipr~ai-crawler-policy-auditor/run-sync-get-dataset-items": {
"post": {
"operationId": "run-sync-get-dataset-items-junipr-ai-crawler-policy-auditor",
"x-openai-isConsequential": false,
"summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
"tags": [
"Run Actor"
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/inputSchema"
}
}
}
},
"parameters": [
{
"name": "token",
"in": "query",
"required": true,
"schema": {
"type": "string"
},
"description": "Enter your Apify token here"
}
],
"responses": {
"200": {
"description": "OK"
}
}
}
},
"/acts/junipr~ai-crawler-policy-auditor/runs": {
"post": {
"operationId": "runs-sync-junipr-ai-crawler-policy-auditor",
"x-openai-isConsequential": false,
"summary": "Executes an Actor and returns information about the initiated run in response.",
"tags": [
"Run Actor"
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/inputSchema"
}
}
}
},
"parameters": [
{
"name": "token",
"in": "query",
"required": true,
"schema": {
"type": "string"
},
"description": "Enter your Apify token here"
}
],
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/runsResponseSchema"
}
}
}
}
}
}
},
"/acts/junipr~ai-crawler-policy-auditor/run-sync": {
"post": {
"operationId": "run-sync-junipr-ai-crawler-policy-auditor",
"x-openai-isConsequential": false,
"summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
"tags": [
"Run Actor"
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/inputSchema"
}
}
}
},
"parameters": [
{
"name": "token",
"in": "query",
"required": true,
"schema": {
"type": "string"
},
"description": "Enter your Apify token here"
}
],
"responses": {
"200": {
"description": "OK"
}
}
}
}
},
"components": {
"schemas": {
"inputSchema": {
"type": "object",
"properties": {
"targets": {
"title": "Policy Targets",
"type": "array",
"description": "Robots.txt and page-signal snapshots to audit. Defaults to a local fixture that makes no network calls.",
"items": {
"type": "object"
},
"default": [
{
"sourceId": "sample-ai-ready-site",
"sourceUrl": "https://example.com",
"targetPath": "/",
"robotsTxt": "User-agent: *\nAllow: /\n\nUser-agent: GPTBot\nAllow: /\n\nUser-agent: ChatGPT-User\nAllow: /\n\nUser-agent: OAI-SearchBot\nAllow: /\n\nUser-agent: ClaudeBot\nAllow: /\n\nUser-agent: Claude-User\nAllow: /\n\nUser-agent: anthropic-ai\nAllow: /\n\nUser-agent: PerplexityBot\nAllow: /\n\nUser-agent: Google-Extended\nAllow: /\n\nUser-agent: GoogleOther\nAllow: /\n\nUser-agent: CCBot\nAllow: /\n\nUser-agent: Bytespider\nAllow: /\n\nUser-agent: Meta-ExternalAgent\nAllow: /\n\nSitemap: https://example.com/sitemap.xml",
"llmsTxt": "# Example AI crawler guidance\n\nThis fixture represents a site that allows major AI crawlers and exposes a sitemap.",
"headers": {
"x-robots-tag": "index, follow"
},
"html": "ExampleExample"
}
]
},
"urls": {
"title": "URLs To Fetch",
"type": "array",
"description": "Optional site URLs to fetch only when fetchUrls is true. The actor fetches /robots.txt and /llms.txt with strict byte and timeout caps.",
"items": {
"type": "string"
},
"default": []
},
"fetchUrls": {
"title": "Fetch URLs",
"type": "boolean",
"description": "Fetch robots.txt and llms.txt over HTTP. Default is false to keep zero-config runs local, deterministic, and cheap.",
"default": false
},
"expectedAccess": {
"title": "Expected AI Crawler Access",
"enum": [
"allow",
"block",
"neutral"
],
"type": "string",
"description": "Use allow for AI crawler readiness checks, block for opt-out audits, or neutral for inventory-only reporting.",
"default": "allow"
},
"auditedAgents": {
"title": "Audited Crawler Agents",
"type": "array",
"description": "Crawler user-agent tokens to evaluate against robots.txt.",
"items": {
"type": "string"
},
"default": [
"GPTBot",
"ChatGPT-User",
"OAI-SearchBot",
"ClaudeBot",
"Claude-User",
"anthropic-ai",
"PerplexityBot",
"Google-Extended",
"GoogleOther",
"CCBot",
"Bytespider",
"Meta-ExternalAgent",
"Googlebot",
"Bingbot"
]
},
"comparePrevious": {
"title": "Compare Previous Robots",
"type": "boolean",
"description": "Compare previousRobotsTxt against current robotsTxt when a previous snapshot is supplied.",
"default": true
},
"requireLlmsTxt": {
"title": "Require llms.txt",
"type": "boolean",
"description": "Flag targets that do not include llms.txt content.",
"default": false
},
"maxItems": {
"title": "Max Items",
"minimum": 1,
"maximum": 250,
"type": "integer",
"description": "Alias for Max Targets/Maximum Records used by the Junipr actor baseline. When both are provided, actor-specific limits still apply.",
"default": 25
},
"includeReport": {
"title": "Include Report Artifacts",
"type": "boolean",
"description": "Write summary JSON, full result JSON, and Markdown report artifacts to the key-value store.",
"default": true
},
"maxTargets": {
"title": "Max Targets",
"minimum": 1,
"maximum": 250,
"type": "integer",
"description": "Maximum robots or URL targets to audit.",
"default": 25
},
"fetchTimeoutMs": {
"title": "Fetch Timeout Milliseconds",
"minimum": 1000,
"maximum": 30000,
"type": "integer",
"description": "HTTP timeout for optional robots.txt and llms.txt fetches.",
"default": 10000
},
"maxRobotsBytes": {
"title": "Max Robots Bytes",
"minimum": 1000,
"maximum": 1000000,
"type": "integer",
"description": "Maximum bytes to read from a fetched robots.txt file.",
"default": 200000
},
"maxChargeUsd": {
"title": "Maximum Charge (USD)",
"minimum": 0,
"maximum": 1000,
"type": "number",
"description": "Hard local spending cap for actor-start, per-item, and report events. The actor stops before work or withholds uncharged output when the next event would exceed this amount.",
"default": 10
},
"dryRun": {
"title": "Dry Run",
"type": "boolean",
"description": "Validate input and write a dry-run summary without PPE charges or dataset output.",
"default": false
},
"debug": {
"title": "Debug Logs",
"type": "boolean",
"description": "Enable debug logs.",
"default": false
}
}
},
"runsResponseSchema": {
"type": "object",
"properties": {
"data": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"actId": {
"type": "string"
},
"userId": {
"type": "string"
},
"startedAt": {
"type": "string",
"format": "date-time",
"example": "2025-01-08T00:00:00.000Z"
},
"finishedAt": {
"type": "string",
"format": "date-time",
"example": "2025-01-08T00:00:00.000Z"
},
"status": {
"type": "string",
"example": "READY"
},
"meta": {
"type": "object",
"properties": {
"origin": {
"type": "string",
"example": "API"
},
"userAgent": {
"type": "string"
}
}
},
"stats": {
"type": "object",
"properties": {
"inputBodyLen": {
"type": "integer",
"example": 2000
},
"rebootCount": {
"type": "integer",
"example": 0
},
"restartCount": {
"type": "integer",
"example": 0
},
"resurrectCount": {
"type": "integer",
"example": 0
},
"computeUnits": {
"type": "integer",
"example": 0
}
}
},
"options": {
"type": "object",
"properties": {
"build": {
"type": "string",
"example": "latest"
},
"timeoutSecs": {
"type": "integer",
"example": 300
},
"memoryMbytes": {
"type": "integer",
"example": 1024
},
"diskMbytes": {
"type": "integer",
"example": 2048
}
}
},
"buildId": {
"type": "string"
},
"defaultKeyValueStoreId": {
"type": "string"
},
"defaultDatasetId": {
"type": "string"
},
"defaultRequestQueueId": {
"type": "string"
},
"buildNumber": {
"type": "string",
"example": "1.0.0"
},
"containerUrl": {
"type": "string"
},
"usage": {
"type": "object",
"properties": {
"ACTOR_COMPUTE_UNITS": {
"type": "integer",
"example": 0
},
"DATASET_READS": {
"type": "integer",
"example": 0
},
"DATASET_WRITES": {
"type": "integer",
"example": 0
},
"KEY_VALUE_STORE_READS": {
"type": "integer",
"example": 0
},
"KEY_VALUE_STORE_WRITES": {
"type": "integer",
"example": 1
},
"KEY_VALUE_STORE_LISTS": {
"type": "integer",
"example": 0
},
"REQUEST_QUEUE_READS": {
"type": "integer",
"example": 0
},
"REQUEST_QUEUE_WRITES": {
"type": "integer",
"example": 0
},
"DATA_TRANSFER_INTERNAL_GBYTES": {
"type": "integer",
"example": 0
},
"DATA_TRANSFER_EXTERNAL_GBYTES": {
"type": "integer",
"example": 0
},
"PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
"type": "integer",
"example": 0
},
"PROXY_SERPS": {
"type": "integer",
"example": 0
}
}
},
"usageTotalUsd": {
"type": "number",
"example": 0.00005
},
"usageUsd": {
"type": "object",
"properties": {
"ACTOR_COMPUTE_UNITS": {
"type": "integer",
"example": 0
},
"DATASET_READS": {
"type": "integer",
"example": 0
},
"DATASET_WRITES": {
"type": "integer",
"example": 0
},
"KEY_VALUE_STORE_READS": {
"type": "integer",
"example": 0
},
"KEY_VALUE_STORE_WRITES": {
"type": "number",
"example": 0.00005
},
"KEY_VALUE_STORE_LISTS": {
"type": "integer",
"example": 0
},
"REQUEST_QUEUE_READS": {
"type": "integer",
"example": 0
},
"REQUEST_QUEUE_WRITES": {
"type": "integer",
"example": 0
},
"DATA_TRANSFER_INTERNAL_GBYTES": {
"type": "integer",
"example": 0
},
"DATA_TRANSFER_EXTERNAL_GBYTES": {
"type": "integer",
"example": 0
},
"PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
"type": "integer",
"example": 0
},
"PROXY_SERPS": {
"type": "integer",
"example": 0
}
}
}
}
}
}
}
}
}
}