{
  "openapi": "3.0.1",
  "info": {
    "title": "URL Access Checker - robots.txt, Bot Protection & Scrapability",
    "description": "Before your AI agent fetches a URL: may it (robots.txt per crawler, RFC 9309) and can it (Cloudflare, DataDome, PerimeterX, AWS WAF, login, JavaScript)? One row per URL with a verdict, safeToFetch true/false and a one-sentence recommendation the agent can follow. It only looks, never bypasses.",
    "version": "0.1",
    "x-build-id": "bYehqMlXnD03qr0zQ"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/neverempty~url-access-checker/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-neverempty-url-access-checker",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/neverempty~url-access-checker/runs": {
      "post": {
        "operationId": "runs-sync-neverempty-url-access-checker",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/neverempty~url-access-checker/run-sync": {
      "post": {
        "operationId": "run-sync-neverempty-url-access-checker",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "properties": {
          "urls": {
            "title": "URLs",
            "type": "array",
            "description": "URLs to check, one per line (1 to 500 per run). A URL without http:// or https:// is read as https://. The same URL given twice (also when only the #fragment differs) is checked and charged once. If this is empty, the example URL https://en.wikipedia.org/wiki/Robots.txt is checked.",
            "items": {
              "type": "string"
            }
          },
          "userAgents": {
            "title": "Crawler names for robots.txt",
            "type": "array",
            "description": "Crawler names (robots.txt product tokens such as GPTBot, ClaudeBot or Googlebot, or a full User-Agent string) to judge robots.txt for. The first one is the subject of verdict, safeToFetch and recommendation; every name gets its own entry in robotsByAgent. Following RFC 9309, a name that has its own group in robots.txt follows only that group; otherwise the User-agent: * group applies. Empty means * (any crawler).",
            "items": {
              "type": "string"
            }
          },
          "includeAiCrawlers": {
            "title": "Also judge the common AI and search crawlers",
            "type": "boolean",
            "description": "Add GPTBot, OAI-SearchBot, ChatGPT-User, ClaudeBot, Claude-User, Claude-SearchBot, anthropic-ai, Google-Extended, Googlebot, Bingbot, PerplexityBot, Perplexity-User, CCBot, Bytespider, Applebot-Extended, meta-externalagent and Amazonbot to robotsByAgent. It only reads robots.txt, so it costs nothing extra.",
            "default": false
          },
          "checkReach": {
            "title": "Test whether the page can actually be fetched",
            "type": "boolean",
            "description": "Request the page through the chosen routes and report status codes, walls and whether it needs JavaScript. The page is requested only when robots.txt allows this Actor (URLAccessChecker, else User-agent: *); otherwise only the robots.txt answer is returned. Turn off for a robots.txt-only answer.",
            "default": true
          },
          "routes": {
            "title": "Routes to test",
            "type": "array",
            "description": "Where the page is requested from: direct (the Actor's own server, no proxy), datacenter (Apify datacenter proxy) and residential (Apify residential proxy, a new IP for every try). Datacenter and residential tries are charged per try that the site answered, to cover the proxy traffic.",
            "items": {
              "type": "string",
              "enum": [
                "direct",
                "datacenter",
                "residential"
              ],
              "enumTitles": [
                "Direct (no proxy)",
                "Datacenter proxy",
                "Residential proxy"
              ]
            },
            "default": [
              "direct"
            ]
          },
          "attemptsPerRoute": {
            "title": "Tries per route",
            "minimum": 1,
            "maximum": 5,
            "type": "integer",
            "description": "How many times the page is requested on each route (1 to 5). More tries show how reliable a route is (for example 2 of 3). Tries are spaced 0.7 seconds apart. A challenge page is recorded, never solved or retried.",
            "default": 1
          },
          "residentialCountry": {
            "title": "Residential proxy country",
            "type": "string",
            "description": "Optional two-letter country code for residential tries (for example US, DE, JP), useful to see geo-restricted pages. Empty means any country."
          },
          "fetchAs": {
            "title": "Request the page as",
            "enum": [
              "browser",
              "bot"
            ],
            "type": "string",
            "description": "browser sends an ordinary Chrome User-Agent header (what most scrapers and fetch tools send); bot sends an honest URLAccessChecker bot header. robots.txt itself is always requested with the bot header. There is no TLS or browser fingerprint spoofing either way.",
            "default": "browser"
          },
          "onlyChanges": {
            "title": "Monitor mode: only URLs whose robots.txt answer changed",
            "type": "boolean",
            "description": "Return only URLs whose robots.txt answer (allowed or not, and the matched rule, for each crawler name) changed since the last run with the same watch name, plus URLs new to the watch. The first run returns every URL as the starting point. A run where robots.txt could not be read does not count as a change. A run in which nothing changed returns a free row saying so and charges only the run start fee (and any proxy tries).",
            "default": false
          },
          "watchName": {
            "title": "Watch name",
            "type": "string",
            "description": "Name of the remembered state used to compare runs (letters, digits, dot, dash, underscore; up to 40). Setting it (or turning on monitor mode) fills changeType and the previous robots.txt answers. Use a different name for each list of URLs you track on its own schedule. With monitor mode on and no name, the name default is used."
          },
          "resetMonitoringState": {
            "title": "Forget what this watch remembered",
            "type": "boolean",
            "description": "Start this watch over: forget the remembered answers before this run, so every URL is returned as a first check.",
            "default": false
          },
          "maxConcurrency": {
            "title": "URLs checked at the same time",
            "minimum": 1,
            "maximum": 8,
            "type": "integer",
            "description": "How many URLs are checked in parallel (1 to 8). Tries for one URL always run one after another.",
            "default": 4
          },
          "requestTimeoutSecs": {
            "title": "Time limit per request (seconds)",
            "minimum": 5,
            "maximum": 60,
            "type": "integer",
            "description": "How long to wait for one answer (5 to 60 seconds).",
            "default": 20
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}