{
  "openapi": "3.0.1",
  "info": {
    "title": "Website Content Crawler to Markdown by Country",
    "description": "Crawl a website and get each page's main content as clean Markdown or text for RAG, LLM and AI agents, fetched from the country you pick through residential (home) connections. Geo-targeted, with proof of the country for every page. JavaScript when needed. Blocked pages are free.",
    "version": "0.1",
    "x-build-id": "HVHSfkYFlCTKY8Fh1"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/zaiq~website-crawler-by-country/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-zaiq-website-crawler-by-country",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/zaiq~website-crawler-by-country/runs": {
      "post": {
        "operationId": "runs-sync-zaiq-website-crawler-by-country",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/zaiq~website-crawler-by-country/run-sync": {
      "post": {
        "operationId": "run-sync-zaiq-website-crawler-by-country",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "required": [
          "startUrls",
          "country"
        ],
        "properties": {
          "startUrls": {
            "title": "Start URLs",
            "type": "array",
            "description": "Where the crawl starts. By default it follows links under each start URL's folder: https://docs.example.com/guide/ crawls https://docs.example.com/guide/...",
            "items": {
              "type": "object",
              "required": [
                "url"
              ],
              "properties": {
                "url": {
                  "type": "string",
                  "title": "URL of a web page",
                  "format": "uri"
                }
              }
            }
          },
          "country": {
            "title": "Country",
            "type": "string",
            "description": "Every page is fetched through a home (residential) internet connection in this country, confirmed by two IP-location services. Pick one, or type any two-letter country code.",
            "default": "us"
          },
          "maxPages": {
            "title": "Max pages",
            "minimum": 1,
            "maximum": 10000,
            "type": "integer",
            "description": "The crawl stops after this many pages have been delivered. Only delivered pages are charged.",
            "default": 50
          },
          "maxDepth": {
            "title": "Max link depth",
            "minimum": 0,
            "maximum": 100,
            "type": "integer",
            "description": "How many links away from a start URL the crawl may go. 0 fetches only the start URLs.",
            "default": 20
          },
          "crawlScope": {
            "title": "Links to follow",
            "enum": [
              "path",
              "hostname",
              "domain"
            ],
            "type": "string",
            "description": "Which links found on the pages are followed. Include patterns below, when given, replace this.",
            "default": "path"
          },
          "includeUrlPatterns": {
            "title": "Only follow URLs matching",
            "type": "array",
            "description": "Optional. Globs such as https://docs.example.com/api/** or /blog/*, or regular expressions starting with re: (for example re:/v[0-9]+/). * matches anything except /, ** matches anything. When set, these decide which links are followed instead of the option above.",
            "default": [],
            "items": {
              "type": "string"
            }
          },
          "excludeUrlPatterns": {
            "title": "Never follow URLs matching",
            "type": "array",
            "description": "Optional. Same pattern format, for example /tag/** or re:[?&]page=. Always applied.",
            "default": [],
            "items": {
              "type": "string"
            }
          },
          "useSitemaps": {
            "title": "Also find pages in the site's sitemaps",
            "type": "boolean",
            "description": "Read the sitemaps listed in robots.txt (or /sitemap.xml) and queue their pages that are in scope, as well as following links.",
            "default": false
          },
          "renderMode": {
            "title": "JavaScript rendering",
            "enum": [
              "auto",
              "never",
              "always"
            ],
            "type": "string",
            "description": "Automatic fetches each page plainly first and loads it in a real browser only when the plain page is missing content that JavaScript adds; the browser price is charged only when the browser found more text.",
            "default": "auto"
          },
          "outputFormats": {
            "title": "Output",
            "uniqueItems": true,
            "type": "array",
            "description": "Markdown keeps headings, lists, tables, links and code blocks. Full HTML is saved as a file in the run's key-value store and linked from each row.",
            "items": {
              "type": "string",
              "enum": [
                "markdown",
                "text",
                "html"
              ],
              "enumTitles": [
                "Markdown",
                "Plain text",
                "Full HTML (saved as a file)"
              ]
            },
            "default": [
              "markdown"
            ]
          },
          "removeBoilerplate": {
            "title": "Remove navigation, headers, footers and cookie banners",
            "type": "boolean",
            "description": "Keep only the page's main content. Switch off to get all of the page's text.",
            "default": true
          },
          "respectRobotsTxt": {
            "title": "Respect robots.txt",
            "type": "boolean",
            "description": "Skip pages the site's robots.txt disallows for all crawlers (User-agent: *), and wait between requests when it sets a Crawl-delay (up to 10 seconds).",
            "default": true
          },
          "includeFailedPages": {
            "title": "List failed and blocked pages in the results",
            "type": "boolean",
            "description": "Add a free row for each page that failed, was blocked or returned an error (4xx/5xx). Switch off to keep only delivered pages, for example when the results go straight into a vector database.",
            "default": true
          },
          "maxConcurrency": {
            "title": "Pages at once",
            "minimum": 1,
            "maximum": 50,
            "type": "integer",
            "description": "How many pages are fetched at the same time. Never more than 10 at once from one site.",
            "default": 8
          },
          "maxTriesPerPage": {
            "title": "Addresses to try per page",
            "minimum": 1,
            "maximum": 8,
            "type": "integer",
            "description": "If a page fails or meets a bot wall, it is tried again through another address in the country, up to this many times. Failed tries are free.",
            "default": 3
          },
          "acceptLanguage": {
            "title": "Accept-Language header",
            "type": "string",
            "description": "Leave empty to send the country's usual browser language (for example de-DE in Germany). Set it to ask every page for one language."
          },
          "contentSelector": {
            "title": "Main content element (CSS selector)",
            "type": "string",
            "description": "Optional. Take the text only from elements matching this selector, for example article.post or #content. Leave empty to find the main content automatically."
          },
          "removeSelector": {
            "title": "Remove these elements (CSS selector)",
            "type": "string",
            "description": "Optional. Elements to drop before the text is taken, for example .related-posts, .ad-slot."
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}