{
  "openapi": "3.0.1",
  "info": {
    "title": "JavaScript Website Crawler",
    "description": "Crawl a public website in a headless browser and get each page as it looks after its scripts run: clean text, Markdown, links, metadata and chunks.",
    "version": "0.1",
    "x-build-id": "RFQpkVcov4P3hEvBR"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/dami_studio~javascript-website-crawler/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-dami_studio-javascript-website-crawler",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/dami_studio~javascript-website-crawler/runs": {
      "post": {
        "operationId": "runs-sync-dami_studio-javascript-website-crawler",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/dami_studio~javascript-website-crawler/run-sync": {
      "post": {
        "operationId": "run-sync-dami_studio-javascript-website-crawler",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "required": [
          "startUrls"
        ],
        "properties": {
          "startUrls": {
            "title": "Start URLs",
            "type": "array",
            "description": "Public web addresses to start from. Links to other pages of the same site are followed from each one.",
            "items": {
              "type": "object",
              "required": [
                "url"
              ],
              "properties": {
                "url": {
                  "type": "string",
                  "title": "URL of a web page",
                  "format": "uri"
                }
              }
            }
          },
          "maxPages": {
            "title": "Pages per start URL",
            "minimum": 1,
            "maximum": 1000,
            "type": "integer",
            "description": "How many pages to read from each start URL, pages that could not be read included. The run's maximum cost can stop it sooner.",
            "default": 10
          },
          "maxDepth": {
            "title": "Link depth",
            "minimum": 0,
            "maximum": 20,
            "type": "integer",
            "description": "How many clicks away from the start URL to go. 0 reads only the start pages.",
            "default": 1
          },
          "maxPagesPerRun": {
            "title": "Pages for the whole run",
            "minimum": 1,
            "maximum": 100000,
            "type": "integer",
            "description": "A page limit shared by all start URLs together. Leave it empty to keep only the limit per start URL."
          },
          "maxConcurrency": {
            "title": "Pages at once",
            "minimum": 1,
            "maximum": 10,
            "type": "integer",
            "description": "How many pages of one site are worked on at the same time.",
            "default": 3
          },
          "useRobotsTxt": {
            "title": "Respect robots.txt",
            "type": "boolean",
            "description": "Honour the site's robots.txt: paths it disallows are not read, and its crawl delay is kept (10 seconds at most).",
            "default": true
          },
          "requestTimeoutSecs": {
            "title": "Request timeout",
            "minimum": 3,
            "maximum": 60,
            "type": "integer",
            "description": "Seconds the first read of a page may take. The browser then gets twice this, between 20 and 60 seconds, to load the page.",
            "default": 15
          },
          "maxResponseSizeKb": {
            "title": "Maximum page size",
            "minimum": 128,
            "maximum": 8192,
            "type": "integer",
            "description": "The biggest page accepted, in kilobytes. A page that grows past twice this (1 MB at least) once its scripts have run is refused too, as a free note.",
            "default": 2048
          },
          "waitForSelector": {
            "title": "Wait for this element (CSS selector)",
            "type": "string",
            "description": "Wait up to 5 seconds for an element matching this CSS selector before reading the page, for example .product-list. A page where it never appears comes back as a free SELECTOR_NOT_FOUND note."
          },
          "renderWaitSecs": {
            "title": "Extra wait (seconds)",
            "minimum": 0,
            "maximum": 5,
            "type": "integer",
            "description": "Wait this long after the page has settled before reading it, 0 to 5.",
            "default": 0
          },
          "scrollToBottom": {
            "title": "Scroll down to load more",
            "type": "boolean",
            "description": "Scroll to the bottom a few times, so content that only loads as you scroll is there.",
            "default": false
          },
          "expandSections": {
            "title": "Open collapsed sections",
            "type": "boolean",
            "description": "Open details boxes and buttons marked as collapsed, such as FAQ answers. Links, forms and send buttons are never clicked.",
            "default": false
          },
          "blockMedia": {
            "title": "Skip images, fonts and styles",
            "type": "boolean",
            "description": "Don't load images, video, fonts or stylesheets, so pages load faster. If a page comes back with less text with it on, run that page without it.",
            "default": false
          },
          "includeUrlGlobs": {
            "title": "Only follow links matching",
            "type": "array",
            "description": "Follow a link only when it matches one of these patterns. * matches inside one folder, ** across folders. A pattern starting with / is matched against the path and query (/docs/**), a full address against the whole URL. Start URLs are always read. Up to 20 patterns.",
            "items": {
              "type": "string"
            }
          },
          "excludeUrlGlobs": {
            "title": "Never follow links matching",
            "type": "array",
            "description": "Never follow a link that matches one of these patterns, for example /tag/** or **?page=*. Same pattern rules as above.",
            "items": {
              "type": "string"
            }
          },
          "stayUnderStartPath": {
            "title": "Stay inside the start folder",
            "type": "boolean",
            "description": "Only follow links inside the start URL's folder: a crawl started at /docs/ stays in /docs/ and leaves /docs-old/ alone.",
            "default": false
          },
          "includeSubdomains": {
            "title": "Include subdomains",
            "type": "boolean",
            "description": "Treat subdomains of the start site as the same site, so a crawl of example.com also reads docs.example.com and help.example.com.",
            "default": false
          },
          "followNavigationLinks": {
            "title": "Follow menu, header and footer links",
            "type": "boolean",
            "description": "Also follow links in menus, headers and footers. Without it only links in the body of the page are followed, which can leave a docs site with a sidebar at one page.",
            "default": false
          },
          "skipFileLinks": {
            "title": "Skip links to files",
            "type": "boolean",
            "description": "Don't request links that end in a file type: PDFs, office files, images, media, archives, scripts, styles and feeds.",
            "default": false
          },
          "countOnlyGoodPages": {
            "title": "Only real pages use up the limit",
            "type": "boolean",
            "description": "Pages that could not be read don't count toward Pages per start URL. The crawl still stops after twice that many tries.",
            "default": false
          },
          "useSitemap": {
            "title": "Start from the sitemap too",
            "type": "boolean",
            "description": "Also start from the pages in the site's sitemap (named in robots.txt, or /sitemap.xml), sitemap indexes included. A start URL ending in .xml is read as a sitemap.",
            "default": false
          },
          "requestLanguage": {
            "title": "Request language",
            "type": "string",
            "description": "The language list sent with each page, for sites that answer in the visitor's language, for example de-DE,de;q=0.9. Empty means US English."
          },
          "outputFormat": {
            "title": "Text format",
            "enum": [
              "json",
              "markdown",
              "both"
            ],
            "type": "string",
            "description": "Keep clean text, Markdown, or both in each page row.",
            "default": "both"
          },
          "markdownStyle": {
            "title": "Markdown style",
            "enum": [
              "basic",
              "rich"
            ],
            "type": "string",
            "description": "Basic keeps headings, paragraphs and lists. Rich also keeps links, bold and italics, tables and code blocks. Chunks are always cut from the plain text.",
            "default": "basic"
          },
          "contentExtractor": {
            "title": "How the main text is found",
            "enum": [
              "default",
              "article",
              "wholePage"
            ],
            "type": "string",
            "description": "Main content reads the page's main or article element, or the body when there is none. Article reader picks the block with the most paragraph text. Whole page reads the body.",
            "default": "default"
          },
          "keepSelector": {
            "title": "Keep only this part (CSS selector)",
            "type": "string",
            "description": "Keep only the parts of the page that match this CSS selector, for example article .content. When nothing matches, the normal text is kept and keepSelectorMatched is false."
          },
          "removeSelector": {
            "title": "Remove these parts (CSS selector)",
            "type": "string",
            "description": "Remove the parts of the page that match this CSS selector before the text is read, for example .sidebar, .newsletter. Links are still followed from them."
          },
          "removeCookieBanners": {
            "title": "Remove cookie dialogs",
            "type": "boolean",
            "description": "Remove cookie and consent dialogs from the text, so their wording stays out of the first chunk.",
            "default": false
          },
          "dropRepeatedLines": {
            "title": "Drop repeated text blocks",
            "type": "boolean",
            "description": "Drop paragraphs and other text blocks already seen on an earlier page of the same site, such as sidebars and sign-up boxes. repeatedLinesDropped counts what went.",
            "default": false
          },
          "includeHtml": {
            "title": "Include cleaned HTML",
            "type": "boolean",
            "description": "Add the cleaned HTML of the content to each row as html, as it stood after the scripts ran. Rows get several times bigger.",
            "default": false
          },
          "query": {
            "title": "Rank chunks by these words",
            "type": "string",
            "description": "Words that score each chunk's relevance, so you can keep the best chunks for a question. Plain word matching, no AI."
          },
          "chunkSizeChars": {
            "title": "Chunk size",
            "minimum": 300,
            "maximum": 5000,
            "type": "integer",
            "description": "The most characters in each chunk of text.",
            "default": 1200
          },
          "chunkOverlapChars": {
            "title": "Chunk overlap",
            "minimum": 0,
            "maximum": 800,
            "type": "integer",
            "description": "Characters repeated between neighbouring chunks, so a sentence cut in two still reads in one of them.",
            "default": 120
          },
          "saveSkippedUrls": {
            "title": "Save the links that were not followed",
            "type": "boolean",
            "description": "Save the links your rules left out, with the reason, in the key-value store record SKIPPED_URLS. Up to 10,000.",
            "default": false
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}