{ "openapi": "3.0.1", "info": { "title": "Agent Fetch: URL to Markdown for LLMs and RAG", "description": "Returns the main content of any web page URL as clean Markdown for LLMs, RAG and AI agents, with title, canonical URL, language, word count and links. Up to 25 URLs per run, automatic JavaScript rendering, optional JSON Schema field extraction with no LLM. Failed pages are free.", "version": "0.1", "x-build-id": "lxqVi71RPG2uNkvII" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/omargnagy~agent-fetch-markdown/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-omargnagy-agent-fetch-markdown", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/omargnagy~agent-fetch-markdown/runs": { "post": { "operationId": "runs-sync-omargnagy-agent-fetch-markdown", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/omargnagy~agent-fetch-markdown/run-sync": { "post": { "operationId": "run-sync-omargnagy-agent-fetch-markdown", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "urls": { "title": "URLs", "minItems": 1, "maxItems": 25, "type": "array", "description": "Web pages to convert to Markdown, 1 to 25 per run. http and https only. Duplicates and #fragments are removed, so a page is never fetched or charged twice.", "items": { "type": "string" } }, "url": { "title": "URL (single page)", "type": "string", "description": "One page, for callers that send {\"url\": \"...\"}. It is added in front of urls and counts toward the 25-page limit." }, "render": { "title": "Render every page in a headless browser", "type": "boolean", "description": "Load every page in headless Chromium instead of a plain HTTP fetch, charged as a render instead of a fetch. Leave it off: autoRender already handles JavaScript-only pages. Rendering does not get past bot protection.", "default": false }, "autoRender": { "title": "Render JavaScript app shells automatically", "type": "boolean", "description": "When a plain fetch returns an empty JavaScript app shell, load that one page in the browser and charge a render instead of a fetch. Off: such pages come back as needs_render and are not charged.", "default": true }, "includeLinks": { "title": "Include the links list", "type": "boolean", "description": "Add a links array to each successful page: absolute URL, anchor text and whether the link leaves the site. linkCount is always returned.", "default": true }, "includeArticleJson": { "title": "Include the article metadata", "type": "boolean", "description": "Add an article object: excerpt, byline, site name, published and modified dates, description, lead image and the page's JSON-LD blocks. Only keys with a value are included.", "default": false }, "maxMarkdownChars": { "title": "Maximum Markdown characters per page", "minimum": 500, "maximum": 500000, "type": "integer", "description": "Longer Markdown is cut at the last paragraph or line break that fits and ends with a [truncated by maxMarkdownChars] note. charCount, note included, never exceeds this number. Lower it to fit a small context window.", "default": 120000 }, "schema": { "title": "Fields to extract (JSON Schema)", "type": "object", "description": "Optional. A JSON Schema with a properties map, for example {\"properties\":{\"author\":{\"type\":\"string\"}}}. Fields are read from the page's JSON-LD, meta tags and label/value pairs, with no LLM. extract is charged only when the page itself states a field." }, "respectRobotsTxt": { "title": "Respect robots.txt", "type": "boolean", "description": "Skip paths that robots.txt disallows for AgentFetchBot. Skipped pages come back as blocked_by_robots and are not charged. Turn it off only for sites you own or are licensed to crawl.", "default": true }, "timeoutSecs": { "title": "Timeout per page (seconds)", "minimum": 5, "maximum": 120, "type": "integer", "description": "Time limit for loading one page: the HTTP request, or the page load in the browser. A page that runs out of time comes back as status timeout, is not charged, and the rest of the run continues. Converting the page to Markdown is not counted.", "default": 30 }, "userAgent": { "title": "User agent", "type": "string", "description": "Sent with every request. robots.txt is always matched as AgentFetchBot, whatever you send here. Leave it unless a site asks you to identify differently.", "default": "Mozilla/5.0 (compatible; AgentFetchBot/1.0; +https://apify.com/omargnagy/agent-fetch-markdown)" } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }