{ "openapi": "3.0.1", "info": { "title": "Agent Fetch: URL to Clean Markdown", "description": "Turn any URL into clean, LLM-ready markdown: main content only, plus title, canonical URL, language, word count and links. Optional headless rendering for JavaScript pages, and deterministic JSON Schema field extraction with no LLM and no API key. robots.txt respected, failures come back labelled.", "version": "0.1", "x-build-id": "k2BOhj5TCqp37OSBy" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/omargnagy~agent-fetch-markdown/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-omargnagy-agent-fetch-markdown", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/omargnagy~agent-fetch-markdown/runs": { "post": { "operationId": "runs-sync-omargnagy-agent-fetch-markdown", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/omargnagy~agent-fetch-markdown/run-sync": { "post": { "operationId": "run-sync-omargnagy-agent-fetch-markdown", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "urls" ], "properties": { "urls": { "title": "URLs", "minItems": 1, "maxItems": 25, "type": "array", "description": "The pages to convert, up to 25 per run. Only http:// and https:// are accepted; anything else is rejected before a single request is made. Duplicates are removed and the fragment (#section) is dropped, so the same page is never fetched or charged twice.", "items": { "type": "string" } }, "render": { "title": "Render with a headless browser", "type": "boolean", "description": "Off by default. Leave it off for normal pages: a plain HTTP fetch is faster and cheaper. Turn it on only when a page builds its content with JavaScript and the plain fetch comes back nearly empty. When on, every URL in the run is loaded in the Chromium that ships inside this Actor's image, and each page is charged as a render event instead of a fetch event. No proxy is used in either path.", "default": false }, "respectRobotsTxt": { "title": "Respect robots.txt", "type": "boolean", "description": "On by default. The robots.txt of each host is fetched once per run and cached. A path that is disallowed for this Actor's user agent comes back as a record with status blocked_by_robots and is not charged. If robots.txt is missing, empty or unreachable, the URL is treated as allowed, which is what the standard says. Switch this off only for hosts you own or are licensed to crawl.", "default": true }, "timeoutSecs": { "title": "Timeout per URL (seconds)", "minimum": 5, "maximum": 120, "type": "integer", "description": "How long a single page may take before it is given up on. A URL that runs out of time becomes a record with status timeout and is not charged; the rest of the run continues.", "default": 30 }, "userAgent": { "title": "User agent", "type": "string", "description": "Sent with every request and used when matching robots.txt groups. Leave it as it is unless a site has asked you to identify differently.", "default": "Mozilla/5.0 (compatible; AgentFetchBot/1.0; +https://apify.com/omargnagy/agent-fetch-markdown)" }, "includeLinks": { "title": "Include the links list", "type": "boolean", "description": "On by default. Adds a links array to each successful record: the absolute URL, the anchor text and whether the link points off the page's own host. linkCount is always returned even when this is off.", "default": true }, "includeArticleJson": { "title": "Include the article JSON", "type": "boolean", "description": "Off by default. Adds an article object holding the page metadata that the readability pass and the page head expose: excerpt, byline, site name, published and modified time, description, lead image, and any JSON-LD blocks found in the page. Only the keys that actually have a value are included.", "default": false }, "maxMarkdownChars": { "title": "Maximum markdown characters", "minimum": 500, "maximum": 500000, "type": "integer", "description": "Markdown longer than this is cut at a paragraph boundary and a truncation marker is appended. charCount always reports the length after truncation. Raise it for long reference pages, lower it to keep records inside a small context window.", "default": 120000 }, "schema": { "title": "Field extraction schema (JSON Schema)", "type": "object", "description": "Optional. A JSON Schema object with a properties map, for example {\"properties\":{\"author\":{\"type\":\"string\"},\"datePublished\":{\"type\":\"string\",\"format\":\"date\"}}}. Each property is looked for in the page's JSON-LD, its meta and microdata tags, its definition lists and tables, and finally in labelled lines of the page text, then coerced to the declared type. This is deterministic parsing, not an LLM: no model is called, no API key is needed, and a field that cannot be found honestly is left out rather than guessed. When a schema is given and at least one field is found, the record gains an extracted object and an extract event is charged. Leave it empty to skip extraction entirely." } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } } }