{ "openapi": "3.0.1", "info": { "title": "edX Course Catalog Scraper", "description": "Scrapes edX's online course catalog by subject. Returns name, description, skills taxonomy, provider/university, upgrade price, rating, historical enrollment, and every past and upcoming run with instructors and schedule dates.", "version": "0.1", "x-build-id": "Y7JGYVQJNYceybTam" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/scrapyx~edx-courses-scraper/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-scrapyx-edx-courses-scraper", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/scrapyx~edx-courses-scraper/runs": { "post": { "operationId": "runs-sync-scrapyx-edx-courses-scraper", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/scrapyx~edx-courses-scraper/run-sync": { "post": { "operationId": "run-sync-scrapyx-edx-courses-scraper", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "subjects": { "title": "Subjects", "type": "array", "description": "Subject names to crawl — 'computer-science', 'business-management', 'artificial-intelligence', 'data-analysis' and 525 more. Resolved live each run against edX's own sitemap, which encodes the real subject taxonomy in its course URL structure (/learn/{subject}/{course-slug}) — this is read directly off the site, not a fixed list that can go stale. Leave empty to crawl the single largest subject as a demo. An unrecognised subject is REFUSED with suggestions rather than silently returning zero rows, since this target has no keyword search to fall back on.", "default": [], "items": { "type": "string" } }, "maxItems": { "title": "Max courses per subject", "minimum": 0, "type": "integer", "description": "Stop after this many courses per subject. Set to 0 for unlimited — the sitemap enumeration is exhaustive (525 subjects, 5,298 course URLs total; computer-science alone has 350). Size this to what you actually need.", "default": 50 }, "includeCourseDetails": { "title": "Fetch full course details", "type": "boolean", "description": "Fetch each course's detail page: name, description, skills taxonomy, provider/university, education level, language, every past and upcoming run with instructor names and schedule dates, and whether it's free to audit. Turn this off to get just the URL list from the sitemap (fast, but nearly no data — the sitemap carries no titles or metadata of its own).", "default": true }, "maxConcurrency": { "title": "Max concurrent requests", "minimum": 1, "maximum": 16, "type": "integer", "description": "How many requests may be in flight at once. This absorbs latency variance; it is NOT the throttle — the crawl rate is set by 'Minimum interval between requests' below.", "default": 6 }, "minRequestInterval": { "title": "Minimum interval between requests (seconds)", "minimum": 0.05, "maximum": 10, "type": "number", "description": "The actual speed control: the shortest gap between two requests to edX, across all workers. No bot mitigation was observed, which is worth preserving.", "default": 0.4 }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "description": "No bot mitigation was observed on the sitemap or course detail pages (6/6 TLS profiles clean). Residential is still the cloud default, since a target being clean during recon is not a guarantee.", "default": { "useApifyProxy": true, "apifyProxyGroups": [ "RESIDENTIAL" ] } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } } }