{ "openapi": "3.0.1", "info": { "title": "Silverset | SQL, REST & Sheets to Analysis-Ready Data", "description": "Read SQL, REST APIs, Google Sheets, CSV/JSON files, a dataset, or pasted records. Raw rows land in Bronze, composable stages clean them, and analysis-ready Silver rows go to a Dataset or key-value store. Bad rows are quarantined, never dropped. Start with the Analysis-ready preset.", "version": "0.1", "x-build-id": "jcrWRaDOLKXLa2b0h" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/ahmedmohamed~silverset/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-ahmedmohamed-silverset", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/ahmedmohamed~silverset/runs": { "post": { "operationId": "runs-sync-ahmedmohamed-silverset", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/ahmedmohamed~silverset/run-sync": { "post": { "operationId": "run-sync-ahmedmohamed-silverset", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "recipe": { "title": "Recipe", "enum": [ "dataset", "file_url", "google_sheets", "inline", "json", "jsonl", "kvs", "mysql", "postgresql", "rest_api", "sqlite" ], "type": "string", "description": "What you are reading. Fill Where next. Put the password or token in Secret only if this recipe needs one (PostgreSQL, MySQL, REST, Google Sheets)." }, "locator": { "title": "Where", "type": "string", "description": "What you type depends on Recipe. • PostgreSQL / MySQL: user@host/database — optional :port (defaults 5432 / 3306). Password goes in Secret, not here: Where is NOT encrypted, so a password pasted inside a URL is stored in the run's INPUT record in the clear. A postgresql:// or mysql:// URL is also accepted (percent-encode a username containing @, e.g. user%40server); its own password wins over Secret if you include one. • SQLite: a file path, or :memory:. • File over HTTP: the file URL. • REST API: the endpoint URL, or URL | records_path when the array is nested. • Google Sheets: spreadsheetId, or spreadsheetId/Tab (default tab is Sheet1). • Dataset: the dataset id. • Key-value store: the record key, or storeId/key to read another store. • JSON / JSONL: paste the text. • Inline records: a JSON array of objects, e.g. [{\"id\": 1}]." }, "secret": { "title": "Secret", "type": "string", "description": "Password for PostgreSQL/MySQL, Bearer token for REST, or OAuth access token for Google Sheets. Leave empty for files, datasets, paste, and SQLite. Stored encrypted." }, "pii_salt": { "title": "PII hash salt", "type": "string", "description": "Hash pepper for the pii stage's hash strategy. When set, it overrides any salt in pipeline JSON for every pii step. Leave empty to use the stage-config salt — a salt typed into pipeline JSON is stored on the run INPUT in plaintext." }, "destination_type": { "title": "Destination", "enum": [ "apify_dataset", "apify_kv" ], "type": "string", "description": "Where cleaned Silver rows go. Apify Dataset is the usual choice — it is what the Console previews and what Export downloads. Key-value store writes one JSON key per batch instead, which is useful when a later step fetches whole batches by key.", "default": "apify_dataset" }, "destination_config": { "title": "Destination configuration", "type": "object", "description": "Usually leave empty. For apify_dataset: {\"dataset_name\": \"my-silver\"} for a named dataset — note that a named dataset accumulates across runs and full refresh does NOT empty it, so re-running appends a second copy; leave this empty to get this run's own fresh dataset. \"stream_field\" renames the _stream column. For apify_kv: {\"key_prefix\": \"run1-\"} — one key per batch, named {prefix}{stream}-{batch:06d}, so a re-run overwrites the same keys instead of duplicating rows." }, "pipeline_preset": { "title": "Pipeline preset", "enum": [ "pre_analysis", "custom" ], "type": "string", "description": "Start here. Analysis-ready (pre_analysis runs pii -> schema -> validate -> profile, which does not change values) profiles your data — it does not coerce CSV strings to numbers. Open Silver and PROFILE after the run. It drops no records either, with one exception: profile cannot compute a distinct-value key for a value nested past roughly 3000 levels, and that record is quarantined (STAGE_UNCAUGHT_ERROR) rather than reaching Silver — quarantined, never silently lost, and the run continues. Choose Custom only to map, convert types, dedup, or redact. Do not set a custom pipeline while this is still Analysis-ready.", "default": "pre_analysis" }, "pipeline": { "title": "Transform stages", "type": "array", "description": "Only when Pipeline preset is Custom. Ordered stages as [{\"stage\", \"config\"}]. Available: normalize, rename, map, drop, missing, types, derive, validate, dedup, pii, schema, profile, passthrough. Empty list is a passthrough. Put pii before schema/profile. Prefer keep=first on large dedup runs. After normalize.record_path, payload id is shared; key dedup on _record_id for one Silver row per child. See the README for a copy-paste example." }, "streams": { "title": "Streams", "type": "array", "description": "Leave unset to sync every discovered table in full_refresh. To limit tables, list [{\"name\": \"customers\"}]. Empty list means zero tables, not all of them." }, "batch_size": { "title": "Batch size", "minimum": 1, "maximum": 10000, "type": "integer", "description": "Records per batch moved through the pipeline.", "default": 500 }, "sync_key": { "title": "Sync key", "type": "string", "description": "Optional explicit cursor namespace. Set it to keep incremental cursors stable across config edits." }, "reset_cursor": { "title": "Reset cursor", "type": "boolean", "description": "Tick this to re-read an incremental table from scratch. Full-refresh tables (the default) already re-read every run.", "default": false }, "dry_run": { "title": "Dry run", "type": "boolean", "description": "Check config and list tables without writing Bronze, Silver, or a manifest. No row charges.", "default": false }, "quality_policy": { "title": "Quality policy", "enum": [ "fail_fast", "quarantine", "warn" ], "type": "string", "description": "What to do with a row a stage rejects. Fail fast writes every row the failing batch rejected to quarantine (DLQ), then aborts. Quarantine (recommended) writes it to datapipeline-quarantine and continues; MANIFEST.status is partial. Warn writes it to quarantine (DLQ) and counts it on MANIFEST.counts.quarantined; MANIFEST.status stays succeeded. Rejected rows never reach Silver. Missing-drop and dedup remove rows without a quarantine item.", "default": "quarantine" }, "fail_on_partial": { "title": "Fail on partial", "type": "boolean", "description": "Exit non-zero when quarantined records made the run partial. Default off: Apify can show SUCCEEDED while MANIFEST.status is partial. Turn on if a scheduler should fail on rejects. Fail fast already aborts. Warn never counts as partial.", "default": false }, "partial_failure": { "title": "On operational failure", "enum": [ "fail_fast", "continue" ], "type": "string", "description": "Legacy field, governs operational errors (a destination write or a whole stream failing) rather than data errors. fail_fast stops the run; continue dead-letters the batch and keeps going. Leave unset unless you need it — Quality policy above is the control for bad records." }, "source_type": { "title": "Source (API / n8n)", "type": "string", "description": "API and n8n send this instead of Recipe. When it is set, Recipe and Where are ignored. Where to read from. Raw rows land in Bronze before any cursor moves. One of: dataset, file_url, google_sheets, inline, json, jsonl, kvs, rest_api, sql." }, "source_config": { "title": "Source configuration (API / n8n)", "type": "object", "description": "One JSON object describing the source you picked. Copy the line for your source and edit it — the README has the same examples with notes. • sql: {\"tables\": [\"customers\"]} — put the URL in Database connection string below, or set \"connection_string\" here. • file_url: {\"url\": \"https://example.com/data.csv\"}. • rest_api: {\"url\": \"https://api.example.com/v1/items\", \"records_path\": \"data\"} — \"pagination\" defaults to \"none\" (one request, one page); add \"pagination\": \"page\" with \"page_param\"/\"max_pages\" to read more (offset-style APIs like $offset/skip aren't supported yet), since this box is the only way to reach paging today. • google_sheets: {\"spreadsheet_id\": \"1AbC...\", \"sheets\": [\"Sheet1\"]}. • dataset: {\"datasetId\": \"aBcD1234\"}. • kvs: {\"key\": \"OUTPUT\"} — add \"storeId\" to read another run's store. • inline: {\"records\": [{\"id\": 1}]}. • json / jsonl: {\"text\": \"[{\\\"id\\\": 1}]\"}. Credentials go in the encrypted boxes below, not in here. Optional on file_url, json, jsonl and kvs: \"format\" when the extension and Content-Type are both unreliable, and \"repair\": true to fix malformed JSON (trailing commas, single quotes) instead of failing — off by default, because a quiet wrong guess is worse than a stopped run." }, "sql_connection_string": { "title": "SQL database: Connection string", "type": "string", "description": "Used only when Source is SQL database. A SQLAlchemy async URL for a read-only user — postgresql+asyncpg://USER:PASSWORD@HOST:5432/DB, mysql+aiomysql://USER:PASSWORD@HOST:3306/DB, or sqlite+aiosqlite:///data.db for a file. Stored encrypted, which a URL pasted into Source configuration is not. Hidden on the Store form; Recipe + Where builds this for Console runs." }, "rest_api_auth_token": { "title": "REST API: Bearer token", "type": "string", "description": "Used only when Source is REST API, and optional. Sent as Authorization: Bearer … . Stored encrypted. Hidden on the Store form; Secret fills this for REST Console runs." }, "google_sheets_access_token": { "title": "Google Sheets: Access token", "type": "string", "description": "Used only when Source is Google Sheets. An OAuth access token with read scope on that sheet. Fill this or API key, never both. Stored encrypted. Hidden on the Store form; Secret fills this for Sheets Console runs." }, "google_sheets_api_key": { "title": "Google Sheets: API key", "type": "string", "description": "Used only when Source is Google Sheets. Fill this or Access token, never both. Stored encrypted. Hidden on the Store form; send this field from the API when you authenticate with a key instead of Secret." }, "profile_policy": { "title": "Profiling policy", "type": "object", "description": "enabled=false skips the profile stage. sample_size is how many rows schema inspects (default 1000). Profiling never rewrites values.", "default": { "enabled": true, "sample_size": 1000 } }, "bronze": { "title": "Bronze landing", "type": "object", "description": "dataset_name selects the dataset raw records land in before the cursor advances. Bronze is kept separate from Silver on purpose: it is the replay source, so every item carries record_id and run_id." } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } } }