{ "openapi": "3.0.1", "info": { "title": "LLM Dataset Cleaner: Near-Duplicate Dedupe, Language, Tokens", "description": "Clean crawled or scraped text for RAG and fine-tuning: exact and MinHash near-duplicate removal, language filter, quality flags, PII masking, token counts.", "version": "1.0", "x-build-id": "6FSYmhydBHT0fG4d8" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/bongo-seakhoa~llm-dataset-cleaner/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-bongo-seakhoa-llm-dataset-cleaner", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/bongo-seakhoa~llm-dataset-cleaner/runs": { "post": { "operationId": "runs-sync-bongo-seakhoa-llm-dataset-cleaner", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/bongo-seakhoa~llm-dataset-cleaner/run-sync": { "post": { "operationId": "run-sync-bongo-seakhoa-llm-dataset-cleaner", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "datasetId": { "title": "Dataset to clean", "type": "string", "description": "Pick a dataset, for example the output of Website Content Crawler or of the PDF to Markdown Actor. Give exactly one source: a dataset, a key-value store record, or a file URL. With no source at all, the Actor cleans a small bundled synthetic sample (300 records) so you can see the output; the sample is never charged." }, "keyValueStoreId": { "title": "Key-value store with a file (optional)", "type": "string", "description": "A key-value store holding a JSONL, JSON array or CSV file. Use it together with the record key below." }, "keyValueStoreKey": { "title": "Record key (optional)", "type": "string", "description": "Key of the JSONL, JSON or CSV record in the store picked above." }, "sourceUrl": { "title": "File URL (optional)", "type": "string", "description": "A direct http(s) link to a JSONL, JSON array or CSV file (UTF-8, UTF-16 or Windows-1252). Only public internet hosts are fetched. The Actor does not crawl web pages: run Website Content Crawler first and pick its dataset above." }, "preset": { "title": "Input preset", "enum": [ "generic", "website-content-crawler", "pdf-actor-chunks" ], "type": "string", "description": "'website-content-crawler' reads 'markdown' (falling back to 'text'), 'url' and 'metadata.title' and keeps the Markdown heading path for chunks. 'pdf-actor-chunks' reads the chunk rows of the PDF to Markdown Actor (record_type 'chunk'); its other rows are skipped and not charged. 'generic' reads the text fields below.", "default": "generic" }, "textFields": { "title": "Text fields", "type": "array", "description": "Fields holding the text (generic preset). Several fields are joined with a blank line. Dotted paths such as 'metadata.description' reach into nested objects.", "default": [ "text" ], "items": { "type": "string" } }, "idField": { "title": "ID field (optional)", "type": "string", "description": "Field with each record's own ID, copied to record_id. Without it, record_id is the record's position in the source (0-based)." }, "keepFields": { "title": "Fields to copy through", "type": "array", "description": "Other fields copied unchanged into each output row under 'fields'.", "default": [], "items": { "type": "string" } }, "mode": { "title": "Mode", "enum": [ "clean", "estimate", "manifest" ], "type": "string", "description": "'clean' runs the pipeline. 'estimate' only counts the records and shows the projected price; it charges none of this Actor's events (Apify's standard Actor start fee still applies). 'manifest' compares the records with a previous run's manifest and returns an incremental update plan for a RAG index (see the README).", "default": "clean" }, "normalize": { "title": "Normalise text", "type": "boolean", "description": "Unicode NFKC, control characters removed, runs of spaces collapsed, line structure kept. Off: output text is the original text (dedup still compares normalised text).", "default": true }, "exactDedup": { "title": "Remove exact duplicates", "type": "boolean", "description": "Records whose normalised text is identical to an earlier record are rejected as EXACT_DUPLICATE. The first occurrence is kept.", "default": true }, "nearDedup": { "title": "Remove near duplicates", "type": "boolean", "description": "MinHash (128 permutations, 5-word shingles) with LSH. Records at or above the similarity threshold to an earlier record are rejected as NEAR_DUPLICATE. The first occurrence is kept.", "default": true }, "nearDupThreshold": { "title": "Near-duplicate threshold", "minimum": 0.5, "maximum": 0.99, "type": "number", "description": "Estimated Jaccard similarity of 5-word shingles at which a record counts as a near duplicate. Lower values catch looser copies and use more memory.", "default": 0.85 }, "languages": { "title": "Keep only these languages", "type": "array", "description": "ISO 639-1 codes such as 'en' or 'de' ('no' keeps both Norwegian forms, nb and nn). Empty: detect the language only, filter nothing. Records in other languages, or too short to tell, are rejected as LANGUAGE_FILTERED.", "default": [], "items": { "type": "string" } }, "minLanguageConfidence": { "title": "Minimum language confidence", "minimum": 0, "maximum": 1, "type": "number", "description": "Below this confidence the language is reported as null.", "default": 0.5 }, "quality": { "title": "Quality heuristics", "enum": [ "off", "flag", "reject" ], "type": "string", "description": "Rules of thumb (length, letter, symbol and digit ratios, repeated lines, placeholder text, cookie banners). 'flag' only reports them in quality_flags; 'reject' also rejects the record.", "default": "flag" }, "minChars": { "title": "Minimum characters", "minimum": 0, "maximum": 200000, "type": "integer", "description": "Shorter records get the TOO_SHORT flag.", "default": 50 }, "maxChars": { "title": "Maximum characters", "minimum": 1, "maximum": 200000, "type": "integer", "description": "Longer records (after normalising) are always rejected as TOO_LONG, whatever 'Quality rules' says, and are not processed further. At most 200,000 characters (about 50,000 tokens).", "default": 200000 }, "pii": { "title": "Personal data", "enum": [ "off", "flag", "mask" ], "type": "string", "description": "Emails, phone numbers in international format, IBANs (checksum), card numbers (Luhn) and IP addresses. 'flag' counts them per record; 'mask' also replaces them with [EMAIL], [PHONE] and so on, in the text, url, title and kept fields (in manifest mode: in the text). This reduces personal data; it does not remove names, addresses or national IDs.", "default": "flag" }, "piiTypes": { "title": "Personal data types", "type": "array", "description": "Which kinds of personal data to look for.", "items": { "type": "string", "enum": [ "email", "phone", "iban", "card", "ipv4", "ipv6" ], "enumTitles": [ "Email", "Phone", "IBAN", "Card number", "IPv4", "IPv6" ] }, "default": [ "email", "phone", "iban", "card", "ipv4", "ipv6" ] }, "tokenizer": { "title": "Tokenizer", "enum": [ "o200k_base", "cl100k_base" ], "type": "string", "description": "Exact token counts with this OpenAI tokenizer (GPT-4o family: o200k_base; GPT-4 and GPT-3.5: cl100k_base).", "default": "o200k_base" }, "chunking": { "title": "Split kept records into RAG chunks", "type": "boolean", "description": "Kept records become chunk rows (record_type 'chunk') instead of record rows. Each chunk is charged as 'chunk-written' on top of the per-record price.", "default": false }, "chunkSize": { "title": "Chunk size (characters)", "minimum": 100, "maximum": 100000, "type": "integer", "description": "Target maximum length of each chunk. Chunks break at headings, paragraphs and sentences.", "default": 1500 }, "chunkOverlap": { "title": "Chunk overlap (characters)", "minimum": 0, "maximum": 50000, "type": "integer", "description": "Characters repeated between consecutive chunks. At most half the chunk size.", "default": 150 }, "chunkMaxTokens": { "title": "Maximum tokens per chunk (optional)", "minimum": 64, "maximum": 100000, "type": "integer", "description": "When a chunk has more tokens than this (with the tokenizer above), the record is chunked again with a smaller size until every chunk fits, down to 100 characters. A chunk that still does not fit is delivered with over_max_tokens: true." }, "exportFormat": { "title": "Export file", "enum": [ "text-jsonl", "rag-chunks", "none" ], "type": "string", "description": "'text-jsonl' writes cleaned.jsonl with one {\"text\": ...} line per kept record (fine-tuning format). 'rag-chunks' writes chunks.jsonl with every chunk row (needs chunking). rejected.jsonl with the reasons is always written.", "default": "text-jsonl" }, "includeRejectedInDataset": { "title": "Also put rejected records in the dataset", "type": "boolean", "description": "Rejected records always go to rejected.jsonl. On: they also become dataset rows with kept = false.", "default": false }, "maxRecords": { "title": "Maximum records", "minimum": 1, "maximum": 500000, "type": "integer", "description": "Stop after this many records (up to 500,000 per run). The SUMMARY then says RECORD_CAP_REACHED. Split larger datasets over several runs; deduplication only sees records within one run.", "default": 100000 }, "manifestScope": { "title": "Manifest scope", "type": "string", "description": "Manifest mode: a name for the collection this input covers, for example 'docs-site'. Only previous entries of the same scope are compared; others are carried forward untouched." }, "sourceIdField": { "title": "Source ID field", "type": "string", "description": "Manifest mode: the field identifying each item's source. Defaults: 'url' (Website Content Crawler preset), 'source' (PDF Actor chunks), else the ID field." }, "locatorFields": { "title": "Locator fields", "type": "array", "description": "Manifest mode: fields that locate an item inside its source; joined with '|'. Default: 'chunk_index' for PDF Actor chunks, none otherwise.", "items": { "type": "string" } }, "parserVersion": { "title": "Parser version", "type": "string", "description": "Manifest mode: the version of whatever produced the text (for example 'pdf-markdown-actor 1.0'). When it differs from the previous manifest, unchanged items are marked reindex_required.", "default": "unspecified" }, "chunkerVersion": { "title": "Chunker version", "type": "string", "description": "Manifest mode: the version of your chunking settings. A change marks unchanged items reindex_required.", "default": "unspecified" }, "completeSnapshot": { "title": "Input is a complete snapshot of the scope", "type": "boolean", "description": "Manifest mode: only when this is on (and the whole input was read without errors, caps or charge limits) are items missing from the input proposed for deletion. Off: missing items stay 'unknown' and are carried forward.", "default": false }, "previousManifestKeyValueStoreId": { "title": "Previous manifest: key-value store", "type": "string", "description": "Manifest mode: the key-value store of an earlier manifest run (its NEXT_MANIFEST.jsonl). Leave empty for the first run." }, "previousManifestKey": { "title": "Previous manifest: record key", "type": "string", "description": "Key of the manifest in the store above.", "default": "NEXT_MANIFEST.jsonl" }, "previousManifestDatasetId": { "title": "Previous manifest: dataset", "type": "string", "description": "Manifest mode, instead of the store: the dataset of an earlier manifest run. Its manifest_item rows with in_next_manifest = true are read." }, "includeTextInManifest": { "title": "Include text in upsert rows", "type": "boolean", "description": "Manifest mode: put the normalised text into rows whose action is 'upsert', ready to embed.", "default": true } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }