diff --git a/components/apify/README.md b/components/apify/README.md index a34f3990bac5f..1cde1a46a5e99 100644 --- a/components/apify/README.md +++ b/components/apify/README.md @@ -14,4 +14,4 @@ The Apify API unleashes the power to automate web scraping, process data, and or **Real Estate Listings Aggregator** -- Create a workflow where Apify actors periodically fetch the latest real estate listings from multiple websites. Process and normalize the data with Pipedream's code steps, then automatically update a Google Sheet that serves as a central repository for all listings, keeping potential buyers informed with the latest options. +- Create a workflow where Apify Actors periodically fetch the latest real estate listings from multiple websites. Process and normalize the data with Pipedream's code steps, then automatically update a Google Sheet that serves as a central repository for all listings, keeping potential buyers informed with the latest options. diff --git a/components/apify/actions/get-dataset-items/get-dataset-items.mjs b/components/apify/actions/get-dataset-items/get-dataset-items.mjs index c705dd0d72ab1..62d847f8b78e5 100644 --- a/components/apify/actions/get-dataset-items/get-dataset-items.mjs +++ b/components/apify/actions/get-dataset-items/get-dataset-items.mjs @@ -1,11 +1,12 @@ import apify from "../../apify.app.mjs"; import { LIMIT } from "../../common/constants.mjs"; +import { ConfigurationError } from "@pipedream/platform"; export default { key: "apify-get-dataset-items", - name: "Get Dataset Items", + name: "Get dataset items", description: "Returns data stored in a dataset. [See the documentation](https://docs.apify.com/api/v2/dataset-items-get)", - version: "0.0.6", + version: "0.0.7", annotations: { destructiveHint: false, openWorldHint: true, @@ -52,32 +53,44 @@ export default { }, }, async run({ $ }) { - const params = { - limit: LIMIT, - offset: this.offset, - clean: this.clean, - fields: this.fields, - omit: this.omit, - }; + const { + clean, fields, omit, limit, + } = this; + const datasetId = this.datasetId?.replace("/", "~"); + const offset = this.offset ?? 0; + + if (limit !== undefined && limit < 1) { + throw new ConfigurationError("Limit must be 1 or greater."); + } + if (offset < 0) { + throw new ConfigurationError("Offset must be 0 or greater."); + } const results = []; - let total; + let currentOffset = offset; - do { + while (limit === undefined || results.length < limit) { + const pageSize = limit === undefined + ? LIMIT + : Math.min(LIMIT, limit - results.length); const { items } = await this.apify.listDatasetItems({ - datasetId: this.datasetId, - params, + datasetId, + params: { + offset: currentOffset, + limit: pageSize, + clean, + fields, + omit, + }, }); + if (!items?.length) { + break; + } results.push(...items); - if (results.length >= this.limit) { + currentOffset += items.length; + if (items.length < pageSize) { break; } - total = items?.length; - params.offset += LIMIT; - } while (total); - - if (results.length > this.limit) { - results.length = this.limit; } if (results.length > 0) { diff --git a/components/apify/actions/get-kvs-record/get-kvs-record.mjs b/components/apify/actions/get-kvs-record/get-kvs-record.mjs index ae2b6bfda4d55..30a8bc5e5a376 100644 --- a/components/apify/actions/get-kvs-record/get-kvs-record.mjs +++ b/components/apify/actions/get-kvs-record/get-kvs-record.mjs @@ -3,9 +3,14 @@ import apify from "../../apify.app.mjs"; export default { key: "apify-get-kvs-record", - name: "Get Key-Value Store Record", + name: "Get key-value store record", description: "Gets a record from a key-value store.", - version: "0.0.5", + version: "0.0.6", + annotations: { + destructiveHint: false, + openWorldHint: true, + readOnlyHint: true, + }, type: "action", props: { apify, diff --git a/components/apify/actions/run-actor/run-actor.mjs b/components/apify/actions/run-actor/run-actor.mjs index 2bb968289455f..d05e9e2191164 100644 --- a/components/apify/actions/run-actor/run-actor.mjs +++ b/components/apify/actions/run-actor/run-actor.mjs @@ -1,13 +1,19 @@ /* eslint-disable no-unused-vars */ import apify from "../../apify.app.mjs"; -import { parseObject } from "../../common/utils.mjs"; +import { + getMemoryLimits, buildMemoryProp, validateMemory, +} from "../../common/memory.mjs"; import { WEBHOOK_EVENT_TYPES } from "@apify/consts"; +import { ConfigurationError } from "@pipedream/platform"; + +// Max OUTPUT record size (bytes) returned inline; oversized values get a reference object. +const MAX_OUTPUT_BYTES = 256 * 1024; export default { key: "apify-run-actor", name: "Run Actor", description: "Performs an execution of a selected Actor in Apify. [See the documentation](https://docs.apify.com/api/v2#/reference/actors/run-collection/run-actor)", - version: "0.0.7", + version: "0.0.8", annotations: { destructiveHint: false, openWorldHint: true, @@ -19,7 +25,7 @@ export default { actorSource: { type: "string", label: "Search Actors from", - description: "Where to search for Actors. Valid options are Store and Recently used Actors.", + description: "Where to search for Actors. Choose **Apify Store Actors** to browse the public [Apify Store](https://apify.com/store), or **Recently used Actors** to pick from Actors you've run before.", options: [ { label: "Apify Store Actors", @@ -54,10 +60,11 @@ export default { reloadProps: true, optional: true, }, - runAsynchronously: { + // Renamed from `runAsynchronously` for clarity. + waitForFinish: { type: "boolean", - label: "Run Asynchronously", - description: "Set to `true` to run the Actor asynchronously", + label: "Wait for finish", + description: "If `true` (default), the step waits for the Actor run to finish and returns its output. If `false`, the step starts the run and returns immediately with the run details, without waiting.", reloadProps: true, default: true, }, @@ -67,12 +74,6 @@ export default { description: "Optional timeout for the run, in seconds. By default, the run uses a timeout specified in the default run configuration for the Actor.", optional: true, }, - memory: { - type: "string", - label: "Memory (MB)", - description: "Memory limit for the run, in megabytes. The amount of memory can be set to a power of 2 with a minimum of 128. By default, the run uses a memory limit specified in the default run configuration for the Actor.", - optional: true, - }, maxItems: { type: "string", label: "Max Items", @@ -94,7 +95,52 @@ export default { }, }, methods: { + outputByteSize(value) { + if (value == null) return 0; + if (Buffer.isBuffer(value)) return value.length; + if (typeof value === "string") return Buffer.byteLength(value); + try { + return Buffer.byteLength(JSON.stringify(value)); + } catch { + // Unserializable (e.g. circular) -> treat as oversized so we never return it inline. + return Infinity; + } + }, + // Returns { output, capped }, where capped indicates if the value was replaced by a reference. + async capOutputRecord(record, keyValueStoreId, recordKey) { + if (record?.value == null) { + return { + output: undefined, + capped: false, + }; + } + const size = this.outputByteSize(record.value); + if (size <= MAX_OUTPUT_BYTES) { + return { + output: record.value, + capped: false, + }; + } + return { + capped: true, + output: { + truncated: true, + message: + "The OUTPUT record exceeds the safe step-output size and was not returned inline. " + + "Retrieve it via `recordUrl`, or use the Get key-value store record action.", + keyValueStoreId, + recordKey, + contentType: record.contentType, + size, + // getKVSRecordUrl -> apify-client getRecordPublicUrl is async; must await or the + // unresolved Promise serializes to `{}` in the step output. + recordUrl: await this.apify.getKVSRecordUrl(keyValueStoreId, recordKey), + }, + }; + }, getType(type) { + // Pipedream has no float type, so numbers are input as strings + if (type === "number") return "string"; return [ "string", "object", @@ -104,14 +150,25 @@ export default { ? type : "string[]"; }, - async getSchema(actorId, buildTag) { + parseNumericInput(value, key) { + const num = Number(value); + if (value == null || value === "" || Number.isNaN(num)) { + throw new ConfigurationError( + `Input "${key}" must be a valid number, but received: ${JSON.stringify(value)}.`, + ); + } + return num; + }, + async getBuildOrThrow(actorId, buildTag) { const build = await this.apify.getBuild(actorId, buildTag); if (!build) { - throw new Error(`No build found for actor ${actorId}`); + throw new Error(`No build found for Actor ${actorId}`); } - + return build; + }, + extractInputSchema(build, actorId) { // Case 1: schema is already an object - if (build.actorDefinition && build.actorDefinition.input) { + if (build.actorDefinition?.input) { return build.actorDefinition.input; } @@ -123,19 +180,38 @@ export default { : build.inputSchema; } catch (err) { throw new Error( - `Failed to parse inputSchema for actor ${actorId}: ${err.message}`, + `Failed to parse inputSchema for Actor ${actorId}: ${err.message}`, ); } } - // Case 3: no schema at all - throw new Error( - `No input schema found for actor ${actorId}. Has it been built successfully?`, + // Case 3: no schema at all (e.g. apify/hello-world) + const noSchemaError = new Error( + `No input schema found for Actor ${actorId}. Has it been built successfully?`, ); + noSchemaError.noInputSchema = true; + throw noSchemaError; + }, + async getSchema(actorId, buildTag) { + const build = await this.getBuildOrThrow(actorId, buildTag); + return this.extractInputSchema(build, actorId); }, - async prepareData(data) { + async prepareData(data, schema) { + let resolvedSchema = schema; + if (resolvedSchema === undefined) { + // No schema passed by the caller: fetch it, tolerating Actors with none. + try { + resolvedSchema = await this.getSchema(this.actorId, this.buildTag); + } catch (err) { + if (err?.noInputSchema) return data; + throw err; + } + } + // No input schema (e.g. apify/hello-world): send the raw input as-is. + if (!resolvedSchema) return data; + const newData = {}; - const { properties } = await this.getSchema(this.actorId, this.buildTag); + const { properties } = resolvedSchema; // Iterate over properties from the schema because newData might contain additional fields for (const [ @@ -145,22 +221,34 @@ export default { const propValue = data[key]; if (propValue === undefined) continue; + if (value.type === "number" || value.type === "integer") { + if (Array.isArray(propValue)) { + newData[key] = propValue.map((item) => this.parseNumericInput(item, key)); + } else if (propValue !== "") { + newData[key] = this.parseNumericInput(propValue, key); + } + continue; + } + const editor = value.editor || "hidden"; newData[key] = Array.isArray(propValue) - ? propValue.map((item) => this.setValue(editor, item)) - : this.setValue(editor, propValue); + ? propValue.map((item) => this.setValue(editor, item, key)) + : this.setValue(editor, propValue, key); } return newData; }, prepareOptions(value) { if (value.enum && value.enumTitles) { - return value.enum.map((val, i) => ({ - value: val, - label: value.enumTitles[i], - })); + // Drop options with an empty or null label + return value.enum + .map((val, i) => ({ + value: val, + label: value.enumTitles[i], + })) + .filter(({ label }) => label !== "" && label != null); } }, - setValue(editor, item) { + setValue(editor, item, key) { switch (editor) { case "requestListSources": return { @@ -176,7 +264,15 @@ export default { }; case "json": case "schemaBased": - if (typeof item === "string") return JSON.parse(item); + if (typeof item === "string") { + try { + return JSON.parse(item); + } catch { + throw new ConfigurationError( + `Input "${key}" must be valid JSON.`, + ); + } + } return item; default: return item; @@ -185,8 +281,29 @@ export default { }, async additionalProps() { const props = {}; + + // Show a hint if user set actorId to sentinel value "". + // Displayed only when user has no recently-used Actors. + if (this.actorId === "") { + return { + actorHint: { + type: "alert", + alertType: "info", + content: "No Actor selected. Set **Search Actors from** to **Apify Store Actors** above, then choose an Actor to run.", + }, + }; + } + + // Not picked yet (undefined): show nothing rather than erroring. + if (!this.actorId) { + return props; + } + + let memoryLimits = getMemoryLimits(); try { - const schema = await this.getSchema(this.actorId, this.buildTag); + const build = await this.getBuildOrThrow(this.actorId, this.buildTag); + memoryLimits = getMemoryLimits(build); + const schema = this.extractInputSchema(build, this.actorId); const { properties, required: requiredProps = [], } = schema; @@ -239,14 +356,22 @@ export default { } } } catch (e) { + if (!e?.noInputSchema) { + throw e; + } props.properties = { type: "object", label: "Properties", - description: e.message || "Schema not available, showing fallback.", + description: "This Actor has no input schema. Provide a raw JSON input object, or leave it empty to run the Actor with its own defaults.", + optional: true, + default: {}, }; } - if (!this.runAsynchronously) { + // Actor memory dropdown, filtered by per-actor limits. + props.memory = buildMemoryProp(memoryLimits); + + if (this.waitForFinish) { props.outputRecordKey = { type: "string", label: "Output Record Key", @@ -271,9 +396,10 @@ export default { async run({ $ }) { const { apify, + actorSource, actorId, buildTag, - runAsynchronously, + waitForFinish, outputRecordKey, timeout, memory, @@ -281,6 +407,7 @@ export default { maxTotalChargeUsd, webhook, eventTypes, + properties, ...data } = this; @@ -295,28 +422,52 @@ export default { if (!actorDetails.stats?.totalBuilds || actorDetails.stats.totalBuilds === 0) { throw new Error( - `Actor "${actorDetails.title || actorDetails.name}" has no builds. Please build it first before running.`, + `Actor "${actorDetails.title || actorDetails.name}" has no builds yet and can't be run until it's built. Open the Actor in Apify Console and build it (Source → Code → Build), or trigger a build via the Apify CLI/API, then run this step again.`, ); } - if (buildTag) { - const taggedBuilds = actorDetails.taggedBuilds || {}; - if (!taggedBuilds[buildTag]) { - throw new Error( - `Build with tag "${buildTag}" was not found for actor "${actorDetails.title || actorDetails.name}".`, - ); + // Fetch build once for both schema and memory limits. + const build = await this.getBuildOrThrow(actorId, buildTag); + const { + min: minMemory, max: maxMemory, + } = getMemoryLimits(build); + + // Extract the input schema, tolerating Actors that have none (e.g. + // apify/hello-world), which run with the raw input passed through. + let schema = null; + try { + schema = this.extractInputSchema(build, actorId); + } catch (err) { + if (!err?.noInputSchema) { + throw err; } } - // Prepare input - // Use data (dynamic props from schema) if it has any keys, - // otherwise fall back to this.properties (fallback object prop) + // Validate memory is a power of two within allowed limits. + const validatedMemory = validateMemory(memory, { + min: minMemory, + max: maxMemory, + }); + + // Prepare input: use schema-driven fields when present, else the raw JSON + // `properties` fallback (schema-less Actors). Fail clearly on malformed JSON + // instead of silently passing it through as a string. + let fallback = {}; + if (typeof properties === "string") { + if (properties.trim() !== "") { + try { + fallback = JSON.parse(properties); + } catch { + throw new ConfigurationError("The Properties field must contain valid JSON."); + } + } + } else if (properties) { + fallback = properties; + } const rawInput = Object.keys(data).length > 0 ? data - : (this.properties - ? parseObject(this.properties) - : {}); - const input = await this.prepareData(rawInput); + : fallback; + const input = await this.prepareData(rawInput, schema); // Build params safely const params = { @@ -326,8 +477,8 @@ export default { ...(timeout && { timeout: Number(timeout), }), - ...(memory && { - memory: Number(memory), + ...(validatedMemory && { + memory: validatedMemory, }), ...(maxItems && { maxItems: Number(maxItems), @@ -347,8 +498,8 @@ export default { let run; - if (runAsynchronously) { - // async run + if (!waitForFinish) { + // async run — start and return immediately without waiting run = await apify.runActorAsynchronously({ actorId, data: input, @@ -365,20 +516,21 @@ export default { options: params, }); - // Fetch OUTPUT record manually + // Fetch OUTPUT record and guard its size before returning it inline. let output; + let capped = false; if (run.defaultKeyValueStoreId) { - const record = await apify - ._client() - .keyValueStore(run.defaultKeyValueStoreId) - .getRecord(outputRecordKey); - - output = record?.value; + const record = await apify.getKVSRecord(run.defaultKeyValueStoreId, outputRecordKey); + ({ + output, capped, + } = await this.capOutputRecord(record, run.defaultKeyValueStoreId, outputRecordKey)); } - $.export( "$summary", - `The run of an Actor with ID: ${actorId} has finished with status "${run.status}".`, + `The run of an Actor with ID: ${actorId} has finished with status "${run.status}".` + + (capped + ? " OUTPUT was too large to return inline; a reference URL is included." + : ""), ); return { diff --git a/components/apify/actions/run-task-synchronously/run-task-synchronously.mjs b/components/apify/actions/run-task-synchronously/run-task-synchronously.mjs deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/components/apify/actions/run-task/run-task.mjs b/components/apify/actions/run-task/run-task.mjs index e75fbc93e6e50..9bc9168789cb8 100644 --- a/components/apify/actions/run-task/run-task.mjs +++ b/components/apify/actions/run-task/run-task.mjs @@ -1,13 +1,22 @@ import apify from "../../apify.app.mjs"; +import { + buildMemoryProp, getMemoryLimits, validateMemory, +} from "../../common/memory.mjs"; import { ACTOR_JOB_STATUSES, ACTOR_JOB_TERMINAL_STATUSES, WEBHOOK_EVENT_TYPES, } from "@apify/consts"; +import { ConfigurationError } from "@pipedream/platform"; export default { key: "apify-run-task", - name: "Run Task", - description: "Run a specific task and optionally wait for it's termination.", - version: "0.0.4", + name: "Run task", + description: "Run a specific task and optionally wait for its termination.", + version: "0.0.5", + annotations: { + destructiveHint: false, + openWorldHint: true, + readOnlyHint: false, + }, type: "action", props: { apify, @@ -20,6 +29,7 @@ export default { }), ], description: "The ID of the task to run", + reloadProps: true, }, waitForFinish: { type: "boolean", @@ -30,29 +40,47 @@ export default { }, overrideInput: { type: "string", - label: "Override Input", - description: "Optional JSON string to override the default input for the task run. Must be valid JSON.", + label: "Override Input (JSON)", + description: "Optional JSON object to override the task's saved input for this run. Leave empty to use the task's saved input. Must be valid JSON (e.g. `{}` for empty input).", optional: true, }, timeout: { type: "integer", - label: "Timeout", + label: "Timeout (seconds)", description: "Optional timeout for the run, in seconds. By default, the run uses a timeout specified in the task settings.", optional: true, }, - memory: { - type: "integer", - label: "Memory", - description: "Memory limit for the run, in megabytes. The amount of memory can be set to a power of 2 with a minimum of 128. By default, the run uses a memory limit specified in the task settings.", - optional: true, - }, build: { type: "string", label: "Build", description: "Specifies the Actor build to run. It can be either a build tag or build number. By default, the run uses the build specified in the task settings (typically latest).", optional: true, + reloadProps: true, }, }, + methods: { + async taskMemoryLimits() { + if (!this.taskId) { + return getMemoryLimits(); + } + const task = await this.apify.getTask(this.taskId); + const build = task?.actId + ? await this.apify.getBuild(task.actId, this.build || task.options?.build) + : null; + return getMemoryLimits(build); + }, + }, + async additionalProps() { + let memoryLimits; + try { + memoryLimits = await this.taskMemoryLimits(); + } catch { + memoryLimits = getMemoryLimits(); + } + return { + memory: buildMemoryProp(memoryLimits), + }; + }, async run({ $ }) { const POLL_INTERVAL_MS = 30_000; // 30s @@ -63,19 +91,23 @@ export default { try { input = JSON.parse(this.overrideInput); } catch (error) { - throw new Error(`Failed to parse override Input JSON: ${error.message}`); + throw new ConfigurationError(`Failed to parse Override Input JSON: ${error.message}`); } } // Helper: start task const startTask = async () => { + const params = { + timeout: this.timeout, + build: this.build, + }; + // Validate memory when set; empty uses task's saved value. + if (this.memory) { + params.memory = validateMemory(this.memory, await this.taskMemoryLimits()); + } return this.apify.runTask({ taskId: this.taskId, - params: { - timeout: this.timeout, - memory: this.memory, - build: this.build, - }, + params, input, }); }; @@ -94,7 +126,7 @@ export default { // Helper: schedule next poll (rerun) with 30s interval and 1-day cap const schedulePoll = (runId, webhookId) => { const startEpoch = - ($.context.run?.context && $.context.run.context.pollStartMs) || + $.context.run?.context?.pollStartMs || $.context.pollStartMs || Date.now(); @@ -125,7 +157,7 @@ export default { const isRerun = typeof runCtx.runs === "number" && runCtx.runs > 1; // RESUME DATA (if the webhook called the resume_url) - const resumeBody = $.$resume_data && $.$resume_data.body; + const resumeBody = $.$resume_data?.body; // 3) RERUN/RESUME BEHAVIOR if (resumeBody || isRerun) { diff --git a/components/apify/actions/scrape-single-url/scrape-single-url.mjs b/components/apify/actions/scrape-single-url/scrape-single-url.mjs index 1020953700a8c..1c1cb4ad36417 100644 --- a/components/apify/actions/scrape-single-url/scrape-single-url.mjs +++ b/components/apify/actions/scrape-single-url/scrape-single-url.mjs @@ -1,12 +1,13 @@ import apify from "../../apify.app.mjs"; import { WCC_ACTOR_ID } from "../../common/constants.mjs"; import { ACTOR_JOB_STATUSES } from "@apify/consts"; +import { ConfigurationError } from "@pipedream/platform"; export default { key: "apify-scrape-single-url", - name: "Scrape Single URL", + name: "Scrape single URL", description: "Executes a scraper on a specific website and returns its content as HTML. This action is perfect for extracting content from a single page. [See the documentation](https://docs.apify.com/sdk/js/docs/examples/crawl-single-url)", - version: "0.1.3", + version: "0.1.4", annotations: { destructiveHint: false, openWorldHint: true, @@ -24,29 +25,50 @@ export default { crawlerType: { type: "string", label: "Crawler Type", - description: "Select the crawling engine:\n- **Headless web browser** - Useful for modern websites with anti-scraping protections and JavaScript rendering. It recognizes common blocking patterns like CAPTCHAs and automatically retries blocked requests through new sessions. However, running web browsers is more expensive as it requires more computing resources and is slower. It is recommended to use at least 8 GB of RAM.\n- **Stealthy web browser** (default) - Another headless web browser with anti-blocking measures enabled. Try this if you encounter bot protection while scraping. For best performance, use with Apify Proxy residential IPs. \n- **Raw HTTP client** - High-performance crawling mode that uses raw HTTP requests to fetch the pages. It is faster and cheaper, but it might not work on all websites.", + description: "Select the crawling engine:\n- **Adaptive** - Automatically switches between raw HTTP for static pages and a headless browser for dynamic pages to get the maximum performance wherever possible.\n- **Firefox (Headless Browser)** (default) - Headless Firefox with Playwright and anti-blocking measures enabled. Reliable, renders JavaScript content, and best at avoiding blocking, but might be slow. For best performance, use with Apify Proxy residential IPs.\n- **Cheerio (Raw HTTP)** - High-performance crawling mode that uses raw HTTP requests to fetch the pages. Fastest and cheapest, but doesn't render JavaScript content.", options: [ { - label: "Headless browser (stealthy Firefox+Playwright) - Very reliable, best in avoiding blocking, but might be slow", - value: "playwright:firefox", + label: "Adaptive", + value: "playwright:adaptive", }, { - label: "Headless browser (Chrome+Playwright) - Reliable, but might be slow", - value: "playwright:chrome", + label: "Firefox (Headless Browser)", + value: "playwright:firefox", }, { - label: "Raw HTTP client (Cheerio) - Extremely fast, but cannot handle dynamic content", + label: "Cheerio (Raw HTTP)", value: "cheerio", }, - { - label: "The crawler automatically switches between raw HTTP for static pages and Chrome browser (via Playwright) for dynamic pages, to get the maximum performance wherever possible.", - value: "playwright:adaptive", - }, ], default: "playwright:firefox", }, }, + methods: { + // new URL() accepts hosts with empty labels (e.g. "google..com"), so check explicitly + validateUrl(url) { + let parsedUrl; + try { + parsedUrl = new URL(url); + } catch { + throw new ConfigurationError(`Invalid URL "${url}": could not be parsed. Use a valid absolute URL like https://example.com.`); + } + + if (![ + "http:", + "https:", + ].includes(parsedUrl.protocol)) { + throw new ConfigurationError(`Invalid URL "${url}": only http and https protocols are supported. Use a valid absolute URL like https://example.com.`); + } + + if (parsedUrl.hostname.split(".").some((label) => label.length === 0)) { + throw new ConfigurationError(`Invalid URL "${url}": host contains an empty label. Use a valid absolute URL like https://example.com.`); + } + }, + }, async run({ $ }) { + const url = this.url?.trim(); + this.validateUrl(url); + const { status, defaultDatasetId, @@ -60,7 +82,7 @@ export default { maxResults: 1, startUrls: [ { - url: this.url, + url, }, ], }, @@ -74,7 +96,7 @@ export default { datasetId: defaultDatasetId, }); - $.export("$summary", "Run of Web Content Crawler finished successfully."); + $.export("$summary", "Scraped the URL successfully."); return items[0]; }, }; diff --git a/components/apify/actions/set-key-value-store-record/set-key-value-store-record.mjs b/components/apify/actions/set-key-value-store-record/set-key-value-store-record.mjs index faf8cebb9e94f..3de501107aeaa 100644 --- a/components/apify/actions/set-key-value-store-record/set-key-value-store-record.mjs +++ b/components/apify/actions/set-key-value-store-record/set-key-value-store-record.mjs @@ -2,7 +2,7 @@ import apify from "../../apify.app.mjs"; export default { key: "apify-set-key-value-store-record", - name: "Set Key-Value Store Record", + name: "Set key-value store record", description: "Create or update a record in an Apify Key-Value Store. Supports strings, numbers, booleans, null, arrays, and objects. Automatically infers content type (JSON vs. plain text).", version: "0.2.4", @@ -26,7 +26,7 @@ export default { }, key: { type: "string", - label: "Key", + label: "Record key", description: "The key of the record to create or update.", optional: false, }, diff --git a/components/apify/apify.app.mjs b/components/apify/apify.app.mjs index 75ae9935a556f..13f7133a0dc6c 100644 --- a/components/apify/apify.app.mjs +++ b/components/apify/apify.app.mjs @@ -7,7 +7,7 @@ export default { propDefinitions: { keyValueStoreId: { type: "string", - label: "Key-Value Store Id", + label: "Key-Value Store ID", description: "The Id of the key-value store.", async options({ page, unnamed = true, @@ -29,7 +29,7 @@ export default { actorId: { type: "string", label: "Actor", - description: "Select the Actor, enter the Actor ID, or use a tilde-separated combination of the owner's username and the Actor name.", + description: "Select an Actor from the dropdown, enter an Actor ID, or use a tilde-separated owner/name identifier (e.g. `apify~web-scraper`). If the list is empty, switch \"Search Actors from\" to **Apify Store Actors**.", async options({ page, actorSource, }) { @@ -62,7 +62,7 @@ export default { datasetId: { type: "string", label: "Dataset ID", - description: "The ID of the dataset to retrieve items within", + description: "Select a dataset, or enter a Dataset ID or `username/dataset-name`", async options({ page }) { const { items } = await this.listDatasets({ offset: LIMIT * page, @@ -81,15 +81,25 @@ export default { buildTag: { type: "string", label: "Build", - description: "Specifies the Actor build to run. The accepted value is the build tag. If not provided, the default build will be used.", + description: "Actor build to run. Pick a tag from the list or enter a build number (e.g. `0.1.2`). Defaults to the Actor's default build.", async options({ actorId }) { - const { taggedBuilds } = await this.getActor({ + if (!actorId) { + return []; + } + + const { taggedBuilds = {} } = await this.getActor({ actorId, }); return Object.entries(taggedBuilds).map(([ - name, - ]) => name); + tag, + info, + ]) => ({ + label: info?.buildNumber + ? `${tag} (${info.buildNumber})` + : tag, + value: tag, + })); }, }, clean: { @@ -119,15 +129,15 @@ export default { limit: { type: "integer", label: "Limit", - description: "The maximum number of items to return", - default: LIMIT, + description: "The maximum number of items to return. Leave empty to return all items", + min: 1, optional: true, }, offset: { type: "integer", label: "Offset", - description: "The number records to skip before returning results", - default: 0, + description: "The number of records to skip before returning results. Leave empty to start from the first item", + min: 0, optional: true, }, }, @@ -158,6 +168,15 @@ export default { limit: LIMIT, }); + if (page === 0 && items.length === 0 && actorSource !== "store") { + return [ + { + label: "No recent Actors, switch to \"Apify Store Actors\" above", + value: "", + }, + ]; + } + return items.map((actor) => ({ label: this.formatActorOrTaskLabel(actor), value: actor.id, @@ -196,12 +215,15 @@ export default { return this._client().task(taskId) .start(input, params); }, + getTask(taskId) { + return this._client().task(taskId) + .get(); + }, getActor({ actorId }) { return this._client().actor(actorId) .get(); }, - async getBuild(actorId, buildTag) { - // Get actor details + async resolveBuildId(actorId, buildRef) { const actor = await this._client().actor(actorId) .get(); @@ -209,20 +231,35 @@ export default { throw new Error(`Actor ${actorId} not found.`); } - if (!buildTag) { - buildTag = actor.defaultRunOptions.build; + if (!buildRef) { + buildRef = actor.defaultRunOptions.build; } - const { taggedBuilds } = actor; + const taggedBuilds = actor.taggedBuilds ?? {}; - if (taggedBuilds[buildTag]) { - return this._client().build(taggedBuilds[buildTag].buildId) - .get(); - } else { - throw new Error( - `Actor ${actorId} has no build tagged "${buildTag}". Please build the actor first.`, - ); + if (taggedBuilds[buildRef]?.buildId) { + return taggedBuilds[buildRef].buildId; } + + const { items: builds = [] } = await this.listBuilds({ + actorId, + desc: true, + limit: LIMIT, + }); + const match = builds.find(({ buildNumber }) => buildNumber === buildRef); + + if (match) { + return match.id; + } + + throw new Error( + `No build with tag or number "${buildRef}" found for actor ${actorId}.`, + ); + }, + async getBuild(actorId, buildRef) { + const buildId = await this.resolveBuildId(actorId, buildRef); + return this._client().build(buildId) + .get(); }, listActors(opts = {}) { return this._client().store() @@ -240,10 +277,12 @@ export default { return this._client().tasks() .list(opts); }, - listBuilds({ actorId }) { + listBuilds({ + actorId, ...opts + }) { return this._client().actor(actorId) .builds() - .list(); + .list(opts); }, listKeyValueStores(opts = {}) { return this._client().keyValueStores() @@ -267,12 +306,6 @@ export default { return this._client().keyValueStore(kvsId) .getRecordPublicUrl(recordKey); }, - runTaskSynchronously({ - taskId, params, input, - }) { - return this._client().task(taskId) - .call(input, params); - }, setKeyValueStoreRecord({ storeId, key, value, contentType, }) { diff --git a/components/apify/common/constants.mjs b/components/apify/common/constants.mjs index fdf2a04b3a3c2..ef13a469313ea 100644 --- a/components/apify/common/constants.mjs +++ b/components/apify/common/constants.mjs @@ -1,2 +1,25 @@ export const WCC_ACTOR_ID = "aYG0l9s7dbB7j3gbS"; export const LIMIT = 100; + +// Apify platform memory limits: memory must be a power of two, from 128 MB to 32 GB. +// See https://docs.apify.com/actors/running/usage-and-resources#memory +export const MIN_MEMORY_MBYTES = 128; +export const MAX_MEMORY_MBYTES = 32768; + +function memoryLabel(mb) { + return mb >= 1024 + ? `${mb / 1024} GB` + : `${mb} MB`; +} + +// Ordered dropdown options for the run memory limit (128 MB -> 32 GB). +export const MEMORY_MBYTES_OPTIONS = (() => { + const options = []; + for (let mb = MIN_MEMORY_MBYTES; mb <= MAX_MEMORY_MBYTES; mb *= 2) { + options.push({ + label: memoryLabel(mb), + value: mb, + }); + } + return options; +})(); diff --git a/components/apify/common/memory.mjs b/components/apify/common/memory.mjs new file mode 100644 index 0000000000000..19595edb901b1 --- /dev/null +++ b/components/apify/common/memory.mjs @@ -0,0 +1,67 @@ +import { ConfigurationError } from "@pipedream/platform"; +import { + MEMORY_MBYTES_OPTIONS, MIN_MEMORY_MBYTES, MAX_MEMORY_MBYTES, +} from "./constants.mjs"; + +// Converts a value to integer or returns null for invalid/missing input. +function toMemoryInt(value) { + if (value === null || value === undefined || value === "") { + return null; + } + const num = Number(value); + return Number.isInteger(num) + ? num + : null; +} + +// Valid memory steps (powers of two, 128 MB–32 GB) within [min, max], for dropdown and validation. +function memoryOptions({ + min, max, +}) { + const filtered = MEMORY_MBYTES_OPTIONS.filter(({ value }) => value >= min && value <= max); + return filtered.length + ? filtered + : MEMORY_MBYTES_OPTIONS; +} + +// Reads the Actor's declared memory limits from an already-fetched build, +// falling back to the platform limits (128 MB - 32 GB) when not declared. +export function getMemoryLimits(build = null) { + const { + minMemoryMbytes, maxMemoryMbytes, + } = build?.actorDefinition ?? {}; + return { + min: toMemoryInt(minMemoryMbytes) ?? MIN_MEMORY_MBYTES, + max: toMemoryInt(maxMemoryMbytes) ?? MAX_MEMORY_MBYTES, + }; +} + +// Builds the ordered memory dropdown prop for [min, max], with a description +// that reflects the range shown. +export function buildMemoryProp(limits) { + const options = memoryOptions(limits); + const low = options[0].label; + const high = options.at(-1).label; + return { + type: "integer", + label: "Memory (MB)", + description: `Memory limit for the run, in megabytes. Must be a power of two between ${low} and ${high}. By default, the run uses the memory limit specified in the run configuration.`, + optional: true, + options, + }; +} + +// Checks if memory is a valid allowed value; throws if not, else returns the value or undefined. +export function validateMemory(memory, limits) { + if (memory === undefined || memory === null || memory === "") { + return undefined; + } + const options = memoryOptions(limits); + const value = Number(memory); + if (!options.some((option) => option.value === value)) { + throw new ConfigurationError( + `Memory "${memory}" MB is not valid. Choose a power of two between ${options[0].label} and ${options.at(-1).label}.`, + ); + } + return value; +} diff --git a/components/apify/common/utils.mjs b/components/apify/common/utils.mjs deleted file mode 100644 index dcc9cc61f6f41..0000000000000 --- a/components/apify/common/utils.mjs +++ /dev/null @@ -1,24 +0,0 @@ -export const parseObject = (obj) => { - if (!obj) return undefined; - - if (Array.isArray(obj)) { - return obj.map((item) => { - if (typeof item === "string") { - try { - return JSON.parse(item); - } catch (e) { - return item; - } - } - return item; - }); - } - if (typeof obj === "string") { - try { - return JSON.parse(obj); - } catch (e) { - return obj; - } - } - return obj; -}; diff --git a/components/apify/package.json b/components/apify/package.json index c7d06090e0233..4804ff154bb6a 100644 --- a/components/apify/package.json +++ b/components/apify/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/apify", - "version": "0.3.1", + "version": "0.3.2", "description": "Pipedream Apify Components", "main": "apify.app.mjs", "keywords": [ diff --git a/components/apify/sources/common/base.mjs b/components/apify/sources/common/base.mjs index 6d6c2672d96de..67969cdd8a17d 100644 --- a/components/apify/sources/common/base.mjs +++ b/components/apify/sources/common/base.mjs @@ -1,6 +1,8 @@ +import { createHash } from "crypto"; import { WEBHOOK_EVENT_TYPE_GROUPS, WEBHOOK_EVENT_TYPES, } from "@apify/consts"; +import { ConfigurationError } from "@pipedream/platform"; import apify from "../../apify.app.mjs"; export default { @@ -11,19 +13,76 @@ export default { type: "$.interface.http", customResponse: true, }, + eventTypes: { + type: "string[]", + label: "Trigger on run states", + description: "Which terminal run states should fire this trigger. Leave empty to fire on all of them (the default).", + optional: true, + options: WEBHOOK_EVENT_TYPE_GROUPS.ACTOR_RUN_TERMINAL.map((value) => { + const state = value.split(".").pop() + .replaceAll("_", " ") + .toLowerCase(); + return { + label: state.charAt(0).toUpperCase() + state.slice(1), + value, + }; + }), + }, + }, + methods: { + // SHA-256(endpoint + condition target) - stable across re-deploys so Apify + // reuses the same webhook instead of creating duplicates. + getIdempotencyKey() { + const target = Object.values(this.getCondition()).join(":"); + return createHash("sha256") + .update(`${this.http.endpoint}:${target}`) + .digest("hex"); + }, }, hooks: { async activate() { + // Block empty or unset condition targets. + if (Object.values(this.getCondition()).some((value) => !value)) { + throw new ConfigurationError( + this.getEmptyConditionMessage?.() + ?? "No target selected. Select an Actor or task before deploying.", + ); + } + + // Whitelist terminal states only; fall back to all when none are valid. + const selected = (this.eventTypes ?? []) + .filter((type) => WEBHOOK_EVENT_TYPE_GROUPS.ACTOR_RUN_TERMINAL.includes(type)); + const response = await this.apify.createHook({ requestUrl: this.http.endpoint, - eventTypes: WEBHOOK_EVENT_TYPE_GROUPS.ACTOR_RUN_TERMINAL, + eventTypes: selected.length + ? selected + : WEBHOOK_EVENT_TYPE_GROUPS.ACTOR_RUN_TERMINAL, condition: this.getCondition(), + idempotencyKey: this.getIdempotencyKey(), }); this.db.set("webhookId", response.id); }, async deactivate() { const webhookId = this.db.get("webhookId"); - await this.apify.deleteHook(webhookId); + if (!webhookId) { + return; + } + + try { + await this.apify.deleteHook(webhookId); + // Forget the webhook after it's deleted. + this.db.set("webhookId", null); + } catch (error) { + // Already removed on Apify; safe to forget. + if (error?.statusCode === 404) { + console.warn(`Apify webhook ${webhookId} was already removed; clearing stored id.`); + this.db.set("webhookId", null); + return; + } + // On other errors, keep the id and rethrow. + throw error; + } }, }, async run({ body }) { @@ -35,7 +94,7 @@ export default { summary: body.eventType === WEBHOOK_EVENT_TYPES.TEST ? "Webhook test has successfully triggered!" : this.getSummary(body), - id: body.eventData.actorRunId || `${body.userId}-${body.createAt}`, + id: body.eventData.actorRunId || `${body.userId}-${body.createdAt}`, ts: Date.parse(body.createdAt), }); }, diff --git a/components/apify/sources/new-finished-actor-run-instant/new-finished-actor-run-instant.mjs b/components/apify/sources/new-finished-actor-run-instant/new-finished-actor-run-instant.mjs index 16f25cb2e0293..8713c535a2b6a 100644 --- a/components/apify/sources/new-finished-actor-run-instant/new-finished-actor-run-instant.mjs +++ b/components/apify/sources/new-finished-actor-run-instant/new-finished-actor-run-instant.mjs @@ -5,9 +5,9 @@ import apify from "../../apify.app.mjs"; export default { ...common, key: "apify-new-finished-actor-run-instant", - name: "New Finished Actor Run (Instant)", + name: "New finished Actor run (instant)", description: "Emit new event when a selected Actor is run and finishes.", - version: "0.0.8", + version: "0.0.9", type: "source", dedupe: "unique", props: { @@ -17,7 +17,7 @@ export default { actorSource: { type: "string", label: "Search Actors from", - description: "Where to search for Actors. Valid options are Store and Recently used Actors.", + description: "Where to search for Actors. Choose **Apify Store Actors** to browse the public [Apify Store](https://apify.com/store), or **Recently used Actors** to pick from Actors you've run before.", options: [ { label: "Apify Store Actors", @@ -56,6 +56,9 @@ export default { actorId: this.actorId, }; }, + getEmptyConditionMessage() { + return "No Actor selected. If the list was empty, set \"Search Actors from\" to \"Apify Store Actors\" and pick an Actor before deploying."; + }, getSummary(body) { return `A new Actor run ${body.eventData.actorRunId} has finished`; }, diff --git a/components/apify/sources/new-finished-task-run-instant/new-finished-task-run-instant.mjs b/components/apify/sources/new-finished-task-run-instant/new-finished-task-run-instant.mjs index 0cdc8fee3f201..bf258bbda0b5e 100644 --- a/components/apify/sources/new-finished-task-run-instant/new-finished-task-run-instant.mjs +++ b/components/apify/sources/new-finished-task-run-instant/new-finished-task-run-instant.mjs @@ -4,9 +4,9 @@ import sampleEmit from "./test-event.mjs"; export default { ...common, key: "apify-new-finished-task-run-instant", - name: "New Finished Task Run (Instant)", + name: "New finished task run (instant)", description: "Emit new event when a selected task is run and finishes.", - version: "0.0.7", + version: "0.0.8", type: "source", dedupe: "unique", props: { diff --git a/components/apify_oauth/actions/get-dataset-items/get-dataset-items.mjs b/components/apify_oauth/actions/get-dataset-items/get-dataset-items.mjs index 1f977dd20e280..dc35879131a5a 100644 --- a/components/apify_oauth/actions/get-dataset-items/get-dataset-items.mjs +++ b/components/apify_oauth/actions/get-dataset-items/get-dataset-items.mjs @@ -11,7 +11,7 @@ const props = adjustPropDefinitions(others.props, app); export default { ...others, key: "apify_oauth-get-dataset-items", - version: "0.0.2", + version: "0.0.3", name, description, type, diff --git a/components/apify_oauth/actions/get-kvs-record/get-kvs-record.mjs b/components/apify_oauth/actions/get-kvs-record/get-kvs-record.mjs index 03e5f7c855d7d..f829a94bd218c 100644 --- a/components/apify_oauth/actions/get-kvs-record/get-kvs-record.mjs +++ b/components/apify_oauth/actions/get-kvs-record/get-kvs-record.mjs @@ -11,7 +11,7 @@ const props = adjustPropDefinitions(others.props, app); export default { ...others, key: "apify_oauth-get-kvs-record", - version: "0.0.1", + version: "0.0.2", name, description, type, diff --git a/components/apify_oauth/actions/run-actor/run-actor.mjs b/components/apify_oauth/actions/run-actor/run-actor.mjs index 9c9084efa8621..15ff02ef18c55 100644 --- a/components/apify_oauth/actions/run-actor/run-actor.mjs +++ b/components/apify_oauth/actions/run-actor/run-actor.mjs @@ -11,7 +11,7 @@ const props = adjustPropDefinitions(others.props, app); export default { ...others, key: "apify_oauth-run-actor", - version: "0.0.2", + version: "0.0.3", name, description, type, diff --git a/components/apify_oauth/actions/run-task/run-task.mjs b/components/apify_oauth/actions/run-task/run-task.mjs index 433cfd291daee..b2cca4cdba2f0 100644 --- a/components/apify_oauth/actions/run-task/run-task.mjs +++ b/components/apify_oauth/actions/run-task/run-task.mjs @@ -11,7 +11,7 @@ const props = adjustPropDefinitions(others.props, app); export default { ...others, key: "apify_oauth-run-task", - version: "0.0.2", + version: "0.0.3", name, description, type, diff --git a/components/apify_oauth/actions/scrape-single-url/scrape-single-url.mjs b/components/apify_oauth/actions/scrape-single-url/scrape-single-url.mjs index cc45105f7036f..b507c5df850a6 100644 --- a/components/apify_oauth/actions/scrape-single-url/scrape-single-url.mjs +++ b/components/apify_oauth/actions/scrape-single-url/scrape-single-url.mjs @@ -11,7 +11,7 @@ const props = adjustPropDefinitions(others.props, app); export default { ...others, key: "apify_oauth-scrape-single-url", - version: "0.0.2", + version: "0.0.3", name, description, type, diff --git a/components/apify_oauth/sources/new-finished-actor-run-instant/new-finished-actor-run-instant.mjs b/components/apify_oauth/sources/new-finished-actor-run-instant/new-finished-actor-run-instant.mjs index 63c5a3172230b..8d0ca767c2f7e 100644 --- a/components/apify_oauth/sources/new-finished-actor-run-instant/new-finished-actor-run-instant.mjs +++ b/components/apify_oauth/sources/new-finished-actor-run-instant/new-finished-actor-run-instant.mjs @@ -11,7 +11,7 @@ const props = adjustPropDefinitions(others.props, app); export default { ...others, key: "apify_oauth-new-finished-actor-run-instant", - version: "0.0.2", + version: "0.0.3", name, description, type, diff --git a/components/apify_oauth/sources/new-finished-task-run-instant/new-finished-task-run-instant.mjs b/components/apify_oauth/sources/new-finished-task-run-instant/new-finished-task-run-instant.mjs index 9ee6613386a3d..ffc947e8f1a1c 100644 --- a/components/apify_oauth/sources/new-finished-task-run-instant/new-finished-task-run-instant.mjs +++ b/components/apify_oauth/sources/new-finished-task-run-instant/new-finished-task-run-instant.mjs @@ -11,7 +11,7 @@ const props = adjustPropDefinitions(others.props, app); export default { ...others, key: "apify_oauth-new-finished-task-run-instant", - version: "0.0.2", + version: "0.0.3", name, description, type,