tinyfish.ts

// ============================================================
// TinyFish Integration Layer (server-side only)
// ------------------------------------------------------------
// TinyFish (https://docs.tinyfish.ai) runs a hosted browser agent
// against a URL with a natural-language goal and returns JSON.
// SchemeAI uses it, when configured, for read-only extraction from
// public pages of official government sources.
//
// API used: POST {TINYFISH_API_URL}/v1/automation/run
//   headers: X-API-Key: <key>
//   body:    { url, goal, browser_profile, agent_config }
//   reply:   { run_id, status: COMPLETED|FAILED, num_of_steps,
//              result: object|null, error: object|null }
//
// HONEST STATUS MODEL:
//   - If TINYFISH_API_KEY is not set, the integration reports
//     NOT_CONFIGURED. The app NEVER fakes TinyFish results and
//     falls back to the honest DIRECT_FETCH live check.
//   - A run whose status is COMPLETED is only treated as a success
//     when the agent did not report a CAPTCHA / block page.
// ============================================================

export type TinyFishStatus = "NOT_CONFIGURED" | "READY" | "ERROR";

export const TINYFISH_DEFAULT_API_URL = "https://agent.tinyfish.ai";

export interface TinyFishConfig {
  apiKey: string | null;
  apiUrl: string;
}

export function getTinyFishConfig(): TinyFishConfig {
  let apiUrl = (process.env.TINYFISH_API_URL || TINYFISH_DEFAULT_API_URL).trim().replace(/\/+$/, "");
  // engine.tinyfish.com was used by an earlier version of this file and
  // does not resolve; treat it as unset so existing .env files keep working.
  if (/engine\.tinyfish\.com/i.test(apiUrl)) apiUrl = TINYFISH_DEFAULT_API_URL;
  // Accept a base URL given with or without the /v1 suffix.
  apiUrl = apiUrl.replace(/\/v1$/, "");
  return {
    apiKey: process.env.TINYFISH_API_KEY || null,
    apiUrl,
  };
}

export function tinyFishStatus(): { status: TinyFishStatus; message: string } {
  const cfg = getTinyFishConfig();
  if (!cfg.apiKey) {
    return {
      status: "NOT_CONFIGURED",
      message:
        "TinyFish API key is not configured. Add TINYFISH_API_KEY to the environment to enable agent-based live extraction. Until then, SchemeAI uses honest direct HTTP source checks and the verified reference database.",
    };
  }
  return {
    status: "READY",
    message: `TinyFish integration configured (${cfg.apiUrl}/v1/automation/run). Live extraction runs on admin-triggered discovery and scheme refresh.`,
  };
}

export interface TinyFishRunResult<T = Record<string, unknown>> {
  ok: boolean;
  mode: "TINYFISH";
  url: string;
  runId: string | null;
  status: string | null; // COMPLETED | FAILED | HTTP_<code> | NETWORK_ERROR | NOT_CONFIGURED
  steps: number | null;
  durationMs: number;
  result: T | null;
  error?: string;
}

const BLOCK_PATTERN = /captcha|access denied|forbidden|blocked|cloudflare/i;

function unwrapResult(raw: unknown): Record<string, unknown> | null {
  if (!raw || typeof raw !== "object") return null;
  const obj = raw as Record<string, unknown>;
  // Plain-text or list answers arrive under result.result (sometimes as a JSON string).
  if (Object.keys(obj).length === 1 && "result" in obj) {
    const inner = obj.result;
    if (typeof inner === "string") {
      try {
        const parsed = JSON.parse(inner);
        if (parsed && typeof parsed === "object") return parsed as Record<string, unknown>;
      } catch {
        /* not JSON — keep as text */
      }
      return { text: inner };
    }
    if (inner && typeof inner === "object") return inner as Record<string, unknown>;
  }
  return obj;
}

function describeHttpError(status: number, body: string): string {
  let code = "";
  let message = "";
  try {
    const j = JSON.parse(body) as { error?: { code?: string; message?: string } };
    code = j.error?.code ?? "";
    message = j.error?.message ?? "";
  } catch {
    /* non-JSON error body */
  }
  const hint =
    status === 401
      ? "Check TINYFISH_API_KEY."
      : status === 429
        ? "TinyFish rate/concurrency limit reached — retry after current runs finish."
        : status === 403
          ? "The API key is not permitted to use this feature."
          : "";
  return `TinyFish request failed with HTTP ${status}${code ? ` (${code})` : ""}${message ? `: ${message}` : ""}. ${hint}`.trim();
}

/**
 * Run a TinyFish browser agent against a public official URL.
 * Synchronous /v1/automation/run — blocks until the run finishes
 * (typically 15–60s). Never fabricates success.
 */
export async function tinyFishRun<T = Record<string, unknown>>(
  url: string,
  goal: string,
  opts: { timeoutMs?: number; maxDurationSeconds?: number; stealth?: boolean } = {}
): Promise<TinyFishRunResult<T>> {
  const cfg = getTinyFishConfig();
  const started = Date.now();
  const base = { mode: "TINYFISH" as const, url, runId: null, steps: null, result: null };

  if (!cfg.apiKey) {
    return {
      ...base,
      ok: false,
      status: "NOT_CONFIGURED",
      durationMs: 0,
      error: "TinyFish is not configured (missing TINYFISH_API_KEY).",
    };
  }

  const maxDurationSeconds = opts.maxDurationSeconds ?? 90;
  const controller = new AbortController();
  // Client timeout is a little longer than the agent's own wall-clock limit.
  const timeout = setTimeout(() => controller.abort(), opts.timeoutMs ?? (maxDurationSeconds + 30) * 1000);
  try {
    const res = await fetch(`${cfg.apiUrl}/v1/automation/run`, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        "X-API-Key": cfg.apiKey,
      },
      body: JSON.stringify({
        url,
        goal,
        // The stealth fingerprint gets past many sites that block the default browser.
        browser_profile: opts.stealth ? "stealth" : "lite",
        agent_config: { max_duration_seconds: maxDurationSeconds },
      }),
      signal: controller.signal,
    });

    const text = await res.text();
    if (!res.ok) {
      return {
        ...base,
        ok: false,
        status: `HTTP_${res.status}`,
        durationMs: Date.now() - started,
        error: describeHttpError(res.status, text),
      };
    }

    const data = JSON.parse(text) as {
      run_id?: string | null;
      status?: string;
      num_of_steps?: number | null;
      result?: unknown;
      error?: { code?: string; message?: string } | null;
    };
    const result = unwrapResult(data.result);
    const runId = data.run_id ?? null;
    const steps = data.num_of_steps ?? null;

    if (data.status !== "COMPLETED" || !result) {
      return {
        ...base,
        ok: false,
        runId,
        steps,
        status: data.status ?? "UNKNOWN",
        durationMs: Date.now() - started,
        error:
          data.error?.message ??
          (data.error?.code ? `TinyFish run failed (${data.error.code}).` : "TinyFish run returned no result."),
      };
    }

    // COMPLETED means the infrastructure worked, not that the goal succeeded.
    const blocked =
      result.blocked === true ||
      (typeof result.text === "string" && BLOCK_PATTERN.test(result.text) && result.text.length < 400);
    if (blocked && !opts.stealth) {
      clearTimeout(timeout);
      return tinyFishRun<T>(url, goal, { ...opts, stealth: true });
    }
    if (blocked) {
      return {
        ...base,
        ok: false,
        runId,
        steps,
        status: "BLOCKED",
        durationMs: Date.now() - started,
        result: result as T,
        error: "The page showed a CAPTCHA or access-denied screen; nothing was extracted.",
      };
    }

    return {
      ...base,
      ok: true,
      runId,
      steps,
      status: "COMPLETED",
      durationMs: Date.now() - started,
      result: result as T,
    };
  } catch (e) {
    const aborted = e instanceof Error && e.name === "AbortError";
    return {
      ...base,
      ok: false,
      status: aborted ? "TIMEOUT" : "NETWORK_ERROR",
      durationMs: Date.now() - started,
      error: aborted
        ? "TinyFish run did not finish within the client timeout."
        : e instanceof Error
          ? e.message
          : "TinyFish request could not be completed.",
    };
  } finally {
    clearTimeout(timeout);
  }
}

// ------------------------------------------------------------
// Goal prompts. Each one is read-only and returns a fixed JSON
// shape so results can be stored and rendered without guessing.
// ------------------------------------------------------------

const READ_ONLY_RULES =
  "Read public content only. Do not log in, register, enter OTPs, solve CAPTCHAs or submit any form. If a CAPTCHA, login wall or access-denied page appears, stop and set blocked=true.";

export interface SourcePageExtract {
  page_title?: string | null;
  scheme_names?: string[];
  announcements?: Array<{ text: string; date: string | null }>;
  deadlines?: Array<{ label: string; date: string }>;
  blocked?: boolean;
}

/** Source-level discovery: what does the official portal currently list? */
export function tinyFishExtractSourcePage(url: string) {
  return tinyFishRun<SourcePageExtract>(
    url,
    `This is an official Indian government portal. ${READ_ONLY_RULES} Stay on the current page. Return JSON exactly in this shape: {"page_title": string, "scheme_names": [string], "announcements": [{"text": string, "date": string|null}], "deadlines": [{"label": string, "date": string}], "blocked": boolean}. scheme_names lists government schemes or scholarships named on the page. deadlines lists application or verification closing dates shown on the page, with dates copied verbatim. Use empty arrays when nothing is listed. Do not invent values.`
  );
}

export interface SchemePageExtract {
  scheme_name?: string | null;
  benefit_amount?: string | null;
  eligibility?: string[];
  documents_required?: string[];
  application_window?: string | null;
  deadline?: string | null;
  application_url?: string | null;
  last_updated?: string | null;
  blocked?: boolean;
}

/** Scheme-level refresh: current deadline, eligibility and documents from the official page. */
export function tinyFishExtractSchemePage(url: string, schemeName: string) {
  return tinyFishRun<SchemePageExtract>(
    url,
    `Find information about the government scheme "${schemeName}" on this official page. ${READ_ONLY_RULES} You may follow at most two on-site links whose text clearly refers to this scheme. Return JSON exactly in this shape: {"scheme_name": string|null, "benefit_amount": string|null, "eligibility": [string], "documents_required": [string], "application_window": string|null, "deadline": string|null, "application_url": string|null, "last_updated": string|null, "blocked": boolean}. Copy text and dates verbatim from the page. Use null or empty arrays for anything not shown. Do not invent values.`
  );
}

/** Trim a TinyFish result for storage in FetchJob.resultJson. */
export function compactTinyFishResult(r: TinyFishRunResult<unknown>, maxChars = 6000) {
  const json = r.result ? JSON.stringify(r.result) : null;
  return {
    ok: r.ok,
    runId: r.runId,
    status: r.status,
    steps: r.steps,
    durationMs: r.durationMs,
    error: r.error ?? null,
    result: json && json.length > maxChars ? { truncated: json.slice(0, maxChars) } : r.result,
  };
}