Merge branch 'huggingface:main' into add_gb10

Author: stelterlab

packages/inference/src/lib/getProviderHelper.ts

@@ -28,6 +28,8 @@ import type {
 	ImageToImageTaskHelper,
 	ImageToTextTaskHelper,
 	ImageToVideoTaskHelper,
+	ImageTextToImageTaskHelper,
+	ImageTextToVideoTaskHelper,
 	ObjectDetectionTaskHelper,
 	QuestionAnsweringTaskHelper,
 	SentenceSimilarityTaskHelper,
@@ -276,6 +278,14 @@ export function getProviderHelper(
 	provider: InferenceProviderOrPolicy,
 	task: "image-to-video"
 ): ImageToVideoTaskHelper & TaskProviderHelper;
+export function getProviderHelper(
+	provider: InferenceProviderOrPolicy,
+	task: "image-text-to-image"
+): ImageTextToImageTaskHelper & TaskProviderHelper;
+export function getProviderHelper(
+	provider: InferenceProviderOrPolicy,
+	task: "image-text-to-video"
+): ImageTextToVideoTaskHelper & TaskProviderHelper;
 export function getProviderHelper(
 	provider: InferenceProviderOrPolicy,
 	task: "sentence-similarity"

packages/inference/src/providers/providerHelper.ts

@@ -19,6 +19,8 @@ import type {
 	ImageToTextInput,
 	ImageToTextOutput,
 	ImageToVideoInput,
+	ImageTextToImageInput,
+	ImageTextToVideoInput,
 	ObjectDetectionInput,
 	ObjectDetectionOutput,
 	QuestionAnsweringInput,
@@ -54,6 +56,8 @@ import { toArray } from "../utils/toArray.js";
 import type { ImageToImageArgs } from "../tasks/cv/imageToImage.js";
 import type { AutomaticSpeechRecognitionArgs } from "../tasks/audio/automaticSpeechRecognition.js";
 import type { ImageToVideoArgs } from "../tasks/cv/imageToVideo.js";
+import type { ImageTextToImageArgs } from "../tasks/cv/imageTextToImage.js";
+import type { ImageTextToVideoArgs } from "../tasks/cv/imageTextToVideo.js";
 import type { ImageSegmentationArgs } from "../tasks/cv/imageSegmentation.js";
 
 /**
@@ -159,6 +163,18 @@ export interface ImageToVideoTaskHelper {
 	preparePayloadAsync(args: ImageToVideoArgs): Promise<RequestArgs>;
 }
 
+export interface ImageTextToImageTaskHelper {
+	getResponse(response: unknown, url?: string, headers?: HeadersInit): Promise<Blob>;
+	preparePayload(params: BodyParams<ImageTextToImageInput & BaseArgs>): Record<string, unknown>;
+	preparePayloadAsync(args: ImageTextToImageArgs): Promise<RequestArgs>;
+}
+
+export interface ImageTextToVideoTaskHelper {
+	getResponse(response: unknown, url?: string, headers?: HeadersInit): Promise<Blob>;
+	preparePayload(params: BodyParams<ImageTextToVideoInput & BaseArgs>): Record<string, unknown>;
+	preparePayloadAsync(args: ImageTextToVideoArgs): Promise<RequestArgs>;
+}
+
 export interface ImageSegmentationTaskHelper {
 	getResponse(response: unknown, url?: string, headers?: HeadersInit): Promise<ImageSegmentationOutput>;
 	preparePayload(params: BodyParams<ImageSegmentationInput & BaseArgs>): Record<string, unknown> | BodyInit;

packages/inference/src/snippets/getInferenceSnippets.ts

@@ -96,7 +96,10 @@ const HF_PYTHON_METHODS: Partial<Record<WidgetType, string>> = {
 	"image-classification": "image_classification",
 	"image-segmentation": "image_segmentation",
 	"image-to-image": "image_to_image",
+	"image-to-video": "image_to_video",
 	"image-to-text": "image_to_text",
+	"image-text-to-image": "image_text_to_image",
+	"image-text-to-video": "image_text_to_video",
 	"object-detection": "object_detection",
 	"question-answering": "question_answering",
 	"sentence-similarity": "sentence_similarity",
@@ -390,7 +393,9 @@ const snippets: Partial<
 	"fill-mask": snippetGenerator("basic"),
 	"image-classification": snippetGenerator("basicImage"),
 	"image-segmentation": snippetGenerator("basicImage"),
+	"image-text-to-image": snippetGenerator("imageToImage", prepareImageToImageInput),
 	"image-text-to-text": snippetGenerator("conversational"),
+	"image-text-to-video": snippetGenerator("imageToVideo", prepareImageToImageInput),
 	"image-to-image": snippetGenerator("imageToImage", prepareImageToImageInput),
 	"image-to-text": snippetGenerator("basicImage"),
 	"image-to-video": snippetGenerator("imageToVideo", prepareImageToImageInput),

packages/inference/src/tasks/cv/imageTextToImage.ts

@@ -0,0 +1,22 @@
+import type { ImageTextToImageInput } from "@huggingface/tasks";
+import { resolveProvider } from "../../lib/getInferenceProviderMapping.js";
+import { getProviderHelper } from "../../lib/getProviderHelper.js";
+import type { BaseArgs, Options } from "../../types.js";
+import { innerRequest } from "../../utils/request.js";
+
+export type ImageTextToImageArgs = BaseArgs & ImageTextToImageInput;
+
+/**
+ * This task takes an image and text input and outputs a new generated image.
+ * Recommended model: black-forest-labs/FLUX.2-dev
+ */
+export async function imageTextToImage(args: ImageTextToImageArgs, options?: Options): Promise<Blob> {
+	const provider = await resolveProvider(args.provider, args.model, args.endpointUrl);
+	const providerHelper = getProviderHelper(provider, "image-text-to-image");
+	const payload = await providerHelper.preparePayloadAsync(args);
+	const { data: res, requestContext } = await innerRequest<Blob>(payload, providerHelper, {
+		...options,
+		task: "image-text-to-image",
+	});
+	return providerHelper.getResponse(res, requestContext.url, requestContext.info.headers as Record<string, string>);
+}

packages/inference/src/tasks/cv/imageTextToVideo.ts

@@ -0,0 +1,22 @@
+import type { ImageTextToVideoInput } from "@huggingface/tasks";
+import { resolveProvider } from "../../lib/getInferenceProviderMapping.js";
+import { getProviderHelper } from "../../lib/getProviderHelper.js";
+import type { BaseArgs, Options } from "../../types.js";
+import { innerRequest } from "../../utils/request.js";
+
+export type ImageTextToVideoArgs = BaseArgs & ImageTextToVideoInput;
+
+/**
+ * This task takes an image and text input and outputs a generated video.
+ * Recommended model: Lightricks/LTX-Video
+ */
+export async function imageTextToVideo(args: ImageTextToVideoArgs, options?: Options): Promise<Blob> {
+	const provider = await resolveProvider(args.provider, args.model, args.endpointUrl);
+	const providerHelper = getProviderHelper(provider, "image-text-to-video");
+	const payload = await providerHelper.preparePayloadAsync(args);
+	const { data: res, requestContext } = await innerRequest<Blob>(payload, providerHelper, {
+		...options,
+		task: "image-text-to-video",
+	});
+	return providerHelper.getResponse(res, requestContext.url, requestContext.info.headers as Record<string, string>);
+}

packages/inference/src/tasks/index.ts

@@ -14,6 +14,8 @@ export * from "./cv/imageSegmentation.js";
 export * from "./cv/imageToImage.js";
 export * from "./cv/imageToText.js";
 export * from "./cv/imageToVideo.js";
+export * from "./cv/imageTextToImage.js";
+export * from "./cv/imageTextToVideo.js";
 export * from "./cv/objectDetection.js";
 export * from "./cv/textToImage.js";
 export * from "./cv/textToVideo.js";

packages/tasks/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@huggingface/tasks",
 	"packageManager": "pnpm@10.10.0",
-	"version": "0.19.65",
+	"version": "0.19.66",
 	"description": "List of ML tasks for huggingface.co/tasks",
 	"repository": "https://github.com/huggingface/huggingface.js.git",
 	"publishConfig": {

packages/tasks/src/local-apps.ts

@@ -517,13 +517,6 @@ export const LOCAL_APPS = {
 			model.tags.includes("coreml") && model.tags.includes("joyfusion") && model.pipeline_tag === "text-to-image",
 		deeplink: (model) => new URL(`https://joyfusion.app/import_from_hf?repo_id=${model.id}`),
 	},
-	invoke: {
-		prettyLabel: "Invoke",
-		docsUrl: "https://github.com/invoke-ai/InvokeAI",
-		mainTask: "text-to-image",
-		displayOnModelPage: (model) => model.library_name === "diffusers" && model.pipeline_tag === "text-to-image",
-		deeplink: (model) => new URL(`https://models.invoke.ai/huggingface/${model.id}`),
-	},
 	ollama: {
 		prettyLabel: "Ollama",
 		docsUrl: "https://ollama.com",

packages/tasks/src/model-libraries-snippets.ts

@@ -331,6 +331,19 @@ output = model.generate(text)
 sf.write("simple.mp3", output, 44100)`,
 ];
 
+export const dia2 = (model: ModelData): string[] => [
+	`from dia2 import Dia2, GenerationConfig, SamplingConfig
+
+dia = Dia2.from_repo("${model.id}", device="cuda", dtype="bfloat16")
+config = GenerationConfig(
+    cfg_scale=2.0,
+    audio=SamplingConfig(temperature=0.8, top_k=50),
+    use_cuda_graph=True,
+)
+result = dia.generate("[S1] Hello Dia2!", config=config, output_wav="hello.wav", verbose=True)
+`,
+];
+
 export const describe_anything = (model: ModelData): string[] => [
 	`# pip install git+https://github.com/NVlabs/describe-anything
 from huggingface_hub import snapshot_download

packages/tasks/src/model-libraries.ts

@@ -293,6 +293,13 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = {
 		snippets: snippets.dia,
 		filter: false,
 	},
+	dia2: {
+		prettyLabel: "Dia2",
+		repoName: "Dia2",
+		repoUrl: "https://github.com/nari-labs/dia2",
+		snippets: snippets.dia2,
+		filter: false,
+	},
 	"diff-interpretation-tuning": {
 		prettyLabel: "Diff Interpretation Tuning",
 		repoName: "Diff Interpretation Tuning",
@@ -413,6 +420,13 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = {
 		filter: true,
 		countDownloads: `path_extension:"bin"`,
 	},
+	fixer: {
+		prettyLabel: "Fixer",
+		repoName: "Fixer",
+		repoUrl: "https://github.com/nv-tlabs/Fixer",
+		filter: false,
+		countDownloads: `path:"pretrained/pretrained_fixer.pkl"`,
+	},
 	flair: {
 		prettyLabel: "Flair",
 		repoName: "Flair",