diff --git a/zeromodels/models/deepseek_v4/deepseek_v4_layers.py b/zeromodels/models/deepseek_v4/deepseek_v4_layers.py index 8a67d58f..51892093 100644 --- a/zeromodels/models/deepseek_v4/deepseek_v4_layers.py +++ b/zeromodels/models/deepseek_v4/deepseek_v4_layers.py @@ -1,5 +1,4 @@ import keras -import numpy as np from keras import layers, ops MASK_NEG = -1e9 @@ -450,7 +449,9 @@ def __init__( self.compress_inv_freq = ( None if compress_inv_freq is None - else np.asarray(compress_inv_freq, dtype="float32") + else ops.convert_to_numpy( + ops.convert_to_tensor(compress_inv_freq, dtype="float32") + ) ) self.norm_eps = norm_eps self.scaling = head_dim**-0.5 diff --git a/zeromodels/models/efficientdet/efficientdet_image_processor.py b/zeromodels/models/efficientdet/efficientdet_image_processor.py index 5dc132d6..024b1399 100644 --- a/zeromodels/models/efficientdet/efficientdet_image_processor.py +++ b/zeromodels/models/efficientdet/efficientdet_image_processor.py @@ -1,9 +1,7 @@ -from typing import Dict, List, Optional, Tuple, Union +from typing import Dict, Optional, Tuple import keras -import numpy as np from keras import ops -from PIL import Image from zeromodels.base import BaseImageProcessor from zeromodels.utils.image_util import get_data_format, load_image @@ -95,9 +93,7 @@ def preprocess_one(self, image): t = ops.transpose(t, (0, 3, 1, 2)) return t, scale, (orig_h, orig_w) - def call( - self, image: Union[str, np.ndarray, Image.Image, List] - ) -> Dict[str, Union[keras.KerasTensor, np.ndarray]]: + def call(self, image) -> Dict: items = list(image) if isinstance(image, (list, tuple)) else [image] tensors, scales, sizes = [], [], [] for item in items: diff --git a/zeromodels/models/gemma3n/gemma3n_layers.py b/zeromodels/models/gemma3n/gemma3n_layers.py index 27d396da..ba65295a 100644 --- a/zeromodels/models/gemma3n/gemma3n_layers.py +++ b/zeromodels/models/gemma3n/gemma3n_layers.py @@ -1,7 +1,6 @@ import math import keras -import numpy as np from keras import layers, ops @@ -720,9 +719,13 @@ def __init__(self, hidden_size, num_heads, context_left, context_right, **kwargs ) num_timescales = hidden_size // 2 log_increment = math.log(1.0e4) / max(num_timescales - 1, 1) - inv = np.exp(np.arange(num_timescales) * -log_increment).astype("float32") + inv = ops.convert_to_numpy( + ops.exp(ops.arange(num_timescales, dtype="float32") * -log_increment) + ) self.inv_timescales = inv[None, None, :] # [1, 1, num_timescales] - pos = np.arange(self.max_backward, -self.max_forward - 1, -1, dtype="float32") + pos = ops.convert_to_numpy( + ops.arange(self.max_backward, -self.max_forward - 1, -1, dtype="float32") + ) self.pos_indices = pos[None] # [1, F_span] self.max_span_plus_1 = self.pos_indices.shape[1] @@ -815,14 +818,16 @@ def __init__( # Static local causal validity mask [W, C]. w, c = self.chunk_size, self.context_size - lower = np.tril(np.ones((c, w), dtype=bool), k=0).T - upper = np.tril( - np.ones((w, c), dtype=bool), + lower = ops.transpose(ops.tril(ops.ones((c, w)), k=0)) + upper = ops.tril( + ops.ones((w, c)), k=self.max_past_horizon + self.max_future_horizon, ) - self.local_causal_valid_mask = np.ones((w, c), dtype=bool) & lower & upper + self.local_causal_valid_mask = ops.convert_to_numpy( + ops.logical_and(ops.cast(lower, "bool"), ops.cast(upper, "bool")) + ) # Block gather start indices; sliced per call by num_blocks. - self._starts = np.arange(4096) + self._starts = ops.convert_to_numpy(ops.arange(4096)) def build(self, input_shape): self.per_dim_scale = self.add_weight( @@ -845,7 +850,7 @@ def extract_block_context(self, x, num_blocks, seq_len): x = ops.pad(x, pad_cfg) idx = ( self._starts[:num_blocks, None] * self.chunk_size - + np.arange(self.context_size)[None, :] + + ops.convert_to_numpy(ops.arange(self.context_size))[None, :] ) idx = ops.convert_to_tensor(idx.astype("int32")) return ops.take(x, idx, axis=1) # [B, U, C, ...] @@ -879,7 +884,7 @@ def call(self, hidden_states, mask=None): logits = self.relative_position_embedding(query_blocks, key_blocks) logits = ops.tanh(logits / self.logit_cap) * self.logit_cap - neg_inf = ops.cast(float(np.finfo(np.float32).min), "float32") + neg_inf = ops.cast(-3.4028234663852886e38, "float32") # most negative float32 logits = ops.where(final_cond, logits, neg_inf) probs = ops.softmax(logits, axis=-1) # [B,N,U,W,C] diff --git a/zeromodels/models/gemma3n/gemma3n_processor.py b/zeromodels/models/gemma3n/gemma3n_processor.py index 8175df03..39abfec0 100644 --- a/zeromodels/models/gemma3n/gemma3n_processor.py +++ b/zeromodels/models/gemma3n/gemma3n_processor.py @@ -1,5 +1,4 @@ import keras -import numpy as np from keras import ops from zeromodels.base import BaseProcessor @@ -95,7 +94,11 @@ def load_image(self, item): def load_audio(self, item): if item.get("audio") is not None: - return np.asarray(item["audio"], dtype="float32").reshape(-1) + return ops.convert_to_numpy( + ops.reshape( + ops.convert_to_tensor(item["audio"], dtype="float32"), (-1,) + ) + ) if item.get("path") is not None: import soundfile as sf @@ -185,13 +188,10 @@ def call( bos = tok.bos_token_id ids = [[bos] + tok.encode(t) for t in texts] max_len = max(len(x) for x in ids) - input_ids = np.zeros((len(ids), max_len), dtype="int32") - attention_mask = np.zeros((len(ids), max_len), dtype="int32") - for i, seq_ids in enumerate(ids): - input_ids[i, : len(seq_ids)] = seq_ids - attention_mask[i, : len(seq_ids)] = 1 - out["input_ids"] = ops.convert_to_tensor(input_ids) - out["attention_mask"] = ops.convert_to_tensor(attention_mask) + input_ids = [row + [0] * (max_len - len(row)) for row in ids] + attention_mask = [[1] * len(row) + [0] * (max_len - len(row)) for row in ids] + out["input_ids"] = ops.convert_to_tensor(input_ids, dtype="int32") + out["attention_mask"] = ops.convert_to_tensor(attention_mask, dtype="int32") return out def get_config(self): diff --git a/zeromodels/models/glm5_moe/glm5_moe_layers.py b/zeromodels/models/glm5_moe/glm5_moe_layers.py index 4aec22d5..abecb668 100644 --- a/zeromodels/models/glm5_moe/glm5_moe_layers.py +++ b/zeromodels/models/glm5_moe/glm5_moe_layers.py @@ -1,5 +1,4 @@ import keras -import numpy as np from keras import layers, ops from zeromodels.base.base_attention import fused_attention @@ -198,7 +197,7 @@ def call(self, hidden_states): ops.one_hot(group_idx, self.n_group, dtype="float32"), axis=1 ) score_mask = ops.repeat(group_mask, self.num_experts // self.n_group, axis=-1) - choice = ops.where(score_mask > 0, biased, -np.inf) + choice = ops.where(score_mask > 0, biased, float("-inf")) _, top_idx = ops.top_k(choice, self.num_experts_per_tok) top_vals = ops.take_along_axis(scores, top_idx, axis=-1) if self.norm_topk_prob: diff --git a/zeromodels/models/granite_speech5/granite_speech5_processor.py b/zeromodels/models/granite_speech5/granite_speech5_processor.py index 6aa8c263..33866c32 100644 --- a/zeromodels/models/granite_speech5/granite_speech5_processor.py +++ b/zeromodels/models/granite_speech5/granite_speech5_processor.py @@ -1,5 +1,4 @@ import keras -import numpy as np from keras import ops from zeromodels.base import BaseProcessor @@ -55,10 +54,8 @@ def call(self, audio=None, text=None, sampling_rate=16000): label_ids = [self.tokenizer.tokenize(t) for t in texts] max_len = max(len(x) for x in label_ids) pad_id = self.tokenizer.pad_token_id - labels = np.full((len(label_ids), max_len), pad_id, dtype="int32") - for i, seq in enumerate(label_ids): - labels[i, : len(seq)] = seq - out["labels"] = ops.convert_to_tensor(labels) + labels = [list(seq) + [pad_id] * (max_len - len(seq)) for seq in label_ids] + out["labels"] = ops.convert_to_tensor(labels, dtype="int32") return out def batch_decode( @@ -69,7 +66,7 @@ def batch_decode( token_ids, skip_special_tokens=skip_special_tokens ) # Word-level timestamps: one dict per clip, mirroring Whisper's shape. - token_ids = np.asarray(ops.convert_to_numpy(token_ids)).tolist() + token_ids = ops.convert_to_numpy(token_ids).tolist() fs = self.frame_seconds return [ { diff --git a/zeromodels/models/locateanything/locateanything_processor.py b/zeromodels/models/locateanything/locateanything_processor.py index 3e4d7c82..5a02c4d1 100644 --- a/zeromodels/models/locateanything/locateanything_processor.py +++ b/zeromodels/models/locateanything/locateanything_processor.py @@ -1,5 +1,4 @@ import keras -import numpy as np from keras import ops from zeromodels.base import BaseProcessor @@ -198,19 +197,18 @@ def call( out["image_grid_hws"] = ops.convert_to_tensor( image_inputs["image_grid_hws"] ) - grid = [tuple(g) for g in np.asarray(image_inputs["image_grid_hws"])] + grid = [ + tuple(g) for g in ops.convert_to_numpy(image_inputs["image_grid_hws"]) + ] per_text = self.deal_per_text(texts, self.image_token, grid) texts = [self.expand_image_tokens(t, g) for t, g in zip(texts, per_text)] ids = [self.tokenizer.encode(t) for t in texts] max_len = max(len(x) for x in ids) - input_ids = np.zeros((len(ids), max_len), dtype="int32") - attention_mask = np.zeros((len(ids), max_len), dtype="int32") - for i, seq in enumerate(ids): - input_ids[i, : len(seq)] = seq - attention_mask[i, : len(seq)] = 1 - out["input_ids"] = ops.convert_to_tensor(input_ids) - out["attention_mask"] = ops.convert_to_tensor(attention_mask) + input_ids = [list(seq) + [0] * (max_len - len(seq)) for seq in ids] + attention_mask = [[1] * len(seq) + [0] * (max_len - len(seq)) for seq in ids] + out["input_ids"] = ops.convert_to_tensor(input_ids, dtype="int32") + out["attention_mask"] = ops.convert_to_tensor(attention_mask, dtype="int32") return out def post_process_generation(self, generated, task=None, image_size=None, text=None): @@ -229,10 +227,17 @@ def post_process_generation(self, generated, task=None, image_size=None, text=No [x1, y1, x2, y2]}`` or ``{"label": ..., "point": [x, y]}``. """ try: - arr = np.asarray(ops.convert_to_numpy(generated)) + arr = ops.convert_to_numpy(generated) except (TypeError, ValueError): - arr = np.asarray(generated) - sequences = [arr.tolist()] if arr.ndim == 1 else [row.tolist() for row in arr] + arr = None + if arr is not None: + sequences = ( + [arr.tolist()] if arr.ndim == 1 else [row.tolist() for row in arr] + ) + elif generated and isinstance(generated[0], (list, tuple)): + sequences = [list(row) for row in generated] + else: + sequences = [list(generated)] results = [] for seq in sequences: diff --git a/zeromodels/models/maskformer/maskformer_image_processor.py b/zeromodels/models/maskformer/maskformer_image_processor.py index 9e0c2665..3a0283b0 100644 --- a/zeromodels/models/maskformer/maskformer_image_processor.py +++ b/zeromodels/models/maskformer/maskformer_image_processor.py @@ -1,8 +1,7 @@ -from typing import Dict, List, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple import keras -import numpy as np -from PIL import Image +from keras import ops from zeromodels.base import BaseImageProcessor from zeromodels.utils.image_util import get_data_format, load_image @@ -17,7 +16,8 @@ class MaskFormerImageProcessor(BaseImageProcessor): Resizes the longest edge to ``target_size``, pads to a square, rescales to ``[0, 1]``, and applies ImageNet normalization. Uses pure - Keras 3 ops for all tensor operations. + Keras 3 ops for all tensor operations. Accepts a path, a PIL image, or an + array (a 4D array is treated as a single-image batch). Args: target_size: Target square edge length (matches the model's @@ -58,44 +58,38 @@ def variant_size(variant): """ return 512 - def __call__( - self, image: Union[str, np.ndarray, Image.Image] - ) -> Dict[str, keras.KerasTensor]: + def __call__(self, image) -> Dict[str, keras.KerasTensor]: return self.call(image) - def call( - self, image: Union[str, np.ndarray, Image.Image] - ) -> Dict[str, keras.KerasTensor]: - if isinstance(image, np.ndarray) and image.ndim == 4: + def call(self, image) -> Dict[str, keras.KerasTensor]: + if hasattr(image, "ndim") and image.ndim == 4: image = image[0] - image = load_image(image).astype(np.float32) + image = load_image(image) h, w = image.shape[:2] scale = self.target_size / max(h, w) new_h, new_w = int(h * scale), int(w * scale) - image = keras.ops.convert_to_tensor(image, dtype="float32") - image = keras.ops.expand_dims(image, axis=0) - image = keras.ops.image.resize(image, (new_h, new_w), interpolation="bilinear") + image = ops.convert_to_tensor(image, dtype="float32") + image = ops.expand_dims(image, axis=0) + image = ops.image.resize(image, (new_h, new_w), interpolation="bilinear") image = image / 255.0 - padded = keras.ops.zeros( - (1, self.target_size, self.target_size, 3), dtype="float32" - ) - padded = keras.ops.slice_update(padded, (0, 0, 0, 0), image) + padded = ops.zeros((1, self.target_size, self.target_size, 3), dtype="float32") + padded = ops.slice_update(padded, (0, 0, 0, 0), image) - mean = keras.ops.reshape( - keras.ops.convert_to_tensor(self.image_mean, dtype="float32"), + mean = ops.reshape( + ops.convert_to_tensor(self.image_mean, dtype="float32"), (1, 1, 1, 3), ) - std = keras.ops.reshape( - keras.ops.convert_to_tensor(self.image_std, dtype="float32"), + std = ops.reshape( + ops.convert_to_tensor(self.image_std, dtype="float32"), (1, 1, 1, 3), ) padded = (padded - mean) / std if get_data_format(self.data_format) == "channels_first": - padded = keras.ops.transpose(padded, (0, 3, 1, 2)) + padded = ops.transpose(padded, (0, 3, 1, 2)) return {"pixel_values": padded} @@ -104,7 +98,7 @@ def post_process_semantic_segmentation( outputs: Dict[str, keras.KerasTensor], target_sizes: Optional[List[Tuple[int, int]]] = None, label_names: Optional[List[str]] = None, - ) -> List[np.ndarray]: + ) -> list: return maskformer_post_process_semantic( outputs, target_sizes=target_sizes, @@ -134,9 +128,7 @@ def post_process_panoptic_segmentation( ) -def unpad_and_resize_masks( - mask_logits, model_size: int, target_h: int, target_w: int -) -> np.ndarray: +def unpad_and_resize_masks(mask_logits, model_size: int, target_h: int, target_w: int): """Upscale mask logits, remove padding, and resize to the original image. The model predicts masks for a square ``model_size`` input that the @@ -151,25 +143,25 @@ def unpad_and_resize_masks( target_w: Original (unpadded) image width. Returns: - Numpy array of shape ``(1, Q, target_h, target_w)``. + Tensor of shape ``(1, Q, target_h, target_w)``. """ scale = model_size / max(target_h, target_w) resized_h, resized_w = int(target_h * scale), int(target_w * scale) - mask_logits = keras.ops.convert_to_tensor(mask_logits, dtype="float32") + mask_logits = ops.convert_to_tensor(mask_logits, dtype="float32") - mask_4d = keras.ops.transpose(mask_logits, (0, 2, 3, 1)) - mask_full = keras.ops.image.resize( + mask_4d = ops.transpose(mask_logits, (0, 2, 3, 1)) + mask_full = ops.image.resize( mask_4d, (model_size, model_size), interpolation="bilinear" ) - mask_full = keras.ops.transpose(mask_full, (0, 3, 1, 2)) + mask_full = ops.transpose(mask_full, (0, 3, 1, 2)) mask_cropped = mask_full[:, :, :resized_h, :resized_w] - mask_cropped_4d = keras.ops.transpose(mask_cropped, (0, 2, 3, 1)) - mask_final = keras.ops.image.resize( + mask_cropped_4d = ops.transpose(mask_cropped, (0, 2, 3, 1)) + mask_final = ops.image.resize( mask_cropped_4d, (target_h, target_w), interpolation="bilinear" ) - mask_final = keras.ops.transpose(mask_final, (0, 3, 1, 2)) - return keras.ops.convert_to_numpy(mask_final) + mask_final = ops.transpose(mask_final, (0, 3, 1, 2)) + return mask_final def default_label_names(num_classes): @@ -200,7 +192,7 @@ def maskformer_post_process_semantic( target_sizes: Optional[List[Tuple[int, int]]] = None, model_size: int = 512, label_names: Optional[List[str]] = None, -) -> List[np.ndarray]: +) -> list: """Fuse per-query class and mask predictions into semantic label maps. For each image, softmaxes the class logits (dropping the no-object class), @@ -222,7 +214,7 @@ def maskformer_post_process_semantic( mask_logits = outputs["masks_queries_logits"] batch_size = class_logits.shape[0] - results: List[np.ndarray] = [] + results = [] for i in range(batch_size): if target_sizes is None: target_h, target_w = model_size, model_size @@ -232,14 +224,10 @@ def maskformer_post_process_semantic( mask_resized = unpad_and_resize_masks( mask_logits[i : i + 1], model_size, target_h, target_w ) - masks_classes = keras.ops.softmax(class_logits[i], axis=-1)[:, :-1] - masks_probs = keras.ops.sigmoid( - keras.ops.convert_to_tensor(mask_resized[0], dtype="float32") - ) - seg_logits = keras.ops.einsum("qc,qhw->chw", masks_classes, masks_probs) - seg = keras.ops.convert_to_numpy(keras.ops.argmax(seg_logits, axis=0)).astype( - np.int32 - ) + masks_classes = ops.softmax(class_logits[i], axis=-1)[:, :-1] + masks_probs = ops.sigmoid(mask_resized[0]) + seg_logits = ops.einsum("qc,qhw->chw", masks_classes, masks_probs) + seg = ops.convert_to_numpy(ops.argmax(seg_logits, axis=0)).astype("int32") results.append(seg) return results @@ -282,45 +270,52 @@ def maskformer_post_process_panoptic( label_names = default_label_names(int(class_logits.shape[-1])) num_classes = class_logits.shape[-1] - 1 + num_queries = int(class_logits.shape[1]) target_h, target_w = target_size mask_logits_resized = unpad_and_resize_masks( mask_logits, model_size, target_h, target_w ) - scores_all = keras.ops.convert_to_numpy(keras.ops.softmax(class_logits[0], axis=-1)) - pred_scores = np.max(scores_all, axis=-1) - pred_labels = np.argmax(scores_all, axis=-1) - - mask_probs = mask_logits_resized[0] - keep = (pred_labels != num_classes) & (pred_scores > threshold) - mask_probs = mask_probs[keep] - pred_scores = pred_scores[keep] - pred_labels = pred_labels[keep] - - if mask_probs.shape[0] == 0: + scores = ops.softmax(class_logits[0], axis=-1) + pred_scores = ops.max(scores, axis=-1) + pred_labels = ops.argmax(scores, axis=-1) + mask_probs_sig = ops.sigmoid(mask_logits_resized[0]) + + keep = ops.logical_and( + ops.not_equal(pred_labels, num_classes), + ops.greater(pred_scores, threshold), + ) + if int(ops.sum(ops.cast(keep, "int32"))) == 0: return { - "segmentation": np.full(target_size, -1, dtype=np.int32), + "segmentation": ops.convert_to_numpy( + ops.full(target_size, -1, dtype="int32") + ), "segments_info": [], } - - mask_probs_sig = keras.ops.convert_to_numpy( - keras.ops.sigmoid(keras.ops.convert_to_tensor(mask_probs, dtype="float32")) + weighted = ops.reshape(pred_scores, (-1, 1, 1)) * mask_probs_sig + keep_bias = ops.where( + keep, + ops.zeros_like(pred_scores), + ops.full_like(pred_scores, float("-inf")), ) - mask_labels = (pred_scores[:, None, None] * mask_probs_sig).argmax(0) + weighted = weighted + ops.reshape(keep_bias, (-1, 1, 1)) + mask_labels = ops.argmax(weighted, axis=0) - segmentation = np.full(target_size, -1, dtype=np.int32) + segmentation = ops.full(target_size, -1, dtype="int32") segments_info: List[Dict] = [] current_id = 0 stuff_memory: Dict[int, int] = {} - for k in range(pred_labels.shape[0]): + for k in range(num_queries): + if not bool(keep[k]): + continue pred_class = int(pred_labels[k]) - mask_k = mask_labels == k - mask_k_area = int(mask_k.sum()) - original_mask = mask_probs_sig[k] >= mask_threshold - original_area = int(original_mask.sum()) - final_mask = mask_k & original_mask - final_area = int(final_mask.sum()) + mask_k = ops.equal(mask_labels, k) + mask_k_area = int(ops.sum(ops.cast(mask_k, "int32"))) + original_mask = ops.greater_equal(mask_probs_sig[k], mask_threshold) + original_area = int(ops.sum(ops.cast(original_mask, "int32"))) + final_mask = ops.logical_and(mask_k, original_mask) + final_area = int(ops.sum(ops.cast(final_mask, "int32"))) if mask_k_area == 0 or original_area == 0 or final_area == 0: continue @@ -330,11 +325,17 @@ def maskformer_post_process_panoptic( if stuff_classes and pred_class in stuff_classes: if pred_class in stuff_memory: - segmentation[final_mask] = stuff_memory[pred_class] + segmentation = ops.where( + final_mask, + ops.cast(stuff_memory[pred_class], "int32"), + segmentation, + ) continue stuff_memory[pred_class] = current_id - segmentation[final_mask] = current_id + segmentation = ops.where( + final_mask, ops.cast(current_id, "int32"), segmentation + ) name = ( label_names[pred_class] if label_names is not None and pred_class < len(label_names) @@ -350,4 +351,7 @@ def maskformer_post_process_panoptic( ) current_id += 1 - return {"segmentation": segmentation, "segments_info": segments_info} + return { + "segmentation": ops.convert_to_numpy(segmentation), + "segments_info": segments_info, + } diff --git a/zeromodels/models/minimax/minimax_layers.py b/zeromodels/models/minimax/minimax_layers.py index 51a2a241..ab7acb48 100644 --- a/zeromodels/models/minimax/minimax_layers.py +++ b/zeromodels/models/minimax/minimax_layers.py @@ -1,5 +1,4 @@ import keras -import numpy as np from keras import layers, ops @@ -343,9 +342,10 @@ def __init__( base = 1.0 / (2.0 ** (8.0 / num_heads)) factor = 1.0 - layer_idx / (num_layers - 1 + 1e-5) + 1e-5 - self.slope = ( - base ** np.arange(1, num_heads + 1, dtype="float32") * factor - ).reshape(num_heads, 1, 1) + self.slope = ops.reshape( + base ** ops.arange(1, num_heads + 1, dtype="float32") * factor, + (num_heads, 1, 1), + ) self.qkv = layers.Dense(num_heads * head_dim * 3, use_bias=False, name="qkv") self.output_gate = layers.Dense( @@ -414,7 +414,7 @@ def call(self, hidden_states, attention_mask=None, use_cache=False): k_decay = ops.exp(-slope * (cur - r)) diag = r - ops.transpose(r) # (cur, cur), i - j diag = slope[None] * diag[None, None] - diag = ops.where(diag >= 0.0, -diag, -np.inf) + diag = ops.where(diag >= 0.0, -diag, float("-inf")) diag = ops.exp(diag) # (1, H, cur, cur) block_decay = ops.exp(-slope * float(cur)) # (H, 1, 1) diff --git a/zeromodels/models/minimax_m3_vl/minimax_m3_vl_layers.py b/zeromodels/models/minimax_m3_vl/minimax_m3_vl_layers.py index 4db804ae..0a6ea0be 100644 --- a/zeromodels/models/minimax_m3_vl/minimax_m3_vl_layers.py +++ b/zeromodels/models/minimax_m3_vl/minimax_m3_vl_layers.py @@ -1,5 +1,4 @@ import keras -import numpy as np from keras import layers, ops MASK_NEG = -1e9 @@ -363,10 +362,12 @@ def block_keep_mask(self, idx_q, idx_k, position_ids, key_positions): ops.cast(key_positions, "int32")[None, None, None, :] > ops.cast(position_ids, "int32")[:, None, :, None] ) - scores = ops.where(future, -np.inf, scores) + scores = ops.where(future, float("-inf"), scores) if pad: scores = ops.pad( - scores, ((0, 0), (0, 0), (0, 0), (0, pad)), constant_values=-np.inf + scores, + ((0, 0), (0, 0), (0, 0), (0, pad)), + constant_values=float("-inf"), ) scores = ops.reshape( scores, (b, self.index_n_heads, q_len, num_blocks, self.index_block_size) @@ -380,11 +381,11 @@ def block_keep_mask(self, idx_q, idx_k, position_ids, key_positions): local_hot = ops.max( ops.one_hot(local_idx, num_blocks, dtype="float32"), axis=2 ) - block_scores = ops.where(local_hot > 0, np.inf, block_scores) + block_scores = ops.where(local_hot > 0, float("inf"), block_scores) topk = min(self.index_topk_blocks, num_blocks) top_scores, top_idx = ops.top_k(block_scores, topk) - valid = ops.cast(top_scores > -np.inf, "float32") + valid = ops.cast(top_scores > float("-inf"), "float32") keep_blocks = ops.max( ops.one_hot(top_idx, num_blocks, dtype="float32") * valid[..., None], axis=2, @@ -657,22 +658,26 @@ def vision_rope_3d(grid_thw, head_dim, theta, spatial_merge_size): m = spatial_merge_size coords = [] for t, h, w in grid_thw: - hi = np.tile(np.arange(h)[:, None], (1, w)) - hi = hi.reshape(h // m, m, w // m, m).transpose(0, 2, 1, 3).flatten() - wi = np.tile(np.arange(w)[None, :], (h, 1)) - wi = wi.reshape(h // m, m, w // m, m).transpose(0, 2, 1, 3).flatten() - ti = np.repeat(np.arange(t), h * w) - coords.append(np.stack([ti, np.tile(hi, t), np.tile(wi, t)], axis=-1)) - coords = np.concatenate(coords, axis=0).astype("float32") # (N, 3) - inv_freq = 1.0 / (theta ** (np.arange(0, axis_dim, 2, dtype="float32") / axis_dim)) - freqs = np.concatenate( - [coords[:, i : i + 1] * inv_freq[None] for i in range(3)], axis=-1 + hi = ops.tile(ops.reshape(ops.arange(h), (h, 1)), (1, w)) + hi = ops.reshape( + ops.transpose(ops.reshape(hi, (h // m, m, w // m, m)), (0, 2, 1, 3)), (-1,) + ) + wi = ops.tile(ops.reshape(ops.arange(w), (1, w)), (h, 1)) + wi = ops.reshape( + ops.transpose(ops.reshape(wi, (h // m, m, w // m, m)), (0, 2, 1, 3)), (-1,) + ) + ti = ops.repeat(ops.arange(t), h * w) + coords.append(ops.stack([ti, ops.tile(hi, (t,)), ops.tile(wi, (t,))], axis=-1)) + coords = ops.cast(ops.concatenate(coords, axis=0), "float32") # (N, 3) + exponent = ops.cast(ops.arange(0, axis_dim, 2), "float32") / axis_dim + inv_freq = ops.expand_dims( + 1.0 / ops.power(ops.convert_to_tensor(theta, "float32"), exponent), 0 ) - emb = np.concatenate([freqs, freqs], axis=-1) - return ( - ops.convert_to_tensor(np.cos(emb)), - ops.convert_to_tensor(np.sin(emb)), + freqs = ops.concatenate( + [coords[:, i : i + 1] * inv_freq for i in range(3)], axis=-1 ) + emb = ops.concatenate([freqs, freqs], axis=-1) + return ops.cos(emb), ops.sin(emb) @keras.saving.register_keras_serializable(package="zeromodels") diff --git a/zeromodels/models/minimax_m3_vl/minimax_m3_vl_model.py b/zeromodels/models/minimax_m3_vl/minimax_m3_vl_model.py index 40e630bb..902f30d0 100644 --- a/zeromodels/models/minimax_m3_vl/minimax_m3_vl_model.py +++ b/zeromodels/models/minimax_m3_vl/minimax_m3_vl_model.py @@ -88,11 +88,7 @@ def call(self, pixel_values, grid_thw=None): # so it must be a concrete iterable. In the functional graph this call runs # eagerly (see compute_output_spec), so a tensor grid is materialized here. if not isinstance(grid_thw, (list, tuple)): - import numpy as np - - grid_thw = ( - np.asarray(ops.convert_to_numpy(grid_thw)).astype("int64").tolist() - ) + grid_thw = ops.convert_to_numpy(grid_thw).astype("int64").tolist() cos, sin = vision_rope_3d( grid_thw, self.head_dim, self.rope_theta, self.spatial_merge_size ) @@ -531,9 +527,7 @@ def scatter_features(self, inputs_embeds, input_ids, features, token_id): def host_grid(self, grid_thw): if isinstance(grid_thw, (list, tuple)): return [list(map(int, g)) for g in grid_thw] - import numpy as np - - return np.asarray(ops.convert_to_numpy(grid_thw)).astype("int64").tolist() + return ops.convert_to_numpy(grid_thw).astype("int64").tolist() def prepare_inputs(self, inputs): input_ids = ops.cast(ops.convert_to_tensor(inputs["input_ids"]), "int32") diff --git a/zeromodels/models/rf_detr/rf_detr_model.py b/zeromodels/models/rf_detr/rf_detr_model.py index 1a3ab93a..b66e4810 100644 --- a/zeromodels/models/rf_detr/rf_detr_model.py +++ b/zeromodels/models/rf_detr/rf_detr_model.py @@ -1531,14 +1531,12 @@ class RFDETRDetect(BaseModel): @classmethod def transfer_from_hf(cls, keras_model, state_dict): - import numpy as np - from .convert_rf_detr_hf_to_keras import transfer_rf_detr_weights # RF-DETR's custom layers create their weights on the first call, so # build the functional graph on a dummy input before assigning weights. shape = [d if d is not None else 1 for d in keras_model.inputs[0].shape] - keras_model(np.zeros(shape, dtype="float32")) + keras_model(ops.zeros(shape, dtype="float32")) transfer_rf_detr_weights(keras_model, state_dict) @classmethod @@ -1836,12 +1834,10 @@ class RFDETRInstanceSegment(BaseModel): @classmethod def transfer_from_hf(cls, keras_model, state_dict): - import numpy as np - from .convert_rf_detr_hf_to_keras import transfer_rf_detr_seg_weights shape = [d if d is not None else 1 for d in keras_model.inputs[0].shape] - keras_model(np.zeros(shape, dtype="float32")) + keras_model(ops.zeros(shape, dtype="float32")) transfer_rf_detr_seg_weights(keras_model, state_dict) @classmethod diff --git a/zeromodels/models/sam3/sam3_image_processor.py b/zeromodels/models/sam3/sam3_image_processor.py index c5582f62..f7224d4b 100644 --- a/zeromodels/models/sam3/sam3_image_processor.py +++ b/zeromodels/models/sam3/sam3_image_processor.py @@ -1,7 +1,7 @@ from typing import Optional, Tuple import keras -import numpy as np +from keras import ops from zeromodels.base import BaseImageProcessor @@ -61,7 +61,7 @@ def call(self, image): rescale_factor=self.rescale_factor, ) return { - "pixel_values": np.asarray(pixel_values), + "pixel_values": ops.convert_to_numpy(pixel_values), "original_size": original_size, } diff --git a/zeromodels/models/tipsv2/tipsv2_tokenizer.py b/zeromodels/models/tipsv2/tipsv2_tokenizer.py index f7516c89..d383df75 100644 --- a/zeromodels/models/tipsv2/tipsv2_tokenizer.py +++ b/zeromodels/models/tipsv2/tipsv2_tokenizer.py @@ -1,7 +1,6 @@ from typing import List, Union import keras -import numpy as np from keras import ops from tokenizers import Tokenizer from tokenizers.pre_tokenizers import Metaspace @@ -59,8 +58,6 @@ def from_hf(cls, repo, **kwargs): try: path = hf_hub_download(repo, "tokenizer.json") except Exception: - # Original TIPSv2 repos ship only a SentencePiece ``tokenizer.model``; - # build the fast tokenizer.json from it (matches transformers' loader). from transformers import AutoTokenizer src = hf_hub_download(repo, "tokenizer.model") @@ -84,7 +81,7 @@ def detokenize( self, token_ids, skip_special_tokens: bool = True ) -> Union[str, List[str]]: if hasattr(token_ids, "numpy"): - token_ids = token_ids.numpy() + token_ids = ops.convert_to_numpy(token_ids) if hasattr(token_ids, "tolist"): token_ids = token_ids.tolist() @@ -103,18 +100,18 @@ def call(self, inputs): """Encode text -> ``{"input_ids", "attention_mask"}`` (fixed ``max_seq_len``).""" texts = [inputs] if isinstance(inputs, str) else list(inputs) encs = self._tok.encode_batch(texts, add_special_tokens=True) - ids = np.array([e.ids for e in encs], dtype="int32") - mask = np.array([e.attention_mask for e in encs], dtype="int32") return { - "input_ids": ops.convert_to_tensor(ids, dtype="int32"), - "attention_mask": ops.convert_to_tensor(mask, dtype="int32"), + "input_ids": ops.convert_to_tensor([e.ids for e in encs], dtype="int32"), + "attention_mask": ops.convert_to_tensor( + [e.attention_mask for e in encs], dtype="int32" + ), } def batch_decode( self, token_ids_batch, skip_special_tokens: bool = True ) -> List[str]: if hasattr(token_ids_batch, "numpy"): - token_ids_batch = token_ids_batch.numpy() + token_ids_batch = ops.convert_to_numpy(token_ids_batch) out = [] for row in token_ids_batch: row = row.tolist() if hasattr(row, "tolist") else list(row) diff --git a/zeromodels/models/whisper/whisper_model.py b/zeromodels/models/whisper/whisper_model.py index 72c29e07..89e18f2f 100644 --- a/zeromodels/models/whisper/whisper_model.py +++ b/zeromodels/models/whisper/whisper_model.py @@ -1,7 +1,6 @@ from typing import List, Optional, Union import keras -import numpy as np from keras import layers, ops from zeromodels.base import BaseModel, BaseSeq2SeqGeneration @@ -22,47 +21,64 @@ def apply_whisper_timestamp_rules( scores, generated, begin_index, timestamp_begin, eos_token_id, max_initial_index ): - """transformers' ``WhisperTimeStampLogitsProcessor``, host-side (numpy). + """transformers' ``WhisperTimeStampLogitsProcessor`` (host-side grammar, keras ops). Enforces Whisper's timestamp-token grammar on a step's logits so the model emits a valid ``<|t|> text <|t|>`` structure: timestamps come in pairs, never decrease, and a step is forced to a timestamp when their summed probability beats the top text - token. ``scores`` is ``(B, V)``; ``generated`` is ``(B, L)`` with the forced prompt - at ``[:begin_index]``. Coordinates: ``<|notimestamps|>`` is ``timestamp_begin - 1``, - and ``<|0.00|>`` is ``timestamp_begin``. Returns the modified scores. + token. ``scores`` is a ``(B, V)`` tensor; ``generated`` is a list of ``B`` token-id + lists with the forced prompt at ``[:begin_index]``. Coordinates: ``<|notimestamps|>`` + is ``timestamp_begin - 1``, and ``<|0.00|>`` is ``timestamp_begin``. The per-batch + grammar decisions are computed host-side from ``generated``, then applied to the + logits with ``ops.where``. Returns the modified scores tensor. """ neg = -1e9 # matches the model's additive suppress bias; never the argmax - scores = scores.copy() - scores[:, timestamp_begin - 1] = neg # never emit <|notimestamps|> mid-stream - batch, length = generated.shape + batch = len(generated) + length = len(generated[0]) + col = ops.arange(int(scores.shape[-1])) + rows = [] for k in range(batch): - seq = generated[k, begin_index:].tolist() + seq = generated[k][begin_index:] last_ts = len(seq) >= 1 and seq[-1] >= timestamp_begin penult_ts = len(seq) < 2 or seq[-2] >= timestamp_begin + # always suppress <|notimestamps|> mid-stream + row = ops.equal(col, timestamp_begin - 1) if last_ts: if penult_ts: # a pair just closed -> the next token must be text - scores[k, timestamp_begin:] = neg + row = ops.logical_or(row, col >= timestamp_begin) else: # one timestamp open -> the next must be a timestamp (or eos) - scores[k, :eos_token_id] = neg + row = ops.logical_or(row, col < eos_token_id) stamps = [t for t in seq if t >= timestamp_begin] if stamps: # timestamps are non-decreasing ts_last = stamps[-1] if (last_ts and not penult_ts) else stamps[-1] + 1 - scores[k, timestamp_begin:ts_last] = neg + row = ops.logical_or( + row, ops.logical_and(col >= timestamp_begin, col < ts_last) + ) + rows.append(row) + forbidden = ops.stack(rows, axis=0) if length == begin_index: # the very first generated token must be a timestamp - scores[:, :timestamp_begin] = neg + first = col < timestamp_begin if max_initial_index is not None: - scores[:, timestamp_begin + max_initial_index + 1 :] = neg + first = ops.logical_or( + first, col >= timestamp_begin + max_initial_index + 1 + ) + forbidden = ops.logical_or(forbidden, first[None, :]) + scores = ops.where(forbidden, neg, scores) # If the total probability mass on timestamps exceeds the top text token, force one. + mx = ops.max(scores, axis=-1, keepdims=True) logprobs = scores - ( - scores.max(-1, keepdims=True) - + np.log(np.exp(scores - scores.max(-1, keepdims=True)).sum(-1, keepdims=True)) + mx + ops.log(ops.sum(ops.exp(scores - mx), axis=-1, keepdims=True)) ) + force = [] for k in range(batch): ts_lp = logprobs[k, timestamp_begin:] - ts_logprob = ts_lp.max() + np.log(np.exp(ts_lp - ts_lp.max()).sum()) - if ts_logprob > logprobs[k, :timestamp_begin].max(): - scores[k, :timestamp_begin] = neg - return scores + ts_max = ops.max(ts_lp) + ts_logprob = ts_max + ops.log(ops.sum(ops.exp(ts_lp - ts_max))) + force.append(ts_logprob > ops.max(logprobs[k, :timestamp_begin])) + force_ts = ops.logical_and( + ops.stack(force, axis=0)[:, None], (col < timestamp_begin)[None, :] + ) + return ops.where(force_ts, neg, scores) _ACTIVATION_ALIASES = { @@ -680,8 +696,8 @@ def generate( generated = super().generate( features, decoder_start_ids, max_new_tokens=max_new_tokens, eos_token_id=eos ) - prompt_col = np.tile(np.asarray(prompt_ids, dtype=generated.dtype), (batch, 1)) - ids = [list(row) for row in np.concatenate([prompt_col, generated], axis=1)] + gen = ops.convert_to_numpy(generated) + ids = [list(prompt_ids) + gen[k].tolist() for k in range(batch)] if return_ids: return ids return processor.batch_decode(ids, skip_special_tokens=True) @@ -721,21 +737,21 @@ def _generate_with_timestamps( cache, logits = self.build_cache( decoder_start_ids, encoder_hidden_states, prompt_len + max_new_tokens ) - generated = np.tile(np.asarray(prompt_ids, dtype="int32"), (batch, 1)) - done = np.zeros((batch,), dtype=bool) + generated = [list(prompt_ids) for _ in range(batch)] + done = [False] * batch for step in range(max_new_tokens): - scores = np.asarray(ops.convert_to_numpy(logits), dtype="float32") scores = apply_whisper_timestamp_rules( - scores, generated, prompt_len, timestamp_begin, eos, max_initial + logits, generated, prompt_len, timestamp_begin, eos, max_initial ) - nxt = scores.argmax(axis=-1).astype("int32") - nxt = np.where(done, eos, nxt).astype("int32") - generated = np.concatenate([generated, nxt[:, None]], axis=1) - done = done | (nxt == eos) - if bool(done.all()) or step == max_new_tokens - 1: + nxt = ops.convert_to_numpy(ops.argmax(scores, axis=-1)).tolist() + nxt = [eos if done[k] else int(nxt[k]) for k in range(batch)] + for k in range(batch): + generated[k].append(nxt[k]) + done[k] = done[k] or (nxt[k] == eos) + if all(done) or step == max_new_tokens - 1: break logits, cache = self.call_with_cache( - ops.convert_to_tensor(nxt[:, None], dtype="int32"), + ops.convert_to_tensor([[x] for x in nxt], dtype="int32"), cache, prompt_len + step, ) @@ -751,7 +767,7 @@ def _generate_with_timestamps( processor, eos, ) - for k in range(int(generated.shape[0])) + for k in range(len(generated)) ] def _parse_timestamp_segments(