diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index 1b3749fdebf..de21f0b01b2 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -1133,5 +1133,7 @@ jobs: # Test models # TODO: Switch to run_tests.py once CI device provisioning is stable # python -m executorch.backends.samsung.test.utils.run_tests --chipset E9955 - pip install evaluate + # accelerate is for test_mobilebert_qat, whose TrainingArguments needs it; + # transformers reports the same error when it is absent and when it is too old. + pip install evaluate 'accelerate>=1.1.0' python -m unittest discover -s backends/samsung/test/models -p "test_*.py" diff --git a/examples/samsung/scripts/mobilebert_finetune_QAT.py b/examples/samsung/scripts/mobilebert_finetune_QAT.py index 28137beea3e..584388b2107 100644 --- a/examples/samsung/scripts/mobilebert_finetune_QAT.py +++ b/examples/samsung/scripts/mobilebert_finetune_QAT.py @@ -9,6 +9,8 @@ from pathlib import Path from typing import Optional +import datasets.config + import evaluate import numpy as np import requests @@ -16,7 +18,6 @@ import torch import torch.nn as nn import torchao - from datasets import ClassLabel, DatasetDict, load_dataset from executorch.backends.samsung.quantizer import EnnQuantizer, Precision @@ -140,6 +141,14 @@ def preprocess_function(examples): print("Preprocessing data...") tokenized_datasets = raw_datasets.map(preprocess_function, batched=True) + # datasets' torch formatter does `from torchvision.io import VideoReader` + # whenever torchvision merely imports, but OSS torchvision ships that symbol + # only in Meta-internal builds -- its io/__init__ swallows the import with + # `except ImportError: pass`. timm puts torchvision in sys.modules, so + # collating the tensors below raises ImportError. This dataset is text only + # and never decodes an image or a video, so the torchvision path is dead + # weight here. Drop the two lines once datasets guards that import. + datasets.config.TORCHVISION_AVAILABLE = False tokenized_datasets.set_format( type="torch", columns=["input_ids", "attention_mask", "label"] )