diff --git a/.github/workflows/build.sh b/.github/workflows/build.sh index f5a8d3c..1234af4 100755 --- a/.github/workflows/build.sh +++ b/.github/workflows/build.sh @@ -17,8 +17,14 @@ rm -rf html_doc rpms .build_hash if test "${OS_VERSION}" = "centos8"; then export METWORK_BUILD_OS=generic; else export METWORK_BUILD_OS=${OS_VERSION}; fi +case "${BRANCH}" in + ci* | pci*) + export DEP_BRANCH=integration;; + *) + export DEP_BRANCH=${BRANCH};; +esac - yum install -y metwork-mfext-layer-python3_scientific-${BRANCH##release_} + yum install -y metwork-mfext-layer-python3_scientific-${DEP_BRANCH##release_} @@ -49,7 +55,7 @@ fi MODULEHASH=`/opt/metwork-mfext-${TARGET_DIR}/bin/mfext_wrapper module_hash 2>module_hash.debug` if test -f /opt/metwork-mfext-${TARGET_DIR}/.dhash; then cat /opt/metwork-mfext-${TARGET_DIR}/.dhash; fi cat module_hash.debug |sort |uniq ; rm -f module_hash.debug -echo "${MODULEHASH}${DRONE_TAG}${DRONE_BRANCH}" |md5sum |cut -d ' ' -f1 >.build_hash +echo "${MODULEHASH}${DRONE_TAG}${BRANCH}" |md5sum |cut -d ' ' -f1 >.build_hash if test -f "${BUILDCACHE}/build_hash_mfextaddon_python3_ia_${BRANCH}_`cat .build_hash`"; then echo "bypass=true" >> github_output echo "buildcache=null" >> github_output diff --git a/.mergify.yml b/.mergify.yml index 9de7cb1..0c56638 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -107,6 +107,14 @@ pull_request_rules: backport: branches: - release_2.3 + - name: backport to 2.4 + conditions: + - base=integration + - label=backport-to-2.4 + actions: + backport: + branches: + - release_2.4 - name: approved merges actions: diff --git a/.metwork-framework/components.md b/.metwork-framework/components.md index a0a7066..d02af2f 100644 --- a/.metwork-framework/components.md +++ b/.metwork-framework/components.md @@ -16,20 +16,20 @@ | [google-auth-oauthlib](https://github.com/GoogleCloudPlatform/google-auth-library-python-oauthlib) | 1.2.2 | python3_ia | | [google-auth](https://github.com/googleapis/google-auth-library-python) | 2.53.0 | python3_ia | | [google-pasta](https://github.com/google/pasta) | 0.2.0 | python3_ia | -| [grpcio](https://grpc.io) | 1.76.0 | python3_ia | +| [grpcio](https://grpc.io) | 1.84.0 | python3_ia | | [hf-xet](https://github.com/huggingface/xet-core) | 1.4.3 | python3_ia | | [huggingface_hub](https://github.com/huggingface/huggingface_hub) | 1.5.0 | python3_ia | -| [hydra-core](https://github.com/facebookresearch/hydra) | 1.3.5 | python3_ia | +| [hydra-core](https://github.com/facebookresearch/hydra) | 1.3.7 | python3_ia | | [jsonargparse](https://pypi.org/project/jsonargparse) | 4.46.0 | python3_ia | | [jsonnet](https://jsonnet.org) | 0.21.0 | python3_ia | | [keras](https://pypi.org/project/keras) | 3.15.1 | python3_ia | | [libclang](https://github.com/sighingnow/libclang) | 18.1.1 | python3_ia | -| [lightning-fabric](https://github.com/Lightning-AI/lightning) | 2.6.5 | python3_ia | +| [lightning-fabric](https://github.com/Lightning-AI/lightning) | 2.6.6 | python3_ia | | [lightning-utilities](https://github.com/Lightning-AI/utilities) | 0.15.3 | python3_ia | -| [lightning](https://github.com/Lightning-AI/lightning) | 2.6.5 | python3_ia | +| [lightning](https://github.com/Lightning-AI/lightning) | 2.6.6 | python3_ia | | [lit](http://llvm.org) | 18.1.8 | python3_ia | | [marshmallow](https://pypi.org/project/marshmallow) | 3.26.2 | python3_ia | -| [mfai](https://github.com/meteofrance/mfai) | 8.0.0 | python3_ia | +| [mfai](https://github.com/meteofrance/mfai) | 8.0.2 | python3_ia | | [ml_dtypes](https://github.com/jax-ml/ml_dtypes) | 0.6.0 | python3_ia | | [monai](https://project-monai.github.io/) | 1.6.0 | python3_ia | | [mpmath](http://mpmath.org/) | 1.3.0 | python3_ia | @@ -38,25 +38,25 @@ | [nvidia-cuda-cupti](https://developer.nvidia.com/cuda-zone) | 13.0.85 | python3_ia | | [nvidia-cuda-nvrtc](https://developer.nvidia.com/cuda-zone) | 13.0.88 | python3_ia | | [nvidia-cuda-runtime](https://developer.nvidia.com/cuda-zone) | 13.0.96 | python3_ia | -| [nvidia-cudnn-cu13](https://developer.nvidia.com/cuda-zone) | 9.20.0.48 | python3_ia | +| [nvidia-cudnn-cu13](https://developer.nvidia.com/cuda-zone) | 9.24.0.43 | python3_ia | | [nvidia-cufft](https://developer.nvidia.com/cuda-zone) | 12.0.0.61 | python3_ia | | [nvidia-cufile](https://developer.nvidia.com/cuda-zone) | 1.15.1.6 | python3_ia | | [nvidia-curand](https://developer.nvidia.com/cuda-zone) | 10.4.0.35 | python3_ia | | [nvidia-cusolver](https://developer.nvidia.com/cuda-zone) | 12.0.4.66 | python3_ia | | [nvidia-cusparse](https://developer.nvidia.com/cuda-zone) | 12.6.3.3 | python3_ia | | [nvidia-cusparselt-cu13](https://developer.nvidia.com/cusparselt) | 0.8.1 | python3_ia | -| [nvidia-nccl-cu13](https://developer.nvidia.com/cuda-zone) | 2.29.7 | python3_ia | +| [nvidia-nccl-cu13](https://developer.nvidia.com/cuda-zone) | 2.30.7 | python3_ia | | [nvidia-nvjitlink](https://developer.nvidia.com/cuda-zone) | 13.0.88 | python3_ia | | [nvidia-nvshmem-cu13](https://developer.nvidia.com/cuda-zone) | 3.4.5 | python3_ia | | [nvidia-nvtx](https://developer.nvidia.com/cuda-zone) | 13.0.85 | python3_ia | -| [oauthlib](https://github.com/oauthlib/oauthlib) | 3.2.2 | python3_ia | +| [oauthlib](https://github.com/oauthlib/oauthlib) | 4.0.0 | python3_ia | | [omegaconf](https://github.com/omry/omegaconf) | 2.3.1 | python3_ia | | [onnx](https://onnx.ai/) | 1.22.0 | python3_ia | | [onnxruntime](https://onnxruntime.ai) | 1.29.0 | python3_ia | | [onnxscript](https://microsoft.github.io/onnxscript/) | 0.2.5 | python3_ia | | [opt_einsum](https://pypi.org/project/opt_einsum) | 3.4.0 | python3_ia | -| [optree](https://github.com/metaopt/optree) | 0.18.0 | python3_ia | -| [pytorch-lightning](https://github.com/Lightning-AI/lightning) | 2.6.5 | python3_ia | +| [optree](https://github.com/metaopt/optree) | 0.20.0 | python3_ia | +| [pytorch-lightning](https://github.com/Lightning-AI/lightning) | 2.6.6 | python3_ia | | [requests-oauthlib](https://github.com/requests/requests-oauthlib) | 2.0.0 | python3_ia | | [rsa](https://stuvel.eu/rsa) | 4.9 | python3_ia | | [safetensors](https://github.com/huggingface/safetensors) | 0.8.0 | python3_ia | @@ -67,17 +67,15 @@ | [tensorboard-data-server](https://github.com/tensorflow/tensorboard/tree/master/tensorboard/data/server) | 0.7.2 | python3_ia | | [tensorboard](https://github.com/tensorflow/tensorboard) | 2.20.0 | python3_ia | | [tensorboardX](https://github.com/lanpa/tensorboardX) | 2.6.5 | python3_ia | -| [tensorflow](https://www.tensorflow.org/) | 2.21.0 | python3_ia | -| [tf_keras](https://keras.io/) | 2.21.0 | python3_ia | -| [tiktoken](https://github.com/openai/tiktoken) | 0.12.0 | python3_ia | +| [tiktoken](https://github.com/openai/tiktoken) | 0.14.0 | python3_ia | | [timm](https://github.com/huggingface/pytorch-image-models) | 1.0.22 | python3_ia | | [tokenizers](https://github.com/huggingface/tokenizers) | 0.23.1 | python3_ia | | [torch-geometric](https://pyg.org) | 2.7.0 | python3_ia | -| [torch](https://pytorch.org) | 2.13.0 | python3_ia | +| [torch](https://pytorch.org) | 2.14.0 | python3_ia | | [torchmetrics](https://github.com/Lightning-AI/torchmetrics) | 1.9.0 | python3_ia | -| [torchvision](https://github.com/pytorch/vision) | 0.28.0 | python3_ia | +| [torchvision](https://github.com/pytorch/vision) | 0.29.0 | python3_ia | | [transformers](https://github.com/huggingface/transformers) | 5.16.1 | python3_ia | -| [triton](https://github.com/triton-lang/triton/) | 3.7.1 | python3_ia | +| [triton](https://github.com/triton-lang/triton/) | 3.8.0 | python3_ia | | [typeshed_client](https://github.com/JelleZijlstra/typeshed_client) | 2.8.2 | python3_ia | -*(79 components)* +*(77 components)* diff --git a/CHANGELOG-2.4.md b/CHANGELOG-2.4.md new file mode 100644 index 0000000..e2ddab9 --- /dev/null +++ b/CHANGELOG-2.4.md @@ -0,0 +1,3 @@ +# release_2.4 CHANGELOG + + diff --git a/CHANGELOG.md b/CHANGELOG.md index 7431efa..78fe47f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,61 +4,16 @@ ### New Features -- bump keras from 3.9.2 to 3.10.0 (#204) -- bump onnx from 1.17.0 to 1.18.0 (#208) -- bump to torch 2.7.1, triton 3.3.1 and torchvision 0.22.1 (#210) -- bump transformers from 4.52.1 to 4.52.4 (#213) -- bump keras from 3.10.0 to 3.11.3 (fix high CVE-2025-8747) (#219) -- bump transformers from 4.52.4 to 4.53.3 (fix CVE-2025-5197) (#221) -- bump tensorflow (and dependencies) from 2.19.0 to 2.20.0 (#222) -- bump tf_keras from 2.20.0 (yanked) to 2.20.1 (#224) -- bump lightning from 2.5.1.post0 to 2.5.5 (#226) -- bump torch from 2.7.1 to 2.8.0 (fix moderate CVE-2025-3730) (#227) -- bump keras from 3.11.3 to 3.12.0 (fix high CVE-2025-12060) (#229) -- bump safetensors from 0.5.3 to 0.6.2 (#230) -- bump grpcio from 1.71.0 to 1.76.0 (compat Python 3.14) (#232) -- bump ml-dtypes from 0.5.1 to 0.5.4 (compat Python 3.14) (#233) -- bump onnx from 1.18.0 to 1.20.0 (compat Python 3.14) (#234) -- bump optree from 0.15.0 to 0.18.0 (compat Python 3.14) (#235) -- bump timm from 1.0.15 to 1.0.22 (compat Python 3.14) (#236) -- bump torch from 2.8.0 to 2.9.1 (compat Python 3.14) (#237) -- bump lightning from 2.5.5 to 2.6.0 (#239) -- bump keras from 3.12.0 to 3.13.1 (fix high CVE-2026-0897) (#241) -- bump google-auth from 2.40.1 to 2.41.1 (compat cachetools>=6) (#243) -- bump torch, torchvision, triton to 2.10.0, 0.25.0, 3.6.0 (#245) -- bump keras from 3.13.1 to 3.13.2 (fix high CVE-2026-0897) (#246) -- bump lightning from 2.6.0 to 2.6.1 (#249) -- bump onnx from 1.20.0 to 1.20.1 (#250) -- add mfai package and dependencies (#251) -- bump torchmetrics to 1.9.0 and lightning-utilities to 0.15.3 (#256) -- bump onnxruntime from 1.22.0 to 1.24.4 (compat Python 3.14) (#258) -- bump onnx from 1.20.1 to 1.21.0 (fix CVE-2026-34445/28500) (#262) -- bump transformers from 4.53.3 to 5.3.0 (fix moderate CVE-2026-1… (#264) -- add lightning-fabric (#268) -- bump diffusers to 0.38.0 (fix high CVE-2026-44513/CVE-2026-45804) (#269) -- bump lightning from 2.6.1 to 2.6.5 (#272) -- remove tensorflow-hub (not required anymore) (#273) -- bump google-auth from 2.41.1 to 2.53.0 (#275) -- bump tensorflow to 2.21.0 and temporarly remove mfai (#276) -- bump torch from 2.10.0 to 2.12.0 (#277) -- bump bitsandbytes from 0.49.1 to 0.49.2 (#278) -- add mfai 7.3.0 (#279) -- bump keras from 3.13.2 to 3.14.1 (#281) -- bump onnx from 1.21.0 to 1.22.0 (fix moderate CVE-2026-44512) (#287) -- bump transformers from 5.3.0 to 5.5.4 (fix high CVE-2026-5241) -- bump torch from 2.12.0 to 2.13.0 (fix low CVE-2025-3000) (#293) -- bump onnxruntime from 1.24.4 to 1.25.1 (fix high CVE-2026-27904) (#295) -- bump onnxruntime from 1.25.1 to 1.28.0 (#297) -- bump safetensors from 0.8.0rc0 to 0.8.0 (#298) -- bump monai from 1.5.2 to 1.6.0 (fix high GHSA-89gg-p5r5-q6r4) -- bump hydra-core from 1.3.2 to 1.3.4 (#303) -- bump omegaconf from 2.3.0 to 2.3.1 -- bump keras from 3.14.1 to 3.15.1 (fix 2 high CVE) (#309) -- bump sqlparse from 0.5.5 to 0.6.0 (fix 3 high CVE) (#314) -- add mfai 8.0.0 (release 7.3.0 had been temporarly removed) (#316) -- bump hydra-core from 1.3.4 to 1.3.5 (#317) -- bump ml_dtypes from 0.5.4 to 0.6.0 (#318) -- bump transformers from 5.5.4 to 5.16.1 (fix high CVE-2026-9856) (#320) -- bump onnxruntime from 1.28.0 to 1.29.0 (#322) +- various updates for Python 3.14 compatibility +- add onnxruntime 1.24.4 (now Python 3.14 compliant) +- use torch as backend for keras (tensorflow not available) +- reintroduce mfai 8.0.2 +- bump lightning from 2.6.5 to 2.6.6 (fix high CVE-2026-58659) +- bump torch from 2.13.0 to 2.14.0 and triton from 3.7.1 to 3.8.0 (#327) +- bump hydra-core from 1.3.5 to 1.3.7 (#328) +- bump grpcio from 1.76.0 to 1.84.0 (compat. Python 3.15) +- bump grpcio from 1.76.0 to 1.84.0 (compat. Python 3.15) (#330) +- bump tiktoken from 0.12.0 to 0.14.0 (compat. Python 3.15) +- bump oauthlib from 3.2.2 to 4.0.0 (fix 2 moderate CVE) (#333) diff --git a/bootstrap.sh b/bootstrap.sh index 8c83c92..6b52044 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -83,7 +83,7 @@ fi # FIXME: do not hardcode this # FIXME: move to layer root extra_env ? -echo "export PYTHON3_SHORT_VERSION := 3.13" >>adm/root.mk +echo "export PYTHON3_SHORT_VERSION := 3.14" >>adm/root.mk echo "BOOTSTRAP DONE !" echo "MFEXT_HOME=${MFEXT_HOME}" diff --git a/integration_tests/0003_test_import_python3_ia/import_python3_ia.py b/integration_tests/0003_test_import_python3_ia/import_python3_ia.py index e4ac348..d19aff6 100644 --- a/integration_tests/0003_test_import_python3_ia/import_python3_ia.py +++ b/integration_tests/0003_test_import_python3_ia/import_python3_ia.py @@ -23,6 +23,7 @@ import clang #libclang import lightning_fabric import lightning_utilities +import lightning_fabric import lightning import marshmallow import mfai @@ -53,8 +54,6 @@ import tensorboard_data_server import tensorboard import tensorboardX -import tensorflow -import tf_keras import tiktoken import timm import tokenizers diff --git a/layers/layer5_python3_ia/.layerapi2_extra_env b/layers/layer5_python3_ia/.layerapi2_extra_env new file mode 100644 index 0000000..9e628bd --- /dev/null +++ b/layers/layer5_python3_ia/.layerapi2_extra_env @@ -0,0 +1 @@ +KERAS_BACKEND=torch diff --git a/layers/layer5_python3_ia/0500_extra_packages/allow_binary_packages b/layers/layer5_python3_ia/0500_extra_packages/allow_binary_packages index 869ad3d..d387561 100644 --- a/layers/layer5_python3_ia/0500_extra_packages/allow_binary_packages +++ b/layers/layer5_python3_ia/0500_extra_packages/allow_binary_packages @@ -42,7 +42,6 @@ opt-einsum optree protobuf pyarrow -PyYAML requests-oauthlib rich rsa diff --git a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt index d6864e1..e328ceb 100644 --- a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt +++ b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt @@ -14,20 +14,20 @@ flatbuffers==25.12.19 google-auth==2.53.0 google-auth-oauthlib==1.2.2 google-pasta==0.2.0 -grpcio==1.76.0 +grpcio==1.84.0 hf_xet==1.4.3 huggingface_hub==1.5.0 -hydra-core==1.3.5 +hydra-core==1.3.7 jsonargparse==4.46.0 jsonnet==0.21.0 keras==3.15.1 libclang==18.1.1 -lightning==2.6.5 -lightning-fabric==2.6.5 +lightning==2.6.6 +lightning-fabric==2.6.6 lightning-utilities==0.15.3 lit==18.1.8 marshmallow==3.26.2 -mfai==8.0.0 +mfai==8.0.2 ml-dtypes==0.6.0 monai==1.6.0 mpmath==1.3.0 @@ -36,7 +36,7 @@ nvidia_cublas==13.1.1.3 nvidia-cuda-cupti==13.0.85 nvidia-cuda-nvrtc==13.0.88 nvidia-cuda-runtime==13.0.96 -nvidia-cudnn-cu13==9.20.0.48 +nvidia-cudnn-cu13==9.24.0.43 nvidia-cufft==12.0.0.61 nvidia-cufile==1.15.1.6 nvidia-curand==10.4.0.35 @@ -44,17 +44,17 @@ nvidia-cusolver==12.0.4.66 nvidia_cusparse==12.6.3.3 nvidia-cusparselt-cu13==0.8.1 nvidia_nvjitlink==13.0.88 -nvidia-nccl-cu13==2.29.7 +nvidia-nccl-cu13==2.30.7 nvidia-nvshmem-cu13==3.4.5 nvidia-nvtx==13.0.85 -oauthlib==3.2.2 +oauthlib==4.0.0 omegaconf==2.3.1 onnx==1.22.0 onnxruntime==1.29.0 onnxscript==0.2.5 opt-einsum==3.4.0 -optree==0.18.0 -pytorch-lightning==2.6.5 +optree==0.20.0 +pytorch-lightning==2.6.6 requests-oauthlib==2.0.0 rsa==4.9 safetensors==0.8.0 @@ -65,15 +65,13 @@ tabulate==0.9.0 tensorboard==2.20.0 tensorboard-data-server==0.7.2 tensorboardx==2.6.5 -tensorflow==2.21.0 -tf-keras==2.21.0 -tiktoken==0.12.0 +tiktoken==0.14.0 timm==1.0.22 tokenizers==0.23.1 -torch==2.13.0 +torch==2.14.0 torchmetrics==1.9.0 -torchvision==0.28.0 +torchvision==0.29.0 torch-geometric==2.7.0 transformers==5.16.1 -triton==3.7.1 +triton==3.8.0 typeshed-client==2.8.2