build: merge integration branch

.github/workflows/build.sh

@@ -17,8 +17,14 @@ rm -rf html_doc rpms .build_hash
 
     if test "${OS_VERSION}" = "centos8"; then export METWORK_BUILD_OS=generic; else export METWORK_BUILD_OS=${OS_VERSION}; fi
 
+case "${BRANCH}" in
+    ci* | pci*)
+        export DEP_BRANCH=integration;;
+    *)
+        export DEP_BRANCH=${BRANCH};;
+esac
 
-    yum install -y metwork-mfext-layer-python3_scientific-${BRANCH##release_}
+    yum install -y metwork-mfext-layer-python3_scientific-${DEP_BRANCH##release_}
 
 
 
@@ -49,7 +55,7 @@ fi
 MODULEHASH=`/opt/metwork-mfext-${TARGET_DIR}/bin/mfext_wrapper module_hash 2>module_hash.debug`
 if test -f /opt/metwork-mfext-${TARGET_DIR}/.dhash; then cat /opt/metwork-mfext-${TARGET_DIR}/.dhash; fi
 cat module_hash.debug |sort |uniq ; rm -f module_hash.debug
-echo "${MODULEHASH}${DRONE_TAG}${DRONE_BRANCH}" |md5sum |cut -d ' ' -f1 >.build_hash
+echo "${MODULEHASH}${DRONE_TAG}${BRANCH}" |md5sum |cut -d ' ' -f1 >.build_hash
 if test -f "${BUILDCACHE}/build_hash_mfextaddon_python3_ia_${BRANCH}_`cat .build_hash`"; then
     echo "bypass=true" >> github_output
     echo "buildcache=null" >> github_output

.mergify.yml

@@ -107,6 +107,14 @@ pull_request_rules:
       backport:
         branches:
           - release_2.3
+  - name: backport to 2.4
+    conditions:
+      - base=integration
+      - label=backport-to-2.4
+    actions:
+      backport:
+        branches:
+          - release_2.4
 
   - name: approved merges
     actions:

.metwork-framework/components.md

@@ -16,20 +16,20 @@
 | [google-auth-oauthlib](https://github.com/GoogleCloudPlatform/google-auth-library-python-oauthlib) | 1.2.2 | python3_ia |
 | [google-auth](https://github.com/googleapis/google-auth-library-python) | 2.53.0 | python3_ia |
 | [google-pasta](https://github.com/google/pasta) | 0.2.0 | python3_ia |
-| [grpcio](https://grpc.io) | 1.76.0 | python3_ia |
+| [grpcio](https://grpc.io) | 1.84.0 | python3_ia |
 | [hf-xet](https://github.com/huggingface/xet-core) | 1.4.3 | python3_ia |
 | [huggingface_hub](https://github.com/huggingface/huggingface_hub) | 1.5.0 | python3_ia |
-| [hydra-core](https://github.com/facebookresearch/hydra) | 1.3.5 | python3_ia |
+| [hydra-core](https://github.com/facebookresearch/hydra) | 1.3.7 | python3_ia |
 | [jsonargparse](https://pypi.org/project/jsonargparse) | 4.46.0 | python3_ia |
 | [jsonnet](https://jsonnet.org) | 0.21.0 | python3_ia |
 | [keras](https://pypi.org/project/keras) | 3.15.1 | python3_ia |
 | [libclang](https://github.com/sighingnow/libclang) | 18.1.1 | python3_ia |
-| [lightning-fabric](https://github.com/Lightning-AI/lightning) | 2.6.5 | python3_ia |
+| [lightning-fabric](https://github.com/Lightning-AI/lightning) | 2.6.6 | python3_ia |
 | [lightning-utilities](https://github.com/Lightning-AI/utilities) | 0.15.3 | python3_ia |
-| [lightning](https://github.com/Lightning-AI/lightning) | 2.6.5 | python3_ia |
+| [lightning](https://github.com/Lightning-AI/lightning) | 2.6.6 | python3_ia |
 | [lit](http://llvm.org) | 18.1.8 | python3_ia |
 | [marshmallow](https://pypi.org/project/marshmallow) | 3.26.2 | python3_ia |
-| [mfai](https://github.com/meteofrance/mfai) | 8.0.0 | python3_ia |
+| [mfai](https://github.com/meteofrance/mfai) | 8.0.2 | python3_ia |
 | [ml_dtypes](https://github.com/jax-ml/ml_dtypes) | 0.6.0 | python3_ia |
 | [monai](https://project-monai.github.io/) | 1.6.0 | python3_ia |
 | [mpmath](http://mpmath.org/) | 1.3.0 | python3_ia |
@@ -38,25 +38,25 @@
 | [nvidia-cuda-cupti](https://developer.nvidia.com/cuda-zone) | 13.0.85 | python3_ia |
 | [nvidia-cuda-nvrtc](https://developer.nvidia.com/cuda-zone) | 13.0.88 | python3_ia |
 | [nvidia-cuda-runtime](https://developer.nvidia.com/cuda-zone) | 13.0.96 | python3_ia |
-| [nvidia-cudnn-cu13](https://developer.nvidia.com/cuda-zone) | 9.20.0.48 | python3_ia |
+| [nvidia-cudnn-cu13](https://developer.nvidia.com/cuda-zone) | 9.24.0.43 | python3_ia |
 | [nvidia-cufft](https://developer.nvidia.com/cuda-zone) | 12.0.0.61 | python3_ia |
 | [nvidia-cufile](https://developer.nvidia.com/cuda-zone) | 1.15.1.6 | python3_ia |
 | [nvidia-curand](https://developer.nvidia.com/cuda-zone) | 10.4.0.35 | python3_ia |
 | [nvidia-cusolver](https://developer.nvidia.com/cuda-zone) | 12.0.4.66 | python3_ia |
 | [nvidia-cusparse](https://developer.nvidia.com/cuda-zone) | 12.6.3.3 | python3_ia |
 | [nvidia-cusparselt-cu13](https://developer.nvidia.com/cusparselt) | 0.8.1 | python3_ia |
-| [nvidia-nccl-cu13](https://developer.nvidia.com/cuda-zone) | 2.29.7 | python3_ia |
+| [nvidia-nccl-cu13](https://developer.nvidia.com/cuda-zone) | 2.30.7 | python3_ia |
 | [nvidia-nvjitlink](https://developer.nvidia.com/cuda-zone) | 13.0.88 | python3_ia |
 | [nvidia-nvshmem-cu13](https://developer.nvidia.com/cuda-zone) | 3.4.5 | python3_ia |
 | [nvidia-nvtx](https://developer.nvidia.com/cuda-zone) | 13.0.85 | python3_ia |
-| [oauthlib](https://github.com/oauthlib/oauthlib) | 3.2.2 | python3_ia |
+| [oauthlib](https://github.com/oauthlib/oauthlib) | 4.0.0 | python3_ia |
 | [omegaconf](https://github.com/omry/omegaconf) | 2.3.1 | python3_ia |
 | [onnx](https://onnx.ai/) | 1.22.0 | python3_ia |
 | [onnxruntime](https://onnxruntime.ai) | 1.29.0 | python3_ia |
 | [onnxscript](https://microsoft.github.io/onnxscript/) | 0.2.5 | python3_ia |
 | [opt_einsum](https://pypi.org/project/opt_einsum) | 3.4.0 | python3_ia |
-| [optree](https://github.com/metaopt/optree) | 0.18.0 | python3_ia |
-| [pytorch-lightning](https://github.com/Lightning-AI/lightning) | 2.6.5 | python3_ia |
+| [optree](https://github.com/metaopt/optree) | 0.20.0 | python3_ia |
+| [pytorch-lightning](https://github.com/Lightning-AI/lightning) | 2.6.6 | python3_ia |
 | [requests-oauthlib](https://github.com/requests/requests-oauthlib) | 2.0.0 | python3_ia |
 | [rsa](https://stuvel.eu/rsa) | 4.9 | python3_ia |
 | [safetensors](https://github.com/huggingface/safetensors) | 0.8.0 | python3_ia |
@@ -67,17 +67,15 @@
 | [tensorboard-data-server](https://github.com/tensorflow/tensorboard/tree/master/tensorboard/data/server) | 0.7.2 | python3_ia |
 | [tensorboard](https://github.com/tensorflow/tensorboard) | 2.20.0 | python3_ia |
 | [tensorboardX](https://github.com/lanpa/tensorboardX) | 2.6.5 | python3_ia |
-| [tensorflow](https://www.tensorflow.org/) | 2.21.0 | python3_ia |
-| [tf_keras](https://keras.io/) | 2.21.0 | python3_ia |
-| [tiktoken](https://github.com/openai/tiktoken) | 0.12.0 | python3_ia |
+| [tiktoken](https://github.com/openai/tiktoken) | 0.14.0 | python3_ia |
 | [timm](https://github.com/huggingface/pytorch-image-models) | 1.0.22 | python3_ia |
 | [tokenizers](https://github.com/huggingface/tokenizers) | 0.23.1 | python3_ia |
 | [torch-geometric](https://pyg.org) | 2.7.0 | python3_ia |
-| [torch](https://pytorch.org) | 2.13.0 | python3_ia |
+| [torch](https://pytorch.org) | 2.14.0 | python3_ia |
 | [torchmetrics](https://github.com/Lightning-AI/torchmetrics) | 1.9.0 | python3_ia |
-| [torchvision](https://github.com/pytorch/vision) | 0.28.0 | python3_ia |
+| [torchvision](https://github.com/pytorch/vision) | 0.29.0 | python3_ia |
 | [transformers](https://github.com/huggingface/transformers) | 5.16.1 | python3_ia |
-| [triton](https://github.com/triton-lang/triton/) | 3.7.1 | python3_ia |
+| [triton](https://github.com/triton-lang/triton/) | 3.8.0 | python3_ia |
 | [typeshed_client](https://github.com/JelleZijlstra/typeshed_client) | 2.8.2 | python3_ia |
 
-*(79 components)*
+*(77 components)*

CHANGELOG-2.4.md

@@ -0,0 +1,3 @@
+# release_2.4 CHANGELOG
+
+

CHANGELOG.md

@@ -4,61 +4,16 @@
 
 ### New Features
 
-- bump keras from 3.9.2 to 3.10.0 (#204)
-- bump onnx from 1.17.0 to 1.18.0 (#208)
-- bump to torch 2.7.1, triton 3.3.1 and torchvision 0.22.1 (#210)
-- bump transformers from 4.52.1 to 4.52.4 (#213)
-- bump keras from 3.10.0 to 3.11.3 (fix high CVE-2025-8747) (#219)
-- bump transformers from 4.52.4 to 4.53.3 (fix CVE-2025-5197) (#221)
-- bump tensorflow (and dependencies) from 2.19.0 to 2.20.0 (#222)
-- bump tf_keras from 2.20.0 (yanked) to 2.20.1 (#224)
-- bump lightning from 2.5.1.post0 to 2.5.5 (#226)
-- bump torch from 2.7.1 to 2.8.0 (fix moderate CVE-2025-3730) (#227)
-- bump keras from 3.11.3 to 3.12.0 (fix high CVE-2025-12060) (#229)
-- bump safetensors from 0.5.3 to 0.6.2 (#230)
-- bump grpcio from 1.71.0 to 1.76.0 (compat Python 3.14) (#232)
-- bump ml-dtypes from 0.5.1 to 0.5.4 (compat Python 3.14) (#233)
-- bump onnx from 1.18.0 to 1.20.0 (compat Python 3.14) (#234)
-- bump optree from 0.15.0 to 0.18.0 (compat Python 3.14) (#235)
-- bump timm from 1.0.15 to 1.0.22 (compat Python 3.14) (#236)
-- bump torch from 2.8.0 to 2.9.1 (compat Python 3.14) (#237)
-- bump lightning from 2.5.5 to 2.6.0 (#239)
-- bump keras from 3.12.0 to 3.13.1 (fix high CVE-2026-0897) (#241)
-- bump google-auth from 2.40.1 to 2.41.1 (compat cachetools>=6) (#243)
-- bump torch, torchvision, triton to 2.10.0, 0.25.0, 3.6.0 (#245)
-- bump keras from 3.13.1 to 3.13.2 (fix high CVE-2026-0897) (#246)
-- bump lightning from 2.6.0 to 2.6.1 (#249)
-- bump onnx from 1.20.0 to 1.20.1 (#250)
-- add mfai package and dependencies (#251)
-- bump torchmetrics to 1.9.0 and lightning-utilities to 0.15.3 (#256)
-- bump onnxruntime from 1.22.0 to 1.24.4 (compat Python 3.14) (#258)
-- bump onnx from 1.20.1 to 1.21.0 (fix CVE-2026-34445/28500) (#262)
-- bump transformers from 4.53.3 to 5.3.0 (fix moderate CVE-2026-1… (#264)
-- add lightning-fabric (#268)
-- bump diffusers to 0.38.0 (fix high CVE-2026-44513/CVE-2026-45804) (#269)
-- bump lightning from 2.6.1 to 2.6.5 (#272)
-- remove tensorflow-hub (not required anymore) (#273)
-- bump google-auth from 2.41.1 to 2.53.0 (#275)
-- bump tensorflow to 2.21.0 and temporarly remove mfai (#276)
-- bump torch from 2.10.0 to 2.12.0 (#277)
-- bump bitsandbytes from 0.49.1 to 0.49.2 (#278)
-- add mfai 7.3.0 (#279)
-- bump keras from 3.13.2 to 3.14.1 (#281)
-- bump onnx from 1.21.0 to 1.22.0 (fix moderate CVE-2026-44512) (#287)
-- bump transformers from 5.3.0 to 5.5.4 (fix high CVE-2026-5241)
-- bump torch from 2.12.0 to 2.13.0 (fix low CVE-2025-3000) (#293)
-- bump onnxruntime from 1.24.4 to 1.25.1 (fix high CVE-2026-27904) (#295)
-- bump onnxruntime from 1.25.1 to 1.28.0 (#297)
-- bump safetensors from 0.8.0rc0 to 0.8.0 (#298)
-- bump monai from 1.5.2 to 1.6.0 (fix high GHSA-89gg-p5r5-q6r4)
-- bump hydra-core from 1.3.2 to 1.3.4 (#303)
-- bump omegaconf from 2.3.0 to 2.3.1
-- bump keras from 3.14.1 to 3.15.1 (fix 2 high CVE) (#309)
-- bump sqlparse from 0.5.5 to 0.6.0 (fix 3 high CVE) (#314)
-- add mfai 8.0.0 (release 7.3.0 had been temporarly removed) (#316)
-- bump hydra-core from 1.3.4 to 1.3.5 (#317)
-- bump ml_dtypes from 0.5.4 to 0.6.0 (#318)
-- bump transformers from 5.5.4 to 5.16.1 (fix high CVE-2026-9856) (#320)
-- bump onnxruntime from 1.28.0 to 1.29.0 (#322)
+- various updates for Python 3.14 compatibility
+- add onnxruntime 1.24.4 (now Python 3.14 compliant)
+- use torch as backend for keras (tensorflow not available)
+- reintroduce mfai 8.0.2
+- bump lightning from 2.6.5 to 2.6.6 (fix high CVE-2026-58659)
+- bump torch from 2.13.0 to 2.14.0 and triton from 3.7.1 to 3.8.0 (#327)
+- bump hydra-core from 1.3.5 to 1.3.7 (#328)
+- bump grpcio from 1.76.0 to 1.84.0 (compat. Python 3.15)
+- bump grpcio from 1.76.0 to 1.84.0 (compat. Python 3.15) (#330)
+- bump tiktoken from 0.12.0 to 0.14.0 (compat. Python 3.15)
+- bump oauthlib from 3.2.2 to 4.0.0 (fix 2 moderate CVE) (#333)
 
 

bootstrap.sh

@@ -83,7 +83,7 @@ fi
 
 # FIXME: do not hardcode this
 # FIXME: move to layer root extra_env ?
-echo "export PYTHON3_SHORT_VERSION := 3.13" >>adm/root.mk
+echo "export PYTHON3_SHORT_VERSION := 3.14" >>adm/root.mk
 
 echo "BOOTSTRAP DONE !"
 echo "MFEXT_HOME=${MFEXT_HOME}"

integration_tests/0003_test_import_python3_ia/import_python3_ia.py

@@ -23,6 +23,7 @@
 import clang #libclang
 import lightning_fabric
 import lightning_utilities
+import lightning_fabric
 import lightning
 import marshmallow
 import mfai
@@ -53,8 +54,6 @@
 import tensorboard_data_server
 import tensorboard
 import tensorboardX
-import tensorflow
-import tf_keras
 import tiktoken
 import timm
 import tokenizers

layers/layer5_python3_ia/.layerapi2_extra_env

@@ -0,0 +1 @@
+KERAS_BACKEND=torch

layers/layer5_python3_ia/0500_extra_packages/allow_binary_packages

@@ -42,7 +42,6 @@ opt-einsum
 optree
 protobuf
 pyarrow
-PyYAML
 requests-oauthlib
 rich
 rsa

layers/layer5_python3_ia/0500_extra_packages/requirements3.txt

@@ -14,20 +14,20 @@ flatbuffers==25.12.19
 google-auth==2.53.0
 google-auth-oauthlib==1.2.2
 google-pasta==0.2.0
-grpcio==1.76.0
+grpcio==1.84.0
 hf_xet==1.4.3
 huggingface_hub==1.5.0
-hydra-core==1.3.5
+hydra-core==1.3.7
 jsonargparse==4.46.0
 jsonnet==0.21.0
 keras==3.15.1
 libclang==18.1.1
-lightning==2.6.5
-lightning-fabric==2.6.5
+lightning==2.6.6
+lightning-fabric==2.6.6
 lightning-utilities==0.15.3
 lit==18.1.8
 marshmallow==3.26.2
-mfai==8.0.0
+mfai==8.0.2
 ml-dtypes==0.6.0
 monai==1.6.0
 mpmath==1.3.0
@@ -36,25 +36,25 @@ nvidia_cublas==13.1.1.3
 nvidia-cuda-cupti==13.0.85
 nvidia-cuda-nvrtc==13.0.88
 nvidia-cuda-runtime==13.0.96
-nvidia-cudnn-cu13==9.20.0.48
+nvidia-cudnn-cu13==9.24.0.43
 nvidia-cufft==12.0.0.61
 nvidia-cufile==1.15.1.6
 nvidia-curand==10.4.0.35
 nvidia-cusolver==12.0.4.66
 nvidia_cusparse==12.6.3.3
 nvidia-cusparselt-cu13==0.8.1
 nvidia_nvjitlink==13.0.88
-nvidia-nccl-cu13==2.29.7
+nvidia-nccl-cu13==2.30.7
 nvidia-nvshmem-cu13==3.4.5
 nvidia-nvtx==13.0.85
-oauthlib==3.2.2
+oauthlib==4.0.0
 omegaconf==2.3.1
 onnx==1.22.0
 onnxruntime==1.29.0
 onnxscript==0.2.5
 opt-einsum==3.4.0
-optree==0.18.0
-pytorch-lightning==2.6.5
+optree==0.20.0
+pytorch-lightning==2.6.6
 requests-oauthlib==2.0.0
 rsa==4.9
 safetensors==0.8.0
@@ -65,15 +65,13 @@ tabulate==0.9.0
 tensorboard==2.20.0
 tensorboard-data-server==0.7.2
 tensorboardx==2.6.5
-tensorflow==2.21.0
-tf-keras==2.21.0
-tiktoken==0.12.0
+tiktoken==0.14.0
 timm==1.0.22
 tokenizers==0.23.1
-torch==2.13.0
+torch==2.14.0
 torchmetrics==1.9.0
-torchvision==0.28.0
+torchvision==0.29.0
 torch-geometric==2.7.0
 transformers==5.16.1
-triton==3.7.1
+triton==3.8.0
 typeshed-client==2.8.2