From 4c4ef6159ec7b0398a3b84ebbb8a4280c60a0f09 Mon Sep 17 00:00:00 2001 From: thebaptiste Date: Tue, 2 Dec 2025 16:47:22 +0000 Subject: [PATCH 01/38] feat: various updates for Python 3.14 compatibility --- .../0500_extra_packages/requirements3.txt | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt index d125a34..fa56078 100644 --- a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt +++ b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt @@ -7,14 +7,14 @@ flatbuffers==25.2.10 google-auth==2.40.1 google-auth-oauthlib==1.2.2 google-pasta==0.2.0 -grpcio==1.71.0 +grpcio==1.76.0 huggingface_hub==0.31.4 keras==3.12.0 libclang==18.1.1 lightning==2.5.5 lightning-utilities==0.14.3 lit==18.1.8 -ml-dtypes==0.5.1 +ml-dtypes==0.5.4 mpmath==1.3.0 namex==0.0.9 nvidia-cublas-cu12==12.8.4.1 @@ -32,11 +32,10 @@ nvidia-nccl-cu12==2.27.3 nvidia-nvjitlink-cu12==12.8.93 nvidia-nvtx-cu12==12.8.90 oauthlib==3.2.2 -onnx==1.18.0 -onnxruntime==1.22.0 +onnx==1.20.0 onnxscript==0.2.5 opt-einsum==3.4.0 -optree==0.15.0 +optree==0.18.0 pytorch-lightning==2.5.1.post0 requests-oauthlib==2.0.0 rsa==4.9 @@ -45,10 +44,9 @@ segmentation_models_pytorch==0.5.0 sympy==1.14.0 tensorboard==2.20.0 tensorboard-data-server==0.7.2 -tensorflow==2.20.0 tensorflow-hub==0.16.1 tf-keras==2.20.1 -timm==1.0.15 +timm==1.0.22 tokenizers==0.21.1 torch==2.8.0 torchmetrics==1.7.1 From 9669ea0ed90efcbad93f04d6f1332712cc91be58 Mon Sep 17 00:00:00 2001 From: thebaptiste Date: Wed, 3 Dec 2025 09:50:40 +0000 Subject: [PATCH 02/38] build: python 3.14 --- bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap.sh b/bootstrap.sh index 8c83c92..6b52044 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -83,7 +83,7 @@ fi # FIXME: do not hardcode this # FIXME: move to layer root extra_env ? -echo "export PYTHON3_SHORT_VERSION := 3.13" >>adm/root.mk +echo "export PYTHON3_SHORT_VERSION := 3.14" >>adm/root.mk echo "BOOTSTRAP DONE !" echo "MFEXT_HOME=${MFEXT_HOME}" From e741906ab0952704aa19829a781ba82d435f175d Mon Sep 17 00:00:00 2001 From: thebaptiste Date: Thu, 4 Dec 2025 13:56:23 +0000 Subject: [PATCH 03/38] build: various fixes for Python 3.14 --- .metwork-framework/components.md | 25 ++++++++----------- .../import_python3_ia.py | 8 +++--- .../0500_extra_packages/allow_binary_packages | 1 + .../0500_extra_packages/requirements3.txt | 11 ++++---- 4 files changed, 21 insertions(+), 24 deletions(-) diff --git a/.metwork-framework/components.md b/.metwork-framework/components.md index bde4e18..9cf60d6 100644 --- a/.metwork-framework/components.md +++ b/.metwork-framework/components.md @@ -9,14 +9,14 @@ | [google-auth-oauthlib](https://github.com/GoogleCloudPlatform/google-auth-library-python-oauthlib) | 1.2.2 | python3_ia | | [google-auth](https://github.com/googleapis/google-auth-library-python) | 2.40.1 | python3_ia | | [google-pasta](https://github.com/google/pasta) | 0.2.0 | python3_ia | -| [grpcio](https://grpc.io) | 1.71.0 | python3_ia | +| [grpcio](https://grpc.io) | 1.76.0 | python3_ia | | [huggingface_hub](https://github.com/huggingface/huggingface_hub) | 0.31.4 | python3_ia | | [keras](https://pypi.org/project/keras) | 3.12.0 | python3_ia | | [libclang](https://github.com/sighingnow/libclang) | 18.1.1 | python3_ia | | [lightning-utilities](https://github.com/Lightning-AI/utilities) | 0.14.3 | python3_ia | | [lightning](https://github.com/Lightning-AI/lightning) | 2.5.5 | python3_ia | | [lit](http://llvm.org) | 18.1.8 | python3_ia | -| [ml_dtypes](https://github.com/jax-ml/ml_dtypes) | 0.5.1 | python3_ia | +| [ml_dtypes](https://github.com/jax-ml/ml_dtypes) | 0.5.4 | python3_ia | | [mpmath](http://mpmath.org/) | 1.3.0 | python3_ia | | [namex](https://pypi.org/project/namex) | 0.0.9 | python3_ia | | [nvidia-cublas-cu12](https://developer.nvidia.com/cuda-zone) | 12.8.4.1 | python3_ia | @@ -30,15 +30,15 @@ | [nvidia-cusolver-cu12](https://developer.nvidia.com/cuda-zone) | 11.7.3.90 | python3_ia | | [nvidia-cusparse-cu12](https://developer.nvidia.com/cuda-zone) | 12.5.8.93 | python3_ia | | [nvidia-cusparselt-cu12](https://developer.nvidia.com/cusparselt) | 0.7.1 | python3_ia | -| [nvidia-nccl-cu12](https://developer.nvidia.com/cuda-zone) | 2.27.3 | python3_ia | +| [nvidia-nccl-cu12](https://developer.nvidia.com/cuda-zone) | 2.27.5 | python3_ia | | [nvidia-nvjitlink-cu12](https://developer.nvidia.com/cuda-zone) | 12.8.93 | python3_ia | +| [nvidia-nvshmem-cu12](https://developer.nvidia.com/cuda-zone) | 3.3.20 | python3_ia | | [nvidia-nvtx-cu12](https://developer.nvidia.com/cuda-zone) | 12.8.90 | python3_ia | | [oauthlib](https://github.com/oauthlib/oauthlib) | 3.2.2 | python3_ia | -| [onnx](https://onnx.ai/) | 1.18.0 | python3_ia | -| [onnxruntime](https://onnxruntime.ai) | 1.22.0 | python3_ia | +| [onnx](https://onnx.ai/) | 1.20.0 | python3_ia | | [onnxscript](https://microsoft.github.io/onnxscript/) | 0.2.5 | python3_ia | | [opt_einsum](https://pypi.org/project/opt_einsum) | 3.4.0 | python3_ia | -| [optree](https://github.com/metaopt/optree) | 0.15.0 | python3_ia | +| [optree](https://github.com/metaopt/optree) | 0.18.0 | python3_ia | | [pytorch-lightning](https://github.com/Lightning-AI/lightning) | 2.5.1.post0 | python3_ia | | [requests-oauthlib](https://github.com/requests/requests-oauthlib) | 2.0.0 | python3_ia | | [rsa](https://stuvel.eu/rsa) | 4.9 | python3_ia | @@ -47,15 +47,12 @@ | [sympy](https://sympy.org) | 1.14.0 | python3_ia | | [tensorboard-data-server](https://github.com/tensorflow/tensorboard/tree/master/tensorboard/data/server) | 0.7.2 | python3_ia | | [tensorboard](https://github.com/tensorflow/tensorboard) | 2.20.0 | python3_ia | -| [tensorflow-hub](https://github.com/tensorflow/hub) | 0.16.1 | python3_ia | -| [tensorflow](https://www.tensorflow.org/) | 2.20.0 | python3_ia | -| [tf_keras](https://keras.io/) | 2.20.1 | python3_ia | -| [timm](https://github.com/huggingface/pytorch-image-models) | 1.0.15 | python3_ia | +| [timm](https://github.com/huggingface/pytorch-image-models) | 1.0.22 | python3_ia | | [tokenizers](https://github.com/huggingface/tokenizers) | 0.21.1 | python3_ia | -| [torch](https://pytorch.org/) | 2.8.0 | python3_ia | +| [torch](https://pytorch.org) | 2.9.1 | python3_ia | | [torchmetrics](https://github.com/Lightning-AI/torchmetrics) | 1.7.1 | python3_ia | -| [torchvision](https://github.com/pytorch/vision) | 0.23.0 | python3_ia | +| [torchvision](https://github.com/pytorch/vision) | 0.24.1 | python3_ia | | [transformers](https://github.com/huggingface/transformers) | 4.53.3 | python3_ia | -| [triton](https://github.com/triton-lang/triton/) | 3.4.0 | python3_ia | +| [triton](https://github.com/triton-lang/triton/) | 3.5.1 | python3_ia | -*(57 components)* +*(54 components)* diff --git a/integration_tests/0003_test_import_python3_ia/import_python3_ia.py b/integration_tests/0003_test_import_python3_ia/import_python3_ia.py index f7ed77a..61ced67 100644 --- a/integration_tests/0003_test_import_python3_ia/import_python3_ia.py +++ b/integration_tests/0003_test_import_python3_ia/import_python3_ia.py @@ -31,7 +31,7 @@ from nvidia import nvtx import oauthlib import onnx -import onnxruntime +#import onnxruntime not Python 3.14 compliant import onnxscript import opt_einsum import optree @@ -43,9 +43,9 @@ import sympy import tensorboard_data_server import tensorboard -import tensorflow -import tensorflow_hub -import tf_keras +#import tensorflow not Python 3.14 compliant +#import tensorflow_hub not Python 3.14 compliant +#import tf_keras not Python 3.14 compliant import timm import tokenizers import torch diff --git a/layers/layer5_python3_ia/0500_extra_packages/allow_binary_packages b/layers/layer5_python3_ia/0500_extra_packages/allow_binary_packages index 9abbd88..a8c38c0 100644 --- a/layers/layer5_python3_ia/0500_extra_packages/allow_binary_packages +++ b/layers/layer5_python3_ia/0500_extra_packages/allow_binary_packages @@ -25,6 +25,7 @@ nvidia-cusparse-cu12 nvidia-cusparselt-cu12 nvidia-nccl-cu12 nvidia-nvjitlink-cu12 +nvidia-nvshmem-cu12 nvidia-nvtx-cu12 oauthlib onnx diff --git a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt index fa56078..17dae91 100644 --- a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt +++ b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt @@ -28,8 +28,9 @@ nvidia-curand-cu12==10.3.9.90 nvidia-cusolver-cu12==11.7.3.90 nvidia-cusparse-cu12==12.5.8.93 nvidia-cusparselt-cu12==0.7.1 -nvidia-nccl-cu12==2.27.3 +nvidia-nccl-cu12==2.27.5 nvidia-nvjitlink-cu12==12.8.93 +nvidia-nvshmem-cu12==3.3.20 nvidia-nvtx-cu12==12.8.90 oauthlib==3.2.2 onnx==1.20.0 @@ -44,12 +45,10 @@ segmentation_models_pytorch==0.5.0 sympy==1.14.0 tensorboard==2.20.0 tensorboard-data-server==0.7.2 -tensorflow-hub==0.16.1 -tf-keras==2.20.1 timm==1.0.22 tokenizers==0.21.1 -torch==2.8.0 +torch==2.9.1 torchmetrics==1.7.1 -torchvision==0.23.0 +torchvision==0.24.1 transformers==4.53.3 -triton==3.4.0 +triton==3.5.1 From c1922498c3ce9abc7cbca0210f9b27e133414796 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste VESLIN <19856429+thebaptiste@users.noreply.github.com> Date: Thu, 4 Dec 2025 16:10:42 +0100 Subject: [PATCH 04/38] build: add test --- .../0003_test_import_python3_ia/import_python3_ia.py | 1 + 1 file changed, 1 insertion(+) diff --git a/integration_tests/0003_test_import_python3_ia/import_python3_ia.py b/integration_tests/0003_test_import_python3_ia/import_python3_ia.py index 61ced67..2672160 100644 --- a/integration_tests/0003_test_import_python3_ia/import_python3_ia.py +++ b/integration_tests/0003_test_import_python3_ia/import_python3_ia.py @@ -28,6 +28,7 @@ from nvidia import cusparse from nvidia import nccl from nvidia import nvjitlink +from nvidia import nvshmem from nvidia import nvtx import oauthlib import onnx From 1639b86ba33884266b573f240121abf9e8b7ad3b Mon Sep 17 00:00:00 2001 From: thebaptiste Date: Thu, 4 Dec 2025 15:21:47 +0000 Subject: [PATCH 05/38] build: remove keras which has tensorflow as dependency --- .metwork-framework/components.md | 3 +-- .../0003_test_import_python3_ia/import_python3_ia.py | 2 +- layers/layer5_python3_ia/0500_extra_packages/requirements3.txt | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.metwork-framework/components.md b/.metwork-framework/components.md index 9cf60d6..73f1ab5 100644 --- a/.metwork-framework/components.md +++ b/.metwork-framework/components.md @@ -11,7 +11,6 @@ | [google-pasta](https://github.com/google/pasta) | 0.2.0 | python3_ia | | [grpcio](https://grpc.io) | 1.76.0 | python3_ia | | [huggingface_hub](https://github.com/huggingface/huggingface_hub) | 0.31.4 | python3_ia | -| [keras](https://pypi.org/project/keras) | 3.12.0 | python3_ia | | [libclang](https://github.com/sighingnow/libclang) | 18.1.1 | python3_ia | | [lightning-utilities](https://github.com/Lightning-AI/utilities) | 0.14.3 | python3_ia | | [lightning](https://github.com/Lightning-AI/lightning) | 2.5.5 | python3_ia | @@ -55,4 +54,4 @@ | [transformers](https://github.com/huggingface/transformers) | 4.53.3 | python3_ia | | [triton](https://github.com/triton-lang/triton/) | 3.5.1 | python3_ia | -*(54 components)* +*(53 components)* diff --git a/integration_tests/0003_test_import_python3_ia/import_python3_ia.py b/integration_tests/0003_test_import_python3_ia/import_python3_ia.py index 2672160..7927aee 100644 --- a/integration_tests/0003_test_import_python3_ia/import_python3_ia.py +++ b/integration_tests/0003_test_import_python3_ia/import_python3_ia.py @@ -12,7 +12,7 @@ import pasta #google_pasta import grpc #grpcio import clang #libclang -import keras +#import keras not Python 3.14 compliant import ml_dtypes import mpmath import namex diff --git a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt index 17dae91..7615203 100644 --- a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt +++ b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt @@ -9,7 +9,6 @@ google-auth-oauthlib==1.2.2 google-pasta==0.2.0 grpcio==1.76.0 huggingface_hub==0.31.4 -keras==3.12.0 libclang==18.1.1 lightning==2.5.5 lightning-utilities==0.14.3 From 77e8245ea560133f7e02b079522d4848d5fe4f37 Mon Sep 17 00:00:00 2001 From: thebaptiste Date: Fri, 5 Dec 2025 14:42:38 +0000 Subject: [PATCH 06/38] build: add authorized dependencies --- integration_tests/0096_system_extra_dependencies/list.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/integration_tests/0096_system_extra_dependencies/list.txt b/integration_tests/0096_system_extra_dependencies/list.txt index 848c03f..44e3473 100644 --- a/integration_tests/0096_system_extra_dependencies/list.txt +++ b/integration_tests/0096_system_extra_dependencies/list.txt @@ -1,5 +1,6 @@ ld-linux-x86-64.so.2 libacl.so.1 +libfabric.so.1 libasound.so.2 libattr.so.1 libblkid.so.1 @@ -105,6 +106,7 @@ libmagic.so.1 libmlx5.so.1 libmount.so.1 libmpi.so.12 +libmpi.so.40 libncurses.so.6 libncursesw.so.6 libnettle.so.6 @@ -118,6 +120,7 @@ libnssutil3.so libonig.so.5 libopenblas.so.0 libopenjp2.so.7 +liboshmem.so.40 libp11-kit.so.0 libpanelw.so.6 libpango-1.0.so.0 @@ -130,6 +133,7 @@ libpcre2-16.so.0 libpixman-1.so.0 libplc4.so libplds4.so +libpmix.so.2 libpng16.so.16 libpopt.so.0 libpsl.so.5 @@ -160,6 +164,8 @@ libthread_db.so.1 libtiff.so.5 libtinfo.so.6 libtk8.6.so +libucp.so.0 +libucs.so.0 libutil.so.1 libuuid.so.1 libunistring.so.2 From 61ddd06a78f93a6f66ee46ea7d8ee09ca1d8ea22 Mon Sep 17 00:00:00 2001 From: thebaptiste Date: Fri, 5 Dec 2025 15:57:44 +0000 Subject: [PATCH 07/38] build: add system dependencies --- layers/layer5_python3_ia/.system_dependencies | 2 ++ 1 file changed, 2 insertions(+) diff --git a/layers/layer5_python3_ia/.system_dependencies b/layers/layer5_python3_ia/.system_dependencies index 5a2b24c..2aa2ffd 100644 --- a/layers/layer5_python3_ia/.system_dependencies +++ b/layers/layer5_python3_ia/.system_dependencies @@ -1 +1,3 @@ librdmacm.so.1()(64bit)@generic +libfabric.so.1()(64bit)@generic +libmpi.so.40()(64bit)@generic From 28902fe3a9bc3c5ed859426db1fe078d3e820013 Mon Sep 17 00:00:00 2001 From: thebaptiste Date: Fri, 5 Dec 2025 18:13:43 +0000 Subject: [PATCH 08/38] build: fix --- layers/layer5_python3_ia/.system_dependencies | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/layers/layer5_python3_ia/.system_dependencies b/layers/layer5_python3_ia/.system_dependencies index 2aa2ffd..b781afd 100644 --- a/layers/layer5_python3_ia/.system_dependencies +++ b/layers/layer5_python3_ia/.system_dependencies @@ -1,3 +1,3 @@ librdmacm.so.1()(64bit)@generic libfabric.so.1()(64bit)@generic -libmpi.so.40()(64bit)@generic +openmpi@generic From bd61229f847e5f6dfd64ae5cf58f9c3e3ff1c3d3 Mon Sep 17 00:00:00 2001 From: thebaptiste Date: Tue, 9 Dec 2025 16:39:43 +0000 Subject: [PATCH 09/38] build: add authorized dependencies --- integration_tests/0096_system_extra_dependencies/list.txt | 7 +++++++ layers/layer5_python3_ia/.layerapi2_extra_env | 1 + 2 files changed, 8 insertions(+) create mode 100644 layers/layer5_python3_ia/.layerapi2_extra_env diff --git a/integration_tests/0096_system_extra_dependencies/list.txt b/integration_tests/0096_system_extra_dependencies/list.txt index 44e3473..1e31280 100644 --- a/integration_tests/0096_system_extra_dependencies/list.txt +++ b/integration_tests/0096_system_extra_dependencies/list.txt @@ -25,8 +25,11 @@ libdav1d.so.7 libdl.so.2 libdouble-conversion.so.3 libdw.so.1 +libefa.so.1 libelf.so.1 libev.so.4 +libevent-2.1.so.6 +libevent_pthreads-2.1.so.6 libexpat.so.1 libexslt.so.0 libffi.so.6 @@ -117,6 +120,7 @@ libnsl.so.1 libnspr4.so libnss3.so libnssutil3.so +libnuma.so.1 libonig.so.5 libopenblas.so.0 libopenjp2.so.7 @@ -137,6 +141,7 @@ libpmix.so.2 libpng16.so.16 libpopt.so.0 libpsl.so.5 +libpsm2.so.2 libpthread.so.0 libQt5Core.so.5 libQt5Gui.so.5 @@ -164,8 +169,10 @@ libthread_db.so.1 libtiff.so.5 libtinfo.so.6 libtk8.6.so +libucm.so.0 libucp.so.0 libucs.so.0 +libuct.so.0 libutil.so.1 libuuid.so.1 libunistring.so.2 diff --git a/layers/layer5_python3_ia/.layerapi2_extra_env b/layers/layer5_python3_ia/.layerapi2_extra_env new file mode 100644 index 0000000..8830bd8 --- /dev/null +++ b/layers/layer5_python3_ia/.layerapi2_extra_env @@ -0,0 +1 @@ +LD_LIBRARY_PATH={LD_LIBRARY_PATH}:/usr/lib64/openmpi/lib From 5f2473eb7d5e99570bf4e2f04f748518dd1d4d8e Mon Sep 17 00:00:00 2001 From: thebaptiste Date: Fri, 12 Dec 2025 10:30:22 +0000 Subject: [PATCH 10/38] build: add authorized dependencies --- integration_tests/0096_system_extra_dependencies/list.txt | 4 ++++ integration_tests/revert_ldd_not_found.sh | 1 + layers/layer5_python3_ia/.layerapi2_extra_env | 1 - 3 files changed, 5 insertions(+), 1 deletion(-) delete mode 100644 layers/layer5_python3_ia/.layerapi2_extra_env diff --git a/integration_tests/0096_system_extra_dependencies/list.txt b/integration_tests/0096_system_extra_dependencies/list.txt index 1e31280..3e0770a 100644 --- a/integration_tests/0096_system_extra_dependencies/list.txt +++ b/integration_tests/0096_system_extra_dependencies/list.txt @@ -29,6 +29,7 @@ libefa.so.1 libelf.so.1 libev.so.4 libevent-2.1.so.6 +libevent_core-2.1.so.6 libevent_pthreads-2.1.so.6 libexpat.so.1 libexslt.so.0 @@ -122,6 +123,9 @@ libnss3.so libnssutil3.so libnuma.so.1 libonig.so.5 +libopen-orted-mpir.so +libopen-pal.so.40 +libopen-rte.so.40 libopenblas.so.0 libopenjp2.so.7 liboshmem.so.40 diff --git a/integration_tests/revert_ldd_not_found.sh b/integration_tests/revert_ldd_not_found.sh index 6ae7c34..7be3ac9 100755 --- a/integration_tests/revert_ldd_not_found.sh +++ b/integration_tests/revert_ldd_not_found.sh @@ -15,6 +15,7 @@ if test "$1" = "--help"; then exit 0 fi +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib64/openmpi/lib for F in $(find . -type f -name "*.so*" ; find . -type f -wholename "*/bin/*"); do N=$(ldd "${F}" 2>/dev/null |grep "=>" |awk -F '=>' '{print $1;}' |grep -c "$1") if test "$N" -gt 0; then diff --git a/layers/layer5_python3_ia/.layerapi2_extra_env b/layers/layer5_python3_ia/.layerapi2_extra_env deleted file mode 100644 index 8830bd8..0000000 --- a/layers/layer5_python3_ia/.layerapi2_extra_env +++ /dev/null @@ -1 +0,0 @@ -LD_LIBRARY_PATH={LD_LIBRARY_PATH}:/usr/lib64/openmpi/lib From 07171effa3e81b376e9ac04e1a047ff339548560 Mon Sep 17 00:00:00 2001 From: thebaptiste Date: Fri, 12 Dec 2025 12:33:29 +0000 Subject: [PATCH 11/38] build: fix --- integration_tests/0096_system_extra_dependencies/test.sh | 1 + integration_tests/revert_ldd_not_found.sh | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/0096_system_extra_dependencies/test.sh b/integration_tests/0096_system_extra_dependencies/test.sh index 1fc36d3..5e7821e 100755 --- a/integration_tests/0096_system_extra_dependencies/test.sh +++ b/integration_tests/0096_system_extra_dependencies/test.sh @@ -12,6 +12,7 @@ OK_NOT_FOUND=$(cat list_ok_not_found.txt|grep -v "#"|xargs) export PATH=${PATH}:${PWD}/.. RET=0 +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib64/openmpi/lib cd "${MFEXT_HOME}" || exit 1 diff --git a/integration_tests/revert_ldd_not_found.sh b/integration_tests/revert_ldd_not_found.sh index 7be3ac9..6ae7c34 100755 --- a/integration_tests/revert_ldd_not_found.sh +++ b/integration_tests/revert_ldd_not_found.sh @@ -15,7 +15,6 @@ if test "$1" = "--help"; then exit 0 fi -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib64/openmpi/lib for F in $(find . -type f -name "*.so*" ; find . -type f -wholename "*/bin/*"); do N=$(ldd "${F}" 2>/dev/null |grep "=>" |awk -F '=>' '{print $1;}' |grep -c "$1") if test "$N" -gt 0; then From 8be9cb337fa5814fcfaf5b5c4cc33c8f7d41c45a Mon Sep 17 00:00:00 2001 From: thebaptiste Date: Fri, 27 Mar 2026 16:56:18 +0000 Subject: [PATCH 12/38] feat: add onnxruntime 1.24.4 (now Python 3.14 compliant) --- .metwork-framework/components.md | 3 ++- layers/layer5_python3_ia/0500_extra_packages/requirements3.txt | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.metwork-framework/components.md b/.metwork-framework/components.md index 73f1ab5..57f6d47 100644 --- a/.metwork-framework/components.md +++ b/.metwork-framework/components.md @@ -35,6 +35,7 @@ | [nvidia-nvtx-cu12](https://developer.nvidia.com/cuda-zone) | 12.8.90 | python3_ia | | [oauthlib](https://github.com/oauthlib/oauthlib) | 3.2.2 | python3_ia | | [onnx](https://onnx.ai/) | 1.20.0 | python3_ia | +| [onnxruntime](https://onnxruntime.ai) | 1.24.4 | python3_ia | | [onnxscript](https://microsoft.github.io/onnxscript/) | 0.2.5 | python3_ia | | [opt_einsum](https://pypi.org/project/opt_einsum) | 3.4.0 | python3_ia | | [optree](https://github.com/metaopt/optree) | 0.18.0 | python3_ia | @@ -54,4 +55,4 @@ | [transformers](https://github.com/huggingface/transformers) | 4.53.3 | python3_ia | | [triton](https://github.com/triton-lang/triton/) | 3.5.1 | python3_ia | -*(53 components)* +*(54 components)* diff --git a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt index 7615203..8d67682 100644 --- a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt +++ b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt @@ -33,6 +33,7 @@ nvidia-nvshmem-cu12==3.3.20 nvidia-nvtx-cu12==12.8.90 oauthlib==3.2.2 onnx==1.20.0 +onnxruntime==1.24.4 onnxscript==0.2.5 opt-einsum==3.4.0 optree==0.18.0 From d374a56aa0857e6b98341cdce267d12ec0e95798 Mon Sep 17 00:00:00 2001 From: thebaptiste Date: Thu, 30 Apr 2026 22:32:54 +0000 Subject: [PATCH 13/38] build: remove lightning and libclang --- .metwork-framework/components.md | 4 +--- .../0003_test_import_python3_ia/import_python3_ia.py | 6 +++--- .../layer5_python3_ia/0500_extra_packages/requirements3.txt | 2 -- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.metwork-framework/components.md b/.metwork-framework/components.md index 12f4dc4..ddef436 100644 --- a/.metwork-framework/components.md +++ b/.metwork-framework/components.md @@ -22,9 +22,7 @@ | [hydra-core](https://github.com/facebookresearch/hydra) | 1.3.2 | python3_ia | | [jsonargparse](https://pypi.org/project/jsonargparse) | 4.46.0 | python3_ia | | [jsonnet](https://jsonnet.org) | 0.21.0 | python3_ia | -| [libclang](https://github.com/sighingnow/libclang) | 18.1.1 | python3_ia | | [lightning-utilities](https://github.com/Lightning-AI/utilities) | 0.15.3 | python3_ia | -| [lightning](https://github.com/Lightning-AI/lightning) | 2.6.1 | python3_ia | | [lit](http://llvm.org) | 18.1.8 | python3_ia | | [marshmallow](https://pypi.org/project/marshmallow) | 3.26.2 | python3_ia | | [ml_dtypes](https://github.com/jax-ml/ml_dtypes) | 0.5.4 | python3_ia | @@ -74,4 +72,4 @@ | [triton](https://github.com/triton-lang/triton/) | 3.6.0 | python3_ia | | [typeshed_client](https://github.com/JelleZijlstra/typeshed_client) | 2.8.2 | python3_ia | -*(73 components)* +*(71 components)* diff --git a/integration_tests/0003_test_import_python3_ia/import_python3_ia.py b/integration_tests/0003_test_import_python3_ia/import_python3_ia.py index f9a1f90..43625bc 100644 --- a/integration_tests/0003_test_import_python3_ia/import_python3_ia.py +++ b/integration_tests/0003_test_import_python3_ia/import_python3_ia.py @@ -19,13 +19,13 @@ import jsonargparse import _jsonnet #jsonnet #import keras not Python 3.14 compliant -import clang #libclang +#import clang #libclang import lightning_utilities -import lightning +#import lightning import marshmallow #import mfai import ml_dtypes -import monai +#import monai import mpmath import namex from nvidia import cublas diff --git a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt index 484d8b8..40dee7a 100644 --- a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt +++ b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt @@ -20,8 +20,6 @@ huggingface_hub==1.5.0 hydra-core==1.3.2 jsonargparse==4.46.0 jsonnet==0.21.0 -libclang==18.1.1 -lightning==2.6.1 lightning-utilities==0.15.3 lit==18.1.8 marshmallow==3.26.2 From 77d6a571c7251b3e9d63e57bb82a93ad933ead08 Mon Sep 17 00:00:00 2001 From: thebaptiste Date: Fri, 1 May 2026 07:07:35 +0000 Subject: [PATCH 14/38] build: add lightning-fabric and lightning --- .metwork-framework/components.md | 5 ++++- .../0003_test_import_python3_ia/import_python3_ia.py | 5 +++-- .../0500_extra_packages/allow_binary_packages | 4 ++-- .../layer5_python3_ia/0500_extra_packages/requirements3.txt | 3 +++ 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.metwork-framework/components.md b/.metwork-framework/components.md index ddef436..e56022c 100644 --- a/.metwork-framework/components.md +++ b/.metwork-framework/components.md @@ -22,7 +22,10 @@ | [hydra-core](https://github.com/facebookresearch/hydra) | 1.3.2 | python3_ia | | [jsonargparse](https://pypi.org/project/jsonargparse) | 4.46.0 | python3_ia | | [jsonnet](https://jsonnet.org) | 0.21.0 | python3_ia | +| [libclang](https://github.com/sighingnow/libclang) | 18.1.1 | python3_ia | +| [lightning-fabric](https://github.com/Lightning-AI/lightning) | 2.6.1 | python3_ia | | [lightning-utilities](https://github.com/Lightning-AI/utilities) | 0.15.3 | python3_ia | +| [lightning](https://github.com/Lightning-AI/lightning) | 2.6.1 | python3_ia | | [lit](http://llvm.org) | 18.1.8 | python3_ia | | [marshmallow](https://pypi.org/project/marshmallow) | 3.26.2 | python3_ia | | [ml_dtypes](https://github.com/jax-ml/ml_dtypes) | 0.5.4 | python3_ia | @@ -72,4 +75,4 @@ | [triton](https://github.com/triton-lang/triton/) | 3.6.0 | python3_ia | | [typeshed_client](https://github.com/JelleZijlstra/typeshed_client) | 2.8.2 | python3_ia | -*(71 components)* +*(74 components)* diff --git a/integration_tests/0003_test_import_python3_ia/import_python3_ia.py b/integration_tests/0003_test_import_python3_ia/import_python3_ia.py index 43625bc..b3a0fca 100644 --- a/integration_tests/0003_test_import_python3_ia/import_python3_ia.py +++ b/integration_tests/0003_test_import_python3_ia/import_python3_ia.py @@ -19,9 +19,10 @@ import jsonargparse import _jsonnet #jsonnet #import keras not Python 3.14 compliant -#import clang #libclang +import clang #libclang import lightning_utilities -#import lightning +import lightning_fabric +import lightning import marshmallow #import mfai import ml_dtypes diff --git a/layers/layer5_python3_ia/0500_extra_packages/allow_binary_packages b/layers/layer5_python3_ia/0500_extra_packages/allow_binary_packages index b1531dc..e64b300 100644 --- a/layers/layer5_python3_ia/0500_extra_packages/allow_binary_packages +++ b/layers/layer5_python3_ia/0500_extra_packages/allow_binary_packages @@ -17,6 +17,7 @@ Keras keras-nightly Keras-Preprocessing libclang +lightning-fabric nvidia-cublas-cu12 nvidia-cuda-cupti-cu12 nvidia-cuda-nvcc-cu12 @@ -41,6 +42,7 @@ onnxscript opt-einsum optree protobuf +pytorch-lightning PyYAML requests-oauthlib rich @@ -58,5 +60,3 @@ torch tensorflow-io torchvision triton -pytorch-lightning -meson diff --git a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt index 40dee7a..1575795 100644 --- a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt +++ b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt @@ -20,6 +20,9 @@ huggingface_hub==1.5.0 hydra-core==1.3.2 jsonargparse==4.46.0 jsonnet==0.21.0 +libclang==18.1.1 +lightning==2.6.1 +lightning-fabric==2.6.1 lightning-utilities==0.15.3 lit==18.1.8 marshmallow==3.26.2 From c82f55387615f7b134638cd9ca431dcb0a599bb4 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste VESLIN <19856429+thebaptiste@users.noreply.github.com> Date: Wed, 12 Aug 2026 09:49:27 +0200 Subject: [PATCH 15/38] Update components.md --- .metwork-framework/components.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.metwork-framework/components.md b/.metwork-framework/components.md index 6f58c1a..45bdc19 100644 --- a/.metwork-framework/components.md +++ b/.metwork-framework/components.md @@ -88,7 +88,6 @@ | [tensorboard-data-server](https://github.com/tensorflow/tensorboard/tree/master/tensorboard/data/server) | 0.7.2 | python3_ia | | [tensorboard](https://github.com/tensorflow/tensorboard) | 2.20.0 | python3_ia | | [tensorboardX](https://github.com/lanpa/tensorboardX) | 2.6.5 | python3_ia | -| [tf_keras](https://keras.io/) | 2.21.0 | python3_ia | | [tiktoken](https://github.com/openai/tiktoken) | 0.12.0 | python3_ia | | [timm](https://github.com/huggingface/pytorch-image-models) | 1.0.22 | python3_ia | | [tokenizers](https://github.com/huggingface/tokenizers) | 0.22.2 | python3_ia | @@ -101,4 +100,4 @@ | [typeshed_client](https://github.com/JelleZijlstra/typeshed_client) | 2.8.2 | python3_ia | | [uvicorn](https://www.uvicorn.org/) | 0.34.2 | python3_ia | -*(100 components)* \ No newline at end of file +*(99 components)* From 3d0a45133c1064df54073346a36f1f43aded31ac Mon Sep 17 00:00:00 2001 From: Jean-Baptiste VESLIN <19856429+thebaptiste@users.noreply.github.com> Date: Wed, 12 Aug 2026 09:51:06 +0200 Subject: [PATCH 16/38] Update import_python3_ia.py --- .../0003_test_import_python3_ia/import_python3_ia.py | 1 - 1 file changed, 1 deletion(-) diff --git a/integration_tests/0003_test_import_python3_ia/import_python3_ia.py b/integration_tests/0003_test_import_python3_ia/import_python3_ia.py index c188f68..0e01296 100644 --- a/integration_tests/0003_test_import_python3_ia/import_python3_ia.py +++ b/integration_tests/0003_test_import_python3_ia/import_python3_ia.py @@ -71,7 +71,6 @@ import tensorboard_data_server import tensorboard import tensorboardX -import tf_keras import tiktoken import timm import tokenizers From d55123140d10ac622d8ffc672411af70f7bf9c75 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste VESLIN <19856429+thebaptiste@users.noreply.github.com> Date: Wed, 12 Aug 2026 09:53:36 +0200 Subject: [PATCH 17/38] Update import_python3_ia.py --- .../0003_test_import_python3_ia/import_python3_ia.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/0003_test_import_python3_ia/import_python3_ia.py b/integration_tests/0003_test_import_python3_ia/import_python3_ia.py index 0e01296..f3b12e3 100644 --- a/integration_tests/0003_test_import_python3_ia/import_python3_ia.py +++ b/integration_tests/0003_test_import_python3_ia/import_python3_ia.py @@ -29,7 +29,7 @@ import hydra #hydra-core import jsonargparse import _jsonnet #jsonnet -#import keras not Python 3.14 compliant +import keras import clang #libclang import lightning_fabric import lightning_utilities @@ -50,7 +50,7 @@ import oauthlib import omegaconf import onnx -#import onnxruntime not Python 3.14 compliant +import onnxruntime import onnxscript import opentelemetry #opentelemetry_api import opt_einsum From 7ffcf403b3af9cece64358497b2ab69ce4a5a59d Mon Sep 17 00:00:00 2001 From: thebaptiste Date: Wed, 12 Aug 2026 22:00:57 +0000 Subject: [PATCH 18/38] feat: use torch as backend for keras (tensorflow not available) --- layers/layer5_python3_ia/.layerapi2_extra_env | 1 + 1 file changed, 1 insertion(+) create mode 100644 layers/layer5_python3_ia/.layerapi2_extra_env diff --git a/layers/layer5_python3_ia/.layerapi2_extra_env b/layers/layer5_python3_ia/.layerapi2_extra_env new file mode 100644 index 0000000..9e628bd --- /dev/null +++ b/layers/layer5_python3_ia/.layerapi2_extra_env @@ -0,0 +1 @@ +KERAS_BACKEND=torch From 185452e518204974d12400f7c2881658b85ab15d Mon Sep 17 00:00:00 2001 From: thebaptiste Date: Wed, 2 Sep 2026 15:08:45 +0000 Subject: [PATCH 19/38] feat: reintroduce mfai 8.0.2 --- .metwork-framework/components.md | 26 ++----------------- .../import_python3_ia.py | 20 +------------- .../0500_extra_packages/requirements3.txt | 24 +---------------- 3 files changed, 4 insertions(+), 66 deletions(-) diff --git a/.metwork-framework/components.md b/.metwork-framework/components.md index 45bdc19..9225497 100644 --- a/.metwork-framework/components.md +++ b/.metwork-framework/components.md @@ -1,7 +1,6 @@ | Name | Version | Layer | | --- | --- | --- | | [absl-py](https://github.com/abseil/abseil-py) | 2.2.2 | python3_ia | -| [alembic](https://alembic.sqlalchemy.org) | 1.18.4 | python3_ia | | [antlr4-python3-runtime](http://www.antlr.org) | 4.9.3 | python3_ia | | [astunparse](https://github.com/simonpercivall/astunparse) | 1.6.3 | python3_ia | | [axial_attention](https://github.com/lucidrains/axial-attention) | 0.6.1 | python3_ia | @@ -9,27 +8,16 @@ | [cuda-bindings](https://pypi.org/project/cuda-bindings) | 13.3.1 | python3_ia | | [cuda-pathfinder](https://pypi.org/project/cuda-pathfinder) | 1.5.5 | python3_ia | | [cuda-toolkit](https://developer.nvidia.com/cuda-toolkit) | 13.0.3.0 | python3_ia | -| [databricks-sdk](https://pypi.org/project/databricks-sdk) | 0.118.0 | python3_ia | | [dataclasses-json](https://github.com/lidatong/dataclasses-json) | 0.6.7 | python3_ia | | [diffusers](https://github.com/huggingface/diffusers) | 0.38.0 | python3_ia | | [dm-tree](https://github.com/deepmind/tree) | 0.1.9 | python3_ia | -| [docker](https://github.com/docker/docker-py) | 7.1.0 | python3_ia | | [einops](https://github.com/arogozhnikov/einops) | 0.8.1 | python3_ia | -| [fastapi](https://github.com/fastapi/fastapi) | 0.136.3 | python3_ia | -| [flask-cors](https://corydolphin.github.io/flask-cors/) | 6.0.5 | python3_ia | | [flatbuffers](https://google.github.io/flatbuffers/) | 25.12.19 | python3_ia | -| [gitdb](https://github.com/gitpython-developers/gitdb) | 4.0.12 | python3_ia | -| [GitPython](https://github.com/gitpython-developers/GitPython) | 3.1.58 | python3_ia | | [google-auth-oauthlib](https://github.com/GoogleCloudPlatform/google-auth-library-python-oauthlib) | 1.2.2 | python3_ia | | [google-auth](https://github.com/googleapis/google-auth-library-python) | 2.53.0 | python3_ia | | [google-pasta](https://github.com/google/pasta) | 0.2.0 | python3_ia | -| [graphene](https://github.com/graphql-python/graphene) | 3.4.3 | python3_ia | -| [graphql-core](https://github.com/graphql-python/graphql-core) | 3.2.11 | python3_ia | -| [graphql-relay](https://github.com/graphql-python/graphql-relay-py) | 3.2.0 | python3_ia | | [grpcio](https://grpc.io) | 1.76.0 | python3_ia | -| [gunicorn](https://gunicorn.org) | 26.0.0 | python3_ia | | [hf-xet](https://github.com/huggingface/xet-core) | 1.4.3 | python3_ia | -| [huey](https://pypi.org/project/huey) | 3.0.3 | python3_ia | | [huggingface_hub](https://github.com/huggingface/huggingface_hub) | 1.5.0 | python3_ia | | [hydra-core](https://github.com/facebookresearch/hydra) | 1.3.4 | python3_ia | | [jsonargparse](https://pypi.org/project/jsonargparse) | 4.46.0 | python3_ia | @@ -41,6 +29,7 @@ | [lightning](https://github.com/Lightning-AI/lightning) | 2.6.5 | python3_ia | | [lit](http://llvm.org) | 18.1.8 | python3_ia | | [marshmallow](https://pypi.org/project/marshmallow) | 3.26.2 | python3_ia | +| [mfai](https://github.com/meteofrance/mfai) | 8.0.2 | python3_ia | | [ml_dtypes](https://github.com/jax-ml/ml_dtypes) | 0.5.4 | python3_ia | | [monai](https://project-monai.github.io/) | 1.6.0 | python3_ia | | [mpmath](http://mpmath.org/) | 1.3.0 | python3_ia | @@ -65,24 +54,14 @@ | [onnx](https://onnx.ai/) | 1.22.0 | python3_ia | | [onnxruntime](https://onnxruntime.ai) | 1.28.0 | python3_ia | | [onnxscript](https://microsoft.github.io/onnxscript/) | 0.2.5 | python3_ia | -| [opentelemetry-api](https://github.com/open-telemetry/opentelemetry-python/tree/main/opentelemetry-api) | 1.42.1 | python3_ia | -| [opentelemetry-proto](https://github.com/open-telemetry/opentelemetry-python/tree/main/opentelemetry-proto) | 1.42.1 | python3_ia | -| [opentelemetry-sdk](https://github.com/open-telemetry/opentelemetry-python/tree/main/opentelemetry-sdk) | 1.42.1 | python3_ia | -| [opentelemetry-semantic-conventions](https://github.com/open-telemetry/opentelemetry-python/tree/main/opentelemetry-semantic-conventions) | 0.63b1 | python3_ia | | [opt_einsum](https://pypi.org/project/opt_einsum) | 3.4.0 | python3_ia | | [optree](https://github.com/metaopt/optree) | 0.18.0 | python3_ia | -| [prettytable](https://github.com/prettytable/prettytable) | 3.17.0 | python3_ia | -| [python-dotenv](https://pypi.org/project/python-dotenv) | 1.2.2 | python3_ia | | [pytorch-lightning](https://github.com/Lightning-AI/lightning) | 2.6.5 | python3_ia | | [requests-oauthlib](https://github.com/requests/requests-oauthlib) | 2.0.0 | python3_ia | | [rsa](https://stuvel.eu/rsa) | 4.9 | python3_ia | | [safetensors](https://github.com/huggingface/safetensors) | 0.8.0 | python3_ia | | [segmentation_models_pytorch](https://github.com/qubvel-org/segmentation_models.pytorch) | 0.5.0 | python3_ia | | [sentencepiece](https://github.com/google/sentencepiece) | 0.2.1 | python3_ia | -| [skops](http://github.com/skops-dev/skops) | 0.14.0 | python3_ia | -| [smmap](https://github.com/gitpython-developers/smmap) | 5.0.2 | python3_ia | -| [sqlparse](https://pypi.org/project/sqlparse) | 0.5.5 | python3_ia | -| [starlette](https://github.com/Kludex/starlette) | 1.3.1 | python3_ia | | [sympy](https://sympy.org) | 1.14.0 | python3_ia | | [tabulate](https://github.com/astanin/python-tabulate) | 0.9.0 | python3_ia | | [tensorboard-data-server](https://github.com/tensorflow/tensorboard/tree/master/tensorboard/data/server) | 0.7.2 | python3_ia | @@ -98,6 +77,5 @@ | [transformers](https://github.com/huggingface/transformers) | 5.5.4 | python3_ia | | [triton](https://github.com/triton-lang/triton/) | 3.7.1 | python3_ia | | [typeshed_client](https://github.com/JelleZijlstra/typeshed_client) | 2.8.2 | python3_ia | -| [uvicorn](https://www.uvicorn.org/) | 0.34.2 | python3_ia | -*(99 components)* +*(77 components)* diff --git a/integration_tests/0003_test_import_python3_ia/import_python3_ia.py b/integration_tests/0003_test_import_python3_ia/import_python3_ia.py index f3b12e3..d19aff6 100644 --- a/integration_tests/0003_test_import_python3_ia/import_python3_ia.py +++ b/integration_tests/0003_test_import_python3_ia/import_python3_ia.py @@ -1,30 +1,20 @@ import absl #absl_py -import alembic import antlr4 #antlr4-python3-runtime import astunparse import axial_attention import bitsandbytes from cuda import bindings from cuda import pathfinder -import databricks.sdk import dataclasses_json import diffusers -import docker import tree #dm_tree import einops -import fastapi -import flask_cors import flatbuffers import google_auth_oauthlib import google #google_auth import pasta #google_pasta -import graphene -import graphql #graphql_core -import graphql_relay import grpc #grpcio -import gunicorn import hf_xet -import huey import huggingface_hub import hydra #hydra-core import jsonargparse @@ -36,9 +26,8 @@ import lightning_fabric import lightning import marshmallow -#import mfai +import mfai import ml_dtypes -#import mlflow import monai import mpmath import namex @@ -52,20 +41,14 @@ import onnx import onnxruntime import onnxscript -import opentelemetry #opentelemetry_api import opt_einsum import optree -import prettytable -import dotenv #python_dotenv import pytorch_lightning import requests_oauthlib import rsa import safetensors import segmentation_models_pytorch import sentencepiece -import skops -import sqlparse -import starlette import sympy import tabulate import tensorboard_data_server @@ -81,4 +64,3 @@ import transformers import triton import typeshed_client -import uvicorn diff --git a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt index b108760..950aecc 100644 --- a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt +++ b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt @@ -1,5 +1,4 @@ absl-py==2.2.2 -alembic==1.18.4 antlr4_python3_runtime==4.9.3 astunparse==1.6.3 axial_attention==0.6.1 @@ -7,27 +6,16 @@ bitsandbytes==0.49.2 cuda_bindings==13.3.1 cuda_pathfinder==1.5.5 cuda-toolkit==13.0.3 -databricks-sdk==0.118.0 dataclasses_json==0.6.7 diffusers==0.38.0 dm-tree==0.1.9 -docker==7.1.0 einops==0.8.1 -fastapi==0.136.3 -Flask-CORS==6.0.5 flatbuffers==25.12.19 -gitdb==4.0.12 -GitPython==3.1.58 google-auth==2.53.0 google-auth-oauthlib==1.2.2 google-pasta==0.2.0 -graphene==3.4.3 -graphql-core==3.2.11 -graphql-relay==3.2.0 grpcio==1.76.0 -gunicorn==26.0.0 hf_xet==1.4.3 -huey==3.0.3 huggingface_hub==1.5.0 hydra-core==1.3.4 jsonargparse==4.46.0 @@ -39,6 +27,7 @@ lightning-fabric==2.6.5 lightning-utilities==0.15.3 lit==18.1.8 marshmallow==3.26.2 +mfai==8.0.2 ml-dtypes==0.5.4 monai==1.6.0 mpmath==1.3.0 @@ -63,24 +52,14 @@ omegaconf==2.3.1 onnx==1.22.0 onnxruntime==1.28.0 onnxscript==0.2.5 -opentelemetry-api==1.42.1 -opentelemetry-proto==1.42.1 -opentelemetry-sdk==1.42.1 -opentelemetry-semantic-conventions==0.63b1 opt-einsum==3.4.0 optree==0.18.0 -prettytable==3.17.0 -python-dotenv==1.2.2 pytorch-lightning==2.6.5 requests-oauthlib==2.0.0 rsa==4.9 safetensors==0.8.0 segmentation_models_pytorch==0.5.0 sentencepiece==0.2.1 -skops==0.14.0 -smmap==5.0.2 -sqlparse==0.5.5 -starlette==1.3.1 sympy==1.14.0 tabulate==0.9.0 tensorboard==2.20.0 @@ -96,4 +75,3 @@ torch-geometric==2.7.0 transformers==5.5.4 triton==3.7.1 typeshed-client==2.8.2 -uvicorn==0.34.2 From f175418465379f49228e871eac817c0b4ed8f9cc Mon Sep 17 00:00:00 2001 From: metworkbot Date: Tue, 8 Sep 2026 08:39:42 +0000 Subject: [PATCH 20/38] build: sync common files from github_organization_management repository --- .mergify.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.mergify.yml b/.mergify.yml index 9de7cb1..0c56638 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -107,6 +107,14 @@ pull_request_rules: backport: branches: - release_2.3 + - name: backport to 2.4 + conditions: + - base=integration + - label=backport-to-2.4 + actions: + backport: + branches: + - release_2.4 - name: approved merges actions: From 26912ac12b4cf38c7837546b4cb22fc8e1013972 Mon Sep 17 00:00:00 2001 From: metworkbot Date: Wed, 9 Sep 2026 07:18:46 +0000 Subject: [PATCH 21/38] build: changelog automatic update --- CHANGELOG.md | 61 ---------------------------------------------------- 1 file changed, 61 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7431efa..b24080e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,64 +1,3 @@ # CHANGELOG -## [Unreleased] - -### New Features - -- bump keras from 3.9.2 to 3.10.0 (#204) -- bump onnx from 1.17.0 to 1.18.0 (#208) -- bump to torch 2.7.1, triton 3.3.1 and torchvision 0.22.1 (#210) -- bump transformers from 4.52.1 to 4.52.4 (#213) -- bump keras from 3.10.0 to 3.11.3 (fix high CVE-2025-8747) (#219) -- bump transformers from 4.52.4 to 4.53.3 (fix CVE-2025-5197) (#221) -- bump tensorflow (and dependencies) from 2.19.0 to 2.20.0 (#222) -- bump tf_keras from 2.20.0 (yanked) to 2.20.1 (#224) -- bump lightning from 2.5.1.post0 to 2.5.5 (#226) -- bump torch from 2.7.1 to 2.8.0 (fix moderate CVE-2025-3730) (#227) -- bump keras from 3.11.3 to 3.12.0 (fix high CVE-2025-12060) (#229) -- bump safetensors from 0.5.3 to 0.6.2 (#230) -- bump grpcio from 1.71.0 to 1.76.0 (compat Python 3.14) (#232) -- bump ml-dtypes from 0.5.1 to 0.5.4 (compat Python 3.14) (#233) -- bump onnx from 1.18.0 to 1.20.0 (compat Python 3.14) (#234) -- bump optree from 0.15.0 to 0.18.0 (compat Python 3.14) (#235) -- bump timm from 1.0.15 to 1.0.22 (compat Python 3.14) (#236) -- bump torch from 2.8.0 to 2.9.1 (compat Python 3.14) (#237) -- bump lightning from 2.5.5 to 2.6.0 (#239) -- bump keras from 3.12.0 to 3.13.1 (fix high CVE-2026-0897) (#241) -- bump google-auth from 2.40.1 to 2.41.1 (compat cachetools>=6) (#243) -- bump torch, torchvision, triton to 2.10.0, 0.25.0, 3.6.0 (#245) -- bump keras from 3.13.1 to 3.13.2 (fix high CVE-2026-0897) (#246) -- bump lightning from 2.6.0 to 2.6.1 (#249) -- bump onnx from 1.20.0 to 1.20.1 (#250) -- add mfai package and dependencies (#251) -- bump torchmetrics to 1.9.0 and lightning-utilities to 0.15.3 (#256) -- bump onnxruntime from 1.22.0 to 1.24.4 (compat Python 3.14) (#258) -- bump onnx from 1.20.1 to 1.21.0 (fix CVE-2026-34445/28500) (#262) -- bump transformers from 4.53.3 to 5.3.0 (fix moderate CVE-2026-1… (#264) -- add lightning-fabric (#268) -- bump diffusers to 0.38.0 (fix high CVE-2026-44513/CVE-2026-45804) (#269) -- bump lightning from 2.6.1 to 2.6.5 (#272) -- remove tensorflow-hub (not required anymore) (#273) -- bump google-auth from 2.41.1 to 2.53.0 (#275) -- bump tensorflow to 2.21.0 and temporarly remove mfai (#276) -- bump torch from 2.10.0 to 2.12.0 (#277) -- bump bitsandbytes from 0.49.1 to 0.49.2 (#278) -- add mfai 7.3.0 (#279) -- bump keras from 3.13.2 to 3.14.1 (#281) -- bump onnx from 1.21.0 to 1.22.0 (fix moderate CVE-2026-44512) (#287) -- bump transformers from 5.3.0 to 5.5.4 (fix high CVE-2026-5241) -- bump torch from 2.12.0 to 2.13.0 (fix low CVE-2025-3000) (#293) -- bump onnxruntime from 1.24.4 to 1.25.1 (fix high CVE-2026-27904) (#295) -- bump onnxruntime from 1.25.1 to 1.28.0 (#297) -- bump safetensors from 0.8.0rc0 to 0.8.0 (#298) -- bump monai from 1.5.2 to 1.6.0 (fix high GHSA-89gg-p5r5-q6r4) -- bump hydra-core from 1.3.2 to 1.3.4 (#303) -- bump omegaconf from 2.3.0 to 2.3.1 -- bump keras from 3.14.1 to 3.15.1 (fix 2 high CVE) (#309) -- bump sqlparse from 0.5.5 to 0.6.0 (fix 3 high CVE) (#314) -- add mfai 8.0.0 (release 7.3.0 had been temporarly removed) (#316) -- bump hydra-core from 1.3.4 to 1.3.5 (#317) -- bump ml_dtypes from 0.5.4 to 0.6.0 (#318) -- bump transformers from 5.5.4 to 5.16.1 (fix high CVE-2026-9856) (#320) -- bump onnxruntime from 1.28.0 to 1.29.0 (#322) - From 1002b82ea21a0c99375aca3cb33bc3ec47ea7be4 Mon Sep 17 00:00:00 2001 From: metworkbot Date: Wed, 9 Sep 2026 07:19:01 +0000 Subject: [PATCH 22/38] build: changelog automatic update --- CHANGELOG-2.4.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 CHANGELOG-2.4.md diff --git a/CHANGELOG-2.4.md b/CHANGELOG-2.4.md new file mode 100644 index 0000000..e2ddab9 --- /dev/null +++ b/CHANGELOG-2.4.md @@ -0,0 +1,3 @@ +# release_2.4 CHANGELOG + + From fa30b2c000b6ca3f045b05bb8e3081cd35ed01ef Mon Sep 17 00:00:00 2001 From: metworkbot Date: Thu, 10 Sep 2026 07:14:14 +0000 Subject: [PATCH 23/38] build: changelog automatic update --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b24080e..2074a66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ # CHANGELOG +## [Unreleased] + +### New Features + +- various updates for Python 3.14 compatibility +- add onnxruntime 1.24.4 (now Python 3.14 compliant) +- use torch as backend for keras (tensorflow not available) +- reintroduce mfai 8.0.2 + From 8700f463425ad80a5850aec75e366cfda8c38084 Mon Sep 17 00:00:00 2001 From: thebaptiste Date: Thu, 10 Sep 2026 12:12:13 +0000 Subject: [PATCH 24/38] feat: bump lightning from 2.6.5 to 2.6.6 (fix high CVE-2026-58659) --- .metwork-framework/components.md | 6 +++--- .../layer5_python3_ia/0500_extra_packages/requirements3.txt | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.metwork-framework/components.md b/.metwork-framework/components.md index 3e3412b..9a3adae 100644 --- a/.metwork-framework/components.md +++ b/.metwork-framework/components.md @@ -24,9 +24,9 @@ | [jsonnet](https://jsonnet.org) | 0.21.0 | python3_ia | | [keras](https://pypi.org/project/keras) | 3.15.1 | python3_ia | | [libclang](https://github.com/sighingnow/libclang) | 18.1.1 | python3_ia | -| [lightning-fabric](https://github.com/Lightning-AI/lightning) | 2.6.5 | python3_ia | +| [lightning-fabric](https://github.com/Lightning-AI/lightning) | 2.6.6 | python3_ia | | [lightning-utilities](https://github.com/Lightning-AI/utilities) | 0.15.3 | python3_ia | -| [lightning](https://github.com/Lightning-AI/lightning) | 2.6.5 | python3_ia | +| [lightning](https://github.com/Lightning-AI/lightning) | 2.6.6 | python3_ia | | [lit](http://llvm.org) | 18.1.8 | python3_ia | | [marshmallow](https://pypi.org/project/marshmallow) | 3.26.2 | python3_ia | | [mfai](https://github.com/meteofrance/mfai) | 8.0.2 | python3_ia | @@ -56,7 +56,7 @@ | [onnxscript](https://microsoft.github.io/onnxscript/) | 0.2.5 | python3_ia | | [opt_einsum](https://pypi.org/project/opt_einsum) | 3.4.0 | python3_ia | | [optree](https://github.com/metaopt/optree) | 0.18.0 | python3_ia | -| [pytorch-lightning](https://github.com/Lightning-AI/lightning) | 2.6.5 | python3_ia | +| [pytorch-lightning](https://github.com/Lightning-AI/lightning) | 2.6.6 | python3_ia | | [requests-oauthlib](https://github.com/requests/requests-oauthlib) | 2.0.0 | python3_ia | | [rsa](https://stuvel.eu/rsa) | 4.9 | python3_ia | | [safetensors](https://github.com/huggingface/safetensors) | 0.8.0 | python3_ia | diff --git a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt index 8e54d14..e3296c2 100644 --- a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt +++ b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt @@ -22,8 +22,8 @@ jsonargparse==4.46.0 jsonnet==0.21.0 keras==3.15.1 libclang==18.1.1 -lightning==2.6.5 -lightning-fabric==2.6.5 +lightning==2.6.6 +lightning-fabric==2.6.6 lightning-utilities==0.15.3 lit==18.1.8 marshmallow==3.26.2 @@ -54,7 +54,7 @@ onnxruntime==1.29.0 onnxscript==0.2.5 opt-einsum==3.4.0 optree==0.18.0 -pytorch-lightning==2.6.5 +pytorch-lightning==2.6.6 requests-oauthlib==2.0.0 rsa==4.9 safetensors==0.8.0 From 8535dfced8be94018417ae44417ac9fc3e36ea7c Mon Sep 17 00:00:00 2001 From: metworkbot Date: Fri, 11 Sep 2026 07:12:35 +0000 Subject: [PATCH 25/38] build: changelog automatic update --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2074a66..3578053 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,5 +8,6 @@ - add onnxruntime 1.24.4 (now Python 3.14 compliant) - use torch as backend for keras (tensorflow not available) - reintroduce mfai 8.0.2 +- bump lightning from 2.6.5 to 2.6.6 (fix high CVE-2026-58659) From 5b2c5c793250ce6fd1bb58b5df73592f81cc7314 Mon Sep 17 00:00:00 2001 From: metworkbot Date: Fri, 11 Sep 2026 14:13:12 +0000 Subject: [PATCH 26/38] build: sync common files from github_organization_management repository --- .github/workflows/build.sh | 10 ++++++++-- bootstrap.sh | 2 +- .../0096_system_extra_dependencies/test.sh | 1 - 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.sh b/.github/workflows/build.sh index f5a8d3c..f5e1060 100755 --- a/.github/workflows/build.sh +++ b/.github/workflows/build.sh @@ -17,8 +17,14 @@ rm -rf html_doc rpms .build_hash if test "${OS_VERSION}" = "centos8"; then export METWORK_BUILD_OS=generic; else export METWORK_BUILD_OS=${OS_VERSION}; fi +case "${BRANCH}" in + ci* | pci*) + export DEP_BRANCH=integration + *) + export DEP_BRANCH=${BRANCH} +esac;; - yum install -y metwork-mfext-layer-python3_scientific-${BRANCH##release_} + yum install -y metwork-mfext-layer-python3_scientific-${DEP_BRANCH##release_} @@ -49,7 +55,7 @@ fi MODULEHASH=`/opt/metwork-mfext-${TARGET_DIR}/bin/mfext_wrapper module_hash 2>module_hash.debug` if test -f /opt/metwork-mfext-${TARGET_DIR}/.dhash; then cat /opt/metwork-mfext-${TARGET_DIR}/.dhash; fi cat module_hash.debug |sort |uniq ; rm -f module_hash.debug -echo "${MODULEHASH}${DRONE_TAG}${DRONE_BRANCH}" |md5sum |cut -d ' ' -f1 >.build_hash +echo "${MODULEHASH}${DRONE_TAG}${BRANCH}" |md5sum |cut -d ' ' -f1 >.build_hash if test -f "${BUILDCACHE}/build_hash_mfextaddon_python3_ia_${BRANCH}_`cat .build_hash`"; then echo "bypass=true" >> github_output echo "buildcache=null" >> github_output diff --git a/bootstrap.sh b/bootstrap.sh index 6b52044..8c83c92 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -83,7 +83,7 @@ fi # FIXME: do not hardcode this # FIXME: move to layer root extra_env ? -echo "export PYTHON3_SHORT_VERSION := 3.14" >>adm/root.mk +echo "export PYTHON3_SHORT_VERSION := 3.13" >>adm/root.mk echo "BOOTSTRAP DONE !" echo "MFEXT_HOME=${MFEXT_HOME}" diff --git a/integration_tests/0096_system_extra_dependencies/test.sh b/integration_tests/0096_system_extra_dependencies/test.sh index c9519a2..00f2fd2 100755 --- a/integration_tests/0096_system_extra_dependencies/test.sh +++ b/integration_tests/0096_system_extra_dependencies/test.sh @@ -12,7 +12,6 @@ OK_NOT_FOUND=$(cat list_ok_not_found.txt|grep -v "#"|xargs) export PATH=${PATH}:${PWD}/.. RET=0 -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib64/openmpi/lib From bebeca1eb4679c0953f165422e43a6885d4be049 Mon Sep 17 00:00:00 2001 From: metworkbot Date: Fri, 11 Sep 2026 14:32:31 +0000 Subject: [PATCH 27/38] build: sync common files from github_organization_management repository --- .github/workflows/build.sh | 6 +++--- bootstrap.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.sh b/.github/workflows/build.sh index f5e1060..1ea2067 100755 --- a/.github/workflows/build.sh +++ b/.github/workflows/build.sh @@ -19,10 +19,10 @@ rm -rf html_doc rpms .build_hash case "${BRANCH}" in ci* | pci*) - export DEP_BRANCH=integration + export DEP_BRANCH=integration *) - export DEP_BRANCH=${BRANCH} -esac;; + export DEP_BRANCH=${BRANCH} +esac yum install -y metwork-mfext-layer-python3_scientific-${DEP_BRANCH##release_} diff --git a/bootstrap.sh b/bootstrap.sh index 8c83c92..6b52044 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -83,7 +83,7 @@ fi # FIXME: do not hardcode this # FIXME: move to layer root extra_env ? -echo "export PYTHON3_SHORT_VERSION := 3.13" >>adm/root.mk +echo "export PYTHON3_SHORT_VERSION := 3.14" >>adm/root.mk echo "BOOTSTRAP DONE !" echo "MFEXT_HOME=${MFEXT_HOME}" From f0615380451924f0bd6620c96f8f6fea34631182 Mon Sep 17 00:00:00 2001 From: metworkbot Date: Fri, 11 Sep 2026 14:45:19 +0000 Subject: [PATCH 28/38] build: sync common files from github_organization_management repository --- .github/workflows/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.sh b/.github/workflows/build.sh index 1ea2067..1234af4 100755 --- a/.github/workflows/build.sh +++ b/.github/workflows/build.sh @@ -19,9 +19,9 @@ rm -rf html_doc rpms .build_hash case "${BRANCH}" in ci* | pci*) - export DEP_BRANCH=integration + export DEP_BRANCH=integration;; *) - export DEP_BRANCH=${BRANCH} + export DEP_BRANCH=${BRANCH};; esac yum install -y metwork-mfext-layer-python3_scientific-${DEP_BRANCH##release_} From 875ec0262d7612820e331aeeab8ee48a386720e1 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste VESLIN <19856429+thebaptiste@users.noreply.github.com> Date: Sun, 13 Sep 2026 20:04:02 +0200 Subject: [PATCH 29/38] feat: bump torch from 2.13.0 to 2.14.0 and triton from 3.7.1 to 3.8.0 (#327) --- .metwork-framework/components.md | 10 +++++----- .../0500_extra_packages/requirements3.txt | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.metwork-framework/components.md b/.metwork-framework/components.md index 9a3adae..d1e7438 100644 --- a/.metwork-framework/components.md +++ b/.metwork-framework/components.md @@ -38,14 +38,14 @@ | [nvidia-cuda-cupti](https://developer.nvidia.com/cuda-zone) | 13.0.85 | python3_ia | | [nvidia-cuda-nvrtc](https://developer.nvidia.com/cuda-zone) | 13.0.88 | python3_ia | | [nvidia-cuda-runtime](https://developer.nvidia.com/cuda-zone) | 13.0.96 | python3_ia | -| [nvidia-cudnn-cu13](https://developer.nvidia.com/cuda-zone) | 9.20.0.48 | python3_ia | +| [nvidia-cudnn-cu13](https://developer.nvidia.com/cuda-zone) | 9.24.0.43 | python3_ia | | [nvidia-cufft](https://developer.nvidia.com/cuda-zone) | 12.0.0.61 | python3_ia | | [nvidia-cufile](https://developer.nvidia.com/cuda-zone) | 1.15.1.6 | python3_ia | | [nvidia-curand](https://developer.nvidia.com/cuda-zone) | 10.4.0.35 | python3_ia | | [nvidia-cusolver](https://developer.nvidia.com/cuda-zone) | 12.0.4.66 | python3_ia | | [nvidia-cusparse](https://developer.nvidia.com/cuda-zone) | 12.6.3.3 | python3_ia | | [nvidia-cusparselt-cu13](https://developer.nvidia.com/cusparselt) | 0.8.1 | python3_ia | -| [nvidia-nccl-cu13](https://developer.nvidia.com/cuda-zone) | 2.29.7 | python3_ia | +| [nvidia-nccl-cu13](https://developer.nvidia.com/cuda-zone) | 2.30.7 | python3_ia | | [nvidia-nvjitlink](https://developer.nvidia.com/cuda-zone) | 13.0.88 | python3_ia | | [nvidia-nvshmem-cu13](https://developer.nvidia.com/cuda-zone) | 3.4.5 | python3_ia | | [nvidia-nvtx](https://developer.nvidia.com/cuda-zone) | 13.0.85 | python3_ia | @@ -71,11 +71,11 @@ | [timm](https://github.com/huggingface/pytorch-image-models) | 1.0.22 | python3_ia | | [tokenizers](https://github.com/huggingface/tokenizers) | 0.23.1 | python3_ia | | [torch-geometric](https://pyg.org) | 2.7.0 | python3_ia | -| [torch](https://pytorch.org) | 2.13.0 | python3_ia | +| [torch](https://pytorch.org) | 2.14.0 | python3_ia | | [torchmetrics](https://github.com/Lightning-AI/torchmetrics) | 1.9.0 | python3_ia | -| [torchvision](https://github.com/pytorch/vision) | 0.28.0 | python3_ia | +| [torchvision](https://github.com/pytorch/vision) | 0.29.0 | python3_ia | | [transformers](https://github.com/huggingface/transformers) | 5.16.1 | python3_ia | -| [triton](https://github.com/triton-lang/triton/) | 3.7.1 | python3_ia | +| [triton](https://github.com/triton-lang/triton/) | 3.8.0 | python3_ia | | [typeshed_client](https://github.com/JelleZijlstra/typeshed_client) | 2.8.2 | python3_ia | *(77 components)* diff --git a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt index e3296c2..709e0ff 100644 --- a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt +++ b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt @@ -36,7 +36,7 @@ nvidia_cublas==13.1.1.3 nvidia-cuda-cupti==13.0.85 nvidia-cuda-nvrtc==13.0.88 nvidia-cuda-runtime==13.0.96 -nvidia-cudnn-cu13==9.20.0.48 +nvidia-cudnn-cu13==9.24.0.43 nvidia-cufft==12.0.0.61 nvidia-cufile==1.15.1.6 nvidia-curand==10.4.0.35 @@ -44,7 +44,7 @@ nvidia-cusolver==12.0.4.66 nvidia_cusparse==12.6.3.3 nvidia-cusparselt-cu13==0.8.1 nvidia_nvjitlink==13.0.88 -nvidia-nccl-cu13==2.29.7 +nvidia-nccl-cu13==2.30.7 nvidia-nvshmem-cu13==3.4.5 nvidia-nvtx==13.0.85 oauthlib==3.2.2 @@ -68,10 +68,10 @@ tensorboardx==2.6.5 tiktoken==0.12.0 timm==1.0.22 tokenizers==0.23.1 -torch==2.13.0 +torch==2.14.0 torchmetrics==1.9.0 -torchvision==0.28.0 +torchvision==0.29.0 torch-geometric==2.7.0 transformers==5.16.1 -triton==3.7.1 +triton==3.8.0 typeshed-client==2.8.2 From ce138b09406bb4cd3081fe20901c44948ae05b7b Mon Sep 17 00:00:00 2001 From: metworkbot Date: Mon, 14 Sep 2026 07:47:46 +0000 Subject: [PATCH 30/38] build: changelog automatic update --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3578053..2fdbb28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,5 +9,6 @@ - use torch as backend for keras (tensorflow not available) - reintroduce mfai 8.0.2 - bump lightning from 2.6.5 to 2.6.6 (fix high CVE-2026-58659) +- bump torch from 2.13.0 to 2.14.0 and triton from 3.7.1 to 3.8.0 (#327) From f382b2732f76927f201e3ba0b6c27c19416f4fad Mon Sep 17 00:00:00 2001 From: Jean-Baptiste VESLIN <19856429+thebaptiste@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:38:15 +0200 Subject: [PATCH 31/38] feat: bump hydra-core from 1.3.5 to 1.3.7 (#328) --- .metwork-framework/components.md | 2 +- layers/layer5_python3_ia/0500_extra_packages/requirements3.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.metwork-framework/components.md b/.metwork-framework/components.md index d1e7438..fc107a4 100644 --- a/.metwork-framework/components.md +++ b/.metwork-framework/components.md @@ -19,7 +19,7 @@ | [grpcio](https://grpc.io) | 1.76.0 | python3_ia | | [hf-xet](https://github.com/huggingface/xet-core) | 1.4.3 | python3_ia | | [huggingface_hub](https://github.com/huggingface/huggingface_hub) | 1.5.0 | python3_ia | -| [hydra-core](https://github.com/facebookresearch/hydra) | 1.3.5 | python3_ia | +| [hydra-core](https://github.com/facebookresearch/hydra) | 1.3.7 | python3_ia | | [jsonargparse](https://pypi.org/project/jsonargparse) | 4.46.0 | python3_ia | | [jsonnet](https://jsonnet.org) | 0.21.0 | python3_ia | | [keras](https://pypi.org/project/keras) | 3.15.1 | python3_ia | diff --git a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt index 709e0ff..c11c064 100644 --- a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt +++ b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt @@ -17,7 +17,7 @@ google-pasta==0.2.0 grpcio==1.76.0 hf_xet==1.4.3 huggingface_hub==1.5.0 -hydra-core==1.3.5 +hydra-core==1.3.7 jsonargparse==4.46.0 jsonnet==0.21.0 keras==3.15.1 From 9c58bf121c721947497307a100da8e67f29194e8 Mon Sep 17 00:00:00 2001 From: metworkbot Date: Wed, 16 Sep 2026 07:30:26 +0000 Subject: [PATCH 32/38] build: changelog automatic update --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fdbb28..ab5e682 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,5 +10,6 @@ - reintroduce mfai 8.0.2 - bump lightning from 2.6.5 to 2.6.6 (fix high CVE-2026-58659) - bump torch from 2.13.0 to 2.14.0 and triton from 3.7.1 to 3.8.0 (#327) +- bump hydra-core from 1.3.5 to 1.3.7 (#328) From 0e2f9c0b54bb3a2aca961fc21be5b298d0279cd5 Mon Sep 17 00:00:00 2001 From: thebaptiste Date: Fri, 25 Sep 2026 14:56:27 +0000 Subject: [PATCH 33/38] feat: bump grpcio from 1.76.0 to 1.84.0 (compat. Python 3.15) --- .metwork-framework/components.md | 2 +- .../layer5_python3_ia/0500_extra_packages/allow_binary_packages | 1 - layers/layer5_python3_ia/0500_extra_packages/requirements3.txt | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.metwork-framework/components.md b/.metwork-framework/components.md index fc107a4..54df949 100644 --- a/.metwork-framework/components.md +++ b/.metwork-framework/components.md @@ -16,7 +16,7 @@ | [google-auth-oauthlib](https://github.com/GoogleCloudPlatform/google-auth-library-python-oauthlib) | 1.2.2 | python3_ia | | [google-auth](https://github.com/googleapis/google-auth-library-python) | 2.53.0 | python3_ia | | [google-pasta](https://github.com/google/pasta) | 0.2.0 | python3_ia | -| [grpcio](https://grpc.io) | 1.76.0 | python3_ia | +| [grpcio](https://grpc.io) | 1.84.0 | python3_ia | | [hf-xet](https://github.com/huggingface/xet-core) | 1.4.3 | python3_ia | | [huggingface_hub](https://github.com/huggingface/huggingface_hub) | 1.5.0 | python3_ia | | [hydra-core](https://github.com/facebookresearch/hydra) | 1.3.7 | python3_ia | diff --git a/layers/layer5_python3_ia/0500_extra_packages/allow_binary_packages b/layers/layer5_python3_ia/0500_extra_packages/allow_binary_packages index 869ad3d..d387561 100644 --- a/layers/layer5_python3_ia/0500_extra_packages/allow_binary_packages +++ b/layers/layer5_python3_ia/0500_extra_packages/allow_binary_packages @@ -42,7 +42,6 @@ opt-einsum optree protobuf pyarrow -PyYAML requests-oauthlib rich rsa diff --git a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt index c11c064..68cee12 100644 --- a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt +++ b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt @@ -14,7 +14,7 @@ flatbuffers==25.12.19 google-auth==2.53.0 google-auth-oauthlib==1.2.2 google-pasta==0.2.0 -grpcio==1.76.0 +grpcio==1.84.0 hf_xet==1.4.3 huggingface_hub==1.5.0 hydra-core==1.3.7 From bd470da997035d1b1e49b526520f3cf1e5cf38b4 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste VESLIN <19856429+thebaptiste@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:11:23 +0200 Subject: [PATCH 34/38] feat: bump grpcio from 1.76.0 to 1.84.0 (compat. Python 3.15) (#330) --- .metwork-framework/components.md | 2 +- layers/layer5_python3_ia/0500_extra_packages/requirements3.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.metwork-framework/components.md b/.metwork-framework/components.md index 54df949..75f2ccb 100644 --- a/.metwork-framework/components.md +++ b/.metwork-framework/components.md @@ -55,7 +55,7 @@ | [onnxruntime](https://onnxruntime.ai) | 1.29.0 | python3_ia | | [onnxscript](https://microsoft.github.io/onnxscript/) | 0.2.5 | python3_ia | | [opt_einsum](https://pypi.org/project/opt_einsum) | 3.4.0 | python3_ia | -| [optree](https://github.com/metaopt/optree) | 0.18.0 | python3_ia | +| [optree](https://github.com/metaopt/optree) | 0.20.0 | python3_ia | | [pytorch-lightning](https://github.com/Lightning-AI/lightning) | 2.6.6 | python3_ia | | [requests-oauthlib](https://github.com/requests/requests-oauthlib) | 2.0.0 | python3_ia | | [rsa](https://stuvel.eu/rsa) | 4.9 | python3_ia | diff --git a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt index 68cee12..1d278d4 100644 --- a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt +++ b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt @@ -53,7 +53,7 @@ onnx==1.22.0 onnxruntime==1.29.0 onnxscript==0.2.5 opt-einsum==3.4.0 -optree==0.18.0 +optree==0.20.0 pytorch-lightning==2.6.6 requests-oauthlib==2.0.0 rsa==4.9 From 3a9b816ca20b8221e975cdefb04e51b6e45c7ccd Mon Sep 17 00:00:00 2001 From: thebaptiste Date: Fri, 25 Sep 2026 15:17:58 +0000 Subject: [PATCH 35/38] feat: bump tiktoken from 0.12.0 to 0.14.0 (compat. Python 3.15) --- .metwork-framework/components.md | 2 +- layers/layer5_python3_ia/0500_extra_packages/requirements3.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.metwork-framework/components.md b/.metwork-framework/components.md index 75f2ccb..44a0561 100644 --- a/.metwork-framework/components.md +++ b/.metwork-framework/components.md @@ -67,7 +67,7 @@ | [tensorboard-data-server](https://github.com/tensorflow/tensorboard/tree/master/tensorboard/data/server) | 0.7.2 | python3_ia | | [tensorboard](https://github.com/tensorflow/tensorboard) | 2.20.0 | python3_ia | | [tensorboardX](https://github.com/lanpa/tensorboardX) | 2.6.5 | python3_ia | -| [tiktoken](https://github.com/openai/tiktoken) | 0.12.0 | python3_ia | +| [tiktoken](https://github.com/openai/tiktoken) | 0.14.0 | python3_ia | | [timm](https://github.com/huggingface/pytorch-image-models) | 1.0.22 | python3_ia | | [tokenizers](https://github.com/huggingface/tokenizers) | 0.23.1 | python3_ia | | [torch-geometric](https://pyg.org) | 2.7.0 | python3_ia | diff --git a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt index 1d278d4..d53780b 100644 --- a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt +++ b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt @@ -65,7 +65,7 @@ tabulate==0.9.0 tensorboard==2.20.0 tensorboard-data-server==0.7.2 tensorboardx==2.6.5 -tiktoken==0.12.0 +tiktoken==0.14.0 timm==1.0.22 tokenizers==0.23.1 torch==2.14.0 From 5eff172ef85650932d6080787d4b05a5d9aad72b Mon Sep 17 00:00:00 2001 From: metworkbot Date: Sat, 26 Sep 2026 07:19:30 +0000 Subject: [PATCH 36/38] build: changelog automatic update --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab5e682..1e6c7ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,5 +11,8 @@ - bump lightning from 2.6.5 to 2.6.6 (fix high CVE-2026-58659) - bump torch from 2.13.0 to 2.14.0 and triton from 3.7.1 to 3.8.0 (#327) - bump hydra-core from 1.3.5 to 1.3.7 (#328) +- bump grpcio from 1.76.0 to 1.84.0 (compat. Python 3.15) +- bump grpcio from 1.76.0 to 1.84.0 (compat. Python 3.15) (#330) +- bump tiktoken from 0.12.0 to 0.14.0 (compat. Python 3.15) From b3cb02ef85ea68dbfb11f5789177c229f7333ed5 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste VESLIN <19856429+thebaptiste@users.noreply.github.com> Date: Thu, 1 Oct 2026 17:26:03 +0200 Subject: [PATCH 37/38] feat: bump oauthlib from 3.2.2 to 4.0.0 (fix 2 moderate CVE) (#333) CVE fixed : - CVE-2026-49264 (moderate) - CVE-2026-49265 (moderate) --- .metwork-framework/components.md | 2 +- layers/layer5_python3_ia/0500_extra_packages/requirements3.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.metwork-framework/components.md b/.metwork-framework/components.md index 44a0561..d02af2f 100644 --- a/.metwork-framework/components.md +++ b/.metwork-framework/components.md @@ -49,7 +49,7 @@ | [nvidia-nvjitlink](https://developer.nvidia.com/cuda-zone) | 13.0.88 | python3_ia | | [nvidia-nvshmem-cu13](https://developer.nvidia.com/cuda-zone) | 3.4.5 | python3_ia | | [nvidia-nvtx](https://developer.nvidia.com/cuda-zone) | 13.0.85 | python3_ia | -| [oauthlib](https://github.com/oauthlib/oauthlib) | 3.2.2 | python3_ia | +| [oauthlib](https://github.com/oauthlib/oauthlib) | 4.0.0 | python3_ia | | [omegaconf](https://github.com/omry/omegaconf) | 2.3.1 | python3_ia | | [onnx](https://onnx.ai/) | 1.22.0 | python3_ia | | [onnxruntime](https://onnxruntime.ai) | 1.29.0 | python3_ia | diff --git a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt index d53780b..e328ceb 100644 --- a/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt +++ b/layers/layer5_python3_ia/0500_extra_packages/requirements3.txt @@ -47,7 +47,7 @@ nvidia_nvjitlink==13.0.88 nvidia-nccl-cu13==2.30.7 nvidia-nvshmem-cu13==3.4.5 nvidia-nvtx==13.0.85 -oauthlib==3.2.2 +oauthlib==4.0.0 omegaconf==2.3.1 onnx==1.22.0 onnxruntime==1.29.0 From f59024182faf14e2d1aed3b2b8978b8e98be7fe5 Mon Sep 17 00:00:00 2001 From: metworkbot Date: Fri, 2 Oct 2026 08:11:49 +0000 Subject: [PATCH 38/38] build: changelog automatic update --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e6c7ab..78fe47f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,5 +14,6 @@ - bump grpcio from 1.76.0 to 1.84.0 (compat. Python 3.15) - bump grpcio from 1.76.0 to 1.84.0 (compat. Python 3.15) (#330) - bump tiktoken from 0.12.0 to 0.14.0 (compat. Python 3.15) +- bump oauthlib from 3.2.2 to 4.0.0 (fix 2 moderate CVE) (#333)