diff --git a/.github/workflows/api-docs.yml b/.github/workflows/api-docs.yml index 3314c31d..0db2b505 100644 --- a/.github/workflows/api-docs.yml +++ b/.github/workflows/api-docs.yml @@ -60,6 +60,15 @@ jobs: run: | # Install package itself to install the samples datasets python -m pip install --upgrade pip + # Add homogeneous TOML support (Python >= 3.12 has standard tomllib) + python -m pip install tomli + python scripts/extract_dependencies_from_pyproject_toml.py -f "pyproject.toml" > requires.txt + # First, install all dependencies except khiops-core and khiops-drivers-* + python -m pip install --user `perl -pe "s/khiops-\S+//g" requires.txt` + # khiops-core and khiops-drivers-* must always be installed from TestPyPI in order to avoid distorting usage statistics + python -m pip install --user --index-url https://test.pypi.org/simple `grep -oE "khiops-\S+" requires.txt | paste -sd ' ' -` + rm -f requires.txt + # Lastly, install khiops-python python -m pip install --user . kh-download-datasets --force-overwrite --version ${{ inputs.khiops-samples-revision || env.DEFAULT_KHIOPS_SAMPLES_REVISION }} kh-status diff --git a/.github/workflows/dev-docker.yml b/.github/workflows/dev-docker.yml index b5d2f796..010616e2 100644 --- a/.github/workflows/dev-docker.yml +++ b/.github/workflows/dev-docker.yml @@ -5,9 +5,6 @@ env: DEFAULT_IMAGE_INCREMENT: 0 DEFAULT_SERVER_REVISION: main DEFAULT_PYTHON_VERSIONS: 3.10 3.11 3.12 3.13 3.14 - DEFAULT_KHIOPS_GCS_DRIVER_REVISION: 0.0.16 - DEFAULT_KHIOPS_S3_DRIVER_REVISION: 0.0.15 - DEFAULT_KHIOPS_AZURE_DRIVER_REVISION: 0.0.6 # XXX : to modify soon on: pull_request: paths: [packaging/docker/khiopspydev/Dockerfile.*, .github/workflows/dev-docker.yml] @@ -16,7 +13,7 @@ on: khiops-revision: type: string default: 11.0.0 - description: Khiops Revision + description: Khiops Revision (for tests against KhiopsDockerRunner) image-increment: type: number default: 0 @@ -37,18 +34,6 @@ on: type: string default: main description: Khiops Server Revision - khiops-gcs-driver-revision: - type: string - default: 0.0.16 - description: Driver version for Google Cloud Storage remote files - khiops-s3-driver-revision: - type: string - default: 0.0.15 - description: Driver version for AWS-S3 remote files - khiops-azure-driver-revision: - type: string - default: 0.0.6 # XXX : to modify soon - description: Driver version for Azure remote files and blobs concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -63,16 +48,28 @@ jobs: packages: write # to write in the Github package registry steps: - name: Set input parameters as env or output + shell: bash run: | set -x echo "KHIOPS_REVISION=${{ inputs.khiops-revision || env.DEFAULT_KHIOPS_REVISION }}" >> "$GITHUB_ENV" echo "IMAGE_INCREMENT=${{ inputs.image-increment || env.DEFAULT_IMAGE_INCREMENT }}" >> "$GITHUB_ENV" - echo "KHIOPSDEV_OS_CODENAME=$(echo '${{ matrix.khiopsdev-os }}' | tr -d '0-9.')" >> "$GITHUB_ENV" + KHIOPSDEV_OS_CODENAME=$(echo '${{ matrix.khiopsdev-os }}' | tr -d '0-9.') + echo "KHIOPSDEV_OS_CODENAME=${KHIOPSDEV_OS_CODENAME}" >> "$GITHUB_ENV" + case ${KHIOPSDEV_OS_CODENAME} in + ubuntu | debian) + # same Dockerfile for the whole family + echo "DOCKER_FILE_NAME=Dockerfile.ubuntu-debian" >> "$GITHUB_ENV" + ;; + rocky) + echo "DOCKER_FILE_NAME=Dockerfile.rocky" >> "$GITHUB_ENV" + ;; + *) + echo "::error::Status error: '${KHIOPSDEV_OS_CODENAME}' is an unexpected OS codename." + exit 1 + ;; + esac echo "SERVER_REVISION=${{ inputs.server-revision || env.DEFAULT_SERVER_REVISION }}" >> "$GITHUB_ENV" echo "IMAGE_URL=ghcr.io/khiopsml/khiops-python/khiopspydev-${{ matrix.khiopsdev-os }}" >> "$GITHUB_ENV" - echo "KHIOPS_GCS_DRIVER_REVISION=${{ inputs.khiops-gcs-driver-revision || env.DEFAULT_KHIOPS_GCS_DRIVER_REVISION }}" >> "$GITHUB_ENV" - echo "KHIOPS_S3_DRIVER_REVISION=${{ inputs.khiops-s3-driver-revision || env.DEFAULT_KHIOPS_S3_DRIVER_REVISION }}" >> "$GITHUB_ENV" - echo "KHIOPS_AZURE_DRIVER_REVISION=${{ inputs.khiops-azure-driver-revision || env.DEFAULT_KHIOPS_AZURE_DRIVER_REVISION }}" >> "$GITHUB_ENV" - name: Checkout khiops-python sources uses: actions/checkout@v4 - name: Set up Docker Buildx @@ -103,15 +100,12 @@ jobs: # added using inputs because /etc/hosts is read-only for alternate builders (buildx via moby buildkit) add-hosts: s3-bucket.localhost:127.0.0.1 context: ./packaging/docker/khiopspydev/ - file: ./packaging/docker/khiopspydev/Dockerfile.${{ env.KHIOPSDEV_OS_CODENAME }} + file: ./packaging/docker/khiopspydev/${{ env.DOCKER_FILE_NAME }} build-args: | "KHIOPS_REVISION=${{ env.KHIOPS_REVISION }}" "KHIOPSDEV_OS=${{ matrix.khiopsdev-os }}" "SERVER_REVISION=${{ env.SERVER_REVISION }}" "PYTHON_VERSIONS=${{ inputs.python-versions || env.DEFAULT_PYTHON_VERSIONS }}" - "KHIOPS_GCS_DRIVER_REVISION=${{ env.KHIOPS_GCS_DRIVER_REVISION }}" - "KHIOPS_S3_DRIVER_REVISION=${{ env.KHIOPS_S3_DRIVER_REVISION }}" - "KHIOPS_AZURE_DRIVER_REVISION=${{ env.KHIOPS_AZURE_DRIVER_REVISION }}" tags: ${{ env.DOCKER_IMAGE_TAGS }} # Push only on manual request push: ${{ inputs.push || false }} diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index a76758f6..92ec5ca9 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -77,23 +77,23 @@ jobs: run: | SAMPLES_REVISION=${{ inputs.samples-revision || env.DEFAULT_SAMPLES_REVISION }} echo "SAMPLES_REVISION=$SAMPLES_REVISION" >> "$GITHUB_ENV" - - name: Checkout Khiops samples + - name: Checkout sources # Checking out the sources is mandatory to be able to extract the dependencies list uses: actions/checkout@v4 with: - repository: khiopsml/khiops-samples - ref: ${{ env.SAMPLES_REVISION }} - token: ${{ secrets.GITHUB_TOKEN }} - path: khiops-samples + # Get Git tags so that versioneer can function correctly + # See issue https://github.com/actions/checkout/issues/701 + fetch-depth: 0 - name: Download package artifact uses: actions/download-artifact@v4 with: name: pip-package - - name: Install package + - name: Install the package and download the samples shell: bash run: | # Allow Pip to write to its cache mkdir -p /github/home/.cache/pip chown -R $(whoami) /github/home/.cache/pip + # Install the Khiops Python library # A virtual env is mandatory under debian @@ -102,10 +102,23 @@ jobs: source khiops-debian-venv/bin/activate fi pip install --upgrade pip + # Add homogeneous TOML support (Python >= 3.12 has standard tomllib) + pip install tomli + python scripts/extract_dependencies_from_pyproject_toml.py -f "pyproject.toml" > requires.txt + # First, install all dependencies except khiops-core and khiops-drivers-* + pip install `perl -pe "s/khiops-\S+//g" requires.txt` + # khiops-core and khiops-drivers-* must always be installed from TestPyPI in order to avoid distorting usage statistics + pip install --index-url https://test.pypi.org/simple `grep -oE "khiops-\S+" requires.txt | paste -sd ' ' -` + rm -f requires.txt + # Lastly, install khiops-python pip install $(ls khiops*.tar.gz) if [[ "${{ matrix.container }}" == "debian13" ]]; then deactivate fi + # Download the samples to `khiops-samples` instead of the default location used by `kh-download-datasets` + wget -O khiops_samples.zip "https://github.com/KhiopsML/khiops-samples/releases/download/${{ env.SAMPLES_REVISION }}/khiops-samples-${{ env.SAMPLES_REVISION }}.zip" + mkdir -p ./khiops-samples && unzip khiops_samples.zip -d ./khiops-samples + rm -f khiops_samples.zip - name: Run tests env: KHIOPS_SAMPLES_DIR: ${{ github.workspace }}/khiops-samples diff --git a/.github/workflows/quick-checks.yml b/.github/workflows/quick-checks.yml index 9d3c9485..a671f21f 100644 --- a/.github/workflows/quick-checks.yml +++ b/.github/workflows/quick-checks.yml @@ -19,11 +19,14 @@ jobs: - name: Install khiops-python dev dependencies run: | # Extract and install package dependency requirements from metadata - pip install pip-tools - python -m piptools compile -o requirements.txt - - # Install dev dependencies - pip install -r requirements.txt + # Add homogeneous TOML support (Python >= 3.12 has standard tomllib) + pip install tomli + python scripts/extract_dependencies_from_pyproject_toml.py -f "pyproject.toml" > requires.txt + # First, install all dependencies except khiops-core and khiops-drivers-* + pip install `perl -pe "s/khiops-\S+//g" requires.txt` + # khiops-core and khiops-drivers-* must always be installed from TestPyPI in order to avoid distorting usage statistics + pip install --index-url https://test.pypi.org/simple `grep -oE "khiops-\S+" requires.txt | paste -sd ' ' -` + rm -f requires.txt # Install black for the samples-generation script pip install black diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bee58ee6..b0e49be4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -97,7 +97,12 @@ jobs: # Add homogeneous TOML support (Python >= 3.12 has standard tomllib) $CONDA install -y -n "$CONDA_ENV" tomli $CONDA run --no-capture-output -n "$CONDA_ENV" python scripts/extract_dependencies_from_pyproject_toml.py -f "pyproject.toml" > requires.txt - $CONDA install -y -n "$CONDA_ENV" `cat requires.txt` + # Conda is currently used to simulate multiple Python environment, let's avoid installing Conda Packages at all but Pip ones instead + $CONDA install -y -n "$CONDA_ENV" pip # Required otherwise the system pip would be run which will install outside the Conda env + # First, install all dependencies except khiops-core and khiops-drivers-* + $CONDA run -n "$CONDA_ENV" pip install `perl -pe "s/khiops-\S+//g" requires.txt` + # khiops-core and khiops-drivers-* must always be installed from TestPyPI in order to avoid distorting usage statistics + $CONDA run -n "$CONDA_ENV" pip install --index-url https://test.pypi.org/simple `grep -oE "khiops-\S+" requires.txt | paste -sd ' ' -` rm -f requires.txt - name: Configure Expensive Tests Setting # Skip expensive tests by default, unless on the `main-v10` or `main` branches @@ -267,8 +272,10 @@ jobs: python scripts/extract_dependencies_from_pyproject_toml.py -f "pyproject.toml" -s "\n" > requires.txt # Install the Python requirements outside a python venv + # khiops-core and khiops-drivers-* must always be installed from TestPyPI in order to avoid distorting usage statistics + # Use of both 'index-url' and '--extra-index-url' options allows indexes prioritization (PEP 766) Get-Content .\requires.txt ` - | ForEach-Object {python -m pip install $_.toString()} + | ForEach-Object {python -m pip install --index-url https://test.pypi.org/simple --extra-index-url https://pypi.org/simple $_.toString()} # Create and activate a python venv python -m venv khiops-windows-venv @@ -276,8 +283,9 @@ jobs: # Install the Python requirements inside a venv # The venv python executable is used here + # Same indexes prioritization as above Get-Content .\requires.txt ` - | ForEach-Object {khiops-windows-venv\Scripts\python -m pip install $_.toString()} + | ForEach-Object {khiops-windows-venv\Scripts\python -m pip install --index-url https://test.pypi.org/simple --extra-index-url https://pypi.org/simple $_.toString()} # Deactivate the python venv deactivate @@ -392,7 +400,10 @@ jobs: # Install tomli for Python < 3.11 pip install tomli python scripts/extract_dependencies_from_pyproject_toml.py -f "pyproject.toml" > requires.txt - pip install `cat requires.txt` + # First, install all dependencies except khiops-core and khiops-drivers-* + pip install `perl -pe "s/khiops-\S+//g" requires.txt` + # khiops-core and khiops-drivers-* must always be installed from TestPyPI in order to avoid distorting usage statistics + pip install --index-url https://test.pypi.org/simple `grep -oE "khiops-\S+" requires.txt | paste -sd ' ' -` rm -f requires.txt if [[ "${{ matrix.container }}" == "debian13" ]]; then deactivate @@ -447,20 +458,6 @@ jobs: echo "::error::Status error: improper setup, as expected: khiops-python has been cloned, not installed from a package" fi - # Run the library against an incompatible Khiops (with a different major version) - # This instance of Khiops is isolated in a dedicated conda environment - CONDA="/root/miniforge3/bin/conda" - # Check an error is raised because of the major version mismatch - # The khiops-python library from the cloned sources is used here - PATTERN=$($CONDA run -n py3_khiops10_conda python -c "import khiops.core as kh; print(kh.get_runner().khiops_version)" 2> >(grep -Ei 'major version.*?does not match')) - if [ -z "$PATTERN" ]; then - echo "::error::Status error: khiops-python should fail because of the major version mismatch" - if [[ "${{ matrix.container }}" == "debian13" ]]; then - deactivate - fi - exit 1; - fi - # Run the remaining integration tests python -m unittest -v tests.test_khiops_integrations diff --git a/CHANGELOG.md b/CHANGELOG.md index fa18a324..d5bc9efd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,8 +13,12 @@ - (General) Support for Azure storage ### Changed +- (General) Full-Pip installation support: `khiops` now depends on the `khiops-core` and optionally on the remote storage driver packages - (`core`) Rename `variable_part_dimensions` to `inner_variable_dimensions` in Coclustering results. +### Removed +- (General) Support of the installation type combining "OS Native Khiops Core" and "Pip Khiops Python library" + ## 11.0.0.3 - 2026-03-06 ### Added diff --git a/khiops/__init__.py b/khiops/__init__.py index 74dbbce6..269ecf4b 100644 --- a/khiops/__init__.py +++ b/khiops/__init__.py @@ -22,10 +22,27 @@ (extension ".khcj") - sklearn: Scikit-Learn estimator classes to learn and use Khiops models """ +from importlib.metadata import PackageNotFoundError, distribution + +from khiops.core import KhiopsEnvironmentError from khiops.core.internals.version import KhiopsVersion __version__ = "11.0.0.3" +# The current Khiops Python library (when packaged as a PyPI package) +# depends on the Khiops binary-only PyPI package +# An error must be raised if this dependency disappear +try: + # FIXME: will always raise an error when : + # the installation method is 'conda' or 'conda-based' + # and a Conda `khiops-core` package was installed not a PyPI package one + # distribution("khiops-core") + pass +except PackageNotFoundError as exc: + raise KhiopsEnvironmentError( + f"The Khiops binaries are not installed properly: " f"{exc}" + ) from exc + def get_compatible_khiops_version(): """Returns the latest Khiops version compatible with this package's version""" diff --git a/khiops/core/internals/filesystems.py b/khiops/core/internals/filesystems.py index f63d20ab..a827116d 100644 --- a/khiops/core/internals/filesystems.py +++ b/khiops/core/internals/filesystems.py @@ -7,6 +7,7 @@ """Classes to interact with local and remote filesystems""" import json import os +import platform import shutil import warnings from abc import ABC, abstractmethod @@ -116,8 +117,8 @@ def _child_uri_info(uri_info, child_name): uri_info : `urllib.parse.ParseResult` URI info structure (output of `urllib.parse.urlparse`) - child_name : str - Name of the new childe node + child_name : `str` + Name of the new child node Returns ------- @@ -127,6 +128,45 @@ def _child_uri_info(uri_info, child_name): return uri_info._replace(path=_child_path(uri_info.path, child_name)) +def _check_khiops_driver_library(remote_storage_type, khiops_drivers_path=None): + """Ensures the dynamic library for the Khiops driver exists + + Parameters + ---------- + remote_storage_type : `str` + Type of the supported storage type + + khiops_drivers_path : `str` + Absolute path to the folder containing all the drivers libraries. + If a relative path (without the leading '/') is passed, + an absolute path is inferred using the current folder which will probably + cause an error. + + this method raises a `RuntimeError` if the dynamic library is not found + """ + assert type(remote_storage_type) == str + assert remote_storage_type in ("s3", "gcs", "azure") + assert type(khiops_drivers_path) == str + + match platform.system(): + case "Windows": + extension = ".dll" + case "Darwin": # MacOS + extension = ".dylib" + case _: # any Linux platform + extension = ".so" + absolute_path = os.path.join( + os.path.abspath(khiops_drivers_path), + f"libkhiopsdriver_file_{remote_storage_type}{extension}", + ) + if not os.path.exists(absolute_path): + raise RuntimeError( + f"Khiops driver for '{remote_storage_type}' " + f"is missing (expected in {absolute_path}). " + "Make sure you installed it." + ) + + ###################### ## Helper Functions ## ###################### @@ -538,6 +578,11 @@ class GoogleCloudStorageResource(FilesystemResource): """ def __init__(self, uri): + + khiops_drivers_path = os.environ.get("KHIOPS_DRIVERS_PATH") + if khiops_drivers_path: + _check_khiops_driver_library("gcs", khiops_drivers_path=khiops_drivers_path) + # Stop initialization if google.cloud module is not available if gcs_import_error is not None: warnings.warn( @@ -635,11 +680,16 @@ class AmazonS3Resource(FilesystemResource): """ def __init__(self, uri): + + khiops_drivers_path = os.environ.get("KHIOPS_DRIVERS_PATH") + if khiops_drivers_path: + _check_khiops_driver_library("s3", khiops_drivers_path=khiops_drivers_path) + # Stop initialization if boto3 could not be imported if boto3_import_error is not None: warnings.warn( "Could not import boto3 python library, " - "make sure you it installed to access S3 files." + "make sure you installed it to access S3 files." ) raise boto3_import_error @@ -778,6 +828,12 @@ def __init__(self, uri): Azure Storage Resource initializer common to Files and Blobs """ + khiops_drivers_path = os.environ.get("KHIOPS_DRIVERS_PATH") + if khiops_drivers_path: + _check_khiops_driver_library( + "azure", khiops_drivers_path=khiops_drivers_path + ) + # Stop initialization if Azure modules are not available if azure_import_error is not None: warnings.warn( diff --git a/khiops/core/internals/runner.py b/khiops/core/internals/runner.py index 46f1fb69..279fe58f 100644 --- a/khiops/core/internals/runner.py +++ b/khiops/core/internals/runner.py @@ -243,11 +243,11 @@ def _infer_khiops_installation_method(trace=False): it was not activated previously nor during the execution and thus the CONDA_PREFIX environment variable is undefined and the path to the `bin` directory inside the conda environment is not in PATH - - 'binary+pip' installs the binaries and the shared libraries system-wide - but will keep the python libraries - in the python system folder - or in the Python folder inside the home directory of the user, - or in a virtual environment (if one is used) + - 'pip' environment containing binaries, shared libraries and the python libraries + can either be : + - system-wide (strongly discouraged) + - or in the Python folder inside the home directory of the user (aka "User site"), + - or in a classical virtual environment (highly encouraged) """ # We are in a Conda environment if @@ -277,10 +277,10 @@ def _infer_khiops_installation_method(trace=False): ): installation_method = "conda-based" else: - installation_method = "binary+pip" + installation_method = "pip" if trace: print(f"Installation method: '{installation_method}'") - assert installation_method in ("conda", "conda-based", "binary+pip") + assert installation_method in ("conda", "conda-based", "pip") return installation_method @@ -953,24 +953,23 @@ def __init__(self): self._initialize_khiops_environment() def _initialize_khiops_environment(self): - # Check the `khiops_env` script - # On Windows native installations, rely on the `KHIOPS_HOME` environment - # variable set by the Khiops Desktop Application installer + # Search for the `khiops_env` script location installation_method = _infer_khiops_installation_method() - if platform.system() == "Windows" and installation_method == "binary+pip": - # KHIOPS_HOME variable by default - if "KHIOPS_HOME" in os.environ: - khiops_env_path = os.path.join( - os.environ["KHIOPS_HOME"], "bin", "khiops_env.cmd" - ) - # Raise error if KHIOPS_HOME is not set + if platform.system() == "Windows" and installation_method == "pip": + sys_executable_direct_parent = Path(sys.executable).parents[0] + probable_khiops_env = os.path.join( + sys_executable_direct_parent, "khiops_env.cmd" + ) + # The script is found in the current environment + if os.path.exists(probable_khiops_env): + khiops_env_path = probable_khiops_env + # Raise error otherwise else: raise KhiopsEnvironmentError( - "No environment variable named 'KHIOPS_HOME' found. " - "Make sure you have installed Khiops >= 10.2.3. " + "No 'khiops_env.cmd' found in the current environment." + "Make sure you have installed properly the Khiops Python library. " "Go to https://khiops.org for more information." ) - # In Conda-based environments, `khiops_env` might not be in the PATH, # hence its path must be inferred elif installation_method == "conda-based": @@ -987,7 +986,8 @@ def _initialize_khiops_environment(self): raise KhiopsEnvironmentError( "The 'khiops_env' script not found for the current " f"'{installation_method}' installation method. Make sure " - "you have installed khiops >= 10.2.3. " + "you have installed properly the Khiops Python library " + "and Khiops (if this later was not installed automatically). " "Go to https://khiops.org for more information." ) @@ -1029,6 +1029,13 @@ def _initialize_khiops_environment(self): elif var_name == "KHIOPS_MPI_COMMAND": self._mpi_command_args = shlex.split(var_value) os.environ["KHIOPS_MPI_COMMAND"] = var_value + # On Windows "KHIOPS_MPI_DLL_PATH" (containing the Intel MPI Library) + # must be appended to "PATH" otherwise Khiops wouldn't find it + # and fail immediately + elif var_name == "KHIOPS_MPI_DLL_PATH": + os.environ["PATH"] = os.pathsep.join( + [var_value, os.environ.get("PATH")] + ) # Propagate all the other environment variables to Khiops binaries else: os.environ[var_name] = var_value @@ -1088,6 +1095,8 @@ def _initialize_khiops_version(self): installation_method = _infer_khiops_installation_method() # Fail immediately if the major versions differ + # It can still occur if the installation method is 'conda' or 'conda-based', + # but never if the installation method is 'pip' (as a dependency is specified) # Note: the installation status will not show at all if self.khiops_version.major != compatible_khiops_version.major: raise KhiopsRuntimeError( @@ -1130,7 +1139,7 @@ def _initialize_khiops_version(self): def _detect_library_installation_incompatibilities(self, library_root_dir_path): """Detects known incompatible installations of this library in the 3 installation modes see `_infer_khiops_installation_method` - (binary+pip, conda, conda-based) + (pip, conda, conda-based) The error_list or warning_list collections are not empty if an issue is detected @@ -1199,7 +1208,7 @@ def _detect_library_installation_incompatibilities(self, library_root_dir_path): "Go to https://khiops.org for instructions.\n" ) error_list.append(error) - # 'binary+pip', 'conda-based' or borderline installations + # 'pip', 'conda-based' or borderline installations else: # ensure a known installer was used otherwise unexpected issues can occur @@ -1220,11 +1229,11 @@ def _detect_library_installation_incompatibilities(self, library_root_dir_path): ) warning_list.append(warning) - # we consider only the 'binary+pip' and 'conda-based' installations here + # we consider only the 'pip' and 'conda-based' installations here # - 'conda-based' installation (similar to a non-activated virtual env) - # - 'binary+pip' installation under a virtual env - # - User site 'binary+pip' installation (without virtual env) - # - system-wide 'binary+pip' installation (without virtual env)... + # - 'pip' installation under a virtual env + # - User site 'pip' installation (without virtual env) + # - system-wide 'pip' installation (without virtual env)... # (an empty string means a borderline installation was found, # no further check cannot be performed) base_dir = _infer_base_dir_for_conda_based_or_pip_installations() @@ -1252,7 +1261,7 @@ def _detect_library_installation_incompatibilities(self, library_root_dir_path): # for conda-based installations python is inside 'base_dir' sys_executable_direct_parent != base_dir_path and - # for 'binary+pip' installations (within a virtual env) + # for 'pip' installations (within a virtual env) # python is inside 'base_dir'/Scripts sys_executable_grand_parent != base_dir_path ) diff --git a/khiops/tools.py b/khiops/tools.py index 1359a35a..efcd9d7d 100644 --- a/khiops/tools.py +++ b/khiops/tools.py @@ -156,9 +156,10 @@ def download_datasets( # Download the sample zip file and extracted to the home dataset dir print(f"Downloading samples from {samples_zip_url}") - with tempfile.NamedTemporaryFile() as temp_zip_file, urllib.request.urlopen( - samples_zip_url - ) as zip_request: + with ( + tempfile.NamedTemporaryFile() as temp_zip_file, + urllib.request.urlopen(samples_zip_url) as zip_request, + ): temp_zip_file.write(zip_request.read()) temp_zip_file.seek(0) with zipfile.ZipFile(temp_zip_file) as temp_zip: diff --git a/packaging/docker/khiopspydev/Dockerfile.debian b/packaging/docker/khiopspydev/Dockerfile.debian deleted file mode 100644 index f16a0142..00000000 --- a/packaging/docker/khiopspydev/Dockerfile.debian +++ /dev/null @@ -1,108 +0,0 @@ -# TODO : avoid duplication in trying to use the same docker file for debian and ubuntu -# 2 distinct files exist now because the releases for debian 12 are used for the remote drivers -# Arguments -ARG KHIOPSDEV_OS -ARG SERVER_REVISION -FROM ghcr.io/khiopsml/khiops/khiopsdev-${KHIOPSDEV_OS}:latest AS khiopsdev -LABEL maintainer="khiops.team@orange.com" -LABEL description="Container for the development of khiops-python" - -# Install dev tools and miniforge (for the unit tests); build and install Khiops -ARG KHIOPS_REVISION -RUN true \ - # Install git (for khiops-python version calculation) and pip \ - && apt-get -y update \ - && apt-get -y --no-install-recommends install git python3-pip \ - # On Debian/Ubuntu systems, python3-venv must be installed to provide ensurepip. - # It is still possible to create venvs using the --without-pip flag but any subsequent - # module installation with pip would be impossible - python3-venv \ - zip pandoc wget ruby-dev \ - # Get Linux distribution codename \ - && if [ -f /etc/os-release ]; then . /etc/os-release; fi \ - # Obtain the Khiops native package \ - && KHIOPS_PKG_FILE=$KHIOPS_REVISION/khiops-core-openmpi_$KHIOPS_REVISION-1-$VERSION_CODENAME.amd64.deb \ - && wget -O KHIOPS_CORE.deb "https://github.com/KhiopsML/khiops/releases/download/${KHIOPS_PKG_FILE}" \ - # Install the Khiops native package : make it always succeed. \ - # If dpkg fails it is due to missing dependencies which will be installed by apt in the next line \ - && (dpkg -i --force-all KHIOPS_CORE.deb || true) \ - && apt-get -f -y install \ - && rm -f KHIOPS_CORE.deb \ - # Set python to python3 \ - && update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \ - # Install miniforge to have multiple Python versions via Conda \ - && mkdir -p /root/miniforge3 && cd /root/miniforge3 \ - && wget https://github.com/conda-forge/miniforge/releases/download/25.9.1-0/Miniforge3-25.9.1-0-Linux-x86_64.sh -O ./Miniforge3_25.9.1-0-Linux-x86_64.sh \ - && echo "07f64c1d908ae036e9f6a81f97704899311c0ae677d83980d664b9781d4cc5fc Miniforge3_25.9.1-0-Linux-x86_64.sh" | sha256sum --check \ - && bash ./Miniforge3_25.9.1-0-Linux-x86_64.sh -b -u -p /root/miniforge3 \ - && rm -rf /root/miniforge3/Miniforge3_25.9.1-0-Linux-x86_64.sh \ - # Make sure that MPI is openmpi \ - && update-alternatives --set mpirun /usr/bin/mpirun.openmpi \ - # Clean build files \ - && rm -fr /var/lib/apt/lists/* \ - && apt-get clean \ - && rm -rf ./khiops \ - && true - -# set up all the supported Python environments under Conda (for the unit tests) -# relying on a variable containing all the versions -ARG PYTHON_VERSIONS -ARG KHIOPS_GCS_DRIVER_REVISION -ARG KHIOPS_S3_DRIVER_REVISION -ARG KHIOPS_AZURE_DRIVER_REVISION - -# Initialize all the Conda environments -RUN true \ - && export CONDA="/root/miniforge3/bin/conda" \ - && /bin/bash -c ' \ - # Update conda to the latest version \ - $CONDA update -n base conda; \ - for version in ${PYTHON_VERSIONS}; \ - do \ - $CONDA create -y -n py${version} python=${version}; \ - done; \ - # Install Khiops from a different major version in a dedicated conda environment \ - # for a specific test regarding compatibility test \ - # The python interpreter version of the base environment is used as no specific version is given \ - $CONDA create -y -n py3_khiops10_conda; \ - $CONDA install -y -n py3_khiops10_conda khiops-core==10.3.2; \ - ' \ - && true - -RUN mkdir -p /scripts -COPY ./run_service.sh ./run_fake_remote_file_servers.sh /scripts/ -RUN chmod +x /scripts/run_service.sh /scripts/run_fake_remote_file_servers.sh && \ - useradd -rm -d /home/ubuntu -s /bin/bash -g root -u 1000 ubuntu - -# remote files drivers installed system-wide -RUN true \ - # Get Linux distribution codename \ - && if [ -f /etc/os-release ]; then . /etc/os-release; fi \ - # Force the installation of the debian 12 versions (bookworm) - && wget -O khiops-gcs.deb https://github.com/KhiopsML/khiopsdriver-gcs/releases/download/${KHIOPS_GCS_DRIVER_REVISION}/khiops-driver-gcs_${KHIOPS_GCS_DRIVER_REVISION}-1-bookworm.amd64.deb \ - && wget -O khiops-s3.deb https://github.com/KhiopsML/khiopsdriver-s3/releases/download/${KHIOPS_S3_DRIVER_REVISION}/khiops-driver-s3_${KHIOPS_S3_DRIVER_REVISION}-1-bookworm.amd64.deb \ - # XXX There is a typo in the latest release tag. Until it is fixed, we need to use the hardcoded folder name - && wget -O khiops-azure.deb https://github.com/KhiopsML/khiopsdriver-azure/releases/download/0.0.7/khiops-driver-azure_${KHIOPS_AZURE_DRIVER_REVISION}-1-bookworm.amd64.deb \ - && (dpkg -i --force-all khiops-gcs.deb khiops-s3.deb khiops-azure.deb || true) \ - && apt-get -f -y install \ - && rm -f khiops-gcs.deb khiops-s3.deb khiops-azure.deb \ - && true - -FROM ghcr.io/khiopsml/khiops-server:${SERVER_REVISION} AS server - -FROM khiopsdev AS base -COPY --from=server /service /usr/bin/service - -# S3 fake file server (only in the ubuntu container) -# Do not use the latest fakes3 version because starting from 1.3 a licence is required -# if fakes3 is no longer compatible think about switching to an alternative and fully compatible server -# (https://github.com/jamhall/s3rver:v3.7.1 is not yet for example) -RUN gem install fakes3:1.2.1 sorted_set -# Avoid resolving a fake s3-bucket.localhost hostname -# Alternate builders (buildx via moby buildkit) mount /etc/hosts read-only, the following command will fail -# echo "127.0.0.1 s3-bucket.localhost" >> /etc/hosts -# You will have to add the `add-hosts` input instead (https://github.com/docker/build-push-action/#inputs) - -# Port on which fakes3 is listening -EXPOSE 4569 - diff --git a/packaging/docker/khiopspydev/Dockerfile.rocky b/packaging/docker/khiopspydev/Dockerfile.rocky index b7994209..775c0733 100644 --- a/packaging/docker/khiopspydev/Dockerfile.rocky +++ b/packaging/docker/khiopspydev/Dockerfile.rocky @@ -7,10 +7,7 @@ LABEL description="Container for the development of khiops-python" # Reuse KHIOPSDEV_OS from previous stage ARG KHIOPSDEV_OS -ARG KHIOPS_REVISION # - Install dev tools and miniforge (for the unit tests) -# - Build and install Khiops -# - Set mpich as the default MPI RUN true \ && useradd -rm -d /home/rocky -s /bin/bash -g root -u 1000 rocky \ # Install git (for khiops-python version calculation), pandoc and pip \ @@ -28,21 +25,6 @@ RUN true \ python3.11-pip \ python3.11-setuptools ; \ fi \ - # Get Linux distribution codename \ - && if [ -f /etc/os-release ]; then . /etc/os-release; fi \ - && IFS='.' read -ra VERSION <<< "$VERSION_ID" \ - && ROCKY_VERSION=${VERSION[0]} \ - # Obtain the Khiops native package \ - && KHIOPS_PKG_FILE=$KHIOPS_REVISION/khiops-core-openmpi-$(echo ${KHIOPS_REVISION} | tr '-' '_')-1.el$ROCKY_VERSION.x86_64.rpm \ - && wget -O KHIOPS_CORE.rpm "https://github.com/KhiopsML/khiops/releases/download/${KHIOPS_PKG_FILE}" \ - # Install the Khiops native package \ - && dnf install KHIOPS_CORE.rpm -y \ - && rm -f KHIOPS_CORE.rpm \ - # Make sure that MPI is openmpi \ - && source /etc/profile.d/modules.sh \ - && module unload mpi \ - # Hard-code OpenMPI module name \ - && module load mpi/openmpi-x86_64 \ # Set python to python3.11 and pip to Pip 3.11 on Rocky 8 and 9 \ && if [ "$KHIOPSDEV_OS" = "rocky8" -o "$KHIOPSDEV_OS" = "rocky9" ]; then \ alternatives --install /usr/bin/python python /usr/bin/python3.11 2; \ @@ -58,7 +40,6 @@ RUN true \ && rm -rf /root/miniforge3/Miniforge3_25.9.1-0-Linux-x86_64.sh \ # Clean build files \ && dnf clean all \ - && rm -rf ./khiops \ && true # set up all the supported Python environments under Conda (for the unit tests) @@ -74,13 +55,7 @@ RUN true \ for version in ${PYTHON_VERSIONS}; \ do \ $CONDA create -y -n py${version} python=${version}; \ - done; \ - # Install Khiops from a different major version in a dedicated conda environment \ - # for a specific test regarding compatibility test \ - # The python interpreter version of the base environment is used as no specific version is given \ - $CONDA create -y -n py3_khiops10_conda; \ - $CONDA install -y -n py3_khiops10_conda khiops-core==10.3.2; \ - ' \ + done;' \ && true RUN mkdir -p /scripts diff --git a/packaging/docker/khiopspydev/Dockerfile.ubuntu b/packaging/docker/khiopspydev/Dockerfile.ubuntu-debian similarity index 68% rename from packaging/docker/khiopspydev/Dockerfile.ubuntu rename to packaging/docker/khiopspydev/Dockerfile.ubuntu-debian index b842a266..5bde832d 100644 --- a/packaging/docker/khiopspydev/Dockerfile.ubuntu +++ b/packaging/docker/khiopspydev/Dockerfile.ubuntu-debian @@ -5,12 +5,12 @@ FROM ghcr.io/khiopsml/khiops/khiopsdev-${KHIOPSDEV_OS}:latest AS khiopsdev LABEL maintainer="khiops.team@orange.com" LABEL description="Container for the development of khiops-python" -# Install dev tools and miniforge (for the unit tests); build and install Khiops +# Install dev tools and miniforge (for the unit tests); install Khiops (only for tests against KhiopsDockerRunner) ARG KHIOPS_REVISION RUN true \ - # Install git (for khiops-python version calculation) and pip \ + # Install git (for khiops-python version calculation), pip and unzip (used when deflating the samples) \ && apt-get -y update \ - && apt-get -y --no-install-recommends install git python3-pip \ + && apt-get -y --no-install-recommends install git python3-pip unzip \ # On Debian/Ubuntu systems, python3-venv must be installed to provide ensurepip. # It is still possible to create venvs using the --without-pip flag but any subsequent # module installation with pip would be impossible @@ -39,15 +39,11 @@ RUN true \ # Clean build files \ && rm -fr /var/lib/apt/lists/* \ && apt-get clean \ - && rm -rf ./khiops \ && true # set up all the supported Python environments under Conda (for the unit tests) # relying on a variable containing all the versions ARG PYTHON_VERSIONS -ARG KHIOPS_GCS_DRIVER_REVISION -ARG KHIOPS_S3_DRIVER_REVISION -ARG KHIOPS_AZURE_DRIVER_REVISION # Initialize all the Conda environments RUN true \ @@ -58,13 +54,7 @@ RUN true \ for version in ${PYTHON_VERSIONS}; \ do \ $CONDA create -y -n py${version} python=${version}; \ - done; \ - # Install Khiops from a different major version in a dedicated conda environment \ - # for a specific test regarding compatibility test \ - # The python interpreter version of the base environment is used as no specific version is given \ - $CONDA create -y -n py3_khiops10_conda; \ - $CONDA install -y -n py3_khiops10_conda khiops-core==10.3.2; \ - ' \ + done;' \ && true RUN mkdir -p /scripts @@ -72,19 +62,6 @@ COPY ./run_service.sh ./run_fake_remote_file_servers.sh /scripts/ RUN chmod +x /scripts/run_service.sh /scripts/run_fake_remote_file_servers.sh && \ useradd -rm -d /home/ubuntu -s /bin/bash -g root -u 1000 ubuntu -# remote files drivers installed system-wide -RUN true \ - # Get Linux distribution codename \ - && if [ -f /etc/os-release ]; then . /etc/os-release; fi \ - && wget -O khiops-gcs.deb https://github.com/KhiopsML/khiopsdriver-gcs/releases/download/${KHIOPS_GCS_DRIVER_REVISION}/khiops-driver-gcs_${KHIOPS_GCS_DRIVER_REVISION}-1-${VERSION_CODENAME}.amd64.deb \ - && wget -O khiops-s3.deb https://github.com/KhiopsML/khiopsdriver-s3/releases/download/${KHIOPS_S3_DRIVER_REVISION}/khiops-driver-s3_${KHIOPS_S3_DRIVER_REVISION}-1-${VERSION_CODENAME}.amd64.deb \ - # XXX There is a typo in the latest release tag. Until it is fixed, we need to use the hardcoded folder name - && wget -O khiops-azure.deb https://github.com/KhiopsML/khiopsdriver-azure/releases/download/0.0.7/khiops-driver-azure_${KHIOPS_AZURE_DRIVER_REVISION}-1-${VERSION_CODENAME}.amd64.deb \ - && (dpkg -i --force-all khiops-gcs.deb khiops-s3.deb khiops-azure.deb || true) \ - && apt-get -f -y install \ - && rm -f khiops-gcs.deb khiops-s3.deb khiops-azure.deb \ - && true - FROM ghcr.io/khiopsml/khiops-server:${SERVER_REVISION} AS server FROM khiopsdev AS base diff --git a/pyproject.toml b/pyproject.toml index c53c39de..11f8d40f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,8 +102,10 @@ classifiers = [ "Programming Language :: Python", "Topic :: Scientific/Engineering", ] -requires-python = ">=3.8" +requires-python = ">=3.10" dependencies = [ + # XXX : replace with the latest khiops-core 11.0.1 stable version + "khiops-core>=11.0.1a4,<12.0.0", # package containing khiops-core binaries # do not use the latest versions, to avoid undesired breaking changes "pandas>=2.3.3,<4.0.0", "scikit-learn>=1.7.2,<1.9.0", @@ -119,14 +121,17 @@ Homepage = "https://khiops.org" s3 = [ # do not use the latest version, to avoid undesired breaking changes "boto3>=1.17.39,<=1.35.69", + "khiops-driver-s3", # package containing the driver binary ] gcs = [ "google-cloud-storage>=1.37.0", + "khiops-driver-gcs", # package containing the driver binary ] azure = [ "azure-core>=1.39.0,<2.0.0", "azure-storage-blob>=12.28.0,<13.0.0", "azure-storage-file-share>=12.24.0,<13.0.0", + "khiops-driver-azure", # package containing the driver binary ] [tool.setuptools.packages.find]