diff --git a/.github/workflows/cd.yml b/.github/workflows/deployer.yml similarity index 100% rename from .github/workflows/cd.yml rename to .github/workflows/deployer.yml diff --git a/.github/workflows/master_cleaner.yml b/.github/workflows/master_cleaner.yml new file mode 100644 index 00000000..e57837d7 --- /dev/null +++ b/.github/workflows/master_cleaner.yml @@ -0,0 +1,29 @@ +name: Master Cleaner + +on: + push: + branches: + - master + +jobs: + formatter: + name: runner / black + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: psf/black@stable + with: + src: "./ezyrb" + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + title: "Format Python code with psf/black push" + commit-message: ":art: Format Python code with psf/black" + body: | + There appear to be some python formatting errors in ${{ github.sha }}. This pull request + uses the [psf/black](https://github.com/psf/black) formatter to fix these issues. + base: ${{ github.head_ref }} # Creates pull request onto pull request or commit branch + branch: actions/black \ No newline at end of file diff --git a/.github/workflows/monthly-tagger.yml b/.github/workflows/monthly-tagger.yml new file mode 100644 index 00000000..c41ff9b9 --- /dev/null +++ b/.github/workflows/monthly-tagger.yml @@ -0,0 +1,46 @@ +name: "Monthly Tagger" + +on: + schedule: + - cron: '20 2 1 * *' + +jobs: + + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-latest, macos-latest, ubuntu-latest] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install Python dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install .[test] + - name: Test with pytest + run: | + python3 -m pytest + + monthly_tag: + runs-on: ubuntu-latest + needs: test + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.PAT_EZYRB_PUSH }} + + - name: Create and push the tag + run: | + python utils/mathlab_versioning.py set --only-date "post$(date +%y%m)" + VERS=$(python utils/mathlab_versioning.py get) + git config --global user.name 'Monthly Tag bot' + git config --global user.email 'mtbot@noreply.github.com' + git add pyproject.toml + git commit -m "monthly version $VERS" + git tag -a "v$VERS" -m "Monthly version $VERS" + git push origin "v$VERS" diff --git a/.github/workflows/tester.yml b/.github/workflows/tester.yml new file mode 100644 index 00000000..bed76eef --- /dev/null +++ b/.github/workflows/tester.yml @@ -0,0 +1,120 @@ +name: "Testing Pull Request" + +on: + pull_request: + branches: + - "master" + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [windows-latest, macos-latest, ubuntu-latest] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + + env: + CFLAGS: "-Wno-error=implicit-function-declaration -Wno-error=incompatible-pointer-types" + CXXFLAGS: "-Wno-error=implicit-function-declaration -Wno-error=incompatible-pointer-types" + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system dependencies (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y libboost-serialization-dev libprotobuf-dev protobuf-compiler libopenblas-dev liblapack-dev + + - name: Set up Java for PyCOMPSs + if: runner.os == 'Linux' + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' + + - name: Install Python dependencies + shell: bash + run: | + python3 -m pip install --upgrade pip + python3 -m pip install setuptools wheel + if [ "$RUNNER_OS" == "Linux" ]; then + python3 -m pip install pycompss --no-build-isolation + fi + python3 -m pip install .[test] + + - name: Test with pytest + shell: bash + run: | + if [ "$RUNNER_OS" == "Linux" ]; then + python3 -m pytest + else + python3 -m pytest --ignore=tests/test_parallel/ + fi + + linter: #################################################################### + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Run Black formatter (check mode) + uses: psf/black@stable + with: + src: "./ezyrb" + + testdocs: ################################################################## + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install system dependencies + run: sudo apt-get update && sudo apt-get install -y pandoc + + - name: Install Python dependencies + run: python3 -m pip install .[docs] + + - name: Build Documentation + run: | + make html + working-directory: docs/ + + coverage: ################################################################## + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + + steps: + - uses: actions/checkout@v4 + + - name: Install system dependencies (Linux) + run: | + sudo apt-get update + sudo apt-get install -y libboost-serialization-dev libprotobuf-dev protobuf-compiler libopenblas-dev liblapack-dev + + - name: Install Python dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install setuptools wheel + python3 -m pip install pycompss --no-build-isolation + python3 -m pip install .[test] + + - name: Generate coverage report + run: | + python3 -m pytest --cov-report term --cov-report xml:cobertura.xml --cov=ezyrb + + - name: Produce the coverage report + if: github.event.pull_request.head.repo.full_name == github.repository + uses: insightsengineering/coverage-action@v2 + with: + path: ./cobertura.xml + threshold: 80.00 + fail: true + publish: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + coverage-summary-title: "Code Coverage Summary" \ No newline at end of file diff --git a/.github/workflows/testing_pr.yml b/.github/workflows/testing_pr.yml deleted file mode 100644 index bbdef74f..00000000 --- a/.github/workflows/testing_pr.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: "Testing Pull Request" - -on: - pull_request: - branches: - - "master" - -jobs: - build: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [windows-latest, macos-latest, ubuntu-latest] - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] - - env: - CFLAGS: "-Wno-error=implicit-function-declaration -Wno-error=incompatible-pointer-types" - CXXFLAGS: "-Wno-error=implicit-function-declaration -Wno-error=incompatible-pointer-types" - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install system dependencies (Linux) - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y libboost-serialization-dev libprotobuf-dev protobuf-compiler libopenblas-dev liblapack-dev - - - name: Set up Java for PyCOMPSs - if: runner.os == 'Linux' - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: '11' - - - name: Install Python dependencies - shell: bash - run: | - python3 -m pip install --upgrade pip - python3 -m pip install setuptools wheel - if [ "$RUNNER_OS" == "Linux" ]; then - python3 -m pip install pycompss --no-build-isolation - fi - python3 -m pip install .[test] - - - name: Test with pytest - shell: bash - run: | - if [ "$RUNNER_OS" == "Linux" ]; then - python3 -m pytest - else - python3 -m pytest --ignore=tests/test_parallel/ - fi \ No newline at end of file diff --git a/.github/workflows/tutorial_exporter.yml b/.github/workflows/tutorial_exporter.yml new file mode 100644 index 00000000..7585faf7 --- /dev/null +++ b/.github/workflows/tutorial_exporter.yml @@ -0,0 +1,140 @@ +name: "Export Tutorials" + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "master" + paths: + - 'tutorials/**/*.ipynb' + +jobs: + # run on push + export_tutorials_on_push: + if: ${{ github.event_name == 'push' }} + permissions: write-all + runs-on: ubuntu-latest + env: + TUTORIAL_TIMEOUT: 1200s + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + # Dependencies for tutorials + python3 -m pip install --upgrade pip .[tutorial,docs] black[jupyter] + - name: Setup FFmpeg + uses: FedericoCarboni/setup-ffmpeg@v2 + + - id: files + uses: jitterbit/get-changed-files@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + format: space-delimited + + - name: Configure git + run: | + git config user.name "github-actions[bot]" + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + + - name: Run formatter + run: black tutorials/ + + - name: Export tutorials to .py and .html + run: | + set -x + for file in ${{ steps.files.outputs.all }}; do + if [[ $file == *.ipynb ]]; then + filename=$(basename $file) + pyfilename=$(echo ${filename%?????})py + timeout --signal=SIGKILL $TUTORIAL_TIMEOUT python -Xfrozen_modules=off -m jupyter nbconvert $file --to python --output $pyfilename --output-dir=$(dirname $file) + htmlfilename=$(echo ${filename%?????} | sed -e 's/-//g')html + htmldir="docs/source"/$(echo ${file%??????????????} | sed -e 's/-//g') + timeout --signal=SIGKILL $TUTORIAL_TIMEOUT python -Xfrozen_modules=off -m jupyter nbconvert --execute $file --to html --output $htmlfilename --output-dir=$htmldir + fi + done + set +x + + - uses: benjlevesque/short-sha@v2.1 + id: short-sha + + - name: Remove unwanted files + run: | + rm -rf build/ tutorials/tutorial4/data/ + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v5.0.2 + with: + labels: maintenance + title: Export tutorial changed in ${{ steps.short-sha.outputs.sha }} + branch: export-tutorial-${{ steps.short-sha.outputs.sha }} + base: ${{ github.head_ref }} + commit-message: export tutorials changed in ${{ steps.short-sha.outputs.sha }} + delete-branch: true + + # run on workflow_dispatch + export_tutorials_workflow_dispatch: + if: ${{ github.event_name == 'workflow_dispatch' }} + permissions: write-all + runs-on: ubuntu-latest + env: + TUTORIAL_TIMEOUT: 1200s + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip .[tutorial,docs] black[jupyter] + + - name: Setup FFmpeg + uses: FedericoCarboni/setup-ffmpeg@v2 + + - name: Configure git + run: | + git config user.name "github-actions[bot]" + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + + - name: Run formatter + run: black tutorials/ + + - name: Export all tutorials to .py and .html + run: | + set -x + # Find all .ipynb files in the tutorials directory + for file in $(find tutorials -type f -name "*.ipynb"); do + filename=$(basename $file) + pyfilename="${filename%.ipynb}.py" + timeout --signal=SIGKILL $TUTORIAL_TIMEOUT python -Xfrozen_modules=off -m jupyter nbconvert $file --to python --output $pyfilename --output-dir=$(dirname $file) + htmlfilename="${filename%.ipynb}.html" + htmldir="docs/source"/$(dirname $file) + timeout --signal=SIGKILL $TUTORIAL_TIMEOUT python -Xfrozen_modules=off -m jupyter nbconvert --execute $file --to html --output $htmlfilename --output-dir=$htmldir + done + set +x + + - uses: benjlevesque/short-sha@v2.1 + id: short-sha + + - name: Remove unwanted files + run: | + rm -rf build/ tutorials/tutorial4/data/ + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v5.0.2 + with: + labels: maintenance + title: Export tutorial changed in ${{ steps.short-sha.outputs.sha }} + branch: export-tutorial-${{ steps.short-sha.outputs.sha }} + base: ${{ github.head_ref }} + commit-message: export tutorials changed in ${{ steps.short-sha.outputs.sha }} + delete-branch: true diff --git a/ezyrb/approximation/kneighbors_regressor.py b/ezyrb/approximation/kneighbors_regressor.py index e3c64c5f..0776289e 100644 --- a/ezyrb/approximation/kneighbors_regressor.py +++ b/ezyrb/approximation/kneighbors_regressor.py @@ -33,7 +33,5 @@ def __init__(self, **kwargs): :param kwargs: Arguments passed to sklearn's KNeighborsRegressor. """ - logger.debug( - "Initializing KNeighborsRegressor with kwargs: %s", kwargs - ) + logger.debug("Initializing KNeighborsRegressor with kwargs: %s", kwargs) self.regressor = Regressor(**kwargs) diff --git a/ezyrb/parallel/__init__.py b/ezyrb/parallel/__init__.py index 91e7e253..87e5ce0a 100644 --- a/ezyrb/parallel/__init__.py +++ b/ezyrb/parallel/__init__.py @@ -18,6 +18,7 @@ from .reduction import Reduction from .pod import POD from .ae import AE + try: from .ae_eddl import AE_EDDL except ImportError: diff --git a/ezyrb/parallel/ae_eddl.py b/ezyrb/parallel/ae_eddl.py index 673c6df2..9c47f2bd 100644 --- a/ezyrb/parallel/ae_eddl.py +++ b/ezyrb/parallel/ae_eddl.py @@ -165,9 +165,7 @@ def __setstate__(self, state): # (n) hidden layers + (n-1) activation layers + (1) input layer n = len(self.layers_encoder) encoder_layer_index = 2 * n - 1 - self.model_Autoencoder = eddl.import_net_from_onnx_file( - self.file_1 - ) + self.model_Autoencoder = eddl.import_net_from_onnx_file(self.file_1) self.encoder = self.model_Autoencoder.layers[encoder_layer_index] self.decoder = self.model_Autoencoder.layers[-1] @@ -339,9 +337,7 @@ def fit(self, values): for j in range(num_batches): # 1) using next_batch eddl.next_batch([values], [xbatch]) - eddl.train_batch( - self.model_Autoencoder, [xbatch], [xbatch] - ) + eddl.train_batch(self.model_Autoencoder, [xbatch], [xbatch]) losses = eddl.get_losses(self.model_Autoencoder) metrics = eddl.get_metrics(self.model_Autoencoder) diff --git a/ezyrb/parallel/pod.py b/ezyrb/parallel/pod.py index 60548fe6..a535cda7 100644 --- a/ezyrb/parallel/pod.py +++ b/ezyrb/parallel/pod.py @@ -18,7 +18,7 @@ # Fallback: Define a 'do-nothing' decorator and dummy constants def task(*args, **kwargs): return lambda f: f - + INOUT = None IN = None diff --git a/ezyrb/parallel/reducedordermodel.py b/ezyrb/parallel/reducedordermodel.py index 8df6e28c..b0571303 100644 --- a/ezyrb/parallel/reducedordermodel.py +++ b/ezyrb/parallel/reducedordermodel.py @@ -43,10 +43,10 @@ def fit(self, *args, **kwargs): """ # Assign the initial training database self.train_full_database = self.database - + self._execute_plugins("fit_preprocessing") self._execute_plugins("fit_before_reduction") - + # Fit reduction and transform self.reduction.fit(self.train_full_database.snapshots_matrix.T) reduced_output = self.reduction.transform( @@ -63,33 +63,40 @@ def fit(self, *args, **kwargs): # Fit approximation on the reduced database self.approximation.fit( - self.train_reduced_database.parameters_matrix, - self.train_reduced_database.snapshots_matrix, - *args, **kwargs + self.train_reduced_database.parameters_matrix, + self.train_reduced_database.snapshots_matrix, + *args, + **kwargs, ) self._execute_plugins("fit_after_approximation") self._execute_plugins("fit_postprocessing") - + return self def predict(self, parameters): r""" Predict the solution for given parameters mu. - - This method distributes the evaluation tasks across the + + This method distributes the evaluation tasks across the available computational nodes using the PyCOMPSs framework. """ - is_db = hasattr(parameters, 'parameters_matrix') - mu = parameters.parameters_matrix if is_db else np.atleast_2d(parameters) - + is_db = hasattr(parameters, "parameters_matrix") + mu = ( + parameters.parameters_matrix if is_db else np.atleast_2d(parameters) + ) + # Setup dummy test_full_database required by some preprocessing plugins - dummy_snaps = np.zeros((len(mu), self.train_full_database.snapshots_matrix.shape[1])) + dummy_snaps = np.zeros( + (len(mu), self.train_full_database.snapshots_matrix.shape[1]) + ) self.test_full_database = Database(mu, dummy_snaps) - # The scaler plugin modifies parameters here BEFORE approximation, + # The scaler plugin modifies parameters here BEFORE approximation, # so we must initialize this object early with dummy snapshots. - dummy_red_snaps = np.zeros((len(mu), self.train_reduced_database.snapshots_matrix.shape[1])) + dummy_red_snaps = np.zeros( + (len(mu), self.train_reduced_database.snapshots_matrix.shape[1]) + ) self.predict_reduced_database = Database(mu, dummy_red_snaps) self._execute_plugins("predict_preprocessing") @@ -99,11 +106,10 @@ def predict(self, parameters): predicted_red_sol = self.approximation.predict( self.predict_reduced_database.parameters_matrix ) - + # Update by creating a NEW database, as snapshots_matrix is read-only self.predict_reduced_database = Database( - self.predict_reduced_database.parameters_matrix, - predicted_red_sol + self.predict_reduced_database.parameters_matrix, predicted_red_sol ) self._execute_plugins("predict_after_approximation") @@ -132,8 +138,8 @@ def test_error(self, test, norm=np.linalg.norm, relative=True): test snapshots. """ predicted_test = self.predict(test.parameters_matrix) - - if hasattr(predicted_test, 'snapshots_matrix'): + + if hasattr(predicted_test, "snapshots_matrix"): pred_snaps = predicted_test.snapshots_matrix else: pred_snaps = predicted_test @@ -144,9 +150,7 @@ def test_error(self, test, norm=np.linalg.norm, relative=True): / norm(test.snapshots_matrix, axis=1) ) else: - return np.mean( - norm(pred_snaps - test.snapshots_matrix, axis=1) - ) + return np.mean(norm(pred_snaps - test.snapshots_matrix, axis=1)) def save(self, fname, save_db=True, save_reduction=True, save_approx=True): """Save the object to `fname` using the pickle module.""" @@ -181,12 +185,12 @@ def kfold_cv_error(self, n_splits, *args, norm=np.linalg.norm, **kwargs): new_db, copy.deepcopy(self.reduction), copy.deepcopy(self.approximation), - plugins=[copy.deepcopy(p) for p in self.plugins] + plugins=[copy.deepcopy(p) for p in self.plugins], ).fit(*args, **kwargs) test = self.database[test_index] pred = rom.predict(test.parameters_matrix) - if hasattr(pred, 'snapshots_matrix'): + if hasattr(pred, "snapshots_matrix"): pred = pred.snapshots_matrix predicted_test.append(pred) original_test.append(test.snapshots_matrix) @@ -219,11 +223,11 @@ def loo_error(self, *args, norm=np.linalg.norm, **kwargs): new_db, copy.deepcopy(self.reduction), copy.deepcopy(self.approximation), - plugins=[copy.deepcopy(p) for p in self.plugins] + plugins=[copy.deepcopy(p) for p in self.plugins], ).fit(*args, **kwargs) pred = rom.predict(test_db.parameters_matrix) - if hasattr(pred, 'snapshots_matrix'): + if hasattr(pred, "snapshots_matrix"): pred = pred.snapshots_matrix predicted_test.append(pred) original_test.append(test_db.snapshots_matrix) @@ -268,4 +272,4 @@ def _simplex_volume(self, vertices): distance = np.transpose([vertices[0] - vi for vi in vertices[1:]]) return np.abs( np.linalg.det(distance) / math.factorial(vertices.shape[1]) - ) \ No newline at end of file + ) diff --git a/ezyrb/reducedordermodel.py b/ezyrb/reducedordermodel.py index 6f6ef9b2..23a3a356 100644 --- a/ezyrb/reducedordermodel.py +++ b/ezyrb/reducedordermodel.py @@ -451,9 +451,7 @@ def test_error(self, test, norm=np.linalg.norm, relative=True): / norm(test.snapshots_matrix, axis=1) ) else: - return np.mean( - norm(predicted_test - test.snapshots_matrix, axis=1) - ) + return np.mean(norm(predicted_test - test.snapshots_matrix, axis=1)) def kfold_cv_error( self, n_splits, *args, norm=np.linalg.norm, relative=True, **kwargs diff --git a/ezyrb/snapshot.py b/ezyrb/snapshot.py index d1712941..ece11d35 100644 --- a/ezyrb/snapshot.py +++ b/ezyrb/snapshot.py @@ -84,9 +84,7 @@ def values(self, new_values): self._values = new_values if new_values is not None: - logger.debug( - "Snapshot values set with length: %d", len(new_values) - ) + logger.debug("Snapshot values set with length: %d", len(new_values)) else: logger.debug("Snapshot values set to None") diff --git a/pyproject.toml b/pyproject.toml index 33e82139..ccd65d88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,10 @@ dev = [ parallel = [ "pycompss" ] +tutorial = [ + "jupyter", + "smithers" +] [project.urls] Homepage = "https://github.com/mathLab/EZyRB"