diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml index c5f256b220e9..76429a377db2 100644 --- a/.github/release-drafter.yml +++ b/.github/release-drafter.yml @@ -1,20 +1,28 @@ name-template: 'v$NEXT_PATCH_VERSION' tag-template: 'v$NEXT_PATCH_VERSION' + +branches: + - develop + categories: - title: ':rocket: Experimental Features' labels: - 'changelog:feature' + - title: ':pill: Bug Fixes' labels: - 'changelog:fix' + - title: ':wrench: Maintenance' labels: - 'changelog:chore' -change-template: '- $TITLE @$AUTHOR (#$NUMBER)' -branches: - - develop + exclude-labels: - 'changelog:none' + +change-template: '- $TITLE @$AUTHOR (#$NUMBER)' + template: | ## Changes + $CHANGES diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 02c94dd1198f..76eb60ebc8f8 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -30,7 +30,7 @@ jobs: - id: compute run: | if [[ "${{ inputs.runner || 'ubuntu-latest' }}" == *arm* ]]; then - echo "flags=" >> $GITHUB_OUTPUT + echo "flags=-Dcpu-arch=armv9-a+simd" >> $GITHUB_OUTPUT echo "werror=" >> $GITHUB_OUTPUT else echo "flags=-Dcpu-arch=skylake" >> $GITHUB_OUTPUT @@ -462,6 +462,7 @@ jobs: uses: docker://ghcr.io/su2code/su2/test-su2:260405-0054 env: OMPI_MCA_osc: pt2pt + LD_LIBRARY_PATH: /github/workspace/install/lib with: entrypoint: install/bin/${{matrix.testdriver}} - name: Post Cleanup diff --git a/.github/workflows/release-management.yml b/.github/workflows/release-management.yml index 8acd87d4b2dd..e8c44d8ac399 100644 --- a/.github/workflows/release-management.yml +++ b/.github/workflows/release-management.yml @@ -2,17 +2,31 @@ name: Release Management on: push: - # branches to consider in the event; optional, defaults to all branches: - develop +permissions: + contents: write + +concurrency: + group: release-management-${{ github.ref }} + cancel-in-progress: true + jobs: - build_and_upload: - name: Build SU2 + build: + name: Build SU2 (${{ matrix.os_bin }}) + runs-on: ubuntu-latest + strategy: fail-fast: false matrix: - os_bin: [macos64, macos64-mpi, linux64-omp, linux64-mpi, win64-omp, win64-mpi] + os_bin: + - macos64 + - macos64-mpi + - linux64-omp + - linux64-mpi + - win64-omp + - win64-mpi include: - os_bin: win64-omp flags: '-Dcpu-arch=haswell -Dwith-omp=true -Dwith-mpi=disabled --cross-file=/hostfiles/hostfile_windows' @@ -26,42 +40,80 @@ jobs: flags: '-Dcpu-arch=haswell -Dwith-omp=true -Dwith-mpi=disabled -Dstatic-cgns-deps=true --cross-file=/hostfiles/hostfile_linux' - os_bin: linux64-mpi flags: '-Dcpu-arch=haswell -Dcustom-mpi=true --cross-file=/hostfiles/hostfile_linux_mpi' - runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + - name: Cache Object Files - uses: actions/cache@v5 + uses: actions/cache@v4 with: path: ccache key: ${{ matrix.os_bin }}-${{ github.sha }} - restore-keys: ${{ matrix.os_bin }} + restore-keys: | + ${{ matrix.os_bin }}- + - name: Build uses: docker://ghcr.io/su2code/su2/build-su2-cross:260405-0054 with: - args: -b ${{ github.sha }} -f "${{matrix.flags}}" + args: -b ${{ github.sha }} -f "${{ matrix.flags }}" + - name: Create Archive run: | cd install - zip -r ../${{matrix.os_bin}}.zip bin/* - # Uploads binaries as artifacts (just as a backup) - - name: Upload Binaries - uses: actions/upload-artifact@v7 + zip -r ../${{ matrix.os_bin }}.zip bin/* + + - name: Upload Build Artifact + uses: actions/upload-artifact@v5 + with: + name: ${{ matrix.os_bin }} + path: ${{ matrix.os_bin }}.zip + + - name: Update Release Draft + id: release_drafter + uses: release-drafter/release-drafter@v6 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + release: + name: Update Draft Release + runs-on: ubuntu-latest + needs: build + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Download All Build Artifacts + uses: actions/download-artifact@v5 with: - name: ${{matrix.os_bin}} - path: ${{matrix.os_bin}}.zip - # Update the release notes of latest draft release - - uses: talbring/jenkins-release-drafter@v5.2.0-jenkins-11 - name: Update Release - id: update_release + path: release-assets + + - name: Flatten Artifact Directory + run: | + find release-assets -name "*.zip" -exec mv {} . \; + + - name: Update Release Draft + id: release_drafter + uses: release-drafter/release-drafter@v6 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # Upload binaries as assets to draft release - - name: Upload Release Asset - id: upload-release-asset - uses: actions/upload-release-asset@v1.0.2 + + - name: Upload Release Assets env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.update_release.outputs.uploadurl }} - asset_path: ${{matrix.os_bin}}.zip - asset_name: SU2-${{ steps.update_release.outputs.tagname }}-${{matrix.os_bin}}.zip - asset_content_type: application/zip + TAG_NAME: ${{ steps.release_drafter.outputs.tag_name }} + run: | + set -e + + for file in *.zip; do + + asset_name="SU2-${TAG_NAME}-${file}" + + echo "Uploading $asset_name" + + gh release upload "$TAG_NAME" \ + "$file#$asset_name" \ + --clobber + + done diff --git a/AUTHORS.md b/AUTHORS.md index 554d925a8e97..d82ccb599fe9 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -10,18 +10,18 @@ Thomas D. Economon (formerly Stanford University) Juan J. Alonso (Stanford University) ``` -## Current Maintainer ## +## Current Maintainers ## The SU2 project is maintained by members of the SU2 Foundation (https://su2foundation.org) ``` -Thomas D. Economon - Executive Director - tom@su2foundation.org -Tim Albring - Director - tim@su2foundation.org -Juan J. Alonso - Director - juan@su2foundation.org -Eran Arad - Director - eran@su2foundation.org -Piero Colonna - Director - piero@su2foundation.org -Pedro Gomes - Director - pedro@su2foundation.org -Daniel Mayer - Director - daniel@su2foundation.org +Thomas D. Economon - Chairperson +Matteo Pini - Vice Chairperson +Nijso Beishuizen - Treasurer +Pedro Gomes - Development Officer +Giulio Gori - Secretary +Nitish Anand - Editorial Officer +Edwin van der Weide - Events Officer ``` in collaboration with the following main contributors and research teams: @@ -60,6 +60,7 @@ Arne Voß Ayush Kumar Beckett Y. Zhou Benjamin S. Kirk +BlueChips Brendan Tracey Brian Munguía Carsten Othmer @@ -69,6 +70,7 @@ Christian Bauer Clark Pederson Daumantas Kavolis Dave Taflin +Davide Di Giusto Eduardo Molina Edwin van der Weide Eitan Aberman @@ -93,6 +95,7 @@ Jairo Paes Cavalcante Filho Jason Howison Jayant Mukhopadhaya Jeffrey van Oostrom +Jesse Li Jessie Lauzon João Loureiro Johannes Blühdorn @@ -172,4 +175,5 @@ sravya91 srcopela tobadavid vfrancesmolla +Yilun Tan ``` diff --git a/Common/include/CConfig.hpp b/Common/include/CConfig.hpp index 71f780bbed07..36e802e99f67 100644 --- a/Common/include/CConfig.hpp +++ b/Common/include/CConfig.hpp @@ -135,7 +135,8 @@ class CConfig { Hold_GridFixed, /*!< \brief Flag hold fixed some part of the mesh during the deformation. */ Axisymmetric, /*!< \brief Flag for axisymmetric calculations */ Enable_Cuda, /*!< \brief Flag for switching GPU computing*/ - Integrated_HeatFlux; /*!< \brief Flag for heat flux BC whether it deals with integrated values.*/ + Integrated_HeatFlux, /*!< \brief Flag for heat flux BC whether it deals with integrated values.*/ + Pressure_Based; /*!< \brief Flag to check if we are using a pressure-based system.*/ su2double Buffet_k; /*!< \brief Sharpness coefficient for buffet sensor.*/ su2double Buffet_lambda; /*!< \brief Offset parameter for buffet sensor.*/ su2double Damp_Engine_Inflow; /*!< \brief Damping factor for the engine inlet. */ @@ -525,6 +526,8 @@ class CConfig { Kind_Gradient_Method_Recon, /*!< \brief Numerical method for computation of spatial gradients used for upwind reconstruction. */ Kind_Deform_Linear_Solver, /*!< Numerical method to deform the grid */ Kind_Deform_Linear_Solver_Prec, /*!< \brief Preconditioner of the linear solver. */ + Kind_Poisson_Linear_Solver, /*!< \brief Numerical solver for the poisson equation. */ + Kind_Poisson_Linear_Solver_Prec, /*!< \brief Preconditioner of the linear solver of the poisson equation. */ Kind_Linear_Solver, /*!< \brief Numerical solver for the implicit scheme. */ Kind_Linear_Solver_Prec, /*!< \brief Preconditioner of the linear solver. */ Kind_DiscAdj_Linear_Solver, /*!< \brief Linear solver for the discrete adjoint system. */ @@ -587,6 +590,9 @@ class CConfig { Kind_Upwind_Heat, /*!< \brief Upwind scheme for the heat transfer model. */ Kind_Upwind_Template; /*!< \brief Upwind scheme for the template model. */ + PBITER Kind_PBIter; /*< \brief Kind of pressure-based algorithm that is used. */ + INCOMP_SYSTEM Kind_Incomp_System; /*< \brief Kind of incompressible solver. */ + bool MUSCL, /*!< \brief MUSCL scheme (for the runtime eq. system). */ MUSCL_Flow, /*!< \brief MUSCL scheme for the flow equations.*/ MUSCL_Turb, /*!< \brief MUSCL scheme for the turbulence equations.*/ @@ -638,17 +644,33 @@ class CConfig { bool InletUseNormal; /*!< \brief Flag for whether to use the local normal as the flow direction for a pressure inlet. */ su2double Linear_Solver_Error; /*!< \brief Min error of the linear solver for the implicit formulation. */ su2double Deform_Linear_Solver_Error; /*!< \brief Min error of the linear solver for the implicit formulation. */ + su2double Poisson_Linear_Solver_Error; /*!< \brief Min error of the linear solver for the poisson equation. */ su2double Linear_Solver_Smoother_Relaxation; /*!< \brief Relaxation factor for iterative linear smoothers. */ unsigned long Linear_Solver_Iter; /*!< \brief Max iterations of the linear solver for the implicit formulation. */ + unsigned long Poisson_Linear_Solver_Iter; /*!< \brief Max iterations of the linear solver for the poisson solver*/ unsigned long Deform_Linear_Solver_Iter; /*!< \brief Max iterations of the linear solver for the implicit formulation. */ unsigned long Linear_Solver_Restart_Frequency; /*!< \brief Restart frequency of the linear solver for the implicit formulation. */ unsigned long Linear_Solver_Restart_Deflation; /*!< \brief Number of vectors used for deflated restarts. */ unsigned long Linear_Solver_Prec_Threads; /*!< \brief Number of threads per rank for ILU and LU_SGS preconditioners. */ - unsigned short Linear_Solver_ILU_n; /*!< \brief ILU fill=in level. */ - bool Linear_Solver_ILU_levels; /*!< \brief Use level scheduling for OMP parallelization of ILU. */ + + struct CIluOptions { + unsigned short FillIn = 0; /*!< \brief ILU fill-in level. */ + bool LevelScheduling = false; /*!< \brief Use level scheduling for OMP parallelization of ILU. */ + /*!< \brief Number of colored Gauss-Seidel sweeps used to build the GPU ILU factorization; + * the triangular solves are level-scheduled and exact (no sweep count). */ + unsigned short GPUSweeps = 2; + } IluOptions; su2double SemiSpan; /*!< \brief Wing Semi span. */ su2double MSW_Alpha; /*!< \brief Coefficient for blending states in the MSW scheme. */ su2double Roe_Kappa; /*!< \brief Relaxation of the Roe scheme. */ + + struct CSIMPLE_Options { + su2double Transient_Term_Removal_Factor; /*!< \brief Coefficient for removing the transient term from the momentum coefficient. */ + su2double Relaxation_Factor_Pressure; /*!< \brief Relaxation coefficient of the pressure corrections in the SIMPLE solver. */ + bool AutomaticRelaxationFactors; /*!< \brief option for automatically computing relaxation factors for flow corrections in SIMPLE. */ + unsigned short nCorrections_PISO; /*!< \brief Number of corrections used in PISO algorithm. */ + } SIMPLE_Options; + su2double Relaxation_Factor_Adjoint; /*!< \brief Relaxation coefficient for variable updates of adjoint solvers. */ su2double Relaxation_Factor_CHT; /*!< \brief Relaxation coefficient for the update of conjugate heat variables. */ su2double EntropyFix_Coeff; /*!< \brief Entropy fix coefficient. */ @@ -810,11 +832,13 @@ class CConfig { su2double *nBlades; /*!< \brief number of blades for turbomachinery computation. */ unsigned short Geo_Description; /*!< \brief Description of the geometry. */ unsigned short Mesh_FileFormat; /*!< \brief Mesh input format. */ + unsigned short Mesh_Out_FileFormat; /*!< \brief Mesh output format. */ TAB_OUTPUT Tab_FileFormat; /*!< \brief Format of the output files. */ unsigned short output_precision; /*!< \brief .precision(value) for SU2_DOT and HISTORY output */ unsigned short ActDisk_Jump; /*!< \brief Format of the output files. */ unsigned long StartWindowIteration; /*!< \brief Starting Iteration for long time Windowing apporach . */ unsigned short nCFL_AdaptParam; /*!< \brief Number of CFL parameters provided in config. */ + unsigned long outlierMitigationParam[4]; /*!< \brief Parameters of outlier mitigation strategy. */ bool CFL_Adapt; /*!< \brief Use adaptive CFL number. */ bool HB_Precondition; /*!< \brief Flag to turn on harmonic balance source term preconditioning */ su2double RefArea, /*!< \brief Reference area for coefficient computation. */ @@ -1118,6 +1142,7 @@ class CConfig { unsigned short *nSpan_iZones; /*!< \brief number of span-wise sections for each zones */ bool turbMixingPlane; /*!< \brief option for turbulent mixingplane */ bool SpatialFourier; /*!< \brief option for computing the fourier transforms for subsonic non-reflecting BC. */ + bool RelFrame_SlidingPlane; /*!< \brief option for relative frame slidingplane */ bool RampMotionFrame; /*!< \brief option for ramping up or down the motion Frame values */ bool RampOutlet; /*!< \brief option for ramping up or down the outlet values */ bool RampMUSCL; @@ -1147,6 +1172,8 @@ class CConfig { /*--- Multigrid options ---*/ unsigned short nMG_PreSmooth_p{0}, nMG_PostSmooth_p{0}, nMG_CorrecSmooth_p{0}; unsigned short *MG_PreSmooth_p{nullptr}, *MG_PostSmooth_p{nullptr}, *MG_CorrecSmooth_p{nullptr}; + unsigned short nMG_CflScaling_p{0}; + su2double *MG_CflScaling_p{nullptr}; ENUM_STREAMWISE_PERIODIC Kind_Streamwise_Periodic; /*!< \brief Kind of Streamwise periodic flow (pressure drop or massflow) */ bool Streamwise_Periodic_Temperature; /*!< \brief Use real periodicity for Energy equation or otherwise outlet source term. */ @@ -1267,6 +1294,8 @@ class CConfig { unsigned long edgeColorGroupSize; /*!< \brief Size of the edge groups colored for OpenMP parallelization of edge loops. */ bool edgeColoringRelaxDiscAdj; /*!< \brief Allow fallback to smaller edge color group sizes and use more colors for the discrete adjoint. */ + unsigned short rcmNumSeeds; /*!< \brief Number of concurrent BFS fronts used to build the RCM reordering. */ + INLET_SPANWISE_INTERP Kind_InletInterpolationFunction; /*!brief type of spanwise interpolation function to use for the inlet face. */ INLET_INTERP_TYPE Kind_Inlet_InterpolationType; /*!brief type of spanwise interpolation data to use for the inlet face. */ bool PrintInlet_InterpolatedData; /*!brief option for printing the interpolated data file. */ @@ -1527,6 +1556,14 @@ class CConfig { */ void SetMPICommunicator(SU2_MPI::Comm Communicator); + /*! + * \brief Helper function, which checks and opens a binary SU2 file. + * \param[in] val_mesh_filename - Name of the file with the grid information. + * \param[in] readnDim = Whether or not nDim must be read. If false nZone is read. + * \return Number of dimensions or number of zones in the grid. + */ + static unsigned short CheckOpenSU2BinFile(const string& val_mesh_filename, bool readnDim); + /*! * \brief Gets the number of zones in the mesh file. * \param[in] val_mesh_filename - Name of the file with the grid information. @@ -1712,6 +1749,11 @@ class CConfig { */ bool GetCFL_Adapt(void) const { return CFL_Adapt; } + /*! + * \brief Get the outlier mitigation parameters. + */ + const unsigned long* GetOutlierMitigationParam() const { return outlierMitigationParam; } + /*! * \brief Get the value of the limits for the sections. * \return Value of the limits for the sections. @@ -1988,12 +2030,6 @@ class CConfig { */ su2double GetPressure_FreeStreamND(void) const { return Pressure_FreeStreamND; } - /*! - * \brief Get a reference to the non-dimensionalized freestream pressure (used for AD tracking). - * \return Reference to non-dimensionalized freestream pressure. - */ - su2double& GetPressure_FreeStreamND(void) { return Pressure_FreeStreamND; } - /*! * \brief Get the value of the thermodynamic pressure. * \return Thermodynamic pressure. @@ -2019,12 +2055,6 @@ class CConfig { */ su2double GetTemperature_FreeStreamND(void) const { return Temperature_FreeStreamND; } - /*! - * \brief Get a reference to the non-dimensionalized freestream temperature (used for AD tracking). - * \return Reference to non-dimensionalized freestream temperature. - */ - su2double& GetTemperature_FreeStreamND(void) { return Temperature_FreeStreamND; } - /*! * \brief Get the value of the non-dimensionalized vibrational-electronic freestream temperature. * \return Non-dimensionalized vibrational-electronic freestream temperature. @@ -2923,7 +2953,8 @@ class CConfig { */ void SetMGLevels(unsigned short val_nMGLevels) { nMGLevels = val_nMGLevels; - if (Kind_MGCycle == MG_CYCLE::FULL) { + /*--- Clamp so FinestMesh can never point past the last level that still exists. ---*/ + if ((Kind_MGCycle == MG_CYCLE::FULL) || (FinestMesh > val_nMGLevels)) { SetFinestMesh(val_nMGLevels); } } @@ -3992,6 +4023,18 @@ class CConfig { */ ENUM_REGIME GetKind_Regime(void) const { return Kind_Regime; } + /*! + * \brief Kind of incompressible solver formulation. + * \return Kind of incompressible solver. + */ + INCOMP_SYSTEM GetKind_Incomp_System(void) const { return Kind_Incomp_System; } + + /*! + * \brief Kind of iteration used for pressure based iterations. + * \return Kind of iteration used for pressure based iterations. + */ + PBITER GetKind_PBIter(void) const { return Kind_PBIter; } + /*! * \brief Governing equations of the flow (it can be different from the run time equation). * \param[in] val_zone - Zone where the soler is applied. @@ -4342,12 +4385,24 @@ class CConfig { */ unsigned short GetKind_Linear_Solver_Prec(void) const { return Kind_Linear_Solver_Prec; } + /*! + * \brief Get the kind of preconditioner for the linear solver of the poisson problem. + * \return Numerical preconditioner for poisson equation (solving the linear system). + */ + unsigned short GetKind_Poisson_Linear_Solver_Prec(void) const { return Kind_Poisson_Linear_Solver_Prec; } + /*! * \brief Get the kind of solver for the implicit solver. * \return Numerical solver for implicit formulation (solving the linear system). */ unsigned short GetKind_Deform_Linear_Solver(void) const { return Kind_Deform_Linear_Solver; } + /*! + * \brief Get the kind of solver for the poisson equation. + * \return Numerical solver for poisson equation (solving the linear system). + */ + unsigned short GetKind_Poisson_Linear_Solver(void) const { return Kind_Poisson_Linear_Solver; } + /*! * \brief Get min error of the linear solver for the implicit formulation. * \return Min error of the linear solver for the implicit formulation. @@ -4360,12 +4415,24 @@ class CConfig { */ su2double GetDeform_Linear_Solver_Error(void) const { return Deform_Linear_Solver_Error; } + /*! + * \brief Get min error of the linear solver for the poisson equation. + * \return Min error of the linear solver for the poisson equation. + */ + su2double GetPoisson_Linear_Solver_Error(void) const { return Poisson_Linear_Solver_Error; } + /*! * \brief Get max number of iterations of the linear solver for the implicit formulation. * \return Max number of iterations of the linear solver for the implicit formulation. */ unsigned long GetLinear_Solver_Iter(void) const { return Linear_Solver_Iter; } + /*! + * \brief Get max number of iterations of the linear solver for the poisson equation. + * \return Max number of iterations of the linear solver for the poisson equation. + */ + unsigned long GetPoisson_Linear_Solver_Iter(void) const { return Poisson_Linear_Solver_Iter; } + /*! * \brief Get max number of iterations of the linear solver for the implicit formulation. * \return Max number of iterations of the linear solver for the implicit formulation. @@ -4373,15 +4440,15 @@ class CConfig { unsigned long GetDeform_Linear_Solver_Iter(void) const { return Deform_Linear_Solver_Iter; } /*! - * \brief Get the ILU fill-in level for the linear solver. - * \return Fill in level of the ILU preconditioner for the linear solver. + * \brief Get the ILU preconditioner options (fill-in level, OMP level scheduling, GPU build + * sweeps), see CIluOptions. */ - unsigned short GetLinear_Solver_ILU_n(void) const { return Linear_Solver_ILU_n; } + const CIluOptions& GetIluOptions(void) const { return IluOptions; } /*! - * \brief Get whether to use level scheduling for OMP parallelization of ILU. + * \brief Get the SIMPLE (and PISO) algorithm options, see CSIMPLE_Options. */ - bool GetLinear_Solver_ILU_levels(void) const { return Linear_Solver_ILU_levels; } + const CSIMPLE_Options& GetSIMPLE_Options(void) const { return SIMPLE_Options; } /*! * \brief Get restart frequency of the linear solver for the implicit formulation. @@ -5355,6 +5422,12 @@ class CConfig { */ bool GetBoolTurbomachinery(void) const { return (nMarker_Turbomachinery !=0);} + /*! + * \brief Verify if a sliding plane for relative frame is specified from config file. + * \return boolean. + */ + bool GetBoolRelFrame_SlidingPlane(void) const { return (RelFrame_SlidingPlane !=0);} + /*! * \brief number Turbomachinery blades computed using the pitch information. * \return nBlades. @@ -5632,14 +5705,14 @@ class CConfig { unsigned short GetnVar(void); /*! - * \brief Provides the number of variables. - * \return Number of variables. + * \brief Provides the total number of zones. + * \return Total number of zones. */ unsigned short GetnZone(void) const { return nZone; } /*! - * \brief Provides the number of variables. - * \return Number of variables. + * \brief Provides the zone index the configuration belongs to. + * \return Zone index. */ unsigned short GetiZone(void) const { return iZone; } @@ -5703,17 +5776,21 @@ class CConfig { /*--- we keep the original Mesh_FileName ---*/ string meshFilename = Mesh_FileName; - /*--- strip the extension, only if it is .su2 or .cgns ---*/ + /*--- strip the extension, only if it is .su2, .su2b or .cgns ---*/ PrintingToolbox::TrimExtension(".su2",meshFilename); + PrintingToolbox::TrimExtension(".su2b",meshFilename); PrintingToolbox::TrimExtension(".cgns",meshFilename); switch (GetMesh_FileFormat()) { - case SU2: - case RECTANGLE: - case BOX: + case ENUM_GRID::SU2: + case ENUM_GRID::RECTANGLE: + case ENUM_GRID::BOX: meshFilename += ".su2"; break; - case CGNS_GRID: + case ENUM_GRID::SU2_BIN: + meshFilename += ".su2b"; + break; + case ENUM_GRID::CGNS_GRID: meshFilename += ".cgns"; break; default: @@ -5727,6 +5804,9 @@ class CConfig { /*! * \brief Get name of the output grid, this parameter is important for grid * adaptation and deformation. + * \note The returned name does not include the extension, it is the + * responsibility of the caller (usually a CFileWriter) to append it, + * consistent with GetMesh_Out_FileExtension(). * \return File name of the output grid. */ string GetMesh_Out_FileName(void) const { @@ -5734,13 +5814,31 @@ class CConfig { /*--- we keep the original Mesh_Out_FileName ---*/ string meshFilename = Mesh_Out_FileName; - /*--- strip the extension, only if it is .su2 or .cgns ---*/ + /*--- strip the extension, only if it is .su2, .su2b or .cgns ---*/ PrintingToolbox::TrimExtension(".su2",meshFilename); + PrintingToolbox::TrimExtension(".su2b",meshFilename); PrintingToolbox::TrimExtension(".cgns",meshFilename); return meshFilename; } + /*! + * \brief Get the extension (including the leading dot) associated with the + * current mesh output format. + * \return Extension of the output grid file. + */ + string GetMesh_Out_FileExtension(void) const { + switch (GetMesh_Out_FileFormat()) { + case ENUM_GRID::SU2: + return ".su2"; + case ENUM_GRID::SU2_BIN: + return ".su2b"; + default: + SU2_MPI::Error("Unrecognized mesh_out format specified!", CURRENT_FUNCTION); + return ""; + } + } + /*! * \brief Get the name of the file with the solution of the flow problem. * \return Name of the file with the solution of the flow problem. @@ -5776,11 +5874,17 @@ class CConfig { } /*! - * \brief Get the format of the input/output grid. - * \return Format of the input/output grid. + * \brief Get the format of the input grid. + * \return Format of the input grid. */ unsigned short GetMesh_FileFormat(void) const { return Mesh_FileFormat; } + /*! + * \brief Get the format of the output grid. + * \return Format of the output grid. + */ + unsigned short GetMesh_Out_FileFormat(void) const { return Mesh_Out_FileFormat; } + /*! * \brief Get the format of the output solution. * \return Format of the output solution. @@ -10106,6 +10210,11 @@ class CConfig { */ bool GetEdgeColoringRelaxDiscAdj() const { return edgeColoringRelaxDiscAdj; } + /*! + * \brief Get the number of concurrent BFS fronts used to build the RCM reordering, see SetRCM_Ordering. + */ + unsigned short GetRCM_NumSeeds(void) const { return rcmNumSeeds; } + /*! * \brief Get the ParMETIS load balancing tolerance. */ @@ -10128,6 +10237,13 @@ class CConfig { */ short FindInterfaceMarker(unsigned short iInterface) const; + /*! + * \brief Find the marker index (if any) that is part of a mixing plane interface pair. + * \param[in] nMarker - Number of the marker in a zone being tested, starting at 0. + * \return value > 1 if (on this mpi rank) the zone defined by config is part of the mixing plane. + */ + short FindMixingPlaneInterfaceMarker(unsigned short nMarker, unsigned short iMarkerInt) const; + /*! * \brief Get whether or not to save solution data to libROM. * \return True if specified in config file. @@ -10265,4 +10381,12 @@ class CConfig { */ const FluidFlamelet_ParsedOptions& GetFlameletParsedOptions() const { return flamelet_ParsedOptions; } + /*! + * \brief Get the enthalpy BC mode for the flamelet solver. + * FLOW_MARKERS: derive enthalpy BCs from MARKER_ISOTHERMAL/MARKER_HEATFLUX/MARKER_INLET (temperature-based). + * SPECIES_MARKERS: take enthalpy BCs directly from MARKER_WALL_SPECIES/MARKER_INLET_SPECIES. + * \return FLAMELET_ENTHALPY_BC enum value. + */ + FLAMELET_ENTHALPY_BC GetFlamelet_Enthalpy_BC() const { return flamelet_ParsedOptions.enthalpy_bc; } + }; diff --git a/Common/include/basic_types/ad_structure.hpp b/Common/include/basic_types/ad_structure.hpp index 2945e4f9ef84..294af302dc4d 100644 --- a/Common/include/basic_types/ad_structure.hpp +++ b/Common/include/basic_types/ad_structure.hpp @@ -1,7 +1,7 @@ /*! * \file ad_structure.hpp * \brief Main routines for the algorithmic differentiation (AD) structure. - * \author T. Albring, J. Blühdorn + * \author T. Albring, J. Blühdorn, O. Burghardt * \version 8.5.0 "Harrier" * * SU2 Project Website: https://su2code.github.io @@ -36,7 +36,21 @@ * In case there is no reverse type configured, they have no effect at all, * and so the real versions of the routined are after #else. */ + namespace AD { + +enum class TAPE_DEBUG_OPTION { + ALLOW_PREACC, + ACTIVATE_PREACC, + ALLOW_ZONE, + ALLOW_ALL_ZONES, + ACTIVATE_ALL_ZONES, + ACTIVATE_ALL_ERRORS, + MULTIZONE_TAGS, + INIT_RUN, + CHECK_RUN +}; + #ifndef CODI_REVERSE_TYPE using Identifier = int; @@ -287,42 +301,73 @@ inline void SetIndex(Identifier& index, const su2double& data) {} */ inline void SetTag(int tag) {} +/*! + * \brief Gets the current tag. + * \param[in] tag - the number to which the tag is set. + */ +inline int GetTag() { return 0; } + +/*! + * \brief Compute the zone-specific tag. + * \param[in] iZone - Zone index from which the zone-specific tag is formed. + * \return The zone-specific tag. + */ +inline int ComputeTag(unsigned short iZone) { return 0; } + /*! * \brief Sets the tag of a variable to 0. * \param[in] v - the variable whose tag is cleared. */ inline void ClearTagOnVariable(su2double& v) {} +/*! + * \brief Sets the tag of a variable to a specified value. + * \param[in] v - the variable whose tag is set manually. + */ +inline void SetTagOnVariable(su2double& v, int zone_tag = 0, int run_tag = 0) {} + /*! * \brief Struct to store information about errors during a tag debug run. */ -struct ErrorReport {}; +struct DebugControl {}; /*! - * \brief Set a reference to the output file of an ErrorReport. - * \param[in] report - the ErrorReport whose output file is set. - * \param[in] output_file - pointer to the output file. + * \brief Set a pointer to the current DebugControl. + * \param[in] control - pointer to the current debug control. */ -inline void SetDebugReportFile(ErrorReport& report, std::ostream* output_file) {} +inline void SetDebugControl(DebugControl* control) {} /*! - * \brief Set the ErrorReport to which error information from a tag debug recording is written. - * \param[in] report - the ErrorReport to which error information is written. + * \brief Set options which kind of tag mismatches are considered errors and written to file. + * \param[in] option - specification which kind of tag mismatches are considered. + * \param[in] izone - the zone w.r.t. which a tag mismatch is allowed. */ -inline void SetTagErrorCallback(ErrorReport& report) {} +inline void SetTapeDebugOption(TAPE_DEBUG_OPTION option, unsigned short izone = 0) {} /*! - * \brief Reset the error counter in an ErrorReport. - * \param[in] report - the ErrorReport whose error counter is resetted. + * \brief Activate the error callback by letting the tape call tagErrorCallback whenever a tag mismatch arises. */ -inline void ResetErrorCounter(ErrorReport& report) {} +inline void ActivateTagErrorCallback() {} /*! - * \brief Get the error count of an ErrorReport. - * \param[in] report - the ErrorReport whose pointer to its error counter is returned. + * \brief Set a pointer to the output file of a DebugControl. + * \param[in] control - the DebugControl whose output file is set. + * \param[in] output_file - pointer to the output file. + */ +inline void SetDebugReportFile(DebugControl& control, std::ostream* output_file) {} + +/*! + * \brief Reset the error counter in a DebugControl. + * \param[in] control - the DebugControl whose error counter is resetted. + */ +inline void ResetErrorCounter(DebugControl& control) {} + +/*! + * \brief Get the error count of a DebugControl. + * \param[in] control - the DebugControl whose error count is reported. * \return Value of the error counter. */ -inline unsigned long GetErrorCount(const ErrorReport& report) { return 0; } +inline unsigned long GetErrorCount(const DebugControl& control) { return 0; } /*! * \brief Pushes back the current tape position to the tape position's vector. @@ -744,37 +789,142 @@ FORCEINLINE void ResumePreaccumulation(bool wasActive) { SU2_OMP_SAFE_GLOBAL_ACCESS(PreaccEnabled = true;) } -struct ErrorReport { +#ifdef CODI_TAG_TAPE + +struct DebugControl { + int current_tag; + bool multizone_tags = false; + bool init_run = false; + bool allow_preacc = false; + bool allow_zones = false; + unsigned short allow_izone = 0; unsigned long ErrorCounter = 0; std::ostream* out = &std::cout; }; -FORCEINLINE void ResetErrorCounter(ErrorReport& report) { report.ErrorCounter = 0; } +struct AdjustDebugControl { + TAPE_DEBUG_OPTION option; + void (*adjust)(DebugControl*, unsigned short); +}; -FORCEINLINE void SetDebugReportFile(ErrorReport& report, std::ostream* output_file) { report.out = output_file; } +static const AdjustDebugControl debug_control_adjustments[] = { + {TAPE_DEBUG_OPTION::ALLOW_PREACC, [](DebugControl* c, unsigned short) { c->allow_preacc = true; }}, + {TAPE_DEBUG_OPTION::ACTIVATE_PREACC, [](DebugControl* c, unsigned short) { c->allow_preacc = false; }}, + {TAPE_DEBUG_OPTION::ALLOW_ZONE, [](DebugControl* c, unsigned short zone) { c->allow_izone = zone + 1; }}, + {TAPE_DEBUG_OPTION::ALLOW_ALL_ZONES, [](DebugControl* c, unsigned short) { c->allow_zones = true; }}, + {TAPE_DEBUG_OPTION::ACTIVATE_ALL_ZONES, + [](DebugControl* c, unsigned short) { + c->allow_zones = false; + c->allow_izone = 0; + }}, + {TAPE_DEBUG_OPTION::ACTIVATE_ALL_ERRORS, + [](DebugControl* c, unsigned short) { + c->allow_preacc = false; + c->allow_zones = false; + c->allow_izone = 0; + }}, + {TAPE_DEBUG_OPTION::INIT_RUN, [](DebugControl* c, unsigned short) { c->init_run = true; }}, + {TAPE_DEBUG_OPTION::CHECK_RUN, [](DebugControl* c, unsigned short) { c->init_run = false; }}, + {TAPE_DEBUG_OPTION::MULTIZONE_TAGS, [](DebugControl* c, unsigned short) { c->multizone_tags = true; }}, +}; -FORCEINLINE unsigned long GetErrorCount(const ErrorReport& report) { return report.ErrorCounter; } +FORCEINLINE void ResetErrorCounter(DebugControl& control) { control.ErrorCounter = 0; } -#ifdef CODI_TAG_TAPE +FORCEINLINE void SetDebugReportFile(DebugControl& control, std::ostream* output_file) { control.out = output_file; } + +FORCEINLINE unsigned long GetErrorCount(const DebugControl& control) { return control.ErrorCounter; } + +extern DebugControl* current_control; + +FORCEINLINE void SetDebugControl(DebugControl* control) { current_control = control; } + +FORCEINLINE void SetTag(int tag) { + current_control->current_tag = tag; + AD::getTape().setCurTag(tag); +} + +FORCEINLINE int GetTag() { return current_control->current_tag; } + +FORCEINLINE int ComputeTag(unsigned short iZone) { + if (current_control->init_run) { + return (current_control->multizone_tags) ? ((int)iZone + 1) * 10 + 1 : 1; + } else { + return (current_control->multizone_tags) ? ((int)iZone + 1) * 10 + 2 : 2; + } +} -FORCEINLINE void SetTag(int tag) { AD::getTape().setCurTag(tag); } FORCEINLINE void ClearTagOnVariable(su2double& v) { AD::getTape().clearTagOnVariable(v); } +FORCEINLINE void SetTagOnVariable(su2double& v, int zone_tag = 0, int run_tag = 0) { + int tag = v.getIdentifier().tag; + int tens = (zone_tag > 0) ? zone_tag + 1 : tag / 10; + int ones = (run_tag > 0 && run_tag < 10) ? run_tag : tag % 10; + if (tag != 0) { + v.getIdentifier().tag = tens * 10 + ones; + } +} + static void tagErrorCallback(const int& correctTag, const int& wrongTag, void* userData) { - auto* report = static_cast(userData); + auto* status = static_cast(userData); + + bool throw_mismatch_error = true; - report->ErrorCounter += 1; - *(report->out) << "Use of variable with bad tag '" << wrongTag << "', should be '" << correctTag << "'." << std::endl; + /*--- The callback could be due to a preaccumulation tag mismatch that we maybe want to allow, ... ---*/ + if (status->allow_preacc) { + if (correctTag == 1337 || wrongTag == 1337) { + throw_mismatch_error = false; + } + } + + /*--- ... or to a mismatch in the zone part of the tag. ---*/ + if (status->allow_zones) { + throw_mismatch_error = false; + } else if (status->allow_izone > 0) { + if (wrongTag / 10 == status->allow_izone) { + throw_mismatch_error = false; + } + } + + if (throw_mismatch_error) { + status->ErrorCounter += 1; + *(status->out) << "Use of variable with bad tag '" << std::setw(2) << std::setfill('0') << wrongTag + << "', should be '" << std::setw(2) << std::setfill('0') << correctTag << "'." << std::endl; + } } -FORCEINLINE void SetTagErrorCallback(ErrorReport& report) { - AD::getTape().setTagErrorCallback(tagErrorCallback, &report); +FORCEINLINE void SetTapeDebugOption(TAPE_DEBUG_OPTION option, unsigned short izone = 0) { + if (current_control == nullptr) { + return; + } + for (const AdjustDebugControl& entry : debug_control_adjustments) { + if (entry.option == option) { + entry.adjust(current_control, izone); + break; + } + } +} + +FORCEINLINE void ActivateTagErrorCallback() { + if (current_control != NULL) { + AD::getTape().setTagErrorCallback(tagErrorCallback, current_control); + } else { + std::cout << "No tape debug control set!" << std::endl; + } } #else +struct DebugControl {}; +FORCEINLINE void ResetErrorCounter(DebugControl& control) {} +FORCEINLINE void SetDebugReportFile(DebugControl& control, std::ostream* output_file) {} +FORCEINLINE unsigned long GetErrorCount(const DebugControl& control) { return 0; } +FORCEINLINE void SetDebugControl(DebugControl* status) {} +FORCEINLINE int GetTag() { return 0; } FORCEINLINE void SetTag(int tag) {} +FORCEINLINE int ComputeTag(unsigned short iZone) { return 0; } FORCEINLINE void ClearTagOnVariable(su2double& v) {} -FORCEINLINE void SetTagErrorCallback(ErrorReport report) {} +FORCEINLINE void SetTagOnVariable(su2double& v, int zone_tag = 0, int run_tag = 0) {} +FORCEINLINE void SetTapeDebugOption(TAPE_DEBUG_OPTION option, unsigned short izone = 0) {} +FORCEINLINE void ActivateTagErrorCallback() {} #endif // CODI_TAG_TAPE diff --git a/Common/include/basic_types/codi_host_only.hpp b/Common/include/basic_types/codi_host_only.hpp new file mode 100644 index 000000000000..e7b6768da44a --- /dev/null +++ b/Common/include/basic_types/codi_host_only.hpp @@ -0,0 +1,49 @@ +/*! + * \file codi_host_only.hpp + * \brief Keeps CoDiPack out of nvcc's device pass. + * \author P. Gomes + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#pragma once + +/*--- Must be the first include of every .cu, before anything can pull in codi.hpp. + * + * When __CUDA_ARCH__ is defined, i.e. only in the device pass, CoDiPack stamps + * "__device__ __host__" on every one of its functions. nvcc then has to generate device + * code for the entire tape, which does not work: the tape uses function-scope statics with + * dynamic initializers (illegal in device code), takes the address of host statics, and + * calls into std::map and std::bitset. The kernels never touch an active type, so none of + * that is wanted in the first place. + * + * CODI_INLINE expands CODI_CUDAFunctionAttributes at each declaration, so emptying the + * macro before CoDiPack is parsed leaves it host-only. Including the defining header first + * makes the copy in config.h a no-op (it is "#pragma once"), so this definition is the one + * that survives. The host pass is unaffected either way, it never defines __CUDA_ARCH__ + * and so already sees the same host-only declarations as the .cpp translation units. ---*/ +#if defined(CODI_REVERSE_TYPE) || defined(CODI_FORWARD_TYPE) +#include "codi/tools/cuda/cudaFunctionAttributes.hpp" + +#undef CODI_CUDAFunctionAttributes +#define CODI_CUDAFunctionAttributes +#endif diff --git a/Common/include/basic_types/datatype_structure.hpp b/Common/include/basic_types/datatype_structure.hpp index 623c410e5510..d9b6bb0b58fb 100644 --- a/Common/include/basic_types/datatype_structure.hpp +++ b/Common/include/basic_types/datatype_structure.hpp @@ -135,6 +135,10 @@ template <> struct Passive { FORCEINLINE static passivedouble Value(const su2double& val) { return GetValue(val); } }; +template +FORCEINLINE auto PassiveValue(const T& val) { + return Passive::Value(val); +} /*! * \brief Casts the primitive value to int (uses GetValue, already implemented for each type). diff --git a/Common/include/code_config.hpp b/Common/include/code_config.hpp index 491777a01f3d..61d143f40a7b 100644 --- a/Common/include/code_config.hpp +++ b/Common/include/code_config.hpp @@ -26,7 +26,9 @@ */ #pragma once +#include #include +#include #if defined(_MSC_VER) #define PRAGMIZE(X) __pragma(X) @@ -48,6 +50,13 @@ #define NEVERINLINE inline #endif +/*--- Marks a function callable from both host and device code, a no-op outside nvcc. ---*/ +#ifdef __CUDACC__ +#define SU2_CUDA_HOST_DEVICE __host__ __device__ +#else +#define SU2_CUDA_HOST_DEVICE +#endif + #if defined(__INTEL_COMPILER) /*--- Disable warnings related to inline attributes. ---*/ #pragma warning disable 2196 @@ -94,6 +103,44 @@ FORCEINLINE Out su2staticcast_p(In ptr) { #define HAVE_OMP #endif +/*--- Detect whether the CUDA kernels are part of this build. They work on su2mixedfloat + * and passivedouble, so reverse AD is fine: su2_gpu_capable_v is false for the active type + * and CSysMatrix stays on the host. Forward AD is not, there su2mixedfloat is + * su2double (see below) and the active type would reach the kernels. The .cu sources are + * compiled with the CoDiPack defines like everything else, see codi_host_only.hpp. ---*/ +#if defined(HAVE_CUDA) && !defined(CODI_FORWARD_TYPE) +#define SU2_ENABLE_CUDA_KERNELS +#endif + +/*--- No full single precision for AD builds. ---*/ +#if (defined(CODI_REVERSE_TYPE) || defined(CODI_FORWARD_TYPE)) && defined(USE_SINGLE_PRECISION) +#undef USE_SINGLE_PRECISION +#endif + +/*--- Default integer types. Currently used for rank-local sparse patterns. ---*/ +using su2uint = uint32_t; +using su2int = int32_t; + +/*--- This type can be used for (rare) compatibility cases or for + * computations that are intended to be (always) passive. ---*/ +#ifdef USE_SINGLE_PRECISION +using passivedouble = float; +#else +using passivedouble = double; +#endif + +/*--- std::min/max do not compile if the arguments have inconsistent types, which + * happens in single precision due to floating point literals (double by default). + * These overloads delegate to fmin/fmax which do not have that problem. ---*/ +#ifdef USE_SINGLE_PRECISION +namespace std { +FORCEINLINE float min(const float& a, const double& b) { return fmin(a, static_cast(b)); } +FORCEINLINE float min(const double& b, const float& a) { return fmin(a, static_cast(b)); } +FORCEINLINE float max(const float& a, const double& b) { return fmax(a, static_cast(b)); } +FORCEINLINE float max(const double& b, const float& a) { return fmax(a, static_cast(b)); } +} // namespace std +#endif + /*--- Depending on the datatype defined during the configuration, * include the correct definition, and create the main typedef. ---*/ @@ -131,13 +178,9 @@ using su2double = codi::RealReverseTag; #include "codi.hpp" using su2double = codi::RealForward; #else // primal / direct / no AD -using su2double = double; +using su2double = passivedouble; #endif -/*--- This type can be used for (rare) compatibility cases or for - * computations that are intended to be (always) passive. ---*/ -using passivedouble = double; - /*--- Define a type for potentially lower precision operations. ---*/ #ifndef CODI_FORWARD_TYPE #ifdef USE_MIXED_PRECISION diff --git a/Common/include/containers/C2DContainer.hpp b/Common/include/containers/C2DContainer.hpp index 85d6d463e115..380e43cfba7b 100644 --- a/Common/include/containers/C2DContainer.hpp +++ b/Common/include/containers/C2DContainer.hpp @@ -391,6 +391,8 @@ class C2DContainer static constexpr bool IsRowMajor = (Store == StorageType::RowMajor); static constexpr bool IsColumnMajor = (Store == StorageType::ColumnMajor); static constexpr size_t StaticSize = StaticRows * StaticCols; + static constexpr size_t StaticNRows = StaticRows; + static constexpr size_t StaticNCols = StaticCols; /*! * \brief Scalar iterator to the inner dimension of the container, read-only. diff --git a/Common/include/containers/CLookUpTable.hpp b/Common/include/containers/CLookUpTable.hpp index c41819521538..e6bf86d074d2 100644 --- a/Common/include/containers/CLookUpTable.hpp +++ b/Common/include/containers/CLookUpTable.hpp @@ -260,29 +260,6 @@ class CLookUpTable { void InterpolateToNearestNeighbors(const su2double val_CV1, const su2double val_CV2, const std::string& name_var, su2double* var_val, const unsigned long i_level = 0); - /*! - * \brief Determine if a point P(val_CV1,val_CV2) is inside the triangle val_id_triangle. - * \param[in] val_CV1 - First coordinate of point P(val_CV1,val_CV2) to check. - * \param[in] val_CV2 - Second coordinate of point P(val_CV1,val_CV2) to check. - * \param[in] val_id_triangle - ID of the triangle to check. - * \returns True if the point is in the triangle, false if it is outside. - */ - bool IsInTriangle(su2double val_CV1, su2double val_CV2, unsigned long val_id_triangle, unsigned long i_level = 0); - - /*! - * \brief Compute the area of a triangle given the 3 points of the triangle. - * \param[in] x1 - The coordinates of the points P1(x1,y1), P2(x2,y2) and P3(x3,y3). - * \param[in] y1 - The coordinates of the points P1(x1,y1), P2(x2,y2) and P3(x3,y3). - * \param[in] x2 - The coordinates of the points P1(x1,y1), P2(x2,y2) and P3(x3,y3). - * \param[in] y2 - The coordinates of the points P1(x1,y1), P2(x2,y2) and P3(x3,y3). - * \param[in] x3 - The coordinates of the points P1(x1,y1), P2(x2,y2) and P3(x3,y3). - * \param[in] y3 - The coordinates of the points P1(x1,y1), P2(x2,y2) and P3(x3,y3). - * \returns The absolute value of the area of the triangle. - */ - inline su2double TriArea(su2double x1, su2double y1, su2double x2, su2double y2, su2double x3, su2double y3) { - return abs((x1 * (y2 - y3) + x2 * (y3 - y1) + x3 * (y1 - y2)) * 0.5); - } - /*! * \brief Compute the values of the first and second controlling variable based on normalized query coordinates * \param[in] inclusion_levels - Pair containing lower(first) and upper(second) table inclusion level indices. diff --git a/Common/include/containers/CTrapezoidalMap.hpp b/Common/include/containers/CTrapezoidalMap.hpp index a083b80d9feb..996eaa03eeb8 100644 --- a/Common/include/containers/CTrapezoidalMap.hpp +++ b/Common/include/containers/CTrapezoidalMap.hpp @@ -1,7 +1,8 @@ /*! * \file CTrapezoidalMap.hpp - * \brief Implementation of the trapezoidal map for tabulation and lookup of fluid properties - * \author D. Mayer, T. Economon + * \brief Memory-efficient trapezoidal map for 2D lookup table queries, + * based on the LUT implementation of P. Gomes (https://github.com/pcarruscag/LUT). + * \author T. Kiymaz, P. Gomes * \version 8.5.0 "Harrier" * * SU2 Project Website: https://su2code.github.io @@ -27,82 +28,485 @@ #pragma once -#include +#include +#include +#include +#include +#include +#include #include -#include "../../Common/include/linear_algebra/blas_structure.hpp" -#include "../../Common/include/toolboxes/CSquareMatrixCM.hpp" +#include "../basic_types/datatype_structure.hpp" +#include "C2DContainer.hpp" + +namespace su2_lut { + +using IntT = int32_t; +using RealT = su2double; + +/*--- Row-major integer matrices with a compile-time number of columns. ---*/ +template +using MatrixNi = C2DContainer; +using Matrix2i = MatrixNi<2>; +using Matrix3i = MatrixNi<3>; +using VectorInt = std::vector; +using VectorReal = std::vector; + +/*--- The map is defined by the limits of the bands in the x direction and a CSR of + * the edge IDs in each band, sorted by the edge y position at the band midpoint. ---*/ +struct TrapezoidalMap { + VectorInt offsets, edge_id; + VectorReal x_bands, edge_y; +}; /*! - * \class CTrapezoidalMap - * \ingroup LookUpInterp - * \brief Construction of trapezoidal map for tabulated lookup - * \author: D. Mayer, T. Economon - * \version 8.5.0 "Harrier" + * \brief Orders points by ascending x coordinates and updates triangle indices. */ -class CTrapezoidalMap { - protected: - /* The unique values of x which exist in the data */ - std::vector unique_bands_x; +inline void ReorderPoints(Matrix3i& triangles, VectorReal& x, VectorReal& y) { + const IntT n_pts = static_cast(x.size()); + + std::vector perm(n_pts); + std::iota(perm.begin(), perm.end(), 0); + std::sort(perm.begin(), perm.end(), + [&x, &y](const auto i, const auto j) { return x[i] != x[j] ? x[i] < x[j] : y[i] < y[j]; }); + + auto reorder = [n_pts, &perm](const auto& v) { + VectorReal tmp(n_pts); + for (IntT i = 0; i < n_pts; ++i) { + tmp[i] = v[perm[i]]; + } + return tmp; + }; + x = reorder(x); + y = reorder(y); + + std::vector inv_perm(n_pts); + for (IntT i = 0; i < n_pts; ++i) { + inv_perm[perm[i]] = i; + } + for (IntT i = 0; i < static_cast(triangles.rows()); ++i) { + for (IntT j = 0; j < 3; ++j) { + triangles(i, j) = inv_perm[triangles(i, j)]; + } + } +} + +/*! + * \brief Extracts unique edges from triangles. Edges are defined by two point IDs and + * up to two adjacent triangles (boundary edges have the second triangle ID < 0). + */ +inline void ExtractEdges(const Matrix3i& triangles, Matrix2i& edge_pts, Matrix2i& edge_faces) { + std::vector> edges; + edges.resize(3 * triangles.rows()); + + for (IntT i_tri = 0; i_tri < static_cast(triangles.rows()); ++i_tri) { + for (IntT i = 0; i < 3; ++i) { + const IntT j = (i + 1) % 3; + const IntT i_pt = std::min(triangles(i_tri, i), triangles(i_tri, j)); + const IntT j_pt = std::max(triangles(i_tri, i), triangles(i_tri, j)); + edges[3 * i_tri + i] = {i_pt, j_pt, i_tri}; + } + } + + /*--- Sort to identify duplicates. ---*/ + std::sort(edges.begin(), edges.end(), + [](const auto& a, const auto& b) { return a[0] != b[0] ? (a[0] < b[0]) : (a[1] < b[1]); }); + + auto is_equal = [](const auto& a, const auto& b) { return a[0] == b[0] && a[1] == b[1]; }; + + IntT n_edges = 1; + for (IntT i = 1; i < static_cast(edges.size()); ++i) { + n_edges += static_cast(!is_equal(edges[i], edges[i - 1])); + } + + edge_pts.resize(n_edges, 2); + edge_faces.resize(n_edges, 2); + IntT pos = 0; + + auto new_edge = [&](const auto& edge) { + edge_pts(pos, 0) = edge[0]; + edge_pts(pos, 1) = edge[1]; + edge_faces(pos, 0) = edge[2]; + edge_faces(pos, 1) = -1; + ++pos; + }; + + new_edge(edges[0]); + for (IntT i = 1; i < static_cast(edges.size()); ++i) { + if (is_equal(edges[i], edges[i - 1])) { + edge_faces(pos - 1, 1) = edges[i][2]; + } else { + new_edge(edges[i]); + } + } +} + +/*! + * \brief Detects the x bands of the map. One band per unique x coordinate is used unless + * that would exceed max_bands, in which case equal-width bands are used to limit memory. + * \return Tuple of (n_bands, x_bands). + */ +inline auto DetectBands(const VectorReal& x, IntT max_bands = 0) { + if (max_bands <= 0) { + max_bands = std::min(IntT{5000}, static_cast(4.0 * std::sqrt(static_cast(x.size())))); + } + + IntT n_unique = 1; + for (IntT i = 1; i < static_cast(x.size()); ++i) { + if (x[i] != x[i - 1]) n_unique++; + } + + if (n_unique <= max_bands) { + const IntT n_bands = n_unique - 1; + VectorReal x_bands(n_unique); + IntT pos = 0; + x_bands[pos] = x[0]; + for (IntT i = 1; i < static_cast(x.size()); ++i) { + if (x[i] != x_bands[pos]) { + x_bands[++pos] = x[i]; + } + } + return std::make_tuple(n_bands, std::move(x_bands)); + } + + const RealT x_min = x.front(); + const RealT x_max = x.back(); + const RealT band_width = (x_max - x_min) / max_bands; + + VectorReal x_bands(max_bands + 1); + for (IntT i = 0; i <= max_bands; ++i) { + x_bands[i] = x_min + i * band_width; + } + x_bands[max_bands] = x_max; + + return std::make_tuple(max_bands, std::move(x_bands)); +} + +/*! + * \brief Builds the trapezoidal map for a set of edges (points must be ordered by x). + */ +inline void BuildTrapezoidalMap(const Matrix2i& edge_pts, const VectorReal& x, const VectorReal& y, + TrapezoidalMap& map) { + auto& x_bands = map.x_bands; + auto& offsets = map.offsets; + auto& edge_id = map.edge_id; + auto& edge_y = map.edge_y; + + auto clear_map = [&]() { + x_bands.clear(); + offsets.clear(); + edge_id.clear(); + edge_y.clear(); + }; + + const auto [n_bands, bands] = DetectBands(x); + x_bands = std::move(bands); + + if (n_bands <= 0) { + clear_map(); + return; + } + + auto find_band = [&x_bands, n_bands = n_bands](RealT x_val) -> IntT { + auto it = std::lower_bound(x_bands.begin(), x_bands.end(), x_val); + const IntT idx = static_cast(it - x_bands.begin()); + return std::min(std::max(IntT{0}, idx - 1), n_bands - 1); + }; + + /*--- Count edges per band. Each edge is stored in every band between the bands of its + * two endpoints (inclusive), a superset of the bands it overlaps. ---*/ + auto& counts = offsets; + counts.clear(); + counts.resize(n_bands + 1, 0); + + for (IntT i = 0; i < static_cast(edge_pts.rows()); ++i) { + const IntT band_0 = find_band(x[edge_pts(i, 0)]); + const IntT band_1 = find_band(x[edge_pts(i, 1)]); + + for (IntT j = std::min(band_0, band_1); j <= std::max(band_0, band_1); ++j) { + ++counts[j + 1]; + } + } + + /*--- Convert counts to offsets (CSR format). ---*/ + for (IntT i = 2; i < static_cast(offsets.size()); ++i) { + offsets[i] += offsets[i - 1]; + } + + /*--- Give up (build failure) rather than allocating an excessive amount of memory. ---*/ + const size_t memory_mb = static_cast(offsets.back()) * (sizeof(IntT) + sizeof(RealT)) / (1024 * 1024); + if (memory_mb > 2048) { + clear_map(); + return; + } + + edge_id.resize(offsets.back()); + edge_y.resize(offsets.back()); + auto pos = offsets; + + for (IntT i_edge = 0; i_edge < static_cast(edge_pts.rows()); ++i_edge) { + const IntT pt_0 = edge_pts(i_edge, 0); + const IntT pt_1 = edge_pts(i_edge, 1); + const RealT x_0 = x[pt_0], y_0 = y[pt_0]; + const RealT x_1 = x[pt_1], y_1 = y[pt_1]; + + const IntT band_0 = find_band(x_0); + const IntT band_1 = find_band(x_1); + + const RealT dx = x_1 - x_0; + const bool vertical = std::abs(SU2_TYPE::GetValue(dx)) < 1e-30; + const RealT dy_dx = vertical ? RealT{0} : (y_1 - y_0) / dx; + + for (IntT j = std::min(band_0, band_1); j <= std::max(band_0, band_1); ++j) { + edge_id[pos[j]] = i_edge; + const RealT x_mid = (x_bands[j] + x_bands[j + 1]) / 2; + edge_y[pos[j]] = vertical ? RealT((y_0 + y_1) / 2) : RealT(y_0 + dy_dx * (x_mid - x_0)); + ++pos[j]; + } + } - su2activematrix edge_limits_x; - su2activematrix edge_limits_y; + /*--- Sort the edges in each band by y coordinate. ---*/ + std::vector> tmp; + for (IntT i = 0; i < n_bands; ++i) { + const IntT begin = offsets[i]; + const IntT end = offsets[i + 1]; + if (begin >= end) continue; - su2vector > edge_to_triangle; + tmp.resize(end - begin); + for (auto k = begin; k < end; ++k) { + tmp[k - begin] = {edge_id[k], edge_y[k]}; + } + std::sort(tmp.begin(), tmp.end(), [](const auto& a, const auto& b) { return a.second < b.second; }); + for (auto k = begin; k < end; ++k) { + edge_id[k] = tmp[k - begin].first; + edge_y[k] = tmp[k - begin].second; + } + } +} + +/*! + * \brief Returns the IDs of the edges directly below and above a query point + * (either ID can be -1 if the point is at a boundary). + */ +inline auto QueryTrapezoidalMap(const TrapezoidalMap& map, const Matrix2i& edge_pts, const VectorReal& x_coords, + const VectorReal& y_coords, const RealT& x, const RealT& y) { + if (map.x_bands.size() < 2 || map.offsets.empty()) { + return std::make_pair(IntT{-1}, IntT{-1}); + } + + const auto& x_bands = map.x_bands; + const IntT n_bands = static_cast(x_bands.size()) - 1; + auto it = std::lower_bound(x_bands.begin(), x_bands.end(), x); + const IntT d = static_cast(it - x_bands.begin()); + const IntT band_idx = std::min(std::max(IntT{0}, d - 1), n_bands - 1); + + RealT best_y_below = -1e300; + RealT best_y_above = 1e300; + IntT edge_below = -1; + IntT edge_above = -1; + + const IntT begin = map.offsets[band_idx]; + const IntT end = map.offsets[band_idx + 1]; - /* The value that each edge which intersects the band takes within that - * same band. Used to sort the edges */ - su2vector > > y_edge_at_band_mid; + for (IntT k = begin; k < end; ++k) { + const IntT e_id = map.edge_id[k]; - double memory_footprint = 0; + const IntT p0 = edge_pts(e_id, 0); + const IntT p1 = edge_pts(e_id, 1); + const RealT x0 = x_coords[p0], y0 = y_coords[p0]; + const RealT x1 = x_coords[p1], y1 = y_coords[p1]; + + if (x < std::min(x0, x1) - 1e-10 || x > std::max(x0, x1) + 1e-10) { + continue; + } + + /*--- y position of the edge at the query x. ---*/ + RealT edge_y_at_x; + const RealT dx = x1 - x0; + if (std::abs(SU2_TYPE::GetValue(dx)) < 1e-30) { + edge_y_at_x = (y0 + y1) / 2.0; + } else { + edge_y_at_x = y0 + (x - x0) / dx * (y1 - y0); + } + + if (edge_y_at_x <= y + 1e-10 && edge_y_at_x > best_y_below) { + best_y_below = edge_y_at_x; + edge_below = e_id; + } + if (edge_y_at_x >= y - 1e-10 && edge_y_at_x < best_y_above) { + best_y_above = edge_y_at_x; + edge_above = e_id; + } + } + + return std::make_pair(edge_below, edge_above); +} + +/*! + * \brief Returns the IDs of the triangles adjacent to two query edges (up to 3 triangles). + */ +inline auto AdjacentTriangles(const IntT edge_0, const IntT edge_1, const Matrix2i& edge_faces) { + std::array tris = {-1, -1, -1}; + IntT pos = 0; + + auto insert = [&tris, &pos](const IntT t) { + if (t < 0) return; + for (IntT i = 0; i < pos; ++i) { + if (t == tris[i]) return; + } + tris[pos++] = t; + }; + + auto get_tris = [&edge_faces](const IntT e) { + if (e < 0) return std::array{IntT{-1}, IntT{-1}}; + return std::array{edge_faces(e, 0), edge_faces(e, 1)}; + }; + + for (const auto e : {edge_0, edge_1}) { + for (const auto t : get_tris(e)) { + insert(t); + } + } + return tris; +} + +/*! + * \brief Computes the barycentric coordinates of point (x_q, y_q) in a triangle. + */ +inline auto TriangleCoords(const IntT i_tri, const Matrix3i& triangles, const VectorReal& x, const VectorReal& y, + const RealT x_q, const RealT y_q) { + const IntT p0 = triangles(i_tri, 0); + const IntT p1 = triangles(i_tri, 1); + const IntT p2 = triangles(i_tri, 2); + + const RealT x0 = x[p0], y0 = y[p0]; + const RealT x1 = x[p1], y1 = y[p1]; + const RealT x2 = x[p2], y2 = y[p2]; + + const RealT dx1 = x1 - x0, dy1 = y1 - y0; + const RealT dx2 = x2 - x0, dy2 = y2 - y0; + + auto cross = [](const RealT ux, const RealT uy, const RealT vx, const RealT vy) { return ux * vy - uy * vx; }; + + const RealT det = cross(dx1, dy1, dx2, dy2); + if (std::abs(SU2_TYPE::GetValue(det)) < 1e-30) { + return std::array{RealT{0}, RealT{0}, RealT{0}}; + } + + const RealT inv_det = 1.0 / det; + const RealT a = (cross(x_q, y_q, dx2, dy2) - cross(x0, y0, dx2, dy2)) * inv_det; + const RealT b = (cross(x0, y0, dx1, dy1) - cross(x_q, y_q, dx1, dy1)) * inv_det; + + return std::array{1 - a - b, a, b}; +} + +/*! + * \brief Checks if a point is inside a triangle based on its barycentric coordinates. + */ +inline bool InTriangle(const std::array& coords, const RealT tol = 0.0) { + return coords[0] >= -tol && coords[1] >= -tol && coords[2] >= -tol; +} + +/*! + * \brief Finds the triangle containing a point using the trapezoidal map. + */ +inline IntT FindTriangle(const TrapezoidalMap& map, const Matrix3i& triangles, const Matrix2i& edge_pts, + const Matrix2i& edge_faces, const VectorReal& x, const VectorReal& y, const RealT x_q, + const RealT y_q, std::array& bary_out) { + const auto [e_below, e_above] = QueryTrapezoidalMap(map, edge_pts, x, y, x_q, y_q); + const auto candidates = AdjacentTriangles(e_below, e_above, edge_faces); + + const RealT tol = 1e-12; + for (const auto t : candidates) { + if (t < 0) continue; + + const auto coords = TriangleCoords(t, triangles, x, y, x_q, y_q); + if (InTriangle(coords, tol)) { + bary_out = coords; + return t; + } + } + + bary_out = {0.0, 0.0, 0.0}; + return -1; +} + +} // namespace su2_lut + +/*! + * \class CTrapezoidalMap + * \ingroup LookUpInterp + * \brief Trapezoidal map for finding the triangle containing a query point in a 2D triangulation. + */ +class CTrapezoidalMap { + private: + su2_lut::Matrix3i triangles; + su2_lut::Matrix2i edge_pts, edge_faces; + su2_lut::VectorReal x_coords, y_coords; + su2_lut::TrapezoidalMap map; + + unsigned long n_points = 0; + unsigned long n_triangles = 0; public: CTrapezoidalMap() = default; - CTrapezoidalMap(const su2double* samples_x, const su2double* samples_y, const unsigned long size, - const std::vector >& edges, - const su2vector >& edge_to_triangle, bool display = false); - /*! - * \brief return the index to the triangle that contains the coordinates (val_x,val_y) - * \param[in] val_x - x-coordinate or first independent variable - * \param[in] val_y - y-coordinate or second independent variable - * \param[out] val_index - index to the triangle + * \brief Build the trapezoidal map from a triangulation. + * \return True on success. */ - unsigned long GetTriangle(const su2double val_x, const su2double val_y); + bool Build(unsigned long num_points, unsigned long num_triangles, const su2double* x, const su2double* y, + const unsigned long* connectivity) { + n_points = num_points; + n_triangles = num_triangles; - /*! - * \brief get the indices of the vertical coordinate band (xmin,xmax) in the 2D search space - * that contains the coordinate val_x - * \param[in] val_x - x-coordinate or first independent variable - * \param[out] val_band - a pair(i_low,i_up) , the lower index and upper index between which the value val_x - * can be found - */ - std::pair GetBand(const su2double val_x); + if (num_points == 0 || num_triangles == 0) return false; - /*! - * \brief for a given coordinate (val_x,value), known to be in the band (xmin,xmax) with band index (i_low,i_up), - * find the edges in the band (these edges come from the triangulation) that enclose the coordinate - * \param[in] val_band - pair i_low,i_up - * \param[in] val_x - x-coordinate or first independent variable - * \param[in] val_y - y-coordinate or first independent variable - * \param[out] pair (edge_low,edge_up) - lower edge and upper edge of a triangle that encloses the coordinate - */ - std::pair GetEdges(std::pair val_band, su2double val_x, - su2double val_y) const; + x_coords.assign(x, x + num_points); + y_coords.assign(y, y + num_points); + + triangles.resize(num_triangles, 3); + for (size_t i = 0; i < 3 * num_triangles; ++i) { + triangles.data()[i] = static_cast(connectivity[i]); + } + + su2_lut::ReorderPoints(triangles, x_coords, y_coords); + su2_lut::ExtractEdges(triangles, edge_pts, edge_faces); + su2_lut::BuildTrapezoidalMap(edge_pts, x_coords, y_coords, map); + + return !map.x_bands.empty() && !map.offsets.empty(); + } /*! - * \brief determine if the x-coordinate falls within the bounds xmin,xmax of the table - * \param[in] val_x - x-coordinate or first independent variable - * \param[out] bool - true if val_x is within (xmin,xmax) + * \brief Find the triangle containing a query point. + * \return True if the point is inside the triangulation. */ - inline bool IsInsideHullX(su2double val_x) { - return (val_x >= unique_bands_x.front()) && (val_x <= unique_bands_x.back()); + bool FindTriangle(su2double val_x, su2double val_y, unsigned long& triangle_id, + std::array& bary_coords) const { + if (n_triangles == 0 || n_points == 0 || map.x_bands.empty()) { + bary_coords = {0.0, 0.0, 0.0}; + return false; + } + + std::array bary; + const su2_lut::IntT tri_id = + su2_lut::FindTriangle(map, triangles, edge_pts, edge_faces, x_coords, y_coords, val_x, val_y, bary); + + if (tri_id < 0) return false; + + triangle_id = static_cast(tri_id); + bary_coords = {bary[0], bary[1], bary[2]}; + return true; } /*! - * \brief get memory footprint of trapezoidal map. - * \return - memory footprint in mega bytes. + * \brief Get the memory footprint of the map in MB. */ - double GetMemoryFootprint() const { return memory_footprint; } + double GetMemoryFootprint() const { + const size_t bytes = + (map.edge_id.size() + map.offsets.size() + edge_pts.size() + edge_faces.size() + triangles.size()) * + sizeof(su2_lut::IntT) + + (map.edge_y.size() + map.x_bands.size() + x_coords.size() + y_coords.size()) * sizeof(su2_lut::RealT); + return double(bytes) / (1024.0 * 1024.0); + } }; diff --git a/Common/include/containers/container_decorators.hpp b/Common/include/containers/container_decorators.hpp index a9e66857eb33..331e87b9a0f4 100644 --- a/Common/include/containers/container_decorators.hpp +++ b/Common/include/containers/container_decorators.hpp @@ -62,6 +62,33 @@ class CMatrixView { const Scalar* operator[](Index i) const noexcept { return &m_ptr[i * m_cols]; } const Scalar& operator()(Index i, Index j) const noexcept { return m_ptr[i * m_cols + j]; } + /*! + * \brief Return copy of data in a static size container (see C2DContainer::get). + * \param[in] i - Row of the view (e.g. point index, whole-mesh usage). + * \param[in] start - Starting column to copy the data (amount determined by container size). + */ + template + StaticContainer get(Index i, Index start = 0) const noexcept { + constexpr size_t Size = StaticContainer::StaticSize; + static_assert(Size, "This method requires a static output type."); + StaticContainer ret; + for (size_t k = 0; k < Size; ++k) ret.data()[k] = m_ptr[i * m_cols + start + k]; + return ret; + } + + /*! + * \brief SIMD gather version of get, one row per lane. + */ + template + StaticContainer get(simd::Array i, Index start = 0) const noexcept { + constexpr size_t Size = StaticContainer::StaticSize; + static_assert(Size, "This method requires a static output type."); + StaticContainer ret; + for (size_t lane = 0; lane < N; ++lane) + for (size_t k = 0; k < Size; ++k) ret.data()[k][lane] = m_ptr[i[lane] * m_cols + start + k]; + return ret; + } + template ::value> = 0> Scalar* operator[](Index i) noexcept { return &m_ptr[i * m_cols]; diff --git a/Common/include/geometry/CGeometry.hpp b/Common/include/geometry/CGeometry.hpp index 7958110fb23b..9d3e574b4328 100644 --- a/Common/include/geometry/CGeometry.hpp +++ b/Common/include/geometry/CGeometry.hpp @@ -73,6 +73,19 @@ using namespace std; * \author F. Palacios */ class CGeometry { + public: + /*! + * \brief Aggregates the full symmetric CSR and its LDU split (L strictly-lower, U strictly-upper). + * Built together lazily via GetSparsePattern; all three are always valid once non-empty. + */ + struct LDUSparsePattern { + CCompressedSparsePatternUL csr; /*!< Full symmetric pattern (with diagonal pointer). */ + CCompressedSparsePatternUL l; /*!< Strictly-lower part. */ + CCompressedSparsePatternUL u; /*!< Strictly-upper part. */ + + bool empty() const { return csr.empty(); } + }; + protected: enum : size_t { OMP_MIN_SIZE = 32 }; /*!< \brief Chunk size for small loops. */ enum : size_t { MAXNDIM = 3 }; @@ -187,12 +200,15 @@ class CGeometry { /*--- Sparsity patterns associated with the geometry. ---*/ - CCompressedSparsePatternUL finiteVolumeCSRFill0, /*!< \brief 0-fill FVM sparsity. */ - finiteVolumeCSRFillN, /*!< \brief N-fill FVM sparsity (e.g. for ILUn preconditioner). */ - finiteElementCSRFill0, /*!< \brief 0-fill FEM sparsity. */ - finiteElementCSRFillN; /*!< \brief N-fill FEM sparsity (e.g. for ILUn preconditioner). */ + LDUSparsePattern finiteVolumePatternFill0; /*!< \brief FVM sparsity with 0-fill (structural pattern). */ + LDUSparsePattern finiteVolumePatternFillN; /*!< \brief FVM sparsity with N-fill (e.g. for ILU-N). */ + LDUSparsePattern finiteElementPatternFill0; /*!< \brief FEM sparsity with 0-fill (structural pattern). */ + LDUSparsePattern finiteElementPatternFillN; /*!< \brief FEM sparsity with N-fill (e.g. for ILU-N). */ - CEdgeToNonZeroMapUL edgeToCSRMap; /*!< \brief Map edges to CSR entries referenced by them (i,j) and (j,i). */ + su2vector finiteVolumeLToUTranspMap; /*!< \brief FVM L-entry -> U-entry of its transpose. */ + su2vector finiteVolumeUToLTranspMap; /*!< \brief FVM U-entry -> L-entry of its transpose. */ + su2vector finiteElementLToUTranspMap; /*!< \brief FEM L-entry -> U-entry of its transpose. */ + su2vector finiteElementUToLTranspMap; /*!< \brief FEM U-entry -> L-entry of its transpose. */ /*--- Edge and element colorings. ---*/ @@ -878,7 +894,7 @@ class CGeometry { inline virtual void GatherInOutAverageValues(CConfig* config, bool allocate) {} /*! - * \brief Store all the turboperformance in the solver in ZONE_0. + * \brief Store all the turboperformance in the solver in final zone. * \param[in] donor_geometry - Solution from the donor mesh. * \param[in] target_geometry - Solution from the target mesh. * \param[in] donorZone - counter of the donor solution @@ -1868,21 +1884,23 @@ class CGeometry { * \param[in] fillLvl - Level of fill of the pattern. * \return Reference to the sparse pattern. */ - const CCompressedSparsePatternUL& GetSparsePattern(ConnectivityType type, unsigned long fillLvl = 0); + const LDUSparsePattern& GetSparsePattern(ConnectivityType type, unsigned long fillLvl = 0); /*! - * \brief Get the edge to sparse pattern map. - * \note This method builds the map and required pattern (0-fill FVM) if that has not been done yet. - * \return Reference to the map. + * \brief Get the bijective map from L-entry indices to U-entry indices of their transposes. + * \note Requires symmetric pattern. Builds both LU transpose maps if not already built. + * \param[in] type - Finite volume or finite element. + * \return Reference to the l_to_u map. */ - const CEdgeToNonZeroMapUL& GetEdgeToSparsePatternMap(); + const su2vector& GetLToUTransposeSparsePatternMap(ConnectivityType type); /*! - * \brief Get the transpose of the (main, i.e 0 fill) sparse pattern (e.g. CSR becomes CSC). + * \brief Get the bijective map from U-entry indices to L-entry indices of their transposes. + * \note Requires symmetric pattern. Builds both LU transpose maps if not already built. * \param[in] type - Finite volume or finite element. - * \return Reference to the map. + * \return Reference to the u_to_l map. */ - const su2vector& GetTransposeSparsePatternMap(ConnectivityType type); + const su2vector& GetUToLTransposeSparsePatternMap(ConnectivityType type); /*! * \brief Get the edge coloring. @@ -1960,7 +1978,8 @@ class CGeometry { * \param[in] config_container - Definition of the particular problem. * \param[in] geometry_container - Geometrical definition of the problem. */ - static void ComputeWallDistance(const CConfig* const* config_container, CGeometry**** geometry_container); + static void ComputeWallDistance(const CConfig* const* config_container, CGeometry**** geometry_container, + const int record_zone = -1); /*! * \brief Set the amount of nonconvex elements in the mesh. diff --git a/Common/include/geometry/meshreader/CSU2ASCIIMeshReaderBase.hpp b/Common/include/geometry/meshreader/CSU2ASCIIMeshReaderBase.hpp index f1c35b2d9392..8d4cbb4dd385 100644 --- a/Common/include/geometry/meshreader/CSU2ASCIIMeshReaderBase.hpp +++ b/Common/include/geometry/meshreader/CSU2ASCIIMeshReaderBase.hpp @@ -30,50 +30,19 @@ #include -#include "CMeshReaderBase.hpp" +#include "CSU2MeshReaderBase.hpp" /*! * \class CSU2ASCIIMeshReaderBase * \brief Base class for the reading of a native SU2 ASCII grid. * \author T. Economon */ -class CSU2ASCIIMeshReaderBase : public CMeshReaderBase { +class CSU2ASCIIMeshReaderBase : public CSU2MeshReaderBase { protected: enum class FileSection { POINTS, ELEMENTS, MARKERS }; /*!< \brief Different sections of the file. */ std::array SectionOrder{}; /*!< \brief Order of the sections in the file. */ - const unsigned short myZone; /*!< \brief Current SU2 zone index. */ - const unsigned short nZones; /*!< \brief Total number of zones in the SU2 file. */ - - const string meshFilename; /*!< \brief Name of the SU2 ASCII mesh file being read. */ - ifstream mesh_file; /*!< \brief File object for the SU2 ASCII mesh file. */ - - bool actuator_disk; /*!< \brief Boolean for whether we have an actuator disk to split. */ - - unsigned long ActDiskNewPoints = - 0; /*!< \brief Total number of new grid points to add due to actuator disk splitting. */ - - su2double Xloc = 0.0; /*!< \brief X-coordinate of the CG of the actuator disk surface. */ - su2double Yloc = 0.0; /*!< \brief X-coordinate of the CG of the actuator disk surface. */ - su2double Zloc = 0.0; /*!< \brief X-coordinate of the CG of the actuator disk surface. */ - - vector ActDisk_Bool; /*!< \brief Flag to identify the grid points on the actuator disk. */ - - vector ActDiskPoint_Back; /*!< \brief Vector containing the global index for the new grid points added - to the back of the actuator disk. */ - vector VolumePoint_Inv; /*!< \brief Vector containing the inverse mapping from the global index to the - added point index for the actuator disk. */ - - vector CoordXActDisk; /*!< \brief X-coordinates of the new grid points added by splitting the actuator disk - (size = ActDiskNewPoints). */ - vector CoordYActDisk; /*!< \brief Y-coordinates of the new grid points added by splitting the actuator disk - (size = ActDiskNewPoints). */ - vector CoordZActDisk; /*!< \brief Z-coordinates of the new grid points added by splitting the actuator disk - (size = ActDiskNewPoints). */ - - vector CoordXVolumePoint; /*!< \brief X-coordinates of the volume elements touching the actuator disk. */ - vector CoordYVolumePoint; /*!< \brief Y-coordinates of the volume elements touching the actuator disk. */ - vector CoordZVolumePoint; /*!< \brief Z-coordinates of the volume elements touching the actuator disk. */ + ifstream mesh_file; /*!< \brief File object for the SU2 ASCII mesh file. */ /*! * \brief Reads all SU2 ASCII mesh metadata and checks for errors. diff --git a/Common/include/geometry/meshreader/CSU2BinaryMeshReaderBase.hpp b/Common/include/geometry/meshreader/CSU2BinaryMeshReaderBase.hpp new file mode 100644 index 000000000000..ca66e6aa80c8 --- /dev/null +++ b/Common/include/geometry/meshreader/CSU2BinaryMeshReaderBase.hpp @@ -0,0 +1,133 @@ +/*! + * \file CSU2BinaryMeshReaderBase.hpp + * \brief Header file for the class CSU2BinaryMeshReaderBase. + * The implementations are in the CSU2BinaryMeshReaderBase.cpp file. + * \author T. Economon, E. van der Weide + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#pragma once + +#include "CSU2MeshReaderBase.hpp" +#include "../../../include/toolboxes/SwapBytes.hpp" + +/*! + * \class CSU2BinaryMeshReaderBase + * \brief Base class for the reading of a native SU2 binary grid. + * \author T. Economon, E. van der Weide + */ +class CSU2BinaryMeshReaderBase : public CSU2MeshReaderBase { + protected: + constexpr static int SU2_STRING_SIZE = + SU2_BINARY_STRING_SIZE; /*!< \brief Size of the strings in the SU2 binary mesh file. */ + + FILE* mesh_file; /*!< \brief File object for the SU2 binary mesh file. */ + bool swap_bytes; /*!< \brief Whether or not byte swapping must be used. */ + int size_conn_type; /*!< \brief Size, in bytes of the connectivity type. */ + + /*! + * \brief Reads the connectivity type used in the binary file and check if + * byte swapping must be applied. + */ + void ReadConnectivityType(); + + /*! + * \brief Reads all SU2 binary mesh metadata and checks for errors. + * \param[in] config - Problem configuration for the current zone. + */ + void ReadMetadata(CConfig* config); + + /*! + * \brief Reads the grid points from an SU2 zone into linear partitions across all ranks. + */ + virtual void ReadPointCoordinates(); + + /*! + * \brief Reads the interior volume elements from one section of an SU2 zone into linear partitions across all ranks. + */ + virtual void ReadVolumeElementConnectivity(); + + /*! + * \brief Reads the surface (boundary) elements from the SU2 zone. + */ + virtual void ReadSurfaceElementConnectivity(); + + /*! + * \brief Helper function to find the current zone in an SU2 binary mesh object. + */ + void FastForwardToMyZone(); + + /*! + * \brief Portable, 64-bit safe replacement for fseek. Needed because binary + * SU2 grid files can exceed 2 GiB even on platforms where "long" is + * only 32 bits wide (e.g. Windows). + */ + static int FileSeek64(FILE* file, int64_t offset, int whence); + + /*! + * \brief Portable, 64-bit safe replacement for ftell. + */ + static int64_t FileTell64(FILE* file); + + /*! + * \brief Function to read one entity of the connectivity type from the binary file. + * \return uint64_t version of the the data. + */ + uint64_t ReadBinaryNEntities(); + + /*! + * \brief Template function to read data from the binary file. + */ + template + void ReadBinaryData(T* data, const size_t nItems) { + /*--- Read the actual data. ---*/ + auto ret = fread(data, sizeof(T), nItems, mesh_file); + if (ret != nItems) SU2_MPI::Error(string("Error while reading the file ") + meshFilename, CURRENT_FUNCTION); + + /*--- Apply byte swapping, if needed. ---*/ + if (swap_bytes) SwapBytes((char*)data, sizeof(T), nItems); + } + + private: + /*! + * \brief Read the meta data for a zone, advancing the file position past it. + * \param[in] storeMetadata - Whether to store the read values in the + * dimension, numberOfGlobalElements, numberOfGlobalPoints and + * numberOfMarkers members. Must be false when only skipping + * past a lower-numbered zone (from + * FastForwardToMyZone), so that doing so does not clobber the + * metadata already read for the current (target) zone. + */ + void ReadMetadataZone(bool storeMetadata); + + public: + /*! + * \brief Constructor of the CSU2BinaryMeshReaderBase class. + */ + CSU2BinaryMeshReaderBase(CConfig* val_config, unsigned short val_iZone, unsigned short val_nZone); + + /*! + * \brief Destructor of the CSU2BinaryMeshReaderBase class. + */ + ~CSU2BinaryMeshReaderBase(void) override; +}; diff --git a/Common/include/geometry/meshreader/CSU2BinaryMeshReaderFEM.hpp b/Common/include/geometry/meshreader/CSU2BinaryMeshReaderFEM.hpp new file mode 100644 index 000000000000..d4112d09e589 --- /dev/null +++ b/Common/include/geometry/meshreader/CSU2BinaryMeshReaderFEM.hpp @@ -0,0 +1,66 @@ +/*! + * \file CSU2BinaryMeshReaderFEM.hpp + * \brief Header file for the class CSU2BinaryMeshReaderFEM. + * The implementations are in the CSU2BinaryMeshReaderFEM.cpp file. + * \author T. Economon, E. van der Weide + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#pragma once + +#include "CSU2BinaryMeshReaderBase.hpp" + +/*! + * \class CSU2BinaryMeshReaderFEM + * \brief Reads a native SU2 binary grid into linear partitions for the finite element solver (FEM). + * \author T. Economon, E. van der Weide + */ +class CSU2BinaryMeshReaderFEM : public CSU2BinaryMeshReaderBase { + private: + /*! + * \brief Reads the grid points from an SU2 zone into linear partitions across all ranks. + */ + void ReadPointCoordinates(); + + /*! + * \brief Reads the interior volume elements from one section of an SU2 zone into linear partitions across all ranks. + */ + void ReadVolumeElementConnectivity(); + + /*! + * \brief Reads the surface (boundary) elements from one section of an SU2 zone into linear partitions across all + * ranks. + */ + void ReadSurfaceElementConnectivity(); + + public: + /*! + * \brief Constructor of the CSU2BinaryMeshReaderFEM class. + */ + CSU2BinaryMeshReaderFEM(CConfig* val_config, unsigned short val_iZone, unsigned short val_nZone); + + /*! + * \brief Destructor of the CSU2BinaryMeshReaderFEM class. + */ + ~CSU2BinaryMeshReaderFEM(void) override; +}; diff --git a/Common/include/geometry/meshreader/CSU2BinaryMeshReaderFVM.hpp b/Common/include/geometry/meshreader/CSU2BinaryMeshReaderFVM.hpp new file mode 100644 index 000000000000..894bd41486d8 --- /dev/null +++ b/Common/include/geometry/meshreader/CSU2BinaryMeshReaderFVM.hpp @@ -0,0 +1,55 @@ +/*! + * \file CSU2BinaryMeshReaderFVM.hpp + * \brief Header file for the class CSU2BinaryMeshReaderFVM. + * The implementations are in the CSU2BinaryMeshReaderFVM.cpp file. + * \author T. Economon, E. van der Weide + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#pragma once + +#include "CSU2BinaryMeshReaderBase.hpp" + +/*! + * \class CSU2BinaryMeshReaderFVM + * \brief Reads a native SU2 binary grid into linear partitions for the finite volume solver (FVM). + * \author T. Economon, E. van der Weide + */ +class CSU2BinaryMeshReaderFVM : public CSU2BinaryMeshReaderBase { + private: + /*! + * \brief Splits a single surface actuator disk boundary into two separate markers (repeated points). + */ + void SplitActuatorDiskSurface(); + + public: + /*! + * \brief Constructor of the CSU2BinaryMeshReaderFVM class. + */ + CSU2BinaryMeshReaderFVM(CConfig* val_config, unsigned short val_iZone, unsigned short val_nZone); + + /*! + * \brief Destructor of the CSU2BinaryMeshReaderFVM class. + */ + ~CSU2BinaryMeshReaderFVM(void) override; +}; diff --git a/Common/include/geometry/meshreader/CSU2MeshReaderBase.hpp b/Common/include/geometry/meshreader/CSU2MeshReaderBase.hpp new file mode 100644 index 000000000000..0f80b73b23d9 --- /dev/null +++ b/Common/include/geometry/meshreader/CSU2MeshReaderBase.hpp @@ -0,0 +1,84 @@ +/*! + * \file CSU2MeshReaderBase.hpp + * \brief Header file for the class CSU2MeshReaderBase. + * The implementations are in the CSU2MeshReaderBase.cpp file. + * \author T. Economon + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#pragma once + +#include + +#include "CMeshReaderBase.hpp" + +/*! + * \class CSU2MeshReaderBase + * \brief Base class for the reading of a native SU2 grid. + * \author T. Economon + */ +class CSU2MeshReaderBase : public CMeshReaderBase { + protected: + const unsigned short myZone; /*!< \brief Current SU2 zone index. */ + const unsigned short nZones; /*!< \brief Total number of zones in the SU2 file. */ + + const string meshFilename; /*!< \brief Name of the SU2 ASCII mesh file being read. */ + + bool actuator_disk; /*!< \brief Boolean for whether we have an actuator disk to split. */ + + unsigned long ActDiskNewPoints = + 0; /*!< \brief Total number of new grid points to add due to actuator disk splitting. */ + + su2double Xloc = 0.0; /*!< \brief X-coordinate of the CG of the actuator disk surface. */ + su2double Yloc = 0.0; /*!< \brief X-coordinate of the CG of the actuator disk surface. */ + su2double Zloc = 0.0; /*!< \brief X-coordinate of the CG of the actuator disk surface. */ + + vector ActDisk_Bool; /*!< \brief Flag to identify the grid points on the actuator disk. */ + + vector ActDiskPoint_Back; /*!< \brief Vector containing the global index for the new grid points added + to the back of the actuator disk. */ + vector VolumePoint_Inv; /*!< \brief Vector containing the inverse mapping from the global index to the + added point index for the actuator disk. */ + + vector CoordXActDisk; /*!< \brief X-coordinates of the new grid points added by splitting the actuator disk + (size = ActDiskNewPoints). */ + vector CoordYActDisk; /*!< \brief Y-coordinates of the new grid points added by splitting the actuator disk + (size = ActDiskNewPoints). */ + vector CoordZActDisk; /*!< \brief Z-coordinates of the new grid points added by splitting the actuator disk + (size = ActDiskNewPoints). */ + + vector CoordXVolumePoint; /*!< \brief X-coordinates of the volume elements touching the actuator disk. */ + vector CoordYVolumePoint; /*!< \brief Y-coordinates of the volume elements touching the actuator disk. */ + vector CoordZVolumePoint; /*!< \brief Z-coordinates of the volume elements touching the actuator disk. */ + + public: + /*! + * \brief Constructor of the CSU2MeshReaderBase class. + */ + CSU2MeshReaderBase(CConfig* val_config, unsigned short val_iZone, unsigned short val_nZone); + + /*! + * \brief Destructor of the CSU2MeshReaderBase class. + */ + ~CSU2MeshReaderBase(void) override; +}; diff --git a/Common/include/grid_movement/CVolumetricMovementFactory.hpp b/Common/include/grid_movement/CVolumetricMovementFactory.hpp index 702decb25e97..970d780676f8 100644 --- a/Common/include/grid_movement/CVolumetricMovementFactory.hpp +++ b/Common/include/grid_movement/CVolumetricMovementFactory.hpp @@ -8,7 +8,7 @@ * The SU2 Project is maintained by the SU2 Foundation * (http://su2foundation.org) * - * Copyright 2012-2024, SU2 Contributors (cf. AUTHORS.md) + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) * * SU2 is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/Common/include/interface_interpolation/CInterpolator.hpp b/Common/include/interface_interpolation/CInterpolator.hpp index 49d6841abe61..045e28c8d3fd 100644 --- a/Common/include/interface_interpolation/CInterpolator.hpp +++ b/Common/include/interface_interpolation/CInterpolator.hpp @@ -99,7 +99,13 @@ class CInterpolator { coefficient.resize(nDonor); } }; - vector > targetVertices; /*! \brief Donor information per marker per vertex of the target. */ + vector> targetVertices; /*! \brief Donor information per marker per vertex of the target. */ + + struct CSpanDonorInfo { + size_t donorSpan; // Refers to donor span + su2double coefficient; // Refers to coefficient + }; + vector> targetSpans; // > /*! * \brief Constructor of the class. @@ -125,13 +131,18 @@ class CInterpolator { * \note Main method that derived classes must implement. * \param[in] config - Definition of the particular problem. */ - virtual void SetTransferCoeff(const CConfig* const* config) = 0; + virtual void SetTransferCoeff(CGeometry**** geometry, const CConfig* const* config) = 0; /*! * \brief Print information about the interpolation. */ virtual void PrintStatistics(void) const {} + /*! + * \brief Write mixing plane interpolation details to file + */ + inline virtual void WriteInterpolationDetails(const string& filename, const CConfig* const* config){}; + /*! * \brief Check whether an interface should be processed or not, i.e. if it is part of the zones. * \param[in] val_markDonor - Marker tag from donor zone. diff --git a/Common/include/interface_interpolation/CInterpolatorFactory.hpp b/Common/include/interface_interpolation/CInterpolatorFactory.hpp index fdc76ceccf47..e3c86cff5a32 100644 --- a/Common/include/interface_interpolation/CInterpolatorFactory.hpp +++ b/Common/include/interface_interpolation/CInterpolatorFactory.hpp @@ -43,5 +43,5 @@ namespace CInterpolatorFactory { */ CInterpolator* CreateInterpolator(CGeometry**** geometry_container, const CConfig* const* config, const CInterpolator* transpInterpolator, unsigned iZone, unsigned jZone, - bool verbose = true); + bool mixing_plane, bool verbose = true); } // namespace CInterpolatorFactory diff --git a/Common/include/interface_interpolation/CIsoparametric.hpp b/Common/include/interface_interpolation/CIsoparametric.hpp index 0b540f4e66c0..6b05dc851be0 100644 --- a/Common/include/interface_interpolation/CIsoparametric.hpp +++ b/Common/include/interface_interpolation/CIsoparametric.hpp @@ -64,7 +64,7 @@ class CIsoparametric final : public CInterpolator { * \brief Set up transfer matrix defining relation between two meshes * \param[in] config - Definition of the particular problem. */ - void SetTransferCoeff(const CConfig* const* config) override; + void SetTransferCoeff(CGeometry**** geometry, const CConfig* const* config) override; /*! * \brief Print information about the interpolation. diff --git a/Common/include/interface_interpolation/CMirror.hpp b/Common/include/interface_interpolation/CMirror.hpp index 1b569483a802..0f0bb2af4f80 100644 --- a/Common/include/interface_interpolation/CMirror.hpp +++ b/Common/include/interface_interpolation/CMirror.hpp @@ -54,5 +54,5 @@ class CMirror final : public CInterpolator { * \brief Set up transfer matrix defining relation between two meshes * \param[in] config - Definition of the particular problem. */ - void SetTransferCoeff(const CConfig* const* config) override; + void SetTransferCoeff(CGeometry**** geometry, const CConfig* const* config) override; }; diff --git a/Common/include/interface_interpolation/CMixingPlane.hpp b/Common/include/interface_interpolation/CMixingPlane.hpp new file mode 100644 index 000000000000..a706bf2fe786 --- /dev/null +++ b/Common/include/interface_interpolation/CMixingPlane.hpp @@ -0,0 +1,101 @@ +/*! + * \file CMixingPlane.hpp + * \brief Header of mixing plane interpolation methods. + * \author J. Kelly + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#pragma once +#include "CInterpolator.hpp" +#include "../option_structure.hpp" + +/*! + * \brief Mixing plane interpolation. + * \note This contains several interpolation methods used in the mixing plane interpolation + * and enables the mixing state class structure for proper recording in AD mode + * \ingroup Interfaces + */ +class CMixingPlane final : public CInterpolator { + public: + CMixingPlane(CGeometry**** geometry_container, const CConfig* const* config, unsigned int iZone, unsigned int jZone); + + void SetTransferCoeff(CGeometry**** geometry, const CConfig* const* config) override; + + inline CSpanDonorInfo MapMatchingSpan(unsigned short iSpanTarget) { return {iSpanTarget, 0.0}; } + + inline CSpanDonorInfo MapNearestSpan(const su2double iSpanTargetValue, const su2double* spanValuesDonor, + unsigned long nSpanDonor) { + unsigned short tSpan = 0; // Nearest donor span index + auto minDist = std::numeric_limits::max(); + + for (auto iSpanDonor = 0u; iSpanDonor < nSpanDonor - 1; iSpanDonor++) { + const auto dist = abs(iSpanTargetValue - spanValuesDonor[iSpanDonor]); + if (dist < minDist) { + minDist = dist; + tSpan = iSpanDonor; + } + } + return {tSpan, 0.0}; + }; + + inline CSpanDonorInfo MapLinearInterpolationSpan(const su2double iSpanTargetValue, const su2double* spanValuesDonor, + unsigned long nSpanDonor, int rank) { + unsigned short kSpan = 0; // Lower bound donor span for interpolation + auto minDist = std::numeric_limits::max(); + su2double coeff = 0.0; // Interpolation coefficient + + if (iSpanTargetValue <= spanValuesDonor[0]) { + PrintClampingWarning(rank, true); + return {0, 0.0}; + } + + if (iSpanTargetValue >= spanValuesDonor[nSpanDonor - 1]) { + PrintClampingWarning(rank, false); + return {nSpanDonor - 1, 0.0}; + } + + for (auto iSpanDonor = 0u; iSpanDonor < nSpanDonor - 1; iSpanDonor++) { + const auto dist = abs(iSpanTargetValue - spanValuesDonor[iSpanDonor]); + if (dist < minDist && iSpanTargetValue >= spanValuesDonor[iSpanDonor]) { + kSpan = iSpanDonor; + minDist = dist; + } + } + coeff = (iSpanTargetValue - spanValuesDonor[kSpan]) / (spanValuesDonor[kSpan + 1] - spanValuesDonor[kSpan]); + return {kSpan, coeff}; + }; + + inline void PrintClampingWarning(int rank, bool atHub) { + if (rank != MASTER_NODE) return; + cout << "Warning! Target spans exist outside the bounds of donor spans! Clamping interpolator..." << endl; + cout << (atHub ? "This is an issue at the hub." : "This is an issue at the shroud.") << endl; + cout << "Setting coeff = 0.0 and transferring endwall value!" << endl; + }; + + /*! + * \brief Write interpolation details to file. + * \param[in] filename - Name of output file. + * \param[in] config - Configuration for all zones. + */ + void WriteInterpolationDetails(const string& filename, const CConfig* const* config) override; +}; diff --git a/Common/include/interface_interpolation/CNearestNeighbor.hpp b/Common/include/interface_interpolation/CNearestNeighbor.hpp index 12e5cafd73bd..f75507a6f4d9 100644 --- a/Common/include/interface_interpolation/CNearestNeighbor.hpp +++ b/Common/include/interface_interpolation/CNearestNeighbor.hpp @@ -63,7 +63,7 @@ class CNearestNeighbor final : public CInterpolator { * \brief Set up transfer matrix defining relation between two meshes. * \param[in] config - Definition of the particular problem. */ - void SetTransferCoeff(const CConfig* const* config) override; + void SetTransferCoeff(CGeometry**** geometry, const CConfig* const* config) override; /*! * \brief Print interpolation statistics. diff --git a/Common/include/interface_interpolation/CRadialBasisFunction.hpp b/Common/include/interface_interpolation/CRadialBasisFunction.hpp index 97a3e8524548..b783b10d882d 100644 --- a/Common/include/interface_interpolation/CRadialBasisFunction.hpp +++ b/Common/include/interface_interpolation/CRadialBasisFunction.hpp @@ -56,7 +56,7 @@ class CRadialBasisFunction final : public CInterpolator { * \brief Set up transfer matrix defining relation between two meshes * \param[in] config - Definition of the particular problem. */ - void SetTransferCoeff(const CConfig* const* config) override; + void SetTransferCoeff(CGeometry**** geometry, const CConfig* const* config) override; /*! * \brief Print information about the interpolation. diff --git a/Common/include/interface_interpolation/CSlidingMesh.hpp b/Common/include/interface_interpolation/CSlidingMesh.hpp index 328052bd167f..34474bad910a 100644 --- a/Common/include/interface_interpolation/CSlidingMesh.hpp +++ b/Common/include/interface_interpolation/CSlidingMesh.hpp @@ -49,7 +49,7 @@ class CSlidingMesh final : public CInterpolator { * \brief Set up transfer matrix defining relation between two meshes * \param[in] config - Definition of the particular problem. */ - void SetTransferCoeff(const CConfig* const* config) override; + void SetTransferCoeff(CGeometry**** geometry, const CConfig* const* config) override; private: /*! diff --git a/Common/include/linear_algebra/CMatrixInverse.hpp b/Common/include/linear_algebra/CMatrixInverse.hpp new file mode 100644 index 000000000000..6d6dfa739595 --- /dev/null +++ b/Common/include/linear_algebra/CMatrixInverse.hpp @@ -0,0 +1,93 @@ +/*! + * \file CMatrixInverse.hpp + * \brief Dense small-matrix inversion via Gauss-Jordan elimination, shared between the host + * (CSysMatrix::MatrixInverse) and device (CSysPreconditionerGPU.cu) implementations. + * \author F. Palacios, A. Bueno, T. Economon, P. Gomes + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#pragma once + +#include + +#include "../code_config.hpp" + +namespace SU2_LinAlg { + +/*! + * \brief Regularize a pivot that is too small to prevent divide-by-zero, on host and device + * this needs to clamp to the same value so that the two produce the same factors. + */ +template +SU2_CUDA_HOST_DEVICE inline void RegularizePivot(ScalarType& pivot) { + const float eps = 1e-12; +#ifdef __CUDA_ARCH__ + if (fabs(pivot) < eps) pivot = copysign(ScalarType(eps), pivot); +#else + if (std::abs(pivot) < eps) pivot = std::copysign(ScalarType(eps), pivot); +#endif +} + +/*! + * \brief Invert the \p nVar by \p nVar dense matrix \p matrix into \p inverse via Gauss-Jordan + * elimination with partial pivoting on the diagonal. + * \note \p matrix is used as scratch space and destroyed, \p inverse must not alias it. + */ +template +SU2_CUDA_HOST_DEVICE inline void MatrixInverse(unsigned long nVar, ScalarType* matrix, ScalarType* inverse) { +#define A(I, J) matrix[(I)*nVar + (J)] +#define M(I, J) inverse[(I)*nVar + (J)] + + /*--- Initialize the inverse with the identity. ---*/ + for (auto iVar = 0ul; iVar < nVar; iVar++) + for (auto jVar = 0ul; jVar < nVar; jVar++) M(iVar, jVar) = ScalarType(iVar == jVar); + + /*--- Transform system in Upper Matrix. ---*/ + for (auto iVar = 1ul; iVar < nVar; iVar++) { + for (auto jVar = 0ul; jVar < iVar; jVar++) { + RegularizePivot(A(jVar, jVar)); + + const ScalarType weight = A(iVar, jVar) / A(jVar, jVar); + for (auto kVar = jVar; kVar < nVar; kVar++) A(iVar, kVar) -= weight * A(jVar, kVar); + + /*--- At this stage M is lower triangular so not all cols need updating. ---*/ + for (auto kVar = 0ul; kVar <= jVar; kVar++) M(iVar, kVar) -= weight * M(jVar, kVar); + } + } + + /*--- Backwards substitution. ---*/ + for (auto iVar = nVar; iVar > 0ul;) { + iVar--; // unsigned type + for (auto jVar = iVar + 1; jVar < nVar; jVar++) + for (auto kVar = 0ul; kVar < nVar; kVar++) M(iVar, kVar) -= A(iVar, jVar) * M(jVar, kVar); + + RegularizePivot(A(iVar, iVar)); + + for (auto kVar = 0ul; kVar < nVar; kVar++) M(iVar, kVar) /= A(iVar, iVar); + } + +#undef A +#undef M +} + +} // namespace SU2_LinAlg diff --git a/Common/include/linear_algebra/CMatrixVectorProduct.hpp b/Common/include/linear_algebra/CMatrixVectorProduct.hpp index 878bb132b984..52614a45770b 100644 --- a/Common/include/linear_algebra/CMatrixVectorProduct.hpp +++ b/Common/include/linear_algebra/CMatrixVectorProduct.hpp @@ -82,7 +82,17 @@ class CSysMatrixVectorProduct final : public CMatrixVectorProduct { */ inline CSysMatrixVectorProduct(const CSysMatrix& matrix_ref, CGeometry* geometry_ref, const CConfig* config_ref) - : matrix(matrix_ref), geometry(geometry_ref), config(config_ref) {} + : matrix(matrix_ref), geometry(geometry_ref), config(config_ref) { + /*--- The matrix does not change while this object lives, so it crosses the bus once, + * here. The vectors are uploaded by CSysSolve, see HandleTemporariesIn. ---*/ +#ifdef SU2_ENABLE_CUDA_KERNELS + if constexpr (su2_gpu_capable_v) { + if (config->GetCUDA()) { + SU2_DEVICE_REGION(matrix.HtDTransfer();) + } + } +#endif + } /*! * \note This class cannot be default constructed as that would leave us with invalid pointers. @@ -95,17 +105,6 @@ class CSysMatrixVectorProduct final : public CMatrixVectorProduct { * \param[out] v - CSysVector that is the result of the product */ inline void operator()(const CSysVector& u, CSysVector& v) const override { - if (config->GetCUDA()) { -#ifdef HAVE_CUDA - matrix.GPUMatrixVectorProduct(u, v, geometry, config); -#else - SU2_MPI::Error( - "\nError in launching Matrix-Vector Product Function\nENABLE_CUDA is set to YES\nPlease compile with CUDA " - "options enabled in Meson to access GPU Functions", - CURRENT_FUNCTION); -#endif - } else { - matrix.MatrixVectorProduct(u, v, geometry, config); - } + matrix.MatrixVectorProduct(u, v, geometry, config); } }; diff --git a/Common/include/linear_algebra/CPastixWrapper.hpp b/Common/include/linear_algebra/CPastixWrapper.hpp index 7ee570010f0b..44fb9d503db3 100644 --- a/Common/include/linear_algebra/CPastixWrapper.hpp +++ b/Common/include/linear_algebra/CPastixWrapper.hpp @@ -28,6 +28,8 @@ #pragma once +#include "../code_config.hpp" + #ifdef HAVE_PASTIX #ifdef CODI_FORWARD_TYPE @@ -62,21 +64,28 @@ class CPastixWrapper { vector perm; /*!< \brief Ordering computed by PaStiX. */ vector workvec; /*!< \brief RHS vector which then becomes the solution. */ - pastix_int_t iparm[IPARM_SIZE]; /*!< \brief Integer parameters for PaStiX. */ - passivedouble dparm[DPARM_SIZE]; /*!< \brief Floating point parameters for PaStiX. */ + vector csr_row_ptr; /*!< \brief Owned CSR row pointers (built from LDU). */ + vector csr_col_ind; /*!< \brief Owned CSR column indices (built from LDU). */ + + pastix_int_t iparm[IPARM_SIZE]; /*!< \brief Integer parameters for PaStiX. */ + double dparm[DPARM_SIZE]; /*!< \brief Floating point parameters for PaStiX. */ struct { unsigned long nVar = 0; unsigned long nPoint = 0; unsigned long nPointDomain = 0; - const unsigned long* rowptr = nullptr; - const unsigned long* colidx = nullptr; - const ScalarType* values = nullptr; + unsigned long blkSz = 0; /*!< \brief Block size (nVar * nVar) for value assembly. */ + + const su2uint* row_ptr_l = nullptr; /*!< \brief LDU lower row pointers (geometry-owned). */ + const su2uint* row_ptr_u = nullptr; /*!< \brief LDU upper row pointers (geometry-owned). */ + const ScalarType* d = nullptr; /*!< \brief Diagonal blocks (matrix-owned). */ + const ScalarType* l = nullptr; /*!< \brief Lower blocks (matrix-owned). */ + const ScalarType* u = nullptr; /*!< \brief Upper blocks (matrix-owned). */ unsigned long size_rhs() const { return nPointDomain * nVar; } - } matrix; /*!< \brief Pointers and sizes of the input matrix. */ + } matrix; /*!< \brief Dimensions and LDU pointers captured from the owning CSysMatrix. */ - bool issetup{}; /*!< \brief Signals that the matrix data has been provided. */ + bool issetup{}; /*!< \brief Signals that the structure has been provided. */ bool isinitialized{}; /*!< \brief Signals that the sparsity pattern has been set. */ bool isfactorized{}; /*!< \brief Signals that a factorization has been computed. */ bool transpose{}; /*!< \brief Solve A^T x = b instead of A x = b. */ @@ -110,6 +119,11 @@ class CPastixWrapper { */ void Initialize(CGeometry* geometry, const CConfig* config); + /*! + * \brief Assemble CSR values from the stored LDU pointers directly into the values buffer. + */ + void AssembleValues(); + public: CPastixWrapper() = default; @@ -125,23 +139,43 @@ class CPastixWrapper { ~CPastixWrapper() { Clean(); } /*! - * \brief Set matrix data, only once. - * \param[in] nVar - DOF per point. + * \brief Returns true once SetLDU has been called. + */ + bool IsSetup() const { return issetup; } + + /*! + * \brief Set LDU structure and value pointers; builds and owns assembled CSR (called once). + * \param[in] nVar - DOF per point (square blocks: nVar x nVar). * \param[in] nPoint - Total number of points including halos. - * \param[in] nPointDomain - Number of internal points. - * \param[in] rowptr - Array, where column index data starts for each matrix row. - * \param[in] colidx - Non zeros column indices. - * \param[in] values - Matrix coefficients. + * \param[in] nPointDomain - Number of internal points (domain rows). + * \param[in] row_ptr_l/u - LDU lower/upper row pointers (geometry-owned, must outlive wrapper). + * \param[in] col_ind_l/u - LDU lower/upper column indices (geometry-owned). + * \param[in] d/l/u - LDU value blocks (matrix-owned, must outlive wrapper). */ - void SetMatrix(unsigned long nVar, unsigned long nPoint, unsigned long nPointDomain, const unsigned long* rowptr, - const unsigned long* colidx, const ScalarType* values) { + void SetLDU(unsigned long nVar, unsigned long nPoint, unsigned long nPointDomain, const su2uint* row_ptr_l, + const su2uint* col_ind_l, const su2uint* row_ptr_u, const su2uint* col_ind_u, const ScalarType* d, + const ScalarType* l, const ScalarType* u) { if (issetup) return; matrix.nVar = nVar; matrix.nPoint = nPoint; matrix.nPointDomain = nPointDomain; - matrix.rowptr = rowptr; - matrix.colidx = colidx; - matrix.values = values; + matrix.row_ptr_l = row_ptr_l; + matrix.row_ptr_u = row_ptr_u; + matrix.d = d; + matrix.l = l; + matrix.u = u; + matrix.blkSz = nVar * nVar; + + const unsigned long nnz_domain = row_ptr_l[nPointDomain] + nPointDomain + row_ptr_u[nPointDomain]; + csr_row_ptr.resize(nPointDomain + 1); + csr_col_ind.reserve(nnz_domain); + for (auto i = 0ul; i < nPointDomain; ++i) { + csr_row_ptr[i] = static_cast(csr_col_ind.size()); + for (auto k = row_ptr_l[i]; k < row_ptr_l[i + 1]; ++k) csr_col_ind.push_back(col_ind_l[k]); + csr_col_ind.push_back(i); + for (auto k = row_ptr_u[i]; k < row_ptr_u[i + 1]; ++k) csr_col_ind.push_back(col_ind_u[k]); + } + csr_row_ptr[nPointDomain] = static_cast(csr_col_ind.size()); issetup = true; } diff --git a/Common/include/linear_algebra/CPreconditioner.hpp b/Common/include/linear_algebra/CPreconditioner.hpp index e4fc7cf159fa..d8cbfd9a806c 100644 --- a/Common/include/linear_algebra/CPreconditioner.hpp +++ b/Common/include/linear_algebra/CPreconditioner.hpp @@ -37,6 +37,60 @@ /// \addtogroup SpLinSys /// @{ +/*! + * \brief Applies a preconditioner that only has a host implementation to vectors that live + * on the device: bring the input down, apply, put the result back. + * \note This is what keeps LU-SGS, Linelet and PaStiX usable on the GPU path. The + * transfers are issued by one thread with the team synchronized around them, the apply + * itself is the normal OpenMP parallel host code. + */ +template +inline void ApplyPreconditionerOnHost(const CSysVector& u, CSysVector& v, Apply&& apply) { +#ifdef SU2_ENABLE_CUDA_KERNELS + if constexpr (su2_gpu_capable_v) { + if (VecExpr::UseDeviceExpressions()) { + /*--- The host code must not see the device pointers of any expression it builds, so + * the switch is flipped for the duration of the apply. It is written inside the + * regions, by one thread, and published to the team by the trailing barrier. ---*/ + SU2_DEVICE_REGION(u.DtHTransfer(); VecExpr::SetUseDeviceExpressions(false);) + + apply(); + + SU2_DEVICE_REGION(VecExpr::SetUseDeviceExpressions(true); v.HtDTransfer();) + return; + } + } +#endif + apply(); +} + +/*! + * \brief Mirror of ApplyPreconditionerOnHost: applies a device preconditioner to host vectors. + * \note For callers that drive the Krylov solvers themselves and so never went through + * CSysSolve::Solve, which is what normally leaves the vectors on the device (Newton-Krylov). + * Device expressions are on for the duration so that a nested solve also uses the device copies. + * Only \p u is uploaded, \p v is always overwritten by the apply. + */ +template +inline void ApplyPreconditionerOnDevice(const CSysVector& u, CSysVector& v, bool useCuda, + Apply&& apply) { +#ifdef SU2_ENABLE_CUDA_KERNELS + if constexpr (su2_gpu_capable_v) { + if (useCuda && !VecExpr::UseDeviceExpressions()) { + SU2_DEVICE_REGION(u.HtDTransfer(); VecExpr::SetUseDeviceExpressions(true);) + + apply(); + + SU2_DEVICE_REGION(VecExpr::SetUseDeviceExpressions(false); v.DtHTransfer();) + return; + } + } +#else + (void)useCuda; +#endif + apply(); +} + /*! * \class CPreconditioner * \brief Abstract base class for defining a preconditioning operation. @@ -77,6 +131,30 @@ class CPreconditioner { template CPreconditioner::~CPreconditioner() {} +/*! + * \class CIdentityPreconditioner + * \brief No-op preconditioner used when Krylov solvers run without preconditioning. + * \note Also serves Q_IDENTITY: Build() requests quantization of the diagonal blocks, needed by + * the matrix-vector product shared with the Krylov solver even though this preconditioner's own + * operation is a no-op. CSysMatrix::QuantizeDiagonalBlocks() when quantization is off. + */ +template +class CIdentityPreconditioner final : public CPreconditioner { + private: + CSysMatrix& sparse_matrix; + + public: + inline explicit CIdentityPreconditioner(CSysMatrix& matrix_ref) : sparse_matrix(matrix_ref) {} + + CIdentityPreconditioner() = delete; + + inline void operator()(const CSysVector& u, CSysVector& v) const override { v = u; } + + inline bool IsIdentity() const override { return true; } + + inline void Build() override { sparse_matrix.QuantizeDiagonalBlocks(); } +}; + /*! * \class CJacobiPreconditioner * \brief Specialization of preconditioner that uses CSysMatrix class. @@ -118,7 +196,9 @@ class CJacobiPreconditioner final : public CPreconditioner { } /*! - * \note Request the associated matrix to build the preconditioner. + * \note Request the associated matrix to build the preconditioner. Also serves Q_JACOBI: + * BuildJacobiPreconditioner() quantizes the diagonal blocks itself when the matrix was + * set up for it. */ inline void Build() override { sparse_matrix.BuildJacobiPreconditioner(); } }; @@ -160,6 +240,7 @@ class CILUPreconditioner final : public CPreconditioner { * \param[out] v - CSysVector that is the result of the preconditioning. */ inline void operator()(const CSysVector& u, CSysVector& v) const override { + /*--- No host bracket, ILU has a device implementation and ComputeILUPreconditioner dispatches to it. ---*/ sparse_matrix.ComputeILUPreconditioner(u, v, geometry, config); } @@ -208,6 +289,11 @@ class CLU_SGSPreconditioner final : public CPreconditioner { inline void operator()(const CSysVector& u, CSysVector& v) const override { sparse_matrix.ComputeLU_SGSPreconditioner(u, v, geometry, config); } + + /*! + * \note Also serves Q_LU_SGS: quantizes the diagonal blocks, no-op for plain LU_SGS. + */ + inline void Build() override { sparse_matrix.BuildLU_SGSPreconditioner(); } }; /*! @@ -247,7 +333,7 @@ class CLineletPreconditioner final : public CPreconditioner { * \param[out] v - CSysVector that is the result of the preconditioning. */ inline void operator()(const CSysVector& u, CSysVector& v) const override { - sparse_matrix.ComputeLineletPreconditioner(u, v, geometry, config); + ApplyPreconditionerOnHost(u, v, [&] { sparse_matrix.ComputeLineletPreconditioner(u, v, geometry, config); }); } /*! @@ -297,7 +383,7 @@ class CPastixPreconditioner final : public CPreconditioner { * \param[out] v - CSysVector that is the result of the preconditioning. */ inline void operator()(const CSysVector& u, CSysVector& v) const override { - sparse_matrix.ComputePastixPreconditioner(u, v, geometry, config); + ApplyPreconditionerOnHost(u, v, [&] { sparse_matrix.ComputePastixPreconditioner(u, v, geometry, config); }); } /*! @@ -332,13 +418,19 @@ CPreconditioner* CPreconditioner::Create(ENUM_LINEAR_SOL CPreconditioner* prec = nullptr; switch (kind) { + case IDENTITY: + case Q_IDENTITY: + prec = new CIdentityPreconditioner(jacobian); + break; case JACOBI: + case Q_JACOBI: prec = new CJacobiPreconditioner(jacobian, geometry, config); break; case LINELET: prec = new CLineletPreconditioner(jacobian, geometry, config); break; case LU_SGS: + case Q_LU_SGS: prec = new CLU_SGSPreconditioner(jacobian, geometry, config); break; case ILU: diff --git a/Common/include/linear_algebra/CSysMatrix.hpp b/Common/include/linear_algebra/CSysMatrix.hpp index ecb26959a06b..70dd943690ec 100644 --- a/Common/include/linear_algebra/CSysMatrix.hpp +++ b/Common/include/linear_algebra/CSysMatrix.hpp @@ -29,13 +29,16 @@ #pragma once #include "../CConfig.hpp" +#include "../code_config.hpp" #include "CSysVector.hpp" #include "CPastixWrapper.hpp" #include "../toolboxes/graph_toolbox.hpp" +#include #include #include #include +#include /*--- In forward mode the matrix is not of a built-in type. ---*/ #if defined(HAVE_MKL) && !defined(CODI_FORWARD_TYPE) @@ -108,6 +111,142 @@ struct CSysMatrixComms { MPI_QUANTITIES commType = MPI_QUANTITIES::SOLUTION_MATRIX); }; +/*! + * \brief Reconstruct the float row-scale from a stored uint8 binary exponent. + * \p e is stored already biased, i.e. it is the IEEE 754 biased-exponent field itself, so + * shifting it into place with a zero mantissa gives the exact power of two 2^(e-127). + * This is the inverse of the encoding in EncodeQuantBlock. + * \note Branches on __CUDA_ARCH__, plain memcpy compiles for the device but does not work! + */ +SU2_CUDA_HOST_DEVICE FORCEINLINE float DecodeQuantScale(uint8_t e) noexcept { + const uint32_t bits = static_cast(e) << 23; +#ifdef __CUDA_ARCH__ + return __uint_as_float(bits); +#else + float scale; + memcpy(&scale, &bits, sizeof(bits)); + return scale; +#endif +} + +/*! + * \brief Encode one row of an nVar×nVar block into int8 quantized storage: \p qs receives the + * row's scale exponent, \p qv (nVar entries) the clamped int8 values for row \p r. + * \p f(r,c) is called twice per entry (max-abs scan then encoding); it should be cheap. + * \note Shared with the device and thus same __CUDA_ARCH__ branches as DecodeQuantScale. \p f's + * return type is cast to float directly on device (only ever instantiated there for plain + * ScalarType, never AD-active); on host it goes through SU2_TYPE::PassiveValue first, since + * ScalarType can be AD-active there (quantized_mode is only compiled out for reverse-mode + * AD, not forward-mode, see quantized_offdiag_needed in CSysMatrix.cpp) and PassiveValue is + * host-only (not SU2_CUDA_HOST_DEVICE). + */ +template +SU2_CUDA_HOST_DEVICE FORCEINLINE void EncodeQuantRow(const F& f, uint8_t& qs, int8_t* __restrict qv, unsigned long nVar, + unsigned long r) noexcept { +#ifdef __CUDA_ARCH__ +#define EQR_PASSIVE(ROW, COL) f(ROW, COL) +#else +#define EQR_PASSIVE(ROW, COL) SU2_TYPE::PassiveValue(f(ROW, COL)) + using std::max; + using std::min; +#endif + constexpr uint32_t eps_bits = 0x34000000u; // ~1.2e-7 + uint32_t max_abs_bits = eps_bits; + for (auto c = 0ul; c < nVar; ++c) { + const auto fv = static_cast(EQR_PASSIVE(r, c)); +#ifdef __CUDA_ARCH__ + const uint32_t fb = __float_as_uint(fv); +#else + uint32_t fb; + memcpy(&fb, &fv, sizeof(fb)); +#endif + /*--- Masking the mantissa as well as the sign leaves the exponent alone in place, which is + * all the scale needs (the max of the exponents is the exponent of the max). ---*/ + max_abs_bits = max(max_abs_bits, fb & 0x7F800000u); + } + /*--- Add 1 (round up the exponent) and subtract 7 (to divide by 128. which is the int8 range + * for "qv") = -6. The 127 float offset is NOT removed, so the stored value is the biased + * exponent of the scale and DecodeQuantScale can use it as exponent bits directly. The + * eps_bits floor puts the result in [98, 249], so it always fits in uint8 without clamping. ---*/ + qs = static_cast((max_abs_bits >> 23) - 6); + /*--- 1/scale = 2^-(qs-127), whose biased exponent is 254 - qs. Because max_abs_bits holds the + * exponent already shifted into place with a zero mantissa, that whole expression collapses to + * one subtraction: (254 - ((max_abs_bits >> 23) - 6)) << 23 == (260 << 23) - max_abs_bits. ---*/ + const uint32_t inv_bits = 0x82000000u /* 260 << 23 */ - max_abs_bits; +#ifdef __CUDA_ARCH__ + const float inv_rscale = __uint_as_float(inv_bits); +#else + float inv_rscale; + memcpy(&inv_rscale, &inv_bits, sizeof(inv_rscale)); +#endif + for (auto c = 0ul; c < nVar; ++c) { + /*--- Truncate and add 0.5 away from 0, equivalent to roundf, but inline. ---*/ + const float t = max(-128.f, min(127.f, static_cast(EQR_PASSIVE(r, c) * inv_rscale))); + qv[c] = static_cast(t + copysignf(0.5f, t)); + } +#undef EQR_PASSIVE +} + +/*! + * \brief Encode one nVar×nVar block into per-row int8 quantized storage, see EncodeQuantRow (each + * row's scale/quantization is independent, this just loops over all of them serially for + * the host path). + */ +template +SU2_CUDA_HOST_DEVICE FORCEINLINE void EncodeQuantBlock(const F& f, uint8_t* __restrict qs, int8_t* __restrict qv, + unsigned long nVar) noexcept { + for (auto r = 0ul; r < nVar; ++r) EncodeQuantRow(f, qs[r], qv + r * nVar, nVar, r); +} + +/*! + * \brief View of one matrix block, const-correct via the ScalarType template parameter. + * \c CBlockView is read-only; \c CBlockView is mutable + * and exposes \c apply(f) for writing with on-the-fly quantized encoding. + * Evaluates to \c false if the block is absent from the sparsity pattern. + */ +template +struct CBlockView { + using QuantType = std::conditional_t, const int8_t, int8_t>; + using QuantScaleType = std::conditional_t, const uint8_t, uint8_t>; + + ScalarType* ptr = nullptr; ///< Full-precision block; non-null iff not quantized. + QuantScaleType* qs = nullptr; ///< Per-row biased binary exponent; non-null iff quantized. + QuantType* qv = nullptr; ///< Quantized values (row-major); non-null iff quantized. + unsigned long nVar = 0; + + /*! \brief False when the block is not present in the sparsity pattern. */ + explicit operator bool() const { return ptr || qs; } + + /*! \brief Return entry (row \p i, col \p j), decoding quantization if necessary. */ + std::remove_const_t operator()(unsigned long i, unsigned long j) const { + using T = std::remove_const_t; + if (ptr) return ptr[i * nVar + j]; + return static_cast(qv[i * nVar + j] * DecodeQuantScale(qs[i])); + } + + /*! + * \brief Write the block from callable \p f(i,j). + * \p Overwrite=true overwrites (or quantizes for Q_LU_SGS off-diagonal blocks); + * \p Overwrite=false accumulates into non-quantized storage only — accumulating into + * quantized storage would require decode-accumulate-encode and is a silent no-op. + * Only enabled for mutable (non-const ScalarType) views. + */ + template > = 0> + void apply(const F& f) const { + if (ptr) { + for (auto i = 0ul; i < nVar; ++i) + for (auto j = 0ul; j < nVar; ++j) { + if constexpr (Overwrite) + ptr[i * nVar + j] = f(i, j); + else + ptr[i * nVar + j] += f(i, j); + } + } else if constexpr (Overwrite) { + if (qs) EncodeQuantBlock(f, qs, qv, nVar); + } + } +}; + /*! * \class CSysMatrix * \ingroup SpLinSys @@ -138,37 +277,153 @@ class CSysMatrix { unsigned long nVar; /*!< \brief Number of variables (and rows of the blocks). */ unsigned long nEqn; /*!< \brief Number of equations (and columns of the blocks). */ - ScalarType* matrix; /*!< \brief Entries of the sparse matrix. */ - unsigned long nnz; /*!< \brief Number of possible nonzero entries in the matrix. */ - const unsigned long* row_ptr; /*!< \brief Pointers to the first element in each row. */ - const unsigned long* dia_ptr; /*!< \brief Pointers to the diagonal element in each row. */ - const unsigned long* col_ind; /*!< \brief Column index for each of the elements in val(). */ - const unsigned long* col_ptr; /*!< \brief The transpose of col_ind, pointer to blocks with the same column index. */ - - ScalarType* d_matrix; /*!< \brief Device Pointer to store the matrix values on the GPU. */ - const unsigned long* d_row_ptr; /*!< \brief Device Pointers to the first element in each row. */ - const unsigned long* d_col_ind; /*!< \brief Device Column index for each of the elements in val(). */ - bool useCuda = false; /*!< \brief Boolean that indicates whether user has enabled CUDA or not. - Mainly used to conditionally free GPU memory in the class destructor. */ - - ScalarType* ILU_matrix; /*!< \brief Entries of the ILU sparse matrix. */ - unsigned long nnz_ilu; /*!< \brief Number of possible nonzero entries in the matrix (ILU). */ - const unsigned long* row_ptr_ilu; /*!< \brief Pointers to the first element in each row (ILU). */ - const unsigned long* dia_ptr_ilu; /*!< \brief Pointers to the diagonal element in each row (ILU). */ - const unsigned long* col_ind_ilu; /*!< \brief Column index for each of the elements in val() (ILU). */ - unsigned short ilu_fill_in; /*!< \brief Fill in level for the ILU preconditioner. */ - - /*!< \brief Level structure for alternative shared memory parallelization of ILU. */ + /*! + * \brief Aggregates value arrays and sparse-structure pointers for an LDU-partitioned matrix. + * Each CSysMatrix holds three LDU instances: the host matrix (mat), its + * device copy (gpu), and the ILU factorization (ilu). Ownership of the value arrays + * (d/l/u) and whether the pointers address host or device memory is managed by + * CSysMatrix. Also reused with T = QuantType to group the quantized scale/blocks + * storage (q_scale, q_blocks, d_q_scale, d_q_blocks) the same way; for those the pattern + * fields (row_ptr_l, col_ind_l, row_ptr_u, col_ind_u, nnz_l, nnz_u) are simply left + * unused, since the sparsity pattern is already available from mat/gpu. + */ + template + struct LDU { + T* d = nullptr; /*!< \brief Diagonal block values. */ + T* l = nullptr; /*!< \brief Strictly-lower block values. */ + T* u = nullptr; /*!< \brief Strictly-upper block values. */ + const su2uint* row_ptr_l = nullptr; /*!< \brief Row pointers for L (geometry-owned or GPU copy). */ + const su2uint* col_ind_l = nullptr; /*!< \brief Column indices for L. */ + const su2uint* row_ptr_u = nullptr; /*!< \brief Row pointers for U. */ + const su2uint* col_ind_u = nullptr; /*!< \brief Column indices for U. */ + unsigned long nnz_l = 0; /*!< \brief Number of L nonzeros. */ + unsigned long nnz_u = 0; /*!< \brief Number of U nonzeros. */ + }; + + LDU mat; /*!< \brief Host matrix (values owned via aligned_alloc; pattern from geometry). */ + LDU gpu; /*!< \brief Device matrix (all pointers to GPU memory). */ + LDU ilu; /*!< \brief ILU factorization, host (values owned; pattern from geometry). */ + LDU gpu_ilu; /*!< \brief ILU factorization, device (values and pattern in GPU memory). */ + ScalarType* d_invM = nullptr; /*!< \brief Device inverse diagonal blocks for the Jacobi or LU-SGS preconditioner. */ + + /*--- Quantized off-diagonal storage (used when quantized_mode == true). ---*/ + using QuantType = int8_t; + /*!< \brief Row scales are stored as the biased float exponent, hence unsigned, see + * DecodeQuantScale. */ + using QuantScaleType = uint8_t; + + /*! \brief Set by Initialize() when preconditioner == Q_LU_SGS, Q_JACOBI or Q_IDENTITY. + * mat.l and mat.u are NOT allocated; off-diagonal blocks live in q_scale/q_blocks + * below. Only the matrix-vector product (used by the Krylov solver and, for Q_LU_SGS, + * by the sweeps) reads the quantized blocks; the Jacobi preconditioner never touches + * them since it only applies the (full precision) inverse diagonal, and the identity + * preconditioner does not touch the matrix at all. */ +#ifndef CODI_REVERSE_TYPE + bool quantized_mode = false; +#else + static constexpr bool quantized_mode = false; +#endif + /*!< \brief Per-row biased exponents; .l/.u sized [nnz_l/u * nVar], .d [nPoint * nVar]. .l/.u are + * populated during assembly (quantized on the fly); .d is populated by + * QuantizeDiagonalBlocks(). .l/.u are pinned (cudaMallocHost) rather than + * aligned_alloc when useCuda, so HtDTransfer()'s async uploads them. */ + LDU q_scale; + /*!< \brief Quantized block entries; .l/.u sized [nnz_l/u * nVar * nEqn], .d [nPoint * nVar * nEqn]. */ + LDU q_blocks; + + /*!< \brief Device mirrors of the quantized storage, only allocated when quantized_mode. */ + LDU d_q_scale; + LDU d_q_blocks; + + bool useCuda = false; /*!< \brief Whether CUDA is enabled. */ + + /*!< \brief Whether the inverse diagonal blocks are only needed on the device. False for the + * Linelet preconditioner, which builds the Jacobi one but reads invM on the host. */ + bool jacobi_on_device = false; + + const su2uint* l_to_u_transp; /*!< \brief L-entry index -> U-entry index of its transpose. */ + const su2uint* u_to_l_transp; /*!< \brief U-entry index -> L-entry index of its transpose. */ + + /*! + * \brief Lookup table from edges to the L-index in the LDU split. + * U-index == edge index by construction (edges are ordered 1:1 with the U pattern). + * Therefore, edge_ptr_l == u_to_l_transp, but we keep a separate member for clarity. + */ + const su2uint* edge_ptr_l; + + unsigned short ilu_fill_in; /*!< \brief Fill level for the ILU preconditioner. */ + + /*!< \brief Level structure of the ILU dependency graph: rows within a level are independent, + * rows in level k only depend on rows in levels < k. The same table drives the forward + * (increasing level) and backward (decreasing level) substitution, because the U pattern is + * the transpose of the L pattern. Used directly by the host/OMP substitution, and flattened + * into ilu_level_ptr / d_precond_level_idx below for the GPU triangular solves. */ CCompressedSparsePatternUL levels_ilu; + /*!< \brief Coloring of the (domain-only) ILU dependency graph, used only by the GPU iterative + * factorization (see IluFactorColorKernel and ilu_color_ptr / d_ilu_color_idx below). The + * host/OMP path and the GPU triangular solves use levels_ilu instead. */ + CCompressedSparsePatternUL color_ilu; + + /*!< \brief Number of colored Gauss-Seidel sweeps used to build the ILU factorization on the + * device, see IluFactorColorKernel. Fixed (not adaptive) so the result is reproducible; set + * from config in Initialize(). The triangular solves have no equivalent sweep count: they are + * exact, one pass per level (see IluForwardKernel / IluBackwardKernel). */ + unsigned short ilu_gpu_sweeps = 1; + + vector ilu_color_ptr; /*!< \brief Start of each color in d_ilu_color_idx, size nColors+1. */ + su2uint* d_ilu_color_idx = nullptr; /*!< \brief Row indices, grouped by color. */ + + vector precond_level_ptr; /*!< \brief Start of each level in d_precond_level_idx, size nLevels+1. */ + su2uint* d_precond_level_idx = nullptr; /*!< \brief Row indices, grouped by level. */ + + /*--- The per-color (factorization) and per-level (triangular solves) kernel launch sequences + * are identical on every call: same grid/block sizes, same device pointers (all fixed members, + * allocated once). Each is captured once into a CUDA graph and replayed to remove + * host-side launch overhead without changing the parallelization. ---*/ + mutable struct CUgraphExec_st* ilu_build_graph_exec = nullptr; + + /*!< \brief Whether a build may refine the factors already on the device instead of computing + * them exactly. TransposeInPlace() clears it for good: from then on this matrix is used in + * both orientations, and the factors of one are a bad starting point for the other, which the + * ilu_gpu_sweeps colored sweeps cannot recover from. It is not restored after a build because + * the orientation flips again on the next one (and the solver refills the matrix in between + * without going through TransposeInPlace). Only the discrete adjoint transposes, so the primal + * keeps refining as before. */ + mutable bool ilu_can_refine = true; + mutable struct CUgraphExec_st* precond_fwd_graph_exec = nullptr; // ILU or LU-SGS forward only + mutable struct CUgraphExec_st* precond_bwd_graph_exec = nullptr; // LU-SGS backward only + mutable const ScalarType* precond_fwd_graph_vec = nullptr; /*!< \brief Pointers the apply graph + * was captured with, to detect when + * it must be recaptured (the + * executable graph itself is then + * updated in place, not rebuilt, + * see InstantiateOrUpdateGraph). */ + mutable ScalarType* precond_fwd_graph_prod = nullptr; + mutable ScalarType* precond_bwd_graph_prod = nullptr; + + /*--- Non-default stream, needed for two uses: (1) the preconditioner build/apply CUDA graphs + * below, since the legacy default stream cannot be captured into a graph; (2) HtDTransfer's + * async H2D transfer of the quantized L/U blocks, so that transfer can run concurrently (copy + * engine) with kernels issued on the default stream (e.g. QuantizeDiagonalBlocksGPU, on the SM) + * instead of queueing behind them on the same stream. The two are mutually exclusive for ILU + * (never quantized) but not for Q_LU_SGS, which uses both; sharing one stream still needs no + * extra synchronization, and in fact gives the right answer for free: the apply graph is + * launched into aux_stream, hence ordered after the transfer of the quantized blocks its + * kernels read. htd_event marks the end of the H2D transfer specifically, so a *default*-stream + * kernel that reads the result (the quantized SpMV) can wait on it without a host-side + * block. ---*/ + mutable struct CUstream_st* aux_stream = nullptr; + mutable struct CUevent_st* htd_event = nullptr; + ScalarType* invM; /*!< \brief Inverse of (Jacobi) preconditioner. */ /*--- Temporary (hence mutable) working memory used in the Linelet preconditioner, outer vector is for threads ---*/ - mutable vector > + mutable vector> LineletUpper; /*!< \brief Pointers to the upper blocks of the tri-diag system (working memory). */ - mutable vector > + mutable vector> LineletInvDiag; /*!< \brief Inverse of the diagonal blocks of the tri-diag system (working memory). */ - mutable vector > + mutable vector> LineletVector; /*!< \brief Solution and RHS of the tri-diag system (working memory). */ #ifdef USE_MKL @@ -188,21 +443,6 @@ class CSysMatrix { mutable CPastixWrapper pastix_wrapper; #endif - /*! - * \brief Auxilary object to wrap the edge map pointer used in fast block updates, i.e. without linear searches. - */ - struct { - const unsigned long* ptr = nullptr; - unsigned long nEdge = 0; - - operator bool() { return nEdge != 0; } - - inline unsigned long operator()(unsigned long edge, unsigned long node) const { return ptr[2 * edge + node]; } - inline unsigned long ij(unsigned long edge) const { return ptr[2 * edge]; } - inline unsigned long ji(unsigned long edge) const { return ptr[2 * edge + 1]; } - - } edge_ptr; - /*! * \brief Handle type conversion for when we Set, Add, etc. blocks, preserving derivative information (if supported by * types). @@ -290,7 +530,7 @@ class CSysMatrix { * \param[in,out] matrix - On entry the system matrix, on exit the factorized matrix. * \param[in,out] vec - On entry the rhs, on exit the solution. */ - void Gauss_Elimination(ScalarType* matrix, ScalarType* vec) const; + void GaussElimination(ScalarType* matrix, ScalarType* vec) const; /*! * \brief Invert a small dense matrix. @@ -305,7 +545,7 @@ class CSysMatrix { * \param[in] rhs - Right-hand-side of the linear system. * \return Solution of the linear system (overwritten on rhs). */ - inline void Gauss_Elimination(unsigned long block_i, ScalarType* rhs) const; + inline void GaussElimination(unsigned long block_i, ScalarType* rhs) const; /*! * \brief Inverse diagonal block. @@ -322,20 +562,11 @@ class CSysMatrix { inline const ScalarType* InvertDiagonalBlockILUMatrix(unsigned long block_i); /*! - * \brief Copies the block (i, j) of the matrix-by-blocks structure in the internal variable *block. - * \param[in] block_i - Indexes of the block in the matrix-by-blocks structure. - * \param[in] block_j - Indexes of the block in the matrix-by-blocks structure. + * \brief Returns the start of the ILU block or nullptr if (i,j) is not a nonzero. + * \param[in] block_i/j - Indexes of the block in the matrix-by-blocks structure. */ inline ScalarType* GetBlock_ILUMatrix(unsigned long block_i, unsigned long block_j); - /*! - * \brief Set the value of a block in the sparse matrix. - * \param[in] block_i - Indexes of the block in the matrix-by-blocks structure. - * \param[in] block_j - Indexes of the block in the matrix-by-blocks structure. - * \param[in] **val_block - Block to set to A(i, j). - */ - inline void SetBlock_ILUMatrix(unsigned long block_i, unsigned long block_j, ScalarType* val_block); - /*! * \brief Performs the product of i-th row of the upper part of a sparse matrix by a vector. * \param[in] vec - Vector to be multiplied by the upper part of the sparse matrix A. @@ -372,6 +603,102 @@ class CSysMatrix { */ void RowProduct(const CSysVector& vec, unsigned long row_i, ScalarType* prod) const; + /*! + * \brief Computes product += A_k * vec using the quantized representation of block k. + * \note Only valid after QuantizeDiagonalBlocks() has been called. + * \param[in] k - Block index in the CSR flat storage. + * \param[in] vec - Input vector (nEqn entries). + * \param[in,out] prod - Accumulation output (nVar entries). + */ + inline void QuantizedMatVecAdd(const QuantScaleType* qs, const QuantType* qv, const ScalarType* vec, + ScalarType* prod) const; + + /*! \brief Quantize one nVar×nVar block (row-major) into the int8 scale+value arrays. + * Called on the hot assembly path (SetBlocks/UpdateBlocks in Q_LU_SGS mode). */ + inline void QuantizeBlock(const ScalarType* blk, QuantScaleType* qs, QuantType* qv) const { + EncodeQuantBlock([&](unsigned long r, unsigned long c) { return blk[r * nVar + c]; }, qs, qv, nVar); + } + + /*! \brief Full-row product using quantized L/D/U (Q_LU_SGS SpMV path). */ + inline void QuantizedRowProduct(const CSysVector& vec, unsigned long row_i, ScalarType* prod) const; + + /*! \brief Upper-triangle product using quantized U (Q_LU_SGS backward sweep). */ + inline void QuantizedUpperProduct(const CSysVector& vec, unsigned long row_i, unsigned long col_ub, + ScalarType* prod) const; + + /*! \brief Lower-triangle product using quantized L (Q_LU_SGS forward sweep). */ + inline void QuantizedLowerProduct(const CSysVector& vec, unsigned long row_i, unsigned long col_lb, + ScalarType* prod) const; + + /*! \brief Diagonal product using quantized D (Q_LU_SGS backward sweep). */ + inline void QuantizedDiagonalProduct(const CSysVector& vec, unsigned long row_i, ScalarType* prod) const; + + /*! \brief Gauss elimination on the quantized diagonal block: decodes q_blocks.d into a local + * ScalarType buffer and delegates to the scalar GaussElimination overload. */ + inline void QuantizedGaussElimination(unsigned long block_i, ScalarType* rhs) const; + + /*--- Hooks for GPU versions (implemented is in CSysMatrixGPU.cu). ---*/ + + /*! + * \brief Performs the product of a sparse matrix by a CSysVector on the device. + */ + void MatrixVectorProductGPU(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, + const CConfig* config) const; + + /*! + * \brief Quantize the diagonal blocks directly on the device. + * \note Requires the device matrix to be up to date, see HtDTransfer. + */ + void QuantizeDiagonalBlocksGPU(); + + /*! + * \brief Build the Jacobi preconditioner on the device, from the device copy of the matrix. + * \note Requires the device matrix to be up to date, see HtDTransfer. + */ + void BuildJacobiPreconditionerGPU(); + + /*! + * \brief Apply the Jacobi preconditioner on the GPU/device side. + */ + void ComputeJacobiPreconditionerGPU(const CSysVector& vec, CSysVector& prod, + CGeometry* geometry, const CConfig* config) const; + + /*! + * \brief Build the ILU preconditioner on the device, from the device copy of the matrix. + * \note Requires the device matrix to be up to date, see HtDTransfer. + */ + void BuildILUPreconditionerGPU(); + + /*! + * \brief Apply the ILU preconditioner on the device. + */ + void ComputeILUPreconditionerGPU(const CSysVector& vec, CSysVector& prod) const; + + /*! + * \brief Build the LU-SGS preconditioner on the device + */ + void BuildLU_SGSPreconditionerGPU(); + + /*! + * \brief Apply the LU-SGS preconditioner forward pass on the device + */ + void ComputeLU_SGSForwardGPU(const CSysVector& vec, CSysVector& prod) const; + + /*! + * \brief Apply the LU-SGS preconditioner backward pass on the device + */ + void ComputeLU_SGSBackwardGPU(CSysVector& prod) const; + + /*! + * \brief Apply the forward pass of the LU-SGS preconditioner + */ + void ComputeLU_SGSPreconditionerForward(const CSysVector& vec, CSysVector& prod) const; + + /*! + * \brief Apply the backward pass of the LU-SGS preconditioner + */ + void ComputeLU_SGSPreconditionerBackward(CSysVector& prod) const; + public: /*! * \brief Constructor of the class. @@ -392,11 +719,21 @@ class CSysMatrix { * \param[in] neqn - Number of equations (and columns of the blocks). * \param[in] geometry - Geometrical definition of the problem. * \param[in] config - Definition of the particular problem. - * \param[in] needTranspPtr - If "col_ptr" should be created, used for "SetDiagonalAsColumnSum". + * \param[in] needTranspPtr - If the L/U transpose maps should be built, used for "SetDiagonalAsColumnSum". + * \param[in] allow_quant - Quantization is only possible with solvers that "set and forget" the off-diagonal + * blocks of the matrix. Solvers that perform multiple updates would lose too much information, so + * that pattern is not supported with quantization (the code will hit null pointers). It is up to + * the solver to declare whether it will "set and forget". + * \param[in] override_prec - Decide if, and with what argument to override the preconditioner. */ void Initialize(unsigned long npoint, unsigned long npointdomain, unsigned short nvar, unsigned short neqn, bool EdgeConnect, CGeometry* geometry, const CConfig* config, bool needTranspPtr = false, - bool grad_mode = false); + bool allow_quant = false, std::optional override_prec = std::nullopt); + + /*! + * \brief Compresses off-diagonal blocks into quantized form for use with USE_QUANTIZATION. + */ + void QuantizeDiagonalBlocks(); /*! * \brief Sets to zero all the entries of the sparse matrix. @@ -421,15 +758,19 @@ class CSysMatrix { * \return Pointer to location in memory where the block starts. */ FORCEINLINE const ScalarType* GetBlock(unsigned long block_i, unsigned long block_j) const { - /*--- The position of the diagonal block is known which allows halving the search space. ---*/ - const auto end = (block_j < block_i) ? dia_ptr[block_i] : row_ptr[block_i + 1]; - for (auto index = (block_j < block_i) ? row_ptr[block_i] : dia_ptr[block_i]; index < end; ++index) - if (col_ind[index] == block_j) return &matrix[index * nVar * nEqn]; + if (block_i == block_j) return &mat.d[block_i * nVar * nEqn]; + if (block_j < block_i) { + for (auto index = mat.row_ptr_l[block_i]; index < mat.row_ptr_l[block_i + 1]; ++index) + if (mat.col_ind_l[index] == block_j) return &mat.l[index * nVar * nEqn]; + return nullptr; + } + for (auto index = mat.row_ptr_u[block_i]; index < mat.row_ptr_u[block_i + 1]; ++index) + if (mat.col_ind_u[index] == block_j) return &mat.u[index * nVar * nEqn]; return nullptr; } /*! - * \brief Get a pointer to the start of block "ij", non-const version + * \brief Get a pointer to the start of block "ij", non-const version. */ FORCEINLINE ScalarType* GetBlock(unsigned long block_i, unsigned long block_j) { const CSysMatrix& const_this = *this; @@ -437,131 +778,98 @@ class CSysMatrix { } /*! - * \brief Gets the value of a particular entry in block "ij". - * \param[in] block_i - Row index. - * \param[in] block_j - Column index. - * \param[in] iVar - Row of the block. - * \param[in] jVar - Column of the block. - * \return Value of the block entry. - */ - FORCEINLINE ScalarType GetBlock(unsigned long block_i, unsigned long block_j, unsigned short iVar, - unsigned short jVar) const { - auto mat_ij = GetBlock(block_i, block_j); - if (!mat_ij) return 0.0; - return mat_ij[iVar * nEqn + jVar]; + * \brief Read-only view of block (block_i, block_j). In Q_LU_SGS mode values are decoded + * on access inside CBlockView::operator()(i,j); no temporary copy is made. + * \return A CBlockView that evaluates to false if the block is absent. + */ + FORCEINLINE CBlockView GetBlockView(unsigned long block_i, unsigned long block_j) const { +#define GET_BLOCK_VIEW_IMPL \ + if (!quantized_mode || block_i == block_j) { \ + return {GetBlock(block_i, block_j), nullptr, nullptr, nVar}; \ + } \ + if (block_j < block_i) { \ + for (auto k = mat.row_ptr_l[block_i]; k < mat.row_ptr_l[block_i + 1]; ++k) \ + if (mat.col_ind_l[k] == block_j) return {nullptr, &q_scale.l[k * nVar], &q_blocks.l[k * nVar * nVar], nVar}; \ + } else { \ + for (auto k = mat.row_ptr_u[block_i]; k < mat.row_ptr_u[block_i + 1]; ++k) \ + if (mat.col_ind_u[k] == block_j) return {nullptr, &q_scale.u[k * nVar], &q_blocks.u[k * nVar * nVar], nVar}; \ + } \ + return {} + GET_BLOCK_VIEW_IMPL; } /*! - * \brief Set the value of a block (in flat format) in the sparse matrix with scaling. - * \note If the template param Overwrite is false we add to the block (bij += alpha*b). - * \param[in] block_i - Row index. - * \param[in] block_j - Column index. - * \param[in] val_block - Block to set to A(i, j). - * \param[in] alpha - Scale factor. + * \overload Non const version of GetBlockView. */ - template ::value> = 0> - inline void SetBlock(unsigned long block_i, unsigned long block_j, const OtherType* val_block, - OtherType alpha = 1.0) { - auto mat_ij = GetBlock(block_i, block_j); - if (!mat_ij) return; - SU2_OMP_SIMD - for (auto iVar = 0ul; iVar < nVar * nEqn; ++iVar) { - mat_ij[iVar] = (Overwrite ? ScalarType(0) : mat_ij[iVar]) + PassiveAssign(alpha * val_block[iVar]); - } - } - - /*! - * \brief Add a scaled block (in flat format) to the sparse matrix (see SetBlock). - * \param[in] block_i - Row index. - * \param[in] block_j - Column index. - * \param[in] val_block - Block to set to A(i, j). - * \param[in] alpha - Scale factor. - */ - template ::value> = 0> - inline void AddBlock(unsigned long block_i, unsigned long block_j, const OtherType* val_block, - OtherType alpha = 1.0) { - SetBlock(block_i, block_j, val_block, alpha); + FORCEINLINE CBlockView GetBlockView(unsigned long block_i, unsigned long block_j) { + GET_BLOCK_VIEW_IMPL; +#undef GET_BLOCK_VIEW_IMPL } /*! * \brief Set the value of a scaled block in the sparse matrix. - * \note If the template param Overwrite is false we add to the block (bij += alpha*b). + * \note This is an templated overload for C2Dcontainer specialization su2matrix. + * It assumes that MatrixType supports a member type Scalar and access operator(i, j). + * If the template param Overwrite is false we add to the block (bij += alpha*b). * \param[in] block_i - Row index. * \param[in] block_j - Column index. * \param[in] val_block - Block to set to A(i, j). * \param[in] alpha - Scale factor. */ - template - inline void SetBlock(unsigned long block_i, unsigned long block_j, const OtherType* const* val_block, - OtherType alpha = 1.0) { - auto mat_ij = GetBlock(block_i, block_j); - if (!mat_ij) return; - for (auto iVar = 0ul; iVar < nVar; ++iVar) { - for (auto jVar = 0ul; jVar < nEqn; ++jVar) { - *mat_ij = (Overwrite ? ScalarType(0) : *mat_ij) + PassiveAssign(alpha * val_block[iVar][jVar]); - ++mat_ij; - } - } + template + inline void SetBlock(unsigned long block_i, unsigned long block_j, MatrixType& val_block, + std::decay_t alpha = 1.0) { + auto view = GetBlockView(block_i, block_j); + if (!view) return; + view.template apply( + [&](unsigned long i, unsigned long j) { return PassiveAssign(alpha * val_block(i, j)); }); } /*! - * \brief Adds a scaled block to the sparse matrix (see SetBlock). - * \param[in] block_i - Row index. - * \param[in] block_j - Column index. - * \param[in] val_block - Block to add to A(i, j). - * \param[in] alpha - Scale factor. + * \overload val_block is a pointer instead of a matrix type. */ - template - inline void AddBlock(unsigned long block_i, unsigned long block_j, const OtherType* const* val_block, - OtherType alpha = 1.0) { - SetBlock(block_i, block_j, val_block, alpha); + template ::value> = 0> + inline void SetBlock(unsigned long block_i, unsigned long block_j, const OtherType* val_block, + std::decay_t alpha = 1.0) { + auto view = GetBlockView(block_i, block_j); + if (!view) return; + view.template apply( + [&](unsigned long i, unsigned long j) { return PassiveAssign(alpha * val_block[i * nEqn + j]); }); } /*! - * \brief Subtracts the specified block to the sparse matrix (see AddBlock). - * \param[in] block_i - Row index. - * \param[in] block_j - Column index. - * \param[in] val_block - Block to subtract to A(i, j). + * \overload val_block is a double pointer instead of matrix type. */ - template - inline void SubtractBlock(unsigned long block_i, unsigned long block_j, const OtherType* const* val_block) { - AddBlock(block_i, block_j, val_block, OtherType(-1)); + template + inline void SetBlock(unsigned long block_i, unsigned long block_j, const OtherType* const* val_block, + std::decay_t alpha = 1.0) { + auto view = GetBlockView(block_i, block_j); + if (!view) return; + view.template apply( + [&](unsigned long i, unsigned long j) { return PassiveAssign(alpha * val_block[i][j]); }); } /*! - * \brief Set the value of a scaled block in the sparse matrix. - * \note This is an templated overload for C2Dcontainer specialization su2matrix. - * It assumes that MatrixType supports a member type Scalar and access operator[][]. - * If the template param Overwrite is false we add to the block (bij += alpha*b). + * \brief Add a scaled block (in flat format) to the sparse matrix (see SetBlock). * \param[in] block_i - Row index. * \param[in] block_j - Column index. * \param[in] val_block - Block to set to A(i, j). * \param[in] alpha - Scale factor. */ - template - inline void SetBlock(unsigned long block_i, unsigned long block_j, MatrixType& val_block, - typename MatrixType::Scalar alpha = 1.0) { - auto mat_ij = GetBlock(block_i, block_j); - if (!mat_ij) return; - for (auto iVar = 0ul; iVar < nVar; ++iVar) { - for (auto jVar = 0ul; jVar < nEqn; ++jVar) { - *mat_ij = (Overwrite ? ScalarType(0) : *mat_ij) + PassiveAssign(alpha * val_block(iVar, jVar)); - ++mat_ij; - } - } + template + inline void AddBlock(unsigned long block_i, unsigned long block_j, const T& val_block, OtherType alpha = 1.0) { + SetBlock(block_i, block_j, val_block, alpha); } /*! - * \brief Adds a scaled block to the sparse matrix (see SetBlock). + * \brief Subtracts the specified block to the sparse matrix (see AddBlock). * \param[in] block_i - Row index. * \param[in] block_j - Column index. - * \param[in] val_block - Block to add to A(i, j). - * \param[in] alpha - Scale factor. + * \param[in] val_block - Block to subtract to A(i, j). */ - template - inline void AddBlock(unsigned long block_i, unsigned long block_j, MatrixType& val_block, - typename MatrixType::Scalar alpha = 1.0) { - SetBlock(block_i, block_j, val_block, alpha); + template + inline void SubtractBlock(unsigned long block_i, unsigned long block_j, const T& val_block) { + AddBlock(block_i, block_j, val_block, -1); } /*! @@ -574,10 +882,11 @@ class CSysMatrix { */ inline void GetBlocks(unsigned long iEdge, unsigned long iPoint, unsigned long jPoint, ScalarType*& bii, ScalarType*& bij, ScalarType*& bji, ScalarType*& bjj) { - bii = &matrix[dia_ptr[iPoint] * nVar * nEqn]; - bjj = &matrix[dia_ptr[jPoint] * nVar * nEqn]; - bij = &matrix[edge_ptr(iEdge, 0) * nVar * nEqn]; - bji = &matrix[edge_ptr(iEdge, 1) * nVar * nEqn]; + const auto blkSz = nVar * nEqn; + bii = &mat.d[iPoint * blkSz]; + bjj = &mat.d[jPoint * blkSz]; + bij = &mat.u[iEdge * blkSz]; + bji = &mat.l[edge_ptr_l[iEdge] * blkSz]; } /*! @@ -590,20 +899,45 @@ class CSysMatrix { * \param[in] block_j - Adds to ij, subs from jj. * \param[in] scale - Scale blocks during update (axpy type op). */ - template + template inline void UpdateBlocks(unsigned long iEdge, unsigned long iPoint, unsigned long jPoint, const MatrixType& block_i, const MatrixType& block_j, OtherType scale = 1) { - ScalarType *bii, *bij, *bji, *bjj; - GetBlocks(iEdge, iPoint, jPoint, bii, bij, bji, bjj); + const auto blkSz = nVar * nEqn; + auto* bii = &mat.d[iPoint * blkSz]; + auto* bjj = &mat.d[jPoint * blkSz]; unsigned long iVar, jVar, offset = 0; + if (quantized_mode) { + assert(OverwriteOffDiag); + /*--- Diagonal: full-precision accumulation. Off-diagonal: quantize on the fly. ---*/ + ScalarType bij_buf[MAXNVAR * MAXNVAR], bji_buf[MAXNVAR * MAXNVAR]; + for (iVar = 0; iVar < nVar; iVar++) + for (jVar = 0; jVar < nEqn; jVar++, ++offset) { + bii[offset] += PassiveAssign(block_i[iVar][jVar] * scale); + bjj[offset] -= PassiveAssign(block_j[iVar][jVar] * scale); + bij_buf[offset] = PassiveAssign(block_j[iVar][jVar] * scale); + bji_buf[offset] = -PassiveAssign(block_i[iVar][jVar] * scale); + } + QuantizeBlock(bij_buf, &q_scale.u[iEdge * nVar], &q_blocks.u[iEdge * blkSz]); + const auto k_l = edge_ptr_l[iEdge]; + QuantizeBlock(bji_buf, &q_scale.l[k_l * nVar], &q_blocks.l[k_l * blkSz]); + return; + } + + auto* bij = &mat.u[iEdge * blkSz]; + auto* bji = &mat.l[edge_ptr_l[iEdge] * blkSz]; for (iVar = 0; iVar < nVar; iVar++) { for (jVar = 0; jVar < nEqn; jVar++) { bii[offset] += PassiveAssign(block_i[iVar][jVar] * scale); - bij[offset] += PassiveAssign(block_j[iVar][jVar] * scale); - bji[offset] -= PassiveAssign(block_i[iVar][jVar] * scale); bjj[offset] -= PassiveAssign(block_j[iVar][jVar] * scale); + if constexpr (OverwriteOffDiag) { + bij[offset] = PassiveAssign(block_j[iVar][jVar] * scale); + bji[offset] = -PassiveAssign(block_i[iVar][jVar] * scale); + } else { + bij[offset] += PassiveAssign(block_j[iVar][jVar] * scale); + bji[offset] -= PassiveAssign(block_i[iVar][jVar] * scale); + } ++offset; } } @@ -615,7 +949,7 @@ class CSysMatrix { template inline void UpdateBlocksSub(unsigned long iEdge, unsigned long iPoint, unsigned long jPoint, const MatrixType& block_i, const MatrixType& block_j) { - UpdateBlocks(iEdge, iPoint, jPoint, block_i, block_j, -1); + UpdateBlocks(iEdge, iPoint, jPoint, block_i, block_j, -1); } /*! @@ -623,12 +957,13 @@ class CSysMatrix { * \note Nothing is updated if the mask is 0. */ template - FORCEINLINE void UpdateBlocks(simd::Array iEdge, simd::Array iPoint, simd::Array jPoint, - const MatTypeSIMD& block_i, const MatTypeSIMD& block_j, simd::Array mask = 1) { + FORCEINLINE void SetBlocks(simd::Array iEdge, simd::Array iPoint, simd::Array jPoint, + const MatTypeSIMD& block_i, const MatTypeSIMD& block_j, simd::Array mask = 1) { static_assert(MatTypeSIMD::StaticSize, "This method requires static size blocks."); static_assert(MatTypeSIMD::IsRowMajor, "Block storage is not compatible with matrix."); constexpr size_t blkSz = MatTypeSIMD::StaticSize; assert(blkSz == nVar * nEqn); + constexpr size_t nVar = MatTypeSIMD::StaticNRows; /*--- "Transpose" the blocks, scale, and possibly convert types, * giving the compiler the chance to vectorize all of these. ---*/ @@ -646,20 +981,32 @@ class CSysMatrix { for (size_t k = 0; k < N; ++k) { if (mask[k] == 0) continue; - /*--- Fetch the blocks. ---*/ - auto bii = &matrix[dia_ptr[iPoint[k]] * blkSz]; - auto bjj = &matrix[dia_ptr[jPoint[k]] * blkSz]; - auto bij = &matrix[edge_ptr(iEdge[k], 0) * blkSz]; - auto bji = &matrix[edge_ptr(iEdge[k], 1) * blkSz]; - - /*--- Update, block i was negated during transpose in the - * hope the assignments below become non-temporal stores. ---*/ - SU2_OMP_SIMD - for (size_t i = 0; i < blkSz; ++i) { - bii[i] -= blk_i[k][i]; - bjj[i] -= blk_j[k][i]; - bij[i] = blk_j[k][i]; - bji[i] = blk_i[k][i]; + auto bii = &mat.d[iPoint[k] * blkSz]; + auto bjj = &mat.d[jPoint[k] * blkSz]; + + if (quantized_mode) { + SU2_OMP_SIMD + for (size_t i = 0; i < blkSz; ++i) { + bii[i] -= blk_i[k][i]; + bjj[i] -= blk_j[k][i]; + } + EncodeQuantBlock([&, k](unsigned long r, unsigned long c) { return blk_j[k][r * nVar + c]; }, + &q_scale.u[iEdge[k] * nVar], &q_blocks.u[iEdge[k] * blkSz], nVar); + const auto k_l = edge_ptr_l[iEdge[k]]; + EncodeQuantBlock([&, k](unsigned long r, unsigned long c) { return blk_i[k][r * nVar + c]; }, + &q_scale.l[k_l * nVar], &q_blocks.l[k_l * blkSz], nVar); + } else { + auto bij = &mat.u[iEdge[k] * blkSz]; + auto bji = &mat.l[edge_ptr_l[iEdge[k]] * blkSz]; + /*--- Update, block i was negated during transpose in the + * hope the assignments below become non-temporal stores. ---*/ + SU2_OMP_SIMD + for (size_t i = 0; i < blkSz; ++i) { + bii[i] -= blk_i[k][i]; + bjj[i] -= blk_j[k][i]; + bij[i] = blk_j[k][i]; + bji[i] = blk_i[k][i]; + } } } } @@ -677,11 +1024,25 @@ class CSysMatrix { template inline void SetBlocks(unsigned long iEdge, const MatrixType& block_i, const MatrixType& block_j, OtherType scale = 1) { - ScalarType* bij = &matrix[edge_ptr(iEdge, 0) * nVar * nEqn]; - ScalarType* bji = &matrix[edge_ptr(iEdge, 1) * nVar * nEqn]; - + const auto blkSz = nVar * nEqn; unsigned long iVar, jVar, offset = 0; + if (quantized_mode) { + assert(Overwrite); + ScalarType bij_buf[MAXNVAR * MAXNVAR], bji_buf[MAXNVAR * MAXNVAR]; + for (iVar = 0; iVar < nVar; iVar++) + for (jVar = 0; jVar < nEqn; jVar++, ++offset) { + bij_buf[offset] = PassiveAssign(block_j[iVar][jVar] * scale); + bji_buf[offset] = -PassiveAssign(block_i[iVar][jVar] * scale); + } + QuantizeBlock(bij_buf, &q_scale.u[iEdge * nVar], &q_blocks.u[iEdge * blkSz]); + const auto k_l = edge_ptr_l[iEdge]; + QuantizeBlock(bji_buf, &q_scale.l[k_l * nVar], &q_blocks.l[k_l * blkSz]); + return; + } + + ScalarType* bij = &mat.u[iEdge * blkSz]; + ScalarType* bji = &mat.l[edge_ptr_l[iEdge] * blkSz]; for (iVar = 0; iVar < nVar; iVar++) { for (jVar = 0; jVar < nEqn; jVar++) { bij[offset] = (Overwrite ? ScalarType(0) : bij[offset]) + PassiveAssign(block_j[iVar][jVar] * scale); @@ -708,6 +1069,80 @@ class CSysMatrix { SetBlocks(iEdge, block_i, block_j, -1); } + /*! + * \brief Set the four blocks of an edge, for fluxes whose i and j contributions are independent. + * \note The diagonal blocks are accumulated, the off-diagonal blocks are set. + */ + template + inline void SetBlocks(unsigned long iEdge, unsigned long iPoint, unsigned long jPoint, const MatrixType& jac_ii, + const MatrixType& jac_ij, const MatrixType& jac_ji, const MatrixType& jac_jj, + OtherType mask = 1) { + const auto blkSz = nVar * nEqn; + auto* bii = &mat.d[iPoint * blkSz]; + auto* bjj = &mat.d[jPoint * blkSz]; + unsigned long iVar, jVar, offset = 0; + + if (quantized_mode) { + ScalarType bij_buf[MAXNVAR * MAXNVAR], bji_buf[MAXNVAR * MAXNVAR]; + for (iVar = 0; iVar < nVar; iVar++) + for (jVar = 0; jVar < nEqn; jVar++, ++offset) { + bii[offset] += PassiveAssign(jac_ii[iVar][jVar] * mask); + bjj[offset] += PassiveAssign(jac_jj[iVar][jVar] * mask); + bij_buf[offset] = PassiveAssign(jac_ij[iVar][jVar] * mask); + bji_buf[offset] = PassiveAssign(jac_ji[iVar][jVar] * mask); + } + QuantizeBlock(bij_buf, &q_scale.u[iEdge * nVar], &q_blocks.u[iEdge * blkSz]); + const auto k_l = edge_ptr_l[iEdge]; + QuantizeBlock(bji_buf, &q_scale.l[k_l * nVar], &q_blocks.l[k_l * blkSz]); + return; + } + + auto* bij = &mat.u[iEdge * blkSz]; + auto* bji = &mat.l[edge_ptr_l[iEdge] * blkSz]; + for (iVar = 0; iVar < nVar; iVar++) { + for (jVar = 0; jVar < nEqn; jVar++) { + bii[offset] += PassiveAssign(jac_ii[iVar][jVar] * mask); + bjj[offset] += PassiveAssign(jac_jj[iVar][jVar] * mask); + bij[offset] = PassiveAssign(jac_ij[iVar][jVar] * mask); + bji[offset] = PassiveAssign(jac_ji[iVar][jVar] * mask); + ++offset; + } + } + } + + /*! + * \brief Set the off-diagonal blocks of an edge, the diagonal being assembled elsewhere. + */ + template + inline void SetOffDiagBlocks(unsigned long iEdge, const MatrixType& jac_ij, const MatrixType& jac_ji, + OtherType mask = 1) { + const auto blkSz = nVar * nEqn; + unsigned long iVar, jVar, offset = 0; + + if (quantized_mode) { + ScalarType bij_buf[MAXNVAR * MAXNVAR], bji_buf[MAXNVAR * MAXNVAR]; + for (iVar = 0; iVar < nVar; iVar++) + for (jVar = 0; jVar < nEqn; jVar++, ++offset) { + bij_buf[offset] = PassiveAssign(jac_ij[iVar][jVar] * mask); + bji_buf[offset] = PassiveAssign(jac_ji[iVar][jVar] * mask); + } + QuantizeBlock(bij_buf, &q_scale.u[iEdge * nVar], &q_blocks.u[iEdge * blkSz]); + const auto k_l = edge_ptr_l[iEdge]; + QuantizeBlock(bji_buf, &q_scale.l[k_l * nVar], &q_blocks.l[k_l * blkSz]); + return; + } + + auto* bij = &mat.u[iEdge * blkSz]; + auto* bji = &mat.l[edge_ptr_l[iEdge] * blkSz]; + for (iVar = 0; iVar < nVar; iVar++) { + for (jVar = 0; jVar < nEqn; jVar++) { + bij[offset] = PassiveAssign(jac_ij[iVar][jVar] * mask); + bji[offset] = PassiveAssign(jac_ji[iVar][jVar] * mask); + ++offset; + } + } + } + /*! * \brief SIMD version, does the update for multiple edges. * \note Nothing is updated if the mask is 0. @@ -719,6 +1154,7 @@ class CSysMatrix { static_assert(MatTypeSIMD::IsRowMajor, "Block storage is not compatible with matrix."); constexpr size_t blkSz = MatTypeSIMD::StaticSize; assert(blkSz == nVar * nEqn); + constexpr size_t nVar = MatTypeSIMD::StaticNRows; /*--- "Transpose" the blocks, scale, and possibly convert types, * giving the compiler the chance to vectorize all of these. ---*/ @@ -736,16 +1172,22 @@ class CSysMatrix { for (size_t k = 0; k < N; ++k) { if (mask[k] == 0) continue; - /*--- Fetch the blocks. ---*/ - auto bij = &matrix[edge_ptr(iEdge[k], 0) * blkSz]; - auto bji = &matrix[edge_ptr(iEdge[k], 1) * blkSz]; - - /*--- Update, block i was negated during transpose in the - * hope the assignments below become non-temporal stores. ---*/ - SU2_OMP_SIMD - for (size_t i = 0; i < blkSz; ++i) { - bij[i] = blk_j[k][i]; - bji[i] = blk_i[k][i]; + if (quantized_mode) { + EncodeQuantBlock([&, k](unsigned long r, unsigned long c) { return blk_j[k][r * nVar + c]; }, + &q_scale.u[iEdge[k] * nVar], &q_blocks.u[iEdge[k] * blkSz], nVar); + const auto k_l = edge_ptr_l[iEdge[k]]; + EncodeQuantBlock([&, k](unsigned long r, unsigned long c) { return blk_i[k][r * nVar + c]; }, + &q_scale.l[k_l * nVar], &q_blocks.l[k_l * blkSz], nVar); + } else { + ScalarType* bij = &mat.u[iEdge[k] * blkSz]; + ScalarType* bji = &mat.l[edge_ptr_l[iEdge[k]] * blkSz]; + /*--- Update, block i was negated during transpose in the + * hope the assignments below become non-temporal stores. ---*/ + SU2_OMP_SIMD + for (size_t i = 0; i < blkSz; ++i) { + bij[i] = blk_j[k][i]; + bji[i] = blk_i[k][i]; + } } } } @@ -760,7 +1202,7 @@ class CSysMatrix { */ template inline void SetBlock2Diag(unsigned long block_i, const OtherType& val_block, T alpha = 1.0) { - auto mat_ii = &matrix[dia_ptr[block_i] * nVar * nEqn]; + auto mat_ii = &mat.d[block_i * nVar * nEqn]; for (auto iVar = 0ul; iVar < nVar; iVar++) for (auto jVar = 0ul; jVar < nEqn; jVar++) { @@ -793,8 +1235,8 @@ class CSysMatrix { */ template inline void AddVal2Diag(unsigned long block_i, OtherType val_matrix) { - for (auto iVar = 0ul; iVar < nVar; iVar++) - matrix[dia_ptr[block_i] * nVar * nVar + iVar * (nVar + 1)] += PassiveAssign(val_matrix); + auto d = &mat.d[block_i * nVar * nVar]; + for (auto iVar = 0ul; iVar < nVar; iVar++) d[iVar * (nVar + 1)] += PassiveAssign(val_matrix); } /*! @@ -806,7 +1248,7 @@ class CSysMatrix { */ template inline void AddVal2Diag(unsigned long block_i, unsigned long iVar, OtherType val) { - matrix[dia_ptr[block_i] * nVar * nVar + iVar * (nVar + 1)] += PassiveAssign(val); + mat.d[block_i * nVar * nVar + iVar * (nVar + 1)] += PassiveAssign(val); } /*! @@ -817,13 +1259,11 @@ class CSysMatrix { */ template inline void SetVal2Diag(unsigned long block_i, OtherType val_matrix) { - unsigned long iVar, index = dia_ptr[block_i] * nVar * nVar; - /*--- Clear entire block before setting its diagonal. ---*/ SU2_OMP_SIMD - for (iVar = 0; iVar < nVar * nVar; iVar++) matrix[index + iVar] = 0.0; + for (auto iVar = 0ul; iVar < nVar * nVar; iVar++) mat.d[block_i * nVar * nVar + iVar] = 0.0; - for (iVar = 0; iVar < nVar; iVar++) matrix[index + iVar * (nVar + 1)] = PassiveAssign(val_matrix); + AddVal2Diag(block_i, val_matrix); } /*! @@ -876,49 +1316,6 @@ class CSysMatrix { void MatrixVectorProduct(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, const CConfig* config) const; - /*! - * \brief Performs the product of a sparse matrix by a CSysVector. - * \param[in] vec - CSysVector to be multiplied by the sparse matrix A. - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] config - Definition of the particular problem. - * \param[out] prod - Result of the product. - */ - void GPUMatrixVectorProduct(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, - const CConfig* config) const; - - /*! - * \brief Performs first step of the LU_SGS Preconditioner building - * \param[in] vec - CSysVector to be multiplied by the sparse matrix A. - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] config - Definition of the particular problem. - * \param[out] prod - Result of the product. - */ - void GPUFirstSymmetricIteration(ScalarType& vec, ScalarType& prod, CGeometry* geometry, const CConfig* config) const; - - /*! - * \brief Performs second step of the LU_SGS Preconditioner building - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] config - Definition of the particular problem. - * \param[out] prod - Result of the product. - */ - void GPUSecondSymmetricIteration(ScalarType& prod, CGeometry* geometry, const CConfig* config) const; - - /*! - * \brief Performs Gaussian Elimination between diagional blocks of the matrix and the prod vector - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] config - Definition of the particular problem. - * \param[out] prod - Result of the product. - */ - void GPUGaussElimination(ScalarType& prod, CGeometry* geometry, const CConfig* config) const; - - /*! - * \brief Multiply CSysVector by the preconditioner all of which are stored on the device - * \param[in] vec - CSysVector to be multiplied by the preconditioner. - * \param[out] prod - Result of the product A*vec. - */ - void GPUComputeLU_SGSPreconditioner(ScalarType& vec, ScalarType& prod, CGeometry* geometry, - const CConfig* config) const; - /*! * \brief Build the Jacobi preconditioner. */ @@ -949,6 +1346,11 @@ class CSysMatrix { void ComputeILUPreconditioner(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, const CConfig* config) const; + /*! + * \brief Build the LU-SGS preconditioner. + */ + void BuildLU_SGSPreconditioner(); + /*! * \brief Multiply CSysVector by the preconditioner * \param[in] vec - CSysVector to be multiplied by the preconditioner. diff --git a/Common/include/linear_algebra/CSysMatrix.inl b/Common/include/linear_algebra/CSysMatrix.inl index 163e6fb08da0..fd6d6aeff119 100644 --- a/Common/include/linear_algebra/CSysMatrix.inl +++ b/Common/include/linear_algebra/CSysMatrix.inl @@ -34,21 +34,16 @@ template FORCEINLINE ScalarType* CSysMatrix::GetBlock_ILUMatrix(unsigned long block_i, unsigned long block_j) { - /*--- The position of the diagonal block is known which allows halving the search space. ---*/ - const auto end = (block_j < block_i) ? dia_ptr_ilu[block_i] : row_ptr_ilu[block_i + 1]; - for (auto index = (block_j < block_i) ? row_ptr_ilu[block_i] : dia_ptr_ilu[block_i]; index < end; ++index) - if (col_ind_ilu[index] == block_j) return &ILU_matrix[index * nVar * nVar]; + if (block_i == block_j) return &ilu.d[block_i * nVar * nVar]; + const auto* __restrict row_ptr = block_j < block_i ? ilu.row_ptr_l : ilu.row_ptr_u; + const auto* __restrict col_ind = block_j < block_i ? ilu.col_ind_l : ilu.col_ind_u; + auto* __restrict vals = block_j < block_i ? ilu.l : ilu.u; + for (auto k = row_ptr[block_i]; k < row_ptr[block_i + 1]; ++k) { + if (col_ind[k] == block_j) return vals + k * nVar * nVar; + } return nullptr; } -template -FORCEINLINE void CSysMatrix::SetBlock_ILUMatrix(unsigned long block_i, unsigned long block_j, - ScalarType* val_block) { - auto ilu_ij = GetBlock_ILUMatrix(block_i, block_j); - if (!ilu_ij) return; - MatrixCopy(val_block, ilu_ij); -} - namespace { template @@ -142,19 +137,30 @@ FORCEINLINE void CSysMatrix::MatrixMatrixProduct(const ScalarType* m #undef __MATVECPROD_SIGNATURE__ template -FORCEINLINE void CSysMatrix::Gauss_Elimination(unsigned long block_i, ScalarType* rhs) const { +FORCEINLINE void CSysMatrix::GaussElimination(unsigned long block_i, ScalarType* rhs) const { /*--- Copy block, as the algorithm modifies the matrix ---*/ ScalarType block[MAXNVAR * MAXNVAR]; - MatrixCopy(&matrix[dia_ptr[block_i] * nVar * nVar], block); + MatrixCopy(&mat.d[block_i * nVar * nVar], block); + GaussElimination(block, rhs); +} - Gauss_Elimination(block, rhs); +template +FORCEINLINE void CSysMatrix::QuantizedGaussElimination(unsigned long block_i, ScalarType* rhs) const { + ScalarType block[MAXNVAR * MAXNVAR]; + const QuantScaleType* __restrict qs = &q_scale.d[block_i * nVar]; + const QuantType* __restrict qv = &q_blocks.d[block_i * nVar * nVar]; + for (auto r = 0ul; r < nVar; ++r) { + const float row_scale = DecodeQuantScale(qs[r]); + for (auto c = 0ul; c < nVar; ++c) block[r * nVar + c] = static_cast(qv[r * nVar + c] * row_scale); + } + GaussElimination(block, rhs); } template FORCEINLINE void CSysMatrix::InverseDiagonalBlock(unsigned long block_i, ScalarType* invBlock) const { /*--- Copy block, as the algorithm modifies the matrix ---*/ ScalarType block[MAXNVAR * MAXNVAR]; - MatrixCopy(&matrix[dia_ptr[block_i] * nVar * nVar], block); + MatrixCopy(&mat.d[block_i * nVar * nVar], block); MatrixInverse(block, invBlock); } @@ -162,21 +168,38 @@ FORCEINLINE void CSysMatrix::InverseDiagonalBlock(unsigned long bloc template FORCEINLINE const ScalarType* CSysMatrix::InvertDiagonalBlockILUMatrix(unsigned long block_i) { /*--- Copy block, as the algorithm modifies the matrix ---*/ - auto* Uii = &ILU_matrix[dia_ptr_ilu[block_i] * nVar * nVar]; + auto* Uii = &ilu.d[block_i * nVar * nVar]; ScalarType block[MAXNVAR * MAXNVAR]; MatrixCopy(Uii, block); MatrixInverse(block, Uii); return Uii; } +template +FORCEINLINE void CSysMatrix::QuantizedMatVecAdd(const QuantScaleType* __restrict qs, + const QuantType* __restrict qv, + const ScalarType* __restrict vec, + ScalarType* __restrict prod) const { + for (auto r = 0ul; r < nVar; ++r) { + const float row_scale = DecodeQuantScale(qs[r]); + auto sum = ScalarType(0); + for (auto c = 0ul; c < nVar; ++c) sum += qv[r * nVar + c] * vec[c]; + prod[r] += row_scale * sum; + } +} + template FORCEINLINE void CSysMatrix::RowProduct(const CSysVector& vec, unsigned long row_i, ScalarType* prod) const { for (auto iVar = 0ul; iVar < nVar; iVar++) prod[iVar] = 0.0; - for (auto index = row_ptr[row_i]; index < row_ptr[row_i + 1]; index++) { - auto col_j = col_ind[index]; - MatrixVectorProductAdd(&matrix[index * nVar * nEqn], &vec[col_j * nEqn], prod); + for (auto k = mat.row_ptr_l[row_i]; k < mat.row_ptr_l[row_i + 1]; k++) { + MatrixVectorProductAdd(&mat.l[k * nVar * nEqn], &vec[mat.col_ind_l[k] * nEqn], prod); + } + MatrixVectorProductAdd(&mat.d[row_i * nVar * nEqn], &vec[row_i * nEqn], prod); + + for (auto k = mat.row_ptr_u[row_i]; k < mat.row_ptr_u[row_i + 1]; k++) { + MatrixVectorProductAdd(&mat.u[k * nVar * nEqn], &vec[mat.col_ind_u[k] * nEqn], prod); } } @@ -185,11 +208,12 @@ FORCEINLINE void CSysMatrix::UpperProduct(const CSysVector= nPointDomain) - MatrixVectorProductAdd(&matrix[index * nVar * nEqn], &vec[col_j * nEqn], prod); + for (auto index = mat.row_ptr_u[row_i]; index < mat.row_ptr_u[row_i + 1]; index++) { + auto col_j = mat.col_ind_u[index]; + + if (col_j < col_ub || col_j >= nPointDomain) { + MatrixVectorProductAdd(&mat.u[index * nVar * nEqn], &vec[col_j * nEqn], prod); + } } } @@ -198,14 +222,64 @@ FORCEINLINE void CSysMatrix::LowerProduct(const CSysVector= col_lb) MatrixVectorProductAdd(&matrix[index * nVar * nEqn], &vec[col_j * nEqn], prod); + for (auto index = mat.row_ptr_l[row_i]; index < mat.row_ptr_l[row_i + 1]; index++) { + auto col_j = mat.col_ind_l[index]; + if (col_j >= col_lb) { + MatrixVectorProductAdd(&mat.l[index * nVar * nEqn], &vec[col_j * nEqn], prod); + } } } template FORCEINLINE void CSysMatrix::DiagonalProduct(const CSysVector& vec, unsigned long row_i, ScalarType* prod) const { - MatrixVectorProduct(&matrix[dia_ptr[row_i] * nVar * nEqn], &vec[row_i * nEqn], prod); + MatrixVectorProduct(&mat.d[row_i * nVar * nEqn], &vec[row_i * nEqn], prod); +} + +template +FORCEINLINE void CSysMatrix::QuantizedRowProduct(const CSysVector& vec, unsigned long row_i, + ScalarType* prod) const { + for (auto iVar = 0ul; iVar < nVar; iVar++) prod[iVar] = 0.0; + + for (auto k = mat.row_ptr_l[row_i]; k < mat.row_ptr_l[row_i + 1]; k++) { + QuantizedMatVecAdd(&q_scale.l[k * nVar], &q_blocks.l[k * nVar * nEqn], &vec[mat.col_ind_l[k] * nEqn], prod); + } + QuantizedMatVecAdd(&q_scale.d[row_i * nVar], &q_blocks.d[row_i * nVar * nEqn], &vec[row_i * nEqn], prod); + + for (auto k = mat.row_ptr_u[row_i]; k < mat.row_ptr_u[row_i + 1]; k++) { + QuantizedMatVecAdd(&q_scale.u[k * nVar], &q_blocks.u[k * nVar * nEqn], &vec[mat.col_ind_u[k] * nEqn], prod); + } +} + +template +FORCEINLINE void CSysMatrix::QuantizedUpperProduct(const CSysVector& vec, unsigned long row_i, + unsigned long col_ub, ScalarType* prod) const { + for (auto iVar = 0ul; iVar < nVar; iVar++) prod[iVar] = 0.0; + + for (auto index = mat.row_ptr_u[row_i]; index < mat.row_ptr_u[row_i + 1]; index++) { + auto col_j = mat.col_ind_u[index]; + if (col_j < col_ub || col_j >= nPointDomain) { + QuantizedMatVecAdd(&q_scale.u[index * nVar], &q_blocks.u[index * nVar * nEqn], &vec[col_j * nEqn], prod); + } + } +} + +template +FORCEINLINE void CSysMatrix::QuantizedLowerProduct(const CSysVector& vec, unsigned long row_i, + unsigned long col_lb, ScalarType* prod) const { + for (auto iVar = 0ul; iVar < nVar; iVar++) prod[iVar] = 0.0; + + for (auto index = mat.row_ptr_l[row_i]; index < mat.row_ptr_l[row_i + 1]; index++) { + auto col_j = mat.col_ind_l[index]; + if (col_j >= col_lb) { + QuantizedMatVecAdd(&q_scale.l[index * nVar], &q_blocks.l[index * nVar * nEqn], &vec[col_j * nEqn], prod); + } + } +} + +template +FORCEINLINE void CSysMatrix::QuantizedDiagonalProduct(const CSysVector& vec, + unsigned long row_i, ScalarType* prod) const { + for (auto iVar = 0ul; iVar < nVar; iVar++) prod[iVar] = 0.0; + QuantizedMatVecAdd(&q_scale.d[row_i * nVar], &q_blocks.d[row_i * nVar * nEqn], &vec[row_i * nEqn], prod); } diff --git a/Common/include/linear_algebra/CSysSolve.hpp b/Common/include/linear_algebra/CSysSolve.hpp index 91e9b11f4e26..1f9bc851b92e 100644 --- a/Common/include/linear_algebra/CSysSolve.hpp +++ b/Common/include/linear_algebra/CSysSolve.hpp @@ -240,13 +240,50 @@ class CSysSolve { */ void WriteWarning(ScalarType res_calc, ScalarType res_true, ScalarType tol) const; + /*! + * \brief Moves the linear system to the device, if the GPU path is in use. + * \note This and DownloadSolution are the only places where b and x cross the bus. The + * work vectors of the solvers never do, they are allocated on the device and read back + * only through the reductions. + */ + void UploadSystem(bool useCuda) const { +#ifdef SU2_ENABLE_CUDA_KERNELS + if constexpr (su2_gpu_capable_v) { + if (!useCuda) return; + BEGIN_SU2_DEVICE_REGION { + LinSysRes_ptr->HtDTransfer(); + LinSysSol_ptr->HtDTransfer(); + VecExpr::SetUseDeviceExpressions(true); + } + END_SU2_DEVICE_REGION + } +#endif + } + + /*! + * \brief Brings the solution back from the device and returns to host evaluation. + */ + void DownloadSolution(bool useCuda) const { +#ifdef SU2_ENABLE_CUDA_KERNELS + if constexpr (su2_gpu_capable_v) { + if (!useCuda) return; + BEGIN_SU2_DEVICE_REGION { + LinSysSol_ptr->DtHTransfer(); + VecExpr::SetUseDeviceExpressions(false); + } + END_SU2_DEVICE_REGION + } +#endif + } + /*! * \brief Used by Solve for compatibility between passive and active CSysVector. * \param[in] LinSysRes - Linear system residual * \param[in,out] LinSysSol - Linear system solution + * \param[in] useCuda - Whether to move the system to the device for the solve. */ template - void HandleTemporariesIn(const CSysVector& LinSysRes, CSysVector& LinSysSol) { + void HandleTemporariesIn(const CSysVector& LinSysRes, CSysVector& LinSysSol, bool useCuda) { SU2_ZONE_SCOPED if constexpr (std::is_same_v) { /*--- Same type specialization, temporary variables are not required. ---*/ @@ -255,6 +292,7 @@ class CSysSolve { LinSysSol_ptr = &LinSysSol; } END_SU2_OMP_SAFE_GLOBAL_ACCESS + UploadSystem(useCuda); } else { /*--- Copy data, the solution is also copied as it serves as initial condition. ---*/ LinSysRes_tmp.PassiveCopy(LinSysRes); @@ -266,16 +304,19 @@ class CSysSolve { LinSysSol_ptr = &LinSysSol_tmp; } END_SU2_OMP_SAFE_GLOBAL_ACCESS + UploadSystem(useCuda); } } /*! * \brief Used by Solve for compatibility between passive and active CSysVector. * \param[out] LinSysSol - Linear system solution + * \param[in] useCuda - Whether the system was solved on the device. */ template - void HandleTemporariesOut(CSysVector& LinSysSol) { + void HandleTemporariesOut(CSysVector& LinSysSol, bool useCuda) { SU2_ZONE_SCOPED + DownloadSolution(useCuda); if constexpr (std::is_same_v) { /*--- Same type specialization, temporary variables are not required. ---*/ BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS { @@ -432,9 +473,9 @@ class CSysSolve { template > = 0> unsigned long Solve_b(MatrixType& Jacobian, const CSysVector& LinSysRes, CSysVector& LinSysSol, CGeometry* geometry, const CConfig* config, bool directCall = true) { - HandleTemporariesIn(LinSysRes, LinSysSol); + HandleTemporariesIn(LinSysRes, LinSysSol, false); auto iter = Solve_b(Jacobian, *LinSysRes_ptr, *LinSysSol_ptr, geometry, config, directCall); - HandleTemporariesOut(LinSysSol); + HandleTemporariesOut(LinSysSol, false); return iter; } @@ -442,13 +483,13 @@ class CSysSolve { * \brief Get the number of iterations. * \return The number of iterations done by Solve or Solve_b */ - inline unsigned long GetIterations(void) const { return Iterations; } + inline unsigned long GetIterations() const { return Iterations; } /*! * \brief Get the final residual. * \return The residual at the end of Solve or Solve_b */ - inline ScalarType GetResidual(void) const { return Residual; } + inline ScalarType GetResidual() const { return Residual; } /*! * \brief Set the type of the tolerance for stoping the linear solvers (RELATIVE or ABSOLUTE). diff --git a/Common/include/linear_algebra/CSysVector.hpp b/Common/include/linear_algebra/CSysVector.hpp index 1498b549bbb8..28c750049580 100644 --- a/Common/include/linear_algebra/CSysVector.hpp +++ b/Common/include/linear_algebra/CSysVector.hpp @@ -31,12 +31,39 @@ #include #include +#include "../code_config.hpp" #include "../parallelization/mpi_structure.hpp" #include "../parallelization/omp_structure.hpp" #include "../parallelization/vectorization.hpp" #include "vector_expressions.hpp" #include "../../include/CConfig.hpp" +#ifdef __CUDACC__ +#include "GPUComms.cuh" +#endif + +template +class CSysVector; + +/*! + * \brief True for the plain floating-point scalar types the GPU vector kernels + * (CSysVectorGPU.cu) are instantiated for, in builds where those kernels + * exist at all. AD active types are never dispatched to the device: the + * tape/expression machinery those types pull in is not compatible with + * nvcc's device-code compilation, and device-resident autodiff is not + * supported by this GPU path. + * \note In an AD build this is false even for su2mixedfloat, because the .cu + * translation units are not linked into the AD libraries (see + * SU2_ENABLE_CUDA_KERNELS in code_config.hpp). + */ +#ifdef SU2_ENABLE_CUDA_KERNELS +template +inline constexpr bool su2_gpu_capable_v = std::is_floating_point_v; +#else +template +inline constexpr bool su2_gpu_capable_v = false; +#endif + /*! * \brief OpenMP worksharing construct used in CSysVector for loops. * \note The loop will only run in parallel if methods are called from a @@ -59,6 +86,106 @@ #define END_CSYSVEC_PARFOR #endif +/*! + * \brief Brackets device work so that it is issued by a single thread with the whole team + * synchronized before and after. + * \note The GPU is one shared resource and the device path does not use OpenMP worksharing. + * Issuing from one thread keeps kernel launches ordered on the default stream and, above + * all, stops part of a team from entering a worksharing construct that the rest skipped. + * Correctness relies on all threads reaching the same vector operations in the same order, + * which is the assumption the "nowait" clause on CSYSVEC_PARFOR already makes. These + * regions must not be nested; they are used by the operations of this class and by the + * matrix-vector product and preconditioner wrappers, and by nothing above those. + */ +#define SU2_DEVICE_REGION(...) SU2_OMP_SAFE_GLOBAL_ACCESS(__VA_ARGS__) +#define BEGIN_SU2_DEVICE_REGION BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS +#define END_SU2_DEVICE_REGION END_SU2_OMP_SAFE_GLOBAL_ACCESS + +namespace VecExpr { + +enum class DeviceAssignOp { Assign, Add, Subtract, Multiply, Divide }; + +/*! + * \brief Whether vector expressions are currently evaluated on the device. + * \note Defined in CSysVectorGPU.cu. This is a plain global, not per thread: every thread + * of a team has to agree on it or they would split over the worksharing constructs below. + * It is switched by CSysSolve at the same boundary that uploads and downloads the vectors + * (HandleTemporariesIn/Out), and nowhere else, so it does not change while a solve runs. + */ +#ifdef SU2_ENABLE_CUDA_KERNELS +bool UseDeviceExpressions(); +void SetUseDeviceExpressions(bool use); +#else +inline bool UseDeviceExpressions() { return false; } +inline void SetUseDeviceExpressions(bool) {} +#endif + +/*! + * \brief How a CSysVector is captured inside an expression: a bare pointer to whichever + * storage the expression is going to be evaluated from. + * \note Capturing by value (rather than a reference to the vector) is what makes an + * arbitrary expression tree trivially copyable, and therefore passable by value to the + * assignment kernel. The choice of storage is fixed when the expression is built, which is + * sound because there is no fallback: while UseDeviceExpressions() holds, every expression + * is evaluated by a kernel. + */ +template +class CVectorView : public CVecExpr, Scalar> { + private: + const Scalar* data = nullptr; + + public: + static constexpr bool StoreAsRef = false; + + CVectorView(const CSysVector& vector); + + SU2_CUDA_HOST_DEVICE FORCEINLINE const Scalar& operator[](size_t i) const { return data[i]; } +}; + +template +struct store_type> { + using type = CVectorView; +}; + +template +struct store_type> { + using type = CVectorView; +}; + +template +void AssignDeviceExpression(Scalar* data, unsigned long size, const CVecExpr& expr); + +#ifdef __CUDACC__ +template +__global__ void DeviceAssignKernel(Scalar* data, unsigned long size, T expr) { + const unsigned long i = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (i >= size) return; + + if constexpr (Op == DeviceAssignOp::Assign) { + data[i] = expr[i]; + } else if constexpr (Op == DeviceAssignOp::Add) { + data[i] += expr[i]; + } else if constexpr (Op == DeviceAssignOp::Subtract) { + data[i] -= expr[i]; + } else if constexpr (Op == DeviceAssignOp::Multiply) { + data[i] *= expr[i]; + } else { + data[i] /= expr[i]; + } +} + +template +inline void AssignDeviceExpression(Scalar* data, unsigned long size, const CVecExpr& expr) { + if (size == 0) return; + constexpr unsigned block_size = 256; + const auto grid_size = static_cast((size + block_size - 1) / block_size); + DeviceAssignKernel<<>>(data, size, expr.derived()); + gpuErrChk(cudaPeekAtLastError()); +} +#endif + +} // namespace VecExpr + /*! * \class CSysVector * \ingroup SpLinSys @@ -110,6 +237,48 @@ class CSysVector : public VecExpr::CVecExpr, ScalarType> } } + /*! + * \brief Evaluates an expression into the device storage of this vector. + * \note The kernel has to be instantiated for the expression type in CSysVectorGPU.cu, + * a shape that is not in that list is an undefined symbol at link time. + */ + template + CSysVector& AssignDevice(const VecExpr::CVecExpr& expr) { +#ifdef SU2_ENABLE_CUDA_KERNELS + if constexpr (su2_gpu_capable_v) { + BEGIN_SU2_DEVICE_REGION { + VecExpr::store_t stored_expr(expr.derived()); + VecExpr::AssignDeviceExpression(d_vec_val, nElm, stored_expr); + } + END_SU2_DEVICE_REGION + } +#endif + return *this; + } + + template + CSysVector& AssignDevice(ScalarType val) { +#ifdef SU2_ENABLE_CUDA_KERNELS + if constexpr (su2_gpu_capable_v) { + BEGIN_SU2_DEVICE_REGION + VecExpr::AssignDeviceExpression(d_vec_val, nElm, VecExpr::Bcast(val)); + END_SU2_DEVICE_REGION + } +#endif + return *this; + } + + /*! + * \brief GPU helper for `dot`. + */ + ScalarType dotGPU(const CSysVector& other) const; + + /*! + * \brief GPU helper for multiDot. + */ + static su2matrix multiDotGPU(const std::vector>& V, size_t i0, size_t n, + const std::vector>& W, size_t m); + public: static constexpr bool StoreAsRef = true; /*! \brief Required by CVecExpr. */ @@ -235,15 +404,15 @@ class CSysVector : public VecExpr::CVecExpr, ScalarType> void DtHTransfer(bool trigger = true) const; /*! - * \brief Sets all the elements of the GPU vector to a certain value - * \param[in] trigger - boolean value that decides whether to conduct the transfer or not. True by default. + * \brief return device pointer that points to the CSysVector values in GPU memory */ - void GPUSetVal(ScalarType val, bool trigger = true) const; + inline ScalarType* GetDevicePointer() const { return d_vec_val; } /*! - * \brief return device pointer that points to the CSysVector values in GPU memory + * \brief return host pointer that points to the CSysVector values, counterpart of + * GetDevicePointer */ - inline ScalarType* GetDevicePointer() const { return d_vec_val; } + inline const ScalarType* GetHostPointer() const { return vec_val; } /*! * \brief return the number of local elements in the CSysVector @@ -301,6 +470,11 @@ class CSysVector : public VecExpr::CVecExpr, ScalarType> * \param[in] other - Another vector. */ CSysVector& operator=(const CSysVector& other) { +#ifdef SU2_ENABLE_CUDA_KERNELS + if constexpr (su2_gpu_capable_v) { + if (VecExpr::UseDeviceExpressions()) return AssignDevice(other); + } +#endif CSYSVEC_PARFOR for (auto i = 0ul; i < nElm; ++i) vec_val[i] = other.vec_val[i]; END_CSYSVEC_PARFOR @@ -311,25 +485,31 @@ class CSysVector : public VecExpr::CVecExpr, ScalarType> * \brief Compound assignement operations with scalars and expressions. * \param[in] val/expr - Scalar value or expression. */ -#define MAKE_COMPOUND(OP) \ - CSysVector& operator OP(ScalarType val) { \ - CSYSVEC_PARFOR \ - for (auto i = 0ul; i < nElm; ++i) vec_val[i] OP val; \ - END_CSYSVEC_PARFOR \ - return *this; \ - } \ - template \ - CSysVector& operator OP(const VecExpr::CVecExpr& expr) { \ - CSYSVEC_PARFOR \ - for (auto i = 0ul; i < nElm; ++i) vec_val[i] OP expr.derived()[i]; \ - END_CSYSVEC_PARFOR \ - return *this; \ +#define MAKE_COMPOUND(OP, ASSIGN_OP) \ + CSysVector& operator OP(ScalarType val) { \ + if constexpr (su2_gpu_capable_v) { \ + if (VecExpr::UseDeviceExpressions()) return AssignDevice(val); \ + } \ + CSYSVEC_PARFOR \ + for (auto i = 0ul; i < nElm; ++i) vec_val[i] OP val; \ + END_CSYSVEC_PARFOR \ + return *this; \ + } \ + template \ + CSysVector& operator OP(const VecExpr::CVecExpr& expr) { \ + if constexpr (su2_gpu_capable_v) { \ + if (VecExpr::UseDeviceExpressions()) return AssignDevice(expr); \ + } \ + CSYSVEC_PARFOR \ + for (auto i = 0ul; i < nElm; ++i) vec_val[i] OP expr.derived()[i]; \ + END_CSYSVEC_PARFOR \ + return *this; \ } - MAKE_COMPOUND(=) - MAKE_COMPOUND(+=) - MAKE_COMPOUND(-=) - MAKE_COMPOUND(*=) - MAKE_COMPOUND(/=) + MAKE_COMPOUND(=, VecExpr::DeviceAssignOp::Assign) + MAKE_COMPOUND(+=, VecExpr::DeviceAssignOp::Add) + MAKE_COMPOUND(-=, VecExpr::DeviceAssignOp::Subtract) + MAKE_COMPOUND(*=, VecExpr::DeviceAssignOp::Multiply) + MAKE_COMPOUND(/=, VecExpr::DeviceAssignOp::Divide) #undef MAKE_COMPOUND /*! @@ -344,6 +524,22 @@ class CSysVector : public VecExpr::CVecExpr, ScalarType> */ template ScalarType dot(const VecExpr::CVecExpr& expr) const { +#ifdef SU2_ENABLE_CUDA_KERNELS + if constexpr (su2_gpu_capable_v) { + using DeviceExpr = std::remove_cv_t>; + static_assert(std::is_same_v, + "On the device the dot product needs a real device pointer (dotGPU takes a " + "materialized vector, not an expression template), so it only takes vectors. " + "Assign the expression to a vector first."); + if (VecExpr::UseDeviceExpressions()) { + /*--- dotGPU reduces over MPI, which has to happen once for the team, so the result + * is published through the same scratch slot the host reduction below uses. ---*/ + SU2_DEVICE_REGION(dot_scratch[0] = dotGPU(expr.derived());) + return dot_scratch[0]; + } + } +#endif + /*--- All threads get the same "view" of the vectors. ---*/ SU2_OMP_BARRIER @@ -502,5 +698,13 @@ class CSysVector : public VecExpr::CVecExpr, ScalarType> } }; +namespace VecExpr { + +template +CVectorView::CVectorView(const CSysVector& vector) + : data(UseDeviceExpressions() ? vector.GetDevicePointer() : vector.GetHostPointer()) {} + +} // namespace VecExpr + #undef CSYSVEC_PARFOR #undef END_CSYSVEC_PARFOR diff --git a/Common/include/linear_algebra/GPUComms.cuh b/Common/include/linear_algebra/GPUComms.cuh index 138543818771..39268c10e5a3 100644 --- a/Common/include/linear_algebra/GPUComms.cuh +++ b/Common/include/linear_algebra/GPUComms.cuh @@ -9,7 +9,7 @@ * The SU2 Project is maintained by the SU2 Foundation * (http://su2foundation.org) * -* Copyright 2012-2024, SU2 Contributors (cf. AUTHORS.md) +* Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) * * SU2 is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -25,6 +25,11 @@ * License along with SU2. If not, see . */ +#pragma once + +#ifndef SU2_COMMON_LINEAR_ALGEBRA_GPUCOMMS_CUH +#define SU2_COMMON_LINEAR_ALGEBRA_GPUCOMMS_CUH + #include #include @@ -51,3 +56,5 @@ inline void gpuAssert(cudaError_t code, const char *file, int line, bool abort=t } #define gpuErrChk(ans) { gpuAssert((ans), __FILE__, __LINE__); } + +#endif // SU2_COMMON_LINEAR_ALGEBRA_GPUCOMMS_CUH diff --git a/Common/include/linear_algebra/vector_expressions.hpp b/Common/include/linear_algebra/vector_expressions.hpp index a0d0ce289013..d59dfdf51ae8 100644 --- a/Common/include/linear_algebra/vector_expressions.hpp +++ b/Common/include/linear_algebra/vector_expressions.hpp @@ -28,6 +28,7 @@ #pragma once #include "../basic_types/datatype_structure.hpp" +#include "../code_config.hpp" #include #include #include @@ -59,7 +60,7 @@ class CVecExpr { /*! * \brief Cast the expression to Derived, usually to allow evaluation via operator[]. */ - FORCEINLINE const Derived& derived() const { return static_cast(*this); } + SU2_CUDA_HOST_DEVICE FORCEINLINE const Derived& derived() const { return static_cast(*this); } // Allowed from C++14, allows nested expression propagation without // manually calling derived() on the expression being evaluated. @@ -76,8 +77,8 @@ class Bcast : public CVecExpr, Scalar> { public: static constexpr bool StoreAsRef = false; - FORCEINLINE Bcast(const Scalar& x_) : x(x_) {} - FORCEINLINE const Scalar& operator[](size_t) const { return x; } + SU2_CUDA_HOST_DEVICE FORCEINLINE Bcast(const Scalar& x_) : x(x_) {} + SU2_CUDA_HOST_DEVICE FORCEINLINE const Scalar& operator[](size_t) const { return x; } }; /*! @@ -101,7 +102,11 @@ struct add_lref_if { using type = remove_reference_t&; }; template -using store_t = typename add_lref_if::type; +struct store_type { + using type = typename add_lref_if::type; +}; +template +using store_t = typename store_type::type; /*--- Namespace from which the math function implementations come. ---*/ @@ -120,19 +125,19 @@ namespace math = ::std; /*--- Macro to create expression classes (EXPR) and overloads (FUN) for unary * functions, based on their coefficient-wise implementation (IMPL). ---*/ -#define MAKE_UNARY_FUN(FUN, EXPR, IMPL) \ - /*!--- Expression class. ---*/ \ - template \ - class EXPR : public CVecExpr, Scalar> { \ - store_t u; \ - \ - public: \ - static constexpr bool StoreAsRef = false; \ - FORCEINLINE EXPR(const U& u_) : u(u_) {} \ - FORCEINLINE auto operator[](size_t i) const RETURNS(IMPL(u[i])) \ - }; \ - /*!--- Function overload, returns an expression object. ---*/ \ - template \ +#define MAKE_UNARY_FUN(FUN, EXPR, IMPL) \ + /*!--- Expression class. ---*/ \ + template \ + class EXPR : public CVecExpr, Scalar> { \ + store_t u; \ + \ + public: \ + static constexpr bool StoreAsRef = false; \ + FORCEINLINE EXPR(const U& u_) : u(u_) {} \ + SU2_CUDA_HOST_DEVICE FORCEINLINE auto operator[](size_t i) const RETURNS(IMPL(u[i])) \ + }; \ + /*!--- Function overload, returns an expression object. ---*/ \ + template \ FORCEINLINE auto FUN(const CVecExpr& u) RETURNS(EXPR(u.derived())) #define sign_impl(x) Scalar(1 - 2 * (x < 0)) @@ -158,7 +163,7 @@ MAKE_UNARY_FUN(sign, sign_, sign_impl) public: \ static constexpr bool StoreAsRef = false; \ FORCEINLINE EXPR(const U& u_, const V& v_) : u(u_), v(v_) {} \ - FORCEINLINE auto operator[](size_t i) const RETURNS(IMPL(u[i], v[i])) \ + SU2_CUDA_HOST_DEVICE FORCEINLINE auto operator[](size_t i) const RETURNS(IMPL(u[i], v[i])) \ }; \ /*!--- Vector with vector function overload. ---*/ \ template \ diff --git a/Common/include/option_structure.hpp b/Common/include/option_structure.hpp index 73266c7dacdb..03b00b23e3d8 100644 --- a/Common/include/option_structure.hpp +++ b/Common/include/option_structure.hpp @@ -77,7 +77,7 @@ const unsigned int MAX_PARAMETERS = 10; /*!< \brief Maximum number of para const unsigned int MAX_NUMBER_PERIODIC = 10; /*!< \brief Maximum number of periodic boundary conditions. */ const unsigned int MAX_STRING_SIZE = 400; /*!< \brief Maximum size of a generic string. */ const unsigned int MAX_NUMBER_FFD = 15; /*!< \brief Maximum number of FFDBoxes for the FFD. */ -enum: unsigned int{MAX_SOLS = 13}; /*!< \brief Maximum number of solutions at the same time (dimension of solution container array). */ +enum: unsigned int{MAX_SOLS = 14}; /*!< \brief Maximum number of solutions at the same time (dimension of solution container array). */ const unsigned int MAX_TERMS = 7; /*!< \brief Maximum number of terms in the numerical equations (dimension of solver container array). */ const unsigned int MAX_ZONES = 3; /*!< \brief Maximum number of zones. */ const unsigned int MAX_FE_KINDS = 7; /*!< \brief Maximum number of Finite Elements. */ @@ -195,6 +195,74 @@ inline unsigned short nPointsOfElementType(unsigned short elementType) { } const int CGNS_STRING_SIZE = 33; /*!< \brief Length of strings used in the CGNS format. */ + +/*--- Layout of the header of the native SU2 binary solution/restart format, shared by + CSU2BinaryFileWriter and the routines that read those files so they cannot drift + apart. The header is SU2_RESTART_HEADER_SIZE ints: a magic number, the number of + variables, the number of points, the size in bytes of the floating point data that + follows, and one spare. + + The 4th and 5th ints used to be the number of ints and of doubles of a metadata + trailer (1 and 5, later 1 and 8) that the writer appended after the data. That + trailer is no longer written (metadata goes to a separate ASCII file) and both + ints have been 0 since, but old files in circulation still have 1 and 5 or 8 + there, which is why 1 is accepted below as meaning double precision. ---*/ +const int SU2_RESTART_MAGIC_NUMBER = 535532; /*!< \brief Hex representation of "SU2". */ +const int SU2_RESTART_HEADER_SIZE = 5; /*!< \brief Number of ints in the header. */ +const int SU2_RESTART_PRECISION_IDX = 3; /*!< \brief Position of the precision field. */ +const int SU2_RESTART_METADATA_IDX = 4; /*!< \brief Position of the legacy metadata count. */ +const int SU2_RESTART_MAX_METADATA = 8; /*!< \brief Most metadata doubles a trailer ever had. */ + +/*! + * \brief Size in bytes of the floating point data of a native SU2 binary solution file. + * \param[in] precisionField - The SU2_RESTART_PRECISION_IDX entry of the file header. + * \return 8 for double precision, 4 for single precision. + * \note Files written before the field had this meaning have a 0 or a 1 there (see above) + * and were always double precision. + */ +inline int GetSU2BinaryScalarSize(int precisionField) { + if (precisionField == 0 || precisionField == 1) return static_cast(sizeof(double)); + if (precisionField != static_cast(sizeof(double)) && precisionField != static_cast(sizeof(float))) { + SU2_MPI::Error("Invalid floating point precision in the header of a binary SU2 solution file.", CURRENT_FUNCTION); + } + return precisionField; +} + +/*! + * \brief Number of metadata scalars in the trailer of a native SU2 binary solution file. + * \param[in] precisionField - The SU2_RESTART_PRECISION_IDX entry of the file header. + * \param[in] metadataField - The SU2_RESTART_METADATA_IDX entry of the file header. + * \return Number of scalars of the trailer, preceded by one int (the iteration number), + * or 0 for the files that do not have one. + * \note Only files that still use the two ints as trailer counts have a trailer, and in + * those the precision field is the number of trailer ints, which was always 1 (see above). + */ +inline int GetSU2BinaryMetadataSize(int precisionField, int metadataField) { + if (precisionField != 1) return 0; + return std::min(metadataField, SU2_RESTART_MAX_METADATA); +} + +/*! + * \brief Convert floating point data read from a native SU2 binary solution file, which + * may have been written by a build of different precision, to the precision of this build. + * \param[in] buffer - Raw data as read from the file, of size count*scalarSize bytes. + * \param[in] scalarSize - Size in bytes of the scalars in the file, see GetSU2BinaryScalarSize. + * \param[in] count - Number of scalars. + * \param[out] data - Converted data, must not overlap with buffer. + */ +inline void SU2BinaryDataToPassive(const void* buffer, int scalarSize, unsigned long count, passivedouble* data) { + if (scalarSize == static_cast(sizeof(float))) { + const auto* src = static_cast(buffer); + for (unsigned long i = 0; i < count; ++i) data[i] = src[i]; + } else { + const auto* src = static_cast(buffer); + for (unsigned long i = 0; i < count; ++i) data[i] = src[i]; + } +} + +const int SU2_BINARY_STRING_SIZE = 65; /*!< \brief Length of strings (e.g. marker names) used in the native + SU2 binary mesh format. Shared by CSU2BinaryMeshReaderBase + and CSU2MeshBinaryFileWriter so they cannot drift apart. */ const int SU2_CONN_SIZE = 10; /*!< \brief Size of the connectivity array that is allocated for each element that we read from a mesh file in the format [[globalID vtkType n0 n1 n2 n3 n4 n5 n6 n7 n8]. */ const int SU2_CONN_SKIP = 2; /*!< \brief Offset to skip the globalID and VTK type at the start of the element connectivity list for each CGNS element. */ @@ -261,6 +329,7 @@ enum class MAIN_SOLVER { FEM_RANS, /*!< \brief Definition of the finite element Reynolds-averaged Navier-Stokes' (RANS) solver. */ FEM_LES, /*!< \brief Definition of the finite element Large Eddy Simulation Navier-Stokes' (LES) solver. */ MULTIPHYSICS, + POISSON_EQUATION, /*!< \brief Definition of the Poisson equation solver. */ NEMO_EULER, /*!< \brief Definition of the NEMO Euler solver. */ NEMO_NAVIER_STOKES, /*!< \brief Definition of the NEMO NS solver. */ }; @@ -336,6 +405,31 @@ static const MapType MatComp_Map = { MakePair("NEARLY_INCOMPRESSIBLE", STRUCT_COMPRESS::NEARLY_INCOMP) }; +/*! + * \brief Type of incompressible solver + */ +enum class INCOMP_SYSTEM { + DENSITY_BASED, /*!< \brief Density-based. */ + PRESSURE_BASED, /*!< \brief Pressure-based. */ +}; +static const MapType Incomp_Map = { + MakePair("DENSITY_BASED", INCOMP_SYSTEM::DENSITY_BASED) + MakePair("PRESSURE_BASED", INCOMP_SYSTEM::PRESSURE_BASED) +}; + +/*! + * \brief Type of iteration + */ +enum class PBITER { + SIMPLE, /*!< \brief SIMPLE algorithm. */ + SIMPLEC, /*!< \brief SIMPLEC algorithm. */ +}; + +static const MapType PBIter_Map = { + MakePair("SIMPLE", PBITER::SIMPLE) + MakePair("SIMPLEC", PBITER::SIMPLEC) +}; + /*! * \brief Types of interpolators */ @@ -476,6 +570,7 @@ enum RUNTIME_TYPE { RUNTIME_ADJRAD_SYS = 24, /*!< \brief One-physics case, the code is solving the adjoint radiation model. */ RUNTIME_SPECIES_SYS = 25, /*!< \brief One-physics case, the code is solving the species model. */ RUNTIME_ADJSPECIES_SYS = 26,/*!< \brief One-physics case, the code is solving the adjoint species model. */ + RUNTIME_POISSON_SYS = 27, /*!< \brief One-physics case, the code is solving the poisson equation. */ }; enum SOLVER_TYPE : const int { @@ -494,6 +589,7 @@ enum RUNTIME_TYPE { ADJSPECIES_SOL=12, /*!< \brief Position of the adjoint of the species solver. */ FEA_SOL=0, /*!< \brief Position of the Finite Element flow solution in the solver container array. */ ADJFEA_SOL=1, /*!< \brief Position of the continuous adjoint Finite Element flow solution in the solver container array. */ + POISSON_SOL=13, /*!< \brief Position of the poisson solution in the solver container array */ TEMPLATE_SOL=0, /*!< \brief Position of the template solution. */ }; @@ -825,7 +921,8 @@ enum class CENTERED { LAX, /*!< \brief Lax-Friedrich centered numerical method. */ JST_MAT, /*!< \brief JST with matrix dissipation. */ JST_KE, /*!< \brief Kinetic Energy preserving Jameson-Smith-Turkel centered numerical method. */ - LD2 /*!< \brief Low-Dissipation Low-Dispersion (LD2) centered scheme. */ + LD2, /*!< \brief Low-Dissipation Low-Dispersion (LD2) centered scheme. */ + CDS /*!< \brief Central Difference Scheme used for pressure based solver. */ }; static const MapType Centered_Map = { MakePair("NONE", CENTERED::NONE) @@ -834,6 +931,7 @@ static const MapType Centered_Map = { MakePair("JST_MAT", CENTERED::JST_MAT) MakePair("LAX-FRIEDRICH", CENTERED::LAX) MakePair("LD2", CENTERED::LD2) + MakePair("CDS", CENTERED::CDS) }; @@ -860,7 +958,8 @@ enum class UPWIND { AUSMPLUSUP, /*!< \brief AUSM+ -up numerical method (All Speed) */ AUSMPLUSUP2, /*!< \brief AUSM+ -up2 numerical method (All Speed) */ AUSMPLUSM, /*!< \breif AUSM+M numerical method. (NEMO Only)*/ - BOUNDED_SCALAR /*!< \brief Scalar advection numerical method. */ + BOUNDED_SCALAR, /*!< \brief Scalar advection numerical method. */ + UDS /*!< \brief Upwind Difference Scheme used for pressure based solver. */ }; static const MapType Upwind_Map = { MakePair("NONE", UPWIND::NONE) @@ -882,6 +981,7 @@ static const MapType Upwind_Map = { MakePair("SLAU2", UPWIND::SLAU2) MakePair("FDS", UPWIND::FDS) MakePair("LAX-FRIEDRICH", UPWIND::LAX_FRIEDRICH) + MakePair("UDS", UPWIND::UDS) }; /*! @@ -1117,9 +1217,18 @@ struct CMGOptions { std::vector MG_PreSmooth; /*!< \brief Multigrid pre-smoothing iterations per level. */ std::vector MG_PostSmooth; /*!< \brief Multigrid post-smoothing iterations per level. */ std::vector MG_CorrecSmooth; /*!< \brief Multigrid Jacobi correction-smoothing per level. */ + std::vector MG_CflScaling; /*!< \brief Per-level CFL scaling factors relative to the previous (finer) level. Entry [i] scales level i+1 from level i. Size = nMGLevels. */ bool MG_Smooth_EarlyExit{false}; /*!< \brief Enable early exit for MG smoothing iterations. */ bool MG_Smooth_Output{false}; /*!< \brief Output compact per-cycle smoothing summary. */ + su2double MG_Smooth_StagnationTol{0.0}; /*!< \brief Stagnation early exit: stop if current_rms >= prev_rms * tol. 0 = disabled. */ bool MG_Implicit_Lines{false}; /*!< \brief Enable implicit-lines agglomeration from walls. */ + unsigned long MG_Implicit_Lines_MaxLength{20}; /*!< \brief Maximum nodes on a wall-normal implicit line (including wall seed). */ + unsigned long MG_Startup_Iter{100}; /*!< \brief Iterations per mesh during FMG startup, and the length of each level's CFL ramp. 0 = no iteration budget. */ + su2double MG_Startup_Convergence{-2.0}; /*!< \brief FMG: orders of magnitude (log10) that CONV_FIELD must drop on the + active level before promoting to the next finer one. Negative is a + drop, as for CONV_RESIDUAL_MINVAL. */ + su2double MG_Startup_Stagnation{0.99}; /*!< \brief FMG: promote when the residual ratio between successive iterations exceeds this. 0 = disabled. */ + unsigned long MG_Startup_Stagnation_Iter{5}; /*!< \brief FMG: consecutive stalled iterations required before promoting. 0 = disabled. */ }; /*! @@ -1441,12 +1550,30 @@ static const MapType Flamelet_Init_Map = { MakePair("SPARK", FLAMELET_INIT_TYPE::SPARK) }; +/*! + * \brief Selects the source of wall/inlet enthalpy boundary conditions for the flamelet solver. + * SPECIES_MARKERS (default): inlet H is taken directly from MARKER_INLET_SPECIES; wall enthalpy BC + * is obtained from MARKER_WALL_SPECIES. + * FLOW_MARKERS: inlet H is derived from the MARKER_INLET temperature via a Newton iteration on the + * LUT (reverse lookup using Z,T) from MARKER_ISOTHERMAL or MARKER_HEATFLUX. + */ +enum class FLAMELET_ENTHALPY_BC { + FLOW_MARKERS, /*!< \brief Derive inlet H from MARKER_INLET T (LUT Newton); walls from MARKER_ISOTHERMAL/MARKER_HEATFLUX. */ + SPECIES_MARKERS, /*!< \brief Take inlet H directly from MARKER_INLET_SPECIES; walls from MARKER_WALL_SPECIES (default). */ +}; + +static const MapType Flamelet_Enthalpy_BC_Map = { + MakePair("FLOW_MARKERS", FLAMELET_ENTHALPY_BC::FLOW_MARKERS) + MakePair("SPECIES_MARKERS", FLAMELET_ENTHALPY_BC::SPECIES_MARKERS) +}; + /*! * \brief Structure containing parsed options for flamelet fluid model. */ struct FluidFlamelet_ParsedOptions { ///TODO: Add python wrapper initialization option FLAMELET_INIT_TYPE ignition_method = FLAMELET_INIT_TYPE::NONE; /*!< \brief Method for solution ignition for flamelet problems. */ + FLAMELET_ENTHALPY_BC enthalpy_bc = FLAMELET_ENTHALPY_BC::SPECIES_MARKERS; /*!< \brief Source of enthalpy BCs: species markers (default, backward-compatible) or flow markers. */ unsigned short n_scalars = 0; /*!< \brief Number of transported scalars for flamelet LUT approach. */ unsigned short n_lookups = 0; /*!< \brief Number of lookup variables, for visualization only. */ unsigned short n_table_sources = 0; /*!< \brief Number of transported scalar source terms for LUT. */ @@ -1467,6 +1594,7 @@ struct FluidFlamelet_ParsedOptions { su2double* spark_reaction_rates; /*!< \brief Source terms for flamelet spark ignition option. */ unsigned short nspark; /*!< \brief Number of source terms for spark initialization. */ bool preferential_diffusion = false; /*!< \brief Preferential diffusion physics for flamelet solver.*/ + bool thickenedflame_correction{true}; /*!< \brief Thickened flame correction. */ su2double Flame_T_ignition = 5000; /*!< \brief Ignition temperature for the flame, used for initialization. */ }; @@ -2123,6 +2251,9 @@ enum ENUM_OBJECTIVE { TOPOL_DISCRETENESS = 63, /*!< \brief Measure of the discreteness of the current topology. */ TOPOL_COMPLIANCE = 64, /*!< \brief Measure of the discreteness of the current topology. */ STRESS_PENALTY = 65, /*!< \brief Penalty function of VM stresses above a maximum value. */ + ENTROPY_GENERATION = 80, /*!< \brief Entropy generation turbomachinery objective function. */ + TOTAL_PRESSURE_LOSS = 81, /*!< \brief Total pressure loss turbomachinery objective function. */ + KINETIC_ENERGY_LOSS = 82 /*!< \breif Kinetic energy loss coefficient turbomachinery objective function. */ }; static const MapType Objective_Map = { MakePair("DRAG", DRAG_COEFFICIENT) @@ -2165,24 +2296,32 @@ static const MapType Objective_Map = { MakePair("TOPOL_DISCRETENESS", TOPOL_DISCRETENESS) MakePair("TOPOL_COMPLIANCE", TOPOL_COMPLIANCE) MakePair("STRESS_PENALTY", STRESS_PENALTY) + MakePair("ENTROPY_GENERATION", ENTROPY_GENERATION) + MakePair("TOTAL_PRESSURE_LOSS", TOTAL_PRESSURE_LOSS) + MakePair("KINETIC_ENERGY_LOSS", KINETIC_ENERGY_LOSS) }; /*! - * \brief Types of input file formats + * \brief Types of grid file formats */ -enum ENUM_INPUT { - SU2 = 1, /*!< \brief SU2 input format. */ - CGNS_GRID = 2, /*!< \brief CGNS input format for the computational grid. */ - RECTANGLE = 3, /*!< \brief 2D rectangular mesh with N x M points of size Lx x Ly. */ - BOX = 4 /*!< \brief 3D box mesh with N x M x L points of size Lx x Ly x Lz. */ +enum ENUM_GRID { + SU2 = 1, /*!< \brief SU2 ascii format. */ + SU2_BIN = 2, /*!< \brief SU2 binary format. */ + CGNS_GRID = 3, /*!< \brief CGNS format for the computational grid. */ + RECTANGLE = 4, /*!< \brief 2D rectangular mesh with N x M points of size Lx x Ly. */ + BOX = 5 /*!< \brief 3D box mesh with N x M x L points of size Lx x Ly x Lz. */ }; -static const MapType Input_Map = { - MakePair("SU2", SU2) - MakePair("CGNS", CGNS_GRID) - MakePair("RECTANGLE", RECTANGLE) - MakePair("BOX", BOX) +static const MapType Input_Map = { + MakePair("SU2", ENUM_GRID::SU2) + MakePair("SU2B", ENUM_GRID::SU2_BIN) + MakePair("CGNS", ENUM_GRID::CGNS_GRID) + MakePair("RECTANGLE", ENUM_GRID::RECTANGLE) + MakePair("BOX", ENUM_GRID::BOX) +}; +static const MapType OutputMesh_Map = { + MakePair("SU2", ENUM_GRID::SU2) + MakePair("SU2B", ENUM_GRID::SU2_BIN) }; - /*! * \brief Type of solution output file formats @@ -2198,7 +2337,8 @@ enum class OUTPUT_TYPE { PARAVIEW_LEGACY_BINARY, /*!< \brief Paraview binary format for the solution output. */ SURFACE_PARAVIEW_ASCII, /*!< \brief Paraview ASCII format for the solution output. */ SURFACE_PARAVIEW_LEGACY_BINARY, /*!< \brief Paraview binary format for the solution output. */ - MESH, /*!< \brief SU2 mesh format. */ + MESH, /*!< \brief SU2 ASCII mesh format. */ + MESH_BINARY, /*!< \brief SU2 binary mesh format. */ RESTART_BINARY, /*!< \brief SU2 binary restart format. */ RESTART_ASCII, /*!< \brief SU2 ASCII restart format. */ PARAVIEW_XML, /*!< \brief Paraview XML with binary data format */ @@ -2224,6 +2364,7 @@ static const MapType Output_Map = { MakePair("SURFACE_PARAVIEW", OUTPUT_TYPE::SURFACE_PARAVIEW_XML) MakePair("PARAVIEW_MULTIBLOCK", OUTPUT_TYPE::PARAVIEW_MULTIBLOCK) MakePair("MESH", OUTPUT_TYPE::MESH) + MakePair("MESH_BINARY", OUTPUT_TYPE::MESH_BINARY) MakePair("RESTART_ASCII", OUTPUT_TYPE::RESTART_ASCII) MakePair("RESTART", OUTPUT_TYPE::RESTART_BINARY) MakePair("CGNS", OUTPUT_TYPE::CGNS) @@ -2492,19 +2633,30 @@ static const MapType Sens_Smoothing_Map = { * \brief Types of preconditioners for the linear solver */ enum ENUM_LINEAR_SOLVER_PREC { + IDENTITY, /*!< \brief No preconditioner. */ JACOBI, /*!< \brief Jacobi preconditioner. */ LU_SGS, /*!< \brief LU SGS preconditioner. */ LINELET, /*!< \brief Line implicit preconditioner. */ ILU, /*!< \brief ILU(k) preconditioner. */ + Q_LU_SGS, /*!< \brief LU-SGS with quantized (int8) off-diagonal storage; L/U are never allocated as ScalarType. */ + Q_JACOBI, /*!< \brief Jacobi with quantized (int8) off-diagonal storage; same matvec quantization as Q_LU_SGS, + the diagonal inverse is still computed and applied at full precision. */ + Q_IDENTITY, /*!< \brief No preconditioner, but the matrix-vector product still uses quantized (int8) + off-diagonal storage, same matvec quantization as Q_LU_SGS/Q_JACOBI. */ PASTIX_ILU=10, /*!< \brief PaStiX ILU(k) preconditioner. */ PASTIX_LU_P, /*!< \brief PaStiX LU as preconditioner. */ PASTIX_LDLT_P, /*!< \brief PaStiX LDLT as preconditioner. */ }; static const MapType Linear_Solver_Prec_Map = { + MakePair("NONE", IDENTITY) + MakePair("IDENTITY", IDENTITY) MakePair("JACOBI", JACOBI) MakePair("LU_SGS", LU_SGS) MakePair("LINELET", LINELET) MakePair("ILU", ILU) + MakePair("Q_LU_SGS", Q_LU_SGS) + MakePair("Q_JACOBI", Q_JACOBI) + MakePair("Q_IDENTITY", Q_IDENTITY) MakePair("PASTIX_ILU", PASTIX_ILU) MakePair("PASTIX_LU", PASTIX_LU_P) MakePair("PASTIX_LDLT", PASTIX_LDLT_P) @@ -2639,7 +2791,7 @@ enum class CHECK_TAPE_VARIABLES { }; static const MapType CheckTapeVariables_Map = { MakePair("SOLVER_VARIABLES", CHECK_TAPE_VARIABLES::SOLVER_VARIABLES) - MakePair("SOLVER_VARIABLES_AND_MESH_COORDINATES", CHECK_TAPE_VARIABLES::MESH_COORDINATES) + MakePair("MESH_COORDINATES", CHECK_TAPE_VARIABLES::MESH_COORDINATES) }; enum class RECORDING { @@ -2647,11 +2799,7 @@ enum class RECORDING { SOLUTION_VARIABLES, MESH_COORDS, MESH_DEFORM, - SOLUTION_AND_MESH, - TAG_INIT_SOLVER_VARIABLES, - TAG_CHECK_SOLVER_VARIABLES, - TAG_INIT_SOLVER_AND_MESH, - TAG_CHECK_SOLVER_AND_MESH + SOLUTION_AND_MESH }; /*! @@ -2727,6 +2875,9 @@ enum class MPI_QUANTITIES { MESH_DISPLACEMENTS , /*!< \brief Mesh displacements at the interface. */ SOLUTION_TIME_N , /*!< \brief Solution at time n. */ SOLUTION_TIME_N1 , /*!< \brief Solution at time n-1. */ + MOM_COEFF , /*!< \brief Momentum coefficient for the Rhie-Chow scheme. */ + MOM_CORRECTION , /*!< \brief Momentum correction for the pressure-based poisson solver (used when computing HbyA). */ + HBYA_CORRECTION , /*!< \brief HbyA correction for the pressure-based poisson solver. */ }; /*! @@ -2854,6 +3005,7 @@ enum class LINEAR_SOLVER_MODE { STANDARD, /*!< \brief Operate in standard mode. */ MESH_DEFORM, /*!< \brief Operate in mesh deformation mode. */ GRADIENT_MODE, /*!< \brief Operate in gradient smoothing mode. */ + POISSON, /*!< \brief Operate in poisson solver mode. */ }; /*! diff --git a/Common/include/option_structure.inl b/Common/include/option_structure.inl index cf85994db070..fe1b2df385ee 100644 --- a/Common/include/option_structure.inl +++ b/Common/include/option_structure.inl @@ -1101,6 +1101,21 @@ struct CStringValuesListHelper { }; // Class where the option is represented by (string, N * "some type", string, N * "some type", ...) +/*! + * \brief Whether a config token is a numeric value rather than a name. + * + * Options that interleave marker names with numbers have to tell the two apart. Testing the first + * character for a letter is not enough: mesh formats such as CGNS routinely produce boundary names + * that begin with a digit (4000_QUAD_4_Bdy6), which such a test reads as a value. Requiring the + * whole token to parse as a number is unambiguous for every name that is not purely numeric. + */ +inline bool IsNumericToken(const std::string& token) { + if (token.empty()) return false; + char* end = nullptr; + std::strtod(token.c_str(), &end); + return (end != token.c_str()) && (*end == '\0'); +} + template class COptionStringValuesList final : public COptionBase { const string name; // identifier for the option @@ -1143,15 +1158,20 @@ class COptionStringValuesList final : public COptionBase { return ""; } - /*--- Determine the number of strings: A new string is found if the first char in the option is a letter. - * This will fail in if a string starts with a number! Additionally, determine the number of values that - * are prescribed per string. ---*/ + /*--- Determine the number of strings: a field that does not parse as a number starts a new string, + * anything that does is one of its values. Testing only the first character for a letter would + * misread the digit-leading marker names that CGNS meshes produce. Additionally, determine the + * number of values that are prescribed per string. ---*/ vector num_vals_per_string; /*--- Loop through the fields of the option. ---*/ for (const auto& val : option_value) { - if (isalpha(val[0])) { + if (!IsNumericToken(val)) { num_vals_per_string.push_back(0); } else { + if (num_vals_per_string.empty()) + SU2_MPI::Error(name + string(" must begin with a marker name, but starts with the value \"") + val + + string("\". A marker whose name is purely numeric cannot be told apart from a value."), + CURRENT_FUNCTION); num_vals_per_string.back()++; } } @@ -1360,15 +1380,15 @@ class COptionWallSpecies : public COptionBase { /*--- Determine the number of markers and species per marker. * Format: marker1, TYPE1, value1, TYPE2, value2, ..., marker2, TYPE1, value1, ... - * Each marker name starts with a letter, each TYPE is an enum string (starts with letter), - * and each value is numeric. Pattern: marker, (TYPE, value) x N ---*/ + * Marker names and TYPE keywords are non-numeric fields, values are numeric. + * Pattern: marker, (TYPE, value) x N ---*/ vector marker_indices; // Indices where markers start vector species_counts; // Number of species per marker // Find all marker positions (strings starting with a letter that are not TYPE keywords) for (unsigned short i = 0; i < totalVals; i++) { - if (isalpha(option_value[i][0])) { + if (!IsNumericToken(option_value[i])) { // Check if this could be a TYPE keyword (i.e., is it in the enum map?) if (this->m.find(option_value[i]) != m.end()) { continue; // This is a TYPE keyword, not a marker diff --git a/Common/include/parallelization/mpi_structure.cpp b/Common/include/parallelization/mpi_structure.cpp index 03152e709e45..e04759b9f952 100644 --- a/Common/include/parallelization/mpi_structure.cpp +++ b/Common/include/parallelization/mpi_structure.cpp @@ -190,7 +190,7 @@ template class CBaseMPIWrapper; #if defined CODI_REVERSE_TYPE template class CBaseMPIWrapper; #endif -#if defined USE_MIXED_PRECISION +#if defined(USE_MIXED_PRECISION) && !defined(USE_SINGLE_PRECISION) template class CBaseMPIWrapper; #endif diff --git a/Common/include/parallelization/mpi_structure.hpp b/Common/include/parallelization/mpi_structure.hpp index afe0c666dcbf..242aa00d7b0f 100644 --- a/Common/include/parallelization/mpi_structure.hpp +++ b/Common/include/parallelization/mpi_structure.hpp @@ -61,6 +61,11 @@ #ifdef HAVE_MPI +#ifdef USE_SINGLE_PRECISION +#undef MPI_DOUBLE +#define MPI_DOUBLE MPI_FLOAT +#endif + /*--- Depending on the datatype used, the correct MPI wrapper class is defined. * For the default (double type) case this results in using the normal MPI routines. ---*/ #if defined CODI_REVERSE_TYPE || defined CODI_FORWARD_TYPE @@ -71,10 +76,10 @@ using namespace medi; #include class CMediMPIWrapper; -typedef CMediMPIWrapper SU2_MPI; +using SU2_MPI = CMediMPIWrapper; -typedef codi::CoDiMpiTypes MediTypes; -typedef MediTypes::Tool MediTool; +using MediTypes = codi::CoDiMpiTypes; +using MediTool = MediTypes::Tool; extern MediTypes* mediTypes; #define AMPI_ADOUBLE ((medi::MpiTypeInterface*)mediTypes->MPI_TYPE) @@ -91,12 +96,12 @@ using SU2_MPI = CBaseMPIWrapper; */ class CBaseMPIWrapper { public: - typedef MPI_Request Request; - typedef MPI_Status Status; - typedef MPI_Datatype Datatype; - typedef MPI_Op Op; - typedef MPI_Comm Comm; - typedef MPI_Win Win; + using Request = MPI_Request; + using Status = MPI_Status; + using Datatype = MPI_Datatype; + using Op = MPI_Op; + using Comm = MPI_Comm; + using Win = MPI_Win; protected: static int Rank, Size, MinRankError; @@ -256,7 +261,7 @@ class CBaseMPIWrapper { static inline passivedouble Wtime(void) { return MPI_Wtime(); } }; -typedef MPI_Comm SU2_Comm; +using SU2_Comm = MPI_Comm; #if defined CODI_REVERSE_TYPE || defined CODI_FORWARD_TYPE @@ -267,8 +272,8 @@ typedef MPI_Comm SU2_Comm; class CMediMPIWrapper : public CBaseMPIWrapper { public: - typedef AMPI_Request Request; - typedef AMPI_Status Status; + using Request = AMPI_Request; + using Status = AMPI_Status; static inline void Init(int* argc, char*** argv) { AMPI_Init(argc, argv); @@ -492,10 +497,10 @@ class CMediMPIWrapper : public CBaseMPIWrapper { template class CBaseMPIWrapper { public: - typedef int Comm; - typedef int Datatype; - typedef int Request; - typedef int Op; + using Comm = int; + using Datatype = int; + using Request = int; + using Op = int; struct Status { int MPI_TAG; @@ -632,7 +637,7 @@ struct SelectMPIWrapper { #endif /*--- Specialize for the low precision type. ---*/ -#if defined(USE_MIXED_PRECISION) +#if defined(USE_MIXED_PRECISION) && !defined(USE_SINGLE_PRECISION) template <> struct SelectMPIWrapper { #if defined HAVE_MPI diff --git a/Common/include/parallelization/omp_structure.hpp b/Common/include/parallelization/omp_structure.hpp index af581bd1d76c..1763169bef27 100644 --- a/Common/include/parallelization/omp_structure.hpp +++ b/Common/include/parallelization/omp_structure.hpp @@ -134,6 +134,7 @@ void omp_finalize(); /*--- Convenience macros (do not use excessive nesting). ---*/ #define SU2_OMP_ATOMIC SU2_OMP(atomic) +#define SU2_OMP_ATOMIC_WRITE SU2_OMP(atomic write) #ifndef HAVE_OPDI @@ -283,7 +284,10 @@ inline void atomicAdd(T rhs, T& lhs) { #define ATOMIC_COMPARE_FALLBACK /*--- Atomic max, shared = max(shared, local). ---*/ -#ifdef _OPENMP +/*--- nvcc's host pass drops the clause from "#pragma omp atomic compare", which the host + * compiler then rejects. The .cu sources do not use these functions, so they simply get + * the critical section fallback below. ---*/ +#if defined(_OPENMP) && !defined(__CUDACC__) #if _OPENMP >= ATOMIC_COMPARE_SINCE /*--- Atomic min/max are supported for arithmetic types. ---*/ template ::value> = 0> @@ -305,7 +309,7 @@ inline void atomicMax(const T& local, T& shared) { } /*--- Atomic min, shared = min(shared, local). ---*/ -#ifdef _OPENMP +#if defined(_OPENMP) && !defined(__CUDACC__) #if _OPENMP >= ATOMIC_COMPARE_SINCE template ::value> = 0> inline void atomicMin(const T& local, T& shared) { diff --git a/Common/include/parallelization/vectorization.hpp b/Common/include/parallelization/vectorization.hpp index 34747913843a..b11d19780af9 100644 --- a/Common/include/parallelization/vectorization.hpp +++ b/Common/include/parallelization/vectorization.hpp @@ -35,6 +35,9 @@ #ifdef __SSE2__ #include "x86intrin.h" #endif +#if defined(__ARM_NEON) || defined(__ARM_NEON__) +#include +#endif namespace simd { /// \addtogroup SIMD @@ -47,7 +50,7 @@ using namespace VecExpr; constexpr size_t PREFERRED_SIZE = 64; #elif defined(__AVX__) constexpr size_t PREFERRED_SIZE = 32; -#elif defined(__SSE2__) +#elif defined(__SSE2__) || defined(__ARM_NEON) || defined(__ARM_NEON__) constexpr size_t PREFERRED_SIZE = 16; #else constexpr size_t PREFERRED_SIZE = 8; @@ -200,9 +203,11 @@ struct SIXTEEN {}; /*--- Constants for bitwise implementations. ---*/ /*--- abs forces the sign bit to 0 ("x" & 0b0111...). ---*/ -constexpr auto abs_mask_d = 0x7FFFFFFFFFFFFFFFL; +constexpr uint64_t abs_mask_d = 0x7FFFFFFFFFFFFFFFL; +constexpr uint32_t abs_mask_s = 0x7FFFFFFFU; /*--- negation flips the sign bit ("x" ^ 0b1000...). ---*/ -constexpr auto sign_mask_d = 0x8000000000000000L; +constexpr uint64_t sign_mask_d = 0x8000000000000000L; +constexpr uint32_t sign_mask_s = 0x80000000U; #ifdef __SSE2__ /*! @@ -248,6 +253,46 @@ FORCEINLINE __m128d sign_p(__m128d x) { return _mm_or_pd(ones_2d, _mm_and_pd(x, #include "special_vectorization.hpp" +/*! + * Create specialization for array of 4 floats (this should be always available). + */ +#define ARRAY_T Array +#define SCALAR_T float +#define REGISTER_T __m128 +#define SIZE_TAG SizeTag::FOUR() + +static const __m128 abs_mask_4s = _mm_castsi128_ps(_mm_set1_epi32(abs_mask_s)); +static const __m128 sign_mask_4s = _mm_castsi128_ps(_mm_set1_epi32(sign_mask_s)); +static const __m128 ones_4s = _mm_set1_ps(1); + +FORCEINLINE __m128 set1_p(SizeTag::FOUR, float p) { return _mm_set1_ps(p); } +FORCEINLINE __m128 load_p(SizeTag::FOUR, const float* p) { return _mm_load_ps(p); } +FORCEINLINE __m128 loadu_p(SizeTag::FOUR, const float* p) { return _mm_loadu_ps(p); } +FORCEINLINE void store_p(float* p, __m128 x) { _mm_store_ps(p, x); } +FORCEINLINE void storeu_p(float* p, __m128 x) { _mm_storeu_ps(p, x); } +FORCEINLINE void stream_p(float* p, __m128 x) { _mm_stream_ps(p, x); } + +FORCEINLINE __m128 add_p(__m128 a, __m128 b) { return _mm_add_ps(a, b); } +FORCEINLINE __m128 sub_p(__m128 a, __m128 b) { return _mm_sub_ps(a, b); } +FORCEINLINE __m128 mul_p(__m128 a, __m128 b) { return _mm_mul_ps(a, b); } +FORCEINLINE __m128 div_p(__m128 a, __m128 b) { return _mm_div_ps(a, b); } +FORCEINLINE __m128 max_p(__m128 a, __m128 b) { return _mm_max_ps(a, b); } +FORCEINLINE __m128 min_p(__m128 a, __m128 b) { return _mm_min_ps(a, b); } + +FORCEINLINE __m128 eq_p(__m128 a, __m128 b) { return _mm_and_ps(ones_4s, _mm_cmpeq_ps(a, b)); } +FORCEINLINE __m128 lt_p(__m128 a, __m128 b) { return _mm_and_ps(ones_4s, _mm_cmplt_ps(a, b)); } +FORCEINLINE __m128 le_p(__m128 a, __m128 b) { return _mm_and_ps(ones_4s, _mm_cmple_ps(a, b)); } +FORCEINLINE __m128 ne_p(__m128 a, __m128 b) { return _mm_and_ps(ones_4s, _mm_cmpneq_ps(a, b)); } +FORCEINLINE __m128 ge_p(__m128 a, __m128 b) { return _mm_and_ps(ones_4s, _mm_cmpge_ps(a, b)); } +FORCEINLINE __m128 gt_p(__m128 a, __m128 b) { return _mm_and_ps(ones_4s, _mm_cmpgt_ps(a, b)); } + +FORCEINLINE __m128 sqrt_p(__m128 x) { return _mm_sqrt_ps(x); } +FORCEINLINE __m128 abs_p(__m128 x) { return _mm_and_ps(x, abs_mask_4s); } +FORCEINLINE __m128 neg_p(__m128 x) { return _mm_xor_ps(x, sign_mask_4s); } +FORCEINLINE __m128 sign_p(__m128 x) { return _mm_or_ps(ones_4s, _mm_and_ps(x, sign_mask_4s)); } + +#include "special_vectorization.hpp" + #endif // __SSE2__ #ifdef __AVX__ @@ -291,6 +336,46 @@ FORCEINLINE __m256d sign_p(__m256d x) { return _mm256_or_pd(ones_4d, _mm256_and_ #include "special_vectorization.hpp" +/*! + * Create specialization for array of 8 floats. + */ +#define ARRAY_T Array +#define SCALAR_T float +#define REGISTER_T __m256 +#define SIZE_TAG SizeTag::EIGHT() + +static const __m256 abs_mask_8s = _mm256_castsi256_ps(_mm256_set1_epi32(abs_mask_s)); +static const __m256 sign_mask_8s = _mm256_castsi256_ps(_mm256_set1_epi32(sign_mask_s)); +static const __m256 ones_8s = _mm256_set1_ps(1); + +FORCEINLINE __m256 set1_p(SizeTag::EIGHT, float p) { return _mm256_set1_ps(p); } +FORCEINLINE __m256 load_p(SizeTag::EIGHT, const float* p) { return _mm256_load_ps(p); } +FORCEINLINE __m256 loadu_p(SizeTag::EIGHT, const float* p) { return _mm256_loadu_ps(p); } +FORCEINLINE void store_p(float* p, __m256 x) { _mm256_store_ps(p, x); } +FORCEINLINE void storeu_p(float* p, __m256 x) { _mm256_storeu_ps(p, x); } +FORCEINLINE void stream_p(float* p, __m256 x) { _mm256_stream_ps(p, x); } + +FORCEINLINE __m256 add_p(__m256 a, __m256 b) { return _mm256_add_ps(a, b); } +FORCEINLINE __m256 sub_p(__m256 a, __m256 b) { return _mm256_sub_ps(a, b); } +FORCEINLINE __m256 mul_p(__m256 a, __m256 b) { return _mm256_mul_ps(a, b); } +FORCEINLINE __m256 div_p(__m256 a, __m256 b) { return _mm256_div_ps(a, b); } +FORCEINLINE __m256 max_p(__m256 a, __m256 b) { return _mm256_max_ps(a, b); } +FORCEINLINE __m256 min_p(__m256 a, __m256 b) { return _mm256_min_ps(a, b); } + +FORCEINLINE __m256 eq_p(__m256 a, __m256 b) { return _mm256_and_ps(ones_8s, _mm256_cmp_ps(a, b, 0)); } +FORCEINLINE __m256 lt_p(__m256 a, __m256 b) { return _mm256_and_ps(ones_8s, _mm256_cmp_ps(a, b, 1)); } +FORCEINLINE __m256 le_p(__m256 a, __m256 b) { return _mm256_and_ps(ones_8s, _mm256_cmp_ps(a, b, 2)); } +FORCEINLINE __m256 ne_p(__m256 a, __m256 b) { return _mm256_and_ps(ones_8s, _mm256_cmp_ps(a, b, 4)); } +FORCEINLINE __m256 ge_p(__m256 a, __m256 b) { return _mm256_and_ps(ones_8s, _mm256_cmp_ps(a, b, 13)); } +FORCEINLINE __m256 gt_p(__m256 a, __m256 b) { return _mm256_and_ps(ones_8s, _mm256_cmp_ps(a, b, 14)); } + +FORCEINLINE __m256 sqrt_p(__m256 x) { return _mm256_sqrt_ps(x); } +FORCEINLINE __m256 abs_p(__m256 x) { return _mm256_and_ps(x, abs_mask_8s); } +FORCEINLINE __m256 neg_p(__m256 x) { return _mm256_xor_ps(x, sign_mask_8s); } +FORCEINLINE __m256 sign_p(__m256 x) { return _mm256_or_ps(ones_8s, _mm256_and_ps(x, sign_mask_8s)); } + +#include "special_vectorization.hpp" + #endif // __AVX__ #ifdef __AVX512F__ @@ -338,8 +423,145 @@ FORCEINLINE __m512d sign_p(__m512d x) { return _mm512_or_pd(ones_8d, _mm512_and_ #include "special_vectorization.hpp" +/*! + * Create specialization for array of 16 floats. + */ +#define ARRAY_T Array +#define SCALAR_T float +#define REGISTER_T __m512 +#define SIZE_TAG SizeTag::SIXTEEN() + +static const __m512 abs_mask_16s = _mm512_castsi512_ps(_mm512_set1_epi32(abs_mask_s)); +static const __m512 sign_mask_16s = _mm512_castsi512_ps(_mm512_set1_epi32(sign_mask_s)); +static const __m512 ones_16s = _mm512_set1_ps(1); + +FORCEINLINE __m512 set1_p(SizeTag::SIXTEEN, float p) { return _mm512_set1_ps(p); } +FORCEINLINE __m512 load_p(SizeTag::SIXTEEN, const float* p) { return _mm512_load_ps(p); } +FORCEINLINE __m512 loadu_p(SizeTag::SIXTEEN, const float* p) { return _mm512_loadu_ps(p); } +FORCEINLINE void store_p(float* p, __m512 x) { _mm512_store_ps(p, x); } +FORCEINLINE void storeu_p(float* p, __m512 x) { _mm512_storeu_ps(p, x); } +FORCEINLINE void stream_p(float* p, __m512 x) { _mm512_stream_ps(p, x); } + +FORCEINLINE __m512 add_p(__m512 a, __m512 b) { return _mm512_add_ps(a, b); } +FORCEINLINE __m512 sub_p(__m512 a, __m512 b) { return _mm512_sub_ps(a, b); } +FORCEINLINE __m512 mul_p(__m512 a, __m512 b) { return _mm512_mul_ps(a, b); } +FORCEINLINE __m512 div_p(__m512 a, __m512 b) { return _mm512_div_ps(a, b); } +FORCEINLINE __m512 max_p(__m512 a, __m512 b) { return _mm512_max_ps(a, b); } +FORCEINLINE __m512 min_p(__m512 a, __m512 b) { return _mm512_min_ps(a, b); } + +template +FORCEINLINE __m512 cmp_p(__m512 a, __m512 b) { + return _mm512_mask_blend_ps(_mm512_cmp_ps_mask(a, b, opCode), _mm512_setzero_ps(), ones_16s); +} +FORCEINLINE __m512 eq_p(__m512 a, __m512 b) { return cmp_p<0>(a, b); } +FORCEINLINE __m512 lt_p(__m512 a, __m512 b) { return cmp_p<1>(a, b); } +FORCEINLINE __m512 le_p(__m512 a, __m512 b) { return cmp_p<2>(a, b); } +FORCEINLINE __m512 ne_p(__m512 a, __m512 b) { return cmp_p<4>(a, b); } +FORCEINLINE __m512 ge_p(__m512 a, __m512 b) { return cmp_p<13>(a, b); } +FORCEINLINE __m512 gt_p(__m512 a, __m512 b) { return cmp_p<14>(a, b); } + +FORCEINLINE __m512 sqrt_p(__m512 x) { return _mm512_sqrt_ps(x); } +FORCEINLINE __m512 abs_p(__m512 x) { return _mm512_and_ps(x, abs_mask_16s); } +FORCEINLINE __m512 neg_p(__m512 x) { return _mm512_xor_ps(x, sign_mask_16s); } +FORCEINLINE __m512 sign_p(__m512 x) { return _mm512_or_ps(ones_16s, _mm512_and_ps(x, sign_mask_16s)); } + +#include "special_vectorization.hpp" + #endif // __AVX512F__ +#if defined(__ARM_NEON) || defined(__ARM_NEON__) +/*! + * Create specialization for array of 2 doubles. + */ +#define ARRAY_T Array +#define SCALAR_T double +#define REGISTER_T float64x2_t +#define SIZE_TAG SizeTag::TWO() + +static const uint64x2_t abs_mask_2d_u = vdupq_n_u64(abs_mask_d); +static const uint64x2_t sign_mask_2d_u = vdupq_n_u64(sign_mask_d); +static const uint64x2_t ones_2d_u = vreinterpretq_u64_f64(vdupq_n_f64(1.0)); +static const uint64x2_t ones_2u = vdupq_n_u64(~0ULL); + +FORCEINLINE float64x2_t set1_p(SizeTag::TWO, double p) { return vdupq_n_f64(p); } +FORCEINLINE float64x2_t load_p(SizeTag::TWO, const double* p) { return vld1q_f64(p); } +FORCEINLINE float64x2_t loadu_p(SizeTag::TWO, const double* p) { return vld1q_f64(p); } +FORCEINLINE void store_p(double* p, float64x2_t x) { vst1q_f64(p, x); } +FORCEINLINE void storeu_p(double* p, float64x2_t x) { vst1q_f64(p, x); } +/*--- No direct NEON equivalent to streaming stores. ---*/ +FORCEINLINE void stream_p(double* p, float64x2_t x) { vst1q_f64(p, x); } + +FORCEINLINE float64x2_t add_p(float64x2_t a, float64x2_t b) { return vaddq_f64(a, b); } +FORCEINLINE float64x2_t sub_p(float64x2_t a, float64x2_t b) { return vsubq_f64(a, b); } +FORCEINLINE float64x2_t mul_p(float64x2_t a, float64x2_t b) { return vmulq_f64(a, b); } +FORCEINLINE float64x2_t div_p(float64x2_t a, float64x2_t b) { return vdivq_f64(a, b); } +FORCEINLINE float64x2_t max_p(float64x2_t a, float64x2_t b) { return vmaxq_f64(a, b); } +FORCEINLINE float64x2_t min_p(float64x2_t a, float64x2_t b) { return vminq_f64(a, b); } + +/*--- Comparisons return uint64x2_t masks. Convert to 0.0 / 1.0. ---*/ +FORCEINLINE float64x2_t int2float(uint64x2_t a) { return vreinterpretq_f64_u64(a); } +FORCEINLINE float64x2_t cmp2float(uint64x2_t cmp) { return int2float(vandq_u64(ones_2d_u, cmp)); } +FORCEINLINE float64x2_t eq_p(float64x2_t a, float64x2_t b) { return cmp2float(vceqq_f64(a, b)); } +FORCEINLINE float64x2_t lt_p(float64x2_t a, float64x2_t b) { return cmp2float(vcltq_f64(a, b)); } +FORCEINLINE float64x2_t le_p(float64x2_t a, float64x2_t b) { return cmp2float(vcleq_f64(a, b)); } +FORCEINLINE float64x2_t ne_p(float64x2_t a, float64x2_t b) { return cmp2float(veorq_u64(ones_2u, vceqq_f64(a, b))); } +FORCEINLINE float64x2_t ge_p(float64x2_t a, float64x2_t b) { return cmp2float(vcgeq_f64(a, b)); } +FORCEINLINE float64x2_t gt_p(float64x2_t a, float64x2_t b) { return cmp2float(vcgtq_f64(a, b)); } + +FORCEINLINE float64x2_t sqrt_p(float64x2_t x) { return vsqrtq_f64(x); } +FORCEINLINE float64x2_t abs_p(float64x2_t x) { return int2float(vandq_u64(vreinterpretq_u64_f64(x), abs_mask_2d_u)); } +FORCEINLINE float64x2_t neg_p(float64x2_t x) { return int2float(veorq_u64(vreinterpretq_u64_f64(x), sign_mask_2d_u)); } +FORCEINLINE float64x2_t sign_p(float64x2_t x) { + return int2float(vorrq_u64(ones_2d_u, vandq_u64(vreinterpretq_u64_f64(x), sign_mask_2d_u))); +} + +#include "special_vectorization.hpp" + +/*! + * Create specialization for array of 4 floats. + */ +#define ARRAY_T Array +#define SCALAR_T float +#define REGISTER_T float32x4_t +#define SIZE_TAG SizeTag::FOUR() + +static const uint32x4_t abs_mask_4s_u = vdupq_n_u32(abs_mask_s); +static const uint32x4_t sign_mask_4s_u = vdupq_n_u32(sign_mask_s); +static const uint32x4_t ones_4s_u = vreinterpretq_u32_f32(vdupq_n_f32(1.0f)); + +FORCEINLINE float32x4_t set1_p(SizeTag::FOUR, float p) { return vdupq_n_f32(p); } +FORCEINLINE float32x4_t load_p(SizeTag::FOUR, const float* p) { return vld1q_f32(p); } +FORCEINLINE float32x4_t loadu_p(SizeTag::FOUR, const float* p) { return vld1q_f32(p); } +FORCEINLINE void store_p(float* p, float32x4_t x) { vst1q_f32(p, x); } +FORCEINLINE void storeu_p(float* p, float32x4_t x) { vst1q_f32(p, x); } +FORCEINLINE void stream_p(float* p, float32x4_t x) { vst1q_f32(p, x); } +FORCEINLINE float32x4_t add_p(float32x4_t a, float32x4_t b) { return vaddq_f32(a, b); } +FORCEINLINE float32x4_t sub_p(float32x4_t a, float32x4_t b) { return vsubq_f32(a, b); } +FORCEINLINE float32x4_t mul_p(float32x4_t a, float32x4_t b) { return vmulq_f32(a, b); } +FORCEINLINE float32x4_t div_p(float32x4_t a, float32x4_t b) { return vdivq_f32(a, b); } +FORCEINLINE float32x4_t max_p(float32x4_t a, float32x4_t b) { return vmaxq_f32(a, b); } +FORCEINLINE float32x4_t min_p(float32x4_t a, float32x4_t b) { return vminq_f32(a, b); } + +FORCEINLINE float32x4_t int2float(uint32x4_t a) { return vreinterpretq_f32_u32(a); } +FORCEINLINE float32x4_t cmp2float(uint32x4_t cmp) { return int2float(vandq_u32(ones_4s_u, cmp)); } +FORCEINLINE float32x4_t eq_p(float32x4_t a, float32x4_t b) { return cmp2float(vceqq_f32(a, b)); } +FORCEINLINE float32x4_t lt_p(float32x4_t a, float32x4_t b) { return cmp2float(vcltq_f32(a, b)); } +FORCEINLINE float32x4_t le_p(float32x4_t a, float32x4_t b) { return cmp2float(vcleq_f32(a, b)); } +FORCEINLINE float32x4_t ne_p(float32x4_t a, float32x4_t b) { return cmp2float(vmvnq_u32(vceqq_f32(a, b))); } +FORCEINLINE float32x4_t ge_p(float32x4_t a, float32x4_t b) { return cmp2float(vcgeq_f32(a, b)); } +FORCEINLINE float32x4_t gt_p(float32x4_t a, float32x4_t b) { return cmp2float(vcgtq_f32(a, b)); } + +FORCEINLINE float32x4_t sqrt_p(float32x4_t x) { return vsqrtq_f32(x); } +FORCEINLINE float32x4_t abs_p(float32x4_t x) { return int2float(vandq_u32(vreinterpretq_u32_f32(x), abs_mask_4s_u)); } +FORCEINLINE float32x4_t neg_p(float32x4_t x) { return int2float(veorq_u32(vreinterpretq_u32_f32(x), sign_mask_4s_u)); } +FORCEINLINE float32x4_t sign_p(float32x4_t x) { + return int2float(vorrq_u32(ones_4s_u, vandq_u32(vreinterpretq_u32_f32(x), sign_mask_4s_u))); +} + +#include "special_vectorization.hpp" + +#endif // __ARM_NEON__ + #undef ARRAY_BOILERPLATE /// @} diff --git a/SU2_CFD/include/numerics/turbulent/transition/trans_convection.hpp b/Common/include/toolboxes/SwapBytes.hpp similarity index 67% rename from SU2_CFD/include/numerics/turbulent/transition/trans_convection.hpp rename to Common/include/toolboxes/SwapBytes.hpp index 2338c2ca837b..c09a91e8b12a 100644 --- a/SU2_CFD/include/numerics/turbulent/transition/trans_convection.hpp +++ b/Common/include/toolboxes/SwapBytes.hpp @@ -1,8 +1,7 @@ /*! - * \file trans_convection.hpp - * \brief Delarations of numerics classes for discretization of - * convective fluxes in transition problems. - * \author S. Kang + * \file SwapBytes.hpp + * \brief Function to swap bytes of primitive data types. + * \author P. Gomes * \version 8.5.0 "Harrier" * * SU2 Project Website: https://su2code.github.io @@ -28,13 +27,13 @@ #pragma once -#include "../turb_convection.hpp" +#include +#include /*! - * \class CUpwSca_TransLM - * \brief Re-use the SST convective fluxes for the scalar upwind discretization of LM transition model equations. - * \ingroup ConvDiscr + * \brief Change storage of buffer to/from big endian from/to little endian + * \param buffer - Pointer to the beginning of the buffer + * \param nBytes - The size in bytes of an data entry + * \param nVar - The number of entries */ -template -using CUpwSca_TransLM = CUpwSca_TurbSST; - +void SwapBytes(char* buffer, size_t nBytes, unsigned long nVar); diff --git a/Common/include/toolboxes/allocation_toolbox.hpp b/Common/include/toolboxes/allocation_toolbox.hpp index 9c357405b757..66db0a04035a 100644 --- a/Common/include/toolboxes/allocation_toolbox.hpp +++ b/Common/include/toolboxes/allocation_toolbox.hpp @@ -144,4 +144,39 @@ inline T* gpu_alloc_cpy(const T* src_ptr, size_t size) noexcept { return static_cast(ptr); } + +/*! + * \brief Page-locked ("pinned") host memory allocation. + * \note Unlike regular (pageable) host memory, cudaMemcpyAsync from/to a pinned buffer is + * actually asynchronous with respect to the host thread; from pageable memory the driver + * silently falls back to a synchronous staged copy. Only worth it for host buffers that + * are the source/destination of an async transfer meant to overlap with other host work. + * \param[in] size in bytes. + * \tparam ZeroInit, initialize memory to 0. + * \return Pointer to memory, always use pinned_free to deallocate. + */ +template +inline T* pinned_alloc(size_t size) noexcept { + void* ptr = nullptr; + +#if defined(HAVE_CUDA) + gpuErrChk(cudaMallocHost((void**)(&ptr), size)); + if (ZeroInit) memset(ptr, 0, size); +#else + return 0; +#endif + + return static_cast(ptr); +} + +/*! + * \brief Free memory allocated with pinned_alloc. + * \param[in] ptr, pointer to memory we want to release. + */ +template +inline void pinned_free(T* ptr) noexcept { +#ifdef HAVE_CUDA + gpuErrChk(cudaFreeHost((void*)ptr)); +#endif +} } // namespace GPUMemoryAllocation diff --git a/Common/include/toolboxes/graph_toolbox.hpp b/Common/include/toolboxes/graph_toolbox.hpp index 344e08ccf7ce..f1c36969ada7 100644 --- a/Common/include/toolboxes/graph_toolbox.hpp +++ b/Common/include/toolboxes/graph_toolbox.hpp @@ -27,6 +27,7 @@ #pragma once +#include "../code_config.hpp" #include "../containers/C2DContainer.hpp" #include "../parallelization/omp_structure.hpp" @@ -333,9 +334,9 @@ class CCompressedSparsePattern { template using CEdgeToNonZeroMap = C2DContainer; -using CCompressedSparsePatternUL = CCompressedSparsePattern; -using CCompressedSparsePatternL = CCompressedSparsePattern; -using CEdgeToNonZeroMapUL = CEdgeToNonZeroMap; +using CCompressedSparsePatternUL = CCompressedSparsePattern; +using CCompressedSparsePatternL = CCompressedSparsePattern; +using CEdgeToNonZeroMapUL = CEdgeToNonZeroMap; /*! * \brief Build a sparse pattern from geometry information, of type FVM or FEM, @@ -417,30 +418,86 @@ CCompressedSparsePattern buildCSRPattern(Geometry_t& geometry, Connecti } /*! - * \brief Build a lookup table of the absolute positions of the non zero entries - * of a compressed sparse pattern, accessed when visiting the FVM edges - * of a grid. The table can then be used for fast access (avoids searches) - * to the non zero entries of a sparse matrix associated with the pattern. - * \param[in] geometry - Definition of the grid. - * \param[in] pattern - Sparse pattern. - * \return nEdge by 2 matrix. + * \brief Extract the strictly-lower part of a symmetric compressed sparse pattern. + * For each row i, the lower entries are those at positions [outerPtr[i], diagPtr[i]). + * \param[in] csr - Full symmetric pattern with diagonal pointer already built. + * \return Strictly-lower CSR pattern. */ -template -CEdgeToNonZeroMap mapEdgesToSparsePattern(Geometry_t& geometry, - const CCompressedSparsePattern& pattern) { - assert(!pattern.empty()); +template +CCompressedSparsePattern buildLowerPattern(const CCompressedSparsePattern& csr) { + assert(!csr.empty()); + const auto nPoint = csr.getOuterSize(); + const auto* outerPtr = csr.outerPtr(); + const auto* innerIdx = csr.innerIdx(); + const auto* diagPtr = csr.diagPtr(); + + su2vector outerPtrL(nPoint + 1); + outerPtrL(0) = 0; + for (auto i = 0ul; i < nPoint; ++i) outerPtrL(i + 1) = outerPtrL(i) + static_cast(diagPtr[i] - outerPtr[i]); + + su2vector innerIdxL(outerPtrL(nPoint)); + Index_t k = 0; + for (auto i = 0ul; i < nPoint; ++i) + for (auto p = outerPtr[i]; p < diagPtr[i]; ++p) innerIdxL(k++) = innerIdx[p]; - CEdgeToNonZeroMap edgeMap(geometry.GetnEdge(), 2); + return CCompressedSparsePattern(std::move(outerPtrL), std::move(innerIdxL)); +} + +/*! + * \brief Extract the strictly-upper part of a symmetric compressed sparse pattern. + * For each row i, the upper entries are those at positions (diagPtr[i], outerPtr[i+1]). + * \param[in] csr - Full symmetric pattern with diagonal pointer already built. + * \return Strictly-upper CSR pattern. + */ +template +CCompressedSparsePattern buildUpperPattern(const CCompressedSparsePattern& csr) { + assert(!csr.empty()); + const auto nPoint = csr.getOuterSize(); + const auto* outerPtr = csr.outerPtr(); + const auto* innerIdx = csr.innerIdx(); + const auto* diagPtr = csr.diagPtr(); + + su2vector outerPtrU(nPoint + 1); + outerPtrU(0) = 0; + for (auto i = 0ul; i < nPoint; ++i) + outerPtrU(i + 1) = outerPtrU(i) + static_cast(outerPtr[i + 1] - diagPtr[i] - 1); + + su2vector innerIdxU(outerPtrU(nPoint)); + Index_t k = 0; + for (auto i = 0ul; i < nPoint; ++i) + for (auto p = diagPtr[i] + 1; p < outerPtr[i + 1]; ++p) innerIdxU(k++) = innerIdx[p]; - for (Index_t iEdge = 0; iEdge < geometry.GetnEdge(); ++iEdge) { - Index_t iPoint = geometry.edges->GetNode(iEdge, 0); - Index_t jPoint = geometry.edges->GetNode(iEdge, 1); + return CCompressedSparsePattern(std::move(outerPtrU), std::move(innerIdxU)); +} - edgeMap(iEdge, 0) = pattern.quickFindInnerIdx(iPoint, jPoint); - edgeMap(iEdge, 1) = pattern.quickFindInnerIdx(jPoint, iPoint); +/*! + * \brief Build bijective maps between strictly-lower (L) and strictly-upper (U) non-zero entries + * that are each other's transposes. Requires a symmetric pattern. + * l_to_u[k_l] = k_u such that U-entry k_u is the transpose of L-entry k_l, and vice-versa. + * \param[in] pattern_l - Strictly-lower CSR pattern. + * \param[in] pattern_u - Strictly-upper CSR pattern. + * \param[out] l_to_u - For each L-entry index, the U-entry index of its transpose. + * \param[out] u_to_l - For each U-entry index, the L-entry index of its transpose. + */ +template +void buildLUTransposeMaps(const CCompressedSparsePattern& pattern_l, + const CCompressedSparsePattern& pattern_u, su2vector& l_to_u, + su2vector& u_to_l) { + const auto nnz_l = pattern_l.getNumNonZeros(); + const auto nnz_u = pattern_u.getNumNonZeros(); + assert(nnz_l == nnz_u && "L and U must have the same NNZ (symmetric pattern)."); + + l_to_u.resize(nnz_l); + u_to_l.resize(nnz_u); + + for (Index_t i = 0; i < pattern_l.getOuterSize(); ++i) { + for (Index_t k_l = pattern_l.outerPtr()[i]; k_l < pattern_l.outerPtr()[i + 1]; ++k_l) { + const Index_t j = pattern_l.innerIdx()[k_l]; // j < i (strictly lower) + const Index_t k_u = pattern_u.quickFindInnerIdx(j, i); // (j,i) is in U since j +template struct GridColor { static_assert(std::is_integral::value); @@ -623,7 +680,7 @@ struct GridColor { * \brief A way to represent natural coloring {0,1,2,...,size-1} with zero * overhead (behaves like looping with an integer index, after optimization...). */ -template +template struct DummyGridColor { static_assert(std::is_integral::value); diff --git a/Common/include/toolboxes/random_toolbox.hpp b/Common/include/toolboxes/random_toolbox.hpp index 6ad052d9e52d..f2ad39a7aa2d 100644 --- a/Common/include/toolboxes/random_toolbox.hpp +++ b/Common/include/toolboxes/random_toolbox.hpp @@ -8,7 +8,7 @@ * The SU2 Project is maintained by the SU2 Foundation * (http://su2foundation.org) * - * Copyright 2012-2025, SU2 Contributors (cf. AUTHORS.md) + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) * * SU2 is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index 0f7a5c6821fa..d5c403d9ab11 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -37,6 +37,7 @@ #include "../include/basic_types/ad_structure.hpp" #include "../include/toolboxes/printing_toolbox.hpp" +#include "../include/toolboxes/SwapBytes.hpp" using namespace PrintingToolbox; @@ -598,12 +599,60 @@ void CConfig::addPythonOption(const string& name) { option_map.insert(pair(name, val)); } +unsigned short CConfig::CheckOpenSU2BinFile(const string& val_mesh_filename, bool readnDim) { + /*--- Check if the mesh file can be opened for binary reading. ---*/ + FILE *mesh_file = fopen(val_mesh_filename.c_str(), "rb"); + if ( !mesh_file ) + SU2_MPI::Error("There is no geometry file called " + val_mesh_filename, + CURRENT_FUNCTION); + + /*--- Read the size of the connectivity type and determine whether + or not byte swapping must be applied. The size of the connectivity + type must be either 4 or 8. ---*/ + int size_conn_type; + auto ret = fread(&size_conn_type, sizeof(int), 1, mesh_file); + if (ret != 1) + SU2_MPI::Error("Error while reading the file " + val_mesh_filename, + CURRENT_FUNCTION); + + bool swap_bytes = false; + if ((size_conn_type != 4) && (size_conn_type != 8)) { + SwapBytes((char *) &size_conn_type, sizeof(int), 1); + swap_bytes = true; + } + + if ((size_conn_type != 4) && (size_conn_type != 8)) + SU2_MPI::Error("The file " + val_mesh_filename + + " is not a valid SU2 binary file", CURRENT_FUNCTION); + + /*--- Skip the number of zones and the zone ID, + if the number of dimensions must be read. ---*/ + if( readnDim ) { + if( fseek(mesh_file, 2*sizeof(int), SEEK_CUR) ) + SU2_MPI::Error("Failed to jump forward in the file" + val_mesh_filename, + CURRENT_FUNCTION); + } + + /*--- Read the information to be returned. ---*/ + int info; + ret = fread(&info, sizeof(int), 1, mesh_file); + if (ret != 1) + SU2_MPI::Error("Error while reading the file " + val_mesh_filename, + CURRENT_FUNCTION); + if ( swap_bytes) + SwapBytes((char *) &info, sizeof(int), 1); + + fclose(mesh_file); + + return (unsigned short) info; +} + unsigned short CConfig::GetnZone(const string& val_mesh_filename, unsigned short val_format) { int nZone = 1; /* Default value if nothing is specified. */ switch (val_format) { - case SU2: { + case ENUM_GRID::SU2: { /*--- Local variables for reading the SU2 file. ---*/ string text_line; @@ -612,8 +661,8 @@ unsigned short CConfig::GetnZone(const string& val_mesh_filename, unsigned short /*--- Check if the mesh file can be opened for reading. ---*/ mesh_file.open(val_mesh_filename.c_str(), ios::in); if (mesh_file.fail()) - SU2_MPI::Error(string("There is no geometry file called ") + val_mesh_filename, - CURRENT_FUNCTION); + SU2_MPI::Error("There is no geometry file called " + val_mesh_filename, + CURRENT_FUNCTION); /*--- Read the SU2 mesh file until the zone data is reached or when it can be decided that it is not present. ---*/ @@ -639,7 +688,15 @@ unsigned short CConfig::GetnZone(const string& val_mesh_filename, unsigned short } - case CGNS_GRID: { + case ENUM_GRID::SU2_BIN: { + + /*--- Open and check the grid file and read the number of zones + at the correct location. */ + nZone = CheckOpenSU2BinFile(val_mesh_filename, false); + break; + } + + case ENUM_GRID::CGNS_GRID: { #ifdef HAVE_CGNS @@ -706,11 +763,11 @@ unsigned short CConfig::GetnZone(const string& val_mesh_filename, unsigned short break; } - case RECTANGLE: { + case ENUM_GRID::RECTANGLE: { nZone = 1; break; } - case BOX: { + case ENUM_GRID::BOX: { nZone = 1; break; } @@ -722,10 +779,10 @@ unsigned short CConfig::GetnZone(const string& val_mesh_filename, unsigned short unsigned short CConfig::GetnDim(const string& val_mesh_filename, unsigned short val_format) { - short nDim = -1; + int nDim = -1; switch (val_format) { - case SU2: { + case ENUM_GRID::SU2: { /*--- Local variables for reading the SU2 file. ---*/ string text_line; @@ -734,7 +791,7 @@ unsigned short CConfig::GetnDim(const string& val_mesh_filename, unsigned short /*--- Open grid file ---*/ mesh_file.open(val_mesh_filename.c_str(), ios::in); if (mesh_file.fail()) { - SU2_MPI::Error(string("The SU2 mesh file named ") + val_mesh_filename + string(" was not found."), CURRENT_FUNCTION); + SU2_MPI::Error("The SU2 mesh file named " + val_mesh_filename + " was not found.", CURRENT_FUNCTION); } /*--- Read the SU2 mesh file until the dimension data is reached @@ -760,14 +817,22 @@ unsigned short CConfig::GetnDim(const string& val_mesh_filename, unsigned short /*--- Throw an error if the dimension was not found. ---*/ if (nDim == -1) { - SU2_MPI::Error(val_mesh_filename + string(" is not an SU2 mesh file or has the wrong format \n ('NDIME=' not found). Please check."), + SU2_MPI::Error(val_mesh_filename + " is not an SU2 mesh file or has the wrong format \n ('NDIME=' not found). Please check.", CURRENT_FUNCTION); } break; } - case CGNS_GRID: { + case ENUM_GRID::SU2_BIN: { + + /*--- Open and check the grid file and read the number of dimensions + at the correct location. */ + nDim = CheckOpenSU2BinFile(val_mesh_filename, true); + break; + } + + case ENUM_GRID::CGNS_GRID: { #ifdef HAVE_CGNS @@ -816,11 +881,11 @@ unsigned short CConfig::GetnDim(const string& val_mesh_filename, unsigned short break; } - case RECTANGLE: { + case ENUM_GRID::RECTANGLE: { nDim = 2; break; } - case BOX: { + case ENUM_GRID::BOX: { nDim = 3; break; } @@ -1137,6 +1202,11 @@ void CConfig::SetConfig_Options() { /*!\brief SST_OPTIONS \n DESCRIPTION: Specify SA turbulence model options/corrections. \n Options: see \link SA_Options_Map \endlink \n DEFAULT: NONE \ingroup Config*/ addEnumListOption("SA_OPTIONS", nSA_Options, SA_Options, SA_Options_Map); + /*!\brief KIND_INCOMP_SYSTEM \n DESCRIPTION: Incomp type \n OPTIONS: see \link Incomp_Map \endlink DEFAULT: NONE \ingroup Config*/ + addEnumOption("KIND_INCOMP_SYSTEM", Kind_Incomp_System, Incomp_Map, INCOMP_SYSTEM::DENSITY_BASED); + /*!\brief KIND_PB_ITER \n DESCRIPTION: Kind_PBIter \n OPTIONS: see \link PBIter_Map \endlink \ingroup Config*/ + addEnumOption("KIND_PB_ITER", Kind_PBIter, PBIter_Map, PBITER::SIMPLE); + /*!\brief ROUGHSST_OPTIONS \n DESCRIPTION: Specify type of boundary condition for rough walls for SST turbulence model. \n Options: see \link ROUGHSST_Options_Map \endlink \n DEFAULT: wilcox1998 \ingroup Config*/ addEnumOption("KIND_ROUGHSST_MODEL", Kind_RoughSST_Model, RoughSST_Model_Map, ROUGHSST_MODEL::WILCOX1998); /*!\brief KIND_TRANS_MODEL \n DESCRIPTION: Specify transition model OPTIONS: see \link Trans_Model_Map \endlink \n DEFAULT: NONE \ingroup Config*/ @@ -1311,9 +1381,9 @@ void CConfig::SetConfig_Options() { /* DESCRIPTION: Definition of the turbulent thermal conductivity model (CONSTANT_PRANDTL_TURB (default), NONE). */ addEnumOption("TURBULENT_CONDUCTIVITY_MODEL", Kind_ConductivityModel_Turb, TurbConductivityModel_Map, CONDUCTIVITYMODEL_TURB::CONSTANT_PRANDTL); - /*--- Options related to Constant Thermal Conductivity Model ---*/ + /*--- Options related to Constant Thermal Conductivity Model ---*/ - /* DESCRIPTION: default value for AIR */ + /* DESCRIPTION: default value for AIR */ addDoubleListOption("THERMAL_CONDUCTIVITY_CONSTANT", nThermal_Conductivity_Constant , Thermal_Conductivity_Constant); /*--- Options related to temperature polynomial coefficients for fluid models. ---*/ @@ -1410,15 +1480,22 @@ void CConfig::SetConfig_Options() { /*!\brief FLAME_INIT_METHOD \n DESCRIPTION: Ignition method for flamelet solver \n DEFAULT: no ignition; cold flow only. */ addEnumOption("FLAME_INIT_METHOD", flamelet_ParsedOptions.ignition_method, Flamelet_Init_Map, FLAMELET_INIT_TYPE::NONE); + /*!\brief FLAME_ENTHALPY_BC \n DESCRIPTION: enthalpy BC for thermal walls. FLOW_MARKERS (default): enthalpy derived + from MARKER_ISOTHERMAL temperature or MARKER_HEATFLUX via GetEnthFromTemp. SPECIES_MARKERS: enthalpy and all other + scalars taken directly from MARKER_WALL_SPECIES or the Python wrapper (SetMarkerCustomScalar). \n DEFAULT: FLOW_MARKERS \ingroup Config */ + addEnumOption("FLAME_ENTHALPY_BC", flamelet_ParsedOptions.enthalpy_bc, Flamelet_Enthalpy_BC_Map, FLAMELET_ENTHALPY_BC::FLOW_MARKERS); /*!\brief FLAME_INIT \n DESCRIPTION: flame front initialization using the flamelet model \ingroup Config*/ - addDoubleArrayOption("FLAME_INIT", flamelet_ParsedOptions.flame_init.size(), false, flamelet_ParsedOptions.flame_init.begin()); + addDoubleArrayOption("FLAME_INIT", flamelet_ParsedOptions.flame_init.size(), false, flamelet_ParsedOptions.flame_init.data()); /*!\brief SPARK_INIT \n DESCRIPTION: spark initialization using the flamelet model \ingroup Config*/ - addDoubleArrayOption("SPARK_INIT", flamelet_ParsedOptions.spark_init.size(), false, flamelet_ParsedOptions.spark_init.begin()); + addDoubleArrayOption("SPARK_INIT", flamelet_ParsedOptions.spark_init.size(), false, flamelet_ParsedOptions.spark_init.data()); /*!\brief SPARK_REACTION_RATES \n DESCRIPTION: Net source term values applied to species within spark area during spark ignition. \ingroup Config*/ addDoubleListOption("SPARK_REACTION_RATES", flamelet_ParsedOptions.nspark, flamelet_ParsedOptions.spark_reaction_rates); + /*!\brief THICKENED_FLAME_CORRECTION \n DESCRIPTION: Coarse grid correction for source terms and diffusive fluxes in reacting flows. \ingroup Config*/ + addBoolOption("THICKENED_FLAME_CORRECTION", flamelet_ParsedOptions.thickenedflame_correction, true); + /*!\brief FLAME_INIT_IGNITION \n DESCRIPTION: Ignition temperature for the flame initialization \ingroup Config*/ addDoubleOption("FLAME_INIT_IGNITION", flamelet_ParsedOptions.Flame_T_ignition, 5000.0); @@ -1675,6 +1752,8 @@ void CConfig::SetConfig_Options() { addStringListOption("MARKER_MIXINGPLANE_INTERFACE", nMarker_MixingPlaneInterface, Marker_MixingPlaneInterface); /*!\brief TURBULENT_MIXINGPLANE \n DESCRIPTION: Activate mixing plane also for turbulent quantities \ingroup Config*/ addBoolOption("TURBULENT_MIXINGPLANE", turbMixingPlane, false); + /*!\brief RELATIVE_FRAME_SLIDINGPLANE \n DESCRIPTION: Activate sliding plane for relative frame \ingroup Config*/ + addBoolOption("RELATIVE_FRAME_SLIDINGPLANE", RelFrame_SlidingPlane, false); /*!\brief MARKER_TURBOMACHINERY \n DESCRIPTION: Identify the boundaries for which the turbomachinery settings are applied. \ingroup Config*/ addTurboPerfOption("MARKER_TURBOMACHINERY", nMarker_Turbomachinery, Marker_TurboBoundIn, Marker_TurboBoundOut, Marker_Turbomachinery); /*!\brief NUM_SPANWISE_SECTIONS \n DESCRIPTION: Integer number of spanwise sections to compute 3D turbo BC and Performance for turbomachinery */ @@ -1815,6 +1894,11 @@ void CConfig::SetConfig_Options() { addDoubleOption("CFL_NUMBER", CFLFineGrid, 1.25); /* DESCRIPTION: Max time step in local time stepping simulations */ addDoubleOption("MAX_DELTA_TIME", Max_DeltaTime, 1000000); + /* !\brief OUTLIER_MITIGATION_PARAM + * DESCRIPTION: Parameters of the outlier mitigation strategy: start iteration, update frequency, print frequency, + * and number of standard deviations (N * sigma) to identify outliers statistically. \ingroup Config*/ + outlierMitigationParam[0] = 999999; outlierMitigationParam[1] = 5; outlierMitigationParam[2] = 2; outlierMitigationParam[3] = 5; + addULongArrayOption("OUTLIER_MITIGATION_PARAM", 4, true, outlierMitigationParam); /* DESCRIPTION: Activate The adaptive CFL number. */ addBoolOption("CFL_ADAPT", CFL_Adapt, false); /* !\brief CFL_ADAPT_PARAM @@ -1894,10 +1978,22 @@ void CConfig::SetConfig_Options() { addDoubleOption("LINEAR_SOLVER_ERROR", Linear_Solver_Error, 1E-6); /* DESCRIPTION: Maximum number of iterations of the linear solver for the implicit formulation */ addUnsignedLongOption("LINEAR_SOLVER_ITER", Linear_Solver_Iter, 10); + /*!\brief LINEAR_SOLVER + * \n DESCRIPTION: Linear solver for the poisson system \n OPTIONS: see \link Linear_Solver_Map \endlink \n DEFAULT: FGMRES \ingroup Config*/ + addEnumOption("POISSON_LINEAR_SOLVER", Kind_Poisson_Linear_Solver, Linear_Solver_Map, FGMRES); + /*!\brief LINEAR_SOLVER_PREC + * \n DESCRIPTION: Preconditioner for the Krylov linear solvers \n OPTIONS: see \link Linear_Solver_Prec_Map \endlink \n DEFAULT: LU_SGS \ingroup Config*/ + addEnumOption("POISSON_LINEAR_SOLVER_PREC", Kind_Poisson_Linear_Solver_Prec, Linear_Solver_Prec_Map, ILU); + /* DESCRIPTION: Minimum error threshold for the poisson linear solver */ + addDoubleOption("POISSON_LINEAR_SOLVER_ERROR", Poisson_Linear_Solver_Error, 1E-6); + /* DESCRIPTION: Maximum number of iterations of the poisson linear solver */ + addUnsignedLongOption("POISSON_LINEAR_SOLVER_ITER", Poisson_Linear_Solver_Iter, 10); /* DESCRIPTION: Fill in level for the ILU preconditioner */ - addUnsignedShortOption("LINEAR_SOLVER_ILU_FILL_IN", Linear_Solver_ILU_n, 0); + addUnsignedShortOption("LINEAR_SOLVER_ILU_FILL_IN", IluOptions.FillIn, 0); /* DESCRIPTION: Use level scheduling for OMP parallelization of the ILU preconditioner */ - addBoolOption("LINEAR_SOLVER_ILU_LEVEL_SCHEDULING", Linear_Solver_ILU_levels, false); + addBoolOption("LINEAR_SOLVER_ILU_LEVEL_SCHEDULING", IluOptions.LevelScheduling, false); + /* DESCRIPTION: Number of colored Gauss-Seidel sweeps used to build the GPU ILU factorization */ + addUnsignedShortOption("LINEAR_SOLVER_ILU_GPU_SWEEPS", IluOptions.GPUSweeps, 2); /* DESCRIPTION: Maximum number of iterations of the linear solver for the implicit formulation */ addUnsignedLongOption("LINEAR_SOLVER_RESTART_FREQUENCY", Linear_Solver_Restart_Frequency, 10); /* DESCRIPTION: Number of vectors used for deflated restarts */ @@ -1908,6 +2004,14 @@ void CConfig::SetConfig_Options() { addUnsignedLongOption("LINEAR_SOLVER_PREC_THREADS", Linear_Solver_Prec_Threads, 0); /* DESCRIPTION: Use an inner linear solver. */ addEnumOption("LINEAR_SOLVER_INNER", Kind_Linear_Solver_Inner, Inner_Linear_Solver_Map, LINEAR_SOLVER_INNER::NONE); + /* DESCRIPTION: Relaxation of the pressure corrections for the SIMPLE algorithm */ + addDoubleOption("RELAXATION_FACTOR_PRESSURE", SIMPLE_Options.Relaxation_Factor_Pressure, 1.0); + /* DESCRIPTION: Removal factor for the transient term in the momentum coefficients for the poisson solver. */ + addDoubleOption("TRANSIENT_TERM_REMOVAL_FACTOR", SIMPLE_Options.Transient_Term_Removal_Factor, 0.0); + /*!\DESCRIPTION: Automatically compute relaxation factors for flow corrections in the SIMPLE algorithm */ + addBoolOption("USE_AUTOMATIC_RELAXATION_FACTORS", SIMPLE_Options.AutomaticRelaxationFactors, false); + /* DESCRIPTION: Number of corrections in the PISO algorithm (pressure based). */ + addUnsignedShortOption("PISO_CORRECTIONS", SIMPLE_Options.nCorrections_PISO, 1); /* DESCRIPTION: Relaxation factor for updates of adjoint variables. */ addDoubleOption("RELAXATION_FACTOR_ADJOINT", Relaxation_Factor_Adjoint, 1.0); /* DESCRIPTION: Relaxation of the CHT coupling */ @@ -1981,16 +2085,36 @@ void CConfig::SetConfig_Options() { addDoubleOption("MG_DAMP_PROLONGATION", Damp_Correc_Prolong, 0.5); /*!\brief MG_SMOOTH_EARLY_EXIT\n DESCRIPTION: Enable early exit for MG smoothing when RMS drops below threshold. DEFAULT: NO \ingroup Config*/ addBoolOption("MG_SMOOTH_EARLY_EXIT", MGOptions.MG_Smooth_EarlyExit, true); - /*!\brief MG_SMOOTH_RES_THRESHOLD\n DESCRIPTION: Smoothing stops when current_rms < threshold * initial_rms. DEFAULT: 0.1 \ingroup Config*/ - addDoubleOption("MG_SMOOTH_RES_THRESHOLD", MGOptions.MG_Smooth_Res_Threshold, 0.5); + /*!\brief MG_SMOOTH_RES_THRESHOLD\n DESCRIPTION: Early exit smoothing when current_rms drops below threshold * initial_rms. DEFAULT: 0.9 \ingroup Config*/ + addDoubleOption("MG_SMOOTH_RES_THRESHOLD", MGOptions.MG_Smooth_Res_Threshold, 0.9); /*!\brief MG_SMOOTH_OUTPUT\n DESCRIPTION: Print compact per-cycle smoothing iteration summary. DEFAULT: NO \ingroup Config*/ addBoolOption("MG_SMOOTH_OUTPUT", MGOptions.MG_Smooth_Output, false); + /*!\brief MG_SMOOTH_STAGNATION_TOL\n DESCRIPTION: Stop smoothing if current_rms >= previous_rms * this value. Values < 1.0 enable early exit on stagnation, 1.0 only exits on defect growth. DEFAULT: 0.99 \ingroup Config*/ + addDoubleOption("MG_SMOOTH_STAGNATION_TOL", MGOptions.MG_Smooth_StagnationTol, 0.99); /*!\brief MG_SMOOTH_COEFF\n DESCRIPTION: Smoothing coefficient for the correction prolongation Jacobi smoother. DEFAULT: 1.25 \ingroup Config*/ addDoubleOption("MG_SMOOTH_COEFF", MGOptions.MG_Smooth_Coeff, 1.25); /*!\brief MG_MIN_MESHSIZE\n DESCRIPTION: Minimum number of CVs on the coarsest multigrid level. Levels that would produce fewer CVs are not created. DEFAULT: 50 \ingroup Config*/ addUnsignedLongOption("MG_MIN_MESHSIZE", MGOptions.MG_Min_MeshSize, 500); /*!\brief MG_IMPLICIT_LINES\n DESCRIPTION: Enable agglomeration along implicit lines from wall seeds. DEFAULT: NO \ingroup Config*/ addBoolOption("MG_IMPLICIT_LINES", MGOptions.MG_Implicit_Lines, false); + /*!\brief MG_IMPLICIT_LINES_MAX_LENGTH\n DESCRIPTION: Maximum number of nodes on a wall-normal implicit agglomeration line (including the wall seed node). DEFAULT: 20 \ingroup Config*/ + addUnsignedLongOption("MG_IMPLICIT_LINES_MAX_LENGTH", MGOptions.MG_Implicit_Lines_MaxLength, 20); + /*!\brief MG_STARTUP_ITER\n DESCRIPTION: Max number of iterations spent on each mesh during the Full + * Multigrid (FMG) startup phase. DEFAULT: 100 \ingroup Config*/ + addUnsignedLongOption("MG_STARTUP_ITER", MGOptions.MG_Startup_Iter, 100); + /*!\brief MG_STARTUP_CONVERGENCE\n DESCRIPTION: During the startup phase of Full-MG, leave the current level once + * CONV_FIELD has dropped by this many orders of magnitude relative to its value when the level became active. + * DEFAULT: -2 \ingroup Config*/ + addDoubleOption("MG_STARTUP_CONVERGENCE", MGOptions.MG_Startup_Convergence, -2.0); + /*!\brief MG_STARTUP_STAGNATION\n DESCRIPTION: Full-MG promotion on stagnation. If the active level's residual ratio + * between successive iterations exceeds this value for MG_STARTUP_STAGNATION_ITER consecutive iterations, promote to + * the next finer level without waiting out MG_STARTUP_ITER. 0 disables it. DEFAULT: 0.99 \ingroup Config*/ + addDoubleOption("MG_STARTUP_STAGNATION", MGOptions.MG_Startup_Stagnation, 0.99); + /*!\brief MG_STARTUP_STAGNATION_ITER\n DESCRIPTION: Consecutive stalled iterations required before Full-MG promotes + * on stagnation. 0 disables it, as MG_STARTUP_STAGNATION= 0 does. DEFAULT: 5 \ingroup Config*/ + addUnsignedLongOption("MG_STARTUP_STAGNATION_ITER", MGOptions.MG_Startup_Stagnation_Iter, 5); + /*!\brief MG_CFL_SCALING\n DESCRIPTION: Per-level CFL scaling factors for coarse MG levels. Entry i is the ratio CFL(i+1)/CFL(i). If fewer values than nMGLevels are given, the last value is repeated. DEFAULT: 0.25 (i.e., 1/4 per level) \ingroup Config*/ + addDoubleListOption("MG_CFL_SCALING", nMG_CflScaling_p, MG_CflScaling_p); /*!\par CONFIG_CATEGORY: Spatial Discretization \ingroup Config*/ /*--- Options related to the spatial discretization ---*/ @@ -2025,7 +2149,7 @@ void CConfig::SetConfig_Options() { addDoubleOption("MUSCL_KAPPA_FLOW", MUSCL_Kappa_Flow, 0.0); /*!\brief RAMP_MUSCL \n DESCRIPTION: Enable ramping of the MUSCL scheme from 1st to 2nd order using specified method*/ addBoolOption("RAMP_MUSCL", RampMUSCL, false); - /*! brief RAMP_OUTLET_COEFF \n DESCRIPTION: the 1st coeff is the ramp start iteration, + /*! brief RAMP_MUSCL_COEFF \n DESCRIPTION: the 1st coeff is the ramp start iteration, * the 2nd coeff is the iteration update frequenct, 3rd coeff is the total number of iterations */ RampMUSCLParam.rampMUSCLCoeff[0] = 0.0; RampMUSCLParam.rampMUSCLCoeff[1] = 1.0; RampMUSCLParam.rampMUSCLCoeff[2] = 500.0; addULongArrayOption("RAMP_MUSCL_COEFF", 3, false, RampMUSCLParam.rampMUSCLCoeff); @@ -2212,9 +2336,11 @@ void CConfig::SetConfig_Options() { /*!\brief ACTDISK_JUMP \n DESCRIPTION: The jump is given by the difference in values or a ratio */ addEnumOption("ACTDISK_JUMP", ActDisk_Jump, Jump_Map, DIFFERENCE); /*!\brief MESH_FORMAT \n DESCRIPTION: Mesh input file format \n OPTIONS: see \link Input_Map \endlink \n DEFAULT: SU2 \ingroup Config*/ - addEnumOption("MESH_FORMAT", Mesh_FileFormat, Input_Map, SU2); + addEnumOption("MESH_FORMAT", Mesh_FileFormat, Input_Map, ENUM_GRID::SU2); /* DESCRIPTION: Mesh input file */ addStringOption("MESH_FILENAME", Mesh_FileName, string("mesh")); + /*!\brief MESH_OUT_FORMAT \n DESCRIPTION: Mesh output file format \n OPTIONS: see \link OutputMesh_Map \endlink \n DEFAULT: SU2 \ingroup Config*/ + addEnumOption("MESH_OUT_FORMAT", Mesh_Out_FileFormat, OutputMesh_Map, ENUM_GRID::SU2); /*!\brief MESH_OUT_FILENAME \n DESCRIPTION: Mesh output file name. Used when converting, scaling, or deforming a mesh. \n DEFAULT: mesh_out \ingroup Config*/ addStringOption("MESH_OUT_FILENAME", Mesh_Out_FileName, string("mesh_out")); @@ -3102,6 +3228,9 @@ void CConfig::SetConfig_Options() { /* DESCRIPTION: Allow fallback to smaller edge color group sizes for the discrete adjoint and allow more colors. */ addBoolOption("EDGE_COLORING_RELAX_DISC_ADJ", edgeColoringRelaxDiscAdj, true); + /* DESCRIPTION: Number of concurrent BFS fronts used to build the RCM reordering (1 is standard single-seed RCM). */ + addUnsignedShortOption("RCM_NUM_SEEDS", rcmNumSeeds, 1); + /*--- options that are used for libROM ---*/ /*!\par CONFIG_CATEGORY:libROM options \ingroup Config*/ @@ -3544,6 +3673,31 @@ void CConfig::SetPostprocessing(SU2_COMPONENT val_software, unsigned short val_i Multizone_Problem = YES; } + /*--- The solver vectors stay on the device but the halo exchange is host-side, so more than + * one rank would use stale halos. Use OpenMP for the host parts instead. ---*/ + if (Enable_Cuda && size > 1) { + SU2_MPI::Error("ENABLE_CUDA= YES is not supported with more than one MPI rank,\n" + " the halo exchange only happens on the host.\n" + " Use a single rank with OpenMP threads, e.g. 'SU2_CFD -t config.cfg'.", + CURRENT_FUNCTION); + } + + /*--- nvcc cannot compile the CoDiPack types, so the kernels are only built into the primal + * solver (see SU2_ENABLE_CUDA_KERNELS). Catch it here, not minutes into the run. ---*/ + if (Enable_Cuda) { +#ifndef SU2_ENABLE_CUDA_KERNELS +#ifdef HAVE_CUDA + SU2_MPI::Error("ENABLE_CUDA= YES is not available in the AD and direct differentiation solvers,\n" + " the CUDA kernels are only built into SU2_CFD.", + CURRENT_FUNCTION); +#else + SU2_MPI::Error("ENABLE_CUDA= YES but SU2 was not compiled with CUDA support,\n" + " reconfigure the build with -Denable-cuda=true.", + CURRENT_FUNCTION); +#endif +#endif + } + /*--- Set the default output files ---*/ if (!OptionIsSet("OUTPUT_FILES")){ nVolumeOutputFiles = 3; @@ -4053,10 +4207,15 @@ void CConfig::SetPostprocessing(SU2_COMPONENT val_software, unsigned short val_i nMGLevels = 0; if (!OptionIsSet("LINEAR_SOLVER_ILU_LEVEL_SCHEDULING")) { /*--- Different default behavior for this solver type. ---*/ - Linear_Solver_ILU_levels = true; + IluOptions.LevelScheduling = true; } } + if (IluOptions.GPUSweeps == 0) { + SU2_MPI::Error("LINEAR_SOLVER_ILU_GPU_SWEEPS must be at least 1; 0 sweeps never factorizes the preconditioner.", + CURRENT_FUNCTION); + } + Radiation = (Kind_Radiation != RADIATION_MODEL::NONE); /*--- Check for unsupported features. ---*/ @@ -4065,6 +4224,46 @@ void CConfig::SetPostprocessing(SU2_COMPONENT val_software, unsigned short val_i SU2_MPI::Error("Harmonic Balance not yet implemented for the incompressible solver.", CURRENT_FUNCTION); } + /*--- The pressure-based solver's Poisson equation only runs on the finest grid, its Rhie-Chow + * mass flux has no pseudo-transient term, it has no adjoint, and its marker switches have no + * PERIODIC_BOUNDARY case. Fail here instead of silently ignoring the option or erroring deep + * inside the first iteration. Gated on the incompressible regime (not just the option's raw + * value) since KIND_INCOMP_SYSTEM is read regardless of solver family, and a compressible or + * SU2_DEF config that happens to carry a leftover PRESSURE_BASED line (e.g. copied from + * config_template.cfg before it defaulted to DENSITY_BASED) must not hard-error here. ---*/ + if (Kind_Regime == ENUM_REGIME::INCOMPRESSIBLE && Kind_Incomp_System == INCOMP_SYSTEM::PRESSURE_BASED) { + if (nMGLevels > 0) { + SU2_MPI::Error("KIND_INCOMP_SYSTEM= PRESSURE_BASED does not support MGLEVEL > 0,\n" + " the Poisson solver is single-grid only.", CURRENT_FUNCTION); + } + if (Time_Domain) { + SU2_MPI::Error("KIND_INCOMP_SYSTEM= PRESSURE_BASED does not support TIME_DOMAIN= YES,\n" + " it converges to a physically wrong solution instead of failing.", + CURRENT_FUNCTION); + } + if (DiscreteAdjoint || ContinuousAdjoint) { + SU2_MPI::Error("KIND_INCOMP_SYSTEM= PRESSURE_BASED has no adjoint formulation.", CURRENT_FUNCTION); + } + if (nMarker_PerBound > 0) { + SU2_MPI::Error("KIND_INCOMP_SYSTEM= PRESSURE_BASED does not support MARKER_PERIODIC.", CURRENT_FUNCTION); + } + if (Kind_Streamwise_Periodic != ENUM_STREAMWISE_PERIODIC::NONE) { + SU2_MPI::Error("KIND_INCOMP_SYSTEM= PRESSURE_BASED does not support streamwise periodicity.", + CURRENT_FUNCTION); + } + + /*--- A_p already carries Vol/dt when SIMPLEC's A_p-Sum_A_nb correction runs, so at the 0.0 + * default that correction collapses to roughly Vol/dt and the pressure correction becomes + * vanishingly weak at low CFL. ---*/ + if (Kind_PBIter == PBITER::SIMPLEC && !OptionIsSet("TRANSIENT_TERM_REMOVAL_FACTOR")) { + SIMPLE_Options.Transient_Term_Removal_Factor = 1.0; + if (rank == MASTER_NODE) { + cout << "WARNING: KIND_PB_ITER= SIMPLEC without TRANSIENT_TERM_REMOVAL_FACTOR set - " + << "defaulting it to 1.0, its intended companion value for SIMPLEC." << endl; + } + } + } + /*--- Check for Fluid model consistency ---*/ if (standard_air) { @@ -4246,6 +4445,10 @@ void CConfig::SetPostprocessing(SU2_COMPONENT val_software, unsigned short val_i } /* --- Check for NEMO compatibility issues ---*/ + if (nemo && Kind_Turb_Model != TURB_MODEL::NONE) { + SU2_MPI::Error("A turbulence model is not yet available for the NEMO solver.", CURRENT_FUNCTION); + } + if (Kind_FluidModel == SU2_NONEQ && (Kind_TransCoeffModel != TRANSCOEFFMODEL::WILKE && Kind_TransCoeffModel != TRANSCOEFFMODEL::SUTHERLAND && Kind_TransCoeffModel != TRANSCOEFFMODEL::GUPTAYOS) ) { SU2_MPI::Error("Transport model not available for NEMO solver using SU2TCLIB. Please use the WILKE, SUTHERLAND or GUPTAYOS transport model instead.", CURRENT_FUNCTION); } @@ -4743,6 +4946,11 @@ void CConfig::SetPostprocessing(SU2_COMPONENT val_software, unsigned short val_i } } + /*--- Only the direct problem promotes a Full-MG startup. Downgrade before FinestMesh is + * derived from the cycle, or it stays on the coarsest level for the entire run. ---*/ + + if (Restart || ((Kind_MGCycle == MG_CYCLE::FULL) && ContinuousAdjoint)) Kind_MGCycle = MG_CYCLE::V; + FinestMesh = MESH_0; if (Kind_MGCycle == MG_CYCLE::FULL) FinestMesh = nMGLevels; @@ -4785,6 +4993,16 @@ void CConfig::SetPostprocessing(SU2_COMPONENT val_software, unsigned short val_i [](unsigned short ) { return (unsigned short)0; }); fillSmooth(nMG_CorrecSmooth_p, MG_CorrecSmooth_p, MGOptions.MG_CorrecSmooth, [](unsigned short ) { return (unsigned short)0; }); + + /*--- Fill MG_CflScaling to size nMGLevels (one entry per coarse level transition). ---*/ + MGOptions.MG_CflScaling.resize(nMGLevels); + if (nMG_CflScaling_p != 0) { + for (unsigned short i = 0; i < nMGLevels; ++i) + MGOptions.MG_CflScaling[i] = (i < nMG_CflScaling_p) ? MG_CflScaling_p[i] : MG_CflScaling_p[nMG_CflScaling_p - 1]; + } else { + for (unsigned short i = 0; i < nMGLevels; ++i) + MGOptions.MG_CflScaling[i] = 0.25; + } } /*--- Override MG Smooth parameters ---*/ @@ -4794,8 +5012,6 @@ void CConfig::SetPostprocessing(SU2_COMPONENT val_software, unsigned short val_i MGOptions.MG_PostSmooth[nMGLevels] = 0; MGOptions.MG_CorrecSmooth[nMGLevels] = 0; - if (Restart) Kind_MGCycle = MG_CYCLE::V; - if (ContinuousAdjoint) { if (Kind_Solver == MAIN_SOLVER::EULER) Kind_Solver = MAIN_SOLVER::ADJ_EULER; if (Kind_Solver == MAIN_SOLVER::NAVIER_STOKES) Kind_Solver = MAIN_SOLVER::ADJ_NAVIER_STOKES; @@ -5662,6 +5878,14 @@ void CConfig::SetPostprocessing(SU2_COMPONENT val_software, unsigned short val_i Kind_Solver != MAIN_SOLVER::MULTIPHYSICS) SU2_MPI::Error("Species transport currently only available for compressible and incompressible flow.", CURRENT_FUNCTION); + /*--- The dual-time density history is recomputed via the fluid model, which needs the species + solution; the species solver only exists on the finest grid. ---*/ + if ((Kind_Regime == ENUM_REGIME::INCOMPRESSIBLE) && (Kind_DensityModel != INC_DENSITYMODEL::CONSTANT) && + (TimeMarching == TIME_MARCHING::DT_STEPPING_1ST || TimeMarching == TIME_MARCHING::DT_STEPPING_2ND) && + (nMGLevels > 0)) + SU2_MPI::Error("Dual-time stepping with species-dependent variable density does not support MGLEVEL > 0.", + CURRENT_FUNCTION); + /*--- Species specific OF currently can only handle one entry in Marker_Analyze. ---*/ for (unsigned short iObj = 0; iObj < nObj; iObj++) { if ((Kind_ObjFunc[iObj] == SURFACE_SPECIES_0 || @@ -5772,10 +5996,6 @@ void CConfig::SetPostprocessing(SU2_COMPONENT val_software, unsigned short val_i } } - if (Kind_Regime == ENUM_REGIME::COMPRESSIBLE && GetBounded_Scalar()) { - SU2_MPI::Error("BOUNDED_SCALAR discretization can only be used for incompressible problems.", CURRENT_FUNCTION); - } - } void CConfig::SetMarkers(SU2_COMPONENT val_software) { @@ -7034,6 +7254,7 @@ void CConfig::SetOutput(SU2_COMPONENT val_software, unsigned short val_izone) { case TOPOL_DISCRETENESS: cout << "Topology discreteness objective function." << endl; break; case TOPOL_COMPLIANCE: cout << "Topology compliance objective function." << endl; break; case STRESS_PENALTY: cout << "Stress penalty objective function." << endl; break; + case ENTROPY_GENERATION: cout << "Entropy generation objective function." << endl; break; } } else { @@ -7370,18 +7591,22 @@ void CConfig::SetOutput(SU2_COMPONENT val_software, unsigned short val_izone) { } } switch (Kind_Linear_Solver_Prec) { - case ILU: cout << "Using a ILU("<< Linear_Solver_ILU_n <<") preconditioning."<< endl; break; - case LINELET: cout << "Using a linelet preconditioning."<< endl; break; - case LU_SGS: cout << "Using a LU-SGS preconditioning."<< endl; break; - case JACOBI: cout << "Using a Jacobi preconditioning."<< endl; break; + case ILU: cout << "Using ILU("<< IluOptions.FillIn <<") preconditioning."<< endl; break; + case LINELET: cout << "Using linelet preconditioning."<< endl; break; + case LU_SGS: cout << "Using LU-SGS preconditioning."<< endl; break; + case Q_LU_SGS: cout << "Using LU-SGS preconditioning with matrix quantization."<< endl; break; + case JACOBI: cout << "Using Jacobi preconditioning."<< endl; break; + case Q_JACOBI: cout << "Using Jacobi preconditioning with matrix quantization."<< endl; break; } break; case SMOOTHER: switch (Kind_Linear_Solver_Prec) { - case ILU: cout << "A ILU(" << Linear_Solver_ILU_n << ")"; break; + case ILU: cout << "A ILU(" << IluOptions.FillIn << ")"; break; case LINELET: cout << "A Linelet"; break; case LU_SGS: cout << "A LU-SGS"; break; + case Q_LU_SGS: cout << "A quantized LU-SGS"; break; case JACOBI: cout << "A Jacobi"; break; + case Q_JACOBI: cout << "A quantized Jacobi"; break; } cout << " method is used for smoothing the linear system." << endl; break; @@ -7530,10 +7755,12 @@ void CConfig::SetOutput(SU2_COMPONENT val_software, unsigned short val_izone) { MGTable.AddColumn("Presmooth", 10); MGTable.AddColumn("PostSmooth", 10); MGTable.AddColumn("CorrectSmooth", 10); + MGTable.AddColumn("CFL Scaling", 12); MGTable.SetAlign(PrintingToolbox::CTablePrinter::RIGHT); MGTable.PrintHeader(); for (unsigned short iLevel = 0; iLevel < nMGLevels+1; iLevel++) { - MGTable << iLevel << MGOptions.MG_PreSmooth[iLevel] << MGOptions.MG_PostSmooth[iLevel] << MGOptions.MG_CorrecSmooth[iLevel]; + const string cflStr = (iLevel == 0) ? "-" : std::to_string(MGOptions.MG_CflScaling[iLevel-1]).substr(0,6); + MGTable << iLevel << MGOptions.MG_PreSmooth[iLevel] << MGOptions.MG_PostSmooth[iLevel] << MGOptions.MG_CorrecSmooth[iLevel] << cflStr; } MGTable.PrintFooter(); } @@ -7655,7 +7882,7 @@ void CConfig::SetOutput(SU2_COMPONENT val_software, unsigned short val_izone) { } if (val_software == SU2_COMPONENT::SU2_DEF) { - cout << "Output mesh file name: " << GetMesh_Out_FileName() << ".su2. " << endl; + cout << "Output mesh file name: " << GetMesh_Out_FileName() << GetMesh_Out_FileExtension() << ". " << endl; switch (GetDeform_Stiffness_Type()) { case INVERSE_VOLUME: cout << "Cell stiffness scaled by inverse of the cell volume." << endl; @@ -8636,6 +8863,7 @@ CConfig::~CConfig() { delete [] nBlades; delete [] FreeStreamTurboNormal; + } /*--- Input is the filename base, output is the completed filename. ---*/ @@ -8779,6 +9007,9 @@ string CConfig::GetObjFunc_Extension(string val_filename) const { case TOPOL_DISCRETENESS: AdjExt = "_topdisc"; break; case TOPOL_COMPLIANCE: AdjExt = "_topcomp"; break; case STRESS_PENALTY: AdjExt = "_stress"; break; + case ENTROPY_GENERATION: AdjExt = "_entg"; break; + case TOTAL_PRESSURE_LOSS: AdjExt = "_tot_press_loss"; break; + case KINETIC_ENERGY_LOSS: AdjExt = "_kin_en_loss"; break; } } else{ @@ -8805,6 +9036,7 @@ unsigned short CConfig::GetContainerPosition(unsigned short val_eqsystem) { case RUNTIME_ADJSPECIES_SYS:return ADJSPECIES_SOL; case RUNTIME_ADJFEA_SYS: return ADJFEA_SOL; case RUNTIME_RADIATION_SYS: return RAD_SOL; + case RUNTIME_POISSON_SYS: return POISSON_SOL; case RUNTIME_MULTIGRID_SYS: return 0; } return 0; @@ -8943,6 +9175,13 @@ void CConfig::SetGlobalParam(MAIN_SOLVER val_solver, } break; + case MAIN_SOLVER::POISSON_EQUATION: + if (val_system == RUNTIME_POISSON_SYS) { + SetKind_ConvNumScheme(NONE, CENTERED::NONE, UPWIND::NONE, LIMITER::NONE, NONE, 0.0, NONE); + SetKind_TimeIntScheme(EULER_IMPLICIT); + } + break; + case MAIN_SOLVER::FEM_ELASTICITY: case MAIN_SOLVER::DISC_ADJ_FEM: if (val_system == RUNTIME_FEA_SYS) { @@ -10079,6 +10318,23 @@ short CConfig::FindInterfaceMarker(unsigned short iInterface) const { return -1; } +short CConfig::FindMixingPlaneInterfaceMarker(unsigned short nMarker, unsigned short iMarkerInt) const { + short mark; + for (auto iMarker = 0; iMarker < nMarker; iMarker++){ + /*--- If the tag GetMarker_All_MixingPlaneInterface equals the index we are looping at ---*/ + if (GetMarker_All_MixingPlaneInterface(iMarker) == iMarkerInt){ + /*--- We have identified the local index of the marker ---*/ + /*--- Store the identifier for the marker ---*/ + mark = iMarker; + /*--- Exit the for loop: we have found the local index for Mixing-Plane interface ---*/ + return mark; + } + /*--- If the tag hasn't matched any tag within the donor markers ---*/ + mark = -1; + } + return mark; +} + void CConfig::GEMM_Tick(double *val_start_time) const { #ifdef PROFILE diff --git a/Common/src/basic_types/ad_structure.cpp b/Common/src/basic_types/ad_structure.cpp index d22ea67f373b..2e1224847976 100644 --- a/Common/src/basic_types/ad_structure.cpp +++ b/Common/src/basic_types/ad_structure.cpp @@ -48,6 +48,10 @@ SU2_OMP(threadprivate(PreaccHelper)) ExtFuncHelper FuncHelper; +#ifdef CODI_TAG_TAPE +DebugControl* current_control = NULL; +#endif // CODI_TAG_TAPE + #endif void Initialize() { diff --git a/Common/src/containers/CLookUpTable.cpp b/Common/src/containers/CLookUpTable.cpp index a9cd03d96641..50b57924d8fb 100644 --- a/Common/src/containers/CLookUpTable.cpp +++ b/Common/src/containers/CLookUpTable.cpp @@ -46,10 +46,7 @@ CLookUpTable::CLookUpTable(const string& var_file_name_lut, string name_CV1_in, FindTableLimits(name_CV1, name_CV2); - if (rank == MASTER_NODE) - cout << "Detecting all unique edges and setting edge to triangle connectivity " - "..." - << endl; + if (rank == MASTER_NODE) cout << "Detecting all unique edges and setting edge to triangle connectivity ..." << endl; IdentifyUniqueEdges(); @@ -62,16 +59,10 @@ CLookUpTable::CLookUpTable(const string& var_file_name_lut, string name_CV1_in, if (rank == MASTER_NODE) switch (table_dim) { case 2: - cout << "Building a trapezoidal map for the (" + name_CV1 + ", " + name_CV2 + - ") " - "space ..." - << endl; + cout << "Building a trapezoidal map for the (" + name_CV1 + ", " + name_CV2 + ") space ..." << endl; break; case 3: - cout << "Building trapezoidal map stack for the (" + name_CV1 + ", " + name_CV2 + - ") " - "space ..." - << endl; + cout << "Building trapezoidal map stack for the (" + name_CV1 + ", " + name_CV2 + ") space ..." << endl; break; default: break; @@ -79,38 +70,36 @@ CLookUpTable::CLookUpTable(const string& var_file_name_lut, string name_CV1_in, trap_map_x_y.resize(n_table_levels); su2double startTime = SU2_MPI::Wtime(); - unsigned short barwidth = 65; - bool display_map_info = (n_table_levels < 2); double tmap_memory_footprint = 0; + for (auto i_level = 0ul; i_level < n_table_levels; i_level++) { - trap_map_x_y[i_level] = - CTrapezoidalMap(GetDataP(name_CV1, i_level), GetDataP(name_CV2, i_level), table_data[i_level].cols(), - edges[i_level], edge_to_triangle[i_level], display_map_info); - tmap_memory_footprint += trap_map_x_y[i_level].GetMemoryFootprint(); - /* Display a progress bar to monitor table generation process */ - if (rank == MASTER_NODE) { - su2double progress = su2double(i_level) / n_table_levels; - auto completed = floor(progress * barwidth); - auto to_do = barwidth - completed; - cout << "[" << setfill('=') << setw(completed); - cout << '>'; - cout << setfill(' ') << setw(to_do) << std::right << "] " << 100 * progress << "%\r"; - cout.flush(); + const auto n_pts = n_points[i_level]; + const auto n_tris = n_triangles[i_level]; + + std::vector x_coords(n_pts); + std::vector y_coords(n_pts); + for (auto i_point = 0ul; i_point < n_pts; ++i_point) { + x_coords[i_point] = table_data[i_level][idx_CV1][i_point]; + y_coords[i_point] = table_data[i_level][idx_CV2][i_point]; + } + + std::vector tri_conn(3 * n_tris); + for (auto i_tri = 0ul; i_tri < n_tris; ++i_tri) { + tri_conn[3 * i_tri + 0] = triangles[i_level][i_tri][0]; + tri_conn[3 * i_tri + 1] = triangles[i_level][i_tri][1]; + tri_conn[3 * i_tri + 2] = triangles[i_level][i_tri][2]; } + + if (!trap_map_x_y[i_level].Build(n_pts, n_tris, x_coords.data(), y_coords.data(), tri_conn.data())) + SU2_MPI::Error( + "Construction of trapezoidal map failed for level " + std::to_string(i_level) + " of table " + file_name_lut, + CURRENT_FUNCTION); + tmap_memory_footprint += trap_map_x_y[i_level].GetMemoryFootprint(); } su2double stopTime = SU2_MPI::Wtime(); if (rank == MASTER_NODE) { - switch (table_dim) { - case 2: - cout << "\nConstruction of trapezoidal map took " << stopTime - startTime << " seconds\n" << endl; - break; - case 3: - cout << "\nConstruction of trapezoidal map stack took " << stopTime - startTime << " seconds\n" << endl; - break; - default: - break; - } + cout << "Construction of trapezoidal map took " << stopTime - startTime << " seconds\n"; cout << "Trapezoidal map memory footprint: " << tmap_memory_footprint << " MB\n"; cout << "Table data memory footprint: " << memory_footprint_data << " MB\n" << endl; } @@ -617,18 +606,8 @@ bool CLookUpTable::LookUp_XY(const vector& idx_var, vector= *limits_table_x[iLevel].first && val_CV1 <= *limits_table_x[iLevel].second) && - (val_CV2 >= *limits_table_y[iLevel].first && val_CV2 <= *limits_table_y[iLevel].second)) { - /* if so, try to find the triangle that holds the (prog, enth) point */ - id_triangle = trap_map_x_y[iLevel].GetTriangle(val_CV1, val_CV2); - - /* check if point is inside a triangle (if table domain is non-rectangular, - * the previous range check might be true but the point could still be outside of the domain) */ - return IsInTriangle(val_CV1, val_CV2, id_triangle, iLevel); - } - return false; + std::array bary_coords; + return trap_map_x_y[iLevel].FindTriangle(val_CV1, val_CV2, id_triangle, bary_coords); } void CLookUpTable::GetInterpCoeffs(su2double val_CV1, su2double val_CV2, const su2activematrix& interp_mat_inv, @@ -783,26 +762,6 @@ void CLookUpTable::InterpolateToNearestNeighbors(const su2double val_CV1, const InterpolateToNearestNeighbors(val_CV1, val_CV2, names_var, val_names_var, i_level); } -bool CLookUpTable::IsInTriangle(su2double val_CV1, su2double val_CV2, unsigned long val_id_triangle, - unsigned long i_level) { - su2double tri_x_0 = table_data[i_level][idx_CV1][triangles[i_level][val_id_triangle][0]]; - su2double tri_y_0 = table_data[i_level][idx_CV2][triangles[i_level][val_id_triangle][0]]; - - su2double tri_x_1 = table_data[i_level][idx_CV1][triangles[i_level][val_id_triangle][1]]; - su2double tri_y_1 = table_data[i_level][idx_CV2][triangles[i_level][val_id_triangle][1]]; - - su2double tri_x_2 = table_data[i_level][idx_CV1][triangles[i_level][val_id_triangle][2]]; - su2double tri_y_2 = table_data[i_level][idx_CV2][triangles[i_level][val_id_triangle][2]]; - - su2double area_tri = TriArea(tri_x_0, tri_y_0, tri_x_1, tri_y_1, tri_x_2, tri_y_2); - - su2double area_0 = TriArea(val_CV1, val_CV2, tri_x_1, tri_y_1, tri_x_2, tri_y_2); - su2double area_1 = TriArea(tri_x_0, tri_y_0, val_CV1, val_CV2, tri_x_2, tri_y_2); - su2double area_2 = TriArea(tri_x_0, tri_y_0, tri_x_1, tri_y_1, val_CV1, val_CV2); - - return (abs(area_tri - (area_0 + area_1 + area_2)) < area_tri * 1e-10); -} - bool CLookUpTable::CheckForVariables(const std::vector& vars_to_check) const { for (const string& var_to_check : vars_to_check) { if (!std::any_of(names_var.begin(), names_var.end(), diff --git a/Common/src/containers/CTrapezoidalMap.cpp b/Common/src/containers/CTrapezoidalMap.cpp deleted file mode 100644 index 75721ee749af..000000000000 --- a/Common/src/containers/CTrapezoidalMap.cpp +++ /dev/null @@ -1,283 +0,0 @@ -/*! - * \file CTrapezoidalMap.cpp - * \brief Implementation of the trapezoidal map for tabulation and lookup of fluid properties - * \author D. Mayer, T. Economon, N. Beishuizen - * \version 8.5.0 "Harrier" - * - * SU2 Project Website: https://su2code.github.io - * - * The SU2 Project is maintained by the SU2 Foundation - * (http://su2foundation.org) - * - * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) - * - * SU2 is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * SU2 is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with SU2. If not, see . - */ - -#include -#include - -#include "../../Common/include/option_structure.hpp" -#include "../../Common/include/containers/CTrapezoidalMap.hpp" - -using namespace std; - -/* Trapezoidal map implementation. Reference: - * M. de Berg, O. Cheong M. van Kreveld, M. Overmars, - * Computational Geometry, Algorithms and Applications pp. 121-146 (2008) - * NOTE: the current implementation is actually the simpler 'slab' approach. - */ -CTrapezoidalMap::CTrapezoidalMap(const su2double* samples_x, const su2double* samples_y, const unsigned long size, - vector > const& edges, - su2vector > const& val_edge_to_triangle, bool display) { - int rank = SU2_MPI::GetRank(); - su2double startTime = SU2_MPI::Wtime(); - - edge_to_triangle = su2vector >(val_edge_to_triangle); - - unique_bands_x.assign(samples_x, samples_x + size); - - /* sort x_bands and make them unique */ - sort(unique_bands_x.begin(), unique_bands_x.end()); - - auto iter = unique(unique_bands_x.begin(), unique_bands_x.end()); - - unique_bands_x.resize(distance(unique_bands_x.begin(), iter)); - - edge_limits_x.resize(edges.size(), 2); - edge_limits_y.resize(edges.size(), 2); - - /* store x and y values of each edge in a vector for a slight speed up - * as it prevents some uncoalesced accesses */ - for (unsigned long j = 0; j < edges.size(); j++) { - edge_limits_x[j][0] = samples_x[edges[j][0]]; - edge_limits_x[j][1] = samples_x[edges[j][1]]; - edge_limits_y[j][0] = samples_y[edges[j][0]]; - edge_limits_y[j][1] = samples_y[edges[j][1]]; - } - - /* number of bands */ - unsigned long n_bands_x = unique_bands_x.size() - 1; - /* band index */ - unsigned long i_band = 0; - /* number of edges */ - unsigned long n_edges = edges.size(); - /* edge index */ - unsigned long i_edge = 0; - unsigned long j_edge = 0; - /* counter for edges intersects */ - unsigned long n_intersects = 0; - /* lower and upper x value of each band */ - su2double band_lower_x = 0; - su2double band_upper_x = 0; - - su2double x_0; - su2double y_0; - su2double dy_edge; - su2double dx_edge; - su2double x_band_mid; - - /* y values of all intersecting edges for every band */ - y_edge_at_band_mid.resize(unique_bands_x.size() - 1); - - /* loop over bands */ - while (i_band < n_bands_x) { - band_lower_x = unique_bands_x[i_band]; - band_upper_x = unique_bands_x[i_band + 1]; - i_edge = 0; - n_intersects = 0; - - /* loop over edges and determine which edges appear in current band */ - while (i_edge < n_edges) { - /* check if edge intersects the band - * (vertical edges are automatically discarded) */ - if (((edge_limits_x[i_edge][0] <= band_lower_x) and (edge_limits_x[i_edge][1] >= band_upper_x)) or - ((edge_limits_x[i_edge][1] <= band_lower_x) and (edge_limits_x[i_edge][0] >= band_upper_x))) { - y_edge_at_band_mid[i_band].emplace_back(0.0, 0); - - x_0 = edge_limits_x[i_edge][0]; - y_0 = edge_limits_y[i_edge][0]; - - dy_edge = edge_limits_y[i_edge][1] - edge_limits_y[i_edge][0]; - dx_edge = edge_limits_x[i_edge][1] - edge_limits_x[i_edge][0]; - x_band_mid = (band_lower_x + band_upper_x) / 2.0; - - y_edge_at_band_mid[i_band][n_intersects].first = y_0 + dy_edge / dx_edge * (x_band_mid - x_0); - - /* save edge index so it can later be recalled when searching */ - y_edge_at_band_mid[i_band][n_intersects].second = i_edge; - - n_intersects++; - } - i_edge++; - } - - /* sort edges by their y values. - * note that these y values are unique (i.e. edges cannot - * intersect in a band) */ - sort(y_edge_at_band_mid[i_band].begin(), y_edge_at_band_mid[i_band].end()); - - i_band++; - } - - su2double stopTime = SU2_MPI::Wtime(); - - /* calculate size of trapezoidal map components */ - double size_unique_bands = sizeof(su2double) * unique_bands_x.size() / 1e6; - double size_edge_limits_x = sizeof(su2double) * edge_limits_x.size() * 2 / 1e6; - double size_edge_limits_y = sizeof(su2double) * edge_limits_y.size() * 2 / 1e6; - - double size_edge_to_triangle = 0; - for (i_edge = 0; i_edge < edge_to_triangle.size(); i_edge++) - for (j_edge = 0; j_edge < edge_to_triangle[i_edge].size(); j_edge++) - size_edge_to_triangle += sizeof(unsigned long) / 1e6; - - double size_y_edge_at_band_mid = 0; - for (unsigned long i_y = 0; i_y < y_edge_at_band_mid.size(); i_y++) - for (unsigned long j_y = 0; j_y < y_edge_at_band_mid[i_y].size(); j_y++) - size_y_edge_at_band_mid += sizeof(su2double) / 1e6 + sizeof(unsigned long) / 1e6; - - memory_footprint = - size_unique_bands + size_edge_limits_x + size_edge_limits_y + size_edge_to_triangle + size_y_edge_at_band_mid; - - /* print size of trapezoidal map components to screen */ - if ((rank == MASTER_NODE) && display) { - cout << setfill(' '); - cout << "\n" << endl; - cout << "+------------------------------------------------------------------+\n"; - cout << "| Trapezoidal map info |\n"; - cout << "+------------------------------------------------------------------+" << endl; - - cout << "| Time to construct trapezoidal map: " << setw(22) << right << stopTime - startTime << " sec" - << " |" << endl; - cout << "| Size of unique_bands in memory: " << setw(22) << size_unique_bands << " MB " - << " |" << endl; - cout << "| Size of edge_limits_x in memory: " << setw(22) << size_edge_limits_x << " MB " - << " |" << endl; - cout << "| Size of edge_limits_y in memory: " << setw(22) << size_edge_limits_y << " MB " - << " |" << endl; - cout << "| Size of edge_to_triangle in memory: " << setw(22) << size_edge_to_triangle << " MB " - << " |" << endl; - cout << "| Size of y_edge_at_band_mid in memory: " << setw(22) << size_y_edge_at_band_mid << " MB " - << " |" << endl; - cout << "| Total: " << setw(22) << memory_footprint << " MB " - << " |" << endl; - cout << "+------------------------------------------------------------------+" << endl; - cout << "\n" << endl; - } -} - -unsigned long CTrapezoidalMap::GetTriangle(const su2double val_x, const su2double val_y) { - /* find x band in which val_x sits */ - pair band = GetBand(val_x); - - /* within that band, find edges which enclose the (val_x, val_y) point */ - pair edges = GetEdges(band, val_x, val_y); - - /* identify the adjacent triangles using the two edges */ - std::array triangles_edge_low; - for (unsigned long i = 0; i < edge_to_triangle[edges.first].size(); i++) - triangles_edge_low[i] = edge_to_triangle[edges.first][i]; - - std::array triangles_edge_up; - for (unsigned long i = 0; i < edge_to_triangle[edges.second].size(); i++) - triangles_edge_up[i] = edge_to_triangle[edges.second][i]; - - sort(triangles_edge_low.begin(), triangles_edge_low.end()); - sort(triangles_edge_up.begin(), triangles_edge_up.end()); - - /* The intersection of the faces to which upper or lower belongs is the face that both belong to. */ - vector triangle; - set_intersection(triangles_edge_up.begin(), triangles_edge_up.end(), triangles_edge_low.begin(), - triangles_edge_low.end(), std::back_inserter(triangle)); - - /*--- We failed to find an intersection, so take the lower triangle inside the band enclosing the point---*/ - if (triangle.size() < 1) { - triangle.resize(1, triangles_edge_low[0]); - } - - return triangle[0]; -} - -pair CTrapezoidalMap::GetBand(const su2double val_x) { - unsigned long i_low = 0; - unsigned long i_up = 0; - su2double val_x_sample = val_x; - /* check if val_x is in x-bounds of the table, if not then project val_x to either x-min or x-max */ - if (val_x_sample < unique_bands_x.front()) val_x_sample = unique_bands_x.front(); - if (val_x_sample > unique_bands_x.back()) val_x_sample = unique_bands_x.back(); - - std::pair::iterator, std::vector::iterator> bounds; - bounds = std::equal_range(unique_bands_x.begin(), unique_bands_x.end(), val_x_sample); - - /*--- if upper bound = 0, then use the range [0,1] ---*/ - i_up = max(1, bounds.first - unique_bands_x.begin()); - i_low = i_up - 1; - - return make_pair(i_low, i_up); -} - -pair CTrapezoidalMap::GetEdges(pair val_band, - su2double val_x, su2double val_y) const { - su2double next_y; - su2double y_edge_low; - su2double y_edge_up; - su2double x_edge_low; - su2double x_edge_up; - - unsigned long i_band_low = val_band.first; - - unsigned long next_edge; - - unsigned long j_low = 0; - unsigned long j_mid = 0; - unsigned long j_up = 0; - - j_up = y_edge_at_band_mid[i_band_low].size() - 1; - j_low = 0; - - while (j_up - j_low > 1) { - j_mid = (j_up + j_low) / 2; - - // Select the edge associated with the x band (i_band_low) - // Search for the RunEdge in the y direction (second value is index of - // edge) - next_edge = y_edge_at_band_mid[i_band_low][j_mid].second; - - y_edge_low = edge_limits_y[next_edge][0]; - y_edge_up = edge_limits_y[next_edge][1]; - x_edge_low = edge_limits_x[next_edge][0]; - x_edge_up = edge_limits_x[next_edge][1]; - - // The search variable in j should be interpolated in i as well - next_y = y_edge_low + (y_edge_up - y_edge_low) / (x_edge_up - x_edge_low) * (val_x - x_edge_low); - - if (next_y > val_y) { - j_up = j_mid; - - } else if (next_y < val_y) { - j_low = j_mid; - - } else if (next_y == val_y) { - j_low = j_mid; - j_up = j_low + 1; - break; - } - } - - unsigned long edge_low = y_edge_at_band_mid[i_band_low][j_low].second; - unsigned long edge_up = y_edge_at_band_mid[i_band_low][j_up].second; - - return make_pair(edge_low, edge_up); -} diff --git a/Common/src/containers/meson.build b/Common/src/containers/meson.build index 4c8d4fe618a4..0743fd6b60c0 100644 --- a/Common/src/containers/meson.build +++ b/Common/src/containers/meson.build @@ -1,3 +1,2 @@ -common_src += files(['CTrapezoidalMap.cpp', - 'CFileReaderLUT.cpp', +common_src += files(['CFileReaderLUT.cpp', 'CLookUpTable.cpp']) diff --git a/Common/src/fem/fem_geometry_structure.cpp b/Common/src/fem/fem_geometry_structure.cpp index 4a98ca15cd45..ae71919060a2 100644 --- a/Common/src/fem/fem_geometry_structure.cpp +++ b/Common/src/fem/fem_geometry_structure.cpp @@ -37,8 +37,15 @@ /* Prototypes for Lapack functions, if MKL or LAPACK is used. */ #if defined(HAVE_MKL) || defined(HAVE_LAPACK) -extern "C" void dpotrf_(char*, int*, passivedouble*, int*, int*); -extern "C" void dpotri_(char*, int*, passivedouble*, int*, int*); +#ifdef USE_SINGLE_PRECISION +#define POTRF_IMPL spotrf_ +#define POTRI_IMPL spotri_ +#else +#define POTRF_IMPL dpotrf_ +#define POTRI_IMPL dpotri_ +#endif +extern "C" void POTRF_IMPL(char*, int*, passivedouble*, int*, int*); +extern "C" void POTRI_IMPL(char*, int*, passivedouble*, int*, int*); #endif bool CPointFEM::operator<(const CPointFEM& other) const { @@ -5361,7 +5368,7 @@ void CMeshFEM_DG::MetricTermsVolumeElements(CConfig* config) { than a standard inverse. */ char uplo = 'L'; int NN = nDOFs, errorCode; - dpotrf_(&uplo, &NN, massMat.data(), &NN, &errorCode); + POTRF_IMPL(&uplo, &NN, massMat.data(), &NN, &errorCode); if (errorCode != 0) { ostringstream message; if (errorCode < 0) { @@ -5376,7 +5383,7 @@ void CMeshFEM_DG::MetricTermsVolumeElements(CConfig* config) { SU2_MPI::Error(message.str(), CURRENT_FUNCTION); } - dpotri_(&uplo, &NN, massMat.data(), &NN, &errorCode); + POTRI_IMPL(&uplo, &NN, massMat.data(), &NN, &errorCode); if (errorCode != 0) { ostringstream message; if (errorCode < 0) { diff --git a/Common/src/geometry/CGeometry.cpp b/Common/src/geometry/CGeometry.cpp index 258e5612be89..360da1aaaa1e 100644 --- a/Common/src/geometry/CGeometry.cpp +++ b/Common/src/geometry/CGeometry.cpp @@ -4123,39 +4123,39 @@ void CGeometry::SetGridVelocity(const CConfig* config) { } } -const CCompressedSparsePatternUL& CGeometry::GetSparsePattern(ConnectivityType type, unsigned long fillLvl) { +const CGeometry::LDUSparsePattern& CGeometry::GetSparsePattern(ConnectivityType type, unsigned long fillLvl) { bool fvm = (type == ConnectivityType::FiniteVolume); - - CCompressedSparsePatternUL* pattern = nullptr; - - if (fillLvl == 0) - pattern = fvm ? &finiteVolumeCSRFill0 : &finiteElementCSRFill0; - else - pattern = fvm ? &finiteVolumeCSRFillN : &finiteElementCSRFillN; - - if (pattern->empty()) { - *pattern = buildCSRPattern(*this, type, fillLvl); - pattern->buildDiagPtr(); + auto& grp = fillLvl == 0 ? (fvm ? finiteVolumePatternFill0 : finiteElementPatternFill0) + : (fvm ? finiteVolumePatternFillN : finiteElementPatternFillN); + if (grp.empty()) { + grp.csr = buildCSRPattern(*this, type, static_cast(fillLvl)); + grp.csr.buildDiagPtr(); + grp.l = buildLowerPattern(grp.csr); + grp.u = buildUpperPattern(grp.csr); } - - return *pattern; + return grp; } -const CEdgeToNonZeroMapUL& CGeometry::GetEdgeToSparsePatternMap() { - if (edgeToCSRMap.empty()) { - if (finiteVolumeCSRFill0.empty()) { - finiteVolumeCSRFill0 = buildCSRPattern(*this, ConnectivityType::FiniteVolume, 0ul); - } - edgeToCSRMap = mapEdgesToSparsePattern(*this, finiteVolumeCSRFill0); +const su2vector& CGeometry::GetLToUTransposeSparsePatternMap(ConnectivityType type) { + bool fvm = (type == ConnectivityType::FiniteVolume); + auto& l_to_u = fvm ? finiteVolumeLToUTranspMap : finiteElementLToUTranspMap; + if (l_to_u.empty()) { + auto& u_to_l = fvm ? finiteVolumeUToLTranspMap : finiteElementUToLTranspMap; + const auto& pat = GetSparsePattern(type); + buildLUTransposeMaps(pat.l, pat.u, l_to_u, u_to_l); } - return edgeToCSRMap; + return l_to_u; } -const su2vector& CGeometry::GetTransposeSparsePatternMap(ConnectivityType type) { - /*--- Yes the const cast is weird but it is still better than repeating code. ---*/ - auto& pattern = const_cast(GetSparsePattern(type)); - pattern.buildTransposePtr(); - return pattern.transposePtr(); +const su2vector& CGeometry::GetUToLTransposeSparsePatternMap(ConnectivityType type) { + bool fvm = (type == ConnectivityType::FiniteVolume); + auto& u_to_l = fvm ? finiteVolumeUToLTranspMap : finiteElementUToLTranspMap; + if (u_to_l.empty()) { + auto& l_to_u = fvm ? finiteVolumeLToUTranspMap : finiteElementLToUTranspMap; + const auto& pat = GetSparsePattern(type); + buildLUTransposeMaps(pat.l, pat.u, l_to_u, u_to_l); + } + return u_to_l; } const CCompressedSparsePatternUL& CGeometry::GetEdgeColoring(su2double* efficiency, bool maximizeEdgeColorGroupSize) { @@ -4540,7 +4540,8 @@ su2double NearestNeighborDistance(CGeometry* geometry, const CConfig* config, co } } // namespace -void CGeometry::ComputeWallDistance(const CConfig* const* config_container, CGeometry**** geometry_container) { +void CGeometry::ComputeWallDistance(const CConfig* const* config_container, CGeometry**** geometry_container, + const int record_zone) { int nZone = config_container[ZONE_0]->GetnZone(); bool allEmpty = true; vector wallDistanceNeeded(nZone, false); @@ -4608,6 +4609,11 @@ void CGeometry::ComputeWallDistance(const CConfig* const* config_container, CGeo } for (int iZone = 0; iZone < nZone; iZone++) { + /*--- When recording for a specific zone, only compute nearest-neighbor distances (which read vertex + * normals) for that zone. Reading normals from other zones at this tape position would create + * cross-zone AD dependencies before those zones have had their geometry updated. ---*/ + if (record_zone >= 0 && iZone != record_zone) continue; + /*--- For the FEM solver, we use a different mesh structure ---*/ MAIN_SOLVER kindSolver = config_container[iZone]->GetKind_Solver(); if (!wallDistanceNeeded[iZone] || kindSolver == MAIN_SOLVER::FEM_LES || kindSolver == MAIN_SOLVER::FEM_RANS) { diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index a55c3972cfa3..684da742b130 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -896,6 +896,9 @@ void CMultiGridGeometry::SetPoint_Connectivity(const CGeometry* fine_grid) { } } + /*--- See CPhysicalGeometry::SetPoint_Connectivity for why we sort. ---*/ + sort(points[iCoarsePoint].begin(), points[iCoarsePoint].end()); + /*--- Set the number of neighbors variable, this is important for JST and multigrid in parallel ---*/ nodes->SetnNeighbor(iCoarsePoint, points[iCoarsePoint].size()); @@ -1005,7 +1008,7 @@ void CMultiGridGeometry::SetControlVolume(const CGeometry* fine_grid, unsigned s auto iFinePoint = nodes->GetChildren_CV(iCoarsePoint, iChildren); Coarse_Volume += fine_grid->nodes->GetVolume(iFinePoint); } - nodes->SetVolume(iCoarsePoint, Coarse_Volume); + nodes->SetVolume(iCoarsePoint, max(Coarse_Volume, EPS)); } /*--- Update or not the values of faces at the edge ---*/ @@ -1280,24 +1283,28 @@ su2double CMultiGridGeometry::ComputeLocalCurvature(const CGeometry* fine_grid, void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const CGeometry* fine_grid, const CConfig* config, CMultiGridQueue& MGQueue_InnerCV) { /*--- Parameters ---*/ - const su2double ANGLE_THRESHOLD_DEG = 20.0; /*!< Stop line if direction deviates more than this. */ - constexpr unsigned long MAX_LINE_LENGTH = 20; /*!< Max nodes on implicit line (including wall). */ + const su2double ANGLE_THRESHOLD_DEG = 20.0; /*!< Stop line if direction deviates more than this. */ + const unsigned long MAX_LINE_LENGTH = config->GetMGOptions().MG_Implicit_Lines_MaxLength; const su2double cos_threshold = cos(ANGLE_THRESHOLD_DEG * PI_NUMBER / 180.0); const unsigned long nPointFine = fine_grid->GetnPoint(); - /*--- Collect implicit lines starting at wall vertices. + /*--- Collect implicit lines starting at viscous (no-slip) wall vertices only. + * Seeding from non-wall boundaries (farfield, inlet, outlet, symmetry) would + * claim interior BL cells before the wall lines can reach them, leaving + * wall-seeded lines with length < 3 (discarded). Restricting to viscous walls + * ensures the boundary-layer cells are agglomerated wall-first. * Each line: [wall_node, interior_1, interior_2, ...]. * The wall node (index 0) is already agglomerated by boundary agglomeration; * only interior nodes (index >= 1) are paired into coarse CVs. ---*/ vector> lines; for (auto iMarker = 0u; iMarker < fine_grid->GetnMarker(); iMarker++) { - /*--- Skip non-physical markers ---*/ - if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE || - config->GetMarker_All_KindBC(iMarker) == INTERNAL_BOUNDARY || - config->GetMarker_All_KindBC(iMarker) == NEARFIELD_BOUNDARY) - continue; + /*--- Only seed lines from viscous (no-slip) wall markers. + * Non-wall boundaries (farfield, inlet, outlet, symmetry) must NOT seed + * lines because they would prematurely claim boundary-layer interior nodes. ---*/ + const auto bc = config->GetMarker_All_KindBC(iMarker); + if (bc != HEAT_FLUX && bc != ISOTHERMAL && bc != CHT_WALL_INTERFACE && bc != SMOLUCHOWSKI_MAXWELL) continue; for (auto iVertex = 0ul; iVertex < fine_grid->GetnVertex(iMarker); iVertex++) { const auto iPoint = fine_grid->vertex[iMarker][iVertex]->GetNode(); diff --git a/Common/src/geometry/CPhysicalGeometry.cpp b/Common/src/geometry/CPhysicalGeometry.cpp index 53720985fe26..0f5dbc59afb7 100644 --- a/Common/src/geometry/CPhysicalGeometry.cpp +++ b/Common/src/geometry/CPhysicalGeometry.cpp @@ -33,6 +33,8 @@ #include "../../include/toolboxes/geometry_toolbox.hpp" #include "../../include/geometry/meshreader/CSU2ASCIIMeshReaderFEM.hpp" #include "../../include/geometry/meshreader/CSU2ASCIIMeshReaderFVM.hpp" +#include "../../include/geometry/meshreader/CSU2BinaryMeshReaderFEM.hpp" +#include "../../include/geometry/meshreader/CSU2BinaryMeshReaderFVM.hpp" #include "../../include/geometry/meshreader/CCGNSMeshReaderFVM.hpp" #include "../../include/geometry/meshreader/CCGNSMeshReaderFEM.hpp" #include "../../include/geometry/meshreader/CRectangularMeshReaderFEM.hpp" @@ -50,6 +52,8 @@ #include "../../include/geometry/primal_grid/CPrism.hpp" #include "../../include/geometry/primal_grid/CVertexMPI.hpp" +#include "../../../Common/include/tracy_structure.hpp" + #include #include #include @@ -80,6 +84,7 @@ CPhysicalGeometry::CPhysicalGeometry(CConfig* config, unsigned short val_iZone, switch (val_format) { case SU2: + case SU2_BIN: case CGNS_GRID: case RECTANGLE: case BOX: @@ -3460,6 +3465,12 @@ void CPhysicalGeometry::Read_Mesh(CConfig* config, const string& val_mesh_filena else Mesh = new CSU2ASCIIMeshReaderFVM(config, val_iZone, val_nZone); break; + case SU2_BIN: + if (fem_solver) + Mesh = new CSU2BinaryMeshReaderFEM(config, val_iZone, val_nZone); + else + Mesh = new CSU2BinaryMeshReaderFVM(config, val_iZone, val_nZone); + break; case CGNS_GRID: if (fem_solver) Mesh = new CCGNSMeshReaderFEM(config, val_iZone, val_nZone); @@ -4463,6 +4474,11 @@ void CPhysicalGeometry::SetPoint_Connectivity() { } } + /*--- Sort the neighbors in ascending order so that the edge numbering done in + * SetEdges matches the upper-CSR ordering of the sparse pattern. This makes + * the edge->upper-block map the identity for the CSysMatrix LDU storage. ---*/ + sort(points[iPoint].begin(), points[iPoint].end()); + /*--- Set the number of neighbors variable, this is important for JST and multigrid in parallel. ---*/ nodes->SetnNeighbor(iPoint, points[iPoint].size()); } @@ -4491,9 +4507,15 @@ void CPhysicalGeometry::SetRCM_Ordering(CConfig* config) { InQueue[iPoint] = true; } + const auto numSeeds = std::max(1, config->GetRCM_NumSeeds()); + constexpr auto unreached = std::numeric_limits::max(); + vector dist; + if (numSeeds > 1) dist.assign(nPoint, unreached); + vector component, bfsQueue; + /*--- Repeat as many times as necessary to handle disconnected graphs. ---*/ while (Result.size() < nPointDomain) { - /*--- Select the node with the lowest degree in the grid. ---*/ + /*--- Select the node with the lowest degree in the grid as the first seed. ---*/ auto AddPoint = nPoint; auto MinDegree = std::numeric_limits::max(); for (auto iPoint = 0ul; iPoint < nPointDomain; iPoint++) { @@ -4507,11 +4529,54 @@ void CPhysicalGeometry::SetRCM_Ordering(CConfig* config) { SU2_MPI::Error("RCM ordering failed", CURRENT_FUNCTION); } - /*--- Seed the queue with the minimum degree node. ---*/ - Result.push_back(AddPoint); - InQueue[AddPoint] = true; + /*--- Farthest-point sampling: grow the seed set with up to numSeeds-1 more points, each + * the node with the largest BFS distance (within this connected component) from every + * seed picked so far. Starting the RCM growth from several spread-out fronts instead of + * one bounds the number of levels by the covering radius of the seed set rather than the + * full component diameter, while keeping the RCM ordering local (and hence bandwidth and + * ILU quality) around each front. + * The distance from each point to its nearest seed is maintained incrementally: the first + * seed does a full BFS of the component, each later seed only relaxes the points it is + * strictly closer to than all previous seeds, keeping the total work close to a single + * BFS instead of one BFS per seed. ---*/ + vector Seeds(1, AddPoint); + if (numSeeds > 1) { + auto relaxFrom = [&](unsigned long seed) { + dist[seed] = 0; + bfsQueue.clear(); + bfsQueue.push_back(seed); + for (auto iBfs = 0ul; iBfs < bfsQueue.size(); ++iBfs) { + const auto iPoint = bfsQueue[iBfs]; + for (auto iNode = 0u; iNode < nodes->GetnPoint(iPoint); iNode++) { + const auto jPoint = nodes->GetPoint(iPoint, iNode); + if (!InQueue[jPoint] && dist[iPoint] + 1 < dist[jPoint]) { + dist[jPoint] = dist[iPoint] + 1; + bfsQueue.push_back(jPoint); + } + } + } + }; + relaxFrom(AddPoint); + /*--- The first BFS reaches exactly the connected component of the seed. ---*/ + component = bfsQueue; + for (auto iSeed = 1u; iSeed < numSeeds; ++iSeed) { + auto farthest = AddPoint; + for (const auto iPoint : component) + if (dist[iPoint] > dist[farthest]) farthest = iPoint; + /*--- The component is already fully covered by the existing seeds. ---*/ + if (dist[farthest] == 0) break; + Seeds.push_back(farthest); + relaxFrom(farthest); + } + } + + /*--- Seed the queue with all selected fronts. ---*/ + for (auto seed : Seeds) { + Result.push_back(seed); + InQueue[seed] = true; + } - /*--- Loop until reorganizing all nodes connected to AddPoint. This will + /*--- Loop until reorganizing all nodes connected to the seeds. This will * also terminate early once the ordering + queue include all points. ---*/ while (QueueStart < Result.size() && Result.size() < nPointDomain) { /*--- Move the start of the queue, equivalent to taking from the front of @@ -5682,9 +5747,9 @@ void CPhysicalGeometry::SetTurboVertex(CConfig* config, unsigned short val_iZone } } if (marker_flag == INFLOW) { - multizone_filename = "TURBOMACHINERY/spanwise_division_inflow.dat"; + multizone_filename = "TURBOMACHINERY/spanwise_division_inflow"; } else { - multizone_filename = "TURBOMACHINERY/spanwise_division_outflow.dat"; + multizone_filename = "TURBOMACHINERY/spanwise_division_outflow"; } char buffer[50]; @@ -7456,10 +7521,8 @@ void CPhysicalGeometry::ComputeMeshQualityStatistics(const CConfig* config) { /*--- Compute the angle between the unit normal associated with the edge and the unit vector pointing from iPoint to jPoint. ---*/ - su2double dotProduct = 0.0; - for (unsigned short iDim = 0; iDim < nDim; iDim++) { - dotProduct += (Normal[iDim] / area) * (edgeVector[iDim] / distance); - } + su2double dotProduct = GeometryToolbox::DotProduct(nDim, Normal, edgeVector.data()); + dotProduct = min(max(-1.0, dotProduct / (area * distance)), 1.0); /*--- The definition of orthogonality is an area-weighted average of 90 degrees minus the angle between the face area unit normal and @@ -7875,8 +7938,8 @@ void CPhysicalGeometry::SetSensitivity(CConfig* config) { char str_buf[CGNS_STRING_SIZE], fname[100]; unsigned short iVar; strcpy(fname, filename.c_str()); - int nRestart_Vars = 5, nFields; - int* Restart_Vars = new int[5]; + int nRestart_Vars = SU2_RESTART_HEADER_SIZE, nFields; + int* Restart_Vars = new int[SU2_RESTART_HEADER_SIZE]; passivedouble* Restart_Data = nullptr; int Restart_Iter = 0; passivedouble Restart_Meta_Passive[8] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; @@ -7906,7 +7969,7 @@ void CPhysicalGeometry::SetSensitivity(CConfig* config) { /*--- Check that this is an SU2 binary file. SU2 binary files have the hex representation of "SU2" as the first int in the file. ---*/ - if (Restart_Vars[0] != 535532) { + if (Restart_Vars[0] != SU2_RESTART_MAGIC_NUMBER) { SU2_MPI::Error(string("File ") + string(fname) + string(" is not a binary SU2 restart file.\n") + string("SU2 reads/writes binary restart files by default.\n") + string("Note that backward compatibility for ASCII restart files is\n") + @@ -7914,9 +7977,11 @@ void CPhysicalGeometry::SetSensitivity(CConfig* config) { CURRENT_FUNCTION); } - /*--- Store the number of fields for simplicity. ---*/ + /*--- Store the number of fields for simplicity. The file may have been written by + a build of different precision, in which case the data needs to be converted. ---*/ nFields = Restart_Vars[1]; + const int scalarSize = GetSU2BinaryScalarSize(Restart_Vars[SU2_RESTART_PRECISION_IDX]); /*--- Read the variable names from the file. Note that we are adopting a fixed length of 33 for the string length to match with CGNS. This is @@ -7938,28 +8003,45 @@ void CPhysicalGeometry::SetSensitivity(CConfig* config) { /*--- Read in the data for the restart at all local points. ---*/ - ret = fread(Restart_Data, sizeof(passivedouble), nFields * GetnPointDomain(), fhw); - if (ret != static_cast(nFields) * GetnPointDomain()) { + const unsigned long nScalars = static_cast(nFields) * GetnPointDomain(); + + if (scalarSize == static_cast(sizeof(passivedouble))) { + ret = fread(Restart_Data, scalarSize, nScalars, fhw); + } else { + vector buffer(nScalars * scalarSize); + ret = fread(buffer.data(), scalarSize, nScalars, fhw); + SU2BinaryDataToPassive(buffer.data(), scalarSize, nScalars, Restart_Data); + } + if (ret != nScalars) { SU2_MPI::Error("Error reading restart file.", CURRENT_FUNCTION); } - /*--- Compute (negative) displacements and grab the metadata. ---*/ + /*--- Grab the metadata trailer, which only old files have. Without it the iteration + number and the metadata keep the zeros they were initialized with. ---*/ - ret = sizeof(int) + 8 * sizeof(passivedouble); - fseek(fhw, -ret, SEEK_END); + const int nMeta = + GetSU2BinaryMetadataSize(Restart_Vars[SU2_RESTART_PRECISION_IDX], Restart_Vars[SU2_RESTART_METADATA_IDX]); + if (nMeta > 0) { + /*--- Compute (negative) displacements and jump to the trailer. ---*/ - /*--- Read the external iteration. ---*/ + ret = sizeof(int) + nMeta * scalarSize; + fseek(fhw, -ret, SEEK_END); - ret = fread(&Restart_Iter, sizeof(int), 1, fhw); - if (ret != 1) { - SU2_MPI::Error("Error reading restart file.", CURRENT_FUNCTION); - } + /*--- Read the external iteration. ---*/ - /*--- Read the metadata. ---*/ + ret = fread(&Restart_Iter, sizeof(int), 1, fhw); + if (ret != 1) { + SU2_MPI::Error("Error reading restart file.", CURRENT_FUNCTION); + } - ret = fread(Restart_Meta_Passive, sizeof(passivedouble), 8, fhw); - if (ret != 8) { - SU2_MPI::Error("Error reading restart file.", CURRENT_FUNCTION); + /*--- Read the metadata. ---*/ + + double meta_buf[SU2_RESTART_MAX_METADATA]; /*--- Correctly aligned for either precision. ---*/ + ret = fread(meta_buf, scalarSize, nMeta, fhw); + if (ret != static_cast(nMeta)) { + SU2_MPI::Error("Error reading restart file.", CURRENT_FUNCTION); + } + SU2BinaryDataToPassive(meta_buf, scalarSize, nMeta, Restart_Meta_Passive); } /*--- Close the file. ---*/ @@ -8002,7 +8084,7 @@ void CPhysicalGeometry::SetSensitivity(CConfig* config) { /*--- Check that this is an SU2 binary file. SU2 binary files have the hex representation of "SU2" as the first int in the file. ---*/ - if (Restart_Vars[0] != 535532) { + if (Restart_Vars[0] != SU2_RESTART_MAGIC_NUMBER) { SU2_MPI::Error(string("File ") + string(fname) + string(" is not a binary SU2 restart file.\n") + string("SU2 reads/writes binary restart files by default.\n") + string("Note that backward compatibility for ASCII restart files is\n") + @@ -8010,9 +8092,11 @@ void CPhysicalGeometry::SetSensitivity(CConfig* config) { CURRENT_FUNCTION); } - /*--- Store the number of fields for simplicity. ---*/ + /*--- Store the number of fields for simplicity. The file may have been written by + a build of different precision, in which case the data needs to be converted. ---*/ nFields = Restart_Vars[1]; + const int scalarSize = GetSU2BinaryScalarSize(Restart_Vars[SU2_RESTART_PRECISION_IDX]); /*--- Read the variable names from the file. Note that we are adopting a fixed length of 33 for the string length to match with CGNS. This is @@ -8046,9 +8130,12 @@ void CPhysicalGeometry::SetSensitivity(CConfig* config) { delete[] mpi_str_buf; - /*--- We're writing only su2doubles in the data portion of the file. ---*/ + /*--- The data portion of the file holds scalars of the precision recorded in the + header, which is not necessarily that of this build. Describe them as opaque + blocks of bytes so that the file views do not depend on the build precision. ---*/ - etype = MPI_DOUBLE; + MPI_Type_contiguous(scalarSize, MPI_BYTE, &etype); + MPI_Type_commit(&etype); /*--- We need to ignore the 4 ints describing the nVar_Restart and nPoints, along with the string names of the variables. ---*/ @@ -8066,11 +8153,11 @@ void CPhysicalGeometry::SetSensitivity(CConfig* config) { for (iPoint_Global = 0; iPoint_Global < GetGlobal_nPointDomain(); iPoint_Global++) { if (GetGlobal_to_Local_Point(iPoint_Global) > -1) { blocklen[counter] = nFields; - displace[counter] = iPoint_Global * nFields * sizeof(passivedouble); + displace[counter] = iPoint_Global * nFields * scalarSize; counter++; } } - MPI_Type_create_hindexed(GetnPointDomain(), blocklen, displace, MPI_DOUBLE, &filetype); + MPI_Type_create_hindexed(GetnPointDomain(), blocklen, displace, etype, &filetype); MPI_Type_commit(&filetype); /*--- Set the view for the MPI file write, i.e., describe the location in @@ -8082,31 +8169,45 @@ void CPhysicalGeometry::SetSensitivity(CConfig* config) { Restart_Data = new passivedouble[nFields * GetnPointDomain()]; - /*--- Collective call for all ranks to read from their view simultaneously. ---*/ + /*--- Collective call for all ranks to read from their view simultaneously, + converting the data if the file precision does not match this build. ---*/ - MPI_File_read_all(fhw, Restart_Data, nFields * GetnPointDomain(), MPI_DOUBLE, &status); + const unsigned long nScalars = static_cast(nFields) * GetnPointDomain(); + + if (scalarSize == static_cast(sizeof(passivedouble))) { + MPI_File_read_all(fhw, Restart_Data, nScalars, etype, &status); + } else { + vector buffer(nScalars * scalarSize); + MPI_File_read_all(fhw, buffer.data(), nScalars, etype, &status); + SU2BinaryDataToPassive(buffer.data(), scalarSize, nScalars, Restart_Data); + } - /*--- Free the derived datatype. ---*/ + /*--- Free the derived datatypes. ---*/ MPI_Type_free(&filetype); + MPI_Type_free(&etype); /*--- Reset the file view before writing the metadata. ---*/ MPI_File_set_view(fhw, 0, MPI_BYTE, MPI_BYTE, (char*)"native", MPI_INFO_NULL); - /*--- Access the metadata. ---*/ + /*--- Access the metadata trailer, which only old files have. Without it the iteration + number and the metadata keep the zeros they were initialized with. ---*/ - if (rank == MASTER_NODE) { + const int nMeta = + GetSU2BinaryMetadataSize(Restart_Vars[SU2_RESTART_PRECISION_IDX], Restart_Vars[SU2_RESTART_METADATA_IDX]); + if (nMeta > 0 && rank == MASTER_NODE) { /*--- External iteration. ---*/ disp = (nRestart_Vars * sizeof(int) + nFields * CGNS_STRING_SIZE * sizeof(char) + - nFields * Restart_Vars[2] * sizeof(passivedouble)); + static_cast(nFields) * Restart_Vars[2] * scalarSize); MPI_File_read_at(fhw, disp, &Restart_Iter, 1, MPI_INT, MPI_STATUS_IGNORE); /*--- Additional doubles for AoA, AoS, etc. ---*/ - disp = (nRestart_Vars * sizeof(int) + nFields * CGNS_STRING_SIZE * sizeof(char) + - nFields * Restart_Vars[2] * sizeof(passivedouble) + 1 * sizeof(int)); - MPI_File_read_at(fhw, disp, Restart_Meta_Passive, 8, MPI_DOUBLE, MPI_STATUS_IGNORE); + disp += sizeof(int); + double meta_buf[SU2_RESTART_MAX_METADATA]; /*--- Correctly aligned for either precision. ---*/ + MPI_File_read_at(fhw, disp, meta_buf, nMeta * scalarSize, MPI_BYTE, MPI_STATUS_IGNORE); + SU2BinaryDataToPassive(meta_buf, scalarSize, nMeta, Restart_Meta_Passive); } /*--- Communicate metadata. ---*/ @@ -8214,7 +8315,7 @@ void CPhysicalGeometry::SetSensitivity(CConfig* config) { /*--- Check that this is an SU2 binary file. SU2 binary files have the hex representation of "SU2" as the first int in the file. ---*/ - if (magic_number == 535532) { + if (magic_number == SU2_RESTART_MAGIC_NUMBER) { SU2_MPI::Error(string("File ") + string(fname) + string(" is a binary SU2 restart file, expected ASCII.\n") + string("SU2 reads/writes binary restart files by default.\n") + string("Note that backward compatibility for ASCII restart files is\n") + @@ -8252,7 +8353,7 @@ void CPhysicalGeometry::SetSensitivity(CConfig* config) { /*--- Check that this is an SU2 binary file. SU2 binary files have the hex representation of "SU2" as the first int in the file. ---*/ - if (magic_number == 535532) { + if (magic_number == SU2_RESTART_MAGIC_NUMBER) { SU2_MPI::Error(string("File ") + string(fname) + string(" is a binary SU2 restart file, expected ASCII.\n") + string("SU2 reads/writes binary restart files by default.\n") + string("Note that backward compatibility for ASCII restart files is\n") + @@ -10161,7 +10262,6 @@ void CPhysicalGeometry::SetWallDistance(CADTElemClass* WallADT, const CConfig* c if (!WallADT->IsEmpty()) { /*--- Solid wall boundary nodes are present. Compute the wall distance for all nodes. ---*/ - SU2_OMP_PARALLEL { CPHYSGEO_PARFOR for (unsigned long iPoint = 0; iPoint < GetnPoint(); ++iPoint) { diff --git a/Common/src/geometry/meshreader/CSU2ASCIIMeshReaderBase.cpp b/Common/src/geometry/meshreader/CSU2ASCIIMeshReaderBase.cpp index 5a9a6da1fb46..a0eb84174560 100644 --- a/Common/src/geometry/meshreader/CSU2ASCIIMeshReaderBase.cpp +++ b/Common/src/geometry/meshreader/CSU2ASCIIMeshReaderBase.cpp @@ -30,10 +30,7 @@ CSU2ASCIIMeshReaderBase::CSU2ASCIIMeshReaderBase(CConfig* val_config, unsigned short val_iZone, unsigned short val_nZone) - : CMeshReaderBase(val_config, val_iZone, val_nZone), - myZone(val_iZone), - nZones(val_nZone), - meshFilename(config->GetMesh_FileName()) {} + : CSU2MeshReaderBase(val_config, val_iZone, val_nZone) {} CSU2ASCIIMeshReaderBase::~CSU2ASCIIMeshReaderBase(void) = default; diff --git a/Common/src/geometry/meshreader/CSU2BinaryMeshReaderBase.cpp b/Common/src/geometry/meshreader/CSU2BinaryMeshReaderBase.cpp new file mode 100644 index 000000000000..9e0e9c02a65f --- /dev/null +++ b/Common/src/geometry/meshreader/CSU2BinaryMeshReaderBase.cpp @@ -0,0 +1,501 @@ +/*! + * \file CSU2BinaryMeshReaderBase.cpp + * \brief Helper class for the reading of a native SU2 binary grid file. + * \author T. Economon, E. van der Weide + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#include "../../../include/toolboxes/CLinearPartitioner.hpp" +#include "../../../include/geometry/meshreader/CSU2BinaryMeshReaderBase.hpp" + +#include +#include + +CSU2BinaryMeshReaderBase::CSU2BinaryMeshReaderBase(CConfig* val_config, unsigned short val_iZone, + unsigned short val_nZone) + : CSU2MeshReaderBase(val_config, val_iZone, val_nZone) {} + +CSU2BinaryMeshReaderBase::~CSU2BinaryMeshReaderBase(void) = default; + +int CSU2BinaryMeshReaderBase::FileSeek64(FILE* file, int64_t offset, int whence) { +#if defined(_WIN32) + return _fseeki64(file, offset, whence); +#else + return fseeko(file, static_cast(offset), whence); +#endif +} + +int64_t CSU2BinaryMeshReaderBase::FileTell64(FILE* file) { +#if defined(_WIN32) + return _ftelli64(file); +#else + return static_cast(ftello(file)); +#endif +} + +void CSU2BinaryMeshReaderBase::ReadConnectivityType() { + /*--- Initialize the byte swapping to false and + read the size of the connectivity type. ---*/ + swap_bytes = false; + ReadBinaryData(&size_conn_type, 1); + + /*--- Check if byte swapping must be applied. ---*/ + if ((size_conn_type != 4) && (size_conn_type != 8)) { + SwapBytes((char*)&size_conn_type, sizeof(int), 1); + swap_bytes = true; + } + + /*--- The size of the connectivity type must be either 4 or 8. ---*/ + if ((size_conn_type != 4) && (size_conn_type != 8)) + SU2_MPI::Error(string("The file ") + meshFilename + string(" is not a valid SU2 binary file"), CURRENT_FUNCTION); +} + +void CSU2BinaryMeshReaderBase::ReadMetadata(CConfig* config) { + const bool harmonic_balance = config->GetTime_Marching() == TIME_MARCHING::HARMONIC_BALANCE; + const bool multizone_file = config->GetMultizone_Mesh(); + + /*--- Open the grid file and check if it went OK. ---*/ + mesh_file = fopen(meshFilename.c_str(), "rb"); + if (!mesh_file) + SU2_MPI::Error( + string("Error opening SU2 binary grid file ") + meshFilename + string(". Check that the file exists"), + CURRENT_FUNCTION); + + /*--- Read the size of the connectivity type and check if byte swapping + must be applied. ---*/ + ReadConnectivityType(); + + /*--- Check for a harmonic balance simulation. If So, the data of the + first zone can be read. Otherwise jump to the location of the + current zone. ---*/ + if (harmonic_balance) { + if (rank == MASTER_NODE) cout << "Reading time instance " << config->GetiInst() + 1 << "." << endl; + FileSeek64(mesh_file, 2 * sizeof(int), SEEK_SET); + } else { + FastForwardToMyZone(); + if (nZones > 1 && multizone_file) { + if (rank == MASTER_NODE) cout << "Reading zone " << myZone << " from native SU2 binary mesh." << endl; + } + } + + /*--- Read the meta data from the current position. ---*/ + ReadMetadataZone(true); + + /*--- Close the grid file again. ---*/ + fclose(mesh_file); +} + +void CSU2BinaryMeshReaderBase::ReadPointCoordinates() { + /* No support yet for actuator disks */ + if (actuator_disk) SU2_MPI::Error("No support for actuator disks yet", CURRENT_FUNCTION); + + /* Jump over the number of points, because it is already known, and + determine the position in the file where the point section ends. */ + FileSeek64(mesh_file, size_conn_type, SEEK_CUR); + auto pos_end_point = FileTell64(mesh_file) + numberOfGlobalPoints * (dimension * sizeof(double) + size_conn_type); + + /* Define a linear partitioner for the points. */ + CLinearPartitioner pointPartitioner(numberOfGlobalPoints, 0); + + /* Jump to the position in the file where the points are stored + that this rank must read. */ + const auto firstIndex = pointPartitioner.GetFirstIndexOnRank(rank); + FileSeek64(mesh_file, firstIndex * (dimension * sizeof(double) + size_conn_type), SEEK_CUR); + + /* Determine the number of local points and prepare the local + data structure to store the point coordinates. */ + numberOfLocalPoints = pointPartitioner.GetSizeOnRank(rank); + localPointCoordinates.resize(dimension); + for (int k = 0; k < dimension; k++) localPointCoordinates[k].resize(numberOfLocalPoints); + + /*--- Read the point coordinates into our data structure. ---*/ + for (unsigned long i = 0; i < numberOfLocalPoints; ++i) { + double Coords[3]; + ReadBinaryData(Coords, dimension); + FileSeek64(mesh_file, size_conn_type, SEEK_CUR); + + for (unsigned short iDim = 0; iDim < dimension; iDim++) { + localPointCoordinates[iDim][i] = Coords[iDim]; + } + } + + /* Jump to the end of the coordinate section. */ + FileSeek64(mesh_file, pos_end_point, SEEK_SET); +} + +void CSU2BinaryMeshReaderBase::ReadVolumeElementConnectivity() { + /* Jump over the zone ID, number of dimensions and number of elements, + because this information is already known. */ + FileSeek64(mesh_file, size_conn_type + 2 * sizeof(int), SEEK_CUR); + + /* Get a linear partitioner of the elements. */ + CLinearPartitioner elemPartitioner(numberOfGlobalElements, 0); + + /* Determine the position at the end of the offset array, where + the total size of the connectivity is stored. */ + const auto first_index = elemPartitioner.GetFirstIndexOnRank(rank); + const auto pos_size_global_conn = FileTell64(mesh_file) + numberOfGlobalElements * size_conn_type; + + /* Jump to position in the file where the offset data is stored for + the element range this rank will read. Allocate the memory for + this offset array. */ + FileSeek64(mesh_file, first_index * size_conn_type, SEEK_CUR); + numberOfLocalElements = elemPartitioner.GetSizeOnRank(rank); + vector offset(numberOfLocalElements + 1); + + /* Read the offset array from the file. It will be stored as uint64_t, + but it may be stored differently in the file. */ + if (size_conn_type == 4) { + vector tmp(offset.size()); + ReadBinaryData(tmp.data(), tmp.size()); + for (size_t i = 0; i < tmp.size(); ++i) offset[i] = static_cast(tmp[i]); + } else { + ReadBinaryData(offset.data(), offset.size()); + } + + /* Jump to the location where the total size of the connectivity array + is stored and read the size. Determine the location of the end of + the connectivity array. */ + FileSeek64(mesh_file, pos_size_global_conn, SEEK_SET); + const auto size_global_conn = ReadBinaryNEntities(); + const auto pos_end_conn = FileTell64(mesh_file) + size_global_conn * size_conn_type; + + /* Read the connectivity data of the elements this rank should read. + Store the data in uint64_t. */ + const auto size_conn = offset.back() - offset[0]; + vector conn_buff(size_conn); + FileSeek64(mesh_file, offset[0] * size_conn_type, SEEK_CUR); + + if (size_conn_type == 4) { + vector tmp(conn_buff.size()); + ReadBinaryData(tmp.data(), tmp.size()); + for (size_t i = 0; i < tmp.size(); ++i) conn_buff[i] = static_cast(tmp[i]); + } else { + ReadBinaryData(conn_buff.data(), conn_buff.size()); + } + + /* Jump to the end of the connectivity data. */ + FileSeek64(mesh_file, pos_end_conn, SEEK_SET); + +#ifdef HAVE_MPI + + /* Update the offset, such that it corresponds to the data in the + local connectivity buffer. */ + for (size_t i = 1; i < offset.size(); ++i) offset[i] -= offset[0]; + offset[0] = 0; + + /* Get a linear partitioner of the points. */ + CLinearPartitioner pointPartitioner(numberOfGlobalPoints, 0); + + /*--- Determine the ranks on which the elements must actually be stored. + Note that an element can be stored on multiple ranks, as the points + must be surrounded by all its elements. ---*/ + std::vector ranks_elements; + ranks_elements.reserve(numberOfLocalElements); + std::vector number_of_ranks_elements(numberOfLocalElements + 1); + number_of_ranks_elements[0] = 0; + + for (unsigned long i = 0; i < numberOfLocalElements; ++i) { + /* Determine the ranks where this element must be stored by looping + over its nodes. */ + set ranks_this_elem; + for (uint64_t j = offset[i] + 1; j < (offset[i + 1] - 1); ++j) { + auto rank_node = pointPartitioner.GetRankContainingIndex(conn_buff[j]); + ranks_this_elem.insert(static_cast(rank_node)); + } + + /* Store the data. */ + number_of_ranks_elements[i + 1] = number_of_ranks_elements[i] + ranks_this_elem.size(); + for (auto rank_elem : ranks_this_elem) ranks_elements.push_back(rank_elem); + } + + /* Create the send buffers. Both the size of each connectivity + information and the connectivity information itself is stored.*/ + std::vector> send_buf; + send_buf.resize(size); + + for (unsigned long i = 0; i < numberOfLocalElements; ++i) { + for (uint64_t j = number_of_ranks_elements[i]; j < number_of_ranks_elements[i + 1]; ++j) { + const int ii = ranks_elements[j]; + + auto size_this_conn = offset[i + 1] - offset[i]; + send_buf[ii].push_back(size_this_conn); + for (uint64_t k = offset[i]; k < offset[i + 1]; ++k) send_buf[ii].push_back(conn_buff[k]); + } + } + + /* Determine the number of ranks from which this rank will receive data. + Allow for self communication. */ + int nRankRecv; + vector sendToRank(size, 0), sizeSend(size, 1); + for (auto rank_elem : ranks_elements) sendToRank[rank_elem] = 1; + SU2_MPI::Reduce_scatter(sendToRank.data(), &nRankRecv, sizeSend.data(), MPI_INT, MPI_SUM, SU2_MPI::GetComm()); + + /* Explicitly delete the memory that is not needed anymore. */ + vector().swap(offset); + vector().swap(conn_buff); + vector().swap(sizeSend); + + /* Determine the number of ranks to which this rank will send data + and allocate the memory for the send requests. */ + int nRankSend = 0; + for (int i = 0; i < size; ++i) { + if (sendToRank[i]) ++nRankSend; + } + + vector sendReqs(nRankSend); + + /* Send the data using non-blocking sends. */ + nRankSend = 0; + for (int i = 0; i < size; ++i) { + if (sendToRank[i]) { + SU2_MPI::Isend(send_buf[i].data(), send_buf[i].size(), MPI_UNSIGNED_LONG, i, i, SU2_MPI::GetComm(), + &sendReqs[nRankSend]); + ++nRankSend; + } + } + + /* Define the receive buffers and receive the messages. */ + std::vector> recv_buf; + recv_buf.resize(size); + + for (int i = 0; i < nRankRecv; ++i) { + SU2_MPI::Status status; + SU2_MPI::Probe(MPI_ANY_SOURCE, rank, SU2_MPI::GetComm(), &status); + int rankRecv = status.MPI_SOURCE; + + int sizeMess; + SU2_MPI::Get_count(&status, MPI_UNSIGNED_LONG, &sizeMess); + recv_buf[rankRecv].resize(sizeMess); + SU2_MPI::Recv(recv_buf[rankRecv].data(), sizeMess, MPI_UNSIGNED_LONG, rankRecv, rank, SU2_MPI::GetComm(), &status); + } + + /* Complete the non-blocking sends and release the memory of the send buffers. */ + SU2_MPI::Waitall(nRankSend, sendReqs.data(), MPI_STATUSES_IGNORE); + for (int i = 0; i < size; ++i) { + if (sendToRank[i]) vector().swap(send_buf[i]); + } + + /* Synchronize the MPI ranks, because wild cards have been used. */ + SU2_MPI::Barrier(SU2_MPI::GetComm()); + + /*--- Store the information in the receive buffers in the offset and conn_buff + vectors, such that it is consistent with the information without MPI. + Release the memory of the receive buffers afterwards. ---*/ + offset.push_back(0); + for (int i = 0; i < size; ++i) { + if (recv_buf[i].size() > 0) { + size_t ind = 0; + while (ind < recv_buf[i].size()) { + auto n_items = recv_buf[i][ind++]; + offset.push_back(offset.back() + n_items); + for (unsigned long j = 0; j < n_items; ++j, ++ind) conn_buff.push_back(recv_buf[i][ind]); + } + vector().swap(recv_buf[i]); + } + } + +#endif + + /*--- Extract the connectivity data from conn_buf and store the data + in the appropriate member variables. ---*/ + numberOfLocalElements = offset.size() - 1; + array connectivity{}; + + for (unsigned long i = 0; i < numberOfLocalElements; ++i) { + auto ind = offset[i]; + auto size_this_elem = static_cast(offset[i + 1] - offset[i]); + auto VTK_Type = conn_buff[ind++]; + const auto nPointsElem = nPointsOfElementType(static_cast(VTK_Type)); + if (size_this_elem != (nPointsElem + 2)) + SU2_MPI::Error("Wrong number of items in volume connectivity", CURRENT_FUNCTION); + + for (unsigned short j = 0; j < nPointsElem; ++j, ++ind) connectivity[j] = conn_buff[ind]; + auto GlobalIndex = conn_buff[ind]; + + localVolumeElementConnectivity.push_back(GlobalIndex); + localVolumeElementConnectivity.push_back(VTK_Type); + /// TODO: Use a compressed format. + for (unsigned short j = 0; j < N_POINTS_HEXAHEDRON; ++j) { + localVolumeElementConnectivity.push_back(connectivity[j]); + } + } +} + +void CSU2BinaryMeshReaderBase::ReadSurfaceElementConnectivity() { + /* The number of surface markers is already known, so jump over it. */ + FileSeek64(mesh_file, sizeof(int), SEEK_CUR); + + /* Allocate the memory for the first index of the connectivity of the + surface elements and the marker names. Note that all ranks store + the entire surface connectivity. */ + surfaceElementConnectivity.resize(numberOfMarkers); + markerNames.resize(numberOfMarkers); + + array connectivity{}; + + /* Loop over the number of markers. */ + for (unsigned long iMarker = 0; iMarker < numberOfMarkers; ++iMarker) { + /* Read the name of the surface marker. */ + char charStr[SU2_STRING_SIZE]; + ReadBinaryData(charStr, SU2_STRING_SIZE); + charStr[SU2_STRING_SIZE - 1] = '\0'; + markerNames[iMarker] = string(charStr); + + /*--- Throw an error if we find deprecated references to SEND_RECEIVE + boundaries in the mesh. ---*/ + if (markerNames[iMarker] == "SEND_RECEIVE") + SU2_MPI::Error( + "Mesh file contains deprecated SEND_RECEIVE marker!\n" + "Please remove any SEND_RECEIVE markers from the SU2 binary mesh.", + CURRENT_FUNCTION); + + /* Read the number of elements for this boundary marker. */ + const auto nElem_Bound = ReadBinaryNEntities(); + + /*--- Read the offset array from the file. It will be stored as + uint64_t, but it may be stored differently in the file. ---*/ + vector offset(nElem_Bound + 1); + if (size_conn_type == 4) { + vector tmp(offset.size()); + ReadBinaryData(tmp.data(), tmp.size()); + for (size_t i = 0; i < tmp.size(); ++i) offset[i] = static_cast(tmp[i]); + } else { + ReadBinaryData(offset.data(), offset.size()); + } + + /*--- Read the connectivity and store it in a buffer. + Always use uint64_t for this internally. ---*/ + vector conn_buff(offset.back()); + if (size_conn_type == 4) { + vector tmp(conn_buff.size()); + ReadBinaryData(tmp.data(), tmp.size()); + for (size_t i = 0; i < tmp.size(); ++i) conn_buff[i] = static_cast(tmp[i]); + } else { + ReadBinaryData(conn_buff.data(), conn_buff.size()); + } + + /*--- Loop over the surface elements to store the connectivity + in the required data structures. ---*/ + for (unsigned long i = 0; i < nElem_Bound; ++i) { + auto ind = offset[i]; + auto size_this_elem = static_cast(offset[i + 1] - offset[i]); + auto VTK_Type = conn_buff[ind++]; + const auto nPointsElem = nPointsOfElementType(static_cast(VTK_Type)); + if (size_this_elem != (nPointsElem + 1)) + SU2_MPI::Error("Wrong number of items in surface connectivity", CURRENT_FUNCTION); + + if (dimension == 3 && VTK_Type == LINE) { + SU2_MPI::Error( + "Line boundary conditions are not possible for 3D calculations.\n" + "Please check the SU2 binary file.", + CURRENT_FUNCTION); + } + + for (unsigned short j = 0; j < nPointsElem; ++j, ++ind) connectivity[j] = conn_buff[ind]; + + surfaceElementConnectivity[iMarker].push_back(0); + surfaceElementConnectivity[iMarker].push_back(VTK_Type); + for (unsigned short j = 0; j < N_POINTS_HEXAHEDRON; ++j) { + surfaceElementConnectivity[iMarker].push_back(connectivity[j]); + } + } + } +} + +void CSU2BinaryMeshReaderBase::FastForwardToMyZone() { + /*--- Jump to the position where the data starts for the first zone. ---*/ + FileSeek64(mesh_file, 2 * sizeof(int), SEEK_SET); + + /*--- If there is only a single zone, or if this file holds a mesh that is + shared by all problem zones (MULTIZONE_MESH= NO), there is nothing to + skip: every zone reads the same (first) block of data, exactly as the + ASCII reader does. ---*/ + if (nZones == 1 || !config->GetMultizone_Mesh()) return; + + /*--- Loop over the lower numbered zones, skipping their metadata (without + overwriting the current zone's, which is read separately afterwards + by ReadMetadata()). ---*/ + for (int zone = 0; zone < myZone; ++zone) ReadMetadataZone(false); +} + +uint64_t CSU2BinaryMeshReaderBase::ReadBinaryNEntities() { + /*--- Define the return value as an uint64_t. ---*/ + uint64_t nEntities; + + /*--- Read the actual data, depending on the connectivity type. ---*/ + if (size_conn_type == 4) { + uint32_t dummy; + ReadBinaryData(&dummy, 1); + nEntities = static_cast(dummy); + } else { + ReadBinaryData(&nEntities, 1); + } + + return nEntities; +} + +void CSU2BinaryMeshReaderBase::ReadMetadataZone(bool storeMetadata) { + /*--- Skip the zone ID and read the number of dimensions. storeMetadata is + false when this call is only being used to skip over a lower-numbered + zone's data (from FastForwardToMyZone): the file-position arithmetic + below still has to run, but its results must not overwrite the + members already holding the current (target) zone's metadata. ---*/ + int nDim; + FileSeek64(mesh_file, sizeof(int), SEEK_CUR); + ReadBinaryData(&nDim, 1); + if (storeMetadata) dimension = static_cast(nDim); + + /*--- Read the number of elements. ---*/ + const auto nElem = ReadBinaryNEntities(); + if (storeMetadata) numberOfGlobalElements = static_cast(nElem); + + /*--- Jump to the end of the offset section, read the size of + the connectivity and jump over it. ---*/ + FileSeek64(mesh_file, nElem * size_conn_type, SEEK_CUR); + const auto size_conn = ReadBinaryNEntities(); + FileSeek64(mesh_file, size_conn * size_conn_type, SEEK_CUR); + + /*--- Read the number of points and jump over the coordinate section. ---*/ + const auto nPoints = ReadBinaryNEntities(); + if (storeMetadata) numberOfGlobalPoints = static_cast(nPoints); + FileSeek64(mesh_file, nPoints * (nDim * sizeof(double) + size_conn_type), SEEK_CUR); + + /*--- Read the number of markers and loop over them. ---*/ + int nMark; + ReadBinaryData(&nMark, 1); + if (storeMetadata) numberOfMarkers = static_cast(nMark); + + for (int mark = 0; mark < nMark; ++mark) { + /*--- Jump over the name of the marker and read + the number of surface elements. ---*/ + FileSeek64(mesh_file, SU2_STRING_SIZE * sizeof(char), SEEK_CUR); + const auto nElemMark = ReadBinaryNEntities(); + + /*--- Jump to the end of the offset section of this marker, read + the size of the connectivity and jump over it. ---*/ + FileSeek64(mesh_file, nElemMark * size_conn_type, SEEK_CUR); + const auto size_conn_mark = ReadBinaryNEntities(); + FileSeek64(mesh_file, size_conn_mark * size_conn_type, SEEK_CUR); + } +} diff --git a/Common/src/geometry/meshreader/CSU2BinaryMeshReaderFEM.cpp b/Common/src/geometry/meshreader/CSU2BinaryMeshReaderFEM.cpp new file mode 100644 index 000000000000..cbbc4826e9d1 --- /dev/null +++ b/Common/src/geometry/meshreader/CSU2BinaryMeshReaderFEM.cpp @@ -0,0 +1,73 @@ +/*! + * \file CSU2BinaryMeshReaderFEM.cpp + * \brief Reads a native SU2 binary grid into linear partitions for the + * finite element solver (FEM). + * \author T. Economon, E. van der Weide + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#include "../../../include/toolboxes/CLinearPartitioner.hpp" +#include "../../../include/geometry/meshreader/CSU2BinaryMeshReaderFEM.hpp" +#include "../../../include/fem/fem_standard_element.hpp" + +CSU2BinaryMeshReaderFEM::CSU2BinaryMeshReaderFEM(CConfig* val_config, unsigned short val_iZone, + unsigned short val_nZone) + : CSU2BinaryMeshReaderBase(val_config, val_iZone, val_nZone) { + /* Read the basic metadata and perform some basic error checks. */ + ReadMetadata(val_config); + + /*--- Open the file with the mesh and go to the place where the data + of the current zone is stored. ---*/ + mesh_file = fopen(meshFilename.c_str(), "rb"); + if (!mesh_file) + SU2_MPI::Error( + string("Error opening SU2 binary grid file ") + meshFilename + string(". Check that the file exists"), + CURRENT_FUNCTION); + FastForwardToMyZone(); + + /*--- Read the volume connectivity and distribute it + linearly over the MPI ranks. ---*/ + ReadVolumeElementConnectivity(); + + /*--- Read the coordinates of the points that are needed + on this MPI rank. ---*/ + ReadPointCoordinates(); + + /*--- Read the surface connectivity and store the surface elements whose + corresponding volume element is stored on this MPI rank. ---*/ + ReadSurfaceElementConnectivity(); + + fclose(mesh_file); +} + +CSU2BinaryMeshReaderFEM::~CSU2BinaryMeshReaderFEM() = default; + +void CSU2BinaryMeshReaderFEM::ReadPointCoordinates() { SU2_MPI::Error("Not implemented yet", CURRENT_FUNCTION); } + +void CSU2BinaryMeshReaderFEM::ReadVolumeElementConnectivity() { + SU2_MPI::Error("Not implemented yet", CURRENT_FUNCTION); +} + +void CSU2BinaryMeshReaderFEM::ReadSurfaceElementConnectivity() { + SU2_MPI::Error("Not implemented yet", CURRENT_FUNCTION); +} diff --git a/Common/src/geometry/meshreader/CSU2BinaryMeshReaderFVM.cpp b/Common/src/geometry/meshreader/CSU2BinaryMeshReaderFVM.cpp new file mode 100644 index 000000000000..4c235416ebdc --- /dev/null +++ b/Common/src/geometry/meshreader/CSU2BinaryMeshReaderFVM.cpp @@ -0,0 +1,66 @@ +/*! + * \file CSU2BinaryMeshReaderFVM.cpp + * \brief Reads a native SU2 binary grid into linear partitions for the + * finite volume solver (FVM). + * \author T. Economon + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#include "../../../include/geometry/meshreader/CSU2BinaryMeshReaderFVM.hpp" + +CSU2BinaryMeshReaderFVM::CSU2BinaryMeshReaderFVM(CConfig* val_config, unsigned short val_iZone, + unsigned short val_nZone) + : CSU2BinaryMeshReaderBase(val_config, val_iZone, val_nZone) { + actuator_disk = (((config->GetnMarker_ActDiskInlet() != 0) || (config->GetnMarker_ActDiskOutlet() != 0)) && + ((config->GetKind_SU2() == SU2_COMPONENT::SU2_CFD) || + ((config->GetKind_SU2() == SU2_COMPONENT::SU2_DEF) && (config->GetActDisk_SU2_DEF())))); + if (config->GetActDisk_DoubleSurface()) actuator_disk = false; + + /* Read the basic metadata and perform some basic error checks. */ + ReadMetadata(val_config); + + /* If the mesh contains an actuator disk as a single surface, + we need to first split the surface into repeated points and update + the connectivity for each element touching the surface. */ + if (actuator_disk) SplitActuatorDiskSurface(); + + /* Read and store the points, interior elements, and surface elements. + We store only the points and interior elements on our rank's linear + partition, but the master stores the entire set of surface connectivity. */ + mesh_file = fopen(meshFilename.c_str(), "rb"); + if (!mesh_file) + SU2_MPI::Error( + string("Error opening SU2 binary grid file ") + meshFilename + string(". Check that the file exists"), + CURRENT_FUNCTION); + + FastForwardToMyZone(); + ReadVolumeElementConnectivity(); + ReadPointCoordinates(); + ReadSurfaceElementConnectivity(); + + fclose(mesh_file); +} + +CSU2BinaryMeshReaderFVM::~CSU2BinaryMeshReaderFVM() = default; + +void CSU2BinaryMeshReaderFVM::SplitActuatorDiskSurface() { SU2_MPI::Error("Not implemented yet", CURRENT_FUNCTION); } diff --git a/Common/src/geometry/meshreader/CSU2MeshReaderBase.cpp b/Common/src/geometry/meshreader/CSU2MeshReaderBase.cpp new file mode 100644 index 000000000000..752d9cbdfd0d --- /dev/null +++ b/Common/src/geometry/meshreader/CSU2MeshReaderBase.cpp @@ -0,0 +1,37 @@ +/*! + * \file CSU2MeshReaderBase.cpp + * \brief Helper class for the reading of a native SU2 grid file. + * \author T. Economon + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#include "../../../include/toolboxes/CLinearPartitioner.hpp" +#include "../../../include/geometry/meshreader/CSU2MeshReaderBase.hpp" + +CSU2MeshReaderBase::CSU2MeshReaderBase(CConfig* val_config, unsigned short val_iZone, unsigned short val_nZone) + : CMeshReaderBase(val_config, val_iZone, val_nZone), + myZone(val_iZone), + nZones(val_nZone), + meshFilename(config->GetMesh_FileName()) {} + +CSU2MeshReaderBase::~CSU2MeshReaderBase(void) = default; diff --git a/Common/src/geometry/meshreader/meson.build b/Common/src/geometry/meshreader/meson.build index 543bdfcf97a7..d6c8342960c9 100644 --- a/Common/src/geometry/meshreader/meson.build +++ b/Common/src/geometry/meshreader/meson.build @@ -9,4 +9,8 @@ common_src += files(['CBoxMeshReaderFEM.cpp', 'CRectangularMeshReaderFVM.cpp', 'CSU2ASCIIMeshReaderBase.cpp', 'CSU2ASCIIMeshReaderFEM.cpp', - 'CSU2ASCIIMeshReaderFVM.cpp']) + 'CSU2ASCIIMeshReaderFVM.cpp', + 'CSU2BinaryMeshReaderBase.cpp', + 'CSU2BinaryMeshReaderFEM.cpp', + 'CSU2BinaryMeshReaderFVM.cpp', + 'CSU2MeshReaderBase.cpp']) diff --git a/Common/src/grid_movement/CSurfaceMovement.cpp b/Common/src/grid_movement/CSurfaceMovement.cpp index 77f5110540e9..6eb1b09ab409 100644 --- a/Common/src/grid_movement/CSurfaceMovement.cpp +++ b/Common/src/grid_movement/CSurfaceMovement.cpp @@ -3902,7 +3902,7 @@ void CSurfaceMovement::SetAirfoil(CGeometry* boundary, CConfig* config) { su2double *VarCoord, *Coord, NewYCoord, NewXCoord, *Coord_i, *Coord_ip1, yp1, ypn, Airfoil_Coord[2] = {0.0, 0.0}, factor, coeff = 10000, Upper, Lower, Arch = 0.0, TotalArch = 0.0, x_i, x_ip1, y_i, y_ip1; - passivedouble AirfoilScale; + double AirfoilScale; vector Svalue, Xcoord, Ycoord, Xcoord2, Ycoord2, Xcoord_Aux, Ycoord_Aux; bool AddBegin = true, AddEnd = true; char AirfoilFile[256], AirfoilFormat[15], MeshOrientation[15], AirfoilClose[15]; @@ -5046,7 +5046,7 @@ void CSurfaceMovement::WriteFFDInfo(CSurfaceMovement** surface_movement, CGeomet if (rank == MASTER_NODE) { /*--- Read the name of the output file ---*/ - auto str = config[ZONE_0]->GetMesh_Out_FileName() + ".su2"; + auto str = config[ZONE_0]->GetMesh_Out_FileName() + config[ZONE_0]->GetMesh_Out_FileExtension(); output_file.precision(15); output_file.open(str, ios::out | ios::app); diff --git a/Common/src/interface_interpolation/CInterpolatorFactory.cpp b/Common/src/interface_interpolation/CInterpolatorFactory.cpp index f17d2d3af941..4364b44e16bc 100644 --- a/Common/src/interface_interpolation/CInterpolatorFactory.cpp +++ b/Common/src/interface_interpolation/CInterpolatorFactory.cpp @@ -32,11 +32,12 @@ #include "../../include/interface_interpolation/CNearestNeighbor.hpp" #include "../../include/interface_interpolation/CRadialBasisFunction.hpp" #include "../../include/interface_interpolation/CSlidingMesh.hpp" +#include "../../include/interface_interpolation/CMixingPlane.hpp" namespace CInterpolatorFactory { CInterpolator* CreateInterpolator(CGeometry**** geometry_container, const CConfig* const* config, const CInterpolator* transpInterpolator, unsigned iZone, unsigned jZone, - bool verbose) { + bool mixing_plane, bool verbose) { CInterpolator* interpolator = nullptr; /*--- Only print information on master node. ---*/ @@ -47,36 +48,41 @@ CInterpolator* CreateInterpolator(CGeometry**** geometry_container, const CConfi if (verbose) cout << " Setting coupling "; - /*--- Conservative interpolation is not applicable to the sliding - * mesh approach so that case is handled first. Then we either - * return a CMirror if the target requires conservative inter- - * polation, or the type of interpolator defined by "type". ---*/ + if (mixing_plane) { + if (verbose) cout << "using a mixing plane interpolation." << endl; + interpolator = new CMixingPlane(geometry_container, config, iZone, jZone); + } else { // Really awful thing to do + /*--- Conservative interpolation is not applicable to the sliding + * mesh approach so that case is handled first. Then we either + * return a CMirror if the target requires conservative inter- + * polation, or the type of interpolator defined by "type". ---*/ - if (type == INTERFACE_INTERPOLATOR::WEIGHTED_AVERAGE) { - if (verbose) cout << "using a sliding mesh approach." << endl; - interpolator = new CSlidingMesh(geometry_container, config, iZone, jZone); - } else if (config[jZone]->GetConservativeInterpolation()) { - if (verbose) cout << "using the mirror approach, \"transposing\" coefficients from opposite mesh." << endl; - interpolator = new CMirror(geometry_container, config, transpInterpolator, iZone, jZone); - } else { - switch (type) { - case INTERFACE_INTERPOLATOR::ISOPARAMETRIC: - if (verbose) cout << "using the isoparametric approach." << endl; - interpolator = new CIsoparametric(geometry_container, config, iZone, jZone); - break; + if (type == INTERFACE_INTERPOLATOR::WEIGHTED_AVERAGE) { + if (verbose) cout << "using a sliding mesh approach." << endl; + interpolator = new CSlidingMesh(geometry_container, config, iZone, jZone); + } else if (config[jZone]->GetConservativeInterpolation()) { + if (verbose) cout << "using the mirror approach, \"transposing\" coefficients from opposite mesh." << endl; + interpolator = new CMirror(geometry_container, config, transpInterpolator, iZone, jZone); + } else { + switch (type) { + case INTERFACE_INTERPOLATOR::ISOPARAMETRIC: + if (verbose) cout << "using the isoparametric approach." << endl; + interpolator = new CIsoparametric(geometry_container, config, iZone, jZone); + break; - case INTERFACE_INTERPOLATOR::NEAREST_NEIGHBOR: - if (verbose) cout << "using a nearest neighbor approach." << endl; - interpolator = new CNearestNeighbor(geometry_container, config, iZone, jZone); - break; + case INTERFACE_INTERPOLATOR::NEAREST_NEIGHBOR: + if (verbose) cout << "using a nearest neighbor approach." << endl; + interpolator = new CNearestNeighbor(geometry_container, config, iZone, jZone); + break; - case INTERFACE_INTERPOLATOR::RADIAL_BASIS_FUNCTION: - if (verbose) cout << "using a radial basis function approach." << endl; - interpolator = new CRadialBasisFunction(geometry_container, config, iZone, jZone); - break; + case INTERFACE_INTERPOLATOR::RADIAL_BASIS_FUNCTION: + if (verbose) cout << "using a radial basis function approach." << endl; + interpolator = new CRadialBasisFunction(geometry_container, config, iZone, jZone); + break; - default: - SU2_MPI::Error("Unknown type of interpolation.", CURRENT_FUNCTION); + default: + SU2_MPI::Error("Unknown type of interpolation.", CURRENT_FUNCTION); + } } } diff --git a/Common/src/interface_interpolation/CIsoparametric.cpp b/Common/src/interface_interpolation/CIsoparametric.cpp index 83e737e7a72a..797cbd0a9c7a 100644 --- a/Common/src/interface_interpolation/CIsoparametric.cpp +++ b/Common/src/interface_interpolation/CIsoparametric.cpp @@ -37,7 +37,7 @@ using namespace GeometryToolbox; CIsoparametric::CIsoparametric(CGeometry**** geometry_container, const CConfig* const* config, unsigned int iZone, unsigned int jZone) : CInterpolator(geometry_container, config, iZone, jZone) { - SetTransferCoeff(config); + SetTransferCoeff(geometry_container, config); } void CIsoparametric::PrintStatistics() const { @@ -46,7 +46,7 @@ void CIsoparametric::PrintStatistics() const { << " Interpolation clipped for " << ErrorCounter << " (" << ErrorRate << "%) target vertices." << endl; } -void CIsoparametric::SetTransferCoeff(const CConfig* const* config) { +void CIsoparametric::SetTransferCoeff(CGeometry**** geometry, const CConfig* const* config) { const su2double matchingVertexTol = 1e-12; // 1um^2 const int nProcessor = size; diff --git a/Common/src/interface_interpolation/CMirror.cpp b/Common/src/interface_interpolation/CMirror.cpp index ed4236ce6a02..36f4b3ed64ae 100644 --- a/Common/src/interface_interpolation/CMirror.cpp +++ b/Common/src/interface_interpolation/CMirror.cpp @@ -40,10 +40,10 @@ CMirror::CMirror(CGeometry**** geometry_container, const CConfig* const* config, to_string(iZone) + string(" and ") + to_string(jZone) + string("."), CURRENT_FUNCTION); } - SetTransferCoeff(config); + SetTransferCoeff(geometry_container, config); } -void CMirror::SetTransferCoeff(const CConfig* const* config) { +void CMirror::SetTransferCoeff(CGeometry**** geometry, const CConfig* const* config) { const int nProcessor = size; vector allNumVertexTarget(nProcessor); diff --git a/Common/src/interface_interpolation/CMixingPlane.cpp b/Common/src/interface_interpolation/CMixingPlane.cpp new file mode 100644 index 000000000000..d44b598c4aee --- /dev/null +++ b/Common/src/interface_interpolation/CMixingPlane.cpp @@ -0,0 +1,239 @@ +/*! + * \file CMixingPlane.cpp + * \brief Implementation of mixing plane interpolation methods. + * \author J. Kelly + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#include "../../include/interface_interpolation/CMixingPlane.hpp" +#include "../../include/CConfig.hpp" +#include "../../include/geometry/CGeometry.hpp" +#include "../../include/toolboxes/geometry_toolbox.hpp" + +CMixingPlane::CMixingPlane(CGeometry**** geometry_container, const CConfig* const* config, unsigned int iZone, + unsigned int jZone) + : CInterpolator(geometry_container, config, iZone, jZone) { + SetTransferCoeff(geometry_container, config); +} + +void CMixingPlane::SetTransferCoeff(CGeometry**** geometry, const CConfig* const* config) { + const auto nMarkerInt = config[donorZone]->GetnMarker_MixingPlaneInterface() / 2; + const auto nDim = donor_geometry->GetnDim(); + + const auto donor_config = config[donorZone]; + const auto target_config = config[targetZone]; + + const auto donor_geometry = geometry[donorZone][INST_0][MESH_0]; + const auto target_geometry = geometry[targetZone][INST_0][MESH_0]; + + // TODO turbo this approach only works if all the turboamchinery marker + // of all zones have the same amount of span wise sections. + // TODO turbo initialization needed for the MPI routine should be place somewhere else. + auto nSpanDonor = donor_config->GetnSpanWiseSections(); + auto nSpanTarget = target_config->GetnSpanWiseSections(); + + targetSpans.resize(config[donorZone]->GetnMarker_MixingPlaneInterface()); + + /*--- On the donor side ---*/ + for (auto iMarkerInt = 1; iMarkerInt < nMarkerInt + 1; iMarkerInt++) { + int markDonor = -1, markTarget = -1; + short donorFlag = 0, targetFlag = 0; + + markDonor = donor_config->FindMixingPlaneInterfaceMarker(donor_geometry->GetnMarker(), iMarkerInt); + donorFlag = (markDonor != -1) ? donor_config->GetMarker_All_MixingPlaneInterface(markDonor) : -1; + + markTarget = target_config->FindMixingPlaneInterfaceMarker(target_geometry->GetnMarker(), iMarkerInt); + targetFlag = (markTarget != -1) ? target_config->GetMarker_All_MixingPlaneInterface(markTarget) : -1; + +#ifdef HAVE_MPI + auto buffMarkerDonor = new int[size]; + auto buffDonorFlag = new int[size]; + auto buffMarkerTarget = new int[size]; + auto buffTargetFlag = new int[size]; + for (int iSize = 0; iSize < size; iSize++) { + buffMarkerDonor[iSize] = -1; + buffDonorFlag[iSize] = -1; + buffMarkerTarget[iSize] = -1; + buffTargetFlag[iSize] = -1; + } + + SU2_MPI::Allgather(&markDonor, 1, MPI_INT, buffMarkerDonor, 1, MPI_INT, SU2_MPI::GetComm()); + SU2_MPI::Allgather(&donorFlag, 1, MPI_INT, buffDonorFlag, 1, MPI_INT, SU2_MPI::GetComm()); + SU2_MPI::Allgather(&markTarget, 1, MPI_INT, buffMarkerTarget, 1, MPI_INT, SU2_MPI::GetComm()); + SU2_MPI::Allgather(&targetFlag, 1, MPI_INT, buffTargetFlag, 1, MPI_INT, SU2_MPI::GetComm()); + + markDonor = -1; + donorFlag = -1; + markTarget = -1; + targetFlag = -1; + + for (int iSize = 0; iSize < size; iSize++) { + if (buffMarkerDonor[iSize] != -1) { + markDonor = buffMarkerDonor[iSize]; + donorFlag = buffDonorFlag[iSize]; + break; + } + } + + for (int iSize = 0; iSize < size; iSize++) { + if (buffMarkerTarget[iSize] != -1) { + markTarget = buffMarkerTarget[iSize]; + targetFlag = buffTargetFlag[iSize]; + break; + } + } + delete[] buffMarkerDonor; + delete[] buffDonorFlag; + delete[] buffMarkerTarget; + delete[] buffTargetFlag; +#endif + if (markTarget == -1 || markDonor == -1) continue; + + nSpanDonor = donor_config->GetnSpanWiseSections(); + nSpanTarget = target_config->GetnSpanWiseSections(); + + targetSpans[iMarkerInt].resize(nSpanTarget + 1); + + const auto spanValuesDonor = donor_geometry->GetSpanWiseValue(donorFlag); + const auto spanValuesTarget = target_geometry->GetSpanWiseValue(targetFlag); + + /*--- Interpolation at hub, shroud & 1D values ---*/ + targetSpans[iMarkerInt][0].donorSpan = 0; + targetSpans[iMarkerInt][0].coefficient = 0.0; + if (nDim > 2) { + targetSpans[iMarkerInt][nSpanTarget - 1].donorSpan = nSpanDonor - 1; + targetSpans[iMarkerInt][nSpanTarget - 1].coefficient = 0.0; + } + targetSpans[iMarkerInt][nSpanTarget].donorSpan = nSpanDonor; + targetSpans[iMarkerInt][nSpanTarget].coefficient = 0.0; + + for (auto iSpanTarget = 1; iSpanTarget < nSpanTarget - 1; iSpanTarget++) { + auto& targetSpan = targetSpans[iMarkerInt][iSpanTarget]; + + switch (donor_config->GetKind_MixingPlaneInterface()) { + case MATCHING: + targetSpan = MapMatchingSpan(iSpanTarget); + break; + + case NEAREST_SPAN: + targetSpan = MapNearestSpan(spanValuesTarget[iSpanTarget], spanValuesDonor, nSpanDonor); + break; + + case LINEAR_INTERPOLATION: { + targetSpan = MapLinearInterpolationSpan(spanValuesTarget[iSpanTarget], spanValuesDonor, nSpanDonor, rank); + break; + } + default: + SU2_MPI::Error("MixingPlane interface option not implemented yet", CURRENT_FUNCTION); + break; + } + } + } +} + +void CMixingPlane::WriteInterpolationDetails(const std::string& filename, const CConfig* const* config) { + // Only write from master process in MPI + if (rank != MASTER_NODE) return; + + std::ofstream outFile(filename); + + if (!outFile.is_open()) { + cout << "Error: Could not open file " << filename << ". Abandoning interpolator writing..." << endl; + return; + } + + const auto donor_config = config[donorZone]; + const auto nMarkerInt = config[donorZone]->GetnMarker_MixingPlaneInterface() / 2; + + outFile << "Mixing-Plane Interpolator Details. Donor Zone = " << donorZone << " Target Zone = " << targetZone + << ". Interpolation Method = "; + switch (donor_config->GetKind_MixingPlaneInterface()) { + case MATCHING: + outFile << "MATCHING\n"; + break; + case NEAREST_SPAN: + outFile << "NEAREST_SPAN\n"; + break; + case LINEAR_INTERPOLATION: + outFile << "LINEAR_INTERPOLATION\n"; + break; + default: + outFile << "UNKNOWN\n"; + } + outFile << "\n"; + outFile << "===============================================================" << endl; + + // Loop through each marker interface + for (auto iMarkerInt = 0; iMarkerInt < nMarkerInt + 1; iMarkerInt++) { + if (targetSpans[iMarkerInt].empty()) continue; + + outFile << "Marker Interface " << iMarkerInt << "\n"; + outFile << "---------------------\n"; + outFile << "Target Span, Donor Span, Interpolation Coefficient\n"; + + for (size_t iSpanTarget = 0; iSpanTarget < targetSpans[iMarkerInt].size(); iSpanTarget++) { + const auto& targetSpan = targetSpans[iMarkerInt][iSpanTarget]; + outFile << iSpanTarget << ", " << targetSpan.donorSpan << ", " << targetSpan.coefficient << "\n"; + } + outFile << "\n"; + } + + // Optional: Write grouped by donor span + outFile << "\n\nGrouped by Donor Span\n"; + outFile << "=====================\n\n"; + + for (auto iMarkerInt = 0; iMarkerInt < nMarkerInt + 1; iMarkerInt++) { + if (targetSpans[iMarkerInt].empty()) continue; + + outFile << "Marker Interface " << iMarkerInt << "\n"; + outFile << "---------------------\n"; + + // Find max donor span + size_t maxDonorSpan = 0; + for (const auto& ts : targetSpans[iMarkerInt]) { + maxDonorSpan = std::max(maxDonorSpan, ts.donorSpan); + } + + // Group by donor span + for (size_t iDonor = 0; iDonor <= maxDonorSpan; iDonor++) { + bool hasTargets = false; + std::ostringstream targets; + + for (size_t iSpanTarget = 0; iSpanTarget < targetSpans[iMarkerInt].size(); iSpanTarget++) { + if (targetSpans[iMarkerInt][iSpanTarget].donorSpan == iDonor) { + if (hasTargets) targets << ", "; + targets << "Target " << iSpanTarget << " (coeff=" << targetSpans[iMarkerInt][iSpanTarget].coefficient << ")"; + hasTargets = true; + } + } + + if (hasTargets) { + outFile << "Donor Span " << iDonor << ": " << targets.str() << "\n"; + } + } + outFile << "\n"; + } + + outFile.close(); + cout << "Interpolation details written to " << filename << endl; +} diff --git a/Common/src/interface_interpolation/CNearestNeighbor.cpp b/Common/src/interface_interpolation/CNearestNeighbor.cpp index 2c245e9fc554..298b34011b30 100644 --- a/Common/src/interface_interpolation/CNearestNeighbor.cpp +++ b/Common/src/interface_interpolation/CNearestNeighbor.cpp @@ -33,7 +33,7 @@ CNearestNeighbor::CNearestNeighbor(CGeometry**** geometry_container, const CConfig* const* config, unsigned int iZone, unsigned int jZone) : CInterpolator(geometry_container, config, iZone, jZone) { - SetTransferCoeff(config); + SetTransferCoeff(geometry_container, config); } void CNearestNeighbor::PrintStatistics() const { @@ -41,7 +41,7 @@ void CNearestNeighbor::PrintStatistics() const { cout << " Avg/max distance to closest donor point: " << AvgDistance << "/" << MaxDistance << endl; } -void CNearestNeighbor::SetTransferCoeff(const CConfig* const* config) { +void CNearestNeighbor::SetTransferCoeff(CGeometry**** geometry, const CConfig* const* config) { /*--- Desired number of donor points. ---*/ const auto nDonor = max(config[donorZone]->GetNumNearestNeighbors(), 1); @@ -111,15 +111,75 @@ void CNearestNeighbor::SetTransferCoeff(const CConfig* const* config) { /*--- Coordinates of the target point. ---*/ const su2double* Coord_i = target_geometry->nodes->GetCoord(Point_Target); + /*--- If the relative-frame sliding plane is active, precompute (once per target vertex, not per + * donor candidate) the rotated target coordinate and the donor-zone rotation matrix: both are + * invariant across every donor candidate visited in the loop below. ---*/ + const bool relframe_sp = + config[targetZone]->GetBoolRelFrame_SlidingPlane() || config[donorZone]->GetBoolRelFrame_SlidingPlane(); + su2double rotCoord_i[3] = {0.0, 0.0, 0.0}; + su2double donorRotMatrix[3][3] = {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}}; + bool rotate_donor = false; + const su2double zeros[3] = {0.0}; + + if (relframe_sp) { + for (unsigned short iDim = 0; iDim < 3; iDim++) rotCoord_i[iDim] = Coord_i[iDim]; + + if (config[targetZone]->GetRotating_Frame() == YES) { + su2double Omega_i[3] = {0.0, 0.0, 0.0}; + su2double dt = config[targetZone]->GetDelta_UnstTimeND(); + unsigned long TimeIter = config[targetZone]->GetTimeIter(); + for (unsigned short iDim = 0; iDim < 3; iDim++) { + Omega_i[iDim] = config[targetZone]->GetRotation_Rate(iDim) / config[targetZone]->GetOmega_Ref(); + } + + /*--- Compute the rotation matrix. Note that the implicit + ordering is rotation about the x-axis, y-axis, then z-axis. ---*/ + su2double Theta = Omega_i[0] * dt * TimeIter; + su2double Phi = Omega_i[1] * dt * TimeIter; + su2double Psi = Omega_i[2] * dt * TimeIter; + su2double rotMatrix[3][3] = {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}}; + GeometryToolbox::RotationMatrix(Theta, Phi, Psi, rotMatrix); + + /*--- Compute transformed point coordinates. ---*/ + GeometryToolbox::Rotate(rotMatrix, zeros, Coord_i, rotCoord_i); + } + + if (config[donorZone]->GetRotating_Frame() == YES) { + rotate_donor = true; + su2double Omega_j[3] = {0.0, 0.0, 0.0}; + su2double dt = config[donorZone]->GetDelta_UnstTimeND(); + unsigned long TimeIter = config[donorZone]->GetTimeIter(); + for (unsigned short iDim = 0; iDim < 3; iDim++) { + Omega_j[iDim] = config[donorZone]->GetRotation_Rate(iDim) / config[donorZone]->GetOmega_Ref(); + } + + /*--- Compute the rotation matrix. Note that the implicit + ordering is rotation about the x-axis, y-axis, then z-axis. ---*/ + su2double Theta = Omega_j[0] * dt * TimeIter; + su2double Phi = Omega_j[1] * dt * TimeIter; + su2double Psi = Omega_j[2] * dt * TimeIter; + GeometryToolbox::RotationMatrix(Theta, Phi, Psi, donorRotMatrix); + } + } + /*--- Compute all distances. ---*/ for (int iProcessor = 0, iDonor = 0; iProcessor < nProcessor; ++iProcessor) { for (auto jVertex = 0ul; jVertex < Buffer_Receive_nVertex_Donor[iProcessor]; ++jVertex) { const auto idx = iProcessor * MaxLocalVertex_Donor + jVertex; const auto pGlobalPoint = Buffer_Receive_GlobalPoint[idx]; const su2double* Coord_j = Buffer_Receive_Coord[idx]; - const auto dist2 = GeometryToolbox::SquaredDistance(nDim, Coord_i, Coord_j); - donorInfo[iDonor++] = DonorInfo(dist2, pGlobalPoint, iProcessor); + /*--- Rotate the donor point before matching if sliding plane for relative frame is activated. ---*/ + if (relframe_sp) { + su2double rotCoord_j[3] = {Coord_j[0], Coord_j[1], Coord_j[2]}; + if (rotate_donor) GeometryToolbox::Rotate(donorRotMatrix, zeros, Coord_j, rotCoord_j); + + const auto dist2 = GeometryToolbox::SquaredDistance(nDim, rotCoord_i, rotCoord_j); + donorInfo[iDonor++] = DonorInfo(dist2, pGlobalPoint, iProcessor); + } else { + const auto dist2 = GeometryToolbox::SquaredDistance(nDim, Coord_i, Coord_j); + donorInfo[iDonor++] = DonorInfo(dist2, pGlobalPoint, iProcessor); + } } } diff --git a/Common/src/interface_interpolation/CRadialBasisFunction.cpp b/Common/src/interface_interpolation/CRadialBasisFunction.cpp index c1d932588f69..f5040f925b0c 100644 --- a/Common/src/interface_interpolation/CRadialBasisFunction.cpp +++ b/Common/src/interface_interpolation/CRadialBasisFunction.cpp @@ -37,17 +37,21 @@ #define HAVE_LAPACK #endif #elif defined(HAVE_LAPACK) +#ifdef USE_SINGLE_PRECISION +#define GEMM_IMPL sgemm_ +#else +#define GEMM_IMPL dgemm_ +#endif // dgemm(opA, opB, m, n, k, alpha, A, lda, B, ldb, beta, C, ldc) -extern "C" void dgemm_(const char*, const char*, const int*, const int*, const int*, const passivedouble*, - const passivedouble*, const int*, const passivedouble*, const int*, const passivedouble*, - passivedouble*, const int*); -#define DGEMM dgemm_ +extern "C" void GEMM_IMPL(const char*, const char*, const int*, const int*, const int*, const passivedouble*, + const passivedouble*, const int*, const passivedouble*, const int*, const passivedouble*, + passivedouble*, const int*); #endif CRadialBasisFunction::CRadialBasisFunction(CGeometry**** geometry_container, const CConfig* const* config, unsigned int iZone, unsigned int jZone) : CInterpolator(geometry_container, config, iZone, jZone) { - SetTransferCoeff(config); + SetTransferCoeff(geometry_container, config); } void CRadialBasisFunction::PrintStatistics() const { @@ -98,7 +102,7 @@ su2double CRadialBasisFunction::Get_RadialBasisValue(RADIAL_BASIS type, const su return rbf; } -void CRadialBasisFunction::SetTransferCoeff(const CConfig* const* config) { +void CRadialBasisFunction::SetTransferCoeff(CGeometry**** geometry, const CConfig* const* config) { /*--- RBF options. ---*/ const auto kindRBF = config[donorZone]->GetKindRadialBasisFunction(); const bool usePolynomial = config[donorZone]->GetRadialBasisFunctionPolynomialOption(); @@ -334,7 +338,7 @@ void CRadialBasisFunction::SetTransferCoeff(const CConfig* const* config) { const int M = interpMat.cols(), N = slabSize, K = funcMat.cols(); // lda = C_inv_trunc.cols() = M; ldb = funcMat.cols() = K; ldc = interpMat.cols() = M; const passivedouble alpha = 1.0, beta = 0.0; - DGEMM(&op, &op, &M, &N, &K, &alpha, C_inv_trunc[0], &M, funcMat[0], &K, &beta, interpMat[0], &M); + GEMM_IMPL(&op, &op, &M, &N, &K, &alpha, C_inv_trunc[0], &M, funcMat[0], &K, &beta, interpMat[0], &M); #else /*--- Naive product, loop order considers short-wide * nature of funcMat and interpMat. ---*/ @@ -480,7 +484,7 @@ void CRadialBasisFunction::ComputeGeneratorMatrix(RADIAL_BASIS type, bool usePol const int M = nVertexDonor, N = nVertexDonor, K = nPolynomial + 1; // lda = C_inv_top.cols() = M; ldb = Q.cols() = M; ldc = C_inv_bot.cols() = M; const passivedouble alpha = -1.0, beta = 1.0; - DGEMM(&opa, &opb, &M, &N, &K, &alpha, C_inv_top[0], &M, Q[0], &M, &beta, C_inv_bot[0], &M); + GEMM_IMPL(&opa, &opb, &M, &N, &K, &alpha, C_inv_top[0], &M, Q[0], &M, &beta, C_inv_bot[0], &M); #else // naive product for (int i = 0; i < nVertexDonor; ++i) for (int j = 0; j < nVertexDonor; ++j) diff --git a/Common/src/interface_interpolation/CSlidingMesh.cpp b/Common/src/interface_interpolation/CSlidingMesh.cpp index a81ff98f31fb..7e287c12f998 100644 --- a/Common/src/interface_interpolation/CSlidingMesh.cpp +++ b/Common/src/interface_interpolation/CSlidingMesh.cpp @@ -33,10 +33,10 @@ CSlidingMesh::CSlidingMesh(CGeometry**** geometry_container, const CConfig* const* config, unsigned int iZone, unsigned int jZone) : CInterpolator(geometry_container, config, iZone, jZone) { - SetTransferCoeff(config); + SetTransferCoeff(geometry_container, config); } -void CSlidingMesh::SetTransferCoeff(const CConfig* const* config) { +void CSlidingMesh::SetTransferCoeff(CGeometry**** geometry, const CConfig* const* config) { /* 0 - Variable declaration */ /* --- General variables --- */ diff --git a/Common/src/interface_interpolation/meson.build b/Common/src/interface_interpolation/meson.build index 8624b3ad4fa8..6cd185ef873c 100644 --- a/Common/src/interface_interpolation/meson.build +++ b/Common/src/interface_interpolation/meson.build @@ -4,4 +4,5 @@ common_src += files(['CInterpolatorFactory.cpp', 'CSlidingMesh.cpp', 'CIsoparametric.cpp', 'CNearestNeighbor.cpp', - 'CRadialBasisFunction.cpp']) + 'CRadialBasisFunction.cpp', + 'CMixingPlane.cpp']) diff --git a/Common/src/linear_algebra/CPastixWrapper.cpp b/Common/src/linear_algebra/CPastixWrapper.cpp index 1633a88bbde9..f213939eaf61 100644 --- a/Common/src/linear_algebra/CPastixWrapper.cpp +++ b/Common/src/linear_algebra/CPastixWrapper.cpp @@ -41,7 +41,7 @@ void CPastixWrapper::Initialize(CGeometry* geometry, const CConfig* if (isinitialized) return; // only need to do this once const unsigned long nVar = matrix.nVar, nPoint = matrix.nPoint, nPointDomain = matrix.nPointDomain; - const unsigned long *row_ptr = matrix.rowptr, *col_ind = matrix.colidx; + const unsigned long *row_ptr = csr_row_ptr.data(), *col_ind = csr_col_ind.data(); const unsigned long nNonZero = row_ptr[nPointDomain]; /*--- Allocate ---*/ @@ -204,12 +204,27 @@ void CPastixWrapper::Initialize(CGeometry* geometry, const CConfig* SU2_MPI::Error("Error analyzing matrix: " + std::to_string(rc), CURRENT_FUNCTION); } - if (mpi_rank == MASTER_NODE && verb > 0) - cout << " +--------------------------------------------------------------------+" << endl; + if (mpi_rank == MASTER_NODE && verb > 0) cout << "+-------------------------------------------------+" << endl; isinitialized = true; } +template +void CPastixWrapper::AssembleValues() { + const auto nDomain = matrix.nPointDomain; + const auto blkSz = matrix.blkSz; + const auto *d = matrix.d, *l = matrix.l, *u = matrix.u; + for (auto iPoint = 0ul; iPoint < nDomain; ++iPoint) { + auto* dst = values.data() + csr_row_ptr[iPoint] * blkSz; + for (auto k = matrix.row_ptr_l[iPoint]; k < matrix.row_ptr_l[iPoint + 1]; ++k, dst += blkSz) + for (auto b = 0ul; b < blkSz; ++b) dst[b] = SU2_TYPE::GetValue(l[k * blkSz + b]); + for (auto b = 0ul; b < blkSz; ++b) dst[b] = SU2_TYPE::GetValue(d[iPoint * blkSz + b]); + dst += blkSz; + for (auto k = matrix.row_ptr_u[iPoint]; k < matrix.row_ptr_u[iPoint + 1]; ++k, dst += blkSz) + for (auto b = 0ul; b < blkSz; ++b) dst[b] = SU2_TYPE::GetValue(u[k * blkSz + b]); + } +} + template void CPastixWrapper::Factorize(CGeometry* geometry, const CConfig* config, unsigned short kind_fact) { /*--- Detect a possible change of settings between direct and adjoint that requires a reset ---*/ @@ -247,30 +262,30 @@ void CPastixWrapper::Factorize(CGeometry* geometry, const CConfig* c if (isfactorized && !factorize) return; // No - /*--- Yes ---*/ + /*--- Yes: assemble LDU blocks into the flat CSR buffer ---*/ + AssembleValues(); if (mpi_rank == MASTER_NODE && verb > 0) { - cout << endl; - cout << " +--------------------------------------------------------------------+" << endl; - cout << " + PaStiX : Parallel Sparse matriX package +" << endl; - cout << " +--------------------------------------------------------------------+" << endl; + cout << "\n+-------------------------------------------------+"; + cout << "\n+ PaStiX : Parallel Sparse matriX package +" << endl; } - const unsigned long szBlk = matrix.nVar * matrix.nVar, nNonZero = values.size(); + const auto blkSz = matrix.blkSz; - /*--- Copy matrix values and swap blocks as required ---*/ - - for (auto i = 0ul; i < nNonZero; ++i) values[i] = SU2_TYPE::GetValue(matrix.values[i]); + /*--- Permute blocks for rows with halo columns into global sorted order. + AssembleValues wrote them in LDU order; copy to tmp then write back sorted. ---*/ + vector tmp; for (auto i = 0ul; i < sort_rows.size(); ++i) { const auto iRow = sort_rows[i]; - const auto begin = matrix.rowptr[iRow]; - - for (auto j = 0ul; j < sort_order[i].size(); ++j) { - const auto target = (begin + j) * szBlk; - const auto source = sort_order[i][j] * szBlk; - - for (auto k = 0ul; k < szBlk; ++k) values[target + k] = SU2_TYPE::GetValue(matrix.values[source + k]); + /*--- colptr is 1-based Fortran numbering: row start = colptr[iRow] - 1. ---*/ + const auto begin = static_cast(colptr[iRow] - 1); + const auto nnz_row = sort_order[i].size(); + + tmp.assign(values.begin() + begin * blkSz, values.begin() + (begin + nnz_row) * blkSz); + for (auto j = 0ul; j < nnz_row; ++j) { + const auto src_pos = sort_order[i][j] - begin; + for (auto k = 0ul; k < blkSz; ++k) values[(begin + j) * blkSz + k] = tmp[src_pos * blkSz + k]; } } @@ -297,8 +312,7 @@ void CPastixWrapper::Factorize(CGeometry* geometry, const CConfig* c SU2_MPI::Error("Error factorizing matrix: " + std::to_string(rc), CURRENT_FUNCTION); } - if (mpi_rank == MASTER_NODE && verb > 0) - cout << " +--------------------------------------------------------------------+" << endl << endl; + if (mpi_rank == MASTER_NODE && verb > 0) cout << "+-------------------------------------------------+\n" << endl; isfactorized = true; } @@ -307,7 +321,7 @@ void CPastixWrapper::Factorize(CGeometry* geometry, const CConfig* c template class CPastixWrapper; #else template class CPastixWrapper; -#ifdef USE_MIXED_PRECISION +#if defined(USE_MIXED_PRECISION) && !defined(USE_SINGLE_PRECISION) template class CPastixWrapper; #endif #endif diff --git a/Common/src/linear_algebra/CSysMatrix.cpp b/Common/src/linear_algebra/CSysMatrix.cpp index f9b5d5427321..503e2ad4893c 100644 --- a/Common/src/linear_algebra/CSysMatrix.cpp +++ b/Common/src/linear_algebra/CSysMatrix.cpp @@ -28,32 +28,85 @@ #include "../../include/linear_algebra/CSysMatrix.inl" #include "../../include/geometry/CGeometry.hpp" +#include "../../include/linear_algebra/CMatrixInverse.hpp" #include "../../include/toolboxes/allocation_toolbox.hpp" #include +#include +#include +#include + +namespace { +/*--- Helper function to regularize small pivots ---*/ +template +FORCEINLINE void RegularizePivot(ScalarType& pivot, unsigned long row, unsigned long col, const char* context) { + const float eps = 1e-12; + if (std::abs(pivot) < eps) { + pivot = std::copysign(eps, SU2_TYPE::GetValue(pivot)); +#ifndef NDEBUG + std::cout << context << ": Regularized small pivot A(" << row << "," << col << ") to " << pivot << std::endl; +#endif + } +} + +/*--- Common failure path for a device dispatch that is not available in this build/scalar type + * combination, called with CURRENT_FUNCTION so the error names the right caller. ---*/ +void GPUNotAvailable(const char* caller) { +#if defined(SU2_ENABLE_CUDA_KERNELS) + SU2_MPI::Error("GPU acceleration is not supported for AD scalar types.", caller); +#elif defined(HAVE_CUDA) + /*--- AD build, the kernels are compiled out; normally rejected by CConfig::SetPostprocessing. ---*/ + SU2_MPI::Error("GPU acceleration is not available in the AD and direct differentiation solvers.", caller); +#else + SU2_MPI::Error( + "ENABLE_CUDA is set to YES but SU2 was not compiled with CUDA support; " + "recompile with CUDA enabled in Meson to use GPU functions.", + caller); +#endif +} +} // namespace template CSysMatrix::CSysMatrix() : rank(SU2_MPI::GetRank()), size(SU2_MPI::GetSize()) { SU2_ZONE_SCOPED nPoint = nPointDomain = nVar = nEqn = 0; - nnz = nnz_ilu = 0; + mat.nnz_l = mat.nnz_u = 0; + gpu.nnz_l = gpu.nnz_u = 0; + ilu.nnz_l = ilu.nnz_u = 0; ilu_fill_in = 0; omp_partitions = nullptr; - matrix = nullptr; - row_ptr = nullptr; - dia_ptr = nullptr; - col_ind = nullptr; - col_ptr = nullptr; - - ILU_matrix = nullptr; - row_ptr_ilu = nullptr; - dia_ptr_ilu = nullptr; - col_ind_ilu = nullptr; + mat.row_ptr_l = nullptr; + mat.col_ind_l = nullptr; + mat.row_ptr_u = nullptr; + mat.col_ind_u = nullptr; + l_to_u_transp = nullptr; + u_to_l_transp = nullptr; + edge_ptr_l = nullptr; + + mat.d = nullptr; + mat.l = nullptr; + mat.u = nullptr; + + gpu.d = nullptr; + gpu.l = nullptr; + gpu.u = nullptr; + gpu.row_ptr_l = nullptr; + gpu.col_ind_l = nullptr; + gpu.row_ptr_u = nullptr; + gpu.col_ind_u = nullptr; + + ilu.l = nullptr; + ilu.d = nullptr; + ilu.u = nullptr; + + q_scale = {}; + q_blocks = {}; invM = nullptr; + d_invM = nullptr; #ifdef USE_MKL MatrixMatrixProductJitter = nullptr; @@ -68,14 +121,55 @@ CSysMatrix::~CSysMatrix() { SU2_ZONE_SCOPED delete[] omp_partitions; - MemoryAllocation::aligned_free(ILU_matrix); - MemoryAllocation::aligned_free(matrix); + auto freeHostLDU = [](auto& m) { + MemoryAllocation::aligned_free(m.d); + MemoryAllocation::aligned_free(m.l); + MemoryAllocation::aligned_free(m.u); + }; + freeHostLDU(mat); + freeHostLDU(ilu); MemoryAllocation::aligned_free(invM); + /*--- q_scale/q_blocks' .l/.u are pinned (cudaMallocHost) rather than aligned_alloc when + * useCuda, .d never is; see the comment in Initialize(). Free each with its matching + * deallocator. ---*/ + auto freeQuantLDU = [this](auto& m) { + MemoryAllocation::aligned_free(m.d); + if (useCuda) { + GPUMemoryAllocation::pinned_free(m.l); + GPUMemoryAllocation::pinned_free(m.u); + } else { + MemoryAllocation::aligned_free(m.l); + MemoryAllocation::aligned_free(m.u); + } + }; + freeQuantLDU(q_scale); + freeQuantLDU(q_blocks); + if (useCuda) { - GPUMemoryAllocation::gpu_free(d_matrix); - GPUMemoryAllocation::gpu_free(d_row_ptr); - GPUMemoryAllocation::gpu_free(d_col_ind); + auto freeLDU = [](auto& m) { + GPUMemoryAllocation::gpu_free(m.d); + GPUMemoryAllocation::gpu_free(m.l); + GPUMemoryAllocation::gpu_free(m.u); + GPUMemoryAllocation::gpu_free(m.row_ptr_l); + GPUMemoryAllocation::gpu_free(m.col_ind_l); + GPUMemoryAllocation::gpu_free(m.row_ptr_u); + GPUMemoryAllocation::gpu_free(m.col_ind_u); + }; + freeLDU(gpu); + freeLDU(gpu_ilu); + freeLDU(d_q_scale); + freeLDU(d_q_blocks); + GPUMemoryAllocation::gpu_free(d_invM); + GPUMemoryAllocation::gpu_free(d_ilu_color_idx); + GPUMemoryAllocation::gpu_free(d_precond_level_idx); +#ifdef SU2_ENABLE_CUDA_KERNELS + if (ilu_build_graph_exec != nullptr) cudaGraphExecDestroy(ilu_build_graph_exec); + if (precond_fwd_graph_exec != nullptr) cudaGraphExecDestroy(precond_fwd_graph_exec); + if (precond_bwd_graph_exec != nullptr) cudaGraphExecDestroy(precond_bwd_graph_exec); + if (aux_stream != nullptr) cudaStreamDestroy(aux_stream); + if (htd_event != nullptr) cudaEventDestroy(htd_event); +#endif } #ifdef USE_MKL @@ -89,13 +183,14 @@ CSysMatrix::~CSysMatrix() { template void CSysMatrix::Initialize(unsigned long npoint, unsigned long npointdomain, unsigned short nvar, unsigned short neqn, bool EdgeConnect, CGeometry* geometry, - const CConfig* config, bool needTranspPtr, bool grad_mode) { + const CConfig* config, bool needTranspPtr, bool allow_quant, + std::optional override_prec) { SU2_ZONE_SCOPED assert(omp_get_thread_num() == 0 && "Only the master thread is allowed to initialize the matrix."); if (npoint == 0) return; - if (matrix != nullptr) { + if (mat.d != nullptr) { SU2_MPI::Error("CSysMatrix can only be initialized once.", CURRENT_FUNCTION); } @@ -119,12 +214,25 @@ void CSysMatrix::Initialize(unsigned long npoint, unsigned long npoi } /*--- No else if, but separate if case! ---*/ - if (config->GetSmoothGradient() && grad_mode) { - prec = config->GetKind_Grad_Linear_Solver_Prec(); + if (override_prec) { + prec = *override_prec; } + useCuda = config->GetCUDA(); + const bool ilu_needed = (prec == ILU); - const bool diag_needed = (prec == JACOBI) || (prec == LINELET); + const bool diag_needed = (prec == JACOBI) || (prec == Q_JACOBI) || (prec == LINELET); + const bool lu_sgs_on_device = useCuda && (prec == LU_SGS || prec == Q_LU_SGS); + + /*--- Linelet also builds the Jacobi preconditioner but reads the inverse diagonal blocks on + * the host, so only plain (or quantized) Jacobi can keep them exclusively on the device. ---*/ + jacobi_on_device = useCuda && (prec == JACOBI || prec == Q_JACOBI); +#ifndef CODI_REVERSE_TYPE + const bool quantized_offdiag_needed = allow_quant && (prec == Q_JACOBI || prec == Q_IDENTITY || prec == Q_LU_SGS); +#else + /*--- No quantization in adjoint mode for now because TransposeInPlace would get complicated. ---*/ + const bool quantized_offdiag_needed = false; +#endif /*--- Basic dimensions. ---*/ nVar = nvar; @@ -132,77 +240,253 @@ void CSysMatrix::Initialize(unsigned long npoint, unsigned long npoi nPoint = npoint; nPointDomain = npointdomain; - /*--- Get sparse structure pointers from geometry, - * the data is managed by CGeometry to allow re-use. ---*/ - - const auto& csr = geometry->GetSparsePattern(type, 0); - - nnz = csr.getNumNonZeros(); - row_ptr = csr.outerPtr(); - col_ind = csr.innerIdx(); - dia_ptr = csr.diagPtr(); - - /*--- Allocate data. ---*/ + /*--- Allocate host data. ---*/ auto allocAndInit = [](ScalarType*& ptr, unsigned long num) { ptr = MemoryAllocation::aligned_alloc(64, num * sizeof(ScalarType)); }; - allocAndInit(matrix, nnz * nVar * nEqn); - - useCuda = config->GetCUDA(); + /*--- L/D/U index structures and value arrays. ---*/ + { + const auto& pat = geometry->GetSparsePattern(type, 0); + mat.row_ptr_l = pat.l.outerPtr(); + mat.col_ind_l = pat.l.innerIdx(); + mat.nnz_l = pat.l.getNumNonZeros(); + mat.row_ptr_u = pat.u.outerPtr(); + mat.col_ind_u = pat.u.innerIdx(); + mat.nnz_u = pat.u.getNumNonZeros(); + } + allocAndInit(mat.d, nPoint * nVar * nEqn); - if (useCuda) { - /*--- Allocate GPU data. ---*/ - auto GPUAllocAndInit = [](ScalarType*& ptr, unsigned long num) { - ptr = GPUMemoryAllocation::gpu_alloc(num * sizeof(ScalarType)); + if (quantized_offdiag_needed) { + /*--- Q_LU_SGS / Q_JACOBI / Q_IDENTITY: no full-precision L/U; off-diagonal blocks live in quantized storage. + * L/U are quantized on-the-fly during assembly; diagonal is quantized in the Build step. ---*/ +#ifndef CODI_REVERSE_TYPE + quantized_mode = true; +#endif + /*--- .l/.u are pinned (page-locked) when useCuda because HtDTransfer() uploads them with + * cudaMemcpyAsync, which is only genuinely asynchronous from pinned host memory. ---*/ + auto allocQ = [](auto*& ptr, unsigned long n) { + using T = std::remove_reference_t; + ptr = MemoryAllocation::aligned_alloc(64, n * sizeof(T)); }; - - auto GPUAllocAndCopy = [](const unsigned long*& ptr, const unsigned long*& src_ptr, unsigned long num) { - ptr = GPUMemoryAllocation::gpu_alloc_cpy(src_ptr, num * sizeof(const unsigned long)); + auto allocPinnedIfCuda = [useCuda = this->useCuda](auto*& ptr, unsigned long n) { + using T = std::remove_reference_t; + if (useCuda) { + ptr = GPUMemoryAllocation::pinned_alloc(n * sizeof(T)); + } else { + ptr = MemoryAllocation::aligned_alloc(64, n * sizeof(T)); + } }; - - GPUAllocAndInit(d_matrix, nnz * nVar * nEqn); - GPUAllocAndCopy(d_row_ptr, row_ptr, (nPointDomain + 1)); - GPUAllocAndCopy(d_col_ind, col_ind, nnz); + allocPinnedIfCuda(q_scale.l, mat.nnz_l * nVar); + allocPinnedIfCuda(q_blocks.l, mat.nnz_l * nVar * nEqn); + allocPinnedIfCuda(q_scale.u, mat.nnz_u * nVar); + allocPinnedIfCuda(q_blocks.u, mat.nnz_u * nVar * nEqn); + allocQ(q_scale.d, nPoint * nVar); + allocQ(q_blocks.d, nPoint * nVar * nEqn); + } else { + allocAndInit(mat.l, mat.nnz_l * nVar * nEqn); + allocAndInit(mat.u, mat.nnz_u * nVar * nEqn); } - if (needTranspPtr) col_ptr = geometry->GetTransposeSparsePatternMap(type).data(); + auto GPUAllocAndInit = [](ScalarType*& ptr, unsigned long num) { + ptr = GPUMemoryAllocation::gpu_alloc(num * sizeof(ScalarType)); + }; + auto GPUAllocAndCopy = [](const su2uint*& ptr, const su2uint* src_ptr, unsigned long num) { + ptr = GPUMemoryAllocation::gpu_alloc_cpy(src_ptr, num * sizeof(su2uint)); + }; + + if (useCuda) { + if (nVar != nEqn) { + SU2_MPI::Error("CUDA CSysMatrix block-LDU SpMV requires square blocks.", CURRENT_FUNCTION); + } + GPUAllocAndInit(gpu.d, nPoint * nVar * nEqn); + GPUAllocAndCopy(gpu.row_ptr_l, mat.row_ptr_l, nPointDomain + 1); + GPUAllocAndCopy(gpu.col_ind_l, mat.col_ind_l, mat.nnz_l); + GPUAllocAndCopy(gpu.row_ptr_u, mat.row_ptr_u, nPointDomain + 1); + GPUAllocAndCopy(gpu.col_ind_u, mat.col_ind_u, mat.nnz_u); + + if (quantized_mode) { + /*--- Device mirrors of the host quantized storage; gpu.l/gpu.u are not allocated (nothing + * would ever read them). d_q_scale.d/d_q_blocks.d are uploaded from the host result once + * QuantizeDiagonalBlocks() has computed it, see the comment on those members. ---*/ + auto GPUAllocQ = [](auto*& ptr, unsigned long n) { + using T = std::remove_reference_t; + ptr = GPUMemoryAllocation::gpu_alloc(n * sizeof(T)); + }; + GPUAllocQ(d_q_scale.l, mat.nnz_l * nVar); + GPUAllocQ(d_q_blocks.l, mat.nnz_l * nVar * nEqn); + GPUAllocQ(d_q_scale.u, mat.nnz_u * nVar); + GPUAllocQ(d_q_blocks.u, mat.nnz_u * nVar * nEqn); + GPUAllocQ(d_q_scale.d, nPoint * nVar); + GPUAllocQ(d_q_blocks.d, nPoint * nVar * nEqn); + } else { + GPUAllocAndInit(gpu.l, mat.nnz_l * nVar * nEqn); + GPUAllocAndInit(gpu.u, mat.nnz_u * nVar * nEqn); + } + } if (type == ConnectivityType::FiniteVolume) { - edge_ptr.ptr = geometry->GetEdgeToSparsePatternMap().data(); - edge_ptr.nEdge = geometry->GetnEdge(); + edge_ptr_l = geometry->GetUToLTransposeSparsePatternMap(type).data(); + } + if (needTranspPtr) { + l_to_u_transp = geometry->GetLToUTransposeSparsePatternMap(type).data(); + u_to_l_transp = geometry->GetUToLTransposeSparsePatternMap(type).data(); } /*--- Get ILU sparse pattern, if fill is 0 no new data is allocated. --*/ if (ilu_needed) { - ilu_fill_in = config->GetLinear_Solver_ILU_n(); - - const auto& csr_ilu = geometry->GetSparsePattern(type, ilu_fill_in); - - row_ptr_ilu = csr_ilu.outerPtr(); - col_ind_ilu = csr_ilu.innerIdx(); - dia_ptr_ilu = csr_ilu.diagPtr(); - nnz_ilu = csr_ilu.getNumNonZeros(); + ilu_fill_in = config->GetIluOptions().FillIn; + ilu_gpu_sweeps = config->GetIluOptions().GPUSweeps; + + const auto& pat_ilu = geometry->GetSparsePattern(type, ilu_fill_in); + ilu.row_ptr_l = pat_ilu.l.outerPtr(); + ilu.col_ind_l = pat_ilu.l.innerIdx(); + ilu.nnz_l = pat_ilu.l.getNumNonZeros(); + ilu.row_ptr_u = pat_ilu.u.outerPtr(); + ilu.col_ind_u = pat_ilu.u.innerIdx(); + ilu.nnz_u = pat_ilu.u.getNumNonZeros(); + + /*--- The GPU triangular solves are level-scheduled (exact, one pass per level), so they need + * levels_ilu unconditionally; the host/OMP path only needs it when both multi-threaded and + * requested via config. ---*/ + if (useCuda || (omp_get_max_threads() > 1 && config->GetIluOptions().LevelScheduling)) { + levels_ilu = computeLevels(pat_ilu.l); + } - if (omp_get_max_threads() > 1 && config->GetLinear_Solver_ILU_levels()) { - levels_ilu = computeLevels(csr_ilu); + /*--- Coloring for the GPU iterative factorization, see IluFactorColorKernel. Colors are + * true independent sets of the (domain-only, symmetric) dependency graph, computed the same + * way SU2 already colors edges/elements for OMP loops, just applied to the ILU pattern + * instead. This does not change the elimination order/pattern (nothing here affects L/U + * membership), only how the build is scheduled on the device. ---*/ + if (useCuda) { + std::vector adjPtr(nPointDomain + 1, 0); + std::vector adjIdx; + adjIdx.reserve(ilu.nnz_l + ilu.nnz_u); + for (auto i = 0ul; i < nPointDomain; ++i) { + adjPtr[i] = static_cast(adjIdx.size()); + for (auto k = ilu.row_ptr_l[i]; k < ilu.row_ptr_l[i + 1]; ++k) adjIdx.push_back(ilu.col_ind_l[k]); + for (auto k = ilu.row_ptr_u[i]; k < ilu.row_ptr_u[i + 1]; ++k) { + const auto j = ilu.col_ind_u[k]; + if (j < nPointDomain) adjIdx.push_back(static_cast(j)); + } + } + adjPtr[nPointDomain] = static_cast(adjIdx.size()); + color_ilu = colorSparsePattern(CCompressedSparsePatternUL(adjPtr, adjIdx), 1, true, false); + + /*--- Report, across ranks, how many colors/levels the GPU ILU ends up scheduled over and how + * wide those groups are on average. Few, wide colors/levels use the GPU efficiently; many + * narrow ones (small average size) serialize into many small kernel launches instead. ---*/ + const auto nColorsLocal = static_cast(color_ilu.getOuterSize()); + const auto nLevelsLocal = static_cast(levels_ilu.getOuterSize()); + unsigned long nColorsMax = 0, nLevelsMax = 0; + SU2_MPI::Reduce(&nColorsLocal, &nColorsMax, 1, MPI_UNSIGNED_LONG, MPI_MAX, MASTER_NODE, SU2_MPI::GetComm()); + SU2_MPI::Reduce(&nLevelsLocal, &nLevelsMax, 1, MPI_UNSIGNED_LONG, MPI_MAX, MASTER_NODE, SU2_MPI::GetComm()); + + const auto avgColorSizeLocal = static_cast(std::lround(double(nPointDomain) / nColorsLocal)); + const auto avgLevelSizeLocal = static_cast(std::lround(double(nPointDomain) / nLevelsLocal)); + unsigned long minAvgColorSize = 0, minAvgLevelSize = 0; + SU2_MPI::Reduce(&avgColorSizeLocal, &minAvgColorSize, 1, MPI_UNSIGNED_LONG, MPI_MIN, MASTER_NODE, + SU2_MPI::GetComm()); + SU2_MPI::Reduce(&avgLevelSizeLocal, &minAvgLevelSize, 1, MPI_UNSIGNED_LONG, MPI_MIN, MASTER_NODE, + SU2_MPI::GetComm()); + + static bool printed = false; + if (rank == MASTER_NODE && !printed) { + cout << "GPU ILU scheduling (worst rank):\n " << nColorsMax << " colors for the factorization (~" + << minAvgColorSize << " points/color on average),\n" + << " " << nLevelsMax << " levels for the triangular solves (~" << minAvgLevelSize + << " points/level on average)." << endl; + printed = true; + } } } /*--- Preconditioners. ---*/ - if (ilu_needed) allocAndInit(ILU_matrix, nnz_ilu * nVar * nEqn); + if (ilu_needed) { + allocAndInit(ilu.l, ilu.nnz_l * nVar * nEqn); + allocAndInit(ilu.d, nPointDomain * nVar * nEqn); + allocAndInit(ilu.u, ilu.nnz_u * nVar * nEqn); + } if (diag_needed) allocAndInit(invM, nPointDomain * nVar * nEqn); + const bool any_precond_on_device = useCuda && (jacobi_on_device || lu_sgs_on_device || ilu_needed); + + if (any_precond_on_device) { + if (nVar != nEqn) { + SU2_MPI::Error("CUDA preconditioners require square blocks.", CURRENT_FUNCTION); + } + if (nVar * nVar > 1024) { + SU2_MPI::Error("CUDA preconditioners use one thread per block entry, nVar is too large.", CURRENT_FUNCTION); + } + + if (jacobi_on_device || lu_sgs_on_device) { + d_invM = GPUMemoryAllocation::gpu_alloc(nPointDomain * nVar * nEqn * sizeof(ScalarType)); + } + + /*--- Flattens a grouped sparse pattern (levels, colors) into a host ptr and device index arrays. + * Used in ILU levels and colors, and LU-SGS levels---*/ + auto FlattenGroupToDevice = [](const auto& grouped, std::vector& group_ptr, unsigned long reserveHint, + unsigned long bound = ~0ul) { + std::vector flat_idx; + flat_idx.reserve(reserveHint); + group_ptr.clear(); + group_ptr.push_back(0); + for (auto group = 0ul; group < grouped.getOuterSize(); ++group) { + for (auto k = 0ul; k < grouped.getNumNonZeros(group); ++k) { + auto idx = grouped.getInnerIdx(group, k); + if (static_cast(idx) >= bound) + continue; // prevent out of bounds in LU-SGS kernels if more than 1 mpi task + flat_idx.push_back(static_cast(idx)); + } + group_ptr.push_back(static_cast(flat_idx.size())); + } + return GPUMemoryAllocation::gpu_alloc_cpy(flat_idx.data(), flat_idx.size() * sizeof(su2uint)); + }; + + if (lu_sgs_on_device) { + // get the zero-filled sparse pattern for the LU-SGS + const auto& pat_lusgs = geometry->GetSparsePattern(type, 0); + + /*--- Compute the levels using the lower pattern for the forward pass and + * reverse the levels for the backward pass. This works if L and U are symmetric, to be verified ---*/ + auto levels_lusgs = computeLevels(pat_lusgs.l); + + /*--- Flatten levels_lusgs. It drives both triangular solves on the device. ---*/ + d_precond_level_idx = FlattenGroupToDevice(levels_lusgs, precond_level_ptr, nPointDomain, nPointDomain); + } + + if (ilu_needed) { + /*--- The factors are built and used on the device, only the pattern and the level table + * are uploaded (once, here) because they do not change. ---*/ + gpu_ilu.nnz_l = ilu.nnz_l; + gpu_ilu.nnz_u = ilu.nnz_u; + GPUAllocAndInit(gpu_ilu.d, nPointDomain * nVar * nEqn); + GPUAllocAndInit(gpu_ilu.l, ilu.nnz_l * nVar * nEqn); + GPUAllocAndInit(gpu_ilu.u, ilu.nnz_u * nVar * nEqn); + GPUAllocAndCopy(gpu_ilu.row_ptr_l, ilu.row_ptr_l, nPointDomain + 1); + GPUAllocAndCopy(gpu_ilu.col_ind_l, ilu.col_ind_l, ilu.nnz_l); + GPUAllocAndCopy(gpu_ilu.row_ptr_u, ilu.row_ptr_u, nPointDomain + 1); + GPUAllocAndCopy(gpu_ilu.col_ind_u, ilu.col_ind_u, ilu.nnz_u); + + /*--- Flatten the coloring, the index type differs from the one of the pattern. It drives + * the factorization on the device. ---*/ + d_ilu_color_idx = FlattenGroupToDevice(color_ilu, ilu_color_ptr, nPointDomain); + + /*--- Flatten levels_ilu the same way. It drives both triangular solves on the device. ---*/ + d_precond_level_idx = FlattenGroupToDevice(levels_ilu, precond_level_ptr, nPointDomain); + } + } + /*--- Thread parallel initialization. ---*/ int num_threads = omp_get_max_threads(); /*--- Set suitable chunk sizes for light static for loops, and heavy dynamic ones, such that threads are approximately evenly loaded. ---*/ - omp_light_size = computeStaticChunkSize(nnz * nVar * nEqn, num_threads, OMP_MAX_SIZE_L); + omp_light_size = computeStaticChunkSize(nPoint * nVar * nEqn, num_threads, OMP_MAX_SIZE_L); omp_heavy_size = computeStaticChunkSize(nPointDomain, num_threads, OMP_MAX_SIZE_H); omp_num_parts = config->GetLinear_Solver_Prec_Threads(); @@ -214,13 +498,16 @@ void CSysMatrix::Initialize(unsigned long npoint, unsigned long npoi /*--- Work estimate based on non-zeros to produce balanced partitions. ---*/ - const auto row_ptr_prec = ilu_needed ? row_ptr_ilu : row_ptr; - const auto nnz_prec = row_ptr_prec[nPointDomain]; - + /*--- Cumulative nnz up to row iPoint for the preconditioner's LDU pattern. ---*/ + auto nnz_up_to = [&](unsigned long iPoint) -> unsigned long { + if (ilu_needed) return ilu.row_ptr_l[iPoint] + iPoint + ilu.row_ptr_u[iPoint]; + return mat.row_ptr_l[iPoint] + iPoint + mat.row_ptr_u[iPoint]; + }; + const auto nnz_prec = nnz_up_to(nPointDomain); const auto nnz_per_part = roundUpDiv(nnz_prec, omp_num_parts); for (auto iPoint = 0ul, part = 0ul; iPoint < nPointDomain; ++iPoint) { - if (row_ptr_prec[iPoint] >= part * nnz_per_part) omp_partitions[part++] = iPoint; + if (nnz_up_to(iPoint) >= part * nnz_per_part) omp_partitions[part++] = iPoint; } for (unsigned long thread = 0; thread < omp_num_parts; ++thread) { @@ -488,14 +775,60 @@ void CSysMatrixComms::Complete(CSysVector& x, CGeometry* geometry, const CCon #endif } +template +void CSysMatrix::QuantizeDiagonalBlocks() { + SU2_ZONE_SCOPED + + if (!quantized_mode) return; + + if (useCuda) { +#ifdef SU2_ENABLE_CUDA_KERNELS + if constexpr (su2_gpu_capable_v) { + /*--- gpu.d is already on the device - HtDTransfer() uploads it unconditionally, since + * Jacobi's own build needs the full precision diagonal regardless of quantization - so + * quantize straight from it here instead of quantizing on the host and uploading the + * result. ---*/ + SU2_DEVICE_REGION(QuantizeDiagonalBlocksGPU();) + return; + } else { + GPUNotAvailable(CURRENT_FUNCTION); + } +#else + GPUNotAvailable(CURRENT_FUNCTION); +#endif + } + + /*--- Q_LU_SGS / Q_JACOBI / Q_IDENTITY: L/U were quantized during assembly; only the diagonal needs quantization + * now. ---*/ + SU2_OMP_FOR_DYN(omp_heavy_size) + for (auto i = 0ul; i < nPointDomain; ++i) + QuantizeBlock(&mat.d[i * nVar * nVar], &q_scale.d[i * nVar], &q_blocks.d[i * nVar * nVar]); + END_SU2_OMP_FOR +} + template void CSysMatrix::SetValZero() { SU2_ZONE_SCOPED - const auto size = nnz * nVar * nEqn; - const auto chunk = roundUpDiv(size, omp_get_num_threads()); - const auto begin = chunk * omp_get_thread_num(); - const auto mySize = min(chunk, size - begin) * sizeof(ScalarType); - memset(&matrix[begin], 0, mySize); + const auto nThreads = static_cast(omp_get_num_threads()); + const auto iThread = static_cast(omp_get_thread_num()); + + auto zeroChunk = [&](auto* arr, unsigned long n) { + if (n == 0) return; + const auto chunk = roundUpDiv(n, nThreads); + const auto begin = min(chunk * iThread, n); + const auto mySize = min(chunk, n - begin) * sizeof(std::remove_pointer_t); + if (mySize) memset(&arr[begin], 0, mySize); + }; + zeroChunk(mat.d, nPoint * nVar * nEqn); + if (!quantized_mode) { + zeroChunk(mat.l, mat.nnz_l * nVar * nEqn); + zeroChunk(mat.u, mat.nnz_u * nVar * nEqn); + } else { + zeroChunk(q_scale.l, mat.nnz_l * nVar); + zeroChunk(q_scale.u, mat.nnz_l * nVar); + zeroChunk(q_blocks.l, mat.nnz_l * nVar * nEqn); + zeroChunk(q_blocks.u, mat.nnz_u * nVar * nEqn); + } SU2_OMP_BARRIER } @@ -503,30 +836,23 @@ template void CSysMatrix::SetValDiagonalZero() { SU2_ZONE_SCOPED SU2_OMP_FOR_STAT(omp_heavy_size) - for (auto iPoint = 0ul; iPoint < nPointDomain; ++iPoint) - for (auto index = 0ul; index < nVar * nEqn; ++index) matrix[dia_ptr[iPoint] * nVar * nEqn + index] = 0.0; + for (auto iVar = 0ul; iVar < nPointDomain * nVar * nEqn; ++iVar) mat.d[iVar] = 0; END_SU2_OMP_FOR } -/*--- Helper function to regularize small pivots ---*/ -template -inline void RegularizePivot(ScalarType& pivot, unsigned long row, unsigned long col, const char* context) { - const float eps = 1e-12; - if (std::abs(pivot) < eps) { - pivot = std::copysign(eps, SU2_TYPE::GetValue(pivot)); -#ifndef NDEBUG - std::cout << context << ": Regularized small pivot A(" << row << "," << col << ") to " << pivot << std::endl; -#endif - } -} - template -void CSysMatrix::Gauss_Elimination(ScalarType* matrix, ScalarType* vec) const { +void CSysMatrix::GaussElimination(ScalarType* matrix, ScalarType* vec) const { #ifdef USE_MKL_LAPACK // With MKL_DIRECT_CALL enabled, this is significantly faster than native code on Intel Architectures. lapack_int ipiv[MAXNVAR]; - LAPACKE_dgetrf(LAPACK_ROW_MAJOR, nVar, nVar, matrix, nVar, ipiv); - LAPACKE_dgetrs(LAPACK_ROW_MAJOR, 'N', nVar, 1, matrix, nVar, ipiv, vec, 1); + if constexpr (std::is_same_v) { + LAPACKE_dgetrf(LAPACK_ROW_MAJOR, nVar, nVar, matrix, nVar, ipiv); + LAPACKE_dgetrs(LAPACK_ROW_MAJOR, 'N', nVar, 1, matrix, nVar, ipiv, vec, 1); + } else { + static_assert(std::is_same_v, "ScalarType not handled"); + LAPACKE_sgetrf(LAPACK_ROW_MAJOR, nVar, nVar, matrix, nVar, ipiv); + LAPACKE_sgetrs(LAPACK_ROW_MAJOR, 'N', nVar, 1, matrix, nVar, ipiv, vec, 1); + } #else #define A(I, J) matrix[(I)*nVar + (J)] @@ -535,7 +861,7 @@ void CSysMatrix::Gauss_Elimination(ScalarType* matrix, ScalarType* v for (auto iVar = 1ul; iVar < nVar; iVar++) { for (auto jVar = 0ul; jVar < iVar; jVar++) { /*--- Regularize pivot if too small to prevent divide-by-zero ---*/ - RegularizePivot(A(jVar, jVar), jVar, jVar, "DEBUG Gauss_Elimination"); + RegularizePivot(A(jVar, jVar), jVar, jVar, "DEBUG GaussElimination"); ScalarType weight = A(iVar, jVar) / A(jVar, jVar); @@ -550,7 +876,7 @@ void CSysMatrix::Gauss_Elimination(ScalarType* matrix, ScalarType* v for (auto jVar = iVar + 1; jVar < nVar; jVar++) vec[iVar] -= A(iVar, jVar) * vec[jVar]; /*--- Regularize diagonal if too small ---*/ - RegularizePivot(A(iVar, iVar), iVar, iVar, "DEBUG Gauss_Elimination backsubst"); + RegularizePivot(A(iVar, iVar), iVar, iVar, "DEBUG GaussElimination backsubst"); vec[iVar] /= A(iVar, iVar); } @@ -561,67 +887,60 @@ void CSysMatrix::Gauss_Elimination(ScalarType* matrix, ScalarType* v template void CSysMatrix::MatrixInverse(ScalarType* matrix, ScalarType* inverse) const { /*--- This is a generalization of Gaussian elimination for multiple rhs' (the basis vectors). - We could call "Gauss_Elimination" multiple times or fully generalize it for multiple rhs, + We could call "GaussElimination" multiple times or fully generalize it for multiple rhs, the performance of both routines would suffer in both cases without the use of exotic templating. And so it feels reasonable to have some duplication here. ---*/ assert((matrix != inverse) && "Output cannot be the same as the input."); + /*--- Inversion ---*/ +#ifdef USE_MKL_LAPACK + // With MKL_DIRECT_CALL enabled, this is significantly faster than native code on Intel Architectures. #define M(I, J) inverse[(I)*nVar + (J)] - /*--- Initialize the inverse with the identity. ---*/ + /*--- Initialize the inverse with the identity, LAPACKE_?getrs solves for it as the rhs. ---*/ for (auto iVar = 0ul; iVar < nVar; iVar++) for (auto jVar = 0ul; jVar < nVar; jVar++) M(iVar, jVar) = ScalarType(iVar == jVar); +#undef M - /*--- Inversion ---*/ -#ifdef USE_MKL_LAPACK - // With MKL_DIRECT_CALL enabled, this is significantly faster than native code on Intel Architectures. lapack_int ipiv[MAXNVAR]; - LAPACKE_dgetrf(LAPACK_ROW_MAJOR, nVar, nVar, matrix, nVar, ipiv); - LAPACKE_dgetrs(LAPACK_ROW_MAJOR, 'N', nVar, nVar, matrix, nVar, ipiv, inverse, nVar); -#else -#define A(I, J) matrix[(I)*nVar + (J)] - - /*--- Transform system in Upper Matrix ---*/ - for (auto iVar = 1ul; iVar < nVar; iVar++) { - for (auto jVar = 0ul; jVar < iVar; jVar++) { - /*--- Regularize pivot if too small to prevent divide-by-zero ---*/ - RegularizePivot(A(jVar, jVar), jVar, jVar, "MatrixInverse"); - - ScalarType weight = A(iVar, jVar) / A(jVar, jVar); - for (auto kVar = jVar; kVar < nVar; kVar++) A(iVar, kVar) -= weight * A(jVar, kVar); - - /*--- at this stage M is lower triangular so not all cols need updating ---*/ - for (auto kVar = 0ul; kVar <= jVar; kVar++) M(iVar, kVar) -= weight * M(jVar, kVar); - } - } - - /*--- Backwards substitution ---*/ - for (auto iVar = nVar; iVar > 0ul;) { - iVar--; // unsigned type - for (auto jVar = iVar + 1; jVar < nVar; jVar++) - for (auto kVar = 0ul; kVar < nVar; kVar++) M(iVar, kVar) -= A(iVar, jVar) * M(jVar, kVar); - - /*--- Regularize diagonal if too small ---*/ - RegularizePivot(A(iVar, iVar), iVar, iVar, "DEBUG MatrixInverse backsubst"); - - for (auto kVar = 0ul; kVar < nVar; kVar++) { - M(iVar, kVar) /= A(iVar, iVar); - } + if constexpr (std::is_same_v) { + LAPACKE_dgetrf(LAPACK_ROW_MAJOR, nVar, nVar, matrix, nVar, ipiv); + LAPACKE_dgetrs(LAPACK_ROW_MAJOR, 'N', nVar, nVar, matrix, nVar, ipiv, inverse, nVar); + } else { + static_assert(std::is_same_v, "ScalarType not handled"); + LAPACKE_sgetrf(LAPACK_ROW_MAJOR, nVar, nVar, matrix, nVar, ipiv); + LAPACKE_sgetrs(LAPACK_ROW_MAJOR, 'N', nVar, nVar, matrix, nVar, ipiv, inverse, nVar); } -#undef A +#else + /*--- Shared with the device implementation, see CMatrixInverse.hpp. ---*/ + SU2_LinAlg::MatrixInverse(nVar, matrix, inverse); #endif -#undef M } template void CSysMatrix::DeleteValsRowi(unsigned long block_i, unsigned long row) { SU2_ZONE_SCOPED - for (auto index = row_ptr[block_i]; index < row_ptr[block_i + 1]; index++) { - for (auto iVar = 0u; iVar < nVar; iVar++) - matrix[index * nVar * nVar + row * nVar + iVar] = 0.0; // Delete row values in the block - if (col_ind[index] == block_i) - matrix[index * nVar * nVar + row * nVar + row] = 1.0; // Set 1 to the diagonal element + const auto blkSz = nVar * nEqn; + + auto* d = &mat.d[block_i * blkSz]; + for (auto iVar = 0u; iVar < nEqn; iVar++) d[row * nEqn + iVar] = 0.0; + d[row * nEqn + row] = 1.0; + + if (quantized_mode) { + for (auto k = mat.row_ptr_l[block_i]; k < mat.row_ptr_l[block_i + 1]; ++k) { + for (auto iVar = 0u; iVar < nEqn; iVar++) q_blocks.l[k * blkSz + row * nEqn + iVar] = 0; + } + for (auto k = mat.row_ptr_u[block_i]; k < mat.row_ptr_u[block_i + 1]; ++k) { + for (auto iVar = 0u; iVar < nEqn; iVar++) q_blocks.u[k * blkSz + row * nEqn + iVar] = 0; + } + } else { + for (auto k = mat.row_ptr_l[block_i]; k < mat.row_ptr_l[block_i + 1]; ++k) { + for (auto iVar = 0u; iVar < nEqn; iVar++) mat.l[k * blkSz + row * nEqn + iVar] = 0; + } + for (auto k = mat.row_ptr_u[block_i]; k < mat.row_ptr_u[block_i + 1]; ++k) { + for (auto iVar = 0u; iVar < nEqn; iVar++) mat.u[k * blkSz + row * nEqn + iVar] = 0; + } } } @@ -629,6 +948,22 @@ template void CSysMatrix::MatrixVectorProduct(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, const CConfig* config) const { SU2_ZONE_SCOPED + + if (useCuda) { +#ifdef SU2_ENABLE_CUDA_KERNELS + if constexpr (su2_gpu_capable_v) { + BEGIN_SU2_DEVICE_REGION + MatrixVectorProductGPU(vec, prod, geometry, config); + END_SU2_DEVICE_REGION + return; + } else { + GPUNotAvailable(CURRENT_FUNCTION); + } +#else + GPUNotAvailable(CURRENT_FUNCTION); +#endif + } + /*--- Some checks for consistency between CSysMatrix and the CSysVectors ---*/ #ifndef NDEBUG if ((nEqn != vec.GetNVar()) || (nVar != prod.GetNVar())) { @@ -645,11 +980,19 @@ void CSysMatrix::MatrixVectorProduct(const CSysVector& v SU2_OMP_BARRIER - SU2_OMP_FOR_DYN(omp_heavy_size) - for (auto row_i = 0ul; row_i < nPointDomain; row_i++) { - RowProduct(vec, row_i, &prod[row_i * nVar]); + if (quantized_mode) { + SU2_OMP_FOR_DYN(omp_heavy_size) + for (auto row_i = 0ul; row_i < nPointDomain; row_i++) { + QuantizedRowProduct(vec, row_i, &prod[row_i * nVar]); + } + END_SU2_OMP_FOR + } else { + SU2_OMP_FOR_DYN(omp_heavy_size) + for (auto row_i = 0ul; row_i < nPointDomain; row_i++) { + RowProduct(vec, row_i, &prod[row_i * nVar]); + } + END_SU2_OMP_FOR } - END_SU2_OMP_FOR /*--- MPI Parallelization. ---*/ @@ -660,6 +1003,25 @@ void CSysMatrix::MatrixVectorProduct(const CSysVector& v template void CSysMatrix::BuildJacobiPreconditioner() { SU2_ZONE_SCOPED + + /*--- Independent of invM (reads/quantizes mat.d, a no-op unless quantized_mode); done first, + * unconditionally, so it runs whichever branch below builds invM (in particular the + * jacobi_on_device one, which returns early). ---*/ + QuantizeDiagonalBlocks(); + + if (jacobi_on_device) { +#ifdef SU2_ENABLE_CUDA_KERNELS + if constexpr (su2_gpu_capable_v) { + SU2_DEVICE_REGION(BuildJacobiPreconditionerGPU();) + return; + } else { + GPUNotAvailable(CURRENT_FUNCTION); + } +#else + GPUNotAvailable(CURRENT_FUNCTION); +#endif + } + /*--- Build Jacobi preconditioner (M = D), compute and store the inverses of the diagonal blocks. ---*/ SU2_OMP_FOR_DYN(omp_heavy_size) for (unsigned long iPoint = 0; iPoint < nPointDomain; iPoint++) @@ -672,6 +1034,20 @@ void CSysMatrix::ComputeJacobiPreconditioner(const CSysVector& prod, CGeometry* geometry, const CConfig* config) const { SU2_ZONE_SCOPED + + if (useCuda) { +#ifdef SU2_ENABLE_CUDA_KERNELS + if constexpr (su2_gpu_capable_v) { + SU2_DEVICE_REGION(ComputeJacobiPreconditionerGPU(vec, prod, geometry, config);) + return; + } else { + GPUNotAvailable(CURRENT_FUNCTION); + } +#else + GPUNotAvailable(CURRENT_FUNCTION); +#endif + } + /*--- Apply Jacobi preconditioner, y = D^{-1} * x, the inverse of the diagonal is already known. ---*/ SU2_OMP_BARRIER SU2_OMP_FOR_DYN(omp_heavy_size) @@ -686,49 +1062,66 @@ void CSysMatrix::ComputeJacobiPreconditioner(const CSysVector void CSysMatrix::BuildILUPreconditioner() { + SU2_ZONE_SCOPED + + if (useCuda) { +#ifdef SU2_ENABLE_CUDA_KERNELS + if constexpr (su2_gpu_capable_v) { + SU2_DEVICE_REGION(BuildILUPreconditionerGPU();) + return; + } else { + GPUNotAvailable(CURRENT_FUNCTION); + } +#else + GPUNotAvailable(CURRENT_FUNCTION); +#endif + } + const auto blockSize = nVar * nVar; ScalarType Lij[MAXNVAR * MAXNVAR], Lij_Ujk[MAXNVAR * MAXNVAR]; /*--- Helper to copy block matrix to compute factorization in-place. ---*/ auto InitIluRow = [&](const auto iPoint) { + MatrixCopy(&mat.d[iPoint * blockSize], &ilu.d[iPoint * blockSize]); + if (ilu_fill_in == 0) { - /*--- ILU0, direct copy to initialize. ---*/ - const auto begin = row_ptr_ilu[iPoint] * blockSize; - const auto end = row_ptr_ilu[iPoint + 1] * blockSize; - SU2_OMP_SIMD - for (unsigned long k = begin; k < end; ++k) ILU_matrix[k] = matrix[k]; + /*--- ILU0: Same sparse pattern, copy L and U blocks directly. ---*/ + auto copy = [&](const su2uint* row_ptr, const ScalarType* mat, ScalarType* ilu) { + const unsigned long begin = row_ptr[iPoint] * blockSize; + const unsigned long end = row_ptr[iPoint + 1] * blockSize; + SU2_OMP_SIMD + for (auto k = begin; k < end; ++k) ilu[k] = mat[k]; + }; + copy(ilu.row_ptr_l, mat.l, ilu.l); + copy(ilu.row_ptr_u, mat.u, ilu.u); return; } - /*--- ILUn, clear or copy the entries of the matrix. ---*/ - auto indexMat = row_ptr[iPoint]; - const auto endMat = row_ptr[iPoint + 1]; - for (auto index = row_ptr_ilu[iPoint]; index < row_ptr_ilu[iPoint + 1];) { - const auto jPoint = col_ind_ilu[index]; - const auto jPointMat = col_ind[indexMat]; - if (jPoint < jPointMat || indexMat == endMat) { - /*--- ILU column has not caught up with matrix column or all matrix columns were used. ---*/ - ZeroMatrix(&ILU_matrix[index * blockSize]); - ++index; - } else { - /*--- Columns match, copy the matrix block. ---*/ - if (jPoint == jPointMat) { - MatrixCopy(&matrix[indexMat * blockSize], &ILU_matrix[index * blockSize]); - ++index; + /*--- ILUn: Merge-scan L and U via shared lambda. ---*/ + auto scatterPart = [&](const su2uint* mat_row_ptr, const su2uint* mat_col_ind, const ScalarType* mat_vals, + const su2uint* ilu_row_ptr, const su2uint* ilu_col_ind, ScalarType* ilu_vals) { + auto km = mat_row_ptr[iPoint], km_end = mat_row_ptr[iPoint + 1]; + for (auto k = ilu_row_ptr[iPoint]; k < ilu_row_ptr[iPoint + 1]; ++k) { + const auto jPoint = ilu_col_ind[k]; + while (km < km_end && mat_col_ind[km] < jPoint) ++km; + if (km < km_end && mat_col_ind[km] == jPoint) { + MatrixCopy(&mat_vals[km * blockSize], &ilu_vals[k * blockSize]); + } else { + ZeroMatrix(&ilu_vals[k * blockSize]); } - /*--- We've either copied the matrix column or it has not caught up with the ILU column. ---*/ - ++indexMat; } - } + }; + scatterPart(mat.row_ptr_l, mat.col_ind_l, mat.l, ilu.row_ptr_l, ilu.col_ind_l, ilu.l); + scatterPart(mat.row_ptr_u, mat.col_ind_u, mat.u, ilu.row_ptr_u, ilu.col_ind_u, ilu.u); }; /*--- Update one row of the LU matrix. ---*/ auto BuildIluRow = [&](const auto iPoint, const auto begin, const auto end) { /*--- For this row (unknown), loop over its lower diagonal entries. ---*/ - for (auto index = row_ptr_ilu[iPoint]; index < dia_ptr_ilu[iPoint]; ++index) { + for (auto kl = ilu.row_ptr_l[iPoint]; kl < ilu.row_ptr_l[iPoint + 1]; ++kl) { /*--- jPoint is the column index (jPoint < iPoint). ---*/ - const auto jPoint = col_ind_ilu[index]; + const auto jPoint = ilu.col_ind_l[kl]; /*--- We only care about the sub matrix within "begin" and "end-1". ---*/ @@ -736,16 +1129,16 @@ void CSysMatrix::BuildILUPreconditioner() { /*--- Multiply the block by the inverse of the corresponding diagonal block. ---*/ - auto* Block_ij = &ILU_matrix[index * blockSize]; - const auto* invUjj = &ILU_matrix[dia_ptr_ilu[jPoint] * blockSize]; + auto* Block_ij = &ilu.l[kl * blockSize]; + const auto* invUjj = &ilu.d[jPoint * blockSize]; MatrixMatrixProduct(Block_ij, invUjj, Lij); /*--- Lij holds Aij*inv(Ujj). Jump to the upper part of the jPoint row. ---*/ - for (auto index_ = dia_ptr_ilu[jPoint] + 1; index_ < row_ptr_ilu[jPoint + 1]; ++index_) { + for (auto ku = ilu.row_ptr_u[jPoint]; ku < ilu.row_ptr_u[jPoint + 1]; ++ku) { /*--- Get the column index (kPoint > jPoint). ---*/ - const auto kPoint = col_ind_ilu[index_]; + const auto kPoint = ilu.col_ind_u[ku]; if (kPoint >= end) break; /*--- If Aik exists, update it: Aik -= Lij * Ujk ---*/ @@ -753,7 +1146,7 @@ void CSysMatrix::BuildILUPreconditioner() { auto* Block_ik = GetBlock_ILUMatrix(iPoint, kPoint); if (Block_ik == nullptr) continue; - const auto* Ujk = &ILU_matrix[index_ * blockSize]; + const auto* Ujk = &ilu.u[ku * blockSize]; MatrixMatrixProduct(Lij, Ujk, Lij_Ujk); MatrixSubtraction(Block_ik, Lij_Ujk, Block_ik); } @@ -813,6 +1206,21 @@ void CSysMatrix::BuildILUPreconditioner() { template void CSysMatrix::ComputeILUPreconditioner(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, const CConfig* config) const { + SU2_ZONE_SCOPED + + if (useCuda) { +#ifdef SU2_ENABLE_CUDA_KERNELS + if constexpr (su2_gpu_capable_v) { + SU2_DEVICE_REGION(ComputeILUPreconditionerGPU(vec, prod);) + return; + } else { + GPUNotAvailable(CURRENT_FUNCTION); + } +#else + GPUNotAvailable(CURRENT_FUNCTION); +#endif + } + /*--- Coherent view of vectors. ---*/ SU2_OMP_BARRIER @@ -825,11 +1233,10 @@ void CSysMatrix::ComputeILUPreconditioner(const CSysVector::ComputeILUPreconditioner(const CSysVector= end) break; - const auto* Block_ij = &ILU_matrix[index * blockSize]; + const auto* Block_ij = &ilu.u[ku * blockSize]; MatrixVectorProductSub(Block_ij, &prod[jPoint * nVar], aux_vec); } @@ -891,11 +1298,69 @@ void CSysMatrix::ComputeILUPreconditioner(const CSysVector +void CSysMatrix::BuildLU_SGSPreconditioner() { + SU2_ZONE_SCOPED + + /*--- Quantize diagonal blocks if mode is active ---*/ + QuantizeDiagonalBlocks(); + + /*--- if on GPU, precompute the inverse of the diagonal D. Otherwise, this is a no-op ---*/ + if (useCuda) { +#ifdef SU2_ENABLE_CUDA_KERNELS + if constexpr (su2_gpu_capable_v) { + SU2_DEVICE_REGION(BuildLU_SGSPreconditionerGPU();) + return; + } else { + GPUNotAvailable(CURRENT_FUNCTION); + } +#else + GPUNotAvailable(CURRENT_FUNCTION); +#endif + } +} + template void CSysMatrix::ComputeLU_SGSPreconditioner(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, const CConfig* config) const { SU2_ZONE_SCOPED + + /*--- First part of the symmetric iteration: (D+L).x* = b ---*/ + ComputeLU_SGSPreconditionerForward(vec, prod); + + /*--- MPI Parallelization ---*/ + + CSysMatrixComms::Initiate(prod, geometry, config); + CSysMatrixComms::Complete(prod, geometry, config); + + /*--- Second part of the symmetric iteration: (D+U).x_(1) = D.x* ---*/ + ComputeLU_SGSPreconditionerBackward(prod); + + /*--- MPI Parallelization ---*/ + + CSysMatrixComms::Initiate(prod, geometry, config); + CSysMatrixComms::Complete(prod, geometry, config); +} + +template +void CSysMatrix::ComputeLU_SGSPreconditionerForward(const CSysVector& vec, + CSysVector& prod) const { + SU2_ZONE_SCOPED + + if (useCuda) { +#ifdef SU2_ENABLE_CUDA_KERNELS + if constexpr (su2_gpu_capable_v) { + SU2_DEVICE_REGION(ComputeLU_SGSForwardGPU(vec, prod);) + return; + } else { + GPUNotAvailable(CURRENT_FUNCTION); + } +#else + GPUNotAvailable(CURRENT_FUNCTION); +#endif + } + /*--- First part of the symmetric iteration: (D+L).x* = b ---*/ /*--- Coherent view of vectors. ---*/ @@ -915,22 +1380,44 @@ void CSysMatrix::ComputeLU_SGSPreconditioner(const CSysVector +void CSysMatrix::ComputeLU_SGSPreconditionerBackward(CSysVector& prod) const { + SU2_ZONE_SCOPED /*--- Second part of the symmetric iteration: (D+U).x_(1) = D.x* ---*/ + if (useCuda) { +#ifdef SU2_ENABLE_CUDA_KERNELS + if constexpr (su2_gpu_capable_v) { + SU2_DEVICE_REGION(ComputeLU_SGSBackwardGPU(prod);) + return; + } else { + GPUNotAvailable(CURRENT_FUNCTION); + } +#else + GPUNotAvailable(CURRENT_FUNCTION); +#endif + } + /*--- OpenMP Parallelization ---*/ SU2_OMP_FOR_STAT(1) for (unsigned long thread = 0; thread < omp_num_parts; ++thread) { @@ -940,21 +1427,27 @@ void CSysMatrix::ComputeLU_SGSPreconditioner(const CSysVector begin;) { - iPoint--; // because of unsigned type - auto idx = iPoint * nVar; - DiagonalProduct(prod, iPoint, dia_prod); // Compute D.x* - UpperProduct(prod, iPoint, row_end, up_prod); // Compute U.x_(n+1) - VectorSubtraction(dia_prod, up_prod, &prod[idx]); // Compute y = D.x*-U.x_(n+1) - Gauss_Elimination(iPoint, &prod[idx]); // Solve D.x* = y + if (quantized_mode) { + for (auto iPoint = row_end; iPoint > begin;) { + iPoint--; + auto idx = iPoint * nVar; + QuantizedDiagonalProduct(prod, iPoint, dia_prod); + QuantizedUpperProduct(prod, iPoint, row_end, up_prod); + VectorSubtraction(dia_prod, up_prod, &prod[idx]); + QuantizedGaussElimination(iPoint, &prod[idx]); + } + } else { + for (auto iPoint = row_end; iPoint > begin;) { + iPoint--; // because of unsigned type + auto idx = iPoint * nVar; + DiagonalProduct(prod, iPoint, dia_prod); // Compute D.x* + UpperProduct(prod, iPoint, row_end, up_prod); // Compute U.x_(n+1) + VectorSubtraction(dia_prod, up_prod, &prod[idx]); // Compute y = D.x*-U.x_(n+1) + GaussElimination(iPoint, &prod[idx]); // Solve D.x* = y + } } } END_SU2_OMP_FOR - - /*--- MPI Parallelization ---*/ - - CSysMatrixComms::Initiate(prod, geometry, config); - CSysMatrixComms::Complete(prod, geometry, config); } template @@ -1031,14 +1524,14 @@ void CSysMatrix::ComputeLineletPreconditioner(const CSysVector::ComputeLineletPreconditioner(const CSysVector 0; --iElem) { @@ -1117,31 +1610,23 @@ void CSysMatrix::EnforceSolutionAtNode(const unsigned long node_i, c * symmetric the entire column may not be eliminated, the result (matrix and vector) is still correct. * The vector is updated with the product of column i by the known (enforced) solution at node i. ---*/ - for (auto index = row_ptr[node_i]; index < row_ptr[node_i + 1]; ++index) { - auto node_j = col_ind[index]; - - /*--- The diagonal block is handled outside the loop. ---*/ - if (node_j == node_i) continue; - - /*--- Delete block j on row i (bij) and ATTEMPT to delete block i on row j (bji). ---*/ - auto bij = &matrix[index * nVar * nVar]; + /*--- Visit off-diagonal columns (L then U; diagonal is handled by SetVal2Diag outside). ---*/ + auto processOffDiag = [&](unsigned long node_j) { + auto bij = GetBlock(node_i, node_j); auto bji = GetBlock(node_j, node_i); - - /*--- The "attempt" part. ---*/ if (bji == nullptr) { node_j = node_i; bji = bij; } - for (auto iVar = 0ul; iVar < nVar; ++iVar) { for (auto jVar = 0ul; jVar < nVar; ++jVar) { - /*--- Column product. ---*/ b[node_j * nVar + iVar] -= bji[iVar * nVar + jVar] * x_i[jVar]; - /*--- Delete blocks. ---*/ bij[iVar * nVar + jVar] = bji[iVar * nVar + jVar] = 0.0; } } - } + }; + for (auto k = mat.row_ptr_l[node_i]; k < mat.row_ptr_l[node_i + 1]; ++k) processOffDiag(mat.col_ind_l[k]); + for (auto k = mat.row_ptr_u[node_i]; k < mat.row_ptr_u[node_i + 1]; ++k) processOffDiag(mat.col_ind_u[k]); /*--- Set the diagonal block to the identity. ---*/ SetVal2Diag(node_i, 1.0); @@ -1155,54 +1640,33 @@ template void CSysMatrix::EnforceZeroProjection(unsigned long node_i, const OtherType* n, CSysVector& b) { SU2_ZONE_SCOPED - for (auto index = row_ptr[node_i]; index < row_ptr[node_i + 1]; ++index) { - const auto node_j = col_ind[index]; - - /*--- Remove product components of block j on row i (bij) and ATTEMPT - * to remove solution components of block i on row j (bji). - * This is identical to symmetry correction applied to gradients - * but extended to the entire matrix. ---*/ - - auto bij = &matrix[index * nVar * nVar]; + /*--- Visit all columns (L, diagonal, U) of row node_i. ---*/ + auto processCol = [&](unsigned long node_j, bool isDiag) { + auto bij = GetBlock(node_i, node_j); auto bji = GetBlock(node_j, node_i); - - /*--- Attempt to remove solution components. ---*/ ScalarType nbn{}; if (bji != nullptr) { for (auto iVar = 0ul; iVar < nVar; ++iVar) { ScalarType proj{}; - for (auto jVar = 0ul; jVar < nVar; ++jVar) { - proj += bji[iVar * nVar + jVar] * PassiveAssign(n[jVar]); - } - for (auto jVar = 0ul; jVar < nVar; ++jVar) { - bji[iVar * nVar + jVar] -= proj * PassiveAssign(n[jVar]); - } + for (auto jVar = 0ul; jVar < nVar; ++jVar) proj += bji[iVar * nVar + jVar] * PassiveAssign(n[jVar]); + for (auto jVar = 0ul; jVar < nVar; ++jVar) bji[iVar * nVar + jVar] -= proj * PassiveAssign(n[jVar]); nbn += proj * PassiveAssign(n[iVar]); } } - - /*--- Product components. ---*/ for (auto jVar = 0ul; jVar < nVar; ++jVar) { ScalarType proj{}; - for (auto iVar = 0ul; iVar < nVar; ++iVar) { - proj += bij[iVar * nVar + jVar] * PassiveAssign(n[iVar]); - } - for (auto iVar = 0ul; iVar < nVar; ++iVar) { - bij[iVar * nVar + jVar] -= proj * PassiveAssign(n[iVar]); - } + for (auto iVar = 0ul; iVar < nVar; ++iVar) proj += bij[iVar * nVar + jVar] * PassiveAssign(n[iVar]); + for (auto iVar = 0ul; iVar < nVar; ++iVar) bij[iVar * nVar + jVar] -= proj * PassiveAssign(n[iVar]); } - - /*--- This part doesn't have the "*2" factor because the product components - * were removed from the result of removing the solution components - * instead of from the original block (bji == bij). ---*/ - if (node_i == node_j) { - for (auto iVar = 0ul; iVar < nVar; ++iVar) { - for (auto jVar = 0ul; jVar < nVar; ++jVar) { + if (isDiag) { + for (auto iVar = 0ul; iVar < nVar; ++iVar) + for (auto jVar = 0ul; jVar < nVar; ++jVar) bij[iVar * nVar + jVar] += PassiveAssign(n[iVar]) * nbn * PassiveAssign(n[jVar]); - } - } } - } + }; + for (auto k = mat.row_ptr_l[node_i]; k < mat.row_ptr_l[node_i + 1]; ++k) processCol(mat.col_ind_l[k], false); + processCol(node_i, true); + for (auto k = mat.row_ptr_u[node_i]; k < mat.row_ptr_u[node_i + 1]; ++k) processCol(mat.col_ind_u[k], false); OtherType proj{}; for (auto iVar = 0ul; iVar < nVar; ++iVar) proj += b(node_i, iVar) * n[iVar]; @@ -1212,16 +1676,31 @@ void CSysMatrix::EnforceZeroProjection(unsigned long node_i, const O template void CSysMatrix::SetDiagonalAsColumnSum() { SU2_ZONE_SCOPED + const auto blkSz = nVar * nEqn; SU2_OMP_FOR_DYN(omp_heavy_size) for (auto iPoint = 0ul; iPoint < nPoint; ++iPoint) { - auto block_ii = &matrix[dia_ptr[iPoint] * nVar * nEqn]; - - for (auto k = 0ul; k < nVar * nEqn; ++k) block_ii[k] = 0.0; - - for (auto k = row_ptr[iPoint]; k < row_ptr[iPoint + 1]; ++k) { - auto block_ji = &matrix[col_ptr[k] * nVar * nEqn]; - if (block_ji != block_ii) MatrixSubtraction(block_ii, block_ji, block_ii); + auto* d_i = &mat.d[iPoint * blkSz]; + for (auto k = 0ul; k < blkSz; ++k) d_i[k] = 0.0; + + if (!quantized_mode) { + /*--- For each L entry (iPoint, j): subtract its U-transpose (j, iPoint). ---*/ + for (auto k_l = mat.row_ptr_l[iPoint]; k_l < mat.row_ptr_l[iPoint + 1]; ++k_l) + MatrixSubtraction(d_i, &mat.u[l_to_u_transp[k_l] * blkSz], d_i); + + /*--- For each U entry (iPoint, j): subtract its L-transpose (j, iPoint). ---*/ + for (auto k_u = mat.row_ptr_u[iPoint]; k_u < mat.row_ptr_u[iPoint + 1]; ++k_u) + MatrixSubtraction(d_i, &mat.l[u_to_l_transp[k_u] * blkSz], d_i); + } else { + auto subtractTransp = [&](su2uint k_transp, const QuantScaleType* qs, const QuantType* qv) { + const CBlockView view{nullptr, &qs[k_transp * nVar], &qv[k_transp * blkSz], nVar}; + for (auto i = 0ul; i < nVar; ++i) + for (auto j = 0ul; j < nEqn; ++j) d_i[i * nEqn + j] -= view(i, j); + }; + for (auto k_l = mat.row_ptr_l[iPoint]; k_l < mat.row_ptr_l[iPoint + 1]; ++k_l) + subtractTransp(l_to_u_transp[k_l], q_scale.u, q_blocks.u); + for (auto k_u = mat.row_ptr_u[iPoint]; k_u < mat.row_ptr_u[iPoint + 1]; ++k_u) + subtractTransp(u_to_l_transp[k_u], q_scale.l, q_blocks.l); } } END_SU2_OMP_FOR @@ -1248,38 +1727,34 @@ void CSysMatrix::TransposeInPlace() { /*--- Swap ij with ji and transpose them. ---*/ - if (edge_ptr) { - /*--- The FV way. ---*/ + if (edge_ptr_l) { + /*--- FV path: each edge maps to one U and one L block. ---*/ SU2_OMP_FOR_DYN(omp_heavy_size * 2) - for (auto iEdge = 0ul; iEdge < edge_ptr.nEdge; ++iEdge) { - auto bij = &matrix[edge_ptr(iEdge, 0) * nVar * nVar]; - auto bji = &matrix[edge_ptr(iEdge, 1) * nVar * nVar]; - - swapAndTransp(nVar, bij, bji); + for (auto iEdge = 0ul; iEdge < mat.nnz_l; ++iEdge) { + auto* bij_u = &mat.u[iEdge * nVar * nVar]; + auto* bji_l = &mat.l[edge_ptr_l[iEdge] * nVar * nVar]; + swapAndTransp(nVar, bij_u, bji_l); } END_SU2_OMP_FOR - } else if (col_ptr) { - /*--- If the column pointer was built. ---*/ + } else if (l_to_u_transp) { + /*--- FEM/general path: use the L→U transpose map (one L entry per pair). ---*/ SU2_OMP_FOR_DYN(omp_heavy_size) for (auto iPoint = 0ul; iPoint < nPoint; ++iPoint) { - for (auto k = row_ptr[iPoint]; k < dia_ptr[iPoint]; ++k) { - auto bij = &matrix[k * nVar * nVar]; - auto bji = &matrix[col_ptr[k] * nVar * nVar]; - - swapAndTransp(nVar, bij, bji); + for (auto k_l = mat.row_ptr_l[iPoint]; k_l < mat.row_ptr_l[iPoint + 1]; ++k_l) { + const auto k_u = l_to_u_transp[k_l]; + swapAndTransp(nVar, &mat.u[k_u * nVar * nVar], &mat.l[k_l * nVar * nVar]); } } END_SU2_OMP_FOR } else { - /*--- Slow fallback, needs to search for ji. ---*/ + /*--- Slow fallback: search for each U entry's L partner via GetBlock. ---*/ SU2_OMP_FOR_DYN(omp_heavy_size) for (auto iPoint = 0ul; iPoint < nPoint; ++iPoint) { - for (auto k = dia_ptr[iPoint] + 1ul; k < row_ptr[iPoint + 1]; ++k) { - const auto jPoint = col_ind[k]; - auto bij = &matrix[k * nVar * nVar]; - auto bji = GetBlock(jPoint, iPoint); + for (auto k_u = mat.row_ptr_u[iPoint]; k_u < mat.row_ptr_u[iPoint + 1]; ++k_u) { + const auto jPoint = mat.col_ind_u[k_u]; + auto* bij = &mat.u[k_u * nVar * nVar]; + auto* bji = GetBlock(jPoint, iPoint); assert(bji && "Pattern is not symmetric."); - swapAndTransp(nVar, bij, bji); } } @@ -1290,7 +1765,7 @@ void CSysMatrix::TransposeInPlace() { SU2_OMP_FOR_STAT(omp_heavy_size) for (auto iPoint = 0ul; iPoint < nPoint; ++iPoint) { - auto bii = &matrix[dia_ptr[iPoint] * nVar * nVar]; + auto bii = &mat.d[iPoint * nVar * nVar]; for (auto i = 0ul; i < nVar; ++i) for (auto j = 0ul; j < i; ++j) std::swap(bii[i * nVar + j], bii[j * nVar + i]); } @@ -1301,21 +1776,38 @@ void CSysMatrix::TransposeInPlace() { pastix_wrapper.SetTransposedSolve(); END_SU2_OMP_MASTER #endif + +#ifdef SU2_ENABLE_CUDA_KERNELS + if constexpr (su2_gpu_capable_v) { + if (useCuda) { + BEGIN_SU2_DEVICE_REGION { + HtDTransfer(); + /*--- The factors of one orientation are not a starting point for the other. ---*/ + ilu_can_refine = false; + } + END_SU2_DEVICE_REGION + } + } +#endif } template void CSysMatrix::MatrixMatrixAddition(ScalarType alpha, const CSysMatrix& B) { SU2_ZONE_SCOPED - /*--- Check that the sparse structure is shared between the two matrices, - * comparing pointers is ok as they are obtained from CGeometry. ---*/ - bool ok = (row_ptr == B.row_ptr) && (col_ind == B.col_ind) && (nVar == B.nVar) && (nEqn == B.nEqn) && (nnz == B.nnz); - - if (!ok) { - SU2_MPI::Error("Matrices do not have compatible sparsity.", CURRENT_FUNCTION); - } + /*--- Check that the LDU structure is shared (pointer equality since both come from CGeometry). ---*/ + const bool ok = (mat.row_ptr_l == B.mat.row_ptr_l) && (mat.col_ind_l == B.mat.col_ind_l) && + (mat.row_ptr_u == B.mat.row_ptr_u) && (mat.col_ind_u == B.mat.col_ind_u) && (nVar == B.nVar) && + (nEqn == B.nEqn) && (nPoint == B.nPoint) && (mat.nnz_l == B.mat.nnz_l) && (mat.nnz_u == B.mat.nnz_u); + if (!ok) SU2_MPI::Error("Matrices do not have compatible sparsity.", CURRENT_FUNCTION); SU2_OMP_FOR_STAT(omp_light_size) - for (auto i = 0ul; i < nnz * nVar * nEqn; ++i) matrix[i] += alpha * B.matrix[i]; + for (auto i = 0ul; i < nPoint * nVar * nEqn; ++i) mat.d[i] += alpha * B.mat.d[i]; + END_SU2_OMP_FOR + SU2_OMP_FOR_STAT(omp_light_size) + for (auto i = 0ul; i < mat.nnz_l * nVar * nEqn; ++i) mat.l[i] += alpha * B.mat.l[i]; + END_SU2_OMP_FOR + SU2_OMP_FOR_STAT(omp_light_size) + for (auto i = 0ul; i < mat.nnz_u * nVar * nEqn; ++i) mat.u[i] += alpha * B.mat.u[i]; END_SU2_OMP_FOR } @@ -1324,9 +1816,9 @@ void CSysMatrix::BuildPastixPreconditioner(CGeometry* geometry, cons unsigned short kind_fact) { SU2_ZONE_SCOPED #ifdef HAVE_PASTIX - /*--- Pastix will launch nested threads. ---*/ BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS { - pastix_wrapper.SetMatrix(nVar, nPoint, nPointDomain, row_ptr, col_ind, matrix); + pastix_wrapper.SetLDU(nVar, nPoint, nPointDomain, mat.row_ptr_l, mat.col_ind_l, mat.row_ptr_u, mat.col_ind_u, mat.d, + mat.l, mat.u); pastix_wrapper.Factorize(geometry, config, kind_fact); } END_SU2_OMP_SAFE_GLOBAL_ACCESS @@ -1364,7 +1856,7 @@ void CSysMatrix::ComputePastixPreconditioner(const CSysVector. */ -#include "../../include/linear_algebra/CSysMatrix.hpp" +/*--- Must come first, see the file for why. ---*/ +#include "../../include/basic_types/codi_host_only.hpp" + +#include +#include +#include + +#include "../../include/linear_algebra/CMatrixInverse.hpp" +#include "../../include/linear_algebra/CSysMatrix.inl" #include "../../include/linear_algebra/GPUComms.cuh" -template -__global__ void GPUMatrixVectorProductAdd(matrixType* matrix, vectorType* vec, vectorType* prod, const unsigned long* d_row_ptr, const unsigned long* d_col_ind, unsigned long nPointDomain, unsigned long nVar, unsigned long nEqn) -{ - int row = (blockIdx.x * blockDim.x + threadIdx.x)/32; - int threadNo = threadIdx.x%32; - int activeThreads = nVar * (32/nVar); +namespace { + +/*! + * \brief Apply the Jacobi preconditioner: prod = invM * vec (block-diagonal). Points are batched + * into blocks of ~128 threads (see ComputeJacobiPreconditionerGPU), threadIdx.x mapping + * to (point-within-block, output variable) via divmod by nVar - the same layout as + * BlockLDU_SpMV_kernel. Occupancy was already fine here (one thread per point, full + * warps), but consecutive threads used to land nVar^2 elements apart in invM (one point's + * whole dense block per thread); with this mapping they land nVar elements apart instead, + * a real (if partial, since it is still not stride-1) coalescing win that needs no shared + * memory or synchronization, unlike a fully-coalesced one-thread-per-block-entry version + * would. + */ +template +__global__ void ApplyJacobiPreconditionerKernel(const ScalarType* __restrict__ invM, + const ScalarType* __restrict__ vec, ScalarType* __restrict__ prod, + unsigned long nPointDomain, unsigned long nVar) { + const unsigned long rowsPerBlock = blockDim.x / nVar; + const unsigned long iVar = threadIdx.x % nVar; + const unsigned long iPoint = static_cast(blockIdx.x) * rowsPerBlock + threadIdx.x / nVar; + if (iPoint >= nPointDomain) return; + + const auto* block = &invM[iPoint * nVar * nVar + iVar * nVar]; + const auto* rhs = &vec[iPoint * nVar]; + + auto sum = ScalarType(0); + for (unsigned long jVar = 0; jVar < nVar; ++jVar) sum += block[jVar] * rhs[jVar]; + prod[iPoint * nVar + iVar] = sum; +} + +/*--- ILU. The factorization is scheduled by coloring: colors are true independent sets of the + * ILU dependency graph (no two same-colored rows depend on each other in either direction), so a + * color's rows can be processed with zero races in one kernel launch, but since a color is wider + * and less ordered than a level, one pass over all colors is only an approximation, not an exact + * result — several sweeps (repeated passes) are needed to converge. The triangular solves are + * scheduled by level instead: a level's rows only depend on earlier levels (already finalized), + * so one pass over the levels, in order, is exact — no sweeping needed there. Both the color and + * level rows are scattered through the matrix, hence the indirection through their tables. + * Throughout, one CUDA block works on one row. ---*/ + +/*! + * \brief The pointers of an LDU-partitioned matrix, all in device memory. This mirrors the + * private CSysMatrix::LDU, which the kernels cannot name. + */ +template +struct DeviceLDU { + ScalarType* d; + ScalarType* l; + ScalarType* u; + const su2uint* row_ptr_l; + const su2uint* col_ind_l; + const su2uint* row_ptr_u; + const su2uint* col_ind_u; +}; + +/*! + * \brief Start of block (i,j), or nullptr if it is not a nonzero of the pattern. + */ +template +__device__ FORCEINLINE ScalarType* GetBlockILU(const DeviceLDU& M, unsigned long nVar, + unsigned long block_i, unsigned long block_j) { + const auto blockSize = nVar * nVar; + if (block_i == block_j) return M.d + block_i * blockSize; + + const bool lower = block_j < block_i; + const auto* row_ptr = lower ? M.row_ptr_l : M.row_ptr_u; + const auto* col_ind = lower ? M.col_ind_l : M.col_ind_u; + auto* vals = lower ? M.l : M.u; + + for (auto k = row_ptr[block_i]; k < row_ptr[block_i + 1]; ++k) { + if (col_ind[k] == block_j) return vals + k * blockSize; + } + return nullptr; +} + +/*! + * \brief Parallel Gauss-Jordan matrix inversion, shared by InvertDiagonalBlocksKernel and + * IluFactorColorKernel's diagonal-inversion step below - both already have nVar*nVar + * threads and two blockSize-sized shared buffers on hand at the point they need a diagonal + * block inverted. SU2_LinAlg::MatrixInverse would have to run on a single thread. + * \param i,j Row/column of the block entry this thread owns, in 0..nVar-1 (i.e. threadIdx.x's + * divmod by nVar, same mapping the caller already uses for everything else). + * \param A Destroyed. \param Inv Must be pre-loaded with the identity, must not alias \p A; + * holds the inverse on return. + * \note __syncthreads() (not __syncwarp()) is used so this stays correct even when nVar*nVar + * exceeds one warp (nVar > ~5), at the cost of a full block-wide barrier even for the + * common case where the whole block already fits in one warp. Every thread of the block + * must call this (no divergent early return before it), since every __syncthreads() here + * is a whole-block barrier. + */ +template +__device__ FORCEINLINE void ParallelMatrixInverse(unsigned long nVar, unsigned long i, unsigned long j, + ScalarType* __restrict__ A, ScalarType* __restrict__ Inv) { + for (auto k = 0ul; k < nVar; ++k) { + /*--- Regularize the pivot (shared with the host path, same clamp value). ---*/ + if (i == k && j == k) SU2_LinAlg::RegularizePivot(A[k * nVar + k]); + __syncthreads(); + + /*--- Normalize the pivot row. ---*/ + const ScalarType pivot = A[k * nVar + k]; + if (i == k) { + A[i * nVar + j] /= pivot; + Inv[i * nVar + j] /= pivot; + } + __syncthreads(); + + /*--- Eliminate column k from every other row; A(k,*) and Inv(k,*) are already finalized for + * this step (previous barrier), and each thread only ever writes its own (i,j), so this + * needs no further synchronization until the next pivot's regularization reads A(k+1,k+1). ---*/ + if (i != k) { + const ScalarType factor = A[i * nVar + k]; + A[i * nVar + j] -= factor * A[k * nVar + j]; + Inv[i * nVar + j] -= factor * Inv[k * nVar + j]; + } + __syncthreads(); + } +} + +/*! + * \brief Invert the diagonal blocks of the matrix, device version of InverseDiagonalBlock, via + * ParallelMatrixInverse (see its comment for why this beats a single serial thread). + * \note Grid: one block per row, blockDim.x == nVar*nVar. Dynamic shared memory: 2*nVar*nVar + * scalars (the working copy of the block, and the inverse being accumulated in place of + * the old identity-then-eliminate scheme). + */ +template +__global__ void InvertDiagonalBlocksKernel(unsigned long nRows, unsigned long nVar, + const ScalarType* __restrict__ mat_d, ScalarType* __restrict__ invM) { + const unsigned long iRow = blockIdx.x; + if (iRow >= nRows) return; + + const auto blockSize = nVar * nVar; + const unsigned long tid = threadIdx.x; + const unsigned long i = tid / nVar; + const unsigned long j = tid % nVar; + + /*--- The inversion destroys its input, so it cannot work on the matrix itself. ---*/ + extern __shared__ __align__(sizeof(double)) char smem[]; + auto* A = reinterpret_cast(smem); + auto* Inv = A + blockSize; + + A[tid] = mat_d[iRow * blockSize + tid]; + Inv[tid] = ScalarType(i == j); + __syncthreads(); + + ParallelMatrixInverse(nVar, i, j, A, Inv); + + invM[iRow * blockSize + tid] = Inv[tid]; +} + +/*! + * \brief Quantize the diagonal blocks straight from the device diagonal (gpu.d), device + * counterpart of CSysMatrix::QuantizeBlock (CSysMatrix.cpp). Calls the exact same + * EncodeQuantRow encoding function. One thread per (point, row) - each block-row's + * scale and quantization are independent of every other row. + */ +template +__global__ void QuantizeDiagonalBlocksKernel(unsigned long nRows, unsigned long nVar, + const ScalarType* __restrict__ mat_d, uint8_t* __restrict__ q_scale_d, + int8_t* __restrict__ q_blocks_d) { + const unsigned long rowsPerBlock = blockDim.x / nVar; + const unsigned long iVar = threadIdx.x % nVar; + const unsigned long iPoint = static_cast(blockIdx.x) * rowsPerBlock + threadIdx.x / nVar; + if (iPoint >= nRows) return; + + const auto* blk = mat_d + iPoint * nVar * nVar; + EncodeQuantRow([&](unsigned long r, unsigned long c) { return blk[r * nVar + c]; }, q_scale_d[iPoint * nVar + iVar], + q_blocks_d + iPoint * nVar * nVar + iVar * nVar, nVar, iVar); +} + +/*! + * \brief Factorize the rows of one color, one sweep of an iterative (colored Gauss-Seidel) + * ILU factorization: same order/pattern as the exact level-scheduled algorithm (this + * does not change L/U membership, so it converges to the exact same fixed point), but + * colors are true independent sets (zero dependency between same-colored rows in either + * direction), so a color can be processed with zero races in far fewer, wider launches + * than the number of levels — at the cost of needing several sweeps (repeated passes + * over all colors) instead of one exact pass, because for a fixed order the level count + * is already the minimum number of race-free single-pass groups (Mirsky's theorem). + * \note Every visit of a row (there is one per sweep) resets it from the original matrix first + * (folding in the device version of the InitIluRow helper of BuildILUPreconditioner), + * because the elimination below is a re-evaluation of the row's defining equation using + * the current (possibly stale) values of other rows, not an incremental accumulation. + * Grid: one block per row of the color, blockDim.x == nVar*nVar (one thread per block + * entry, so that the small matrix products are one dot product per thread). Dynamic + * shared memory: 2*nVar*nVar scalars. + */ +template +__global__ void IluFactorColorKernel(const su2uint* __restrict__ color_idx, unsigned long color_begin, + unsigned long color_size, unsigned long nRows, unsigned long nVar, + DeviceLDU A, DeviceLDU M) { + if (blockIdx.x >= color_size) return; + + const unsigned long iRow = color_idx[color_begin + blockIdx.x]; + const auto blockSize = nVar * nVar; + const unsigned long tid = threadIdx.x; + const auto iVar = tid / nVar, jVar = tid % nVar; + + extern __shared__ __align__(sizeof(double)) char smem[]; + auto* Lij = reinterpret_cast(smem); + auto* work = Lij + blockSize; + + /*--- Reset this row to the raw matrix entries (device version of InitIluRow, but for one + * row instead of the whole matrix, since here it runs once per row per sweep). ---*/ + M.d[iRow * blockSize + tid] = A.d[iRow * blockSize + tid]; + auto scatter = [&](const su2uint* a_row_ptr, const su2uint* a_col_ind, const ScalarType* a_vals, + const su2uint* m_row_ptr, const su2uint* m_col_ind, ScalarType* m_vals) { + auto ka = a_row_ptr[iRow]; + const auto ka_end = a_row_ptr[iRow + 1]; + for (auto k = m_row_ptr[iRow]; k < m_row_ptr[iRow + 1]; ++k) { + const auto jPoint = m_col_ind[k]; + while (ka < ka_end && a_col_ind[ka] < jPoint) ++ka; + if (ka < ka_end && a_col_ind[ka] == jPoint) { + m_vals[k * blockSize + tid] = a_vals[ka * blockSize + tid]; + } else { + m_vals[k * blockSize + tid] = ScalarType(0); + } + } + }; + scatter(A.row_ptr_l, A.col_ind_l, A.l, M.row_ptr_l, M.col_ind_l, M.l); + scatter(A.row_ptr_u, A.col_ind_u, A.u, M.row_ptr_u, M.col_ind_u, M.u); + __syncthreads(); + + /*--- For this row (unknown), loop over its lower diagonal entries. ---*/ + for (auto kl = M.row_ptr_l[iRow]; kl < M.row_ptr_l[iRow + 1]; ++kl) { + /*--- All threads must be done with the previous entry: Lij is about to be overwritten, + * and the blocks of this row updated below are read here across threads. ---*/ + __syncthreads(); + + /*--- jPoint is the column index (jPoint < iRow). ---*/ + const unsigned long jPoint = M.col_ind_l[kl]; + + /*--- Multiply the block by the inverse of the corresponding diagonal block. ---*/ + auto* Block_ij = M.l + kl * blockSize; + const auto* invUjj = M.d + jPoint * blockSize; + + ScalarType sum = 0; + for (auto k = 0ul; k < nVar; ++k) sum += Block_ij[iVar * nVar + k] * invUjj[k * nVar + jVar]; + Lij[tid] = sum; + __syncthreads(); + + /*--- Lij holds Aij*inv(Ujj). Jump to the upper part of the jPoint row. ---*/ + for (auto ku = M.row_ptr_u[jPoint]; ku < M.row_ptr_u[jPoint + 1]; ++ku) { + /*--- Get the column index (kPoint > jPoint), halo columns are not factorized. ---*/ + const unsigned long kPoint = M.col_ind_u[ku]; + if (kPoint >= nRows) break; + + /*--- If Aik exists, update it: Aik -= Lij * Ujk ---*/ + auto* Block_ik = GetBlockILU(M, nVar, iRow, kPoint); + if (Block_ik == nullptr) continue; + + /*--- Block_ik cannot alias Block_ij because kPoint > jPoint. ---*/ + const auto* Ujk = M.u + ku * blockSize; + ScalarType prod = 0; + for (auto k = 0ul; k < nVar; ++k) prod += Lij[iVar * nVar + k] * Ujk[k * nVar + jVar]; + Block_ik[tid] -= prod; + } + + /*--- Store Lij in the lower triangular part, each thread only writes its own entry. ---*/ + Block_ij[tid] = Lij[tid]; + } + + /*--- Invert the diagonal entry, Uii, for the rows that depend on it. The loop above may have + * updated it (when kPoint == iRow), so the whole block has to be done first. Lij is free again + * here (its last use, storing it into Block_ij, is done) - reuse it as the identity/inverse + * buffer ParallelMatrixInverse needs, instead of a separate shared allocation. ---*/ + __syncthreads(); + work[tid] = M.d[iRow * blockSize + tid]; + Lij[tid] = ScalarType(iVar == jVar); + __syncthreads(); + + ParallelMatrixInverse(nVar, iVar, jVar, work, Lij); + + M.d[iRow * blockSize + tid] = Lij[tid]; +} + +/*! + * \brief Compute blk[iVar,jVar].x[col,jVar] and sum over neighbor rows on device + * \note used for L.x* and U.x* in ILU and LU-SGS preconditioners + */ +template +__device__ FORCEINLINE ScalarType DeviceSparseBlockMatVec(unsigned long iRow, unsigned long iVar, unsigned long jVar, unsigned long nVar, + const su2uint* __restrict__ row_ptr, const su2uint* __restrict__ col_ind, + const ScalarType* __restrict__ blk, const ScalarType* __restrict__ x, + unsigned long nRows= ~0ul) { + + const auto blockSize = nVar * nVar; + // compute blk[iVar,jVar].x[col,jVar] and sum over row + ScalarType acc = 0; + for (auto k = row_ptr[iRow]; k < row_ptr[iRow + 1]; ++k) { + const unsigned long jPoint = col_ind[k]; + if (jPoint >= nRows) break; //default is largest possible value thus skipped by default + acc += blk[k * blockSize + iVar * nVar + jVar] * x[jPoint * nVar + jVar]; + } + return acc; +} + +/*! + * \brief Compute Quantized blk[iVar,jVar].x[col,jVar] and sum over neighbor rows on device + */ +template +__device__ FORCEINLINE ScalarType QuantizedDeviceSparseBlockMatVec(unsigned long iRow, unsigned long iVar, unsigned long jVar, unsigned long nVar, + const su2uint* __restrict__ row_ptr, const su2uint* __restrict__ col_ind, + const QuantType* __restrict__ q_blk, const QuantScaleType* __restrict__ q_scale, + const ScalarType* __restrict__ x, unsigned long nRows= ~0ul) { + + const auto blockSize = nVar * nVar; + ScalarType acc = 0; + + for (auto k = row_ptr[iRow]; k < row_ptr[iRow + 1]; ++k) { + const unsigned long jPoint = col_ind[k]; + if (jPoint >= nRows) break; //default is largest possible value thus skipped by default + const float scale = DecodeQuantScale(q_scale[k * nVar + iVar]); + ScalarType q_val = static_cast(q_blk[k * blockSize + iVar * nVar + jVar]); // directly cast to ScalarType + acc += scale * q_val * x[jPoint * nVar + jVar]; + } + return acc; +} + + +/*! + * \brief Compute the partial sum across a row on device + * \note used after DeviceSparseBlockMatVec, it completes the dot product for a given iVar + */ +template +__device__ FORCEINLINE ScalarType DeviceReduceBlockRow(const ScalarType* __restrict__ x, unsigned long iVar, unsigned long nVar) { + ScalarType sum = 0; + for (auto j = 0ul; j < nVar; ++j) sum += x[iVar * nVar + j]; + return sum; +} + +/*! + * \brief Compute the block by vector multiplication + */ +template + __device__ FORCEINLINE ScalarType DeviceDenseBlockMatVec(const ScalarType* __restrict__ blk, const ScalarType* __restrict__ x, + ScalarType* __restrict__ partial, unsigned long tid, + unsigned long iVar, unsigned long jVar, unsigned long nVar) { + + // Compute blk.x + partial[tid] = blk[iVar * nVar + jVar] * x[jVar]; + __syncthreads(); + return DeviceReduceBlockRow(partial, iVar, nVar); +} + +/*! + * \brief Exact forward substitution for the rows of one level, (L+I).prod = vec. + * \note Every row in a level only depends on rows in earlier levels, which are already + * finalized (see CSysMatrix::levels_ilu), so one pass over the levels in increasing order + * gives the exact result, unlike the colored factorization above. One thread per block + * entry, so the inner dot product over a neighbor block is spread across nVar threads + * instead of done serially by one; each thread accumulates its own (iVar,jVar) partial + * product across every neighbor with no synchronization at all, and only the final + * nVar-way reduction (summing over jVar for each iVar) needs one __syncthreads(). Grid: + * one block per row of the level, blockDim.x == nVar*nVar. Dynamic shared memory: + * nVar*nVar scalars. + */ +template +__global__ void IluForwardKernel(const su2uint* __restrict__ level_idx, unsigned long level_begin, + unsigned long level_size, unsigned long nVar, DeviceLDU M, + const ScalarType* __restrict__ vec, ScalarType* __restrict__ prod) { + if (blockIdx.x >= level_size) return; + + const unsigned long iRow = level_idx[level_begin + blockIdx.x]; + const unsigned long tid = threadIdx.x; + const auto iVar = tid / nVar, jVar = tid % nVar; + + extern __shared__ __align__(sizeof(double)) char smem[]; + auto* partial = reinterpret_cast(smem); + + partial[tid] = DeviceSparseBlockMatVec(iRow, iVar, jVar, nVar, M.row_ptr_l, M.col_ind_l, M.l, prod); + __syncthreads(); + + if (jVar == 0) prod[iRow * nVar + iVar] = vec[iRow * nVar + iVar] - DeviceReduceBlockRow(partial, iVar, nVar); +} + +/*! + * \brief Exact backward substitution for the rows of one level, U.prod = prod. + * \note The right-hand side is read directly from \p prod: a level is visited exactly once, so + * prod[iRow] is still the untouched forward-solve result when its row is processed (unlike + * a colored sweep, which revisits every row and would need a separate fixed buffer to tell + * the right-hand side apart from a solution estimate). Levels are processed in decreasing + * order so every U-neighbor (a higher row index) is already finalized. Same thread layout + * as IluForwardKernel, plus one extra __syncthreads() before the diagonal multiply (which + * needs every iVar's reduced sum). Grid: one block per row of the level, + * blockDim.x == nVar*nVar. Dynamic shared memory: nVar*nVar + nVar scalars. + */ +template +__global__ void IluBackwardKernel(const su2uint* __restrict__ level_idx, unsigned long level_begin, + unsigned long level_size, unsigned long nRows, unsigned long nVar, + DeviceLDU M, ScalarType* __restrict__ prod) { + if (blockIdx.x >= level_size) return; + + const unsigned long iRow = level_idx[level_begin + blockIdx.x]; + const auto blockSize = nVar * nVar; + const unsigned long tid = threadIdx.x; + const auto iVar = tid / nVar, jVar = tid % nVar; + + extern __shared__ __align__(sizeof(double)) char smem[]; + auto* partial = reinterpret_cast(smem); + auto* aux = partial + blockSize; + + partial[tid] = DeviceSparseBlockMatVec(iRow, iVar, jVar, nVar, M.row_ptr_u, M.col_ind_u, M.u, prod, nRows); + __syncthreads(); + + if (jVar == 0) aux[iVar] = prod[iRow * nVar + iVar] - DeviceReduceBlockRow(partial, iVar, nVar); + __syncthreads(); + + ScalarType out = DeviceDenseBlockMatVec(M.d + iRow * blockSize, aux, partial, tid, iVar, jVar, nVar); + if (jVar == 0) prod[iRow * nVar + iVar] = out; +} + +/*! + * \brief Block-LDU SpMV kernel: y[iRow] = (L + D + U) * x per block-row. Several rows are + * batched into one CUDA block (blockDim.x / nVar of them, see MatrixVectorProductGPU) + * instead of one row per block: nVar is typically ~4-6, so one-row-per-block leaves most + * of a warp's lanes permanently idle and caps occupancy at a few resident (mostly-empty) + * warps per SM, well before DRAM bandwidth is the limit. threadIdx.x indexes + * (row-within-block, output variable) as (threadIdx.x / nVar, threadIdx.x % nVar). + */ +template +__global__ void BlockLDU_SpMV_kernel(unsigned long nRows, unsigned long nVar, + const su2uint* __restrict__ row_ptr_l, + const su2uint* __restrict__ col_ind_l, + const ScalarType* __restrict__ mat_l, + const ScalarType* __restrict__ mat_d, + const su2uint* __restrict__ row_ptr_u, + const su2uint* __restrict__ col_ind_u, + const ScalarType* __restrict__ mat_u, + const ScalarType* __restrict__ x, ScalarType* __restrict__ y) { + const unsigned long rowsPerBlock = blockDim.x / nVar; + const unsigned long iVar = threadIdx.x % nVar; + const unsigned long iRow = static_cast(blockIdx.x) * rowsPerBlock + threadIdx.x / nVar; + if (iRow >= nRows) return; + + ScalarType sum = 0; + /* Lower */ + for (auto k = row_ptr_l[iRow]; k < row_ptr_l[iRow + 1]; ++k) { + const auto col = col_ind_l[k]; + const ScalarType* blk = mat_l + k * nVar * nVar + iVar * nVar; + for (unsigned long jVar = 0; jVar < nVar; ++jVar) sum += blk[jVar] * x[col * nVar + jVar]; + } + /* Diagonal */ + { + const ScalarType* blk = mat_d + iRow * nVar * nVar + iVar * nVar; + for (unsigned long jVar = 0; jVar < nVar; ++jVar) sum += blk[jVar] * x[iRow * nVar + jVar]; + } + /* Upper */ + for (auto k = row_ptr_u[iRow]; k < row_ptr_u[iRow + 1]; ++k) { + const auto col = col_ind_u[k]; + const ScalarType* blk = mat_u + k * nVar * nVar + iVar * nVar; + for (unsigned long jVar = 0; jVar < nVar; ++jVar) sum += blk[jVar] * x[col * nVar + jVar]; + } + y[iRow * nVar + iVar] = sum; +} + +/*! + * \brief Device version of QuantizedRowProduct/QuantizedMatVecAdd (CSysMatrix.inl). Rows are + * batched per block the same way as BlockLDU_SpMV_kernel. + */ +template +__global__ void QuantizedBlockLDU_SpMV_kernel( + unsigned long nRows, unsigned long nVar, const su2uint* __restrict__ row_ptr_l, + const su2uint* __restrict__ col_ind_l, const uint8_t* __restrict__ q_scale_l, + const int8_t* __restrict__ q_blocks_l, const uint8_t* __restrict__ q_scale_d, + const int8_t* __restrict__ q_blocks_d, const su2uint* __restrict__ row_ptr_u, + const su2uint* __restrict__ col_ind_u, const uint8_t* __restrict__ q_scale_u, + const int8_t* __restrict__ q_blocks_u, const ScalarType* __restrict__ x, ScalarType* __restrict__ y) { + const unsigned long rowsPerBlock = blockDim.x / nVar; + const unsigned long iVar = threadIdx.x % nVar; + const unsigned long iRow = static_cast(blockIdx.x) * rowsPerBlock + threadIdx.x / nVar; + if (iRow >= nRows) return; + + auto addBlock = [&](const uint8_t* __restrict__ qs, const int8_t* __restrict__ qv, + const ScalarType* __restrict__ xk) { + const float row_scale = DecodeQuantScale(qs[iVar]); + const int8_t* __restrict__ row = qv + iVar * nVar; + ScalarType partial = 0; + unsigned long jVar = 0; + for (; jVar + 4 <= nVar; jVar += 4) { + /*--- Row bytes are not generally 4-byte aligned (nVar*nVar need not be a multiple of 4), + * so this must go through memcpy rather than a reinterpret_cast deref. ---*/ + uint32_t packed; + memcpy(&packed, row + jVar, sizeof(packed)); + partial += static_cast(packed) * xk[jVar]; + partial += static_cast(packed >> 8) * xk[jVar + 1]; + partial += static_cast(packed >> 16) * xk[jVar + 2]; + partial += static_cast(packed >> 24) * xk[jVar + 3]; + } + for (; jVar < nVar; ++jVar) partial += row[jVar] * xk[jVar]; + return static_cast(row_scale) * partial; + }; + + ScalarType sum = 0; + /* Lower */ + for (auto k = row_ptr_l[iRow]; k < row_ptr_l[iRow + 1]; ++k) { + const auto col = col_ind_l[k]; + sum += addBlock(q_scale_l + k * nVar, q_blocks_l + k * nVar * nVar, x + col * nVar); + } + /* Diagonal */ + sum += addBlock(q_scale_d + iRow * nVar, q_blocks_d + iRow * nVar * nVar, x + iRow * nVar); + /* Upper */ + for (auto k = row_ptr_u[iRow]; k < row_ptr_u[iRow + 1]; ++k) { + const auto col = col_ind_u[k]; + sum += addBlock(q_scale_u + k * nVar, q_blocks_u + k * nVar * nVar, x + col * nVar); + } + y[iRow * nVar + iVar] = sum; +} + +/*! + * \brief Instantiate the freshly captured \p graph into \p exec, or, when \p exec already holds a + * graph with the same topology, push the new node parameters into it in place. + * \note Re-capturing the topology is cheap, instantiating it is not: cudaGraphInstantiate + * allocates and builds the executable graph, at a cost that grows with the node count (one + * node per level here), so doing it on every call would cost more than simply launching the + * kernels and would defeat the purpose of using graphs at all. cudaGraphExecUpdate keeps the + * executable graph and only rewrites the kernel arguments that changed, which is what makes + * the graphs worth having on the flexible-FGMRES path where the vectors change every call. + * The full instantiation stays as the fallback for the first call and for the (unexpected) + * case of the topology actually changing. + */ +inline void InstantiateOrUpdateGraph(cudaGraphExec_t& exec, cudaGraph_t graph, const char* what) { + SU2_ZONE_SCOPED_N("Graph instantiate or update") + if (exec != nullptr) { + cudaGraphExecUpdateResultInfo info{}; + if (cudaGraphExecUpdate(exec, graph, &info) == cudaSuccess) return; + + /*--- A failed update is recoverable (we just instantiate again), but the runtime holds on to + * the error, so consume it before the next gpuErrChk mistakes it for a real failure. ---*/ + cudaGetLastError(); + gpuErrChk(cudaGraphExecDestroy(exec)); + exec = nullptr; + } + gpuErrChk(cudaGraphInstantiate(&exec, graph, nullptr, nullptr, 0)); +} + +} // namespace + +template +void CSysMatrix::ComputeJacobiPreconditionerGPU(const CSysVector& vec, + CSysVector& prod, CGeometry* geometry, + const CConfig* config) const { + (void)geometry; + (void)config; + + SU2_ZONE_SCOPED + + if (d_invM == nullptr) { + SU2_MPI::Error("CUDA Jacobi preconditioner used before BuildJacobiPreconditionerGPU.", CURRENT_FUNCTION); + } + + constexpr unsigned long targetThreadsPerBlock = 128; + const auto rowsPerBlock = std::max(1, targetThreadsPerBlock / nVar); + const auto threadsPerBlock = static_cast(rowsPerBlock * nVar); + const auto blocks = static_cast((nPointDomain + rowsPerBlock - 1) / rowsPerBlock); + ApplyJacobiPreconditionerKernel<<>>(d_invM, vec.GetDevicePointer(), prod.GetDevicePointer(), + nPointDomain, nVar); + /*--- Sync so the zone above actually times the kernel, not just the (async) launch call. ---*/ + gpuErrChk(cudaStreamSynchronize(nullptr)); + gpuErrChk(cudaGetLastError()); +} + +template +void CSysMatrix::QuantizeDiagonalBlocksGPU() { + SU2_ZONE_SCOPED + + if (nPointDomain == 0) return; + + /*--- The matrix is expected to be on the device already, it is uploaded once per solve by + * CSysMatrixVectorProduct, which is created before the preconditioner is built. ---*/ + constexpr unsigned long targetThreadsPerBlock = 128; + const auto rowsPerBlock = std::max(1, targetThreadsPerBlock / nVar); + const auto threadsPerBlock = static_cast(rowsPerBlock * nVar); + const auto blocks = static_cast((nPointDomain + rowsPerBlock - 1) / rowsPerBlock); + QuantizeDiagonalBlocksKernel + <<>>(nPointDomain, nVar, gpu.d, d_q_scale.d, d_q_blocks.d); + /*--- Sync so the zone above actually times the kernel, not just the (async) launch call. ---*/ + gpuErrChk(cudaStreamSynchronize(nullptr)); + gpuErrChk(cudaGetLastError()); +} + +template +void CSysMatrix::BuildJacobiPreconditionerGPU() { + SU2_ZONE_SCOPED + + if (d_invM == nullptr) { + SU2_MPI::Error("CUDA Jacobi preconditioner used without device storage.", CURRENT_FUNCTION); + } + if (nPointDomain == 0) return; + + /*--- The matrix is expected to be on the device already, it is uploaded once per solve by + * CSysMatrixVectorProduct, which is created before the preconditioner is built. ---*/ + const auto blockSize = static_cast(nVar * nVar); + InvertDiagonalBlocksKernel<<(nPointDomain), blockSize, + 2 * blockSize * sizeof(ScalarType)>>>(nPointDomain, nVar, gpu.d, d_invM); + /*--- Sync so the zone above actually times the kernel, not just the (async) launch call. ---*/ + gpuErrChk(cudaStreamSynchronize(nullptr)); + gpuErrChk(cudaGetLastError()); +} + +template +void CSysMatrix::BuildILUPreconditionerGPU() { + SU2_ZONE_SCOPED + + if (gpu_ilu.d == nullptr) { + SU2_MPI::Error("CUDA ILU preconditioner used without device storage.", CURRENT_FUNCTION); + } + if (nPointDomain == 0) return; - int blockRow = (threadNo/nVar)%nVar; + /*--- The matrix is expected to be on the device already, it is uploaded once per solve by + * CSysMatrixVectorProduct, which is created before the preconditioner is built. ---*/ + const DeviceLDU A{gpu.d, gpu.l, gpu.u, gpu.row_ptr_l, + gpu.col_ind_l, gpu.row_ptr_u, gpu.col_ind_u}; + const DeviceLDU M{gpu_ilu.d, gpu_ilu.l, gpu_ilu.u, gpu_ilu.row_ptr_l, + gpu_ilu.col_ind_l, gpu_ilu.row_ptr_u, gpu_ilu.col_ind_u}; - if(row(nVar * nVar); + const auto shared = 2 * blockSize * sizeof(ScalarType); - __syncthreads(); + /*--- The legacy default stream cannot be captured, so the graph lives on its own stream, + * created once. Every launch below is followed by a sync back to the host, so this does not + * change execution order relative to the rest of the (single-stream) solver. ---*/ + if (aux_stream == nullptr) gpuErrChk(cudaStreamCreate(&aux_stream)); - if(row + <<>>(d_precond_level_idx, begin, size, nPointDomain, nVar, A, M); + } + gpuErrChk(cudaStreamSynchronize(aux_stream)); + gpuErrChk(cudaGetLastError()); + return; + } - for(int index = d_row_ptr[row] * nVar * nEqn + threadNo; index < d_row_ptr[row+1] * nVar * nEqn; index+=activeThreads) - { - int blockCol = index%nEqn; - int blockNo = index/(nVar * nEqn); - res += matrix[index] * vec[(d_col_ind[blockNo])*nVar + blockCol]; + /*--- The launch sequence (ilu_gpu_sweeps passes over all colors) is identical on every call: + * the grid and block sizes only depend on the (fixed) sparsity pattern/coloring and the device + * pointers are fixed members, allocated once. Capture it into a CUDA graph the first time and + * replay that from then on, which removes the per-launch host-side overhead without touching + * the parallelization of any individual kernel (unlike a persistent cooperative-groups kernel, + * this does not cap per-color parallelism to an occupancy-resident block count). See + * IluFactorColorKernel for why several sweeps over the colors are needed. Note that factors + * are not reset between calls to BuildILUPreconditionerGPU, so with LINEAR_SOLVER_ILU_GPU_SWEEPS + * set low (even 1), each call refines the previous one's result rather than reconverging from + * scratch, relying on the matrix changing little between outer/pseudo-time iterations. ---*/ + if (ilu_build_graph_exec == nullptr) { + cudaGraph_t graph; + gpuErrChk(cudaStreamBeginCapture(aux_stream, cudaStreamCaptureModeThreadLocal)); + + for (unsigned short sweep = 0; sweep < ilu_gpu_sweeps; ++sweep) { + for (auto color = 0ul; color + 1 < ilu_color_ptr.size(); ++color) { + const auto begin = ilu_color_ptr[color]; + const auto size = ilu_color_ptr[color + 1] - begin; + if (size == 0) continue; + IluFactorColorKernel + <<>>(d_ilu_color_idx, begin, size, nPointDomain, nVar, A, M); } + } + + gpuErrChk(cudaStreamEndCapture(aux_stream, &graph)); + gpuErrChk(cudaGraphInstantiate(&ilu_build_graph_exec, graph, nullptr, nullptr, 0)); + gpuErrChk(cudaGraphDestroy(graph)); + } + + gpuErrChk(cudaGraphLaunch(ilu_build_graph_exec, aux_stream)); + gpuErrChk(cudaStreamSynchronize(aux_stream)); + gpuErrChk(cudaGetLastError()); +} + +template +void CSysMatrix::ComputeILUPreconditionerGPU(const CSysVector& vec, + CSysVector& prod) const { + SU2_ZONE_SCOPED + + if (gpu_ilu.d == nullptr) { + SU2_MPI::Error("CUDA ILU preconditioner used before BuildILUPreconditionerGPU.", CURRENT_FUNCTION); + } + if (nPointDomain == 0) return; + + const DeviceLDU M{gpu_ilu.d, gpu_ilu.l, gpu_ilu.u, gpu_ilu.row_ptr_l, + gpu_ilu.col_ind_l, gpu_ilu.row_ptr_u, gpu_ilu.col_ind_u}; + + auto* d_vec = vec.GetDevicePointer(); + auto* d_prod = prod.GetDevicePointer(); + + const auto nLevels = precond_level_ptr.size() - 1; + + /*--- One thread per block entry, like the factorization kernel: spreads each row's neighbor + * dot products over nVar*nVar threads instead of doing them serially in nVar threads, without + * changing the number of blocks (still one per row), so this does not trade away SM coverage + * the way batching several rows into a block did. ---*/ + const auto threads = static_cast(nVar * nVar); + const auto sharedForward = threads * sizeof(ScalarType); + const auto sharedBackward = (threads + nVar) * sizeof(ScalarType); + + if (aux_stream == nullptr) gpuErrChk(cudaStreamCreate(&aux_stream)); + + /*--- Same idea as BuildILUPreconditionerGPU: the launch sequence only depends on the (fixed) + * level structure, plus the vec/prod device pointers. Unlike the build graph those pointers do + * change: flexible FGMRES applies the preconditioner as precond(V[i], Z[i]), so they walk the + * Krylov basis and differ on every call. The topology does not change though, so we re-record + * and let InstantiateOrUpdateGraph patch the new arguments into the executable graph instead of + * building a new one. ---*/ + if (precond_fwd_graph_exec == nullptr || precond_fwd_graph_vec != d_vec || precond_fwd_graph_prod != d_prod) { + SU2_ZONE_SCOPED_N("ILU graph recapture") + + cudaGraph_t graph; + gpuErrChk(cudaStreamBeginCapture(aux_stream, cudaStreamCaptureModeThreadLocal)); + + /*--- Forward substitution: one exact pass over the levels in increasing order, + * (L+I).prod = vec, see IluForwardKernel. ---*/ + for (auto level = 0ul; level < nLevels; ++level) { + const auto begin = precond_level_ptr[level]; + const auto size = precond_level_ptr[level + 1] - begin; + if (size == 0) continue; + IluForwardKernel + <<>>(d_precond_level_idx, begin, size, nVar, M, d_vec, d_prod); + } + + /*--- Backward substitution: one exact pass over the levels in decreasing order, + * U.prod = prod, see IluBackwardKernel. ---*/ + for (auto level = nLevels; level > 0;) { + --level; + const auto begin = precond_level_ptr[level]; + const auto size = precond_level_ptr[level + 1] - begin; + if (size == 0) continue; + IluBackwardKernel + <<>>(d_precond_level_idx, begin, size, nPointDomain, nVar, M, d_prod); + } + + gpuErrChk(cudaStreamEndCapture(aux_stream, &graph)); + InstantiateOrUpdateGraph(precond_fwd_graph_exec, graph, "ILU preconditioner"); + gpuErrChk(cudaGraphDestroy(graph)); + precond_fwd_graph_vec = d_vec; + precond_fwd_graph_prod = d_prod; + } + + gpuErrChk(cudaGraphLaunch(precond_fwd_graph_exec, aux_stream)); + gpuErrChk(cudaStreamSynchronize(aux_stream)); + gpuErrChk(cudaGetLastError()); +} + +/*! + * \brief Exact forward substitution for the rows of one level, x* = D^{-1}.(b-Lx*) + * \note See notes in IluForwardKernel for more details. + */ +template +__global__ void LU_SGS_ForwardKernel(const su2uint* __restrict__ level_idx, unsigned long level_begin, + unsigned long level_size, unsigned long nVar, DeviceLDU M, + const QuantType* __restrict__ q_l, const QuantScaleType* __restrict__ q_scale_l, + const ScalarType* __restrict__ invD, const ScalarType* __restrict__ vec, + ScalarType* __restrict__ prod) { + if (blockIdx.x >= level_size) return; + + const unsigned long iRow = level_idx[level_begin + blockIdx.x]; + const auto blockSize = nVar * nVar; + const unsigned long tid = threadIdx.x; + const auto iVar = tid / nVar, jVar = tid % nVar; + + extern __shared__ __align__(sizeof(double)) char smem[]; + auto* partial = reinterpret_cast(smem); // serves nVar * nVar threads + auto* aux = partial + blockSize; // skip nVar * nVar threads, serves nVar threads - atomicAdd(&prod[row * nVar + blockRow], res); - } + // Compute L.x* + if constexpr (Quantized) { + partial[tid] = QuantizedDeviceSparseBlockMatVec(iRow, iVar, jVar, nVar, M.row_ptr_l, M.col_ind_l, q_l, q_scale_l, prod); + } else { + partial[tid] = DeviceSparseBlockMatVec(iRow, iVar, jVar, nVar, M.row_ptr_l, M.col_ind_l, M.l, prod); + } + __syncthreads(); + + // Compute y = b - L.x* + if (jVar == 0) aux[iVar] = vec[iRow * nVar + iVar] - DeviceReduceBlockRow(partial, iVar, nVar); + __syncthreads(); + + // Compute x* - D^{-1}.y + ScalarType out = DeviceDenseBlockMatVec(invD + iRow * blockSize, aux, partial, tid, iVar, jVar, nVar); + if (jVar == 0) prod[iRow * nVar + iVar] = out; } -template -void CSysMatrix::HtDTransfer(bool trigger) const -{ - if(trigger) gpuErrChk(cudaMemcpy((void*)(d_matrix), (void*)&matrix[0], (sizeof(ScalarType)*nnz*nVar*nEqn), cudaMemcpyHostToDevice)); + +/*! + * \brief Exact backward substitution for the rows of one level, x* = D^{-1}.(D.x* - U.x) = x* - D^{-1}.U.x + * \note See notes in IluBackwardKernel for more details + */ +template +__global__ void LU_SGS_BackwardKernel(const su2uint* __restrict__ level_idx, unsigned long level_begin, + unsigned long level_size, unsigned long nRows, unsigned long nVar, + DeviceLDU M, const QuantType* __restrict__ q_u, + const QuantScaleType* __restrict__ q_scale_u, const ScalarType* __restrict__ invD, + ScalarType* __restrict__ prod) { + if (blockIdx.x >= level_size) return; + + const unsigned long iRow = level_idx[level_begin + blockIdx.x]; + const auto blockSize = nVar * nVar; + const unsigned long tid = threadIdx.x; + const auto iVar = tid / nVar, jVar = tid % nVar; + + extern __shared__ __align__(sizeof(double)) char smem[]; + auto* partial = reinterpret_cast(smem); // serves nVar * nVar threads + auto* aux = partial + blockSize; // skip nVar * nVar threads, serves nVar threads + + // Compute U.x + if constexpr (Quantized) { + partial[tid] = QuantizedDeviceSparseBlockMatVec(iRow, iVar, jVar, nVar, M.row_ptr_u, M.col_ind_u, q_u, q_scale_u, prod, nRows); + } else { + partial[tid] = DeviceSparseBlockMatVec(iRow, iVar, jVar, nVar, M.row_ptr_u, M.col_ind_u, M.u, prod, nRows); + } + __syncthreads(); + + + if (jVar == 0) aux[iVar] = DeviceReduceBlockRow(partial, iVar, nVar); + __syncthreads(); + + // Compute x* - D^{-1}.(U.x) + ScalarType correction = DeviceDenseBlockMatVec(invD + iRow * blockSize, aux, partial, tid, iVar, jVar, nVar); + if (jVar == 0) prod[iRow * nVar + iVar] -= correction; + +} + +/*! + * \brief Pre-calculates the inverse of the diagonal matrix D, same as for the Jacobi preconditioner + */ +template +void CSysMatrix::BuildLU_SGSPreconditionerGPU() { + SU2_ZONE_SCOPED + if (d_invM == nullptr) { + SU2_MPI::Error("CUDA LU-SGS preconditioner used without device storage.", CURRENT_FUNCTION); + } + if (nPointDomain == 0) return; + + /*--- The matrix is expected to be on the device already, it is uploaded once per solve by + * CSysMatrixVectorProduct, which is created before the preconditioner is built. ---*/ + const auto blockSize = static_cast(nVar * nVar); + InvertDiagonalBlocksKernel<<(nPointDomain), blockSize, + 2 * blockSize * sizeof(ScalarType)>>>(nPointDomain, nVar, gpu.d, d_invM); + /*--- Sync so the zone above actually times the kernel, not just the (async) launch call. ---*/ + gpuErrChk(cudaStreamSynchronize(nullptr)); + gpuErrChk(cudaGetLastError()); +} + +/*! + * \brief Compute the LU-SGS preconditioner forward pass + */ +template +void CSysMatrix::ComputeLU_SGSForwardGPU(const CSysVector& vec, + CSysVector& prod) const { + SU2_ZONE_SCOPED + + if (d_invM == nullptr) { + SU2_MPI::Error("CUDA LU-SGS preconditioner used without device storage.", CURRENT_FUNCTION); + } + if (nPointDomain == 0) return; + + const DeviceLDU M{gpu.d, gpu.l, gpu.u, gpu.row_ptr_l, + gpu.col_ind_l, gpu.row_ptr_u, gpu.col_ind_u}; + + auto* d_vec = vec.GetDevicePointer(); + auto* d_prod = prod.GetDevicePointer(); + + /*--- One thread per block entry, as done in ILU preconditioner ---*/ + const auto threads = static_cast(nVar * nVar); + const auto sharedForward = (threads + nVar) * sizeof(ScalarType); + + if (aux_stream == nullptr) gpuErrChk(cudaStreamCreate(&aux_stream)); + + /*--- First part of the symmetric iteration: (D+L).x* = b ---*/ + if (precond_fwd_graph_exec == nullptr || precond_fwd_graph_vec != d_vec || precond_fwd_graph_prod != d_prod) { + SU2_ZONE_SCOPED_N("LU-SGS fwd graph recapture") + + cudaGraph_t graph; + gpuErrChk(cudaStreamBeginCapture(aux_stream, cudaStreamCaptureModeThreadLocal)); + + const auto nLevels = precond_level_ptr.size() - 1; + /*--- Forward substitution: compute x* = D^{-1}.(vec - L.x*). Whether the off-diagonal blocks + * are quantized is fixed for the lifetime of the matrix (Initialize decides it from the + * preconditioner type), so it selects the kernel instantiation here rather than being tested + * by every thread: inside the kernel it is a compile-time constant and the unused branch is + * not compiled at all. ---*/ + auto RecordSweep = [&](auto quantized) { + for (auto level = 0ul; level < nLevels; ++level) { + const auto begin = precond_level_ptr[level]; + const auto size = precond_level_ptr[level + 1] - begin; + if (size == 0) continue; + LU_SGS_ForwardKernel + <<>>(d_precond_level_idx, begin, size, nVar, M, d_q_blocks.l, + d_q_scale.l, d_invM, d_vec, d_prod); + } + }; + if (quantized_mode) { + RecordSweep(std::true_type{}); + } else { + RecordSweep(std::false_type{}); + } + + gpuErrChk(cudaStreamEndCapture(aux_stream, &graph)); + InstantiateOrUpdateGraph(precond_fwd_graph_exec, graph, "LU-SGS forward"); + gpuErrChk(cudaGraphDestroy(graph)); + precond_fwd_graph_vec = d_vec; + precond_fwd_graph_prod = d_prod; + + } + + gpuErrChk(cudaGraphLaunch(precond_fwd_graph_exec, aux_stream)); + gpuErrChk(cudaStreamSynchronize(aux_stream)); + gpuErrChk(cudaGetLastError()); + } -template -void CSysMatrix::GPUMatrixVectorProduct(const CSysVector& vec, CSysVector& prod, - CGeometry* geometry, const CConfig* config) const - { +/*! + * \brief Compute the LU-SGS preconditioner forward pass + */ +template +void CSysMatrix::ComputeLU_SGSBackwardGPU(CSysVector& prod) const { + SU2_ZONE_SCOPED + + if (d_invM == nullptr) { + SU2_MPI::Error("CUDA LU-SGS preconditioner used without device storage.", CURRENT_FUNCTION); + } + if (nPointDomain == 0) return; + + const DeviceLDU M{gpu.d, gpu.l, gpu.u, gpu.row_ptr_l, + gpu.col_ind_l, gpu.row_ptr_u, gpu.col_ind_u}; - ScalarType* d_vec = vec.GetDevicePointer(); - ScalarType* d_prod = prod.GetDevicePointer(); + auto* d_prod = prod.GetDevicePointer(); - HtDTransfer(); - vec.HtDTransfer(); - prod.GPUSetVal(0.0); + /*--- One thread per block entry, as done in ILU preconditioner ---*/ + const auto threads = static_cast(nVar * nVar); + const auto sharedBackward = (threads + nVar) * sizeof(ScalarType); - dim3 blockDim(KernelParameters::MVP_BLOCK_SIZE,1,1); - int gridx = KernelParameters::round_up_division(KernelParameters::MVP_WARP_SIZE, nPointDomain); - dim3 gridDim(gridx, 1, 1); + if (aux_stream == nullptr) gpuErrChk(cudaStreamCreate(&aux_stream)); - GPUMatrixVectorProductAdd<<>>(d_matrix, d_vec, d_prod, d_row_ptr, d_col_ind, nPointDomain, nVar, nEqn); - gpuErrChk( cudaPeekAtLastError() ); + /*--- Second part of the symmetric iteration: (D+U).x_(1) = D.x* ---*/ + if (precond_bwd_graph_exec == nullptr || precond_bwd_graph_prod != d_prod) { + SU2_ZONE_SCOPED_N("LU-SGS bwd graph recapture") - prod.DtHTransfer(); + cudaGraph_t graph; + gpuErrChk(cudaStreamBeginCapture(aux_stream, cudaStreamCaptureModeThreadLocal)); + const auto nLevels = precond_level_ptr.size() - 1; + /*--- Backward substitution: compute x* = D^{-1}.(D.x* - U.x) = x* - D^{-1}.U.x. Quantization + * selects the kernel instantiation, see the forward sweep. ---*/ + auto RecordSweep = [&](auto quantized) { + for (auto level = nLevels; level > 0;) { + --level; + const auto begin = precond_level_ptr[level]; + const auto size = precond_level_ptr[level + 1] - begin; + if (size == 0) continue; + LU_SGS_BackwardKernel + <<>>(d_precond_level_idx, begin, size, nPointDomain, nVar, M, + d_q_blocks.u, d_q_scale.u, d_invM, d_prod); + } + }; + if (quantized_mode) { + RecordSweep(std::true_type{}); + } else { + RecordSweep(std::false_type{}); + } + + gpuErrChk(cudaStreamEndCapture(aux_stream, &graph)); + InstantiateOrUpdateGraph(precond_bwd_graph_exec, graph, "LU-SGS backward"); + gpuErrChk(cudaGraphDestroy(graph)); + precond_bwd_graph_prod = d_prod; + + } + + gpuErrChk(cudaGraphLaunch(precond_bwd_graph_exec, aux_stream)); + gpuErrChk(cudaStreamSynchronize(aux_stream)); + gpuErrChk(cudaGetLastError()); + +} + +template +void CSysMatrix::HtDTransfer(bool trigger) const { + SU2_ZONE_SCOPED + if (!trigger) return; + gpuErrChk(cudaMemcpy(gpu.d, mat.d, sizeof(ScalarType) * nPoint * nVar * nEqn, cudaMemcpyHostToDevice)); + if (quantized_mode) { + /*--- No gpu.l/gpu.u to transfer (never allocated); mirror the host quantized off-diagonal + * storage instead (the diagonal mirrors, d_q_scale.d/d_q_blocks.d, are not touched here at + * all - QuantizeDiagonalBlocksGPU() populates them straight from gpu.d, just uploaded above, + * with no host round trip). Issued as async copies on a dedicated stream so this transfer can + * run concurrently with whatever the preconditioner's Build() launches next on the default + * stream. ---*/ + if (aux_stream == nullptr) gpuErrChk(cudaStreamCreate(&aux_stream)); + if (htd_event == nullptr) gpuErrChk(cudaEventCreateWithFlags(&htd_event, cudaEventDisableTiming)); + gpuErrChk(cudaMemcpyAsync(d_q_scale.l, q_scale.l, sizeof(QuantScaleType) * mat.nnz_l * nVar, cudaMemcpyHostToDevice, + aux_stream)); + gpuErrChk(cudaMemcpyAsync(d_q_blocks.l, q_blocks.l, sizeof(QuantType) * mat.nnz_l * nVar * nEqn, + cudaMemcpyHostToDevice, aux_stream)); + gpuErrChk(cudaMemcpyAsync(d_q_scale.u, q_scale.u, sizeof(QuantScaleType) * mat.nnz_u * nVar, cudaMemcpyHostToDevice, + aux_stream)); + gpuErrChk(cudaMemcpyAsync(d_q_blocks.u, q_blocks.u, sizeof(QuantType) * mat.nnz_u * nVar * nEqn, + cudaMemcpyHostToDevice, aux_stream)); + gpuErrChk(cudaEventRecord(htd_event, aux_stream)); + } else { + gpuErrChk(cudaMemcpy(gpu.l, mat.l, sizeof(ScalarType) * mat.nnz_l * nVar * nEqn, cudaMemcpyHostToDevice)); + gpuErrChk(cudaMemcpy(gpu.u, mat.u, sizeof(ScalarType) * mat.nnz_u * nVar * nEqn, cudaMemcpyHostToDevice)); + } +} + +template +void CSysMatrix::MatrixVectorProductGPU(const CSysVector& vec, CSysVector& prod, + CGeometry* geometry, const CConfig* config) const { + SU2_ZONE_SCOPED + + ScalarType* d_vec = vec.GetDevicePointer(); + ScalarType* d_prod = prod.GetDevicePointer(); + + /*--- Batch several rows per block (see BlockLDU_SpMV_kernel's comment): nVar is small + * (typically ~4-6), so one row per block would leave most of a warp idle. Aim for ~128 + * threads/block, the largest whole number of rows that fits. ---*/ + constexpr unsigned long targetThreadsPerBlock = 128; + const auto rowsPerBlock = std::max(1, targetThreadsPerBlock / nVar); + dim3 blockDim(static_cast(rowsPerBlock * nVar), 1, 1); + dim3 gridDim(static_cast((nPointDomain + rowsPerBlock - 1) / rowsPerBlock), 1, 1); + if (quantized_mode) { + /*--- Wait (on the device, no host block) for HtDTransfer's async L/U copy on its own stream + * to finish before this default-stream kernel reads d_q_scale.l/.u/d_q_blocks.l/.u. ---*/ + gpuErrChk(cudaStreamWaitEvent(nullptr, htd_event, 0)); + QuantizedBlockLDU_SpMV_kernel<<>>( + nPointDomain, nVar, gpu.row_ptr_l, gpu.col_ind_l, d_q_scale.l, d_q_blocks.l, d_q_scale.d, d_q_blocks.d, + gpu.row_ptr_u, gpu.col_ind_u, d_q_scale.u, d_q_blocks.u, d_vec, d_prod); + } else { + BlockLDU_SpMV_kernel<<>>( + nPointDomain, nVar, gpu.row_ptr_l, gpu.col_ind_l, gpu.l, gpu.d, + gpu.row_ptr_u, gpu.col_ind_u, gpu.u, d_vec, d_prod); + } + /*--- Sync so the zone above actually times the kernel, not just the (async) launch call. ---*/ + gpuErrChk(cudaStreamSynchronize(nullptr)); + gpuErrChk(cudaGetLastError()); } -template class CSysMatrix; //This is a temporary fix for invalid instantiations due to separating the member function from the header file the class is defined in. Will try to rectify it in coming commits. +#define INSTANTIATE_MATRIX(TYPE) \ +template void CSysMatrix::HtDTransfer(bool trigger) const; \ +template void CSysMatrix::MatrixVectorProductGPU(const CSysVector& vec, \ + CSysVector& prod, \ + CGeometry* geometry, \ + const CConfig* config) const; \ +template void CSysMatrix::QuantizeDiagonalBlocksGPU(); \ +template void CSysMatrix::BuildJacobiPreconditionerGPU(); \ +template void CSysMatrix::BuildILUPreconditionerGPU(); \ +template void CSysMatrix::BuildLU_SGSPreconditionerGPU(); \ +template void CSysMatrix::ComputeILUPreconditionerGPU(const CSysVector& vec, \ + CSysVector& prod) const; \ +template void CSysMatrix::ComputeJacobiPreconditionerGPU(const CSysVector& vec, \ + CSysVector& prod, \ + CGeometry* geometry, \ + const CConfig* config) const;\ +template void CSysMatrix::ComputeLU_SGSForwardGPU(const CSysVector& vec, \ + CSysVector& prod) const; \ +template void CSysMatrix::ComputeLU_SGSBackwardGPU(CSysVector& prod) const; +INSTANTIATE_MATRIX(su2mixedfloat) + +#if defined(USE_MIXED_PRECISION) && !defined(USE_SINGLE_PRECISION) +INSTANTIATE_MATRIX(passivedouble) +#endif diff --git a/Common/src/linear_algebra/CSysSolve.cpp b/Common/src/linear_algebra/CSysSolve.cpp index 953bed0ef2af..8ec1b19b23b3 100644 --- a/Common/src/linear_algebra/CSysSolve.cpp +++ b/Common/src/linear_algebra/CSysSolve.cpp @@ -208,15 +208,19 @@ bool CSysSolve::ModGramSchmidt(bool shared_hsbg, int i, su2matrix::multiDot(w, i + 1, 1, w, i + 1); LinearCombination( - shared_hsbg, i + 1, w, [&h_i](int k) { return -h_i(0, k); }, w[i + 1], true); - - const auto& dh_i = CSysVector::multiDot(w, i + 1, 1, w, i + 1); - LinearCombination( - shared_hsbg, i + 1, w, [&dh_i](int k) { return -dh_i(0, k); }, w[i + 1], true); - - for (int k = 0; k < i + 1; k++) SetHsbg(k, i, h_i(0, k) + dh_i(0, k)); + false, i + 1, w, [&h_i](int k) { return -h_i(0, k); }, w[i + 1], true); + if (i < 5) { + for (int k = 0; k < i + 1; k++) SetHsbg(k, i, h_i(0, k)); + } else { + const auto& dh_i = CSysVector::multiDot(w, i + 1, 1, w, i + 1); + LinearCombination( + false, i + 1, w, [&dh_i](int k) { return -dh_i(0, k); }, w[i + 1], true); + for (int k = 0; k < i + 1; k++) SetHsbg(k, i, h_i(0, k) + dh_i(0, k)); + } /*--- The norm of w[i+1] is 0 or NaN: the input vector from mat_vec is * zero or contains NaN. Cannot proceed with orthogonalization. ---*/ @@ -425,7 +429,7 @@ unsigned long CSysSolve::FGMRES_LinSolver(const CSysVector 1; + const bool nestedParallel = !omp_in_parallel() && omp_get_max_threads() > 1 && !VecExpr::UseDeviceExpressions(); /*--- Check the subspace size ---*/ @@ -546,6 +550,8 @@ unsigned long CSysSolve::FGMRES_LinSolver(const CSysVector::FGCRODR_LinSolverImpl(const CSysVector 1; + const bool nestedParallel = !omp_in_parallel() && omp_get_max_threads() > 1 && !VecExpr::UseDeviceExpressions(); /*--- Check the subspace size. ---*/ @@ -822,6 +828,8 @@ unsigned long CSysSolve::FGCRODR_LinSolverImpl(const CSysVector::FGCRODR_LinSolverImpl(const CSysVector::multiDot(V, i0, n, W, k); END_SU2_OMP_PARALLEL } else { @@ -1419,6 +1429,16 @@ unsigned long CSysSolve::Solve(CSysMatrix& Jacobian, con break; } + /*--- Poisson solver mode ---*/ + case LINEAR_SOLVER_MODE::POISSON: { + KindSolver = config->GetKind_Poisson_Linear_Solver(); + KindPrecond = config->GetKind_Poisson_Linear_Solver_Prec(); + MaxIter = config->GetPoisson_Linear_Solver_Iter(); + SolverTol = SU2_TYPE::GetValue(config->GetPoisson_Linear_Solver_Error()); + ScreenOutput = false; + break; + } + /*--- Normal mode assumes that 'lin_sol_mode==LINEAR_SOLVER_MODE::STANDARD', * but does not enforce it to avoid compiler warning. ---*/ default: { @@ -1462,7 +1482,7 @@ unsigned long CSysSolve::Solve(CSysMatrix& Jacobian, con auto externalFunction = [&]() { /*--- Create matrix-vector product, preconditioner, and solve the linear system ---*/ - HandleTemporariesIn(LinSysRes, LinSysSol); + HandleTemporariesIn(LinSysRes, LinSysSol, config->GetCUDA()); auto mat_vec = CSysMatrixVectorProduct(Jacobian, geometry, config); @@ -1537,7 +1557,7 @@ unsigned long CSysSolve::Solve(CSysMatrix& Jacobian, con } END_SU2_OMP_MASTER - HandleTemporariesOut(LinSysSol); + HandleTemporariesOut(LinSysSol, config->GetCUDA()); delete normal_prec; delete nested_prec; @@ -1564,10 +1584,16 @@ unsigned long CSysSolve::Solve(CSysMatrix& Jacobian, con break; case JACOBI: case LINELET: + case Q_JACOBI: + /*--- BuildJacobiPreconditioner() quantizes the diagonal itself when needed. ---*/ if (RequiresTranspose) Jacobian.BuildJacobiPreconditioner(); break; case LU_SGS: - /*--- Nothing to build. ---*/ + case Q_LU_SGS: + /*--- Nothing to build on the host, but the device keeps the inverted diagonal blocks + * and those have to follow the transpose (no-op without CUDA). Transpose path not + * supported for Q_LU_SGS, see CSysMatrix::Initialize. ---*/ + if (RequiresTranspose) Jacobian.BuildLU_SGSPreconditioner(); break; case PASTIX_ILU: case PASTIX_LU_P: @@ -1627,6 +1653,16 @@ unsigned long CSysSolve::Solve_b(CSysMatrix& Jacobian, c break; } + /*--- Poisson solver mode ---*/ + case LINEAR_SOLVER_MODE::POISSON: { + KindSolver = config->GetKind_Poisson_Linear_Solver(); + KindPrecond = config->GetKind_Poisson_Linear_Solver_Prec(); + MaxIter = config->GetPoisson_Linear_Solver_Iter(); + SolverTol = SU2_TYPE::GetValue(config->GetPoisson_Linear_Solver_Error()); + ScreenOutput = false; + break; + } + /*--- Normal mode assumes that 'lin_sol_mode==LINEAR_SOLVER_MODE::STANDARD', * but does not enforce it to avoid compiler warning. ---*/ default: { @@ -1654,6 +1690,10 @@ unsigned long CSysSolve::Solve_b(CSysMatrix& Jacobian, c normal_prec->Build(); } + /*--- The vectors are already of the solver type here, but they still have to cross the + * bus: the matrix and preconditioner operations dispatch to the device on their own. ---*/ + HandleTemporariesIn(LinSysRes, LinSysSol, config->GetCUDA()); + CPreconditioner* nested_prec = nullptr; if (nested) { auto f = [&](const CSysVector& u, CSysVector& v) { @@ -1713,6 +1753,8 @@ unsigned long CSysSolve::Solve_b(CSysMatrix& Jacobian, c break; } + HandleTemporariesOut(LinSysSol, config->GetCUDA()); + delete normal_prec; delete nested_prec; @@ -1728,6 +1770,6 @@ unsigned long CSysSolve::Solve_b(CSysMatrix& Jacobian, c /*--- Explicit instantiations ---*/ template class CSysSolve; -#ifdef USE_MIXED_PRECISION +#if defined(USE_MIXED_PRECISION) && !defined(USE_SINGLE_PRECISION) template class CSysSolve; #endif diff --git a/Common/src/linear_algebra/CSysVector.cpp b/Common/src/linear_algebra/CSysVector.cpp index 61fc3b92a4fc..22449c3759d9 100644 --- a/Common/src/linear_algebra/CSysVector.cpp +++ b/Common/src/linear_algebra/CSysVector.cpp @@ -52,6 +52,9 @@ void CSysVector::Initialize(unsigned long numBlk, unsigned long numB if (vec_val == nullptr) vec_val = MemoryAllocation::aligned_alloc(64, nElm * sizeof(ScalarType)); + /*--- Device storage mirrors the host allocation; free first so that re-initializing a + * vector does not leak it. ---*/ + GPUMemoryAllocation::gpu_free(d_vec_val); d_vec_val = GPUMemoryAllocation::gpu_alloc(nElm * sizeof(ScalarType)); #ifdef HAVE_OMP @@ -73,49 +76,69 @@ const su2matrix& CSysVector::multiDot(const std::vector< const std::vector>& W, const size_t m) { SU2_ZONE_SCOPED - static constexpr size_t BLOCK_SIZE = 1024; + static su2matrix shared; if (n == 0 || m == 0) return shared; - SU2_OMP_BARRIER - const size_t size = V[0].nElmDomain; + su2matrix local; - su2matrix local(n, m); - local.setConstant(0); - - SU2_OMP_FOR_(schedule(static) SU2_NOWAIT) - for (size_t offset = 0; offset < size; offset += BLOCK_SIZE) { - const auto limit = std::min(offset + BLOCK_SIZE, size); - for (size_t i = 0; i < n; ++i) { - const auto& vi = V[i0 + i]; - for (size_t j = 0; j < m; ++j) { - const auto& wj = W[j]; - ScalarType sum = 0.0; - SU2_OMP_SIMD - for (auto k = offset; k < limit; ++k) { - sum += vi[k] * wj[k]; + if (VecExpr::UseDeviceExpressions()) { +#ifdef SU2_ENABLE_CUDA_KERNELS + if constexpr (su2_gpu_capable_v) { + BEGIN_SU2_DEVICE_REGION + local = multiDotGPU(V, i0, n, W, m); + END_SU2_DEVICE_REGION + } else { + SU2_MPI::Error("GPU acceleration is not supported for AD scalar types.", CURRENT_FUNCTION); + } +#else + SU2_MPI::Error( + "\nError in multiDot\nENABLE_CUDA is set to YES\nPlease compile with CUDA options " + "enabled in Meson to access GPU Functions", + CURRENT_FUNCTION); +#endif + } else { + static constexpr size_t BLOCK_SIZE = 1024; + + SU2_OMP_BARRIER + const size_t size = V[0].nElmDomain; + + local.resize(n, m); + local.setConstant(0); + + SU2_OMP_FOR_(schedule(static) SU2_NOWAIT) + for (size_t offset = 0; offset < size; offset += BLOCK_SIZE) { + const auto limit = std::min(offset + BLOCK_SIZE, size); + for (size_t i = 0; i < n; ++i) { + const auto& vi = V[i0 + i]; + for (size_t j = 0; j < m; ++j) { + const auto& wj = W[j]; + ScalarType sum = 0.0; + SU2_OMP_SIMD + for (auto k = offset; k < limit; ++k) { + sum += vi[k] * wj[k]; + } + local(i, j) += sum; } - local(i, j) += sum; } } - } - END_SU2_OMP_FOR + END_SU2_OMP_FOR - /*--- Reduce over all threads in an ordered way to ensure a deterministic result. ---*/ - for (size_t i = 0; i < n; ++i) { - for (size_t j = 0; j < m; ++j) { - W[j].dot_scratch[omp_get_thread_num()] = local(i, j); - } - BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS - for (size_t j = 0; j < m; ++j) { - for (int t = 1; t < omp_get_num_threads(); ++t) { - local(i, j) += W[j].dot_scratch[t]; + /*--- Reduce over all threads in an ordered way to ensure a deterministic result. ---*/ + for (size_t i = 0; i < n; ++i) { + for (size_t j = 0; j < m; ++j) { + W[j].dot_scratch[omp_get_thread_num()] = local(i, j); } + BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS + for (size_t j = 0; j < m; ++j) { + for (int t = 1; t < omp_get_num_threads(); ++t) { + local(i, j) += W[j].dot_scratch[t]; + } + } + END_SU2_OMP_SAFE_GLOBAL_ACCESS } - END_SU2_OMP_SAFE_GLOBAL_ACCESS } - /*--- Single AllReduce of the result, only the master thread communicates. ---*/ SU2_OMP_MASTER { shared.resize(n, m); @@ -144,7 +167,7 @@ CSysVector::~CSysVector() { /*--- Explicit instantiations ---*/ template class CSysVector; -#ifdef USE_MIXED_PRECISION +#if defined(USE_MIXED_PRECISION) && !defined(USE_SINGLE_PRECISION) template class CSysVector; #endif #ifdef CODI_REVERSE_TYPE diff --git a/Common/src/linear_algebra/CSysVectorGPU.cu b/Common/src/linear_algebra/CSysVectorGPU.cu index 94ec17bb88fa..ef699893423e 100644 --- a/Common/src/linear_algebra/CSysVectorGPU.cu +++ b/Common/src/linear_algebra/CSysVectorGPU.cu @@ -1,7 +1,7 @@ /*! * \file CSysVectorGPU.cu * \brief Implementations of Kernels and Functions for Vector Operations on the GPU - * \author A. Raj + * \author A. Raj, D. Di giusto * \version 8.5.0 "Harrier" * * SU2 Project Website: https://su2code.github.io @@ -9,7 +9,7 @@ * The SU2 Project is maintained by the SU2 Foundation * (http://su2foundation.org) * - * Copyright 2012-2024, SU2 Contributors (cf. AUTHORS.md) + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) * * SU2 is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -25,25 +25,419 @@ * License along with SU2. If not, see . */ +/*--- Must come first, see the file for why. ---*/ +#include "../../include/basic_types/codi_host_only.hpp" + #include "../../include/linear_algebra/CSysVector.hpp" #include "../../include/linear_algebra/GPUComms.cuh" +#include +#include + +namespace { + +/*! + * \brief Fixed launch shape for DotKernel, so the number of warps per block (and thus the static + * shared-memory reduction buffer's size) is a compile-time constant - no dynamic shared + * memory needed for a single running sum, unlike MultiDotKernel's per-(i,j) buffer. + */ +constexpr unsigned int DOT_THREADS_PER_BLOCK = 256; +constexpr unsigned int DOT_WARPS_PER_BLOCK = DOT_THREADS_PER_BLOCK / 32u; + +/*! + * \brief Single dot product result[0] = , same single-pass grid-stride + warp-shuffle + + * block-combine + atomicAdd reduction as MultiDotKernel (see its comment), specialized for + * the one-running-sum case: no per-thread array, no dynamic shared memory, just a plain + * register accumulator and a small static warpSums buffer. + */ +template +__global__ void DotKernel(const ScalarType* __restrict__ x, const ScalarType* __restrict__ y, unsigned long size, + ScalarType* __restrict__ result) { + ScalarType sum = 0; + const auto stride = static_cast(blockDim.x) * gridDim.x; + for (auto k = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; k < size; k += stride) { + sum += x[k] * y[k]; + } + + /*--- Warp-level reduction: after this, lane 0 of every warp holds that warp's true sum. ---*/ + for (int offset = 16; offset > 0; offset >>= 1) sum += __shfl_down_sync(0xFFFFFFFFu, sum, offset); + + /*--- Block-level combine: warp leaders stage their partials in static shared memory, thread 0 + * sums them and issues the one atomicAdd this block contributes to the global result + * (pre-zeroed by the caller). ---*/ + __shared__ ScalarType warpSums[DOT_WARPS_PER_BLOCK]; + const unsigned int lane = threadIdx.x % 32u; + const unsigned int warpId = threadIdx.x / 32u; -template -void CSysVector::HtDTransfer(bool trigger) const -{ - if(trigger) gpuErrChk(cudaMemcpy((void*)(d_vec_val), (void*)&vec_val[0], (sizeof(ScalarType)*nElm), cudaMemcpyHostToDevice)); + if (lane == 0) warpSums[warpId] = sum; + __syncthreads(); + + if (threadIdx.x == 0) { + ScalarType blockSum = 0; + for (unsigned int w = 0; w < DOT_WARPS_PER_BLOCK; ++w) blockSum += warpSums[w]; + atomicAdd(result, blockSum); + } } -template -void CSysVector::DtHTransfer(bool trigger) const -{ - if(trigger) gpuErrChk(cudaMemcpy((void*)(&vec_val[0]), (void*)d_vec_val, (sizeof(ScalarType)*nElm), cudaMemcpyDeviceToHost)); +/*! + * \brief Cap on n*m (the number of dot products one MultiDotKernel launch computes), sizing the + * per-thread accumulator array below and the block's shared-memory reduction buffer - + * both compile-time sized, since CUDA has no runtime-sized register/local arrays. + * FGCRODR's Ritz-value path (CSysSolve.cpp) can reach n = m+1 with m = + * LINEAR_SOLVER_RESTART_DEFLATION (user-configurable, default 4 but not uncommon to raise + * into the tens), so n*m grows quadratically with that setting - e.g. m=10 already needs + * 110. Sized generously above realistic usage; multiDotGPU raises a clear SU2_MPI::Error + * rather than silently truncating if a caller ever needs more. + */ +constexpr unsigned int MULTIDOT_MAX_NM = 1024; + +/*! + * \brief Caps on the individual vector counts n and m, bounding the fixed-size pointer arrays + * passed into MultiDotKernel by value as ordinary launch parameters. + * Asymmetric because the two call sites in this codebase produce genuinely asymmetric + * shapes. multiDotGPU always puts whichever of n, m is larger into the "large" kernel + * argument (transposing the result back if needed). + */ +constexpr unsigned int MULTIDOT_MAX_VEC_LARGE = 256; +constexpr unsigned int MULTIDOT_MAX_VEC_SMALL = 64; + +/*! + * \brief Fixed-size array of device pointers, passed to MultiDotKernel by value (see + * MULTIDOT_MAX_VEC_LARGE/SMALL for why). + */ +template +struct MultiDotPointers { + const ScalarType* ptr[MaxCount]; +}; + +/*! + * \brief Compute the aCount*bCount matrix of dot products D(a,b) = in a single pass + * over the data: every thread reads each of the aCount+bCount vectors once per k and + * forms all aCount*bCount products from that same read, so the vectors are only ever read + * once total (aCount+bCount reads of length size, not aCount*bCount), and the reduction + * stays memory-bound. + * \note Each thread accumulates its own private running aCount*bCount sums while striding over k + * (kept in thread-local storage, capped at MULTIDOT_MAX_NM - small enough that it should + * stay resident in registers or L1 for realistic shapes, cheap either way next to the + * K-length main loop's DRAM traffic). Only after that loop do threads combine: a + * warp-shuffle reduction, then one shared-memory combine and one atomicAdd per (a,b) per + * block, so total atomics are O(aCount*bCount * numBlocks), not O(aCount*bCount * size). + * \note A is the "large" argument (up to MULTIDOT_MAX_VEC_LARGE), B the "small" one (up to + * MULTIDOT_MAX_VEC_SMALL) - the caller (multiDotGPU) is responsible for putting the larger + * of its two vector counts into A, and transposing the result back if it had to swap. + */ +template +__global__ void MultiDotKernel(MultiDotPointers A, unsigned int aCount, + MultiDotPointers B, unsigned int bCount, + unsigned long size, ScalarType* __restrict__ D) { + const unsigned int nm = aCount * bCount; + + ScalarType local[MULTIDOT_MAX_NM]; + for (unsigned int t = 0; t < nm; ++t) local[t] = ScalarType(0); + + const auto stride = static_cast(blockDim.x) * gridDim.x; + for (auto k = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; k < size; k += stride) { + for (unsigned int a = 0; a < aCount; ++a) { + const ScalarType v = A.ptr[a][k]; + for (unsigned int b = 0; b < bCount; ++b) local[a * bCount + b] += v * B.ptr[b][k]; + } + } + + /*--- Warp-level reduction: after this, lane 0 of every warp holds that warp's true sum. ---*/ + for (unsigned int t = 0; t < nm; ++t) { + ScalarType val = local[t]; + for (int offset = 16; offset > 0; offset >>= 1) val += __shfl_down_sync(0xFFFFFFFFu, val, offset); + local[t] = val; + } + + /*--- Block-level combine: warp leaders stage their partials in shared memory, thread 0 sums + * them and issues the one atomicAdd per (i,j) this block contributes to the global result + * (pre-zeroed by the caller). ---*/ + extern __shared__ __align__(sizeof(double)) char smem[]; + auto* warpPartials = reinterpret_cast(smem); + const unsigned int lane = threadIdx.x % 32u; + const unsigned int warpId = threadIdx.x / 32u; + const unsigned int warpsPerBlock = blockDim.x / 32u; + + if (lane == 0) { + for (unsigned int t = 0; t < nm; ++t) warpPartials[warpId * nm + t] = local[t]; + } + __syncthreads(); + + if (threadIdx.x == 0) { + for (unsigned int t = 0; t < nm; ++t) { + ScalarType sum = 0; + for (unsigned int w = 0; w < warpsPerBlock; ++w) sum += warpPartials[w * nm + t]; + atomicAdd(&D[t], sum); + } + } } -template -void CSysVector::GPUSetVal(ScalarType val, bool trigger) const -{ - if(trigger) gpuErrChk(cudaMemset((void*)(d_vec_val), val, (sizeof(ScalarType)*nElm))); +} // namespace + +namespace VecExpr { + +namespace { +/*--- Deliberately not thread local: every thread of an OpenMP team has to agree on this, + * otherwise the team splits over the worksharing constructs in CSysVector. It is only + * written by CSysSolve, outside any parallel region over the linear system. ---*/ +bool use_device_expressions = false; +} // namespace + +bool UseDeviceExpressions() { return use_device_expressions; } + +void SetUseDeviceExpressions(bool use) { use_device_expressions = use; } + +} // namespace VecExpr + +template +void CSysVector::HtDTransfer(bool trigger) const { + SU2_ZONE_SCOPED + if (trigger) + gpuErrChk(cudaMemcpy((void*)(d_vec_val), (void*)&vec_val[0], (sizeof(ScalarType) * nElm), cudaMemcpyHostToDevice)); } -template class CSysVector; //This is a temporary fix for invalid instantiations due to separating the member function from the header file the class is defined in. Will try to rectify it in coming commits. +template +void CSysVector::DtHTransfer(bool trigger) const { + SU2_ZONE_SCOPED + if (trigger) + gpuErrChk(cudaMemcpy((void*)(&vec_val[0]), (void*)d_vec_val, (sizeof(ScalarType) * nElm), cudaMemcpyDeviceToHost)); +} + +template +ScalarType CSysVector::dotGPU(const CSysVector& other) const { + SU2_ZONE_SCOPED + /*--- Both operands are already on the device, the caller owns the transfers. This reduces over + * MPI, so it must be called by a single thread (see SU2_DEVICE_REGION). ---*/ + static ScalarType* d_result = nullptr; + if (d_result == nullptr) gpuErrChk(cudaMalloc(&d_result, sizeof(ScalarType))); + + gpuErrChk(cudaMemsetAsync(d_result, 0, sizeof(ScalarType))); + + const auto blocks = static_cast( + std::min((nElmDomain + DOT_THREADS_PER_BLOCK - 1) / DOT_THREADS_PER_BLOCK, 1024)); + DotKernel<<>>(GetDevicePointer(), other.GetDevicePointer(), nElmDomain, + d_result); + + ScalarType local_dot = ScalarType(0); + gpuErrChk(cudaMemcpyAsync(&local_dot, d_result, sizeof(ScalarType), cudaMemcpyDeviceToHost)); + gpuErrChk(cudaStreamSynchronize(nullptr)); + gpuErrChk(cudaGetLastError()); + + ScalarType global_dot = ScalarType(0); + const auto mpi_type = (sizeof(ScalarType) < sizeof(double)) ? MPI_FLOAT : MPI_DOUBLE; + SelectMPIWrapper::W::Allreduce(&local_dot, &global_dot, 1, mpi_type, MPI_SUM, SU2_MPI::GetComm()); + + return global_dot; +} + +/*! + * \brief Multi vector dot product, local(i,j) = , via MultiDotKernel. + * \note Whichever of n, m is larger is passed as MultiDotKernel's "A" (large-capacity) argument + * and the other as "B" (small-capacity) - see MULTIDOT_MAX_VEC_LARGE/SMALL for why - so if + * m > n, V and W (and their counts) are swapped for the call, and the resulting m*n matrix + * is transposed back into the n*m shape the caller expects (cheap: this matrix is at most + * MULTIDOT_MAX_NM elements, nowhere near the size of the reduction itself). + */ +template +su2matrix CSysVector::multiDotGPU(const std::vector>& V, const size_t i0, + const size_t n, const std::vector>& W, + const size_t m) { + const size_t nm = n * m; + if (nm > MULTIDOT_MAX_NM) { + SU2_MPI::Error("CSysVector::multiDotGPU: n*m exceeds MULTIDOT_MAX_NM, raise that constant in " + "CSysVectorGPU.cu.", + CURRENT_FUNCTION); + } + + su2matrix local; + local.resize(n, m); + if (nm == 0) return local; + + const bool swap = m > n; + const size_t aCount = swap ? m : n; + const size_t bCount = swap ? n : m; + if (aCount > MULTIDOT_MAX_VEC_LARGE || bCount > MULTIDOT_MAX_VEC_SMALL) { + SU2_MPI::Error("CSysVector::multiDotGPU: n or m exceeds MULTIDOT_MAX_VEC_LARGE/SMALL, raise " + "those constants in CSysVectorGPU.cu.", + CURRENT_FUNCTION); + } + + const size_t size = V[0].nElmDomain; + + /*--- Persistent device workspace for the output only, cached across calls and freed + * automatically when the program exits (static local destruction), instead of leaking. The + * V/W pointers themselves need no device buffer at all: they go to MultiDotKernel as ordinary + * by-value launch parameters (see MultiDotPointers/MULTIDOT_MAX_VEC_LARGE/SMALL). ---*/ + struct Workspace { + ScalarType* d_D = nullptr; + size_t capacity = 0; + + void EnsureCapacity(size_t nm) { + if (nm > capacity) { + cudaFree(d_D); + gpuErrChk(cudaMalloc(&d_D, nm * sizeof(ScalarType))); + capacity = nm; + } + } + + ~Workspace() { cudaFree(d_D); } + }; + static Workspace ws; + ws.EnsureCapacity(nm); + + MultiDotPointers aPtrs{}; + MultiDotPointers bPtrs{}; + if (!swap) { + for (size_t i = 0; i < n; ++i) aPtrs.ptr[i] = V[i0 + i].GetDevicePointer(); + for (size_t j = 0; j < m; ++j) bPtrs.ptr[j] = W[j].GetDevicePointer(); + } else { + for (size_t i = 0; i < m; ++i) aPtrs.ptr[i] = W[i].GetDevicePointer(); + for (size_t j = 0; j < n; ++j) bPtrs.ptr[j] = V[i0 + j].GetDevicePointer(); + } + + gpuErrChk(cudaMemsetAsync(ws.d_D, 0, nm * sizeof(ScalarType))); + + constexpr unsigned int threadsPerBlock = 256; + const auto blocks = static_cast(std::min((size + threadsPerBlock - 1) / threadsPerBlock, 1024)); + const auto sharedBytes = static_cast(threadsPerBlock / 32u) * nm * sizeof(ScalarType); + + /*--- sharedBytes can exceed the default 48KB static shared-memory limit for large nm (the + * FGCRODR deflation matrix in particular, see MULTIDOT_MAX_NM); opt in to the device's larger + * "dynamic" limit once, the first time it is actually needed, rather than always paying for + * the query. ---*/ + static size_t optedInSharedBytes = 0; + if (sharedBytes > optedInSharedBytes) { + gpuErrChk(cudaFuncSetAttribute(MultiDotKernel, cudaFuncAttributeMaxDynamicSharedMemorySize, + static_cast(sharedBytes))); + optedInSharedBytes = sharedBytes; + } + + MultiDotKernel<<>>( + aPtrs, static_cast(aCount), bPtrs, static_cast(bCount), size, ws.d_D); + + if (!swap) { + gpuErrChk(cudaMemcpyAsync(local.data(), ws.d_D, nm * sizeof(ScalarType), cudaMemcpyDeviceToHost)); + gpuErrChk(cudaStreamSynchronize(nullptr)); + } else { + /*--- D is aCount*bCount = m*n row-major (D(a,b) = ); local is n*m with + * local(i,j) = = D(j,i), i.e. local is D transposed. ---*/ + static std::vector D; + D.resize(nm); + gpuErrChk(cudaMemcpyAsync(D.data(), ws.d_D, nm * sizeof(ScalarType), cudaMemcpyDeviceToHost)); + gpuErrChk(cudaStreamSynchronize(nullptr)); + for (size_t i = 0; i < n; ++i) + for (size_t j = 0; j < m; ++j) local(i, j) = D[j * n + i]; + } + gpuErrChk(cudaGetLastError()); + + return local; +} + +/*--- Every expression the solvers assign to a CSysVector needs its assignment kernel + * instantiated here; the host compiler cannot emit one. A shape that is missing shows up + * as an undefined reference to VecExpr::AssignDeviceExpression at link time, and is fixed + * by adding a line to DEVICE_EXPRESSION_SHAPES below. The aliases use CSysVector (not + * CVectorView) because that is how the operator overloads name their operands; store_t + * turns it into a view when the node is built. ---*/ +namespace { + +template +using Vec = CSysVector; +template +using Sca = VecExpr::Bcast; + +/*--- Leaves and the shapes of the FGMRES/GMRES basis updates. ---*/ +template +using DeviceBcast = Sca; +template +using DeviceView = VecExpr::CVectorView; +template +using DeviceNeg = VecExpr::minus_, S>; + +/*--- vector * scalar and scalar * vector are distinct types, both are used. ---*/ +template +using DeviceScale = VecExpr::mul_, Sca, S>; +template +using DeviceLScale = VecExpr::mul_, Vec, S>; +template +using DeviceDivScale = VecExpr::div_, Sca, S>; + +/*--- Linear combinations, CSysSolve unrolls them up to four terms. ---*/ +template +using DeviceScale2 = VecExpr::add_, DeviceScale, S>; +template +using DeviceScale3 = VecExpr::add_, DeviceScale, S>; +template +using DeviceScale4 = VecExpr::add_, DeviceScale, S>; + +/*--- r = b - A_x, in CG, BCGSTAB, Smoother and FGCRODR. ---*/ +template +using DeviceSub = VecExpr::sub_, Vec, S>; + +/*--- p = beta * p + z, in CG. ---*/ +template +using DeviceLScalePlus = VecExpr::add_, Vec, S>; + +/*--- p = beta * (p - omega * v) + r, in BCGSTAB. ---*/ +template +using DeviceSubLScale = VecExpr::sub_, DeviceLScale, S>; +template +using DeviceLScaleSub = VecExpr::mul_, DeviceSubLScale, S>; +template +using DeviceBcgsDir = VecExpr::add_, Vec, S>; + +} // namespace + +#define DEVICE_EXPRESSION_SHAPES(SCALAR) \ + INSTANTIATE_DEVICE_ASSIGN_EXPR(SCALAR, DeviceBcast); \ + INSTANTIATE_DEVICE_ASSIGN_EXPR(SCALAR, DeviceView); \ + INSTANTIATE_DEVICE_ASSIGN_EXPR(SCALAR, DeviceNeg); \ + INSTANTIATE_DEVICE_ASSIGN_EXPR(SCALAR, DeviceScale); \ + INSTANTIATE_DEVICE_ASSIGN_EXPR(SCALAR, DeviceLScale); \ + INSTANTIATE_DEVICE_ASSIGN_EXPR(SCALAR, DeviceDivScale); \ + INSTANTIATE_DEVICE_ASSIGN_EXPR(SCALAR, DeviceScale2); \ + INSTANTIATE_DEVICE_ASSIGN_EXPR(SCALAR, DeviceScale3); \ + INSTANTIATE_DEVICE_ASSIGN_EXPR(SCALAR, DeviceScale4); \ + INSTANTIATE_DEVICE_ASSIGN_EXPR(SCALAR, DeviceSub); \ + INSTANTIATE_DEVICE_ASSIGN_EXPR(SCALAR, DeviceLScalePlus); \ + INSTANTIATE_DEVICE_ASSIGN_EXPR(SCALAR, DeviceSubLScale); \ + INSTANTIATE_DEVICE_ASSIGN_EXPR(SCALAR, DeviceLScaleSub); \ + INSTANTIATE_DEVICE_ASSIGN_EXPR(SCALAR, DeviceBcgsDir) + +#define INSTANTIATE_DEVICE_ASSIGN(SCALAR, OP, EXPR) \ + template void VecExpr::AssignDeviceExpression>( \ + SCALAR*, unsigned long, const VecExpr::CVecExpr, SCALAR>&) + +#define INSTANTIATE_DEVICE_ASSIGN_EXPR(SCALAR, EXPR) \ + INSTANTIATE_DEVICE_ASSIGN(SCALAR, Assign, EXPR); \ + INSTANTIATE_DEVICE_ASSIGN(SCALAR, Add, EXPR); \ + INSTANTIATE_DEVICE_ASSIGN(SCALAR, Subtract, EXPR); \ + INSTANTIATE_DEVICE_ASSIGN(SCALAR, Multiply, EXPR); \ + INSTANTIATE_DEVICE_ASSIGN(SCALAR, Divide, EXPR) + +DEVICE_EXPRESSION_SHAPES(su2mixedfloat); + +#if defined(USE_MIXED_PRECISION) && !defined(USE_SINGLE_PRECISION) +DEVICE_EXPRESSION_SHAPES(passivedouble); +#endif + +#undef DEVICE_EXPRESSION_SHAPES +#undef INSTANTIATE_DEVICE_ASSIGN_EXPR +#undef INSTANTIATE_DEVICE_ASSIGN + + +template void CSysVector::HtDTransfer(bool trigger) const; +template void CSysVector::DtHTransfer(bool trigger) const; +template su2mixedfloat CSysVector::dotGPU(const CSysVector& other) const; +template su2matrix CSysVector::multiDotGPU( + const std::vector>& V, size_t i0, size_t n, + const std::vector>& W, size_t m); + +#if defined(USE_MIXED_PRECISION) && !defined(USE_SINGLE_PRECISION) +template void CSysVector::HtDTransfer(bool trigger) const; +template void CSysVector::DtHTransfer(bool trigger) const; +template passivedouble CSysVector::dotGPU(const CSysVector& other) const; +template su2matrix CSysVector::multiDotGPU( + const std::vector>& V, size_t i0, size_t n, + const std::vector>& W, size_t m); +#endif diff --git a/Common/src/linear_algebra/blas_structure.cpp b/Common/src/linear_algebra/blas_structure.cpp index 2bd87cb378f9..bb85ed9610ad 100644 --- a/Common/src/linear_algebra/blas_structure.cpp +++ b/Common/src/linear_algebra/blas_structure.cpp @@ -34,11 +34,19 @@ #if (defined(HAVE_MKL) || defined(HAVE_BLAS)) && !(defined(CODI_REVERSE_TYPE) || defined(CODI_FORWARD_TYPE)) /* Function prototypes for the BLAS routines used. */ -extern "C" void dgemm_(char*, char*, const int*, const int*, const int*, const passivedouble*, const passivedouble*, - const int*, const passivedouble*, const int*, const passivedouble*, passivedouble*, const int*); +#ifdef USE_SINGLE_PRECISION +#define GEMM_IMPL sgemm_ +#define GEMV_IMPL sgemv_ +#else +#define GEMM_IMPL dgemm_ +#define GEMV_IMPL dgemv_ +#endif +extern "C" void GEMM_IMPL(char*, char*, const int*, const int*, const int*, const passivedouble*, const passivedouble*, + const int*, const passivedouble*, const int*, const passivedouble*, passivedouble*, + const int*); -extern "C" void dgemv_(char*, const int*, const int*, const passivedouble*, const passivedouble*, const int*, - const passivedouble*, const int*, const passivedouble*, passivedouble*, const int*); +extern "C" void GEMV_IMPL(char*, const int*, const int*, const passivedouble*, const passivedouble*, const int*, + const passivedouble*, const int*, const passivedouble*, passivedouble*, const int*); #endif /* Constructor. Initialize the const member variables, if needed. */ @@ -77,7 +85,7 @@ void CBlasStructure::gemm(const int M, const int N, const int K, const su2double su2double beta = 0.0; char trans = 'N'; - dgemm_(&trans, &trans, &N, &M, &K, &alpha, B, &N, A, &K, &beta, C, &N); + GEMM_IMPL(&trans, &trans, &N, &M, &K, &alpha, B, &N, A, &K, &beta, C, &N); #endif @@ -100,7 +108,7 @@ void CBlasStructure::gemv(const int M, const int N, const su2double* A, const su int inc = 1; char trans = 'T'; - dgemv_(&trans, &N, &M, &alpha, A, &N, x, &inc, &beta, y, &inc); + GEMV_IMPL(&trans, &N, &M, &alpha, A, &N, x, &inc, &beta, y, &inc); #else diff --git a/Common/src/linear_algebra/meson.build b/Common/src/linear_algebra/meson.build index 7b880b29c1e3..c86898903bdf 100644 --- a/Common/src/linear_algebra/meson.build +++ b/Common/src/linear_algebra/meson.build @@ -5,6 +5,8 @@ common_src += files(['CSysSolve_b.cpp', 'CPastixWrapper.cpp', 'blas_structure.cpp']) - if get_option('enable-cuda') - common_src += files(['CSysMatrixGPU.cu', 'CSysVectorGPU.cu',]) +if get_option('enable-cuda') + # Kept apart from common_src because nvcc takes a different set of flags, see + # common_cuda_src in Common/src/meson.build. + common_cuda_src += files(['CSysMatrixGPU.cu', 'CSysVectorGPU.cu']) endif diff --git a/Common/src/meson.build b/Common/src/meson.build index f385c4a32edd..52b02348a95d 100644 --- a/Common/src/meson.build +++ b/Common/src/meson.build @@ -6,6 +6,25 @@ common_src =files(['graph_coloring_structure.cpp', '../include/parallelization/mpi_structure.cpp', '../include/parallelization/omp_structure.cpp']) +# The CUDA sources are kept in their own list because nvcc only accepts the defines out of +# su2_cpp_args, not the warning and tuning flags meant for the host compiler. They go into +# the primal and the reverse AD libraries with the same defines as the .cpp sources next to +# them, so su2double means the same thing everywhere; codi_host_only.hpp is what keeps +# CoDiPack out of the device pass. Forward AD gets no kernels, see SU2_ENABLE_CUDA_KERNELS. +common_cuda_src = [] +common_cuda_cpp_args = [] +foreach arg : su2_cpp_args + if arg.startswith('-D') or arg.startswith('-U') + common_cuda_cpp_args += arg + endif +endforeach + +# Note that the OpenMP flags do reach nvcc (through omp_dep), so the .cu objects agree +# with the rest of the library about HAVE_OMP. They must still not instantiate anything +# containing OpenMP directives, because nvcc's host pass rewrites some of them; the +# device path is synchronized in the headers (see SU2_DEVICE_REGION in CSysVector.hpp), +# which only the .cpp sources instantiate. + subdir('linear_algebra') subdir('toolboxes') subdir('geometry') @@ -21,10 +40,11 @@ subdir('adt') if get_option('enable-normal') common = static_library('SU2Common', - common_src, + common_src, common_cuda_src, install : false, dependencies : su2_deps, - cpp_args: [default_warning_flags, su2_cpp_args]) + cpp_args: [default_warning_flags, su2_cpp_args], + cuda_args: common_cuda_cpp_args) common_dep = declare_dependency(link_with: common, include_directories : common_include) @@ -32,11 +52,15 @@ endif if get_option('enable-autodiff') + # codi_rev_args is only defined for AD builds, and it is all defines, which nvcc accepts. + common_cuda_rev_args = common_cuda_cpp_args + codi_rev_args + commonAD = static_library('SU2CommonAD', - common_src, + common_src, common_cuda_src, install : false, dependencies : [su2_deps, codi_dep], - cpp_args: [default_warning_flags, su2_cpp_args, codi_rev_args]) + cpp_args: [default_warning_flags, su2_cpp_args, codi_rev_args], + cuda_args: common_cuda_rev_args) commonAD_dep = declare_dependency(link_with: commonAD, include_directories : common_include) diff --git a/Common/src/toolboxes/CSquareMatrixCM.cpp b/Common/src/toolboxes/CSquareMatrixCM.cpp index 9cbc407cac63..87f6eb356df9 100644 --- a/Common/src/toolboxes/CSquareMatrixCM.cpp +++ b/Common/src/toolboxes/CSquareMatrixCM.cpp @@ -38,11 +38,20 @@ using namespace std; #endif #elif defined(HAVE_LAPACK) /*--- Lapack / Blas routines used in CSquareMatrixCM. ---*/ -extern "C" void dgetrf_(const int*, const int*, passivedouble*, const int*, int*, int*); -extern "C" void dgetri_(const int*, passivedouble*, const int*, int*, passivedouble*, const int*, int*); -extern "C" void dgemm_(char*, char*, const int*, const int*, const int*, const passivedouble*, const passivedouble*, - const int*, const passivedouble*, const int*, const passivedouble*, passivedouble*, const int*); -#define DGEMM dgemm_ +#ifdef USE_SINGLE_PRECISION +#define GEMM_IMPL sgemm_ +#define GETRF_IMPL sgetrf_ +#define GETRI_IMPL sgetri_ +#else +#define GEMM_IMPL dgemm_ +#define GETRF_IMPL dgetrf_ +#define GETRI_IMPL dgetri_ +#endif +extern "C" void GETRF_IMPL(const int*, const int*, passivedouble*, const int*, int*, int*); +extern "C" void GETRI_IMPL(const int*, passivedouble*, const int*, int*, passivedouble*, const int*, int*); +extern "C" void GEMM_IMPL(char*, char*, const int*, const int*, const int*, const passivedouble*, const passivedouble*, + const int*, const passivedouble*, const int*, const passivedouble*, passivedouble*, + const int*); #endif void CSquareMatrixCM::Transpose() { @@ -59,10 +68,10 @@ void CSquareMatrixCM::Invert() { vector ipiv(sz); vector work(sz); - dgetrf_(&sz, &sz, mat.data(), &sz, ipiv.data(), &info); + GETRF_IMPL(&sz, &sz, mat.data(), &sz, ipiv.data(), &info); if (info != 0) SU2_MPI::Error(string("Matrix is singular"), CURRENT_FUNCTION); - dgetri_(&sz, mat.data(), &sz, ipiv.data(), work.data(), &sz, &info); + GETRI_IMPL(&sz, mat.data(), &sz, ipiv.data(), work.data(), &sz, &info); if (info != 0) SU2_MPI::Error(string("Matrix inversion failed"), CURRENT_FUNCTION); #else @@ -88,7 +97,7 @@ void CSquareMatrixCM::MatMatMult(const char side, const ColMajorMatrix(tmp); vector work(query); /*--- Factorize and invert. ---*/ - dsytrf_(&uplo, &sz, mat.data(), &sz, ipiv.data(), work.data(), &query, &info); + SYTRF_IMPL(&uplo, &sz, mat.data(), &sz, ipiv.data(), work.data(), &query, &info); if (info != 0) SU2_MPI::Error("LDLT factorization failed.", CURRENT_FUNCTION); - dsytri_(&uplo, &sz, mat.data(), &sz, ipiv.data(), work.data(), &info); + SYTRI_IMPL(&uplo, &sz, mat.data(), &sz, ipiv.data(), work.data(), &info); if (info != 0) SU2_MPI::Error("Inversion with LDLT factorization failed.", CURRENT_FUNCTION); #endif } @@ -137,9 +150,9 @@ void CSymmetricMatrix::CalcInv_potri() { const int sz = Size(); int info; - dpotrf_(&uplo, &sz, mat.data(), &sz, &info); + POTRF_IMPL(&uplo, &sz, mat.data(), &sz, &info); if (info != 0) SU2_MPI::Error("LLT factorization failed.", CURRENT_FUNCTION); - dpotri_(&uplo, &sz, mat.data(), &sz, &info); + POTRI_IMPL(&uplo, &sz, mat.data(), &sz, &info); if (info != 0) SU2_MPI::Error("Inversion with LLT factorization failed.", CURRENT_FUNCTION); #endif } @@ -167,7 +180,7 @@ void CSymmetricMatrix::MatMatMult(const char side, const su2passivematrix& mat_i /*--- Right and lower because matrices are in row major order. ---*/ const char side = 'R', uplo = 'L'; const passivedouble alpha = 1.0, beta = 0.0; - DSYMM(&side, &uplo, &N, &M, &alpha, mat.data(), &M, mat_in.data(), &N, &beta, mat_out.data(), &N); + SYMM_IMPL(&side, &uplo, &N, &M, &alpha, mat.data(), &M, mat_in.data(), &N, &beta, mat_out.data(), &N); #else // Naive product for (int i = 0; i < M; ++i) for (int j = 0; j < N; ++j) { @@ -187,7 +200,7 @@ void CSymmetricMatrix::MatMatMult(const char side, const su2passivematrix& mat_i /*--- Left and lower because matrices are in row major order. ---*/ const char side = 'L', uplo = 'L'; const passivedouble alpha = 1.0, beta = 0.0; - DSYMM(&side, &uplo, &N, &M, &alpha, mat.data(), &N, mat_in.data(), &N, &beta, mat_out.data(), &N); + SYMM_IMPL(&side, &uplo, &N, &M, &alpha, mat.data(), &N, mat_in.data(), &N, &beta, mat_out.data(), &N); #else // Naive product for (int i = 0; i < M; ++i) for (int j = 0; j < N; ++j) { diff --git a/Common/src/toolboxes/SwapBytes.cpp b/Common/src/toolboxes/SwapBytes.cpp new file mode 100644 index 000000000000..d60567e09ed0 --- /dev/null +++ b/Common/src/toolboxes/SwapBytes.cpp @@ -0,0 +1,53 @@ +/*! + * \file SwapBytes.cpp + * \brief Function to swap bytes of primitive data types + * \author P. Gomes + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#include "../../include/toolboxes/SwapBytes.hpp" + +/*--- Function to swap bytes, in case we need to convert between + big and little endian storage. ---*/ +void SwapBytes(char* buffer, size_t nBytes, unsigned long nVar) { + /*--- Store half the number of bytes in kk. ---*/ + const int kk = (int)nBytes / 2; + + /*--- Loop over the number of variables in the buffer. ---*/ + for (unsigned long j = 0; j < nVar; j++) { + /*--- Initialize ii and jj, which are used to store the + indices of the bytes to be swapped. ---*/ + unsigned long ii = j * nBytes; + unsigned long jj = ii + nBytes - 1; + + /*--- Swap the bytes. ---*/ + for (int i = 0; i < kk; i++) { + char tmp = buffer[jj]; + buffer[jj] = buffer[ii]; + buffer[ii] = tmp; + + ii++; + jj--; + } + } +} diff --git a/Common/src/toolboxes/meson.build b/Common/src/toolboxes/meson.build index fa5e3b234c34..3240bac986d5 100644 --- a/Common/src/toolboxes/meson.build +++ b/Common/src/toolboxes/meson.build @@ -2,7 +2,8 @@ common_src += files(['CLinearPartitioner.cpp', 'printing_toolbox.cpp', 'C1DInterpolation.cpp', 'CSquareMatrixCM.cpp', - 'CSymmetricMatrix.cpp']) + 'CSymmetricMatrix.cpp', + 'SwapBytes.cpp']) subdir('MMS') subdir('fem') diff --git a/Common/src/wall_model.cpp b/Common/src/wall_model.cpp index 60c975c43ed5..5c8ea30603cb 100644 --- a/Common/src/wall_model.cpp +++ b/Common/src/wall_model.cpp @@ -31,7 +31,12 @@ /* Prototypes for Lapack functions, if MKL or LAPACK is used. */ #if defined(HAVE_MKL) || defined(HAVE_LAPACK) -extern "C" void dgtsv_(int*, int*, passivedouble*, passivedouble*, passivedouble*, passivedouble*, int*, int*); +#ifdef USE_SINGLE_PRECISION +#define GTSV_IMPL sgtsv_ +#else +#define GTSV_IMPL dgtsv_ +#endif +extern "C" void GTSV_IMPL(int*, int*, passivedouble*, passivedouble*, passivedouble*, passivedouble*, int*, int*); #endif CWallModel::CWallModel(CConfig* config) { @@ -193,7 +198,7 @@ void CWallModel1DEQ::WallShearStressAndHeatFlux(const su2double tExchange, const #if (defined(HAVE_MKL) || defined(HAVE_LAPACK)) && !(defined(CODI_REVERSE_TYPE) || defined(CODI_FORWARD_TYPE)) int info, nrhs = 1; - dgtsv_(&numPoints, &nrhs, lower.data(), diagonal.data(), upper.data(), rhs.data(), &numPoints, &info); + GTSV_IMPL(&numPoints, &nrhs, lower.data(), diagonal.data(), upper.data(), rhs.data(), &numPoints, &info); if (info != 0) SU2_MPI::Error("Unsuccessful call to dgtsv_", CURRENT_FUNCTION); #else SU2_MPI::Error("Not compiled with MKL or LAPACK support", CURRENT_FUNCTION); @@ -273,7 +278,7 @@ void CWallModel1DEQ::WallShearStressAndHeatFlux(const su2double tExchange, const /* Solve the matrix problem to get the Enthalpy field */ #if (defined(HAVE_MKL) || defined(HAVE_LAPACK)) && !(defined(CODI_REVERSE_TYPE) || defined(CODI_FORWARD_TYPE)) - dgtsv_(&numPoints, &nrhs, lower.data(), diagonal.data(), upper.data(), rhs.data(), &numPoints, &info); + GTSV_IMPL(&numPoints, &nrhs, lower.data(), diagonal.data(), upper.data(), rhs.data(), &numPoints, &info); if (info != 0) SU2_MPI::Error("Unsuccessful call to dgtsv_", CURRENT_FUNCTION); #else SU2_MPI::Error("Not compiled with MKL or LAPACK support", CURRENT_FUNCTION); diff --git a/SU2_CFD/flamelet_python_bc.md b/SU2_CFD/flamelet_python_bc.md new file mode 100644 index 000000000000..3aa393cee60b --- /dev/null +++ b/SU2_CFD/flamelet_python_bc.md @@ -0,0 +1,225 @@ +# Flamelet Solver — Python Wrapper Wall Boundary Conditions + +This document explains how to use the Python wrapper (`pysu2`) to set +custom per-vertex boundary conditions for the flamelet (FGM) scalar transport +solver, and how the setup differs between the two `FLAMELET_ENTHALPY_BC` modes. + +--- + +## Background + +The flamelet solver transports `nVar` scalar variables. Their order is: + +| Index | Variable | +|-------|----------| +| 0 | `PROGVAR` (progress variable) | +| 1 | `ENTH` (total enthalpy) | +| 2 … n\_CV−1 | additional control variables (e.g. `MIXFRAC`) | +| n\_CV … nVar−1 | auxiliary / user-defined species | + +Two modes control how the enthalpy boundary condition is derived at thermal +walls (`MARKER_ISOTHERMAL`, `MARKER_HEATFLUX`): + +- **`FLOW_MARKERS`** — enthalpy is derived from the flow thermal field (wall + temperature via `MARKER_ISOTHERMAL`, or wall heat flux via `MARKER_HEATFLUX`). +- **`SPECIES_MARKERS`** (default) — enthalpy and all other scalars are taken + directly from `MARKER_WALL_SPECIES`, or from the Python wrapper when + `MARKER_PYTHON_CUSTOM` is active. + +--- + +## Mode 1: `FLAMELET_ENTHALPY_BC = FLOW_MARKERS` + +### How it works + +The C++ function `BC_HeatFlux_Wall` reads the wall heat flux and applies it as +a Neumann condition on `I_ENTH`. When the marker is also listed in +`MARKER_PYTHON_CUSTOM`, it instead reads a **per-vertex heat flux** set by the +Python wrapper. + +``` +driver.SetMarkerCustomNormalHeatFlux(iMarker, iVertex, q_wall) + → geometry->CustomBoundaryHeatFlux[iMarker][iVertex] + → BC_HeatFlux_Wall reads geometry->GetCustomBoundaryHeatFlux(...) +``` + +> `BC_Isothermal_Wall` in `FLOW_MARKERS` mode always converts the wall +> temperature (from `MARKER_ISOTHERMAL`) to enthalpy via `GetEnthFromTemp`. +> Per-vertex customisation of the enthalpy Dirichlet value is not supported in +> this mode — use `SPECIES_MARKERS` instead. + +### Limitations in this mode + +- Only `I_ENTH` is reachable via the Python wrapper. +- Non-enthalpy scalars (`PROGVAR`, auxiliary species) receive implicit zero + Neumann at the wall and cannot be set via Python. + +### config.cfg + +```cfg +FLAMELET_ENTHALPY_BC = FLOW_MARKERS + +% Flow solver wall BC — sets KindBC and provides the fallback heat flux value +% used when MARKER_PYTHON_CUSTOM is not active. +MARKER_HEATFLUX = ( burner_wall, 0.0 ) + +% Enable the per-vertex Python override path. +MARKER_PYTHON_CUSTOM = ( burner_wall ) +``` + +### run.py + +```python +import pysu2 + +driver = pysu2.CSinglezoneDriver("config.cfg", 1, False) + +marker_name = "burner_wall" +iMarker = list(driver.GetMarkerTags()).index(marker_name) +n_vertex = driver.GetNumberMarkerNodes(iMarker) + +def compute_heat_flux(coord): + """Heat flux [W/m²], positive = into the domain.""" + import math + r = math.sqrt(coord[0]**2 + coord[1]**2) + return -5000.0 * math.exp(-r**2 / 0.01) # Gaussian profile, heat out + +for iteration in range(driver.GetNumberIter()): + + for iVertex in range(n_vertex): + node_id = driver.GetMarkerNode(iMarker, iVertex) + coord = driver.GetInitialMeshCoord(node_id) # [x, y(, z)] + q_wall = compute_heat_flux(coord) + driver.SetMarkerCustomNormalHeatFlux(iMarker, iVertex, q_wall) + + driver.Preprocess(iteration) + driver.Run() + driver.Postprocess() + driver.Monitor(iteration) + driver.Output(iteration) + +driver.Finalize() +``` + +--- + +## Mode 2: `FLAMELET_ENTHALPY_BC = SPECIES_MARKERS` (default) + +### How it works + +Both `BC_HeatFlux_Wall` and `BC_Isothermal_Wall` delegate immediately to +`CSpeciesSolver::BC_Wall_Generic`, which processes **all `nVar` scalars** +independently. When `MARKER_PYTHON_CUSTOM` is active, the value for each +scalar is taken from the array set by `driver.SetMarkerCustomScalar()`. + +``` +driver.SetMarkerCustomScalar(iMarker, iVertex, [val_0, val_1, ..., val_nVar-1]) + → CustomBoundaryScalar[iMarker](iVertex, iVar) + → BC_Wall_Generic reads CustomBoundaryScalar for every iVar +``` + +The **type** of BC for each scalar (Dirichlet `VALUE` or Neumann `FLUX`) is +read from `MARKER_WALL_SPECIES` in the config and cannot be changed from +Python. The Python wrapper only overrides the **magnitude** per vertex. + +### config.cfg + +The example below uses `nVar = 3` (PROGVAR, ENTH, one auxiliary species). +Adjust the number of `BC_TYPE, value` pairs to match the actual number of +scalar variables in your setup. + +```cfg +FLAMELET_ENTHALPY_BC = SPECIES_MARKERS + +% Flow solver wall BC — still required to set KindBC. +% The choice between MARKER_ISOTHERMAL and MARKER_HEATFLUX only affects +% the flow solver; the species solver uses BC_Wall_Generic in both cases. +MARKER_ISOTHERMAL = ( burner_wall, 300.0 ) + +% Per-variable BC types for the species solver. +% Format: (marker_name, BC_TYPE, fallback_value, BC_TYPE, fallback_value, ...) +% One BC_TYPE+value pair per scalar variable, in index order. +% BC_TYPE: FLUX → Neumann (value is normal flux density [unit/m²]) +% VALUE → Dirichlet (value is the scalar value at the wall) +% +% When MARKER_PYTHON_CUSTOM is active, the fallback_value is ignored and +% the Python-supplied value is used instead. The BC_TYPE is always from config. +% +% idx 0 PROGVAR : zero Neumann (no progress variable source at the wall) +% idx 1 ENTH : Dirichlet (enthalpy value set per-vertex by Python) +% idx 2 aux_1 : zero Neumann (no auxiliary species flux at the wall) +MARKER_WALL_SPECIES = ( burner_wall, FLUX, 0.0, VALUE, 0.0, FLUX, 0.0 ) + +% Enable the per-vertex Python override path. +MARKER_PYTHON_CUSTOM = ( burner_wall ) +``` + +### run.py + +```python +import pysu2 + +driver = pysu2.CSinglezoneDriver("config.cfg", 1, False) + +marker_name = "burner_wall" +iMarker = list(driver.GetMarkerTags()).index(marker_name) +n_vertex = driver.GetNumberMarkerNodes(iMarker) + +def compute_wall_enthalpy(coord): + """Return wall enthalpy [J/kg] at this vertex.""" + # Example: uniform value corresponding to T=300 K from your LUT. + return 3.5e5 + +for iteration in range(driver.GetNumberIter()): + + for iVertex in range(n_vertex): + node_id = driver.GetMarkerNode(iMarker, iVertex) + coord = driver.GetInitialMeshCoord(node_id) + + enth_wall = compute_wall_enthalpy(coord) + + # Pass one value per scalar variable (length = nVar). + # PROGVAR (idx 0): 0.0 → applied as FLUX → zero Neumann (BC_TYPE from config) + # ENTH (idx 1): enth_wall → applied as VALUE → Dirichlet + # aux_1 (idx 2): 0.0 → applied as FLUX → zero Neumann + driver.SetMarkerCustomScalar(iMarker, iVertex, [0.0, enth_wall, 0.0]) + + driver.Preprocess(iteration) + driver.Run() + driver.Postprocess() + driver.Monitor(iteration) + driver.Output(iteration) + +driver.Finalize() +``` + +--- + +## Comparison + +| | `FLOW_MARKERS` | `SPECIES_MARKERS` | +|---|---|---| +| **Python API** | `SetMarkerCustomNormalHeatFlux` | `SetMarkerCustomScalar` | +| **Storage** | `geometry->CustomBoundaryHeatFlux` | `CustomBoundaryScalar[iMarker](iVertex, iVar)` | +| **What you set** | Per-vertex heat flux (W/m²) for `I_ENTH` only | Per-vertex values for **all** `nVar` scalars | +| **BC type control** | Hard-coded Neumann in C++ | Per-variable via `MARKER_WALL_SPECIES` in config | +| **`MARKER_ISOTHERMAL` walls** | Enthalpy computed from T via `GetEnthFromTemp`; not overridable by Python | Enthalpy set as Dirichlet `VALUE` from Python | +| **Non-enthalpy scalars** | Not reachable via Python | All controlled via the same `SetMarkerCustomScalar` call | +| **Required config keys** | `MARKER_HEATFLUX`, `MARKER_PYTHON_CUSTOM` | `MARKER_ISOTHERMAL` or `MARKER_HEATFLUX`, `MARKER_WALL_SPECIES`, `MARKER_PYTHON_CUSTOM` | + +--- + +## Notes + +- `MARKER_PYTHON_CUSTOM` is a purely additive flag. A marker can appear in + both `MARKER_HEATFLUX` (or `MARKER_ISOTHERMAL`) and `MARKER_PYTHON_CUSTOM` + simultaneously. +- The fallback values in `MARKER_WALL_SPECIES` are used when `py_custom` is + false (e.g., during a plain SU2\_CFD run without the Python driver). They + allow the same config to be used both ways. +- In `SPECIES_MARKERS` mode, `SetMarkerCustomScalar` must supply a vector of + exactly `nVar` values. Indices that correspond to `FLUX`-type variables + should receive `0.0` unless you intentionally want a non-zero flux there. +- The `CHT` (conjugate heat transfer) path (`BC_ConjugateHeat_Interface`) + always uses `BC_Isothermal_Wall_Generic` and is unaffected by + `FLAMELET_ENTHALPY_BC`. Python custom BCs are not applicable to CHT markers. diff --git a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp index 60d040532ca0..ac3bc15e713f 100644 --- a/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp +++ b/SU2_CFD/include/drivers/CDiscAdjMultizoneDriver.hpp @@ -28,45 +28,21 @@ #pragma once #include "CMultizoneDriver.hpp" #include "../../../Common/include/toolboxes/CQuasiNewtonInvLeastSquares.hpp" -#include "../../../Common/include/linear_algebra/CPreconditioner.hpp" -#include "../../../Common/include/linear_algebra/CMatrixVectorProduct.hpp" #include "../../../Common/include/linear_algebra/CSysSolve.hpp" /*! * \brief Block Gauss-Seidel driver for multizone / multiphysics discrete adjoint problems. * \ingroup DiscAdj */ + class CDiscAdjMultizoneDriver : public CMultizoneDriver { protected: -#ifdef CODI_FORWARD_TYPE - using Scalar = su2double; -#else - using Scalar = passivedouble; -#endif - - class AdjointProduct : public CMatrixVectorProduct { - public: - CDiscAdjMultizoneDriver* const driver; - const unsigned short iZone = 0; - mutable unsigned long iInnerIter = 0; - - AdjointProduct(CDiscAdjMultizoneDriver* d, unsigned short i) : driver(d), iZone(i) {} - - inline void operator()(const CSysVector & u, CSysVector & v) const override { - driver->SetAllSolutions(iZone, true, u); - driver->Iterate(iZone, iInnerIter, true); - driver->GetAllSolutions(iZone, true, v); - v -= u; - ++iInnerIter; - } - }; - - class Identity : public CPreconditioner { - public: - inline bool IsIdentity() const override { return true; } - inline void operator()(const CSysVector & u, CSysVector & v) const override { v = u; } - }; + #ifdef CODI_FORWARD_TYPE + using Scalar = su2double; + #else + using Scalar = passivedouble; + #endif /*! * \brief Kinds of recordings. @@ -141,16 +117,6 @@ class CDiscAdjMultizoneDriver : public CMultizoneDriver { */ void StartSolver() override; - /*! - * \brief [Overload] Launch the tape test mode for the discrete adjoint multizone solver. - */ - void TapeTest (); - - /*! - * \brief [Overload] Get error numbers after a tape test run of the discrete adjoint multizone solver. - */ - int TapeTestGatherErrors(AD::ErrorReport& error_report) const; - /*! * \brief Preprocess the multizone iteration */ @@ -161,14 +127,14 @@ class CDiscAdjMultizoneDriver : public CMultizoneDriver { */ void Run() override; -protected: - /*! * \brief Run one inner iteration for a given zone. * \return The result of "monitor". */ bool Iterate(unsigned short iZone, unsigned long iInnerIter, bool KrylovMode = false); +protected: + /*! * \brief Run inner iterations using a Krylov method (GMRES atm). */ @@ -303,4 +269,15 @@ class CDiscAdjMultizoneDriver : public CMultizoneDriver { } } + /*! + * \brief Launch the tape test run of the discrete adjoint multizone solver. + */ + void TapeTest (); + + /*! + * \brief Get the total error count after a tape test run of the discrete adjoint multizone solver. + * \param[in] debug_control - DebugControl from which this rank's contribution to the total error count is read. + * \return The total error count across all ranks. + */ + int TapeTestGatherErrors(AD::DebugControl& debug_control) const; }; diff --git a/SU2_CFD/include/drivers/CDiscAdjSinglezoneDriver.hpp b/SU2_CFD/include/drivers/CDiscAdjSinglezoneDriver.hpp index 137c961f1482..6a2f00932ef1 100644 --- a/SU2_CFD/include/drivers/CDiscAdjSinglezoneDriver.hpp +++ b/SU2_CFD/include/drivers/CDiscAdjSinglezoneDriver.hpp @@ -55,6 +55,14 @@ class CDiscAdjSinglezoneDriver : public CSinglezoneDriver { COutput *direct_output; CNumerics ***numerics; /*!< \brief Container vector with all the numerics. */ + /*! + * \brief Returns true if the objective function does not depend on the main variables. In which case, + * the adjoint variables are 0 and the sensitivities can be computed just with the secondary recording. + */ + bool TrivialFunction() const { + return config_container[ZONE_0]->GetnObj() == 1 && config_container[ZONE_0]->GetKind_ObjFunc() == VOLUME_FRACTION; + } + /*! * \brief Record one iteration of a flow iteration in within multiple zones. * \param[in] kind_recording - Type of recording (full list in ENUM_RECORDING, option_structure.hpp) diff --git a/SU2_CFD/include/drivers/CDriver.hpp b/SU2_CFD/include/drivers/CDriver.hpp index 0dc9dd2f604d..916661ecfb06 100644 --- a/SU2_CFD/include/drivers/CDriver.hpp +++ b/SU2_CFD/include/drivers/CDriver.hpp @@ -77,6 +77,7 @@ class CDriver : public CDriverBase { interpolator_container; /*!< \brief Definition of the interpolation method between non-matching discretizations of the interface. */ CInterface*** interface_container; /*!< \brief Definition of the interface of information and physics. */ + unsigned short** interface_types; /*!< \brief Type of coupling between the distinct (physical) zones. */ bool dry_run; /*!< \brief Flag if SU2_CFD was started as dry-run via "SU2_CFD -d .cfg" */ public: @@ -203,8 +204,8 @@ class CDriver : public CDriverBase { * \param[in] interpolation - Object defining the interpolation. */ void InitializeInterface(CConfig** config, CSolver***** solver, CGeometry**** geometry, - unsigned short** interface_types, CInterface*** interface, - vector>>& interpolation); + unsigned short** interface_types, CInterface*** interface, + vector>>& interpolation); /*! * \brief Definition and allocation of all solver classes. @@ -290,17 +291,14 @@ class CDriver : public CDriverBase { * \param[in] geometry - Geometrical definition of the problem. * \param[in] solver - Container vector with all the solutions. * \param[in] interface - Class defining the physical transfer of information. + * \param[in] iteration - Class defining the iteration strcuture. * \param[in] dummy - Definition of dummy driver */ void PreprocessTurbomachinery(CConfig** config, CGeometry**** geometry, CSolver***** solver, - CInterface*** interface, bool dummy); + CInterface*** interface, CIteration*** iteration, bool dummy); - /*! - * \brief Ramp some simulation settings for turbomachinery problems. - * \param[in] iter - Iteration for the ramp (can be outer or time depending on type of simulation). - * \note TODO This is not compatible with inner iterations because they are delegated to the iteration class. - */ - void RampTurbomachineryValues(unsigned long iter); + void PreprocessTurboVertex(CConfig** config, CGeometry**** geometry, CSolver***** solver, + CInterface*** interface, CIteration*** iteration, bool dummy); /*! * \brief A virtual member. @@ -352,6 +350,7 @@ class CDriver : public CDriverBase { */ void PrintDirectResidual(RECORDING kind_recording); + public: /*! * \brief Set the solution of all solvers (adjoint or primal) in a zone. * \param[in] iZone - Index of the zone. @@ -364,7 +363,7 @@ class CDriver : public CDriverBase { const auto nPoint = geometry_container[iZone][INST_0][MESH_0]->GetnPoint(); for (auto iSol = 0u, offset = 0u; iSol < MAX_SOLS; ++iSol) { auto solver = solver_container[iZone][INST_0][MESH_0][iSol]; - if (!(solver && (solver->GetAdjoint() == adjoint))) continue; + if (!solver || solver->GetAdjoint() != adjoint) continue; for (auto iPoint = 0ul; iPoint < nPoint; ++iPoint) for (auto iVar = 0ul; iVar < solver->GetnVar(); ++iVar) if (!Old) { @@ -395,7 +394,7 @@ class CDriver : public CDriverBase { const auto nPoint = geometry_container[iZone][INST_0][MESH_0]->GetnPoint(); for (auto iSol = 0u, offset = 0u; iSol < MAX_SOLS; ++iSol) { auto solver = solver_container[iZone][INST_0][MESH_0][iSol]; - if (!(solver && (solver->GetAdjoint() == adjoint))) continue; + if (!solver || solver->GetAdjoint() != adjoint) continue; const auto& sol = solver->GetNodes()->GetSolution(); for (auto iPoint = 0ul; iPoint < nPoint; ++iPoint) for (auto iVar = 0ul; iVar < solver->GetnVar(); ++iVar) @@ -419,7 +418,6 @@ class CDriver : public CDriverBase { return nVar; } - public: /*! * \brief Launch the computation for all zones and all physics. */ diff --git a/SU2_CFD/include/drivers/CDriverBase.hpp b/SU2_CFD/include/drivers/CDriverBase.hpp index ba3af37349fd..b9d9c74cb1f7 100644 --- a/SU2_CFD/include/drivers/CDriverBase.hpp +++ b/SU2_CFD/include/drivers/CDriverBase.hpp @@ -60,8 +60,7 @@ class CDriverBase { nZone, /*!< \brief Total number of zones in the problem. */ nDim, /*!< \brief Number of dimensions. */ iInst, /*!< \brief Iterator on instance levels. */ - *nInst, /*!< \brief Total number of instances in the problem (per zone). */ - **interface_types; /*!< \brief Type of coupling between the distinct (physical) zones. */ + *nInst; /*!< \brief Total number of instances in the problem (per zone). */ CConfig* driver_config = nullptr; /*!< \brief Definition of the driver configuration. */ COutput* driver_output = nullptr; /*!< \brief Definition of the driver output. */ diff --git a/SU2_CFD/include/drivers/CMultizoneDriver.hpp b/SU2_CFD/include/drivers/CMultizoneDriver.hpp index 8d7fcd7e8838..d0b1ab976e36 100644 --- a/SU2_CFD/include/drivers/CMultizoneDriver.hpp +++ b/SU2_CFD/include/drivers/CMultizoneDriver.hpp @@ -83,15 +83,6 @@ class CMultizoneDriver : public CDriver { */ bool TransferData(unsigned short donorZone, unsigned short targetZone); - - /*! - * \brief Transfer the local turboperfomance quantities (for each blade row) from all the donorZones to the - * targetZone (ZONE_0). - * \note IMPORTANT: This approach of multi-zone performances rely upon the fact that turbomachinery markers follow - * the natural (stator-rotor) development of the real machine. - */ - void SetTurboPerformance(); - /*! * \brief Check the convergence at the outer level. */ diff --git a/SU2_CFD/include/integration/CIntegration.hpp b/SU2_CFD/include/integration/CIntegration.hpp index b970a8099b01..51dacbc93b2a 100644 --- a/SU2_CFD/include/integration/CIntegration.hpp +++ b/SU2_CFD/include/integration/CIntegration.hpp @@ -113,6 +113,24 @@ class CIntegration { CNumerics ******numerics_container, CConfig **config, unsigned short RunTime_EqSystem, unsigned short iZone, unsigned short iInst) { }; + /*! + * \brief Report the convergence fields on the active mesh level to a Full-MG startup. + * \param[in] convFields - Name and log10 value of each monitored residual field. + * \param[in] config - Definition of the particular problem. + */ + virtual void MonitorFullMG_Startup(const vector >& convFields, + const CConfig *config) { }; + + /*! + * \brief Whether the Full-MG startup is still ramping the CFL of the finest grid. + */ + virtual bool GetFullMG_CFLRamp() const { return false; } + + /*! + * \brief InnerIter at which the currently active Full-MG level became active. + */ + virtual unsigned long GetLevelStartIter() const { return 0; } + /*! * \brief A virtual member. * \param[in] geometry - Geometrical definition of the problem. diff --git a/SU2_CFD/include/integration/CMultiGridIntegration.hpp b/SU2_CFD/include/integration/CMultiGridIntegration.hpp index 5f17525cabf5..eee8c7235fd7 100644 --- a/SU2_CFD/include/integration/CMultiGridIntegration.hpp +++ b/SU2_CFD/include/integration/CMultiGridIntegration.hpp @@ -53,6 +53,18 @@ class CMultiGridIntegration final : public CIntegration { CNumerics ******numerics_container, CConfig **config, unsigned short RunTime_EqSystem, unsigned short iZone, unsigned short iInst) override; + /*! + * \brief Record CONV_FIELD on the active Full-MG level and decide whether it is done. + * \param[in] convFields - Name and log10 value of each monitored residual field. + * \param[in] config - Definition of the particular problem. + */ + void MonitorFullMG_Startup(const vector >& convFields, + const CConfig *config) override; + + bool GetFullMG_CFLRamp() const override { return mg_ramp_mesh0_active; } + + unsigned long GetLevelStartIter() const override { return mg_ramp_level_start_iter; } + private: /*! * \brief Perform a Full-Approximation Storage (FAS) Multigrid. @@ -217,90 +229,141 @@ class CMultiGridIntegration final : public CIntegration { unsigned short RunTime_EqSystem, unsigned long Iteration, unsigned short iZone); /*! - * \brief Compute adaptive CFL for multigrid coarse levels. - * \param[in] config - Problem configuration. - * \param[in] solver_coarse - Coarse grid solver. - * \param[in] geometry_coarse - Coarse grid geometry. - * \param[in] iMesh - Current multigrid level. - * \param[in] CFL_fine - Fine grid CFL value (passive). - * \param[in] CFL_coarse_current - Current coarse grid CFL value (passive). - * \param[in] rms_res_coarse - Coarse-grid RMS residual (already MPI-reduced, from lastPreSmoothRMS). - * \return New CFL value for the coarse grid. + * \brief Adapt both restriction and prolongation damping factors from the global-trend signal. + * \param[in,out] config - Problem configuration. + * \param[in] crossCycleRatio - Current fine_d0 divided by the EMA of fine_d0. */ - passivedouble computeMultigridCFL(CConfig* config, unsigned short iMesh, - passivedouble CFL_fine, passivedouble CFL_coarse_current, - passivedouble rms_res_coarse); + void adaptDampingFactors(CConfig* config, passivedouble crossCycleRatio); /*! - * \brief Adapt the residual restriction damping factor. - * - * Uses \c lastPreSmoothIters[] (filled by the previous multigrid cycle) to assess - * whether the pre-smoother is converging fast or slow on coarse levels, then adjusts - * \c Damp_Res_Restric in \p config accordingly. - * - * Signal logic: - * - any coarse level ran its full configured iterations: reduce damping - * - all coarse levels exited early: increase damping - * - mixed (some full, some partial): no change + * \brief Set the CFL of every coarse multigrid level, ramping level i from the final CFL of + * level i+1 to its own target, which CFL_NUMBER and MG_CFL_SCALING determine. * - * \param[in,out] config - Problem configuration. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in,out] solver_container - Container vector with all the solutions. + * \param[in,out] config - Definition of the particular problem. + * \param[in] RunTime_EqSystem - System of equations which is going to be solved. + * \param[in] iZone - Zone index. + * \param[in] iInst - Instance index. + * \param[in] FinestMesh - Currently active finest mesh level. + * \param[in] FullMG - Whether the Full-MG cycle is active. + * \param[in] mesh0_ramp_window - Result of FullMG_Mesh0RampWindow, computed by the caller. */ - void adaptRestrictionDamping(CConfig* config); + void SetCoarseGridCFL(CGeometry ****geometry, CSolver *****solver_container, CConfig **config, + unsigned short RunTime_EqSystem, unsigned short iZone, unsigned short iInst, + unsigned short FinestMesh, bool FullMG, bool mesh0_ramp_window); /*! - * \brief Adapt the correction prolongation damping factor. - * - * Uses \c lastCorrecSmoothIters[] (filled by the previous multigrid cycle) to assess - * whether the correction smoother is struggling or converging fast, - * then adjusts \c Damp_Correc_Prolong in \p config accordingly. - * - * Signal logic: - * - any level ran its full correction-smooth iterations: reduce damping - * - all levels exited early: increase damping - * - mixed: no change - * - * \param[in,out] config - Problem configuration; \c SetDamp_Correc_Prolong is called to persist the result. + * \brief Whether the finest grid has a Full-MG CFL ramp at all. + * \param[in] config - Definition of the particular problem. + * \param[in] FinestMesh - Currently active finest mesh level. + * \param[in] FullMG - Whether the Full-MG cycle is active. + */ + bool FullMG_Mesh0HasRamp(const CConfig* config, unsigned short FinestMesh, bool FullMG) const { + return FullMG && (FinestMesh == MESH_0) && (config->GetMGOptions().MG_Startup_Iter > 0) && + (mg_ramp_cfl_start > 0.0); + } + + /*! + * \brief Whether the level-0 CFL is still climbing towards the configured number. + */ + bool FullMG_Mesh0Ramping(const CConfig* config, unsigned short FinestMesh, bool FullMG) const { + return FullMG_Mesh0HasRamp(config, FinestMesh, FullMG) && + ((config->GetInnerIter() - mg_ramp_level_start_iter) < config->GetMGOptions().MG_Startup_Iter); + } + + /*! + * \brief The single iteration after the ramp, which writes the configured CFL back. + */ + bool FullMG_Mesh0RampRestore(const CConfig* config, unsigned short FinestMesh, bool FullMG) const { + return FullMG_Mesh0HasRamp(config, FinestMesh, FullMG) && + ((config->GetInnerIter() - mg_ramp_level_start_iter) == config->GetMGOptions().MG_Startup_Iter); + } + + /*! + * \brief Whether the startup owns the level-0 CFL this iteration. + */ + bool FullMG_Mesh0RampWindow(const CConfig* config, unsigned short FinestMesh, bool FullMG) const { + return FullMG_Mesh0Ramping(config, FinestMesh, FullMG) || + FullMG_Mesh0RampRestore(config, FinestMesh, FullMG); + } + + /*! + * \brief Interpolate a scalar solver's solution onto the newly activated Full-MG level. + * \param[in,out] sol_fine - Solver on the level being activated. + * \param[in] sol_coarse - Solver on the level handing over. + * \param[in] geo_fine - Geometry of the level being activated. + * \param[in] geo_coarse - Geometry of the level handing over. + * \param[in] config - Definition of the particular problem. + * \param[in] eddy_viscosity - Carry the eddy viscosity across as well. + */ + void SetProlongated_ScalarSolution(CSolver *sol_fine, CSolver *sol_coarse, CGeometry *geo_fine, + CGeometry *geo_coarse, CConfig *config, bool eddy_viscosity); + + /*! + * \brief Helper function for early-exit logic during pre/post-smoothing. + * \param[in] iSmooth - Current smoothing iteration index. + * \param[in] iMesh - Index of the mesh in multigrid computations. + * \param[in] defect - Current RMS defect value. + * \param[in] mgOpts - Reference to multigrid options. + * \param[in] stag_tol - Stagnation tolerance value. + * \param[in] early_exit - Whether early exit is enabled. + * \param[out] lastRMS - Array to store RMS values [start, end]. + * \param[out] exitReason - Character for early exit reason ('T', 'S', 'A', or ' '). + * \param[out] worstStepRatio - Worst step-to-step ratio seen. + * \param[out] worstStep - Iteration number of worst step. */ - void adaptProlongationDamping(CConfig* config); + void prePostEarlyExit(unsigned short iSmooth, unsigned short iMesh, + passivedouble defect, const CMGOptions& mgOpts, + passivedouble stag_tol, bool early_exit, + passivedouble lastRMS[2], char& exitReason, + passivedouble& worstStepRatio, unsigned short& worstStep); - /*--- CFL adaptation state variables. - * These must be passivedouble: AD::Reset() clears the tape between adjoint recordings, - * but class members survive. If these were su2double their stale AD indices would - * reference the cleared tape, causing invalid memory access during the backward pass. ---*/ static constexpr int MAX_MG_LEVELS = 10; - passivedouble current_avg[MAX_MG_LEVELS] = {}; - passivedouble prev_avg[MAX_MG_LEVELS] = {}; - passivedouble last_res[MAX_MG_LEVELS] = {}; - bool last_was_increase[MAX_MG_LEVELS] = {}; - int oscillation_count[MAX_MG_LEVELS] = {}; - unsigned long last_check_iter[MAX_MG_LEVELS] = {}; - unsigned long last_update_iter[MAX_MG_LEVELS] = {}; - unsigned long last_reset_iter = std::numeric_limits::max(); /*--- Early-exit smoothing state (shared across OMP threads via master write + barrier). ---*/ - bool mg_early_exit_flag = false; /*!< \brief Shared flag for early exit across OMP threads. */ - passivedouble mg_initial_smooth_rms = 0.0; /*!< \brief Initial RMS before current smoothing phase. */ - passivedouble mg_last_smooth_rms = 0.0; /*!< \brief Last computed RMS; cached to avoid redundant Allreduce. */ + bool mg_early_exit_flag = false; /*!< \brief Shared flag for early exit across OMP threads. */ + passivedouble mg_initial_smooth_rms = 0.0; /*!< \brief Initial RMS residual before current smoothing phase (FAS). */ + passivedouble mg_prev_smooth_rms = 0.0; /*!< \brief RMS residual from previous smoothing step; used for stagnation detection. */ + passivedouble mg_fine_rms_ema = 0.0; /*!< \brief EMA of fine-grid pre-smooth RMS across cycles; cross-cycle trend signal. */ + passivedouble last_crossCycleRatio = 1.0; /*!< \brief crossCycleRatio from the most recent cycle; stored for display only. */ /*--- Actual iteration counts per MG level, filled each cycle for the compact output summary. ---*/ unsigned short lastPreSmoothIters[MAX_MG_LEVELS+1] = {}; unsigned short lastPostSmoothIters[MAX_MG_LEVELS+1] = {}; unsigned short lastCorrecSmoothIters[MAX_MG_LEVELS+1] = {}; + /*--- Early-exit reason per level: 'T'=threshold, 'S'=stagnation, ' '=ran to completion. ---*/ + char lastPreSmoothExitReason[MAX_MG_LEVELS+1] = {}; + char lastPostSmoothExitReason[MAX_MG_LEVELS+1] = {}; - /*--- Per-level residual progress flags: true if the final RMS after that phase was lower - * than the initial RMS. Used by the adaptive damping routines to distinguish - * "hit max iters but still converging" from "hit max iters and stagnated". ---*/ - bool lastPreSmoothProgress[MAX_MG_LEVELS+1] = {}; - bool lastPostSmoothProgress[MAX_MG_LEVELS+1] = {}; - bool lastCorrecSmoothProgress[MAX_MG_LEVELS+1] = {}; - - /*--- Per-level start/end RMS for the compact output summary. - * [0] = initial RMS before smoothing, [1] = final RMS after smoothing. - * Filled unconditionally (early-exit path and exhaustion path). - * Must be passivedouble: class members survive tape resets; su2double would - * carry stale AD indices referencing a cleared tape. ---*/ + /*--- Per-level start/end RMS residual for adaptive damping. ---*/ passivedouble lastPreSmoothRMS[MAX_MG_LEVELS+1][2] = {}; passivedouble lastPostSmoothRMS[MAX_MG_LEVELS+1][2] = {}; passivedouble lastCorrecSmoothRMS[MAX_MG_LEVELS+1][2] = {}; + /*--- Per-level worst step-to-step amplification seen inside a smoothing phase. + * step==0 means no intra-smoother ratio was available (fewer than 2 sweeps). ---*/ + passivedouble lastPreSmoothWorstStepRatio[MAX_MG_LEVELS+1] = {}; + passivedouble lastPostSmoothWorstStepRatio[MAX_MG_LEVELS+1] = {}; + unsigned short lastPreSmoothWorstStep[MAX_MG_LEVELS+1] = {}; + unsigned short lastPostSmoothWorstStep[MAX_MG_LEVELS+1] = {}; + + /*! \brief FinestMesh observed on the previous call; 0 also serves as the sentinel forcing a + * reset on the first call, since a FULL cycle starts at FinestMesh == nMGLevels > 0. */ + unsigned short mg_ramp_last_FinestMesh = 0; + unsigned long mg_ramp_level_start_iter = 0; /*!< \brief InnerIter the active FMG level became active at. */ + passivedouble mg_ramp_cfl_start = 0.0; /*!< \brief CFL the level below handed over; 0 before a promotion. */ + bool mg_ramp_mesh0_active = false; /*!< \brief Whether the level-0 CFL ramp is still climbing. */ + + su2double mg_damp_restric_initial = -1.0; /*!< \brief MG_DAMP_RESTRICTION as configured; negative if not captured. */ + su2double mg_damp_prolong_initial = 0.0; /*!< \brief MG_DAMP_PROLONGATION as configured. */ + + /*! \brief Why the active level was last promoted, for the report message. */ + enum class MGStartupPromote { NONE, BUDGET, CONVERGENCE, STAGNATION }; + MGStartupPromote mg_startup_promote_reason = MGStartupPromote::NONE; + + vector mg_startup_conv_start; /*!< \brief Field values when the active level became active. */ + vector mg_startup_conv_prev; /*!< \brief Field values on the previous iteration. */ + unsigned long mg_startup_stall_count = 0; /*!< \brief Consecutive iterations without useful reduction. */ + }; diff --git a/SU2_CFD/include/integration/CNewtonIntegration.hpp b/SU2_CFD/include/integration/CNewtonIntegration.hpp index da8ff67930ad..25bb3139777e 100644 --- a/SU2_CFD/include/integration/CNewtonIntegration.hpp +++ b/SU2_CFD/include/integration/CNewtonIntegration.hpp @@ -152,6 +152,18 @@ class CNewtonIntegration final : public CIntegration { template::value> = 0> inline unsigned long Preconditioner_impl(const CSysVector& u, CSysVector& v, unsigned long iters, Scalar& eps) const { + /*--- Unlike the matrix-free outer product this is a CSysMatrix operation, it can run on the + * device. The outer Krylov vectors are host resident, hence the transfers. ---*/ + unsigned long nIters = 0; + ApplyPreconditionerOnDevice(u, v, config->GetCUDA(), [&] { nIters = PreconditionerApply(u, v, iters, eps); }); + return nIters; + } + + /*! + * \brief The preconditioner on its own, or a nested solve with the approximate Jacobian. + */ + inline unsigned long PreconditionerApply(const CSysVector& u, CSysVector& v, + unsigned long iters, Scalar& eps) const { const auto inner_solver = config->GetKind_Linear_Solver_Inner(); if (iters == 0 || (iters == 1 && inner_solver == LINEAR_SOLVER_INNER::SMOOTHER)) { diff --git a/SU2_CFD/include/integration/ComputeLinSysResRMS.hpp b/SU2_CFD/include/integration/ComputeLinSysResRMS.hpp deleted file mode 100644 index c773f3fe683d..000000000000 --- a/SU2_CFD/include/integration/ComputeLinSysResRMS.hpp +++ /dev/null @@ -1,63 +0,0 @@ -/*! - * \file ComputeLinSysResRMS.hpp - * \brief Helper to compute the global RMS of LinSysRes across all variables and domain points. - * \author Nijso Beishuizen - * \version 8.5.0 "Harrier" - * - * SU2 Project Website: https://su2code.github.io - * - * The SU2 Project is maintained by the SU2 Foundation - * (http://su2foundation.org) - * - * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) - * - * SU2 is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * SU2 is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with SU2. If not, see . - */ - -#pragma once -#include "../../include/solvers/CSolver.hpp" -#include "../../../Common/include/parallelization/omp_structure.hpp" -#include - -/*! - * \brief Compute the global (MPI-reduced) RMS of LinSysRes over all variables and domain points. - * - * \note Thread-safety: This function MUST be called by ALL threads in the current - * OpenMP parallel region, because squaredNorm() uses parallel for + barriers - * internally. Do NOT call from inside BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS. - * The return value is correct only on the master thread (thread 0). - * - * \param[in] solver - Solver whose LinSysRes is evaluated. - * \return Global RMS value (valid on master thread; other threads return 0). - */ -inline passivedouble ComputeLinSysResRMS(const CSolver* solver) { - - /*--- squaredNorm() -> dot() uses OMP parallel for + barriers internally, - * so all threads must participate. ---*/ - const su2double sqNorm = solver->LinSysRes.squaredNorm(); - - /*--- The MPI reduction for nElmDomain must be single-threaded. ---*/ - passivedouble result = 0.0; - BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS - { - unsigned long nElmDomain = solver->LinSysRes.GetNElmDomain(); - unsigned long globalNElmDomain = 0; - SU2_MPI::Allreduce(&nElmDomain, &globalNElmDomain, 1, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); - if (globalNElmDomain > 0) - result = std::sqrt(SU2_TYPE::GetValue(sqNorm) / static_cast(globalNElmDomain)); - } - END_SU2_OMP_SAFE_GLOBAL_ACCESS - - return result; -} diff --git a/SU2_CFD/include/interfaces/CInterface.hpp b/SU2_CFD/include/interfaces/CInterface.hpp index 7480f970ce4a..ed85008e184e 100644 --- a/SU2_CFD/include/interfaces/CInterface.hpp +++ b/SU2_CFD/include/interfaces/CInterface.hpp @@ -29,6 +29,8 @@ #pragma once #include "../../../Common/include/parallelization/mpi_structure.hpp" +#include "../../../Common/include/containers/C2DContainer.hpp" +#include "../../../Common/include/option_structure.hpp" #include #include @@ -36,6 +38,7 @@ #include #include #include +#include #include #include @@ -65,6 +68,8 @@ class CInterface { su2double *Target_Variable = nullptr; bool valAggregated = false; + unsigned short InterfaceType; /*!< \brief The type of interface. */ + /*--- Mixing Plane interface variable ---*/ su2double *SpanValueCoeffTarget = nullptr; unsigned short *SpanLevelDonor = nullptr; @@ -77,7 +82,7 @@ class CInterface { /*! * \brief Constructor of the class. */ - CInterface(void); + CInterface(); /*! * \overload @@ -89,7 +94,7 @@ class CInterface { /*! * \brief Destructor of the class. */ - virtual ~CInterface(void); + virtual ~CInterface(); /*! * \brief Interpolate data and broadcast it into all processors, for nonmatching meshes. @@ -125,6 +130,16 @@ class CInterface { const CConfig *donor_config, unsigned long Marker_Donor, unsigned long Vertex_Donor, unsigned long Point_Donor) = 0; + /*! + * \brief A virtual member. + * \param[in] donor_config - Definition of the problem at the donor mesh. + * \param[in] donor_geometry - Geometry of the donor mesh. + * \param[in] target_config - Definition of the problem at the target mesh. + * \param[in] target_geometry - Geometry of the target mesh. + */ + inline virtual void GetDonor_Velocity_RotatingFrame(const CConfig *donor_config, CGeometry *donor_geometry, + const CConfig *target_config, CGeometry *target_geometry) {} + /*! * \brief Initializes the target variable. * \param[in] target_solution - Solution from the target mesh. @@ -146,6 +161,21 @@ class CInterface { for (auto iVar = 0u; iVar < nVar; iVar++) Target_Variable[iVar] += donorCoeff * bcastVariable[iVar]; } + /*! + * \brief Recovers the target variable at the endwall from the buffer of su2doubles that was broadcast for mixing plane interfaces. + * \param[in] bcastVariable - Broadcast variable. + * \param[in] idx - Index of the target point. + */ + inline virtual void RecoverTarget_SpanEndwall(const su2activevector &bcastVariable, unsigned long idx) { } + + /*! + * \brief Recovers the target variable at the span from the buffer of su2doubles that was broadcast for mixing plane interfaces. + * \param[in] bcastVariable - Broadcast variable. + * \param[in] idx - Index of the target point. + * \param[in] donorCoeff - value of the donor coefficient. + */ + inline virtual void RecoverTarget_Span(const su2activevector &bcastVariable, unsigned long idx, su2double donorCoeff) { } + /*! * \brief A virtual member. * \param[in] target_solution - Solution from the target mesh. @@ -177,51 +207,42 @@ class CInterface { inline virtual void SetSpanWiseLevels(const CConfig *donor_config, const CConfig *target_config) { } /*! - * \brief A virtual member. - * \param[in] target_solution - Solution from the target mesh. + * \brief Interpolate data and broadcast it into all processors, for nonmatching meshes. + * \param[in] interpolator - Object defining the interpolation. + * \param[in] donor_solution - Solution from the donor mesh. * \param[in] target_solution - Solution from the target mesh. - * \param[in] donor_zone - Index of the donorZone. - */ - inline virtual void SetAverageValues(CSolver *donor_solution, CSolver *target_solution, - unsigned short donorZone) { } - - /*! - * \brief Transfer pre-processing for the mixing plane inteface. * \param[in] donor_geometry - Geometry of the donor mesh. * \param[in] target_geometry - Geometry of the target mesh. * \param[in] donor_config - Definition of the problem at the donor mesh. * \param[in] target_config - Definition of the problem at the target mesh. */ - void PreprocessAverage(CGeometry *donor_geometry, CGeometry *target_geometry, - const CConfig *donor_config, const CConfig *target_config, unsigned short iMarkerInt); + inline virtual void BroadcastData_MixingPlane(const CInterpolator& interpolator, + CSolver *donor_solution, CSolver *target_solution, + CGeometry *donor_geometry, CGeometry *target_geometry, + const CConfig *donor_config, const CConfig *target_config) { }; + /*! - * \brief Interpolate data and scatter it into different processors, for matching meshes. - * \param[in] donor_solution - Solution from the donor mesh. - * \param[in] target_solution - Solution from the target mesh. - * \param[in] donor_geometry - Geometry of the donor mesh. - * \param[in] target_geometry - Geometry of the target mesh. - * \param[in] donor_config - Definition of the problem at the donor mesh. - * \param[in] target_config - Definition of the problem at the target mesh. + * \brief Set the contact resistance value for the solid-to-solid heat transfer interface. + * \param[in] val_contact_resistance - Contact resistance value in m^2/W */ - void AllgatherAverage(CSolver *donor_solution, CSolver *target_solution, - CGeometry *donor_geometry, CGeometry *target_geometry, - const CConfig *donor_config, const CConfig *target_config, unsigned short iMarkerInt); + inline virtual void SetContactResistance(su2double val_contact_resistance) {}; /*! - * \brief Interpolate data and scatter it into different processors, for matching meshes. - * \param[in] donor_solution - Solution from the donor mesh. - * \param[in] target_solution - Solution from the target mesh. - * \param[in] donor_geometry - Geometry of the donor mesh. - * \param[in] target_geometry - Geometry of the target mesh. - * \param[in] donor_config - Definition of the problem at the donor mesh. - * \param[in] target_config - Definition of the problem at the target mesh. + * \brief Set the type of an interface + * \param[in] interface_type - The type of interface */ - void GatherAverageValues(CSolver *donor_solution, CSolver *target_solution, unsigned short donorZone); + void SetInterfaceType(unsigned short interface_type) { InterfaceType = interface_type; } /*! - * \brief Set the contact resistance value for the solid-to-solid heat transfer interface. - * \param[in] val_contact_resistance - Contact resistance value in m^2/W + * \brief Get the type of an interface */ - inline virtual void SetContactResistance(su2double val_contact_resistance) {}; + unsigned short GetInterfaceType(void) const { return InterfaceType; } + + /*! + * \brief These can be used to chain interfaces between the same zones but for other variables, + * without having to mix physics in the interface classes. Currently this is used for FSI+CHT. + */ + ENUM_TRANSFER NextInterfaceType = ENUM_TRANSFER::NO_TRANSFER; + CInterface* NextInterface = nullptr; }; diff --git a/SU2_CFD/include/interfaces/cfd/CMixingPlaneInterface.hpp b/SU2_CFD/include/interfaces/cfd/CMixingPlaneInterface.hpp index e57d7c61764c..fad7aa7a4c27 100644 --- a/SU2_CFD/include/interfaces/cfd/CMixingPlaneInterface.hpp +++ b/SU2_CFD/include/interfaces/cfd/CMixingPlaneInterface.hpp @@ -29,6 +29,7 @@ #pragma once #include "../CInterface.hpp" +#include "../../../Common/include/containers/C2DContainer.hpp" /*! * \brief Mixing plane interface for turbomachinery. @@ -36,6 +37,7 @@ */ class CMixingPlaneInterface : public CInterface { public: + unsigned short nMixingVars; /*! * \overload * \param[in] val_nVar - Number of variables that need to be transferred. @@ -49,6 +51,21 @@ class CMixingPlaneInterface : public CInterface { */ void SetSpanWiseLevels(const CConfig *donor_config, const CConfig *target_config) override; + /*! + * \brief Interpolate data and broadcast it into all processors, for nonmatching meshes. + * \param[in] interpolator - Object defining the interpolation. + * \param[in] donor_solution - Solution from the donor mesh. + * \param[in] target_solution - Solution from the target mesh. + * \param[in] donor_geometry - Geometry of the donor mesh. + * \param[in] target_geometry - Geometry of the target mesh. + * \param[in] donor_config - Definition of the problem at the donor mesh. + * \param[in] target_config - Definition of the problem at the target mesh. + */ + void BroadcastData_MixingPlane(const CInterpolator& interpolator, + CSolver *donor_solution, CSolver *target_solution, + CGeometry *donor_geometry, CGeometry *target_geometry, + const CConfig *donor_config, const CConfig *target_config) override; + /*! * \brief Retrieve the variable that will be sent from donor mesh to target mesh. * \param[in] donor_solution - Solution from the donor mesh. @@ -73,14 +90,15 @@ class CMixingPlaneInterface : public CInterface { void SetTarget_Variable(CSolver *target_solution, CGeometry *target_geometry, const CConfig *target_config, unsigned long Marker_Target, unsigned long val_Span, unsigned long Point_Target) override; - /*! - * \brief Store all the turboperformance in the solver in ZONE_0. - * \param[in] donor_solution - Solution from the donor mesh. - * \param[in] target_solution - Solution from the target mesh. - * \param[in] donorZone - counter of the donor solution - */ - void SetAverageValues(CSolver *donor_solution, CSolver *target_solution, unsigned short donorZone) override; - - + inline void RecoverTarget_SpanEndwall(const su2activevector &bcastVariable, unsigned long iSpan) override { + for (auto iVar = 0u; iVar < nMixingVars; iVar++) { + Target_Variable[iVar] = bcastVariable[iSpan * nMixingVars + iVar]; + } + } + inline void RecoverTarget_Span(const su2activevector &bcastVariable, unsigned long iSpan, su2double donorCoeff) override { + for (auto iVar = 0u; iVar < nMixingVars; iVar++) { + Target_Variable[iVar] = (1 - donorCoeff)*bcastVariable[iSpan * nMixingVars + iVar] + donorCoeff * bcastVariable[(iSpan + 1) * nMixingVars + iVar]; + } + } }; diff --git a/SU2_CFD/include/interfaces/cfd/CSlidingInterface.hpp b/SU2_CFD/include/interfaces/cfd/CSlidingInterface.hpp index 6a93ac1bdd70..1cae1287915d 100644 --- a/SU2_CFD/include/interfaces/cfd/CSlidingInterface.hpp +++ b/SU2_CFD/include/interfaces/cfd/CSlidingInterface.hpp @@ -54,6 +54,16 @@ class CSlidingInterface : public CInterface { void GetDonor_Variable(CSolver *donor_solution, CGeometry *donor_geometry, const CConfig *donor_config, unsigned long Marker_Donor, unsigned long Vertex_Donor, unsigned long Point_Donor) override; + /*! + * \brief Rotate the velocity if rotating frame is applied. + * \param[in] donor_config - Definition of the problem at the donor mesh. + * \param[in] donor_geometry - Geometry of the donor mesh. + * \param[in] target_config - Definition of the problem at the target mesh. + * \param[in] target_geometry - Geometry of the target mesh. + */ + void GetDonor_Velocity_RotatingFrame(const CConfig *donor_config, CGeometry *donor_geometry, + const CConfig *target_config, CGeometry *target_geometry) override; + /*! * \brief A virtual member, initializes the target variable for sliding mesh. * \param[in] target_solution - Solution from the target mesh. diff --git a/SU2_CFD/include/iteration/CFluidIteration.hpp b/SU2_CFD/include/iteration/CFluidIteration.hpp index 2a53eba538cb..4adacde2045f 100644 --- a/SU2_CFD/include/iteration/CFluidIteration.hpp +++ b/SU2_CFD/include/iteration/CFluidIteration.hpp @@ -72,6 +72,22 @@ class CFluidIteration : public CIteration { CVolumetricMovement*** grid_movement, CFreeFormDefBox*** FFDBox, unsigned short val_iZone, unsigned short val_iInst) override; + /*! + * \brief Perform auxiliary solvers iterations after the main flow solver. + * \param[in] integration - Container vector with all the integration methods. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] solver - Container vector with all the solutions. + * \param[in] numerics - Description of the numerical method (the way in which the equations are solved). + * \param[in] config - Definition of the particular problem. + * \param[in] val_iZone - Index of the zone. + * \param[in] val_iInst - Index of the instance layer. + * \param[in] main_solver - Main solver. + * \param[in] frozen_visc - Flag for frozen viscosity. + */ + void CommonAuxiliarySolvers(CIntegration**** integration, CGeometry**** geometry, CSolver***** solver, + CNumerics****** numerics, CConfig** config, unsigned short val_iZone, + unsigned short val_iInst, MAIN_SOLVER main_solver, bool frozen_visc); + /*! * \brief Iterate the fluid system for a number of Inner_Iter iterations. * \param[in] output - Pointer to the COutput class. @@ -115,12 +131,7 @@ class CFluidIteration : public CIteration { * \param[in] iZone - The current zone * \param[in] ramp_flag - Flag indicating type of ramp (grid or boundary) */ - void UpdateRamp(CGeometry**** geometry_container, CConfig** config_container, unsigned long iter, unsigned short iZone, RAMP_TYPE ramp_flag); - - /*! - * \brief Computes turboperformance. - */ - void ComputeTurboPerformance(CSolver***** solver, CGeometry**** geometry_container, CConfig** config_container, unsigned long ExtIter); + void UpdateRamps(CGeometry**** geometry_container, CConfig** config_container, unsigned long iter, unsigned short iZone, RAMP_TYPE ramp_flag); /*! * \brief Postprocesses the fluid system before heading to another physics system or the next iteration. diff --git a/SU2_CFD/include/iteration/CIteration.hpp b/SU2_CFD/include/iteration/CIteration.hpp index b570d4b75ab4..06783c821f12 100644 --- a/SU2_CFD/include/iteration/CIteration.hpp +++ b/SU2_CFD/include/iteration/CIteration.hpp @@ -60,9 +60,7 @@ class CIteration { su2double StartTime{0.0}, /*!< \brief Tracking wall time. */ StopTime{0.0}, UsedTime{0.0}; - std::shared_ptr TurbomachineryPerformance; /*!< \brief turbo performance calculator. */ std::shared_ptr TurbomachineryStagePerformance; /*!< \brief turbo stage performance calculator. */ - public: /*! * \brief Constructor of the class. @@ -295,4 +293,22 @@ class CIteration { virtual void RegisterOutput(CSolver***** solver, CGeometry**** geometry, CConfig** config, unsigned short iZone, unsigned short iInst) {} + + /*! + * \brief Computes turboperformance. + */ + void ComputeTurboPerformance(CSolver***** solver, CGeometry**** geometry_container, CConfig** config_container); + + /*! + * \brief Initialises turboperformance classes. + */ + void InitTurboPerformance(CGeometry *geometry, CConfig** config, CFluidModel *fluid, unsigned short val_iZone); + + inline su2vector> GetBladesPerformanceVector(CSolver***** solver, unsigned short nBladeRow){ + su2vector> bladePerformances(nBladeRow); + for (auto iBladeRow = 0u; iBladeRow < nBladeRow; iBladeRow++) { + bladePerformances[iBladeRow] = solver[iBladeRow][INST_0][MESH_0][FLOW_SOL]->GetTurboBladePerformance(); + } + return bladePerformances; + } }; diff --git a/SU2_CFD/include/iteration/CPBFluidIteration.hpp b/SU2_CFD/include/iteration/CPBFluidIteration.hpp new file mode 100644 index 000000000000..9efedea916e5 --- /dev/null +++ b/SU2_CFD/include/iteration/CPBFluidIteration.hpp @@ -0,0 +1,66 @@ +/*! + * \file CPBFluidIteration.hpp + * \brief Headers of the pressure based fluid iteration class. + * \author T. Aalbers + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#pragma once + +#include "CFluidIteration.hpp" + +/*! + * \class CPBFluidIteration + * \ingroup Drivers + * \brief Class for driving a pressure-based iteration of the fluid system. + * \author T. Aalbers + */ +class CPBFluidIteration : public CFluidIteration { +public: + /*! + * \brief Constructor of the class. + * \param[in] config - Definition of the particular problem. + */ + explicit CPBFluidIteration(const CConfig* config) : CFluidIteration(config) {} + + /*! + * \brief Perform a single iteration of the fluid system. + * \param[in] output - Pointer to the COutput class. + * \param[in] integration - Container vector with all the integration methods. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] solver - Container vector with all the solutions. + * \param[in] numerics - Description of the numerical method (the way in which the equations are solved). + * \param[in] config - Definition of the particular problem. + * \param[in] surface_movement - Surface movement classes of the problem. + * \param[in] grid_movement - Volume grid movement classes of the problem. + * \param[in] FFDBox - FFD FFDBoxes of the problem. + * \param[in] val_iZone - Index of the zone. + * \param[in] val_iInst - Index of the instance layer. + */ + void Iterate(COutput* output, CIntegration**** integration, CGeometry**** geometry, CSolver***** solver, + CNumerics****** numerics, CConfig** config, CSurfaceMovement** surface_movement, + CVolumetricMovement*** grid_movement, CFreeFormDefBox*** FFDBox, unsigned short val_iZone, + unsigned short val_iInst) override; + + }; + diff --git a/SU2_CFD/include/numerics/CNumerics.hpp b/SU2_CFD/include/numerics/CNumerics.hpp index 535e40cba96d..ec511a3fa1db 100644 --- a/SU2_CFD/include/numerics/CNumerics.hpp +++ b/SU2_CFD/include/numerics/CNumerics.hpp @@ -519,12 +519,20 @@ class CNumerics { * See: Spalart, P. R., "Strategies for Turbulence Modelling and Simulation", * International Journal of Heat and Fluid Flow, Vol. 21, 2000, pp. 252-263 * + * The QCR correction applies to the turbulent (Boussinesq) stresses only. + * When tau is the total (laminar + turbulent) stress tensor, which is + * proportional to the total viscosity, the turbulent part is recovered by + * scaling the correction with turb_fraction = mu_t / (mu_l + mu_t); at a + * no-slip wall (mu_t = 0) the correction vanishes. Pass 1 only if tau is + * already the turbulent stress tensor. + * * \param[in] nDim: 2D or 3D. * \param[in] gradvel: Velocity gradients. * \param[in,out] tau: Shear stress tensor. + * \param[in] turb_fraction: Turbulent share of the viscosity in tau. */ - template - FORCEINLINE static void AddQCR(size_t nDim, const Mat1& gradvel, Mat2& tau) { + template + FORCEINLINE static void AddQCR(size_t nDim, const Mat1& gradvel, Mat2& tau, Scalar2 turb_fraction) { using Scalar = typename std::decay::type; const Scalar c_cr1 = 0.3; @@ -553,7 +561,7 @@ class CNumerics { for (size_t iDim = 0; iDim < nDim; iDim++) for (size_t jDim = 0; jDim < nDim; jDim++) - tau[iDim][jDim] -= c_cr1 * tauQCR[iDim][jDim]; + tau[iDim][jDim] -= turb_fraction * c_cr1 * tauQCR[iDim][jDim]; } /*! diff --git a/SU2_CFD/include/numerics/flow/convection/pressure_based.hpp b/SU2_CFD/include/numerics/flow/convection/pressure_based.hpp new file mode 100644 index 000000000000..cc618dbb4516 --- /dev/null +++ b/SU2_CFD/include/numerics/flow/convection/pressure_based.hpp @@ -0,0 +1,160 @@ +/*! + * \file pressure_based.hpp + * \brief Declaration of numerics classes for convective schemes for + * the pressure based solver, the implementation is in pressure_based.cpp. + * \author T. Aalbers + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#pragma once + +#include "../../CNumerics.hpp" + + +/*! + * \class CPBConvection_Base + * \brief Class for computing a linear centered scheme. + * \ingroup ConvDiscr + * \author T. Aalbers + */ +class CPBConvection_Base : public CNumerics { +protected: + + bool implicit, dynamic_grid, energy, variable_density; + + unsigned short iDim, jDim, iVar, jVar; + + su2double *AdvectedVelocity = nullptr, AdvectedEnthalpy; + su2double *Flux = nullptr; + su2double **Jacobian_i = nullptr; + su2double **Jacobian_j = nullptr; + + su2double MeanDensity; + su2double dRhodh_i, dRhodh_j, Temperature_i, Temperature_j; + + su2double weight_jacobian_i, weight_jacobian_j; + + /*! + * \brief Function which defines the advected quantities + */ + void virtual ComputeAdvectedQuantities(void) = 0; + + /*! + * \brief Function which defines jacobian weights + */ + void virtual ComputeJacobianWeights(void) = 0; + + /*! + * \brief Function which defines the Jacobian + */ + void ComputeJacobian(su2double val_density, const su2double *val_velocity, + su2double val_enthalpy, su2double val_dRhodh, + su2double val_scale, su2double **val_Proj_Jac_Tensor); + +public: + + /*! + * \brief Constructor of the class. + * \param[in] val_nDim - Number of dimension of the problem. + * \param[in] val_nVar - Number of variables of the problem. + * \param[in] config - Definition of the particular problem. + */ + CPBConvection_Base(unsigned short val_nDim, unsigned short val_nVar, CConfig *config); + + /*! + * \brief Destructor of the class. + */ + virtual ~CPBConvection_Base(void); + + /*! + * \brief Compute the flow residual. + * \param[out] val_resconv - Pointer to the convective residual. + * \param[out] val_Jacobian_i - Jacobian of the numerical method at node i (implicit computation). + * \param[out] val_Jacobian_j - Jacobian of the numerical method at node j (implicit computation). + * \param[in] config - Definition of the particular problem. + */ + ResidualType<> ComputeResidual(const CConfig* config) final; +}; + +/*! + * \class CPBConvection_Central + * \brief Class for computing a centered scheme. + * \ingroup ConvDiscr + * \author T. Aalbers + */ +class CPBConvection_Central : public CPBConvection_Base { + +public: + + /*! + * \brief Constructor of the class. + * \param[in] val_nDim - Number of dimension of the problem. + * \param[in] val_nVar - Number of variables of the problem. + * \param[in] config - Definition of the particular problem. + */ + CPBConvection_Central(unsigned short val_nDim, unsigned short val_nVar, CConfig *config) + : CPBConvection_Base(val_nDim, val_nVar, config) {} + + /*! + * \brief Function which defines the advected quantities + */ + void ComputeAdvectedQuantities(void) final; + + /*! + * \brief Function which defines jacobian weights + */ + void ComputeJacobianWeights(void) final; + +}; + + +/*! + * \class CPBConvection_Upwind + * \brief Class for computing an upwind scheme. + * \ingroup ConvDiscr + * \author T. Aalbers + */ +class CPBConvection_Upwind : public CPBConvection_Base { + +public: + + /*! + * \brief Constructor of the class. + * \param[in] val_nDim - Number of dimension of the problem. + * \param[in] val_nVar - Number of variables of the problem. + * \param[in] config - Definition of the particular problem. + */ + CPBConvection_Upwind(unsigned short val_nDim, unsigned short val_nVar, CConfig *config) + : CPBConvection_Base(val_nDim, val_nVar, config) {} + + /*! + * \brief Function which defines the advected quantities + */ + void ComputeAdvectedQuantities(void) final; + + /*! + * \brief Function which defines jacobian weights + */ + void ComputeJacobianWeights(void) final; + +}; diff --git a/SU2_CFD/include/numerics/heat.hpp b/SU2_CFD/include/numerics/heat.hpp deleted file mode 100644 index 1e9cbf815052..000000000000 --- a/SU2_CFD/include/numerics/heat.hpp +++ /dev/null @@ -1,107 +0,0 @@ -/*! - * \file heat.hpp - * \brief Declarations of numerics classes for heat transfer problems. - * \author F. Palacios, T. Economon - * \version 8.5.0 "Harrier" - * - * SU2 Project Website: https://su2code.github.io - * - * The SU2 Project is maintained by the SU2 Foundation - * (http://su2foundation.org) - * - * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) - * - * SU2 is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * SU2 is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with SU2. If not, see . - */ - -#pragma once - -#include "scalar/scalar_diffusion.hpp" -#include "scalar/scalar_convection.hpp" -#include "../variables/CIncEulerVariable.hpp" - -/*! - * \class CUpwSca_Heat - * \brief Class for doing a scalar upwind solver for the heat convection equation. - * \ingroup ConvDiscr - * \author O. Burghardt. - * \version 8.5.0 "Harrier" - */ -class CUpwSca_Heat final : public CUpwScalar> { - public: - /*! - * \brief Constructor of the class. - * \param[in] val_nDim - Number of dimensions of the problem. - * \param[in] config - Definition of the particular problem. - */ - CUpwSca_Heat(unsigned short val_nDim, const CConfig *config) - : CUpwScalar>(val_nDim, 1, config) {} - - private: - /*! - * \brief Adds extra variables to AD - */ - void ExtraADPreaccIn(void) override {} - - /*! - * \brief Heat-specific specific steps in the ComputeResidual method - * \param[in] config - Definition of the particular problem. - */ - void FinishResidualCalc(const CConfig* config) override { - Flux[0] = a0 * ScalarVar_i[0] + a1 * ScalarVar_j[0]; - Jacobian_i[0][0] = a0; - Jacobian_j[0][0] = a1; - } -}; - -/*! - * \class CAvgGrad_Heat - * \brief Class for computing viscous term using average of gradients without correction (heat equation). - * \ingroup ViscDiscr - * \author O. Burghardt. - * \version 8.5.0 "Harrier" - */ -class CAvgGrad_Heat final : public CAvgGrad_Scalar { - public: - /*! - * \brief Constructor of the class. - * \param[in] val_nDim - Number of dimensions of the problem. - * \param[in] config - Definition of the particular problem. - * \param[in] correct - Whether to correct the gradient. - */ - CAvgGrad_Heat(unsigned short val_nDim, const CConfig *config, bool correct) - : CAvgGrad_Scalar(val_nDim, 1, correct, config) {} - - private: - /*! - * \brief Adds extra variables to AD - */ - void ExtraADPreaccIn(void) override { - AD::SetPreaccIn(*Diffusion_Coeff_i, *Diffusion_Coeff_j); - } - - /*! - * \brief Heat-specific specific steps in the ComputeResidual method - * \param[in] config - Definition of the particular problem. - */ - void FinishResidualCalc(const CConfig* config) override { - const su2double Thermal_Diffusivity_Mean = 0.5 * (*Diffusion_Coeff_i + *Diffusion_Coeff_j); - - Flux[0] = Thermal_Diffusivity_Mean * Proj_Mean_GradScalarVar[0]; - - /*--- Use TSL for Jacobians. ---*/ - Jacobian_i[0][0] = -Thermal_Diffusivity_Mean * proj_vector_ij; - Jacobian_j[0][0] = Thermal_Diffusivity_Mean * proj_vector_ij; - } -}; diff --git a/SU2_CFD/include/numerics/heat_edge_flux.hpp b/SU2_CFD/include/numerics/heat_edge_flux.hpp new file mode 100644 index 000000000000..213d168d97f2 --- /dev/null +++ b/SU2_CFD/include/numerics/heat_edge_flux.hpp @@ -0,0 +1,89 @@ +/*! + * \file heat_edge_flux.hpp + * \brief Heat transport as a third-layer scalar flux, see numerics/scalar/scalar_edge_flux.hpp. + * \author P. Gomes + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#pragma once + +#include "scalar/scalar_edge_flux.hpp" + +/*! + * \class CScalarFlux_Heat + * \ingroup ViscDiscr + * \brief Convection and diffusion of temperature, non-conservative, with a diagonal + * (single-equation) diffusion coefficient. + * \note The temperature has no notion of density weighting, so Conservative is false and the + * inherited CUpwScalarFlux::finalizeFlux, flux(0) = a0*phi_i(0) + a1*phi_j(0), is exactly + * the model's whole convective term; no override is needed. + * \note The solver runs in two modes, weakly-coupled energy equation on a fluid zone or standalone + * conduction on a solid one (CHeatSolver::flow); the diffusion coefficient is the flow's + * thermal conductivity over specific heat, plus a turbulent contribution, in the former, and + * the configured constant thermal diffusivity in the latter. EdgeSide::flowNodes is null in + * the solid case, so this is the only place that may read it, and only when flow is set. + */ +template +class CScalarFlux_Heat final + : public CUpwScalarBase, FlowIndices, nDim, nVar> { + public: + static constexpr bool Conservative = false; + static constexpr bool DiagonalDiffusion = true; + + using Base = CUpwScalarBase; + using Int = typename Base::Int; + + explicit CScalarFlux_Heat(const CConfig& config) + : Base(config), + flow(config.GetFluidProblem()), + prandtlTurb(config.GetPrandtl_Turb()), + constDiffusivity(config.GetThermalDiffusivity()) {} + + /*! + * \brief Thermal diffusivity, an i/j average, identical for both edge sides (TSL Jacobian). + */ + template + FORCEINLINE CPair> coefficients(const FlowIndices& idx, Int iPoint, + const EdgeSide& side_i, Int jPoint, + const EdgeSide& side_j, + const CPair&) const { + Vector D; + if (flow) { + const Double k_i = gatherVariables(iPoint, side_i.flowNodes->GetPrimitive(), idx.ThermalConductivity()); + const Double cp_i = gatherVariables(iPoint, side_i.flowNodes->GetPrimitive(), idx.CpTotal()); + const Double muT_i = gatherVariables(iPoint, side_i.flowNodes->GetPrimitive(), idx.EddyViscosity()); + const Double k_j = gatherVariables(jPoint, side_j.flowNodes->GetPrimitive(), idx.ThermalConductivity()); + const Double cp_j = gatherVariables(jPoint, side_j.flowNodes->GetPrimitive(), idx.CpTotal()); + const Double muT_j = gatherVariables(jPoint, side_j.flowNodes->GetPrimitive(), idx.EddyViscosity()); + D(0) = 0.5 * (k_i / cp_i + muT_i / prandtlTurb + k_j / cp_j + muT_j / prandtlTurb); + } else { + D(0) = constDiffusivity; + } + return {D, D}; + } + + private: + const bool flow; + const su2double prandtlTurb; + const su2double constDiffusivity; +}; diff --git a/SU2_CFD/include/numerics/poisson_edge_flux.hpp b/SU2_CFD/include/numerics/poisson_edge_flux.hpp new file mode 100644 index 000000000000..69aca80d5a68 --- /dev/null +++ b/SU2_CFD/include/numerics/poisson_edge_flux.hpp @@ -0,0 +1,78 @@ +/*! + * \file poisson_edge_flux.hpp + * \brief Pressure correction (Poisson) equation as a third-layer scalar flux, + * see numerics/scalar/scalar_edge_flux.hpp. + * \author T. Aalbers, P. Gomes + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#pragma once + +#include "scalar/scalar_edge_flux.hpp" + +/*! + * \class CScalarFlux_Poisson + * \ingroup ViscDiscr + * \brief Diffusion of the pressure correction, with a diagonal (single-equation) coefficient. + * \note The equation has no convective term at all, so the solver runs this kernel with + * ScalarFluxOptions::convective cleared and the inherited CUpwScalarFlux::finalizeFlux + * is never reached; see CPoissonSolver::Viscous_Residual. + * \note The diffusion coefficient is the momentum coefficient vol/A_p carried by the solver's + * own variables, not a flow primitive, so nothing here reads the flow's primitive row; + * the flow variables are only asked which points carry a strong velocity BC. + */ +template +class CScalarFlux_Poisson final + : public CUpwScalarBase, FlowIndices, nDim, nVar> { + public: + static constexpr bool Conservative = false; + static constexpr bool DiagonalDiffusion = true; + + using Base = CUpwScalarBase; + using Int = typename Base::Int; + + using Base::Base; + + /*! + * \brief Momentum coefficient, an i/j average, identical for both edge sides (TSL Jacobian). + * \note A point under a strong velocity BC has no momentum equation, and so no momentum + * coefficient of its own; the edge uses that of its other node instead. + */ + template + FORCEINLINE CPair> coefficients(const FlowIndices&, Int iPoint, + const EdgeSide& side_i, Int jPoint, + const EdgeSide& side_j, + const CPair&) const { + const bool strong_i = side_i.flowNodes->GetStrongBC(iPoint); + const bool strong_j = side_j.flowNodes->GetStrongBC(jPoint); + + const Double coeff_i = strong_i ? gatherVariables(jPoint, side_j.scalarNodes.GetMomCoeff()) + : gatherVariables(iPoint, side_i.scalarNodes.GetMomCoeff()); + const Double coeff_j = strong_j ? gatherVariables(iPoint, side_i.scalarNodes.GetMomCoeff()) + : gatherVariables(jPoint, side_j.scalarNodes.GetMomCoeff()); + + Vector D; + D(0) = 0.5 * (coeff_i + coeff_j); + return {D, D}; + } +}; diff --git a/SU2_CFD/include/numerics/scalar/scalar_convection.hpp b/SU2_CFD/include/numerics/scalar/scalar_convection.hpp deleted file mode 100644 index e40749a5c1d3..000000000000 --- a/SU2_CFD/include/numerics/scalar/scalar_convection.hpp +++ /dev/null @@ -1,150 +0,0 @@ -/*! - * \file scalar_convection.hpp - * \brief Declarations of numerics classes for discretization of - * convective fluxes in scalar problems. - * \author F. Palacios, T. Economon - * \version 8.5.0 "Harrier" - * - * SU2 Project Website: https://su2code.github.io - * - * The SU2 Project is maintained by the SU2 Foundation - * (http://su2foundation.org) - * - * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) - * - * SU2 is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * SU2 is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with SU2. If not, see . - */ - -#pragma once - -#include "../CNumerics.hpp" - -/*! - * \class CUpwScalar - * \brief Template class for scalar upwind fluxes between nodes i and j. - * \details This class serves as a template for the scalar upwinding residual - * classes. The general structure of a scalar upwinding calculation is the - * same for many different models, which leads to a lot of repeated code. - * By using the template design pattern, these sections of repeated code are - * moved to this shared base class, and the specifics of each model - * are implemented by derived classes. In order to add a new residual - * calculation for a convection residual, extend this class and implement - * the pure virtual functions with model-specific behavior. - * \ingroup ConvDiscr - * \author C. Pederson, A. Bueno., and A. Campos. - */ -template -class CUpwScalar : public CNumerics { - protected: - enum : unsigned short {MAXNVAR = 8}; - - const FlowIndices idx; /*!< \brief Object to manage the access to the flow primitives. */ - su2double a0 = 0.0; /*!< \brief The maximum of the face-normal velocity and 0. */ - su2double a1 = 0.0; /*!< \brief The minimum of the face-normal velocity and 0. */ - su2double Flux[MAXNVAR]; /*!< \brief Final result, diffusive flux/residual. */ - su2double* Jacobian_i[MAXNVAR]; /*!< \brief Flux Jacobian w.r.t. node i. */ - su2double* Jacobian_j[MAXNVAR]; /*!< \brief Flux Jacobian w.r.t. node j. */ - su2double JacobianBuffer[2*MAXNVAR*MAXNVAR]; /*!< \brief Static storage for the two Jacobians. */ - - const bool incompressible = false, dynamic_grid = false; - - /*! - * \brief A pure virtual function. Derived classes must use it to register the additional - * variables they use as preaccumulation inputs, e.g. the density for SST. - */ - virtual void ExtraADPreaccIn() = 0; - - /*! - * \brief Model-specific steps in the ComputeResidual method, derived classes - * compute the Flux and its Jacobians via this method. - * \param[in] config - Definition of the particular problem. - */ - virtual void FinishResidualCalc(const CConfig* config) = 0; - - public: - /*! - * \brief Constructor of the class. - * \param[in] ndim - Number of dimensions of the problem. - * \param[in] nvar - Number of variables of the problem. - * \param[in] config - Definition of the particular problem. - */ - CUpwScalar(unsigned short ndim, unsigned short nvar, const CConfig* config) - : CNumerics(ndim, nvar, config), - idx(ndim, config->GetnSpecies()), - incompressible(config->GetKind_Regime() == ENUM_REGIME::INCOMPRESSIBLE), - dynamic_grid(config->GetDynamic_Grid()) { - if (nVar > MAXNVAR) { - SU2_MPI::Error("Static arrays are too small.", CURRENT_FUNCTION); - } - for (unsigned short iVar = 0; iVar < nVar; iVar++) { - Jacobian_i[iVar] = &JacobianBuffer[iVar * nVar]; - Jacobian_j[iVar] = &JacobianBuffer[iVar * nVar + MAXNVAR * MAXNVAR]; - } - - /*--- Initialize the JacobianBuffer to zero. ---*/ - for (unsigned short iVar = 0; iVar < 2*MAXNVAR*MAXNVAR; iVar++) { - JacobianBuffer[iVar] = 0.0; - } - } - - /*! - * \brief Compute the scalar upwind flux between two nodes i and j. - * \param[in] config - Definition of the particular problem. - * \return A lightweight const-view (read-only) of the residual/flux and Jacobians. - */ - CNumerics::ResidualType<> ComputeResidual(const CConfig* config) final { - AD::StartPreacc(); - AD::SetPreaccIn(Normal, nDim); - AD::SetPreaccIn(ScalarVar_i, nVar); - AD::SetPreaccIn(ScalarVar_j, nVar); - if (dynamic_grid) { - AD::SetPreaccIn(GridVel_i, nDim); - AD::SetPreaccIn(GridVel_j, nDim); - } - AD::SetPreaccIn(&V_i[idx.Velocity()], nDim); - AD::SetPreaccIn(&V_j[idx.Velocity()], nDim); - AD::SetPreaccIn(V_i[idx.Density()]); - AD::SetPreaccIn(V_j[idx.Density()]); - AD::SetPreaccIn(MassFlux); - - ExtraADPreaccIn(); - - if (bounded_scalar) { - a0 = fmax(0.0, MassFlux) / V_i[idx.Density()]; - a1 = fmin(0.0, MassFlux) / V_j[idx.Density()]; - } else { - su2double q_ij = 0.0; - if (dynamic_grid) { - for (unsigned short iDim = 0; iDim < nDim; iDim++) { - su2double Velocity_i = V_i[iDim + idx.Velocity()] - GridVel_i[iDim]; - su2double Velocity_j = V_j[iDim + idx.Velocity()] - GridVel_j[iDim]; - q_ij += 0.5 * (Velocity_i + Velocity_j) * Normal[iDim]; - } - } else { - for (unsigned short iDim = 0; iDim < nDim; iDim++) { - q_ij += 0.5 * (V_i[iDim + idx.Velocity()] + V_j[iDim + idx.Velocity()]) * Normal[iDim]; - } - } - a0 = fmax(0.0, q_ij); - a1 = fmin(0.0, q_ij); - } - - FinishResidualCalc(config); - - AD::SetPreaccOut(Flux, nVar); - AD::EndPreacc(); - - return ResidualType<>(Flux, Jacobian_i, Jacobian_j); - } -}; diff --git a/SU2_CFD/include/numerics/scalar/scalar_diffusion.hpp b/SU2_CFD/include/numerics/scalar/scalar_diffusion.hpp deleted file mode 100644 index 580c0bedcefa..000000000000 --- a/SU2_CFD/include/numerics/scalar/scalar_diffusion.hpp +++ /dev/null @@ -1,155 +0,0 @@ -/*! - * \file scalar_diffusion.hpp - * \brief Declarations of numerics classes for discretization of - * viscous fluxes in scalar problems. - * \author F. Palacios, T. Economon - * \version 8.5.0 "Harrier" - * - * SU2 Project Website: https://su2code.github.io - * - * The SU2 Project is maintained by the SU2 Foundation - * (http://su2foundation.org) - * - * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) - * - * SU2 is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * SU2 is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with SU2. If not, see . - */ - -#pragma once - -#include "../CNumerics.hpp" - -/*! - * \class CNoFlowIndices - * \brief Dummy flow indices class to use CAvgGrad_Scalar when flow variables are not available. - * For example, solid heat transfer problems. - */ -struct CNoFlowIndices { - CNoFlowIndices(int, int) {} - inline int Density() const { return 0; } - inline int LaminarViscosity() const { return 0; } - inline int EddyViscosity() const { return 0; } -}; - -/*! - * \class CAvgGrad_Scalar - * \brief Template class for computing viscous residual of scalar values - * \details This class serves as a template for the scalar viscous residual - * classes. The general structure of a viscous residual calculation is the - * same for many different models, which leads to a lot of repeated code. - * By using the template design pattern, these sections of repeated code are - * moved to a shared base class, and the specifics of each model - * are implemented by derived classes. In order to add a new residual - * calculation for a viscous residual, extend this class and implement - * the pure virtual functions with model-specific behavior. - * \ingroup ViscDiscr - * \author C. Pederson, A. Bueno, and F. Palacios - */ -template -class CAvgGrad_Scalar : public CNumerics { - protected: - enum : unsigned short {MAXNVAR = 8}; - - const FlowIndices idx; /*!< \brief Object to manage the access to the flow primitives. */ - su2double Proj_Mean_GradScalarVar[MAXNVAR]; /*!< \brief Mean_gradScalarVar DOT normal, corrected if required. */ - su2double proj_vector_ij = 0.0; /*!< \brief (Edge_Vector DOT normal)/|Edge_Vector|^2 */ - su2double Flux[MAXNVAR] = {0.0}; /*!< \brief Final result, diffusive flux/residual. */ - su2double* Jacobian_i[MAXNVAR]; /*!< \brief Flux Jacobian w.r.t. node i. */ - su2double* Jacobian_j[MAXNVAR]; /*!< \brief Flux Jacobian w.r.t. node j. */ - su2double JacobianBuffer[2*MAXNVAR*MAXNVAR];/*!< \brief Static storage for the two Jacobians. */ - - const bool correct_gradient = false, incompressible = false; - - /*! - * \brief A pure virtual function; Adds any extra variables to AD - */ - virtual void ExtraADPreaccIn() = 0; - - /*! - * \brief Model-specific steps in the ComputeResidual method, derived classes - * should compute the Flux and Jacobians (i/j) inside this method. - * \param[in] config - Definition of the particular problem. - */ - virtual void FinishResidualCalc(const CConfig* config) = 0; - - public: - /*! - * \brief Constructor of the class. - * \param[in] val_nDim - Number of dimensions of the problem. - * \param[in] val_nVar - Number of variables of the problem. - * \param[in] correct_gradient - Whether to correct gradient for skewness. - * \param[in] config - Definition of the particular problem. - */ - CAvgGrad_Scalar(unsigned short val_nDim, unsigned short val_nVar, bool correct_grad, - const CConfig* config) - : CNumerics(val_nDim, val_nVar, config), - idx(val_nDim, config->GetnSpecies()), - correct_gradient(correct_grad), - incompressible(config->GetKind_Regime() == ENUM_REGIME::INCOMPRESSIBLE) { - if (nVar > MAXNVAR) { - SU2_MPI::Error("Static arrays are too small.", CURRENT_FUNCTION); - } - for (unsigned short iVar = 0; iVar < nVar; iVar++) { - Jacobian_i[iVar] = &JacobianBuffer[iVar * nVar]; - Jacobian_j[iVar] = &JacobianBuffer[iVar * nVar + MAXNVAR * MAXNVAR]; - } - - /*--- Initialize the JacobianBuffer to zero. ---*/ - for (unsigned short iVar = 0; iVar < 2*MAXNVAR*MAXNVAR; iVar++) { - JacobianBuffer[iVar] = 0.0; - } - } - - /*! - * \brief Compute the viscous residual using an average of gradients without correction. - * \param[in] config - Definition of the particular problem. - * \return A lightweight const-view (read-only) of the residual/flux and Jacobians. - */ - ResidualType<> ComputeResidual(const CConfig* config) final { - AD::StartPreacc(); - AD::SetPreaccIn(Coord_i, nDim); - AD::SetPreaccIn(Coord_j, nDim); - AD::SetPreaccIn(Normal, nDim); - AD::SetPreaccIn(ScalarVar_Grad_i, nVar, nDim); - AD::SetPreaccIn(ScalarVar_Grad_j, nVar, nDim); - if (correct_gradient) { - AD::SetPreaccIn(ScalarVar_i, nVar); - AD::SetPreaccIn(ScalarVar_j, nVar); - } - if (!std::is_same::value) { - AD::SetPreaccIn(V_i[idx.Density()], V_i[idx.LaminarViscosity()], V_i[idx.EddyViscosity()]); - AD::SetPreaccIn(V_j[idx.Density()], V_j[idx.LaminarViscosity()], V_j[idx.EddyViscosity()]); - - Density_i = V_i[idx.Density()]; - Density_j = V_j[idx.Density()]; - Laminar_Viscosity_i = V_i[idx.LaminarViscosity()]; - Laminar_Viscosity_j = V_j[idx.LaminarViscosity()]; - Eddy_Viscosity_i = V_i[idx.EddyViscosity()]; - Eddy_Viscosity_j = V_j[idx.EddyViscosity()]; - } - - ExtraADPreaccIn(); - - su2double ProjGradScalarVarNoCorr[MAXNVAR]; - proj_vector_ij = ComputeProjectedGradient(nDim, nVar, Normal, Coord_i, Coord_j, ScalarVar_Grad_i, ScalarVar_Grad_j, - correct_gradient, ScalarVar_i, ScalarVar_j, ProjGradScalarVarNoCorr, - Proj_Mean_GradScalarVar); - FinishResidualCalc(config); - - AD::SetPreaccOut(Flux, nVar); - AD::EndPreacc(); - - return ResidualType<>(Flux, Jacobian_i, Jacobian_j); - } -}; diff --git a/SU2_CFD/include/numerics/scalar/scalar_edge_flux.hpp b/SU2_CFD/include/numerics/scalar/scalar_edge_flux.hpp new file mode 100644 index 000000000000..4d2e79608bdf --- /dev/null +++ b/SU2_CFD/include/numerics/scalar/scalar_edge_flux.hpp @@ -0,0 +1,504 @@ +/*! + * \file scalar_edge_flux.hpp + * \brief Model-agnostic convection and diffusion of a transported scalar, shared by every + * scalar solver (turbulence, transition, species, flamelet, heat). + * \author P. Gomes + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#pragma once + +#include "../../../../Common/include/CConfig.hpp" +#include "../../../../Common/include/containers/container_decorators.hpp" +#include "../util.hpp" +#include "../../variables/CFlowVariable.hpp" + +/*! + * \brief Locates the data of one endpoint of an edge. + * \note The j side of a boundary flux indexes per-marker ghost containers, which have the + * same types as the solver's own, so the kernels read both through one code path. + */ +template +struct EdgeSide { + const VariableType& scalarNodes; /*!< \brief Scalar solver variables. */ + const CFlowVariable* flowNodes; /*!< \brief Flow variables, null for solid heat transfer. */ + CMatrixView coord; /*!< \brief Point coordinates. */ + CMatrixView gridVel; /*!< \brief Empty when the grid is static. */ +}; + +/*! + * \brief Loop invariant flags for a scalar edge flux, built once outside the edge loop so the + * compiler can unswitch the branches they guard. + * \note Built through the named constructors below: the flags are too many and too alike to be + * given positionally, where one transposed pair would change the discretization silently. + */ +struct ScalarFluxOptions { + bool dynamicGrid = false; + bool boundedScalar = false; + bool correctGradient = false; + bool accurateJacobians = false; + bool implicit = false; /*!< \brief Whether the Jacobians are assembled, and so computed. */ + bool convective = true; /*!< \brief Whether the convective scheme contributes. */ + bool viscous = false; /*!< \brief Whether the diffusion term contributes. */ + bool oneSided = false; /*!< \brief Whether only the row of i is assembled. */ + bool muscl = false; /*!< \brief Whether the convective scheme reconstructs. */ + + /*! + * \brief Options of the interior edge loop: both terms, both rows, and reconstruction and + * gradient correction as the configuration asks for them. + */ + static ScalarFluxOptions Interior(const CConfig& config, bool bounded, bool accurateJacobians = false) { + auto opt = common(config); + opt.boundedScalar = bounded; + opt.accurateJacobians = accurateJacobians; + opt.correctGradient = true; + opt.viscous = true; + opt.muscl = config.GetMUSCL(); + return opt; + } + + /*! + * \brief Options of a boundary that imposes a convective flux alone, which is most of them: + * the diffusive term at an inlet or an outlet causes serious convergence problems. + */ + static ScalarFluxOptions BoundaryConvective(const CConfig& config, bool bounded) { + auto opt = common(config); + opt.boundedScalar = bounded; + opt.oneSided = true; + return opt; + } + + /*! + * \brief Options of a boundary that imposes both terms, which is the turbomachinery sites. + * \note They impose no mass flux, so the bounded scheme contributes nothing here whatever the + * configuration says. + */ + static ScalarFluxOptions BoundaryFull(const CConfig& config) { + auto opt = common(config); + opt.correctGradient = true; + opt.viscous = true; + opt.oneSided = true; + return opt; + } + + /*! + * \brief Options of the diffusive pass of a fluid interface, which follows a convective pass + * over the donor vertices. + * \param[in] correctGrad - Whether the projected gradient is corrected for skewness, which the + * models do not agree on at this boundary. + */ + static ScalarFluxOptions BoundaryDiffusive(const CConfig& config, bool correctGrad) { + auto opt = common(config); + opt.correctGradient = correctGrad; + opt.convective = false; + opt.viscous = true; + opt.oneSided = true; + return opt; + } + + private: + static ScalarFluxOptions common(const CConfig& config) { + ScalarFluxOptions opt; + opt.dynamicGrid = config.GetDynamic_Grid(); + opt.implicit = config.GetKind_TimeIntScheme() == EULER_IMPLICIT; + return opt; + } +}; + +/*! + * \brief Thin wrapper giving a Vector the all(iVar) accessor the MUSCL reconstruction helpers + * expect (see CCompressiblePrimitives in numerics_simd/flow/variables.hpp). + */ +template +struct CScalarValues { + static constexpr size_t nVar = Size; /*!< \brief Only used as VarType::nVar when reconstruct's own nVarGrad_ default + applies; every call site here passes an explicit nVarGrad_ instead. */ + Vector all; +}; + +/*! + * \brief Diffusion of a transported scalar, driven by model-supplied coefficients. + * \note The derived class returns the coefficients of both orientations of the edge, as one + * object, so that whatever the two share is computed once. A model whose coefficients + * are the same in both orientations returns the same value twice. A model whose matrix + * is diagonal declares DiagonalDiffusion and returns a vector instead of a matrix. + */ +template +class CAvgGradScalarBase { + protected: + using Int = typename CLaneTraits::Int; + + /*! + * \param[in] rho - Density of both endpoints, read once by ComputeFlux. + */ + template + FORCEINLINE void diffusionTerms(const FlowIndices& idx, const ScalarFluxOptions& opt, Int iPoint, + const EdgeSide& side_i, Int jPoint, + const EdgeSide& side_j, const CPair& rho, + const Vector& normal, const Vector& vector_ij, + EdgeResidual& res) const { + if (!opt.viscous) return; + + constexpr size_t Size = EdgeResidual::Size; + + const Double dist2_ij = fmax(squaredNorm(vector_ij), EPS); + const Double proj_vector_ij = dot(vector_ij, normal) / dist2_ij; + + /*--- Average gradient, corrected for skewness when asked. + * \note Gathered one variable at a time, bounded by res.nVar rather than Size: a static + * model with nVar 1 has Size 2 (the Matrix degeneracy floor), and a + * dynamic one has Size MaxScalarVar, so a single Size-wide read would run past the + * actual width of the gradient container in either case. ---*/ + Matrix avgGrad; + for (size_t iVar = 0; iVar < res.nVar; ++iVar) { + const auto grad_i = gatherVariables(iPoint, side_i.scalarNodes.GetGradient(), iVar); + const auto grad_j = gatherVariables(jPoint, side_j.scalarNodes.GetGradient(), iVar); + for (int iDim = 0; iDim < nDim; ++iDim) avgGrad(iVar, iDim) = 0.5 * (grad_i(iDim) + grad_j(iDim)); + } + + if (opt.correctGradient) { + for (size_t iVar = 0; iVar < res.nVar; ++iVar) { + const Double phi_i = gatherVariables(iPoint, side_i.scalarNodes.GetSolution(), iVar); + const Double phi_j = gatherVariables(jPoint, side_j.scalarNodes.GetSolution(), iVar); + const Double corr = (dot(avgGrad[iVar], vector_ij) - phi_j + phi_i) / dist2_ij; + for (int iDim = 0; iDim < nDim; ++iDim) avgGrad(iVar, iDim) -= corr * vector_ij(iDim); + } + } + + Vector projGrad; + for (size_t iVar = 0; iVar < res.nVar; ++iVar) projGrad(iVar) = dot(avgGrad[iVar], normal); + + /*--- The Jacobians of a conservative model are w.r.t. the conserved (density-weighted) + * variable, which divides the geometric projection by the density of the row being written. ---*/ + Double proj_on_w_i = proj_vector_ij, proj_on_w_j = proj_vector_ij; + if constexpr (Derived::Conservative) { + proj_on_w_i = proj_vector_ij / rho.i; + proj_on_w_j = proj_vector_ij / rho.j; + } + + const auto* self = static_cast(this); + const auto D = self->coefficients(idx, iPoint, side_i, jPoint, side_j, rho); + + for (size_t iVar = 0; iVar < res.nVar; ++iVar) { + if constexpr (Derived::DiagonalDiffusion) { + res.flux_i(iVar) -= D.i(iVar) * projGrad(iVar); + if (opt.implicit) { + res.jac_ii(iVar, iVar) += D.i(iVar) * proj_on_w_i; + if (!opt.oneSided) res.jac_ij(iVar, iVar) -= D.i(iVar) * proj_on_w_j; + } + if (!opt.oneSided) { + res.flux_j(iVar) += D.j(iVar) * projGrad(iVar); + if (opt.implicit) { + res.jac_ji(iVar, iVar) -= D.j(iVar) * proj_on_w_i; + res.jac_jj(iVar, iVar) += D.j(iVar) * proj_on_w_j; + } + } + } else { + for (size_t jVar = 0; jVar < res.nVar; ++jVar) { + res.flux_i(iVar) -= D.i(iVar, jVar) * projGrad(jVar); + if (opt.implicit) { + res.jac_ii(iVar, jVar) += D.i(iVar, jVar) * proj_on_w_i; + if (!opt.oneSided) res.jac_ij(iVar, jVar) -= D.i(iVar, jVar) * proj_on_w_j; + } + if (!opt.oneSided) { + res.flux_j(iVar) += D.j(iVar, jVar) * projGrad(jVar); + if (opt.implicit) { + res.jac_ji(iVar, jVar) -= D.j(iVar, jVar) * proj_on_w_i; + res.jac_jj(iVar, jVar) += D.j(iVar, jVar) * proj_on_w_j; + } + } + } + } + } + + if (opt.implicit && opt.accurateJacobians) { + /*--- Coefficients that depend on the transported variables contribute here, from whatever + * the model chose to carry in the object it returned from coefficients. ---*/ + self->coefficientJacobians(opt, D, projGrad, res); + } + + self->extraDiffusionTerms(idx, opt, iPoint, side_i, jPoint, side_j, rho, normal, vector_ij, res); + } + + /*! + * \brief Contribution of the derivatives of the coefficients themselves. + */ + template + FORCEINLINE void coefficientJacobians(Ts&&...) const {} + + /*! + * \brief Diffusion of a model that transports more than one gradient, of states it + * synthesises from its own containers. + */ + template + FORCEINLINE void extraDiffusionTerms(Ts&&...) const {} +}; + +/*! + * \brief Convective flux shared by every model whose transport equation has the shape + * flux(iVar) = a0 * w_i * phi_i(iVar) + a1 * w_j * phi_j(iVar), which is every + * model except SA and stochastic backscatter (see CScalarFlux_SA). + * \note The weight w is 1 for a non-conservative model, the density for a conservative one. + */ +template +class CUpwScalarFlux : public CAvgGradScalarBase { + protected: + using Int = typename CLaneTraits::Int; + + explicit CUpwScalarFlux(const CConfig&) {} + + /*! + * \brief Upwind convection of the transported variable, weighted by the density for a + * conservative model. + * \param[in] phi - Transported variable of both endpoints, reconstructed if opt.muscl is set; + * read from here rather than side_i/side_j.scalarNodes directly so a model needs + * no reconstruction logic of its own. + * \param[in] rho - Density of both endpoints, reconstructed alongside the velocity when the + * convective scheme reconstructs, so it weights the flux as the velocity does. + * \note The flux is written in terms of the transported variable but the Jacobians are w.r.t. + * the conserved one, which for a conservative model is the density-weighted variable; + * the density therefore multiplies the flux and not the Jacobian. + */ + template + FORCEINLINE void finalizeFlux(const FlowIndices&, const ScalarFluxOptions& opt, Int, const EdgeSide&, + Int, const EdgeSide&, const Double& a0, const Double& a1, + const CPair& rho, const CPair>& phi, + EdgeResidual& res) const { + Double w0 = a0, w1 = a1; + if constexpr (Derived::Conservative) { + w0 *= rho.i; + w1 *= rho.j; + } + + for (size_t iVar = 0; iVar < res.nVar; ++iVar) { + const Double flux = w0 * phi.i.all(iVar) + w1 * phi.j.all(iVar); + + res.flux_i(iVar) += flux; + if (!opt.oneSided) res.flux_j(iVar) -= flux; + + if (opt.implicit) { + res.jac_ii(iVar, iVar) += a0; + if (!opt.oneSided) { + res.jac_ij(iVar, iVar) += a1; + res.jac_ji(iVar, iVar) -= a0; + res.jac_jj(iVar, iVar) -= a1; + } + } + } + } +}; + +/*! + * \brief Upwind convection and diffusion of a transported scalar, accumulated into one + * residual, each term contributing or not according to the options. + */ +template +class CUpwScalarBase : public CUpwScalarFlux { + public: + using Double = Double_; + using Int = typename CLaneTraits::Int; + static constexpr int nDim = nDim_; + static constexpr size_t nVar = nVar_; + + /*! + * \brief Backing size, for the model to size the containers its coefficients() returns; + * nVar itself is Dynamic for a runtime model and never usable as a container size. + */ + static constexpr size_t Size = EdgeResidual::Size; + + /*! + * \brief Whether the model's diffusion coefficients read the density, beyond the reading that + * Conservative already implies. A model that declares neither never gathers it. + */ + static constexpr bool DiffusionReadsDensity = false; + + protected: + using Base = CUpwScalarFlux; + + const FlowIndices idx; + const size_t nEqn; /*!< \brief Equations of the model, which a dynamic one gives to its base. */ + + /*! + * \brief MUSCL reconstruction parameters, read from CConfig once per construction (i.e. once + * per nonlinear iteration, see CScalarSolver::EdgeFluxResidual) instead of per edge; + * this is also where the scalar limiter's freezing (GetLimiterIter) is resolved, by + * collapsing its type to NONE once frozen. The flow limiter is not frozen this way: once + * the flow solver stops recomputing it, it keeps applying the last values it has. + */ + const su2double kappa, umusclRamp, kappaFlow; + const LIMITER limiterType, limiterTypeFlow; + const bool musclFlow; + + public: + /*! + * \brief Constructor, inherited by the model with `using Base::Base`. + * \note Public, not protected: a using-declaration that inherits a constructor keeps the + * base's own access, so the solver that builds the concrete model needs this public + * to build it at all. + */ + explicit CUpwScalarBase(const CConfig& config, size_t nEqn_ = nVar_) + : Base(config), + idx(nDim, config.GetnSpecies()), + nEqn(nEqn_), + kappa(config.GetMUSCL_Kappa()), + umusclRamp(config.GetMUSCLRampValue()), + kappaFlow(config.GetMUSCL_Kappa_Flow()), + limiterType(config.GetInnerIter() <= config.GetLimiterIter() ? config.GetKind_SlopeLimit() : LIMITER::NONE), + limiterTypeFlow(config.GetKind_SlopeLimit_Flow() != LIMITER::VAN_ALBADA_EDGE ? config.GetKind_SlopeLimit_Flow() + : LIMITER::NONE), + musclFlow(config.GetMUSCL_Flow() && config.GetKind_ConvNumScheme_Flow() == SPACE_UPWIND) { + if (nEqn > Size) { + SU2_MPI::Error("Static arrays are too small for the requested equation count.", CURRENT_FUNCTION); + } + } + + template + FORCEINLINE EdgeResidual ComputeFlux(const ScalarFluxOptions& opt, Int iPoint, + const EdgeSide& side_i, Int jPoint, + const EdgeSide& side_j, + const Vector& normal, const Double& massFlux) const { + /*--- Inputs are registered as they are read, by each of the two terms. ---*/ + AD::StartPreacc(); + AD::SetPreaccIn(normal, nDim); + + EdgeResidual res(nEqn); + + /*--- Read once by the reconstruction and by the diffusion. ---*/ + Vector vector_ij; + if (opt.muscl || opt.viscous) { + vector_ij = distanceVector(iPoint, side_i.coord, jPoint, side_j.coord); + } + + /*--- Density of both endpoints, gathered once: the conservative weighting of the convective + * term, the bounded scheme's division of the mass flux, and some models' diffusion + * coefficients all want it, and in reverse mode every gather is a preaccumulation input. ---*/ + CPair rho{Double(1.0), Double(1.0)}; + if (Derived::Conservative || Derived::DiffusionReadsDensity || opt.boundedScalar) { + rho.i = gatherVariables(iPoint, side_i.flowNodes->GetPrimitive(), idx.Density()); + rho.j = gatherVariables(jPoint, side_j.flowNodes->GetPrimitive(), idx.Density()); + } + + if (opt.convective) { + /*--- Upwinding weights of the face normal mass or volume flux, and the density that weights + * a conservative flux, which follows the velocity in being reconstructed or not. ---*/ + Double a0, a1; + CPair rhoConv = rho; + + if (opt.boundedScalar) { + AD::SetPreaccIn(massFlux); + a0 = fmax(0.0, massFlux) / rho.i; + a1 = fmin(0.0, massFlux) / rho.j; + } else { + CPair> u; + u.i.all = gatherVariables(iPoint, side_i.flowNodes->GetPrimitive(), idx.Velocity()); + u.j.all = gatherVariables(jPoint, side_j.flowNodes->GetPrimitive(), idx.Velocity()); + + if (opt.muscl && musclFlow) { + reconstruct(iPoint, jPoint, vector_ij, side_i.flowNodes->GetGradient_Reconstruction(), + side_i.flowNodes->GetLimiter_Primitive(), limiterTypeFlow, idx.Velocity(), u, kappaFlow, + umusclRamp); + + if constexpr (Derived::Conservative) { + /*--- Density is not adjacent to the velocity in the primitive row, so it is a second + * reconstruction of one variable rather than a wider slice of the first. Some upwind + * schemes (see EulerNPrimVarGrad) size the flow's gradient/limiter columns smaller + * than the full primitive count and leave density out of it entirely, so fall back to + * the cell-centered density instead. ---*/ + if (idx.Density() < side_i.flowNodes->GetnPrimVarGrad()) { + CPair> r; + r.i.all(0) = rho.i; + r.j.all(0) = rho.j; + reconstruct<1>(iPoint, jPoint, vector_ij, side_i.flowNodes->GetGradient_Reconstruction(), + side_i.flowNodes->GetLimiter_Primitive(), limiterTypeFlow, idx.Density(), r, kappaFlow, + umusclRamp); + rhoConv.i = r.i.all(0); + rhoConv.j = r.j.all(0); + } + } + } + + /*--- Face normal velocity of the mean of the two points, relative to the grid. ---*/ + Vector vel_ij; + for (int iDim = 0; iDim < nDim; ++iDim) vel_ij(iDim) = 0.5 * (u.i.all(iDim) + u.j.all(iDim)); + + if (opt.dynamicGrid) { + const auto ug_i = gatherVariables(iPoint, side_i.gridVel); + /*--- A boundary's ghost point has no grid velocity of its own: it is spatially + * coincident with i, so it moves with it. ---*/ + const auto ug_j = opt.oneSided ? ug_i : gatherVariables(jPoint, side_j.gridVel); + for (int iDim = 0; iDim < nDim; ++iDim) vel_ij(iDim) -= 0.5 * (ug_i(iDim) + ug_j(iDim)); + } + + const Double q_ij = dot(vel_ij, normal); + a0 = fmax(0.0, q_ij); + a1 = fmin(0.0, q_ij); + } + + /*--- Transported variable of both endpoints, reconstructed if opt.muscl is set. + * Gathered one variable at a time (like the diffusion gradients above) so a static model + * with nVar 1 never reads past the single column its solution container actually has. ---*/ + CPair> phi; + for (size_t iVar = 0; iVar < res.nVar; ++iVar) { + phi.i.all(iVar) = gatherVariables(iPoint, side_i.scalarNodes.GetSolution(), iVar); + phi.j.all(iVar) = gatherVariables(jPoint, side_j.scalarNodes.GetSolution(), iVar); + } + + if (opt.muscl) { + if constexpr (nVar != Dynamic) { + reconstruct(iPoint, jPoint, vector_ij, side_i.scalarNodes.GetGradient_Reconstruction(), + side_i.scalarNodes.GetLimiter(), limiterType, 0, phi, kappa, umusclRamp); + } else { + /*--- A dynamic model's equation count is only known at runtime, so the reconstructed + * width is passed as an argument instead of a template parameter. ---*/ + reconstruct(iPoint, jPoint, vector_ij, side_i.scalarNodes.GetGradient_Reconstruction(), + side_i.scalarNodes.GetLimiter(), limiterType, 0, phi, kappa, umusclRamp, res.nVar); + } + } + + static_cast(this)->finalizeFlux(idx, opt, iPoint, side_i, jPoint, side_j, a0, a1, rhoConv, phi, + res); + } + + Base::diffusionTerms(idx, opt, iPoint, side_i, jPoint, side_j, rho, normal, vector_ij, res); + + setPreaccOut(res.flux_i, res.nVar); + if (!opt.oneSided) setPreaccOut(res.flux_j, res.nVar); + AD::EndPreacc(); + + return res; + } + + /*! + * \brief Compute the flux of an edge and write it to the linear system. + */ + template + FORCEINLINE void ComputeFlux(const ScalarFluxOptions& opt, Int iEdge, Int iPoint, + const EdgeSide& side_i, Int jPoint, const EdgeSide& side_j, + const Vector& normal, const Double& massFlux, UpdateType updateType, + Double updateMask, CSysVector& vector, CSysVector& vectorDiff, + SparseMatrixType& matrix) const { + const auto res = ComputeFlux(opt, iPoint, side_i, jPoint, side_j, normal, massFlux); + + updateLinearSystem(iEdge, iPoint, jPoint, opt.implicit, updateType, updateMask, res, vector, vectorDiff, matrix); + } +}; diff --git a/SU2_CFD/include/numerics/species/flamelet_edge_flux.hpp b/SU2_CFD/include/numerics/species/flamelet_edge_flux.hpp new file mode 100644 index 000000000000..dc634a235ca1 --- /dev/null +++ b/SU2_CFD/include/numerics/species/flamelet_edge_flux.hpp @@ -0,0 +1,172 @@ +/*! + * \file flamelet_edge_flux.hpp + * \brief Flamelet transport as a third-layer scalar flux, see numerics/scalar/scalar_edge_flux.hpp. + * \author P. Gomes + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#pragma once + +#include "species_edge_flux.hpp" + +/*! + * \class CScalarFlux_Flamelet + * \ingroup ViscDiscr + * \brief Convection and diffusion of the flamelet controlling variables and passive species, + * which is species transport plus two preferential diffusion terms. + * \note The preferential diffusion terms read the beta scalars and their gradients from the + * auxiliary variables of the solver's own containers, which the per-marker ghost containers + * of a boundary do not carry. They are an interior edge term: boundaries instantiate + * CScalarFlux_Species, as they did before this model existed. + */ +template +class CScalarFlux_Flamelet final + : public CScalarFluxSpeciesBase, FlowIndices, nDim, + nVar> { + public: + using Base = CScalarFluxSpeciesBase; + using Int = typename Base::Int; + + explicit CScalarFlux_Flamelet(const CConfig& config) + : Base(config), + preferentialDiffusion(config.GetFlameletParsedOptions().preferential_diffusion), + nControlVars(config.GetFlameletParsedOptions().n_control_vars) {} + + /*! + * \brief Preferential diffusion, two terms with the shape of the ordinary diffusion but of + * states the model synthesises: div(D grad(beta - phi)) for each controlling variable, + * and a thermal term div(beta_T D grad(T)) on the enthalpy equation. + * \note The thermal term has no implicit part, matching the treatment of the heat flux it + * models; the first term has the same thin shear layer Jacobian as the ordinary diffusion, + * because it is the same operator applied to a shifted state. + */ + template + FORCEINLINE void extraDiffusionTerms(const FlowIndices& idx, const ScalarFluxOptions& opt, Int iPoint, + const EdgeSide& side_i, Int jPoint, + const EdgeSide& side_j, const CPair& rho, + const Vector& normal, const Vector& vector_ij, + EdgeResidual& res) const { + if (!preferentialDiffusion) return; + + const Double dist2_ij = fmax(squaredNorm(vector_ij), EPS); + const Double proj_vector_ij = dot(vector_ij, normal) / dist2_ij; + const Double proj_on_rho_i = proj_vector_ij / rho.i; + const Double proj_on_rho_j = proj_vector_ij / rho.j; + + const Double diffTurb = Base::turbulentDiffusivity(idx, iPoint, side_i, jPoint, side_j); + + /*--- The gradient of a controlling variable is subtracted from that of its beta scalar, so + * that what is added here is the difference from the ordinary diffusion already applied. ---*/ + for (auto iScalar = 0u; iScalar < nControlVars; ++iScalar) { + const auto iBeta = betaIndex(iScalar); + + const Double phi_i = gatherVariables(iPoint, side_i.scalarNodes.GetAuxVar(), iBeta) - + gatherVariables(iPoint, side_i.scalarNodes.GetSolution(), iScalar); + const Double phi_j = gatherVariables(jPoint, side_j.scalarNodes.GetAuxVar(), iBeta) - + gatherVariables(jPoint, side_j.scalarNodes.GetSolution(), iScalar); + + auto grad_i = gatherVariables(iPoint, side_i.scalarNodes.GetAuxVarGradient(), iBeta); + auto grad_j = gatherVariables(jPoint, side_j.scalarNodes.GetAuxVarGradient(), iBeta); + const auto gradPhi_i = gatherVariables(iPoint, side_i.scalarNodes.GetGradient(), iScalar); + const auto gradPhi_j = gatherVariables(jPoint, side_j.scalarNodes.GetGradient(), iScalar); + for (int iDim = 0; iDim < nDim; ++iDim) { + grad_i(iDim) -= gradPhi_i(iDim); + grad_j(iDim) -= gradPhi_j(iDim); + } + + const Double D_i = gatherVariables(iPoint, side_i.scalarNodes.GetDiffusivity(), iScalar); + const Double D_j = gatherVariables(jPoint, side_j.scalarNodes.GetDiffusivity(), iScalar); + const Double D = 0.5 * (rho.i * D_i + rho.j * D_j) + diffTurb; + + const Double projGrad = projectedGradient(opt, grad_i, grad_j, phi_i, phi_j, normal, vector_ij, dist2_ij); + + res.flux_i(iScalar) -= D * projGrad; + if (!opt.oneSided) res.flux_j(iScalar) += D * projGrad; + + if (opt.implicit) { + res.jac_ii(iScalar, iScalar) += D * proj_on_rho_i; + if (!opt.oneSided) { + res.jac_ij(iScalar, iScalar) -= D * proj_on_rho_j; + res.jac_ji(iScalar, iScalar) -= D * proj_on_rho_i; + res.jac_jj(iScalar, iScalar) += D * proj_on_rho_j; + } + } + } + + /*--- Thermal term, on the enthalpy equation alone, driven by the temperature gradient. ---*/ + if (nControlVars <= I_ENTH) return; + + const Double T_i = gatherVariables(iPoint, side_i.flowNodes->GetPrimitive(), idx.Temperature()); + const Double T_j = gatherVariables(jPoint, side_j.flowNodes->GetPrimitive(), idx.Temperature()); + + const auto gradT_i = gatherVariables(iPoint, side_i.flowNodes->GetGradient_Primitive(), idx.Temperature()); + const auto gradT_j = gatherVariables(jPoint, side_j.flowNodes->GetGradient_Primitive(), idx.Temperature()); + + const Double Dth_i = gatherVariables(iPoint, side_i.scalarNodes.GetAuxVar(), I_BETA_ENTH_THERMAL) * + gatherVariables(iPoint, side_i.scalarNodes.GetDiffusivity(), I_ENTH); + const Double Dth_j = gatherVariables(jPoint, side_j.scalarNodes.GetAuxVar(), I_BETA_ENTH_THERMAL) * + gatherVariables(jPoint, side_j.scalarNodes.GetDiffusivity(), I_ENTH); + const Double Dth = 0.5 * (rho.i * Dth_i + rho.j * Dth_j) + diffTurb; + + const Double projGradT = projectedGradient(opt, gradT_i, gradT_j, T_i, T_j, normal, vector_ij, dist2_ij); + + res.flux_i(I_ENTH) -= Dth * projGradT; + if (!opt.oneSided) res.flux_j(I_ENTH) += Dth * projGradT; + } + + private: + const bool preferentialDiffusion; + const unsigned short nControlVars; + + /*! + * \brief Auxiliary variable holding the beta scalar of a controlling variable. + */ + static FORCEINLINE unsigned short betaIndex(unsigned short iScalar) { + switch (iScalar) { + case I_PROGVAR: + return I_BETA_PROGVAR; + case I_ENTH: + return I_BETA_ENTH; + default: + return I_BETA_MIXFRAC; + } + } + + /*! + * \brief Average gradient of one synthesised state projected on the normal, corrected for + * skewness when asked, which is what the ordinary diffusion does for a transported one. + */ + FORCEINLINE Double projectedGradient(const ScalarFluxOptions& opt, const Vector& grad_i, + const Vector& grad_j, const Double& phi_i, const Double& phi_j, + const Vector& normal, const Vector& vector_ij, + const Double& dist2_ij) const { + Vector avgGrad; + for (int iDim = 0; iDim < nDim; ++iDim) avgGrad(iDim) = 0.5 * (grad_i(iDim) + grad_j(iDim)); + + if (opt.correctGradient) { + const Double corr = (dot(avgGrad, vector_ij) - phi_j + phi_i) / dist2_ij; + for (int iDim = 0; iDim < nDim; ++iDim) avgGrad(iDim) -= corr * vector_ij(iDim); + } + return dot(avgGrad, normal); + } +}; diff --git a/SU2_CFD/include/numerics/species/species_convection.hpp b/SU2_CFD/include/numerics/species/species_convection.hpp deleted file mode 100644 index 114501cb5449..000000000000 --- a/SU2_CFD/include/numerics/species/species_convection.hpp +++ /dev/null @@ -1,85 +0,0 @@ -/*! - * \file species_convection.hpp - * \brief Declarations of numerics classes for discretization of - * convective fluxes in species problems. - * \author T. Kattmann - * \version 8.5.0 "Harrier" - * - * SU2 Project Website: https://su2code.github.io - * - * The SU2 Project is maintained by the SU2 Foundation - * (http://su2foundation.org) - * - * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) - * - * SU2 is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * SU2 is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with SU2. If not, see . - */ - -#pragma once - -#include "../scalar/scalar_convection.hpp" - -/*! - * \class CUpwSca_Species - * \brief Class for doing a scalar upwind solver for the species transport equations. - * \ingroup ConvDiscr - */ -template -class CUpwSca_Species final : public CUpwScalar { - private: - using Base = CUpwScalar; - using Base::nVar; - using Base::nDim; - using Base::V_i; - using Base::V_j; - using Base::a0; - using Base::a1; - using Base::Flux; - using Base::Jacobian_i; - using Base::Jacobian_j; - using Base::ScalarVar_i; - using Base::ScalarVar_j; - using Base::idx; - using Base::bounded_scalar; - - /*! - * \brief Adds any extra variables to AD - */ - void ExtraADPreaccIn() override {} - - /*! - * \brief Species transport specific steps in the ComputeResidual method - * \param[in] config - Definition of the particular problem. - */ - void FinishResidualCalc(const CConfig* config) override { - for (auto iVar = 0u; iVar < nVar; iVar++) { - Flux[iVar] = a0 * V_i[idx.Density()] * ScalarVar_i[iVar] + a1 * V_j[idx.Density()] * ScalarVar_j[iVar]; - - /*--- Jacobians are taken wrt rho*Y not Y alone in the species solver. ---*/ - /*--- Off-diagonal entries are zero. ---*/ - Jacobian_i[iVar][iVar] = a0; - Jacobian_j[iVar][iVar] = a1; - } // iVar - } - - public: - /*! - * \brief Constructor of the class. - * \param[in] val_nDim - Number of dimensions of the problem. - * \param[in] val_nVar - Number of variables of the problem. - * \param[in] config - Definition of the particular problem. - */ - CUpwSca_Species(unsigned short val_nDim, unsigned short val_nVar, const CConfig* config) - : CUpwScalar(val_nDim, val_nVar, config) { bounded_scalar = config->GetBounded_Species(); } -}; diff --git a/SU2_CFD/include/numerics/species/species_diffusion.hpp b/SU2_CFD/include/numerics/species/species_diffusion.hpp deleted file mode 100644 index 424c4a47b8ca..000000000000 --- a/SU2_CFD/include/numerics/species/species_diffusion.hpp +++ /dev/null @@ -1,110 +0,0 @@ -/*! - * \file species_diffusion.hpp - * \brief Declarations of numerics classes for discretization of - * viscous fluxes in species problems. - * \author T. Kattmann - * \version 8.5.0 "Harrier" - * - * SU2 Project Website: https://su2code.github.io - * - * The SU2 Project is maintained by the SU2 Foundation - * (http://su2foundation.org) - * - * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) - * - * SU2 is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * SU2 is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with SU2. If not, see . - */ - -#pragma once - -#include "../scalar/scalar_diffusion.hpp" - -/*! - * \class CAvgGrad_Species - * \brief Class for computing viscous term using average of gradients (species transport model). - * \ingroup ViscDiscr - */ -template -class CAvgGrad_Species final : public CAvgGrad_Scalar { - private: - using Base = CAvgGrad_Scalar; - using Base::nVar; - using Base::Eddy_Viscosity_i; - using Base::Eddy_Viscosity_j; - using Base::Diffusion_Coeff_i; - using Base::Diffusion_Coeff_j; - using Base::Density_i; - using Base::Density_j; - using Base::ScalarVar_i; - using Base::ScalarVar_j; - using Base::Proj_Mean_GradScalarVar; - using Base::proj_vector_ij; - using Base::Flux; - using Base::Jacobian_i; - using Base::Jacobian_j; - - const bool turbulence; - - /*! - * \brief Adds any extra variables to AD - */ - void ExtraADPreaccIn(void) override { - AD::SetPreaccIn(Diffusion_Coeff_i, nVar); - AD::SetPreaccIn(Diffusion_Coeff_j, nVar); - } - - /*! - * \brief Species transport specific steps in the ComputeResidual method - * \param[in] config - Definition of the particular problem. - */ - void FinishResidualCalc(const CConfig* config) override { - for (auto iVar = 0u; iVar < nVar; iVar++) { - - const su2double Diffusivity_Lam = 0.5 * (Density_i * Diffusion_Coeff_i[iVar] + Density_j * Diffusion_Coeff_j[iVar]); - - su2double Diffusivity_Turb = 0.0; - - if (turbulence) { - const su2double Sc_t = config->GetSchmidt_Number_Turbulent(); - Diffusivity_Turb = 0.5 * (Eddy_Viscosity_i / Sc_t + Eddy_Viscosity_j / Sc_t); - } - - const su2double Diffusivity = Diffusivity_Lam + Diffusivity_Turb; - - Flux[iVar] = Diffusivity * Proj_Mean_GradScalarVar[iVar]; - - /*--- Use TSL approx. to compute derivatives of the gradients. ---*/ - - /*--- Off-diagonal entries are all zero. ---*/ - const su2double proj_on_rhoi = proj_vector_ij / Density_i; - Jacobian_i[iVar][iVar] = -Diffusivity * proj_on_rhoi; - - const su2double proj_on_rhoj = proj_vector_ij / Density_j; - Jacobian_j[iVar][iVar] = Diffusivity * proj_on_rhoj; - - } // iVar - } - - public: - /*! - * \brief Constructor of the class. - * \param[in] val_nDim - Number of dimensions of the problem. - * \param[in] val_nVar - Number of variables of the problem. - * \param[in] correct_grad - Whether to correct gradient for skewness. - * \param[in] config - Definition of the particular problem. - */ - CAvgGrad_Species(unsigned short val_nDim, unsigned short val_nVar, bool correct_grad, const CConfig* config) - : CAvgGrad_Scalar(val_nDim, val_nVar, correct_grad, config), - turbulence(config->GetKind_Turb_Model() != TURB_MODEL::NONE) {} -}; diff --git a/SU2_CFD/include/numerics/species/species_edge_flux.hpp b/SU2_CFD/include/numerics/species/species_edge_flux.hpp new file mode 100644 index 000000000000..e744d8c96553 --- /dev/null +++ b/SU2_CFD/include/numerics/species/species_edge_flux.hpp @@ -0,0 +1,113 @@ +/*! + * \file species_edge_flux.hpp + * \brief Species transport as a third-layer scalar flux, see numerics/scalar/scalar_edge_flux.hpp. + * \author P. Gomes + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#pragma once + +#include "../scalar/scalar_edge_flux.hpp" + +/*! + * \class CScalarFluxSpeciesBase + * \ingroup ViscDiscr + * \brief Convection and diffusion of a mass fraction, conservative with a diagonal, + * i/j-symmetric diffusion matrix. Unlike SA/SST/LM, the equation count is only known at + * runtime (one per transported species), so this is the framework's first Dynamic-nVar + * model: nEqn is passed to the base explicitly, and coefficients() loops to it rather than + * to a compile-time nVar. + * \note This carries no finalizeFlux of its own: the inherited CUpwScalarFlux one is exactly + * flux(iVar) = a0*rho_i*Y_i(iVar) + a1*rho_j*Y_j(iVar), Conservative weighting by density, + * which is the model's whole convective term. + * \note It takes the most derived class as a parameter so that the flamelet model, which adds a + * preferential diffusion term to the same coefficients, is a sibling rather than a copy. + */ +template +class CScalarFluxSpeciesBase : public CUpwScalarBase { + public: + static constexpr bool Conservative = true; + static constexpr bool DiagonalDiffusion = true; + + using Base = CUpwScalarBase; + using Int = typename Base::Int; + + explicit CScalarFluxSpeciesBase(const CConfig& config) + : Base(config, config.GetnSpecies()), + turbulence(config.GetKind_Turb_Model() != TURB_MODEL::NONE), + Sc_t(config.GetSchmidt_Number_Turbulent()) {} + + /*! + * \brief Diffusion coefficients, an i/j average of (rho * mass diffusivity) per species, plus a + * turbulent (mu_t/Sc_t) contribution shared by every species, when a turbulence model is + * active; identical for both edge sides. + */ + template + FORCEINLINE CPair> coefficients(const FlowIndices& idx, Int iPoint, + const EdgeSide& side_i, Int jPoint, + const EdgeSide& side_j, + const CPair& rho) const { + const Double diffTurb = turbulentDiffusivity(idx, iPoint, side_i, jPoint, side_j); + + Vector D; + for (size_t iVar = 0; iVar < this->nEqn; ++iVar) { + const Double D_lam_i = gatherVariables(iPoint, side_i.scalarNodes.GetDiffusivity(), iVar); + const Double D_lam_j = gatherVariables(jPoint, side_j.scalarNodes.GetDiffusivity(), iVar); + D(iVar) = 0.5 * (rho.i * D_lam_i + rho.j * D_lam_j) + diffTurb; + } + return {D, D}; + } + + protected: + /*! + * \brief Turbulent contribution to the diffusivity, shared by every species and, in the + * flamelet model, by the preferential diffusion terms. + */ + template + FORCEINLINE Double turbulentDiffusivity(const FlowIndices& idx, Int iPoint, const EdgeSide& side_i, + Int jPoint, const EdgeSide& side_j) const { + if (!turbulence) return Double(0.0); + + const Double muT_i = gatherVariables(iPoint, side_i.flowNodes->GetPrimitive(), idx.EddyViscosity()); + const Double muT_j = gatherVariables(jPoint, side_j.flowNodes->GetPrimitive(), idx.EddyViscosity()); + return 0.5 * (muT_i / Sc_t + muT_j / Sc_t); + } + + private: + const bool turbulence; + const su2double Sc_t; +}; + +/*! + * \class CScalarFlux_Species + * \ingroup ViscDiscr + * \brief Convection and diffusion of the species transport model. + */ +template +class CScalarFlux_Species final + : public CScalarFluxSpeciesBase, FlowIndices, nDim, + nVar> { + public: + using Base = CScalarFluxSpeciesBase; + using Base::Base; +}; diff --git a/SU2_CFD/include/numerics/turbulent/transition/trans_diffusion.hpp b/SU2_CFD/include/numerics/turbulent/transition/trans_diffusion.hpp deleted file mode 100644 index 0d4aee5a9472..000000000000 --- a/SU2_CFD/include/numerics/turbulent/transition/trans_diffusion.hpp +++ /dev/null @@ -1,105 +0,0 @@ -/*! - * \file trans_diffusion.hpp - * \brief Declarations of numerics classes for discretization of - * viscous fluxes in transition problems. - * \author S. Kang - * \version 8.5.0 "Harrier" - * - * SU2 Project Website: https://su2code.github.io - * - * The SU2 Project is maintained by the SU2 Foundation - * (http://su2foundation.org) - * - * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) - * - * SU2 is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * SU2 is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with SU2. If not, see . - */ -#pragma once - - -#include "../../scalar/scalar_diffusion.hpp" - -/*! - * \class CAvgGrad_TransLM - * \brief Class for computing viscous term using average of gradient with correction (LM transition model). - * \ingroup ViscDiscr - * \author S. Kang. - */ -template -class CAvgGrad_TransLM final : public CAvgGrad_Scalar { -private: - using Base = CAvgGrad_Scalar; - using Base::Laminar_Viscosity_i; - using Base::Laminar_Viscosity_j; - using Base::Eddy_Viscosity_i; - using Base::Eddy_Viscosity_j; - using Base::Density_i; - using Base::Density_j; - using Base::ScalarVar_i; - using Base::ScalarVar_j; - using Base::Proj_Mean_GradScalarVar; - using Base::proj_vector_ij; - using Base::Flux; - using Base::Jacobian_i; - using Base::Jacobian_j; - - /*! - * \brief Adds any extra variables to AD - */ - void ExtraADPreaccIn() override {} - - /*! - * \brief LM transition model specific steps in the ComputeResidual method - * \param[in] config - Definition of the particular problem. - */ - void FinishResidualCalc(const CConfig* config) override { - const bool implicit = config->GetKind_TimeIntScheme() == EULER_IMPLICIT; - - /*--- Compute mean effective dynamic viscosity ---*/ - const su2double diff_i_gamma = Laminar_Viscosity_i + Eddy_Viscosity_i; - const su2double diff_j_gamma = Laminar_Viscosity_j + Eddy_Viscosity_j; - const su2double diff_i_ReThetaT = 2.0*(Laminar_Viscosity_i + Eddy_Viscosity_i); - const su2double diff_j_ReThetaT = 2.0*(Laminar_Viscosity_j + Eddy_Viscosity_j); - - const su2double diff_gamma = 0.5*(diff_i_gamma + diff_j_gamma); - const su2double diff_ReThetaT = 0.5*(diff_i_ReThetaT + diff_j_ReThetaT); - - Flux[0] = diff_gamma*Proj_Mean_GradScalarVar[0]; - Flux[1] = diff_ReThetaT*Proj_Mean_GradScalarVar[1]; - - /*--- For Jacobians -> Use of TSL (Thin Shear Layer) approx. to compute derivatives of the gradients ---*/ - if (implicit) { - const su2double proj_on_rho_i = proj_vector_ij/Density_i; - Jacobian_i[0][0] = -diff_gamma*proj_on_rho_i; Jacobian_i[0][1] = 0.0; - Jacobian_i[1][0] = 0.0; Jacobian_i[1][1] = -diff_ReThetaT*proj_on_rho_i; - - const su2double proj_on_rho_j = proj_vector_ij/Density_j; - Jacobian_j[0][0] = diff_gamma*proj_on_rho_j; Jacobian_j[0][1] = 0.0; - Jacobian_j[1][0] = 0.0; Jacobian_j[1][1] = diff_ReThetaT*proj_on_rho_j; - } - } - -public: - /*! - * \brief Constructor of the class. - * \param[in] val_nDim - Number of dimensions of the problem. - * \param[in] val_nVar - Number of variables of the problem. - * \param[in] correct_grad - Whether to correct gradient for skewness. - * \param[in] config - Definition of the particular problem. - */ - CAvgGrad_TransLM(unsigned short val_nDim, unsigned short val_nVar, bool correct_grad, const CConfig* config) - : CAvgGrad_Scalar(val_nDim, val_nVar, correct_grad, config){ - } - -}; diff --git a/SU2_CFD/include/numerics/turbulent/transition/trans_edge_flux.hpp b/SU2_CFD/include/numerics/turbulent/transition/trans_edge_flux.hpp new file mode 100644 index 000000000000..927fcb6f5b02 --- /dev/null +++ b/SU2_CFD/include/numerics/turbulent/transition/trans_edge_flux.hpp @@ -0,0 +1,78 @@ +/*! + * \file trans_edge_flux.hpp + * \brief Langtry-Menter transition model as a third-layer scalar flux, see numerics/scalar/scalar_edge_flux.hpp. + * \author P. Gomes + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#pragma once + +#include "../../scalar/scalar_edge_flux.hpp" + +/*! + * \class CScalarFlux_TransLM + * \ingroup ViscDiscr + * \brief Convection and diffusion of the Langtry-Menter transition model, conservative with a + * diagonal, i/j-symmetric diffusion matrix. The coefficients depend only on the flow's + * mu/mu_t, not on the transported gamma/Re_theta, so no coefficientJacobians override + * is needed. + * \note LM writes no finalizeFlux of its own: the inherited CUpwScalarFlux one is exactly + * flux(iVar) = a0*rho_i*phi_i(iVar) + a1*rho_j*phi_j(iVar), Conservative weighting by + * density, which is the model's whole convective term. + */ +template +class CScalarFlux_TransLM + : public CUpwScalarBase, FlowIndices, nDim, nVar> { + public: + static constexpr bool Conservative = true; + static constexpr bool DiagonalDiffusion = true; + + using Base = CUpwScalarBase; + using Int = typename Base::Int; + using Base::Base; + + /*! + * \brief Diffusion coefficients, an i/j average of (mu+mu_t) for intermittency and of + * 2*(mu+mu_t) for the momentum-thickness Reynolds number; identical for both edge sides. + */ + template + FORCEINLINE CPair> coefficients(const FlowIndices& idx, Int iPoint, + const EdgeSide& side_i, Int jPoint, + const EdgeSide& side_j, + const CPair&) const { + const Double mu_i = gatherVariables(iPoint, side_i.flowNodes->GetPrimitive(), idx.LaminarViscosity()); + const Double mu_j = gatherVariables(jPoint, side_j.flowNodes->GetPrimitive(), idx.LaminarViscosity()); + const Double muT_i = gatherVariables(iPoint, side_i.flowNodes->GetPrimitive(), idx.EddyViscosity()); + const Double muT_j = gatherVariables(jPoint, side_j.flowNodes->GetPrimitive(), idx.EddyViscosity()); + + const Double diff_i_gamma = mu_i + muT_i; + const Double diff_j_gamma = mu_j + muT_j; + const Double diff_i_ReThetaT = 2.0 * (mu_i + muT_i); + const Double diff_j_ReThetaT = 2.0 * (mu_j + muT_j); + + Vector D; + D(0) = 0.5 * (diff_i_gamma + diff_j_gamma); + D(1) = 0.5 * (diff_i_ReThetaT + diff_j_ReThetaT); + return {D, D}; + } +}; diff --git a/SU2_CFD/include/numerics/turbulent/turb_convection.hpp b/SU2_CFD/include/numerics/turbulent/turb_convection.hpp deleted file mode 100644 index 6c1641db87d3..000000000000 --- a/SU2_CFD/include/numerics/turbulent/turb_convection.hpp +++ /dev/null @@ -1,141 +0,0 @@ -/*! - * \file turb_convection.hpp - * \brief Declarations of numerics classes for discretization of - * convective fluxes in turbulence problems. - * \author F. Palacios, T. Economon - * \version 8.5.0 "Harrier" - * - * SU2 Project Website: https://su2code.github.io - * - * The SU2 Project is maintained by the SU2 Foundation - * (http://su2foundation.org) - * - * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) - * - * SU2 is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * SU2 is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with SU2. If not, see . - */ - -#pragma once - -#include "../scalar/scalar_convection.hpp" - -/*! - * \class CUpwSca_TurbSA - * \brief Class for doing a scalar upwind solver for the Spalar-Allmaras turbulence model equations. - * \ingroup ConvDiscr - * \author A. Bueno. - */ -template -class CUpwSca_TurbSA final : public CUpwScalar { -private: - using Base = CUpwScalar; - using Base::a0; - using Base::a1; - using Base::Flux; - using Base::Jacobian_i; - using Base::Jacobian_j; - using Base::ScalarVar_i; - using Base::ScalarVar_j; - using Base::bounded_scalar; - using Base::V_i; - using Base::V_j; - using Base::idx; - using Base::nVar; - - /*! - * \brief Adds any extra variables to AD. - */ - void ExtraADPreaccIn() override {} - - /*! - * \brief SA specific steps in the ComputeResidual method - * \param[in] config - Definition of the particular problem. - */ - void FinishResidualCalc(const CConfig* config) override { - if (config->GetSBSParam().StochasticBackscatter && config->GetSBSParam().SBS_Ctau > 0.0) { - for (unsigned short iVar = 1; iVar < nVar; iVar++) { - Flux[iVar] = (a0 + a1) * 0.5 * (ScalarVar_i[iVar] + ScalarVar_j[iVar]); - Jacobian_i[iVar][iVar] = 0.5 * (a0+a1); - Jacobian_j[iVar][iVar] = 0.5 * (a0+a1); - } - } - Flux[0] = a0*ScalarVar_i[0] + a1*ScalarVar_j[0]; - Jacobian_i[0][0] = a0; - Jacobian_j[0][0] = a1; - } - -public: - /*! - * \brief Constructor of the class. - * \param[in] val_nDim - Number of dimensions of the problem. - * \param[in] val_nVar - Number of variables of the problem. - * \param[in] config - Definition of the particular problem. - */ - CUpwSca_TurbSA(unsigned short val_nDim, unsigned short val_nVar, const CConfig* config) - : CUpwScalar(val_nDim, val_nVar, config) { bounded_scalar = config->GetBounded_Turb(); } -}; - -/*! - * \class CUpwSca_TurbSST - * \brief Class for doing a scalar upwind solver for the Menter SST turbulence model equations. - * \ingroup ConvDiscr - * \author A. Campos. - */ -template -class CUpwSca_TurbSST final : public CUpwScalar { -private: - using Base = CUpwScalar; - using Base::nDim; - using Base::V_i; - using Base::V_j; - using Base::a0; - using Base::a1; - using Base::Flux; - using Base::Jacobian_i; - using Base::Jacobian_j; - using Base::ScalarVar_i; - using Base::ScalarVar_j; - using Base::idx; - using Base::bounded_scalar; - - /*! - * \brief Adds any extra variables to AD - */ - void ExtraADPreaccIn() override {} - - /*! - * \brief SST specific steps in the ComputeResidual method - * \param[in] config - Definition of the particular problem. - */ - void FinishResidualCalc(const CConfig* config) override { - Flux[0] = a0*V_i[idx.Density()]*ScalarVar_i[0] + a1*V_j[idx.Density()]*ScalarVar_j[0]; - Flux[1] = a0*V_i[idx.Density()]*ScalarVar_i[1] + a1*V_j[idx.Density()]*ScalarVar_j[1]; - - Jacobian_i[0][0] = a0; Jacobian_i[0][1] = 0.0; - Jacobian_i[1][0] = 0.0; Jacobian_i[1][1] = a0; - - Jacobian_j[0][0] = a1; Jacobian_j[0][1] = 0.0; - Jacobian_j[1][0] = 0.0; Jacobian_j[1][1] = a1; - } - -public: - /*! - * \brief Constructor of the class. - * \param[in] val_nDim - Number of dimensions of the problem. - * \param[in] val_nVar - Number of variables of the problem. - * \param[in] config - Definition of the particular problem. - */ - CUpwSca_TurbSST(unsigned short val_nDim, unsigned short val_nVar, const CConfig* config) - : CUpwScalar(val_nDim, val_nVar, config) { bounded_scalar = config->GetBounded_Turb(); } -}; diff --git a/SU2_CFD/include/numerics/turbulent/turb_diffusion.hpp b/SU2_CFD/include/numerics/turbulent/turb_diffusion.hpp deleted file mode 100644 index a4b2bbe264b0..000000000000 --- a/SU2_CFD/include/numerics/turbulent/turb_diffusion.hpp +++ /dev/null @@ -1,346 +0,0 @@ -/*! - * \file turb_diffusion.hpp - * \brief Declarations of numerics classes for discretization of - * viscous fluxes in turbulence problems. - * \author F. Palacios, T. Economon - * \version 8.5.0 "Harrier" - * - * SU2 Project Website: https://su2code.github.io - * - * The SU2 Project is maintained by the SU2 Foundation - * (http://su2foundation.org) - * - * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) - * - * SU2 is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * SU2 is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with SU2. If not, see . - */ -#pragma once - -#include "../scalar/scalar_diffusion.hpp" - -/*! - * \class CAvgGrad_TurbSA - * \brief Class for computing viscous term using average of gradients (Spalart-Allmaras Turbulence model). - * \ingroup ViscDiscr - * \author A. Bueno. - */ -template -class CAvgGrad_TurbSA final : public CAvgGrad_Scalar { -private: - using Base = CAvgGrad_Scalar; - using Base::Laminar_Viscosity_i; - using Base::Laminar_Viscosity_j; - using Base::Density_i; - using Base::Density_j; - using Base::ScalarVar_i; - using Base::ScalarVar_j; - using Base::Proj_Mean_GradScalarVar; - using Base::proj_vector_ij; - using Base::Flux; - using Base::Jacobian_i; - using Base::Jacobian_j; - - const su2double sigma = 2.0/3.0; - const su2double cb2 = 0.622; - - const bool use_accurate_jacobians; - - /*! - * \brief Adds any extra variables to AD - */ - void ExtraADPreaccIn() override {} - - /*! - * \brief SA specific steps in the ComputeResidual method - * \param[in] config - Definition of the particular problem. - */ - void FinishResidualCalc(const CConfig* config) override { - const bool implicit = config->GetKind_TimeIntScheme() == EULER_IMPLICIT; - - /*--- Compute mean effective viscosity ---*/ - - /*--- First Term. Normal diffusion, and conservative part of the quadratic diffusion. - * ||grad nu_t||^2 = div(nu_t grad nu_t) - nu_t div grad nu_t ---*/ - const su2double nu_i = Laminar_Viscosity_i/Density_i; - const su2double nu_j = Laminar_Viscosity_j/Density_j; - const su2double nu_e = 0.5 * (nu_i + nu_j + (1 + cb2) * (ScalarVar_i[0] + ScalarVar_j[0])); - const su2double term_1 = nu_e; - - /* Second Term (quadratic diffusion, non conservative). */ - const su2double nu_tilde_i = ScalarVar_i[0]; - const su2double term_2 = cb2 * nu_tilde_i; - - const su2double diffusion_coefficient = term_1 - term_2; - Flux[0] = diffusion_coefficient * Proj_Mean_GradScalarVar[0] / sigma; - - if (implicit) { - /*--- For Jacobians -> Use of TSL approx. to compute derivatives of the gradients ---*/ - Jacobian_i[0][0] = -diffusion_coefficient * proj_vector_ij / sigma; - Jacobian_j[0][0] = diffusion_coefficient * proj_vector_ij / sigma; - - if (use_accurate_jacobians) { - /*--- The diffusion coefficient is also a function of nu_t. ---*/ - const su2double dTerm1_dnut_i = (1 + cb2) * 0.5; - const su2double dTerm1_dnut_j = (1 + cb2) * 0.5; - - const su2double dTerm2_dnut_i = cb2; - const su2double dTerm2_dnut_j = 0.0; - - const su2double dDC_dnut_i = dTerm1_dnut_i - dTerm2_dnut_i; - const su2double dDC_dnut_j = dTerm1_dnut_j - dTerm2_dnut_j; - - Jacobian_i[0][0] += dDC_dnut_i * Proj_Mean_GradScalarVar[0] / sigma; - Jacobian_j[0][0] += dDC_dnut_j * Proj_Mean_GradScalarVar[0] / sigma; - } - } - } - -public: - /*! - * \brief Constructor of the class. - * \param[in] val_nDim - Number of dimensions of the problem. - * \param[in] val_nVar - Number of variables of the problem. - * \param[in] correct_grad - Whether to correct gradient for skewness. - * \param[in] config - Definition of the particular problem. - */ - CAvgGrad_TurbSA(unsigned short val_nDim, unsigned short val_nVar, - bool correct_grad, const CConfig* config) - : CAvgGrad_Scalar(val_nDim, val_nVar, correct_grad, config), - use_accurate_jacobians(config->GetUse_Accurate_Turb_Jacobians()) {} -}; - -/*! - * \class CAvgGrad_TurbSA_Neg - * \brief Class for computing viscous term using average of gradients (Spalart-Allmaras Turbulence model). - * \ingroup ViscDiscr - * \author F. Palacios - */ -template -class CAvgGrad_TurbSA_Neg final : public CAvgGrad_Scalar { -private: - using Base = CAvgGrad_Scalar; - using Base::Laminar_Viscosity_i; - using Base::Laminar_Viscosity_j; - using Base::Density_i; - using Base::Density_j; - using Base::ScalarVar_i; - using Base::ScalarVar_j; - using Base::Proj_Mean_GradScalarVar; - using Base::proj_vector_ij; - using Base::Flux; - using Base::Jacobian_i; - using Base::Jacobian_j; - - const su2double sigma = 2.0/3.0; - const su2double cn1 = 16.0; - const su2double cb2 = 0.622; - - /*! - * \brief Adds any extra variables to AD - */ - void ExtraADPreaccIn() override {} - - /*! - * \brief SA-neg specific steps in the ComputeResidual method - * \param[in] config - Definition of the particular problem. - */ - void FinishResidualCalc(const CConfig* config) override { - const bool implicit = config->GetKind_TimeIntScheme() == EULER_IMPLICIT; - - /*--- Compute mean effective viscosity ---*/ - - const su2double nu_i = Laminar_Viscosity_i/Density_i; - const su2double nu_j = Laminar_Viscosity_j/Density_j; - - const su2double nu_ij = 0.5 * (nu_i + nu_j); - const su2double nu_tilde_i = ScalarVar_i[0]; - const su2double nu_tilde_j = ScalarVar_j[0]; - const su2double nu_tilde_ij = 0.5 * (nu_tilde_i + nu_tilde_j); - - /*--- Following Diskin's implementation from 10.2514/1.J064629, they propose a new fn function - * to be evaluated at the cell to maintain positivity in the diffusion coefficient, which is - * used in both terms. The new fn term averaged across the face reverts to the original fn - * function. ---*/ - - /*--- Second Term (LHS) ---*/ - const su2double zeta_i = ((1 + cb2) * nu_tilde_ij - cb2 * nu_tilde_i) / nu_ij; - su2double fn_i = 1.0; - if (zeta_i < 0.0) { - fn_i = (cn1 + pow(zeta_i,3)) / (cn1 - pow(zeta_i,3)); - } - - const su2double term_1 = (nu_ij + (1 + cb2) * nu_tilde_ij * fn_i); - const su2double term_2 = cb2 * nu_tilde_i * fn_i; - Flux[0] = (term_1 - term_2) * Proj_Mean_GradScalarVar[0] / sigma; - - /*--- For Jacobians -> Use of TSL approx. to compute derivatives of the gradients - * Exact Jacobians were tested on multiple cases but resulted in divergence of all - * simulations, hence only frozen diffusion coefficient (approximate) Jacobians are used. ---*/ - - if (implicit) { - const su2double diffusion_coefficient = (term_1 - term_2); - - const su2double dGrad_dnut_i = -proj_vector_ij; - const su2double dGrad_dnut_j = proj_vector_ij; - - Jacobian_i[0][0] = diffusion_coefficient * dGrad_dnut_i / sigma; - Jacobian_j[0][0] = diffusion_coefficient * dGrad_dnut_j / sigma; - } - } - -public: - /*! - * \brief Constructor of the class. - * \param[in] val_nDim - Number of dimensions of the problem. - * \param[in] val_nVar - Number of variables of the problem. - * \param[in] correct_grad - Whether to correct gradient for skewness. - * \param[in] config - Definition of the particular problem. - */ - CAvgGrad_TurbSA_Neg(unsigned short val_nDim, unsigned short val_nVar, - bool correct_grad, const CConfig* config) - : CAvgGrad_Scalar(val_nDim, val_nVar, correct_grad, config) {} -}; - -/*! - * \class CAvgGrad_TurbSST - * \brief Class for computing viscous term using average of gradient with correction (Menter SST turbulence model). - * \ingroup ViscDiscr - * \author A. Bueno. - */ -template -class CAvgGrad_TurbSST final : public CAvgGrad_Scalar { -private: - using Base = CAvgGrad_Scalar; - using Base::Laminar_Viscosity_i; - using Base::Laminar_Viscosity_j; - using Base::Eddy_Viscosity_i; - using Base::Eddy_Viscosity_j; - using Base::Density_i; - using Base::Density_j; - using Base::ScalarVar_i; - using Base::ScalarVar_j; - using Base::Proj_Mean_GradScalarVar; - using Base::proj_vector_ij; - using Base::Flux; - using Base::Jacobian_i; - using Base::Jacobian_j; - - const su2double sigma_k1; /*!< \brief Constants for the viscous terms, k-w (1), k-eps (2)*/ - const su2double sigma_k2; - const su2double sigma_om1; - const su2double sigma_om2; - const bool use_accurate_jacobians; - - su2double F1_i, F1_j; /*!< \brief Menter's first blending function */ - - /*! - * \brief Adds any extra variables to AD - */ - void ExtraADPreaccIn() override { - AD::SetPreaccIn(F1_i, F1_j); - } - - /*! - * \brief SST specific steps in the ComputeResidual method - * \param[in] config - Definition of the particular problem. - */ - void FinishResidualCalc(const CConfig* config) override { - const bool implicit = config->GetKind_TimeIntScheme() == EULER_IMPLICIT; - - /*--- Compute the blended constant for the viscous terms ---*/ - const su2double sigma_kine_i = F1_i*sigma_k1 + (1.0 - F1_i)*sigma_k2; - const su2double sigma_kine_j = F1_j*sigma_k1 + (1.0 - F1_j)*sigma_k2; - const su2double sigma_omega_i = F1_i*sigma_om1 + (1.0 - F1_i)*sigma_om2; - const su2double sigma_omega_j = F1_j*sigma_om1 + (1.0 - F1_j)*sigma_om2; - - /*--- Compute mean effective dynamic viscosity ---*/ - const su2double diff_i_kine = Laminar_Viscosity_i + sigma_kine_i*Eddy_Viscosity_i; - const su2double diff_j_kine = Laminar_Viscosity_j + sigma_kine_j*Eddy_Viscosity_j; - const su2double diff_i_omega = Laminar_Viscosity_i + sigma_omega_i*Eddy_Viscosity_i; - const su2double diff_j_omega = Laminar_Viscosity_j + sigma_omega_j*Eddy_Viscosity_j; - - const su2double diff_kine = 0.5*(diff_i_kine + diff_j_kine); - const su2double diff_omega_T1 = 0.5*(diff_i_omega + diff_j_omega); - - /*--- We aim to treat the cross-diffusion as a diffusion term rather than a source term. - * Re-writing the cross-diffusion contribution as λ/w ∇w ∇k, where λ = (2 (1- F1) ρ σ_ω2) - * and expanding using the product rule for divergence theorem gives: ∇(w λ/w ∇k) - w ∇(λ/w ∇k). - * Discretising using FVM, gives: (λ)_ij ∇k - w_c (λ/w)_ij ∇k. where w_c is the cell centre value ---*/ - - const su2double lambda_i = 2 * (1 - F1_i) * Density_i * sigma_omega_i; - const su2double lambda_j = 2 * (1 - F1_j) * Density_j * sigma_omega_j; - const su2double lambda_ij = 0.5 * (lambda_i + lambda_j); - const su2double w_ij = 0.5 * (ScalarVar_i[1] + ScalarVar_j[1]); - - const su2double diff_omega_T2 = lambda_ij; - - const su2double diff_omega_T3 = -ScalarVar_i[1] * lambda_ij/w_ij; - - Flux[0] = diff_kine*Proj_Mean_GradScalarVar[0]; - Flux[1] = diff_omega_T1*Proj_Mean_GradScalarVar[1] + (diff_omega_T2 + diff_omega_T3)*Proj_Mean_GradScalarVar[0]; - - /*--- For Jacobians -> Use of TSL (Thin Shear Layer) approx. to compute derivatives of the gradients ---*/ - if (implicit) { - const su2double proj_on_rho_i = proj_vector_ij/Density_i; - const su2double proj_on_rho_j = proj_vector_ij/Density_j; - Jacobian_i[0][0] = -diff_kine*proj_on_rho_i; - Jacobian_i[0][1] = 0.0; - Jacobian_i[1][0] = (diff_omega_T2+diff_omega_T3)*-proj_on_rho_i; - Jacobian_i[1][1] = -diff_omega_T1*proj_on_rho_i; - - Jacobian_j[0][0] = diff_kine*proj_on_rho_j; - Jacobian_j[0][1] = 0.0; - Jacobian_j[1][0] = (diff_omega_T2+diff_omega_T3)*proj_on_rho_j; - Jacobian_j[1][1] = diff_omega_T1*proj_on_rho_j; - - if (use_accurate_jacobians) { - Jacobian_i[0][0] = -diff_kine*proj_on_rho_i; - Jacobian_i[0][1] = 0.0; - Jacobian_i[1][0] = (diff_omega_T2 + diff_omega_T3)*-proj_on_rho_i; - Jacobian_i[1][1] = -proj_on_rho_i * diff_omega_T1 - 2*lambda_ij*ScalarVar_j[1]/pow(ScalarVar_i[1]+ScalarVar_j[1],2) * Proj_Mean_GradScalarVar[0]; - - Jacobian_j[0][0] = diff_kine*proj_on_rho_j; - Jacobian_j[0][1] = 0.0; - Jacobian_j[1][0] = (diff_omega_T2 + diff_omega_T3)*proj_on_rho_j; - Jacobian_j[1][1] = proj_on_rho_j * diff_omega_T1 + 2*lambda_ij*ScalarVar_i[1]/pow(ScalarVar_i[1]+ScalarVar_j[1],2) * Proj_Mean_GradScalarVar[0]; - } - } - } - -public: - /*! - * \brief Constructor of the class. - * \param[in] val_nDim - Number of dimensions of the problem. - * \param[in] val_nVar - Number of variables of the problem. - * \param[in] constants - Constants of the model. - * \param[in] correct_grad - Whether to correct gradient for skewness. - * \param[in] config - Definition of the particular problem. - */ - CAvgGrad_TurbSST(unsigned short val_nDim, unsigned short val_nVar, - const su2double* constants, bool correct_grad, const CConfig* config) - : CAvgGrad_Scalar(val_nDim, val_nVar, correct_grad, config), - sigma_k1(constants[0]), - sigma_k2(constants[1]), - sigma_om1(constants[2]), - sigma_om2(constants[3]), - use_accurate_jacobians(config->GetUse_Accurate_Turb_Jacobians()) { - } - - /*! - * \brief Sets value of first blending function. - */ - void SetF1blending(su2double val_F1_i, su2double val_F1_j) override { - F1_i = val_F1_i; F1_j = val_F1_j; - } -}; diff --git a/SU2_CFD/include/numerics/turbulent/turb_sa_edge_flux.hpp b/SU2_CFD/include/numerics/turbulent/turb_sa_edge_flux.hpp new file mode 100644 index 000000000000..978b0aaaf583 --- /dev/null +++ b/SU2_CFD/include/numerics/turbulent/turb_sa_edge_flux.hpp @@ -0,0 +1,183 @@ +/*! + * \file turb_sa_edge_flux.hpp + * \brief Spalart-Allmaras model as a third-layer scalar flux, see numerics/scalar/scalar_edge_flux.hpp. + * \author P. Gomes + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#pragma once + +#include "../scalar/scalar_edge_flux.hpp" + +/*! + * \class CScalarFlux_SA + * \ingroup ConvDiscr + * \ingroup ViscDiscr + * \brief Convection and diffusion of the Spalart-Allmaras model, non-conservative and with a + * diagonal (but asymmetric) diffusion coefficient. + * \note SA writes its own convective term rather than using the inherited CUpwScalarFlux one, + * because with stochastic backscatter active (nVar 4) the three Langevin equations are + * advected with a centered flux, unlike the plain upwind SA equation itself. + */ +template +class CScalarFlux_SA + : public CUpwScalarBase, FlowIndices, nDim, nVar> { + public: + static constexpr bool Conservative = false; + static constexpr bool DiagonalDiffusion = true; + static constexpr bool DiffusionReadsDensity = true; /*!< \brief The kinematic viscosities below. */ + + using Base = CUpwScalarBase; + using Int = typename Base::Int; + + explicit CScalarFlux_SA(const CConfig& config) + : Base(config), + negativeSA(config.GetSAParsedOptions().version == SA_OPTIONS::NEG), + accurateJacobians(config.GetUse_Accurate_Turb_Jacobians()) {} + + private: + static constexpr passivedouble sigma = 2.0 / 3.0; /*!< \brief Constant of the diffusion term. */ + static constexpr passivedouble cb2 = 0.622; /*!< \brief Constant of the diffusion term. */ + static constexpr passivedouble cn1 = 16.0; /*!< \brief Constant of the SA-neg diffusion correction. */ + + /*!< \brief Whether nu_tilde may go negative (SA_OPTIONS= NEGATIVE), which needs the fn-corrected + * diffusion coefficient below to keep the diffusion term from turning anti-diffusive. */ + const bool negativeSA; + const bool accurateJacobians; + + public: + /*! + * \brief SA convection, plus the centered advection of the backscatter equations when nVar > 1. + */ + template + FORCEINLINE void finalizeFlux(const FlowIndices&, const ScalarFluxOptions& opt, Int, const EdgeSide&, + Int, const EdgeSide&, const Double& a0, const Double& a1, + const CPair&, const CPair>& phi, + EdgeResidual& res) const { + const Double flux = a0 * phi.i.all(0) + a1 * phi.j.all(0); + + res.flux_i(0) += flux; + if (!opt.oneSided) res.flux_j(0) -= flux; + + if (opt.implicit) { + res.jac_ii(0, 0) += a0; + if (!opt.oneSided) { + res.jac_ij(0, 0) += a1; + res.jac_ji(0, 0) -= a0; + res.jac_jj(0, 0) -= a1; + } + } + + /*--- Stochastic backscatter: three Langevin equations, advected with the mean of the two + * upwinding weights and with no diffusion. ---*/ + const Double avg = 0.5 * (a0 + a1); + for (size_t iVar = 1; iVar < res.nVar; ++iVar) { + const Double flux_bs = avg * (phi.i.all(iVar) + phi.j.all(iVar)); + + res.flux_i(iVar) += flux_bs; + if (!opt.oneSided) res.flux_j(iVar) -= flux_bs; + + if (opt.implicit) { + res.jac_ii(iVar, iVar) += avg; + if (!opt.oneSided) { + res.jac_ij(iVar, iVar) += avg; + res.jac_ji(iVar, iVar) -= avg; + res.jac_jj(iVar, iVar) -= avg; + } + } + } + } + + /*! + * \brief fn, the positivity-preserving correction to the SA-neg diffusion coefficient + * (Allmaras, Johnson & Spalart), 1 when nu_tilde is not negative enough to need it. + */ + FORCEINLINE Double fn(const Double& zeta) const { + if (!negativeSA || zeta >= 0.0) return 1.0; + const Double zeta3 = zeta * zeta * zeta; + return (cn1 + zeta3) / (cn1 - zeta3); + } + + /*! + * \brief Diffusion coefficients of both orientations of the edge. + * \note The coefficient is not symmetric: it uses the transported variable of the row it is + * going to be used for (the quadratic, non-conservative part of the diffusion term). + * Coefficients past index 0 are left at zero, the backscatter equations have no diffusion. + */ + template + FORCEINLINE CPair> coefficients(const FlowIndices& idx, Int iPoint, + const EdgeSide& side_i, Int jPoint, + const EdgeSide& side_j, + const CPair& rho) const { + const Double nu_i = gatherVariables(iPoint, side_i.flowNodes->GetPrimitive(), idx.LaminarViscosity()) / rho.i; + const Double nu_j = gatherVariables(jPoint, side_j.flowNodes->GetPrimitive(), idx.LaminarViscosity()) / rho.j; + + const Double nuTilde_i = gatherVariables(iPoint, side_i.scalarNodes.GetSolution(), 0); + const Double nuTilde_j = gatherVariables(jPoint, side_j.scalarNodes.GetSolution(), 0); + + const Double nu_ij = 0.5 * (nu_i + nu_j); + const Double nuTilde_ij = 0.5 * (nuTilde_i + nuTilde_j); + + /*--- fn is only ever != 1 under SA_OPTIONS= NEGATIVE, and then only where the row's own + * nu_tilde pulls the coefficient negative; without it (nu + nu_tilde going anti-diffusive) + * the equation diverges, see Allmaras, Johnson & Spalart's negative SA modification. ---*/ + const Double fn_i = fn(((1.0 + cb2) * nuTilde_ij - cb2 * nuTilde_i) / nu_ij); + const Double fn_j = fn(((1.0 + cb2) * nuTilde_ij - cb2 * nuTilde_j) / nu_ij); + + Vector D_i, D_j; + D_i(0) = (nu_ij + (1.0 + cb2) * nuTilde_ij * fn_i - cb2 * nuTilde_i * fn_i) / sigma; + D_j(0) = (nu_ij + (1.0 + cb2) * nuTilde_ij * fn_j - cb2 * nuTilde_j * fn_j) / sigma; + for (size_t iVar = 1; iVar < nVar; ++iVar) { + D_i(iVar) = 0.0; + D_j(iVar) = 0.0; + } + return {D_i, D_j}; + } + + /*! + * \brief Extra Jacobian terms from the dependence of the diffusion coefficient on nu_tilde. + * \note Skipped for SA-neg, whose fn-corrected coefficient was found (upstream, pre-migration) + * to diverge with exact Jacobians; frozen (TSL-only) Jacobians are used there instead. + * Skipped for standard SA too unless USE_ACCURATE_TURB_JACOBIANS is set, matching the + * pre-migration default of using frozen Jacobians there as well. + */ + template + FORCEINLINE void coefficientJacobians(const ScalarFluxOptions& opt, const Coefficients&, + const Vector& projGrad, EdgeResidual& res) const { + if (negativeSA || !accurateJacobians) return; + + /*--- d(diffusion coefficient of i)/d(nu_tilde_i), and its counterpart w.r.t. nu_tilde_j; + * the coefficient of j is the same expression with i and j swapped, so the same two + * derivatives apply to both orientations. Both are per-edge constants, so the coefficients + * themselves are not read here. ---*/ + const Double dDC_dNuTilde_i = ((1.0 + cb2) * 0.5 - cb2) / sigma; + const Double dDC_dNuTilde_j = (1.0 + cb2) * 0.5 / sigma; + + res.jac_ii(0, 0) -= dDC_dNuTilde_i * projGrad(0); + if (opt.oneSided) return; + + res.jac_ij(0, 0) -= dDC_dNuTilde_j * projGrad(0); + res.jac_ji(0, 0) += dDC_dNuTilde_j * projGrad(0); + res.jac_jj(0, 0) += dDC_dNuTilde_i * projGrad(0); + } +}; diff --git a/SU2_CFD/include/numerics/turbulent/turb_sources.hpp b/SU2_CFD/include/numerics/turbulent/turb_sources.hpp index 2c0cb51a8d21..9f72b6e37cc4 100644 --- a/SU2_CFD/include/numerics/turbulent/turb_sources.hpp +++ b/SU2_CFD/include/numerics/turbulent/turb_sources.hpp @@ -1014,7 +1014,7 @@ class CSourcePieceWise_TurbSST final : public CNumerics { Residual[0] -= dk * Volume; Residual[1] -= dw * Volume; - /*--- Cross diffusion is included in the viscous fluxes, discretisation in turb_diffusion.hpp ---*/ + /*--- Cross diffusion is included in the viscous fluxes, not this source term. ---*/ /*--- Contribution due to 2D axisymmetric formulation ---*/ diff --git a/SU2_CFD/include/numerics/turbulent/turb_sst_edge_flux.hpp b/SU2_CFD/include/numerics/turbulent/turb_sst_edge_flux.hpp new file mode 100644 index 000000000000..b891b6e14d5c --- /dev/null +++ b/SU2_CFD/include/numerics/turbulent/turb_sst_edge_flux.hpp @@ -0,0 +1,147 @@ +/*! + * \file turb_sst_edge_flux.hpp + * \brief Menter SST model as a third-layer scalar flux, see numerics/scalar/scalar_edge_flux.hpp. + * \author P. Gomes + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#pragma once + +#include "../scalar/scalar_edge_flux.hpp" + +/*! + * \class CScalarFlux_SST + * \ingroup ViscDiscr + * \brief Convection and diffusion of the Menter SST model, conservative with a coupled (but + * neither symmetric nor diagonal) 2x2 diffusion matrix. + * \note SST writes no finalizeFlux of its own: the inherited CUpwScalarFlux one is exactly + * flux(iVar) = a0*rho_i*phi_i(iVar) + a1*rho_j*phi_j(iVar), Conservative weighting by + * density, which is the model's whole convective term. + */ +template +class CScalarFlux_SST + : public CUpwScalarBase, FlowIndices, nDim, nVar> { + public: + static constexpr bool Conservative = true; + static constexpr bool DiagonalDiffusion = false; + + using Base = CUpwScalarBase; + using Int = typename Base::Int; + using Base::Base; + + private: + /*--- Fixed regardless of SST_OPTIONS::version: only the production-limiter and source-term + * constants (alfa/gamma) differ by version, not these. ---*/ + static constexpr passivedouble sigma_k1 = 0.85; + static constexpr passivedouble sigma_k2 = 1.0; + static constexpr passivedouble sigma_om1 = 0.5; + static constexpr passivedouble sigma_om2 = 0.856; + + public: + /*! + * \brief Diffusion coefficients of both orientations of the edge, and the terms of the cross + * diffusion that the Jacobian correction below needs, so that neither the gathers nor + * the blending are repeated for it. + * \note The cross term reads the transported omega of whichever point its row is being written + * for, so it is not symmetric: i, read by i's row, uses omega at i; j, read by j's row, + * uses omega at j. Every other entry is an i/j average, so it is the same in both. + */ + struct CCoefficients { + Matrix i, j; + Double lambda_ij, omega_i, omega_j; + }; + + template + FORCEINLINE CCoefficients coefficients(const FlowIndices& idx, Int iPoint, const EdgeSide& side_i, + Int jPoint, const EdgeSide& side_j, + const CPair& rho) const { + const Double mu_i = gatherVariables(iPoint, side_i.flowNodes->GetPrimitive(), idx.LaminarViscosity()); + const Double mu_j = gatherVariables(jPoint, side_j.flowNodes->GetPrimitive(), idx.LaminarViscosity()); + const Double muT_i = gatherVariables(iPoint, side_i.flowNodes->GetPrimitive(), idx.EddyViscosity()); + const Double muT_j = gatherVariables(jPoint, side_j.flowNodes->GetPrimitive(), idx.EddyViscosity()); + + const Double F1_i = gatherVariables(iPoint, side_i.scalarNodes.GetF1blending()); + const Double F1_j = gatherVariables(jPoint, side_j.scalarNodes.GetF1blending()); + + CCoefficients D; + D.omega_i = gatherVariables(iPoint, side_i.scalarNodes.GetSolution(), 1); + D.omega_j = gatherVariables(jPoint, side_j.scalarNodes.GetSolution(), 1); + + const Double sigma_kine_i = F1_i * sigma_k1 + (1.0 - F1_i) * sigma_k2; + const Double sigma_kine_j = F1_j * sigma_k1 + (1.0 - F1_j) * sigma_k2; + const Double sigma_omega_i = F1_i * sigma_om1 + (1.0 - F1_i) * sigma_om2; + const Double sigma_omega_j = F1_j * sigma_om1 + (1.0 - F1_j) * sigma_om2; + + const Double diff_kine = 0.5 * ((mu_i + sigma_kine_i * muT_i) + (mu_j + sigma_kine_j * muT_j)); + const Double diff_omega = 0.5 * ((mu_i + sigma_omega_i * muT_i) + (mu_j + sigma_omega_j * muT_j)); + + const Double lambda_i = 2.0 * (1.0 - F1_i) * rho.i * sigma_omega_i; + const Double lambda_j = 2.0 * (1.0 - F1_j) * rho.j * sigma_omega_j; + D.lambda_ij = 0.5 * (lambda_i + lambda_j); + const Double w_ij = 0.5 * (D.omega_i + D.omega_j); + + /*--- Cross-diffusion coefficient: a divergence-theorem term (diff_omega_T2) plus a cell + * centre correction (diff_omega_T3) that reads the transported omega of the row's own point. ---*/ + const Double diff_omega_T2 = D.lambda_ij; + const Double diff_omega_T3_i = -D.omega_i * D.lambda_ij / w_ij; + const Double diff_omega_T3_j = -D.omega_j * D.lambda_ij / w_ij; + + /*--- D.i(0,1) and D.j(0,1) are left zero: there is no diffusive coupling from omega into + * the k row. ---*/ + D.i = Double(0.0); + D.j = Double(0.0); + D.i(0, 0) = diff_kine; + D.i(1, 1) = diff_omega; + D.i(1, 0) = diff_omega_T2 + diff_omega_T3_i; + + D.j(0, 0) = diff_kine; + D.j(1, 1) = diff_omega; + D.j(1, 0) = diff_omega_T2 + diff_omega_T3_j; + + return D; + } + + /*! + * \brief Extra Jacobian terms from the dependence of the cross-diffusion coefficient on omega. + * \note diff_omega_T3_i and diff_omega_T3_j both depend on omega_i and omega_j through w_ij, so + * each of the four blocks needs a correction beyond the one diffusionTerms already applies + * through projGrad. The correction only depends on which point's omega is being + * differentiated against, not on which row it lands in: differentiating against omega_i + * gives +E_j in both jac_ii and jac_ji, differentiating against omega_j gives -E_i in both + * jac_ij and jac_jj. + */ + template + FORCEINLINE void coefficientJacobians(const ScalarFluxOptions& opt, const CCoefficients& D, + const Vector& projGrad, EdgeResidual& res) const { + const Double denom = pow(D.omega_i + D.omega_j, 2.0); + const Double E_i = 2.0 * D.lambda_ij * D.omega_i / denom * projGrad(0); + const Double E_j = 2.0 * D.lambda_ij * D.omega_j / denom * projGrad(0); + + res.jac_ii(1, 1) += E_j; + if (opt.oneSided) return; + + res.jac_ij(1, 1) -= E_i; + res.jac_ji(1, 1) += E_j; + res.jac_jj(1, 1) -= E_i; + } +}; diff --git a/SU2_CFD/include/numerics/util.hpp b/SU2_CFD/include/numerics/util.hpp new file mode 100644 index 000000000000..92e5144b9c93 --- /dev/null +++ b/SU2_CFD/include/numerics/util.hpp @@ -0,0 +1,638 @@ +/*! + * \file util.hpp + * \brief Generic auxiliary functions. + * \author P. Gomes + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#pragma once + +#include + +#include "../../../Common/include/option_structure.hpp" +#include "../../../Common/include/parallelization/vectorization.hpp" +#include "../../../Common/include/containers/C2DContainer.hpp" +#include "../../../Common/include/linear_algebra/CSysVector.hpp" +#include "../../../Common/include/linear_algebra/CSysMatrix.hpp" + +/*! + * \enum UpdateType + * \brief Ways to update vectors and system matrices. + * COLORING is the typical i/j update, whereas for REDUCTION + * the fluxes are stored and the matrix diagonal is not modified. + */ +enum class UpdateType { COLORING, REDUCTION }; + +#ifdef CODI_FORWARD_TYPE +using SparseMatrixType = CSysMatrix; +#else +using SparseMatrixType = CSysMatrix; +#endif + +/*! + * \brief Alignment of the static containers backing a flux value type. + * \note Yields the type's own alignment for a SIMD array, and the plain type's natural + * alignment for a scalar; C2DContainer's AlignSize also accepts 0 to mean its own + * default, but that reaches `alignas(0)` on the static specializations, which some + * compilers warn about even though it is a no-op, so a real value is passed instead. + */ +template +struct CAlignTraits { + enum : size_t { Align = alignof(Type) }; +}; + +template +struct CAlignTraits> { + enum : size_t { Align = simd::Array::Align }; +}; + +/*! + * \brief Static vector and matrix types. + * \note These should be used instead of C-style arrays. + */ +template +using Vector = C2DContainer::Align, Size, 1>; + +template +using Matrix = C2DContainer::Align, Rows, Cols>; + +/*! + * \brief The flux value type and lane count that go with an index type. + * \note There is exactly one floating type in play, su2double, active under AD; a plain + * integral index reads one of it, a lane-vector index reads a lane-vector of it. This + * is what lets every helper below deduce its value type from the index it is handed, + * instead of a caller naming it explicitly. + */ +template +struct CValueTraits; + +template <> +struct CValueTraits { + using Double = su2double; + static constexpr size_t Size = 1; +}; + +template +struct CValueTraits> { + using Double = simd::Array; + static constexpr size_t Size = N; +}; + +/*! + * \brief Index type and lane count that go with a flux value type, the converse of + * CValueTraits, used where the value type is already known (e.g. a class template + * parameter) and the index type is what needs deriving. + */ +template +struct CLaneTraits; + +template <> +struct CLaneTraits { + using Int = unsigned long; + static constexpr bool IsArray = false; +}; + +template +struct CLaneTraits> { + using Int = simd::Array; + static constexpr bool IsArray = true; +}; + +/*! + * \brief Constexpr version of max. + */ +inline constexpr size_t Max(size_t a, size_t b) { return a > b ? a : b; } + +/*! + * \brief Simple pair type for i/j variables. + */ +template +struct CPair { + T i, j; +}; + +/*! + * \brief Blocks a template parameter from participating in argument deduction. + * \note Deduction never applies a user conversion, so a parameter typed plain Double would + * force a caller passing a bare su2double constant (kappa, a limiter ramp) to have + * already broadcast it. Wrapping the parameter type here defers Double entirely to + * the other, genuinely deduced arguments, and the broadcast then happens as an + * ordinary implicit conversion at the call. + */ +template +struct CIdentity { + using type = T; +}; +template +using CNonDeduced = typename CIdentity::type; + +/*! + * \brief Equation count of a model whose value is only known at runtime. + */ +constexpr size_t Dynamic = size_t(-1); + +/*! + * \brief Backing size of the static arrays of a dynamic model. + * \note The scalar numerics cap the equation count at this value and error above it, so a + * configuration that fits them fits these kernels. + */ +constexpr size_t MaxScalarVar = 8; + +/*! + * \brief Residual of one edge, accumulated by the convective and the diffusive terms. + * \note flux_i and flux_j are the contributions to the rows of i and j. They are opposite + * for a conservative term and independent for a non-conservative one. The Jacobians + * map onto the ii, ij, ji and jj blocks of the edge. A dynamic model sizes the storage + * with the maximum and iterates to nVar, which the scheme sets from the solver. + */ +template +struct EdgeResidual { + /*!< \brief The Matrix a static nVar==1 model would otherwise need degenerates + * to vector-only indexing in C2DContainer (its RowMajor, one-row specialization), so the + * backing is never smaller than 2; the unused padding row/column is simply never visited, + * every loop here and in the model bounds itself to nVar, not Size. */ + static constexpr size_t Size = Max(2, (nVar_ == Dynamic) ? MaxScalarVar : nVar_); + + Vector flux_i, flux_j; + Matrix jac_ii, jac_ij, jac_ji, jac_jj; + const size_t nVar; + + /*! + * \brief Zero the terms of the equations in use, so that both terms can accumulate into them. + * \note A static model zeroes its whole storage with constant trip counts; a dynamic one + * zeroes the leading nVar rows and columns and leaves the rest of the backing untouched. + */ + FORCEINLINE explicit EdgeResidual(size_t nEqn) : nVar(nEqn) { + for (size_t iVar = 0; iVar < nVar; ++iVar) { + flux_i(iVar) = 0.0; + flux_j(iVar) = 0.0; + for (size_t jVar = 0; jVar < nVar; ++jVar) { + jac_ii(iVar, jVar) = 0.0; + jac_ij(iVar, jVar) = 0.0; + jac_ji(iVar, jVar) = 0.0; + jac_jj(iVar, jVar) = 0.0; + } + } + } +}; + +/*! + * \brief Dot product. + */ +template +FORCEINLINE auto dot(ForwardIterator iterator, const T* ptr) -> typename std::decay::type { + typename std::decay::type sum = 0.0; + for (size_t iDim = 0; iDim < nDim; ++iDim) { + sum += *(iterator++) * ptr[iDim]; + } + return sum; +} + +/*! + * \overload Dot product. + */ +template +FORCEINLINE Double dot(ForwardIterator iterator, const Vector& vector) { + return dot(iterator, vector.data()); +} + +/*! + * \overload Dot product. + */ +template +FORCEINLINE Double dot(const Vector& a, const Vector& b) { + return dot(a.data(), b.data()); +} + +/*! + * \brief Squared norm. + */ +template +FORCEINLINE auto squaredNorm(ForwardIterator iterator) -> typename std::decay::type { + typename std::decay::type sum = 0.0; + for (size_t iDim = 0; iDim < nDim; ++iDim) { + sum += pow(*(iterator++), 2); + } + return sum; +} + +/*! + * \overload Squared norm. + */ +template +FORCEINLINE Double squaredNorm(const Vector& vector) { + return squaredNorm(vector.data()); +} + +/*! + * \brief Tangential projection. + */ +template +FORCEINLINE Vector tangentProjection(const Matrix& tensor, + const Vector& unitVector) { + Vector proj; + for (size_t iDim = 0; iDim < nDim; ++iDim) proj(iDim) = dot(tensor[iDim], unitVector); + + Double normalProj = dot(proj, unitVector); + + for (size_t iDim = 0; iDim < nDim; ++iDim) proj(iDim) -= normalProj * unitVector(iDim); + + return proj; +} + +/*! + * \brief Vector norm. + */ +template +FORCEINLINE Double norm(const Vector& vector) { + return sqrt(squaredNorm(vector)); +} + +#ifndef CODI_REVERSE_TYPE +/*! + * \brief Gather a single variable, from column iVar (0 by default) of row iPoint of a + * 2D container, or from index iPoint of a 1D container. + */ +template ::Double> +FORCEINLINE Double gatherVariables(Int iPoint, const Container& vars, size_t iVar = 0) { + return vars.template get>(iPoint, iVar)(0); +} + +/*! + * \brief Gather nVar contiguous variables starting at column iVar (0 by default) of row + * iPoint of a 2D container. + */ +template ::Double> +FORCEINLINE Vector gatherVariables(Int iPoint, const Container& vars, size_t iVar = 0) { + return vars.template get>(iPoint, iVar); +} + +/*! + * \brief Gather an nRows x nCols block of a 3D container, from outer index iPoint and + * starting at middle index iRow. + */ +template ::Double> +FORCEINLINE Matrix gatherVariables(Int iPoint, const Container& vars, size_t iRow = 0) { + return vars.template get>(iPoint, iRow); +} +#else + +namespace { +/*--- Rank of a container, from the element accessors it offers: a 1D container is indexed by + * point alone, a 3D one by (point, row, column). A 3D container also answers a two-argument + * call, but with an offset sub-matrix view rather than a scalar, which is why the rank is + * detected up front instead of trying the access forms in turn. ---*/ +template +struct Is1D : std::false_type {}; +template +struct Is1D()(0ul))>> : std::true_type {}; + +template +struct Is3D : std::false_type {}; +template +struct Is3D()(0ul, 0ul, 0ul))>> : std::true_type {}; + +/*--- One lane of an index or of a gathered value, the whole thing when there are no lanes. ---*/ +FORCEINLINE unsigned long lane(unsigned long iPoint, size_t) { return iPoint; } +template +FORCEINLINE unsigned long lane(const simd::Array& iPoint, size_t k) { return iPoint[k]; } +FORCEINLINE su2double& lane(su2double& x, size_t) { return x; } +template +FORCEINLINE T& lane(simd::Array& x, size_t k) { return x[k]; } + +/*--- Register one source element as a preaccumulation input, passing it through for the copy. + * The registration has to happen here, on the reference into the container's own storage: a + * copy has a fresh identifier of its own, and registering that instead would sever the source + * from the statement EndPreacc() stores. ---*/ +FORCEINLINE const su2double& preaccIn(const su2double& value) { + AD::SetPreaccIn(value); + return value; +} +} // namespace + +template ::Double> +FORCEINLINE Double gatherVariables(Int iPoint, const Container& vars, size_t iVar = 0) { + Double x; + for (size_t k = 0; k < CValueTraits::Size; ++k) { + if constexpr (Is1D::value) + lane(x, k) = preaccIn(vars(lane(iPoint, k))); + else + lane(x, k) = preaccIn(vars(lane(iPoint, k), iVar)); + } + return x; +} + +template ::Double> +FORCEINLINE Vector gatherVariables(Int iPoint, const Container& vars, size_t iVar = 0) { + Vector x; + for (size_t i = 0; i < nVar; ++i) { + for (size_t k = 0; k < CValueTraits::Size; ++k) { + if constexpr (Is3D::value) + lane(x(i), k) = preaccIn(vars(lane(iPoint, k), iVar, i)); + else + lane(x(i), k) = preaccIn(vars(lane(iPoint, k), iVar + i)); + } + } + return x; +} + +template ::Double> +FORCEINLINE Matrix gatherVariables(Int iPoint, const Container& vars, size_t iRow = 0) { + Matrix x; + for (size_t i = 0; i < nRows; ++i) { + for (size_t j = 0; j < nCols; ++j) { + for (size_t k = 0; k < CValueTraits::Size; ++k) { + lane(x(i, j), k) = preaccIn(vars(lane(iPoint, k), iRow + i, j)); + } + } + } + return x; +} +#endif + +/*! + * \brief Register the leading nVar entries of a static vector as preaccumulation outputs. + * \note A lane vector is registered one lane at a time, a scalar in one call; a kernel therefore + * reaches this rather than AD::SetPreaccOut directly, and reads the same whichever value + * type it is bound to. + */ +template +FORCEINLINE void setPreaccOut(Vector& x, size_t nVar) { + if constexpr (!CLaneTraits::IsArray) { + AD::SetPreaccOut(x, static_cast(nVar)); + } else { + AD::SetPreaccOut(x, static_cast(nVar), Double::Size); + } +} + +/*! + * \brief Stop the AD preaccumulation. + */ +template +FORCEINLINE void stopPreacc(Vector& x) { + setPreaccOut(x, nVar); + AD::EndPreacc(); +} + +/*! + * \brief Distance vector, from point i to point j of one container. + */ +template ::Double> +FORCEINLINE Vector distanceVector(Int iPoint, Int jPoint, const Container& coords) { + return distanceVector(iPoint, coords, jPoint, coords); +} + +/*! + * \brief Distance vector, from point i of one container to point j of another. + * \note The two endpoints of a boundary flux read different containers, the solver's own + * and the marker's ghost one; the interior edge loop passes the same container twice. + */ +template ::Double> +FORCEINLINE Vector distanceVector(Int iPoint, const Container& coords_i, Int jPoint, + const Container& coords_j) { + auto coord_i = gatherVariables(iPoint, coords_i); + auto coord_j = gatherVariables(jPoint, coords_j); + Vector vector_ij; + for (size_t iDim = 0; iDim < nDim; ++iDim) { + vector_ij(iDim) = coord_j(iDim) - coord_i(iDim); + } + return vector_ij; +} + +/*! + * \brief Blended difference for U-MUSCL reconstruction. + * \param[in] gradProj - Gradient projection at point i: dot(grad_i, vector_ij). + * \param[in] delta - Centered difference: V_j - V_i. + * \param[in] kappa - Blending parameter. + * \return Blended difference for reconstruction from point i. + */ +template +FORCEINLINE Double umusclProjection(const Double& gradProj, const Double& delta, const CNonDeduced& kappa) { + /*-------------------------------------------------------------------*/ + /*--- The MUSCL kappa-scheme reconstruction is typically written: ---*/ + /*--- V_L = V_i + 0.25 * dV_ij^kap, where ---*/ + /*--- dV_ij^kap = (1-kappa) dV_ij^upw + (1+kappa) dV_ij^cen, ---*/ + /*--- dV_ij^cen = V_j - V_i, ---*/ + /*--- dV_ij^upw = 2 grad(Vi) dot vector_ij - dV_ij^cen. ---*/ + /*--- To maintain proper scaling for edge limiters, the result of ---*/ + /*--- this function is 0.5 * dV_ij^kap. ---*/ + /*-------------------------------------------------------------------*/ + return (1.0 - kappa) * gradProj + kappa * delta; +} + +/*! + * \brief Reads the gradient rows of one point as one block, gathered up front. + * \note This is what a kernel whose variable count is a compile-time constant above one wants: + * one gather of the whole nVarGrad x nDim block instead of nVarGrad of them. + */ +template +struct CGradientBlock { + using Int = typename CLaneTraits::Int; + Matrix rows; + + FORCEINLINE CGradientBlock(Int iPoint, const Gradient_t& gradient, size_t iRow) + : rows(gatherVariables(iPoint, gradient, iRow)) {} + + FORCEINLINE Double project(size_t iVar, const Vector& vector_ij) const { + return dot(rows[iVar], vector_ij); + } +}; + +/*! + * \brief Reads the gradient rows of one point one row at a time. + * \note This is what a runtime variable count forces, since the block shape would not be a + * compile-time constant, and what a single variable forces, since its block would be a + * Matrix: that shape satisfies IsVector and would silently degenerate into + * a lone scalar instead of failing to compile. + */ +template +struct CGradientRows { + using Int = typename CLaneTraits::Int; + const Int iPoint; + const Gradient_t& gradient; + const size_t iRow; + + FORCEINLINE Double project(size_t iVar, const Vector& vector_ij) const { + return dot(gatherVariables(iPoint, gradient, iRow + iVar), vector_ij); + } +}; + +/*! + * \brief Gradient reader of one point, blocked or row by row according to nVarGrad_. + */ +template +FORCEINLINE auto gradientReader(typename CLaneTraits::Int iPoint, const Gradient_t& gradient, size_t iRow) { + if constexpr (nVarGrad_ > 1) { + return CGradientBlock(iPoint, gradient, iRow); + } else { + return CGradientRows{iPoint, gradient, iRow}; + } +} + +/*! + * \brief How the reconstructed differences are limited. + */ +enum class MusclLimiter { NONE, EDGE, POINT }; + +/*! + * \brief U-MUSCL reconstruction of nVarGrad variables, from the gradient rows starting at iRow. + * \note The limiter kind is a template parameter so that the choice is made once, outside the + * loop, by the dispatching overload below. + * \param[in] iRow - Starting row of gradient (and column of limiter) to read, for reconstructing + * a slice of a larger set of gradients (e.g. only the velocity out of the primitives). + * \param[in] nVarGradRuntime - Variable count of a Dynamic model, known only at runtime; ignored + * (falling back to nVarGrad_ or VarType::nVar) when left at its default of 0. + */ +template +FORCEINLINE void muscl(typename CLaneTraits::Int iPoint, typename CLaneTraits::Int jPoint, + const Vector& vector_ij, const Gradient_t& gradient, const Limiter_t& limiter, + size_t iRow, CPair& V, const CNonDeduced& kappa, + const CNonDeduced& umusclRamp, size_t nVarGradRuntime) { + const size_t nVarGrad = nVarGrad_ > 0 ? nVarGrad_ : (nVarGradRuntime > 0 ? nVarGradRuntime : VarType::nVar); + + const auto grad_i = gradientReader(iPoint, gradient, iRow); + const auto grad_j = gradientReader(jPoint, gradient, iRow); + + for (size_t iVar = 0; iVar < nVarGrad; ++iVar) { + /*--- Centered difference, needed for the U-MUSCL projection and the edge limiter. ---*/ + const Double delta_ij = V.j.all(iVar) - V.i.all(iVar); + + /*--- U-MUSCL reconstructed differences, to be halved when applied. ---*/ + const Double proj_i = umusclRamp * umusclProjection(grad_i.project(iVar, vector_ij), delta_ij, kappa); + const Double proj_j = umusclRamp * umusclProjection(grad_j.project(iVar, vector_ij), delta_ij, kappa); + + Double lim_i = 1.0, lim_j = 1.0; + if constexpr (limiterKind == MusclLimiter::EDGE) { + const Double delta_ij_2 = pow(delta_ij, 2) + 1e-6; + lim_i = (delta_ij_2 + proj_i * delta_ij) / (pow(proj_i, 2) + delta_ij_2); + lim_j = (delta_ij_2 + proj_j * delta_ij) / (pow(proj_j, 2) + delta_ij_2); + } else if constexpr (limiterKind == MusclLimiter::POINT) { + lim_i = gatherVariables(iPoint, limiter, iRow + iVar); + lim_j = gatherVariables(jPoint, limiter, iRow + iVar); + } + + /*--- Apply reconstruction: V_L = V_i + 0.5 * lim * dV_ij^kap ---*/ + V.i.all(iVar) += 0.5 * lim_i * proj_i; + V.j.all(iVar) -= 0.5 * lim_j * proj_j; + } +} + +/*! + * \brief Reconstruct a slice of nVarGrad variables starting at column iRow, dispatching on the + * limiter type; shared by the flow and the scalar reconstructions so both call one body. + */ +template +FORCEINLINE void reconstruct(typename CLaneTraits::Int iPoint, typename CLaneTraits::Int jPoint, + const Vector& vector_ij, const Gradient_t& gradient, + const Limiter_t& limiter, LIMITER limiterType, size_t iRow, CPair& V, + const CNonDeduced& kappa, const CNonDeduced& umusclRamp, + size_t nVarGradRuntime = 0) { + switch (limiterType) { + case LIMITER::NONE: + muscl(iPoint, jPoint, vector_ij, gradient, limiter, iRow, V, kappa, umusclRamp, + nVarGradRuntime); + break; + case LIMITER::VAN_ALBADA_EDGE: + muscl(iPoint, jPoint, vector_ij, gradient, limiter, iRow, V, kappa, umusclRamp, + nVarGradRuntime); + break; + default: + muscl(iPoint, jPoint, vector_ij, gradient, limiter, iRow, V, kappa, umusclRamp, + nVarGradRuntime); + break; + } +} + +/*! + * \brief Update the matrix and right-hand-side of a linear system with one conservative flux. + */ +template ::Int> +FORCEINLINE void updateLinearSystem(Int iEdge, Int iPoint, Int jPoint, bool implicit, UpdateType updateType, + Double updateMask, const Vector& flux, + const Matrix& jac_i, const Matrix& jac_j, + CSysVector& vector, SparseMatrixType& matrix) { + if (updateType == UpdateType::COLORING) { + vector.UpdateBlocks(iPoint, jPoint, flux, updateMask); + if (implicit) { + auto wasActive = AD::BeginPassive(); + matrix.SetBlocks(iEdge, iPoint, jPoint, jac_i, jac_j, updateMask); + AD::EndPassive(wasActive); + } + } else { + vector.SetBlock(iEdge, flux, updateMask); + if (implicit) { + auto wasActive = AD::BeginPassive(); + matrix.SetBlocks(iEdge, jac_i, jac_j, updateMask); + AD::EndPassive(wasActive); + } + } +} + +/*! + * \brief Update the matrix and right-hand-side of a linear system with two independent row + * contributions and four independent Jacobian blocks. + * \note It carries a second CSysVector, the target of flux_j under UpdateType::REDUCTION and + * unused under COLORING, where both rows are written directly. + * \note The residual is the same under both update types, the Jacobian is not: REDUCTION writes + * the off-diagonal blocks only and CSysMatrix::SetDiagonalAsColumnSum then derives each + * diagonal block as minus the sum of its column, which equals the jac_ii and jac_jj computed + * here only where the flux is conservative (jac_ii == -jac_ji). A model whose diffusion + * coefficients differ between the two orientations of an edge, i.e. one that evaluates a + * non-conservative term at the point whose row it is writing, therefore converges along a + * slightly different path under the reducer, to the same solution. + */ +template ::Int> +FORCEINLINE void updateLinearSystem(Int iEdge, Int iPoint, Int jPoint, bool implicit, UpdateType updateType, + Double updateMask, const EdgeResidual& res, + CSysVector& vector, CSysVector& vectorDiff, + SparseMatrixType& matrix) { + if (updateType == UpdateType::COLORING) { + vector.AddBlock(iPoint, res.flux_i, updateMask); + vector.AddBlock(jPoint, res.flux_j, updateMask); + if (implicit) { + auto wasActive = AD::BeginPassive(); + matrix.SetBlocks(iEdge, iPoint, jPoint, res.jac_ii, res.jac_ij, res.jac_ji, res.jac_jj, updateMask); + AD::EndPassive(wasActive); + } + } else { + vector.SetBlock(iEdge, res.flux_i, updateMask); + vectorDiff.SetBlock(iEdge, res.flux_j, updateMask); + if (implicit) { + auto wasActive = AD::BeginPassive(); + matrix.SetOffDiagBlocks(iEdge, res.jac_ij, res.jac_ji, updateMask); + AD::EndPassive(wasActive); + } + } +} + +/*! + * \brief Store the (scalar) mass flux of an edge, e.g. for "bounded scalar" transport equations. + * \note No-op if "target" is null. As with CEdge's Nodes/Normal, edges within a SIMD group are + * contiguous (coloring groups are multiples of the SIMD size), so this is a plain vectorized store + * starting at iEdge[0], relying on "target" being padded to a multiple of the SIMD size. + */ +template ::Int> +FORCEINLINE void updateEdgeMassFlux(Int iEdge, const Double& massFlux, su2activevector* target) { + if (target) massFlux.store(&(*target)[iEdge[0]]); +} diff --git a/SU2_CFD/include/numerics_simd/CNumericsSIMD.cpp b/SU2_CFD/include/numerics_simd/CNumericsSIMD.cpp index 52c1d6e743ea..f5ce83935a10 100644 --- a/SU2_CFD/include/numerics_simd/CNumericsSIMD.cpp +++ b/SU2_CFD/include/numerics_simd/CNumericsSIMD.cpp @@ -82,8 +82,8 @@ CNumericsSIMD* createCenteredNumerics(const CConfig& config, int iMesh, const CV case CENTERED::JST_MAT: obj = new CJSTmatScheme(config, iMesh, turbVars); break; - case CENTERED::LD2: - /*--- LD2 implemented only in the incompressible solver. ---*/ + case CENTERED::LD2: case CENTERED::CDS: + /*--- CDS and LD2 implemented only in the incompressible solver. ---*/ break; } return obj; diff --git a/SU2_CFD/include/numerics_simd/CNumericsSIMD.hpp b/SU2_CFD/include/numerics_simd/CNumericsSIMD.hpp index 3379a8ac920e..076c08517b40 100644 --- a/SU2_CFD/include/numerics_simd/CNumericsSIMD.hpp +++ b/SU2_CFD/include/numerics_simd/CNumericsSIMD.hpp @@ -28,14 +28,7 @@ #pragma once #include "../../../Common/include/parallelization/vectorization.hpp" - -/*! - * \enum UpdateType - * \brief Ways to update vectors and system matrices. - * COLORING is the typical i/j update, whereas for REDUCTION - * the fluxes are stored and the matrix diagonal is not modified. - */ -enum class UpdateType {COLORING, REDUCTION}; +#include "../numerics/util.hpp" /*! * \brief Define Double and Int SIMD types. @@ -44,18 +37,10 @@ using Double = simd::Array; using Int = simd::Array; /*--- Forward declare a few classes used in name only by the interface. ---*/ -template class CSysVector; -template class CSysMatrix; class CConfig; class CGeometry; class CVariable; -#ifdef CODI_FORWARD_TYPE -using SparseMatrixType = CSysMatrix; -#else -using SparseMatrixType = CSysMatrix; -#endif - /*! * \class CNumericsSIMD * \ingroup ConvDiscr @@ -74,6 +59,8 @@ class CNumericsSIMD { * \param[in] updateMask - SIMD array of 1's and 0's, the latter prevent the update. * \param[in,out] vector - Target for the fluxes. * \param[in,out] matrix - Target for the flux Jacobians. + * \param[out] edgeMassFluxes - Optional, per-edge mass flux (density-equation flux), + * used by "bounded scalar" discretization of transported scalars (turbulence, species). * \note The update mask is used to handle "remainder" edges (nEdge mod simdSize). */ virtual void ComputeFlux(Int iEdge, @@ -83,7 +70,8 @@ class CNumericsSIMD { UpdateType updateType, Double updateMask, CSysVector& vector, - SparseMatrixType& matrix) const = 0; + SparseMatrixType& matrix, + su2activevector* edgeMassFluxes) const = 0; /*! \brief Destructor of the class. */ virtual ~CNumericsSIMD(void) = default; diff --git a/SU2_CFD/include/numerics_simd/flow/convection/centered.hpp b/SU2_CFD/include/numerics_simd/flow/convection/centered.hpp index 6647e46825f9..faaf56912351 100644 --- a/SU2_CFD/include/numerics_simd/flow/convection/centered.hpp +++ b/SU2_CFD/include/numerics_simd/flow/convection/centered.hpp @@ -67,7 +67,7 @@ class CCenteredBase : public Base { * \brief Special treatment needed to fetch integer data. */ template - FORCEINLINE static Double numNeighbor(simd::Array idx, const CGeometry& geometry) { + FORCEINLINE static Double numNeighbor(const simd::Array& idx, const CGeometry& geometry) { Double n; for (size_t k=0; kGetnNeighbor(idx[k]); return n; @@ -80,14 +80,15 @@ class CCenteredBase : public Base { /*! * \brief Implementation of the base centered flux. */ - void ComputeFlux(Int iEdge, + void ComputeFlux(const Int iEdge, const CConfig& config, const CGeometry& geometry, const CVariable& solution_, - UpdateType updateType, - Double updateMask, + const UpdateType updateType, + const Double updateMask, CSysVector& vector, - SparseMatrixType& matrix) const final { + SparseMatrixType& matrix, + su2activevector* edgeMassFluxes) const final { /*--- Start preaccumulation, inputs are registered * automatically in "gatherVariables". ---*/ @@ -182,6 +183,10 @@ class CCenteredBase : public Base { updateLinearSystem(iEdge, iPoint, jPoint, implicit, updateType, updateMask, flux, jac_i, jac_j, vector, matrix); + + /*--- Store the mass flux (density-equation flux) for bounded-scalar transport. ---*/ + + updateEdgeMassFlux(iEdge, flux(0), edgeMassFluxes); } }; @@ -220,14 +225,14 @@ class CJSTScheme : public CCenteredBase,Decorator> { FORCEINLINE void finalizeFlux(VectorDbl& flux, MatrixDbl& jac_i, MatrixDbl& jac_j, - bool implicit, - Double area, - Double projVel, + const bool implicit, + const Double& area, + const Double& projVel, const PrimVarType& avgV, const CPair& V, const VectorDbl& diffU, - Int iPoint, - Int jPoint, + const Int& iPoint, + const Int& jPoint, const CGeometry& geometry, const CEulerVariable& solution, Ts&...) const { @@ -301,14 +306,14 @@ class CJSTmatScheme : public CCenteredBase,Decorator> { FORCEINLINE void finalizeFlux(VectorDbl& flux, MatrixDbl& jac_i, MatrixDbl& jac_j, - bool implicit, - Double area, - Double projVel, + const bool implicit, + const Double& area, + const Double& projVel, const PrimVarType& avgV, const CPair& V, const VectorDbl& diffU, - Int iPoint, - Int jPoint, + const Int& iPoint, + const Int& jPoint, const CGeometry& geometry, const CEulerVariable& solution, const VectorDbl& unitNormal, @@ -425,14 +430,14 @@ class CJSTkeScheme : public CCenteredBase,Decorator> { FORCEINLINE void finalizeFlux(VectorDbl& flux, MatrixDbl& jac_i, MatrixDbl& jac_j, - bool implicit, - Double area, - Double projVel, + const bool implicit, + const Double& area, + const Double& projVel, const PrimVarType& avgV, const CPair& V, const VectorDbl& diffU, - Int iPoint, - Int jPoint, + const Int& iPoint, + const Int& jPoint, const CGeometry& geometry, const CEulerVariable& solution, Ts&...) const { @@ -496,14 +501,14 @@ class CLaxScheme : public CCenteredBase,Decorator> { FORCEINLINE void finalizeFlux(VectorDbl& flux, MatrixDbl& jac_i, MatrixDbl& jac_j, - bool implicit, - Double area, - Double projVel, + const bool implicit, + const Double& area, + const Double& projVel, const PrimVarType& avgV, const CPair& V, const VectorDbl& diffU, - Int iPoint, - Int jPoint, + const Int& iPoint, + const Int& jPoint, const CGeometry& geometry, const CEulerVariable& solution, Ts&...) const { diff --git a/SU2_CFD/include/numerics_simd/flow/convection/common.hpp b/SU2_CFD/include/numerics_simd/flow/convection/common.hpp index c2203e1f9841..800167d4b0f5 100644 --- a/SU2_CFD/include/numerics_simd/flow/convection/common.hpp +++ b/SU2_CFD/include/numerics_simd/flow/convection/common.hpp @@ -32,149 +32,6 @@ #include "../variables.hpp" #include "../../../variables/CNSVariable.hpp" -/*! - * \brief Blended difference for U-MUSCL reconstruction. - * \param[in] gradProj - Gradient projection at point i: dot(grad_i, vector_ij). - * \param[in] delta - Centered difference: V_j - V_i. - * \param[in] kappa - Blending parameter. - * \return Blended difference for reconstruction from point i. - */ -FORCEINLINE Double umusclProjection(Double gradProj, - Double delta, - Double kappa) { - /*-------------------------------------------------------------------*/ - /*--- The MUSCL kappa-scheme reconstruction is typically written: ---*/ - /*--- V_L = V_i + 0.25 * dV_ij^kap, where ---*/ - /*--- dV_ij^kap = (1-kappa) dV_ij^upw + (1+kappa) dV_ij^cen, ---*/ - /*--- dV_ij^cen = V_j - V_i, ---*/ - /*--- dV_ij^upw = 2 grad(Vi) dot vector_ij - dV_ij^cen. ---*/ - /*--- To maintain proper scaling for edge limiters, the result of ---*/ - /*--- this function is 0.5 * dV_ij^kap. ---*/ - /*-------------------------------------------------------------------*/ - return (1.0 - kappa) * gradProj + kappa * delta; -} - -/*! - * \brief MUSCL reconstruction of the specified variable. - * \note The result should be halved when added to i (or subtracted from j). - * \param[in] grad_i - Gradient vector at point i. - * \param[in] vector_ij - Distance vector from i to j. - * \param[in] delta - Centered difference: V_j - V_i. - * \param[in] iVar - Variable index. - * \param[in] kappa - Blending coefficient. - * \param[in] umusclRamp - MUSCL 1st-2nd order ramp times Newton-Krylov relaxation. - * \return Variable reconstructed from point i. - */ -template -FORCEINLINE Double musclReconstruction(const GradType& grad, - const VectorDbl& vector_ij, - const Double delta, - size_t iVar, - Double kappa, - Double umusclRamp) { - const Double proj = dot(grad[iVar], vector_ij); - return umusclRamp * umusclProjection(proj, delta, kappa); -} - -/*! - * \brief Unlimited reconstruction. - */ -template -FORCEINLINE void musclUnlimited(Int iPoint, - Int jPoint, - const VectorDbl& vector_ij, - const Gradient_t& gradient, - CPair& V, - Double kappa, - Double umusclRamp) { - constexpr auto nVarGrad = nVarGrad_ > 0 ? nVarGrad_ : VarType::nVar; - - auto grad_i = gatherVariables(iPoint, gradient); - auto grad_j = gatherVariables(jPoint, gradient); - - for (size_t iVar = 0; iVar < nVarGrad; ++iVar) { - /*--- Centered difference, needed for U-MUSCL projection ---*/ - const Double delta_ij = V.j.all(iVar) - V.i.all(iVar); - - /*--- U-MUSCL reconstructed variables ---*/ - const Double proj_i = musclReconstruction(grad_i, vector_ij, delta_ij, iVar, kappa, umusclRamp); - const Double proj_j = musclReconstruction(grad_j, vector_ij, delta_ij, iVar, kappa, umusclRamp); - - /*--- Apply reconstruction: V_L = V_i + 0.5 * dV_ij^kap ---*/ - V.i.all(iVar) += 0.5 * proj_i; - V.j.all(iVar) -= 0.5 * proj_j; - } -} - -/*! - * \brief Limited reconstruction with point-based limiter. - */ -template -FORCEINLINE void musclPointLimited(Int iPoint, - Int jPoint, - const VectorDbl& vector_ij, - const Limiter_t& limiter, - const Gradient_t& gradient, - CPair& V, - Double kappa, - Double umusclRamp) { - constexpr auto nVarGrad = nVarGrad_ > 0 ? nVarGrad_ : VarType::nVar; - - auto lim_i = gatherVariables(iPoint, limiter); - auto lim_j = gatherVariables(jPoint, limiter); - - auto grad_i = gatherVariables(iPoint, gradient); - auto grad_j = gatherVariables(jPoint, gradient); - - for (size_t iVar = 0; iVar < nVarGrad; ++iVar) { - /*--- Centered difference, needed for U-MUSCL projection ---*/ - const Double delta_ij = V.j.all(iVar) - V.i.all(iVar); - - /*--- U-MUSCL reconstructed variables ---*/ - const Double proj_i = musclReconstruction(grad_i, vector_ij, delta_ij, iVar, kappa, umusclRamp); - const Double proj_j = musclReconstruction(grad_j, vector_ij, delta_ij, iVar, kappa, umusclRamp); - - /*--- Apply reconstruction: V_L = V_i + 0.5 * lim * dV_ij^kap ---*/ - V.i.all(iVar) += 0.5 * lim_i(iVar) * proj_i; - V.j.all(iVar) -= 0.5 * lim_j(iVar) * proj_j; - } -} - -/*! - * \brief Limited reconstruction with edge-based limiter. - */ -template -FORCEINLINE void musclEdgeLimited(Int iPoint, - Int jPoint, - const VectorDbl& vector_ij, - const Gradient_t& gradient, - CPair& V, - Double kappa, - Double umusclRamp) { - constexpr auto nVarGrad = nVarGrad_ > 0 ? nVarGrad_ : VarType::nVar; - - auto grad_i = gatherVariables(iPoint, gradient); - auto grad_j = gatherVariables(jPoint, gradient); - - for (size_t iVar = 0; iVar < nVarGrad; ++iVar) { - /*--- Centered difference, needed for U-MUSCL projection and limiter ---*/ - const Double delta_ij = V.j.all(iVar) - V.i.all(iVar); - const Double delta_ij_2 = pow(delta_ij, 2) + 1e-6; - - /*--- U-MUSCL reconstructed variables ---*/ - const Double proj_i = musclReconstruction(grad_i, vector_ij, delta_ij, iVar, kappa, umusclRamp); - const Double proj_j = musclReconstruction(grad_j, vector_ij, delta_ij, iVar, kappa, umusclRamp); - - /// TODO: Customize the limiter function. - const Double lim_i = (delta_ij_2 + proj_i*delta_ij) / (pow(proj_i,2) + delta_ij_2); - const Double lim_j = (delta_ij_2 + proj_j*delta_ij) / (pow(proj_j,2) + delta_ij_2); - - /*--- Apply reconstruction: V_L = V_i + 0.5 * lim * dV_ij^kap ---*/ - V.i.all(iVar) += 0.5 * lim_i * proj_i; - V.j.all(iVar) -= 0.5 * lim_j * proj_j; - } -} - /*! * \brief Retrieve primitive variables for points i/j, reconstructing them if needed. * \note Density and enthalpy are recomputed from ideal gas EOS. @@ -188,19 +45,22 @@ FORCEINLINE void musclEdgeLimited(Int iPoint, * \param[in] V1st - Pair of compressible flow primitives for nodes i,j. * \param[in] vector_ij - Distance vector from i to j. * \param[in] solution - Entire solution container (a derived CVariable). + * \param[out] nonPhysical - Signals that the edge is treated as non-physical. * \return Pair of primitive variables. */ template -FORCEINLINE CPair reconstructPrimitives(Int iEdge, Int iPoint, Int jPoint, +FORCEINLINE CPair reconstructPrimitives(const Int& iEdge, + const Int& iPoint, const Int& jPoint, const su2double& gamma, const su2double& gasConst, - bool muscl, + const bool muscl, const su2double& kappa, const su2double& umusclRamp, - LIMITER limiterType, + const LIMITER limiterType, const CPair& V1st, const VectorDbl& vector_ij, - const VariableType& solution) { + const VariableType& solution, + Double& nonPhysical) { static_assert(ReconVarType::nVar <= PrimVarType::nVar); const auto& gradients = solution.GetGradient_Reconstruction(); @@ -216,17 +76,7 @@ FORCEINLINE CPair reconstructPrimitives(Int iEdge, Int iPoint, Int if (muscl) { /*--- Reconstruct density and enthalpy without using their gradients. ---*/ constexpr auto nVarGrad = ReconVarType::nVar - 2; - switch (limiterType) { - case LIMITER::NONE: - musclUnlimited(iPoint, jPoint, vector_ij, gradients, V, kappa, umusclRamp); - break; - case LIMITER::VAN_ALBADA_EDGE: - musclEdgeLimited(iPoint, jPoint, vector_ij, gradients, V, kappa, umusclRamp); - break; - default: - musclPointLimited(iPoint, jPoint, vector_ij, limiters, gradients, V, kappa, umusclRamp); - break; - } + reconstruct(iPoint, jPoint, vector_ij, gradients, limiters, limiterType, 0, V, kappa, umusclRamp); /*--- Recompute density using the reconstructed pressure and temperature. ---*/ V.i.density() = V.i.pressure() / (gasConst * V.i.temperature()); V.j.density() = V.j.pressure() / (gasConst * V.j.temperature()); @@ -261,15 +111,20 @@ FORCEINLINE CPair reconstructPrimitives(Int iEdge, Int iPoint, Int const Double neg_sound_speed = enthalpy * (R+1) < 0.5 * v_squared; /*--- Revert to first order if the state is non-physical. ---*/ - Double bad_recon = fmax(neg_p_or_rho, neg_sound_speed); + nonPhysical = fmax(neg_p_or_rho, neg_sound_speed); /*--- Handle SIMD dimensions 1 by 1. ---*/ for (size_t k = 0; k < Double::Size; ++k) { - bad_recon[k] = solution.UpdateNonPhysicalEdgeCounter(iEdge[k], bad_recon[k]); + nonPhysical[k] = solution.UpdateNonPhysicalEdgeCounter(iEdge[k], nonPhysical[k]); + nonPhysical[k] = fmax(nonPhysical[k], + fmax(solution.OutlierMitigation(iPoint[k]), + solution.OutlierMitigation(jPoint[k])) / VariableType::MAX_OUTLIER_MITIGATION); } for (size_t iVar = 0; iVar < ReconVarType::nVar; ++iVar) { - V.i.all(iVar) = bad_recon * V1st.i.all(iVar) + (1-bad_recon) * V.i.all(iVar); - V.j.all(iVar) = bad_recon * V1st.j.all(iVar) + (1-bad_recon) * V.j.all(iVar); + V.i.all(iVar) = nonPhysical * V1st.i.all(iVar) + (1-nonPhysical) * V.i.all(iVar); + V.j.all(iVar) = nonPhysical * V1st.j.all(iVar) + (1-nonPhysical) * V.j.all(iVar); } + } else { + nonPhysical = 0; } return V; } @@ -278,8 +133,8 @@ FORCEINLINE CPair reconstructPrimitives(Int iEdge, Int iPoint, Int * \brief Compute and return the P tensor (compressible flow, ideal gas). */ template -FORCEINLINE MatrixDbl pMatrix(Double gamma, Double density, const RandomAccessIterator& velocity, - Double projVel, Double speedSound, const VectorDbl& normal) { +FORCEINLINE MatrixDbl pMatrix(const Double& gamma, const Double& density, const RandomAccessIterator& velocity, + const Double& projVel, const Double& speedSound, const VectorDbl& normal) { MatrixDbl pMat; const Double vel2 = 0.5*squaredNorm(velocity); @@ -340,8 +195,9 @@ FORCEINLINE MatrixDbl pMatrix(Double gamma, Double density, const Random * \brief Compute and return the inverse P tensor (compressible flow, ideal gas). */ template -FORCEINLINE MatrixDbl pMatrixInv(Double gamma, Double density, const RandomAccessIterator& velocity, - Double projVel, Double speedSound, const VectorDbl& normal) { +FORCEINLINE MatrixDbl pMatrixInv(const Double& gamma, const Double& density, + const RandomAccessIterator& velocity, const Double& projVel, + const Double& speedSound, const VectorDbl& normal) { MatrixDbl pMatInv; const Double c2 = pow(speedSound,2); @@ -421,9 +277,9 @@ FORCEINLINE VectorDbl inviscidProjFlux(const PrimVarType& V, * \brief Jacobian of the convective flux (compressible flow, ideal gas). */ template -FORCEINLINE MatrixDbl inviscidProjJac(Double gamma, RandomAccessIterator velocity, - Double energy, const VectorDbl& normal, - Double scale) { +FORCEINLINE MatrixDbl inviscidProjJac(const Double& gamma, RandomAccessIterator velocity, + const Double& energy, const VectorDbl& normal, + const Double& scale) { MatrixDbl jac; Double projVel = dot(velocity, normal); @@ -459,9 +315,9 @@ FORCEINLINE MatrixDbl inviscidProjJac(Double gamma, RandomAccessIterator * \brief (Low) Dissipation coefficient for Roe schemes. */ template -FORCEINLINE Double roeDissipation(Int iPoint, - Int jPoint, - ENUM_ROELOWDISS type, +FORCEINLINE Double roeDissipation(const Int& iPoint, + const Int& jPoint, + const ENUM_ROELOWDISS type, const VariableType& solution) { if (type == NO_ROELOWDISS) { return 1.0; @@ -511,10 +367,10 @@ FORCEINLINE Double roeDissipation(Int iPoint, * \brief Correct spectral radius (avgLambda) for stretching. */ template -FORCEINLINE Double correctedSpectralRadius(Int iPoint, - Int jPoint, - Double avgLambda, - T stretchParam, +FORCEINLINE Double correctedSpectralRadius(const Int& iPoint, + const Int& jPoint, + const Double& avgLambda, + const T& stretchParam, const VariableType& solution) { const auto lambda_i = gatherVariables(iPoint, solution.GetLambda()); @@ -531,7 +387,7 @@ FORCEINLINE Double correctedSpectralRadius(Int iPoint, */ template FORCEINLINE void scalarDissipationJacobian(const VariableType& V, - Double gamma, + const Double& gamma, Double dissipConst, MatrixDbl& jac) { /*--- Diagonal entries. ---*/ diff --git a/SU2_CFD/include/numerics_simd/flow/convection/upwind.hpp b/SU2_CFD/include/numerics_simd/flow/convection/upwind.hpp index 36036d7dc5dc..38ec02c902b5 100644 --- a/SU2_CFD/include/numerics_simd/flow/convection/upwind.hpp +++ b/SU2_CFD/include/numerics_simd/flow/convection/upwind.hpp @@ -80,14 +80,15 @@ class CUpwindBase : public Base { /*! * \brief Implementation of the base Roe flux. */ - void ComputeFlux(Int iEdge, + void ComputeFlux(const Int iEdge, const CConfig& config, const CGeometry& geometry, const CVariable& solution_, - UpdateType updateType, - Double updateMask, + const UpdateType updateType, + const Double updateMask, CSysVector& vector, - SparseMatrixType& matrix) const final { + SparseMatrixType& matrix, + su2activevector* edgeMassFluxes) const final { /*--- Start preaccumulation, inputs are registered * automatically in "gatherVariables". ---*/ @@ -118,8 +119,10 @@ class CUpwindBase : public Base { V1st.j.all = gatherVariables(jPoint, solution.GetPrimitive()); /*--- Recompute density and enthalpy instead of reconstructing. ---*/ + Double nonPhysical; auto V = reconstructPrimitives >( - iEdge, iPoint, jPoint, gamma, gasConst, muscl, umusclKappa, umusclRamp, typeLimiter, V1st, vector_ij, solution); + iEdge, iPoint, jPoint, gamma, gasConst, muscl, umusclKappa, umusclRamp, + typeLimiter, V1st, vector_ij, solution, nonPhysical); /*--- Compute conservative variables. ---*/ @@ -132,8 +135,8 @@ class CUpwindBase : public Base { const auto derived = static_cast(this); VectorDbl flux; MatrixDbl jac_i, jac_j; - derived->finalizeFlux(flux, jac_i, jac_j, implicit, area, unitNormal, - normal, V, U, iPoint, jPoint, solution, geometry); + derived->finalizeFlux(flux, jac_i, jac_j, implicit, area, unitNormal, normal, + V, U, iPoint, jPoint, nonPhysical, solution, geometry); /*--- Add the contributions from the base class (static decorator). ---*/ @@ -148,6 +151,10 @@ class CUpwindBase : public Base { updateLinearSystem(iEdge, iPoint, jPoint, implicit, updateType, updateMask, flux, jac_i, jac_j, vector, matrix); + + /*--- Store the mass flux (density-equation flux) for bounded-scalar transport. ---*/ + + updateEdgeMassFlux(iEdge, flux(0), edgeMassFluxes); } }; @@ -191,14 +198,15 @@ class CRoeScheme : public CUpwindBase, Decorator> { FORCEINLINE void finalizeFlux(VectorDbl& flux, MatrixDbl& jac_i, MatrixDbl& jac_j, - bool implicit, - Double area, + const bool implicit, + const Double& area, const VectorDbl& unitNormal, const VectorDbl& normal, const CPair& V, const CPair& U, - Int iPoint, - Int jPoint, + const Int& iPoint, + const Int& jPoint, + const Double& nonPhysical, const CEulerVariable& solution, const CGeometry& geometry, Ts&...) const { @@ -227,10 +235,9 @@ class CRoeScheme : public CUpwindBase, Decorator> { /*--- Apply Mavriplis' entropy correction to eigenvalues. ---*/ - Double maxLambda = abs(projVel) + roeAvg.speedSound; - + Double lambdaMin = fmax(entropyFix, nonPhysical) * (abs(projVel) + roeAvg.speedSound); for (size_t iVar = 0; iVar < nVar; ++iVar) { - lambda(iVar) = fmax(abs(lambda(iVar)), entropyFix*maxLambda); + lambda(iVar) = fmax(abs(lambda(iVar)), lambdaMin); } /*--- Inviscid fluxes and Jacobians. ---*/ @@ -340,14 +347,15 @@ class CMSWScheme : public CUpwindBase, Decorator> { FORCEINLINE void finalizeFlux(VectorDbl& flux, MatrixDbl& jac_i, MatrixDbl& jac_j, - bool implicit, - Double area, + const bool implicit, + const Double& area, const VectorDbl& unitNormal, const VectorDbl& normal, const CPair& V, const CPair& U, - Int iPoint, - Int jPoint, + const Int& iPoint, + const Int& jPoint, + const Double& nonPhysical, const CEulerVariable& solution, const CGeometry& geometry, Ts&...) const { @@ -358,7 +366,7 @@ class CMSWScheme : public CUpwindBase, Decorator> { const auto sj = gatherVariables(jPoint, solution.GetSensor()); const Double dp = fmax(si, sj) - alpha * 0.06; - const Double w = 0.25 * (1 - sign(dp)) * (1 - exp(-100 * abs(dp))); + const Double w = 0.25 * (1 - sign(dp) * (1 - exp(-100 * abs(dp)))) * (1 - nonPhysical); const Double onemw = 1 - w; CPair> Vweighted; diff --git a/SU2_CFD/include/numerics_simd/flow/diffusion/common.hpp b/SU2_CFD/include/numerics_simd/flow/diffusion/common.hpp index 43793d215aaa..78b391d8a2fe 100644 --- a/SU2_CFD/include/numerics_simd/flow/diffusion/common.hpp +++ b/SU2_CFD/include/numerics_simd/flow/diffusion/common.hpp @@ -36,7 +36,7 @@ * \brief Average gradients at i/j points. */ template -FORCEINLINE MatrixDbl averageGradient(Int iPoint, Int jPoint, +FORCEINLINE MatrixDbl averageGradient(const Int& iPoint, const Int& jPoint, const GradientType& gradient) { auto avgGrad = gatherVariables(iPoint, gradient); auto grad_j = gatherVariables(jPoint, gradient); @@ -55,7 +55,7 @@ FORCEINLINE MatrixDbl averageGradient(Int iPoint, Int jPoint, template FORCEINLINE void correctGradient(const PrimitiveType& V, const VectorDbl& vector_ij, - Double dist2_ij, + const Double& dist2_ij, MatrixDbl& avgGrad) { for (size_t iVar = 0; iVar < nVar; ++iVar) { Double corr = (dot(avgGrad[iVar],vector_ij) - V.j.all(iVar) + V.i.all(iVar)) / dist2_ij; @@ -70,7 +70,7 @@ FORCEINLINE void correctGradient(const PrimitiveType& V, * \note Second viscosity term ignored. */ template -FORCEINLINE MatrixDbl stressTensor(Double viscosity, +FORCEINLINE MatrixDbl stressTensor(const Double& viscosity, const MatrixDbl& grad) { /*--- Hydrostatic term. ---*/ Double velDiv = 0.0; @@ -121,7 +121,7 @@ NEVERINLINE void addPerturbedRSM(const PrimitiveType& V, * \brief SA-QCR2000 modification of the stress tensor. */ template -FORCEINLINE void addQCR(const MatrixType& grad, MatrixDbl& tau) { +FORCEINLINE void addQCR(const MatrixType& grad, MatrixDbl& tau, Double turb_fraction) { constexpr passivedouble c_cr1 = 0.3; /*--- Denominator, antisymmetric normalized rotation tensor. ---*/ @@ -146,7 +146,7 @@ FORCEINLINE void addQCR(const MatrixType& grad, MatrixDbl& tau) { } for (size_t iDim = 0; iDim < nDim; ++iDim) for (size_t jDim = 0; jDim < nDim; ++jDim) - tau(iDim,jDim) -= c_cr1 * qcr(iDim,jDim); + tau(iDim,jDim) -= turb_fraction * c_cr1 * qcr(iDim,jDim); } /*! @@ -154,7 +154,7 @@ FORCEINLINE void addQCR(const MatrixType& grad, MatrixDbl& tau) { * wall function) magnitude in the tangential direction. */ template -FORCEINLINE void addTauWall(Int iPoint, Int jPoint, +FORCEINLINE void addTauWall(const Int& iPoint, const Int& jPoint, const Container& tauWall, const VectorDbl& unitNormal, MatrixDbl& tau) { @@ -185,7 +185,7 @@ FORCEINLINE void addTauWall(Int iPoint, Int jPoint, template FORCEINLINE MatrixDbl stressTensorJacobian(const PrimitiveType& V, const VectorDbl& normal, - Double dist_ij) { + const Double& dist_ij) { Double viscosity = V.laminarVisc() + V.eddyVisc(); Double xi = viscosity / (V.density() * dist_ij); MatrixDbl jac; diff --git a/SU2_CFD/include/numerics_simd/flow/diffusion/viscous_fluxes.hpp b/SU2_CFD/include/numerics_simd/flow/diffusion/viscous_fluxes.hpp index 22114093435f..1a8745b10bc2 100644 --- a/SU2_CFD/include/numerics_simd/flow/diffusion/viscous_fluxes.hpp +++ b/SU2_CFD/include/numerics_simd/flow/diffusion/viscous_fluxes.hpp @@ -111,16 +111,16 @@ class CCompressibleViscousFluxBase : public CNumericsSIMD { * \brief Add viscous contributions to flux and jacobians. */ template - FORCEINLINE void viscousTerms(Int iEdge, - Int iPoint, - Int jPoint, + FORCEINLINE void viscousTerms(const Int& iEdge, + const Int& iPoint, + const Int& jPoint, const PrimVarType& avgV, const CPair& V, const CVariable& solution_, const VectorDbl& vector_ij, const CGeometry& geometry, const CConfig& config, - Double area, + const Double& area, const VectorDbl& unitNormal, bool implicit, VectorDbl& flux, @@ -149,8 +149,9 @@ class CCompressibleViscousFluxBase : public CNumericsSIMD { /*--- Stress and heat flux tensors. ---*/ - auto tau = stressTensor(avgV.laminarVisc() + (uq? Double(0.0) : avgV.eddyVisc()), avgGrad); - if(useSA_QCR) addQCR(avgGrad, tau); + const Double eddyVisc = uq? Double(0.0) : avgV.eddyVisc(); + auto tau = stressTensor(avgV.laminarVisc() + eddyVisc, avgGrad); + if(useSA_QCR) addQCR(avgGrad, tau, eddyVisc / (avgV.laminarVisc() + eddyVisc)); if(uq) { Double turb_ke = 0.5*(gatherVariables(iPoint, turbVars->GetSolution()) + gatherVariables(jPoint, turbVars->GetSolution())); @@ -210,9 +211,9 @@ class CCompressibleViscousFluxBase : public CNumericsSIMD { * \overload Average primitives if not provided yet. */ template - FORCEINLINE void viscousTerms(Int iEdge, - Int iPoint, - Int jPoint, + FORCEINLINE void viscousTerms(const Int& iEdge, + const Int& iPoint, + const Int& jPoint, const CPair& V, Ts&... args) const { PrimVarType avgV; @@ -228,9 +229,9 @@ class CCompressibleViscousFluxBase : public CNumericsSIMD { * \overload Compute the i-j vector if not provided yet. */ template - FORCEINLINE void viscousTerms(Int iEdge, - Int iPoint, - Int jPoint, + FORCEINLINE void viscousTerms(const Int& iEdge, + const Int& iPoint, + const Int& jPoint, const PrimVarType& avgV, const CPair& V, const CVariable& solution_, @@ -278,9 +279,9 @@ class CCompressibleViscousFlux : public CCompressibleViscousFluxBase FORCEINLINE VectorDbl energyJacobian(const PrimitiveType& V, const MatrixDbl& dtau, - Double thermalCond, - Double area, - Double dist_ij, + const Double& thermalCond, + const Double& area, + const Double& dist_ij, Ts&... args) const { Double vel2 = 0.5 * squaredNorm(V.velocity()); Double phi = (gamma-1) / V.density(); @@ -334,11 +335,11 @@ class CGeneralCompressibleViscousFlux : public CCompressibleViscousFluxBase FORCEINLINE VectorDbl energyJacobian(const PrimitiveType& V, const MatrixDbl& dtau, - Double thermalCond, - Double area, - Double dist_ij, - Int iPoint, - Int jPoint, + const Double& thermalCond, + const Double& area, + const Double& dist_ij, + const Int& iPoint, + const Int& jPoint, const VariableType& solution) const { Double vel2 = squaredNorm(V.velocity()); Double contraction = 0.0; diff --git a/SU2_CFD/include/numerics_simd/flow/variables.hpp b/SU2_CFD/include/numerics_simd/flow/variables.hpp index fa39c7337f2e..4406a06abf7c 100644 --- a/SU2_CFD/include/numerics_simd/flow/variables.hpp +++ b/SU2_CFD/include/numerics_simd/flow/variables.hpp @@ -113,7 +113,7 @@ struct CRoeVariables { * \brief Compute Roe-averaged variables from pair of primitive variables. */ template -FORCEINLINE CRoeVariables roeAveragedVariables(Double gamma, +FORCEINLINE CRoeVariables roeAveragedVariables(const Double& gamma, const CPair& V, const VectorDbl& normal) { CRoeVariables roeAvg; diff --git a/SU2_CFD/include/numerics_simd/util.hpp b/SU2_CFD/include/numerics_simd/util.hpp index 6e84209ee6db..9db7721803f4 100644 --- a/SU2_CFD/include/numerics_simd/util.hpp +++ b/SU2_CFD/include/numerics_simd/util.hpp @@ -1,6 +1,6 @@ /*! * \file util.hpp - * \brief Generic auxiliary functions. + * \brief Vector, matrix and index types bound to the SIMD Double/Int of CNumericsSIMD.hpp. * \author P. Gomes * \version 8.5.0 "Harrier" * @@ -28,236 +28,10 @@ #pragma once #include "CNumericsSIMD.hpp" -#include "../../../Common/include/containers/C2DContainer.hpp" -#include "../../../Common/include/linear_algebra/CSysVector.hpp" -#include "../../../Common/include/linear_algebra/CSysMatrix.hpp" +#include "../numerics/util.hpp" -/*! - * \brief Static vector and matrix types. - * \note These should be used instead of C-style arrays. - */ -template -using Vector = C2DContainer; - -template using VectorInt = Vector; -template using VectorDbl = Vector; - -template -using Matrix = C2DContainer; - -template using MatrixInt = Matrix; -template using MatrixDbl = Matrix; - -/*! - * \brief Constexpr version of max. - */ -inline constexpr size_t Max(size_t a, size_t b) { return a>b? a : b; } - -/*! - * \brief Simple pair type for i/j variables. - */ -template -struct CPair { - T i, j; -}; - -/*! - * \brief Dot product. - */ -template -FORCEINLINE Double dot(ForwardIterator iterator, const T* ptr) { - Double sum = 0.0; - for (size_t iDim = 0; iDim < nDim; ++iDim) { - sum += *(iterator++) * ptr[iDim]; - } - return sum; -} - -/*! - * \overload Dot product. - */ -template -FORCEINLINE Double dot(ForwardIterator iterator, const VectorDbl& vector) { - return dot(iterator, vector.data()); -} - -/*! - * \overload Dot product. - */ -template -FORCEINLINE Double dot(const VectorDbl& a, const VectorDbl& b) { - return dot(a.data(), b.data()); -} - -/*! - * \brief Squared norm. - */ -template -FORCEINLINE Double squaredNorm(ForwardIterator iterator) { - Double sum = 0.0; - for (size_t iDim = 0; iDim < nDim; ++iDim) { - sum += pow(*(iterator++),2); - } - return sum; -} - -/*! - * \overload Squared norm. - */ -template -FORCEINLINE Double squaredNorm(const VectorDbl& vector) { - return squaredNorm(vector.data()); -} - -/*! - * \brief Tangential projection. - */ -template -FORCEINLINE VectorDbl tangentProjection(const MatrixDbl& tensor, - const VectorDbl& unitVector) { - VectorDbl proj; - for (size_t iDim = 0; iDim < nDim; ++iDim) - proj(iDim) = dot(tensor[iDim], unitVector); - - Double normalProj = dot(proj, unitVector); +template using VectorInt = Vector; +template using VectorDbl = Vector; - for (size_t iDim = 0; iDim < nDim; ++iDim) - proj(iDim) -= normalProj * unitVector(iDim); - - return proj; -} - -/*! - * \brief Vector norm. - */ -template -FORCEINLINE Double norm(const VectorDbl& vector) { return sqrt(squaredNorm(vector)); } - -#ifndef CODI_REVERSE_TYPE -/*! - * \brief Gather a single variable from index iPoint of a 1D container. - */ -template -FORCEINLINE Double gatherVariables(Int iPoint, const Container& vars) { - return *vars.innerIter(iPoint); -} - -/*! - * \brief Gather a vector of variables (size nVar) from row iPoint of a 2D container. - */ -template -FORCEINLINE VectorDbl gatherVariables(Int iPoint, const Container& vars) { - return vars.template get >(iPoint); -} - -/*! - * \brief Gather a matrix of variables from outer index iPoint of a 3D container. - */ -template -FORCEINLINE MatrixDbl gatherVariables(Int iPoint, const Container& vars) { - return vars.template get >(iPoint); -} -#else - -namespace { - template = 0> - FORCEINLINE const su2double& get(const Container& vars, unsigned long iPoint) { return vars(iPoint); } - - /*--- When getting 1 variable from a matrix container, we assume it is the first. ---*/ - template = 0> - FORCEINLINE const su2double& get(const Container& vars, unsigned long iPoint) { return vars(iPoint,0); } -} - -template -FORCEINLINE Double gatherVariables(Int iPoint, const Container& vars) { - Double x; - for (size_t k=0; k -FORCEINLINE VectorDbl gatherVariables(Int iPoint, const Container& vars) { - VectorDbl x; - for (size_t i=0; i -FORCEINLINE MatrixDbl gatherVariables(Int iPoint, const Container& vars) { - MatrixDbl x; - for (size_t i=0; i -FORCEINLINE void stopPreacc(VectorDbl& x) { - AD::SetPreaccOut(x, nVar, Double::Size); - AD::EndPreacc(); -} - -/*! - * \brief Distance vector, from point i to point j. - */ -template -FORCEINLINE VectorDbl distanceVector(Int iPoint, Int jPoint, - const Container& coords) { - auto coord_i = gatherVariables(iPoint, coords); - auto coord_j = gatherVariables(jPoint, coords); - VectorDbl vector_ij; - for (size_t iDim = 0; iDim < nDim; ++iDim) { - vector_ij(iDim) = coord_j(iDim) - coord_i(iDim); - } - return vector_ij; -} - -/*! - * \brief Update the matrix and right-hand-side of a linear system. - */ -template -FORCEINLINE void updateLinearSystem(Int iEdge, - Int iPoint, - Int jPoint, - bool implicit, - UpdateType updateType, - Double updateMask, - const VectorDbl& flux, - const MatrixDbl& jac_i, - const MatrixDbl& jac_j, - CSysVector& vector, - SparseMatrixType& matrix) { - if (updateType == UpdateType::COLORING) { - vector.UpdateBlocks(iPoint, jPoint, flux, updateMask); - if(implicit) { - auto wasActive = AD::BeginPassive(); - matrix.UpdateBlocks(iEdge, iPoint, jPoint, jac_i, jac_j, updateMask); - AD::EndPassive(wasActive); - } - } - else { - vector.SetBlock(iEdge, flux, updateMask); - if(implicit) { - auto wasActive = AD::BeginPassive(); - matrix.SetBlocks(iEdge, jac_i, jac_j, updateMask); - AD::EndPassive(wasActive); - } - } -} +template using MatrixInt = Matrix; +template using MatrixDbl = Matrix; diff --git a/SU2_CFD/include/output/CFlowCompOutput.hpp b/SU2_CFD/include/output/CFlowCompOutput.hpp index ba1e289f8529..ad2dd0217317 100644 --- a/SU2_CFD/include/output/CFlowCompOutput.hpp +++ b/SU2_CFD/include/output/CFlowCompOutput.hpp @@ -53,6 +53,8 @@ class CFlowCompOutput final: public CFlowOutput { */ void LoadHistoryData(CConfig *config, CGeometry *geometry, CSolver **solver) override; + void LoadHistoryData(CConfig *config, CGeometry *geometry, CSolver **solver, unsigned short iZone); + /*! * \brief Set the available volume output fields * \param[in] config - Definition of the particular problem. @@ -74,6 +76,8 @@ class CFlowCompOutput final: public CFlowOutput { */ void SetHistoryOutputFields(CConfig *config) override; + void SetTurbomachineryObjectiveFunctions(CSolver *solver, CConfig *config); + /*! * \brief Check whether the base values for relative residuals should be initialized * \param[in] config - Definition of the particular problem. @@ -101,7 +105,7 @@ class CFlowCompOutput final: public CFlowOutput { * \param[in] OuterIter - Index of current outer iteration * \param[in] InnerIter - Index of current inner iteration */ - void SetTurboPerformance_Output(std::shared_ptr TurboPerf, CConfig *config, unsigned long TimeIter, unsigned long OuterIter, unsigned long InnerIter) override; + void SetTurboPerformance_Output(su2vector> TurboBladePerfs, CConfig *config, unsigned long TimeIter, unsigned long OuterIter, unsigned long InnerIter) override; /*! * \brief Sets the multizone turboperformacne screen output @@ -109,7 +113,7 @@ class CFlowCompOutput final: public CFlowOutput { * \param[in] TurboPerf - Turboperformance class * \param[in] config - Definition of the particular problem */ - void SetTurboMultiZonePerformance_Output(std::shared_ptr TurboStagePerf, std::shared_ptr TurboPerf, CConfig *config) override; + void SetTurboMultiZonePerformance_Output(std::shared_ptr TurboStagePerf, su2vector> TurboBladePerfs, CConfig *config) override; /*! * \brief Loads the turboperformacne history data @@ -117,7 +121,7 @@ class CFlowCompOutput final: public CFlowOutput { * \param[in] TurboPerf - Turboperformance class * \param[in] config - Definition of the particular problem */ - void LoadTurboHistoryData(std::shared_ptr TurboStagePerf, std::shared_ptr TurboPerf, CConfig *config) override; + void LoadTurboHistoryData(std::shared_ptr TurboStagePerf, su2vector> TurboBladePerfs, CConfig *config) override; /*! * \brief Write the kinematic and thermodynamic variables at each spanwise division @@ -126,6 +130,6 @@ class CFlowCompOutput final: public CFlowOutput { * \param[in] config - Descripiton of the particular problem * \param[in] val_iZone - Idientifier of current zone */ - void WriteTurboSpanwisePerformance(std::shared_ptr TurboPerf, CGeometry *geometry, CConfig **config, + void WriteTurboSpanwisePerformance(su2vector> TurboBladePerfs, CGeometry *geometry, CConfig **config, unsigned short val_iZone) override; }; diff --git a/SU2_CFD/include/output/CFlowIncOutput.hpp b/SU2_CFD/include/output/CFlowIncOutput.hpp index 0eb29f2911a1..80e22a80d3b1 100644 --- a/SU2_CFD/include/output/CFlowIncOutput.hpp +++ b/SU2_CFD/include/output/CFlowIncOutput.hpp @@ -42,6 +42,7 @@ class CFlowIncOutput final: public CFlowOutput { bool heat; /*!< \brief Boolean indicating whether have a heat problem*/ bool weakly_coupled_heat; /*!< \brief Boolean indicating whether have a weakly coupled heat equation*/ bool flamelet; /*!< \brief Boolean indicating whether we solve the flamelet equations */ + bool pressure_based; /*!< Boolean indicating whether running the pressure based version */ unsigned short streamwisePeriodic; /*!< \brief Boolean indicating whether it is a streamwise periodic simulation. */ bool streamwisePeriodic_temperature; /*!< \brief Boolean indicating streamwise periodic temperature is used. */ diff --git a/SU2_CFD/include/output/CFlowOutput.hpp b/SU2_CFD/include/output/CFlowOutput.hpp index e79067c79d9b..7e6264f18476 100644 --- a/SU2_CFD/include/output/CFlowOutput.hpp +++ b/SU2_CFD/include/output/CFlowOutput.hpp @@ -44,7 +44,7 @@ class CFlowOutput : public CFVMOutput{ */ CFlowOutput(const CConfig *config, unsigned short nDim, bool femOutput); - /* + /*! * \brief Add turboperformance outputs as history field * \param[in] nZone - Number of zones in problem */ @@ -116,6 +116,14 @@ class CFlowOutput : public CFVMOutput{ */ void LoadHistoryDataScalar(const CConfig* config, const CSolver* const* solver); + /*! + * \brief Recompute history output field values that can be used as objective functions in the (multiphysics) discrete adjoint solver. + * \param[in] config - Definition of the particular problem. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] solver - The container holding all solution data. + */ + void LoadCustomAndComboObjectiveFunctions(CConfig *config, CGeometry *geometry, CSolver **solver) override; + /*! * \brief Add scalar (turbulence/species) volume solution fields for a point (FVMComp, FVMInc, FVMNEMO). * \note The order of fields in restart files is fixed. Therefore the split-up. diff --git a/SU2_CFD/include/output/COutput.hpp b/SU2_CFD/include/output/COutput.hpp index f5df6d6272b7..29032b003026 100644 --- a/SU2_CFD/include/output/COutput.hpp +++ b/SU2_CFD/include/output/COutput.hpp @@ -321,8 +321,10 @@ class COutput { vector oldFunc, /*!< \brief Old value of the coefficient. */ newFunc; /*!< \brief Current value of the coefficient. */ bool convergence; /*!< \brief To indicate if the solver has converged or not. */ + bool convergenceInterrupted; /*!< \brief To indicate that the exit was forced by an interrupt signal instead of the convergence criteria. */ su2double initResidual; /*!< \brief Initial value of the residual to evaluate the convergence level. */ vector convFields; /*!< \brief Name of the field to be monitored for convergence. */ + unsigned long convergenceStartIter = 0; /*!< \brief Iteration the convergence history is counted from. */ /*----------------------------- Adaptive CFL ----------------------------*/ @@ -411,7 +413,7 @@ class COutput { */ void SetHistoryOutput(CGeometry ****geometry, CSolver *****solver_container, CConfig **config, std::shared_ptr TurboStagePerf, - std::shared_ptr TurboPerf, unsigned short val_iZone, + su2vector> TurboBladePerfs, unsigned short val_iZone, unsigned long TimeIter, unsigned long OuterIter, unsigned long InnerIter, unsigned short val_iInst); /*! @@ -426,6 +428,15 @@ class COutput { void SetMultizoneHistoryOutput(COutput** output, CConfig **config, CConfig *driver_config, unsigned long TimeIter, unsigned long OuterIter); + /*! + * \brief Evaluates objective functions in the (multiphysics) discrete adjoint solver. + * \note Uses the same subroutines for objective function evaluation as SetHistoryOutput, but omits unnecessary evaluations (e.g. residuals, convergence data) to avoid AD complications. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] solver_container - Container vector with all the solutions. + * \param[in] config - Definition of the particular problem. + */ + void SetObjectiveFunctionValues(CGeometry *geometry, CSolver **solver_container, CConfig *config); + /*! * \brief Sets the volume output filename * \param[in] filename - the new filename @@ -486,6 +497,28 @@ class COutput { return 0; } + /*! + * \brief Discard the convergence history and count it from the given iteration instead. + * \param[in] Iteration - Iteration the history restarts from. + */ + void ResetConvergenceMonitoring(unsigned long Iteration) { convergenceStartIter = Iteration; } + + /*! + * \brief Names and values of the fields convergence is monitored on that are residuals. + * \return Name and current value of every monitored residual field, in CONV_FIELD order. + */ + vector> GetResidualConvFields() const { + vector> fields; + for (const auto& name : convFields) { + const auto it = historyOutput_Map.find(name); + if (it == historyOutput_Map.end()) continue; + if ((it->second.fieldType != HistoryFieldType::RESIDUAL) && + (it->second.fieldType != HistoryFieldType::AUTO_RESIDUAL)) continue; + fields.emplace_back(name, SU2_TYPE::GetValue(it->second.value)); + } + return fields; + } + /*! * \brief Get the value of particular surface history output field * \param[in] field - Name of the field @@ -565,6 +598,12 @@ class COutput { */ bool GetConvergence() const {return convergence;} + /*! + * \brief Get whether the exit was forced by an interrupt signal (e.g. SIGTERM) instead of the convergence criteria. + * \return Boolean indicating whether an interrupt signal forced the exit. + */ + bool GetConvergenceInterrupted() const {return convergenceInterrupted;} + /*! * \brief Set the value of the convergence flag. * \param[in] conv - New value of the convergence flag. @@ -786,6 +825,12 @@ class COutput { */ void CheckHistoryOutput(unsigned short nZone); + /*! + * \brief Check that the Full-MG startup has a criterion left to promote the active level on. + * \param[in] config - Definition of the particular problem. + */ + void CheckFullMG_Startup(const CConfig *config) const; + /*! * \brief Open the history file and write the header. * \param[in] config - Definition of the particular problem. @@ -967,6 +1012,17 @@ class COutput { */ inline virtual void LoadHistoryData(CConfig *config, CGeometry *geometry, CSolver **solver) {} + /*! + * \brief Recompute history output field values that can be used as objective functions in the (multiphysics) discrete adjoint solver. + * \param[in] config - Definition of the particular problem. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] solver - The container holding all solution data. + */ + inline virtual void LoadCustomAndComboObjectiveFunctions(CConfig *config, CGeometry *geometry, CSolver **solver) { + /*--- Unless LoadCustomAndComboObjectiveFunctions is implemented in a derived output class, we use LoadHistoryData (not ideal for AD). ---*/ + LoadHistoryData(config, geometry, solver); + } + /*! * \brief Load the multizone history output field values * \param[in] output - Container holding the output instances per zone. @@ -982,7 +1038,7 @@ class COutput { * \param[in] OuterIter - Index of current outer iteration * \param[in] InnerIter - Index of current inner iteration */ - inline virtual void SetTurboPerformance_Output(std::shared_ptr TurboPerf, CConfig *config, unsigned long TimeIter, unsigned long OuterIter, unsigned long InnerIter) {} + inline virtual void SetTurboPerformance_Output(su2vector> TurboBladePerfs, CConfig *config, unsigned long TimeIter, unsigned long OuterIter, unsigned long InnerIter) {} /*! * \brief Sets the multizone turboperformacne screen output @@ -990,7 +1046,7 @@ class COutput { * \param[in] TurboPerf - Turboperformance class * \param[in] config - Definition of the particular problem */ - inline virtual void SetTurboMultiZonePerformance_Output(std::shared_ptr TurboStagePerf, std::shared_ptr TurboPerf, CConfig *config) {} + inline virtual void SetTurboMultiZonePerformance_Output(std::shared_ptr TurboStagePerf, su2vector> TurboPerf, CConfig *config) {} /*! * \brief Loads the turboperformacne history data @@ -998,7 +1054,7 @@ class COutput { * \param[in] TurboPerf - Turboperformance class * \param[in] config - Definition of the particular problem */ - inline virtual void LoadTurboHistoryData(std::shared_ptr TurboStagePerf, std::shared_ptr TurboPerf, CConfig *config) {} + inline virtual void LoadTurboHistoryData(std::shared_ptr TurboStagePerf, su2vector> TurboPerf, CConfig *config) {} /*! * \brief Write the kinematic and thermodynamic variables at each spanwise division @@ -1007,7 +1063,7 @@ class COutput { * \param[in] config - Descripiton of the particular problem * \param[in] val_iZone - Idientifier of current zone */ - inline virtual void WriteTurboSpanwisePerformance(std::shared_ptr TurboPerf, CGeometry *geometry, CConfig **config, + inline virtual void WriteTurboSpanwisePerformance(su2vector> TurboBladePerfs, CGeometry *geometry, CConfig **config, unsigned short val_iZone) {}; /*! diff --git a/SU2_CFD/include/output/CTurboOutput.hpp b/SU2_CFD/include/output/CTurboOutput.hpp index 21c69659e8f6..430f239cd8e6 100644 --- a/SU2_CFD/include/output/CTurboOutput.hpp +++ b/SU2_CFD/include/output/CTurboOutput.hpp @@ -96,6 +96,11 @@ class CTurbomachineryState { CTurbomachineryState(unsigned short nDim, su2double area, su2double radius); + inline void SetZeroValues() { + Density = Pressure = Entropy = Enthalpy = Temperature = TotalTemperature = TotalPressure = TotalEnthalpy = 0.0; + AbsFlowAngle = FlowAngle = MassFlow = Rothalpy = TotalRelPressure = 0.0; + } + void ComputeState(CFluidModel& fluidModel, const CTurbomachineryPrimitiveState& primitiveState); const su2double& GetDensity() const { return Density; } @@ -247,16 +252,30 @@ class CTurbomachineryStagePerformance { */ class CTurboOutput { private: - vector>> BladesPerformances; + vector> BladesPerformances; static void ComputePerBlade(vector> const bladePerformances, vector const bladePrimitives); static void ComputePerSpan(shared_ptr const spanPerformances, const CTurbomachineryCombinedPrimitiveStates& spanPrimitives); public: - CTurboOutput(CConfig** config, const CGeometry& geometry, CFluidModel& fluidModel); + CTurboOutput(CConfig** config, const CGeometry& geometry, CFluidModel& fluidModel, unsigned short iBladeRow); + + const vector>& GetBladesPerformances() const { return BladesPerformances; } + + void ComputeTurbomachineryPerformance(vector const primitives, unsigned short iBladeRow); - const vector>>& GetBladesPerformances() const { return BladesPerformances; } + /*! + * \brief Returns true if the given objective function kind is a turbomachinery objective + * that can be evaluated via GetObjectiveValue. + * \param[in] kind - Objective function kind (ENUM_OBJECTIVE value). + */ + static bool IsTurboObjective(unsigned short kind); - void ComputeTurbomachineryPerformance(vector> const primitives); + /*! + * \brief Get the value of a turbomachinery objective function from the tip span performance. + * \param[in] kind - Objective function kind (ENUM_OBJECTIVE value). + * \return The objective function value, or 0.0 for unrecognised kinds. + */ + su2double GetObjectiveValue(unsigned short kind) const; }; \ No newline at end of file diff --git a/SU2_CFD/include/output/filewriter/CParaviewBinaryFileWriter.hpp b/SU2_CFD/include/output/filewriter/CParaviewBinaryFileWriter.hpp index 53617a633d42..507741a0e58f 100644 --- a/SU2_CFD/include/output/filewriter/CParaviewBinaryFileWriter.hpp +++ b/SU2_CFD/include/output/filewriter/CParaviewBinaryFileWriter.hpp @@ -60,16 +60,5 @@ class CParaviewBinaryFileWriter final: public CFileWriter{ * \param[in] val_filename - The name of the file */ void WriteData(string val_filename) override ; - -private: - - /*! - * \brief Change storage of buffer from big endian to little endian - * \param buffer - Pointer to the beginning of the buffer - * \param nBytes - The size in bytes of an data entry - * \param nVar - The number of entries - */ - void SwapBytes(char *buffer, size_t nBytes, unsigned long nVar); - }; diff --git a/SU2_CFD/include/output/filewriter/CSU2MeshBinaryFileWriter.hpp b/SU2_CFD/include/output/filewriter/CSU2MeshBinaryFileWriter.hpp new file mode 100644 index 000000000000..de3c681c713a --- /dev/null +++ b/SU2_CFD/include/output/filewriter/CSU2MeshBinaryFileWriter.hpp @@ -0,0 +1,59 @@ +/*! + * \file CSU2MeshBinaryFileWriter.hpp + * \brief Headers for the SU2 binary mesh file writer class. + * \author E. van der Weide + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ +#pragma once +#include "CFileWriter.hpp" + +class CSU2MeshBinaryFileWriter final: public CFileWriter{ + +private: + unsigned short iZone, //!< Index of the current zone + nZone; //!< Number of zones + +public: + + /*! + * \brief File extension + */ + const static string fileExt; + + /*! + * \brief Construct a file writer using field names, dimension. + * \param[in] valDataSorter - The parallel sorted data to write + * \param[in] valiZone - The index of the current zone + * \param[in] valnZone - The total number of zones + */ + CSU2MeshBinaryFileWriter(CParallelDataSorter* valDataSorter, + unsigned short valiZone, unsigned short valnZone); + + /*! + * \brief Write sorted data to file in SU2 mesh file format + * \param[in] val_filename - The name of the file + */ + void WriteData(string val_filename) override ; + +}; + diff --git a/SU2_CFD/include/solvers/CDiscAdjSolver.hpp b/SU2_CFD/include/solvers/CDiscAdjSolver.hpp index 71758c0f43cf..81d325feaf5c 100644 --- a/SU2_CFD/include/solvers/CDiscAdjSolver.hpp +++ b/SU2_CFD/include/solvers/CDiscAdjSolver.hpp @@ -57,7 +57,7 @@ class CDiscAdjSolver final : public CSolver { su2double Total_Sens_BPress; /*!< \brief Total sensitivity to outlet pressure. */ su2double Total_Sens_Density; /*!< \brief Total sensitivity to initial density (incompressible). */ su2double Total_Sens_ModVel; /*!< \brief Total sensitivity to inlet velocity (incompressible). */ - su2double Mach, Alpha, Beta, Temperature, BPressure, ModVel; + su2double Mach, Alpha, Beta, Pressure, Temperature, BPressure, ModVel; su2double TemperatureRad, Total_Sens_Temp_Rad; CDiscAdjVariable* nodes = nullptr; /*!< \brief The highest level in the variable hierarchy this solver can safely use. */ diff --git a/SU2_CFD/include/solvers/CEulerSolver.hpp b/SU2_CFD/include/solvers/CEulerSolver.hpp index 46eed006b049..345be6d3d3a4 100644 --- a/SU2_CFD/include/solvers/CEulerSolver.hpp +++ b/SU2_CFD/include/solvers/CEulerSolver.hpp @@ -29,6 +29,7 @@ #include "CFVMFlowSolverBase.hpp" #include "../variables/CEulerVariable.hpp" +#include "../output/CTurboOutput.hpp" /*! * \class CEulerSolver @@ -116,6 +117,9 @@ class CEulerSolver : public CFVMFlowSolverBase FluidModel; /*!< \brief fluid model used in the solver. */ + /*!< \brief Variables for outlier detection. */ + su2double MeanTemperature, StdDevTemperature; + /*--- Turbomachinery Solver Variables ---*/ vector AverageFlux; @@ -123,21 +127,16 @@ class CEulerSolver : public CFVMFlowSolverBase AverageVelocity; vector AverageTurboVelocity; vector OldAverageTurboVelocity; - vector ExtAverageTurboVelocity; su2activematrix AveragePressure; su2activematrix OldAveragePressure; su2activematrix RadialEquilibriumPressure; - su2activematrix ExtAveragePressure; su2activematrix AverageDensity; su2activematrix OldAverageDensity; - su2activematrix ExtAverageDensity; su2activematrix AverageNu; su2activematrix AverageKine; su2activematrix AverageOmega; - su2activematrix ExtAverageNu; - su2activematrix ExtAverageKine; - su2activematrix ExtAverageOmega; su2activevector AverageMassFlowRate; + su2activematrix AverageRelTangVelocity; su2activematrix DensityIn; su2activematrix PressureIn; @@ -151,11 +150,57 @@ class CEulerSolver : public CFVMFlowSolverBase > > CkInflow, CkOutflow1, CkOutflow2; + static constexpr unsigned short nMixingStateVars = 8; /*!< \brief Number of averaged variables transferred across mixing plane interfaces. */ + vector MixingState; /* GetTurboBladePerformance() const final { return TurbomachineryPerformance; } + /*! * \brief it take a velocity in the cartesian reference of framework and transform into the turbomachinery frame of reference. * \param[in] cartesianVelocity - cartesian components of velocity vector. @@ -1156,22 +1212,21 @@ class CEulerSolver : public CFVMFlowSolverBaseval_marker. - */ - inline su2double GetExtAverageNu(unsigned short valMarker, unsigned short valSpan) const final { - return ExtAverageNu[valMarker][valSpan]; - } - - /*! - * \brief Provide the average density at the boundary of interest. - * \param[in] val_marker - bound marker. - * \return Value of the Average turbulent Kine on the surface val_marker. - */ - inline su2double GetExtAverageKine(unsigned short valMarker, unsigned short valSpan) const final { - return ExtAverageKine[valMarker][valSpan]; - } - - /*! - * \brief Provide the average density at the boundary of interest. - * \param[in] val_marker - bound marker. - * \return Value of the Average turbulent Omega on the surface val_marker. - */ - inline su2double GetExtAverageOmega(unsigned short valMarker, unsigned short valSpan) const final { - return ExtAverageOmega[valMarker][valSpan]; - } - - /*! - * \brief Set the external average density at the boundary of interest. - * \param[in] val_marker - bound marker. - * \param[in] val_Span - value of the Span. - * \param[in] valDensity - value to set. - */ - inline void SetExtAverageDensity(unsigned short valMarker, - unsigned short valSpan, - su2double valDensity) final { - ExtAverageDensity[valMarker][valSpan] = valDensity; - } - - /*! - * \brief Set the external average density at the boundary of interest. - * \param[in] val_marker - bound marker. - * \param[in] val_Span - value of the Span. - * \param[in] valPressure - value to set. - */ - inline void SetExtAveragePressure(unsigned short valMarker, - unsigned short valSpan, - su2double valPressure) final { - ExtAveragePressure[valMarker][valSpan] = valPressure; - } - - /*! - * \brief Set the external the average turbo velocity average at the boundary of interest. - * \param[in] val_marker - bound marker. - * \return Value of the Average Total Pressure on the surface val_marker. - */ - inline void SetExtAverageTurboVelocity(unsigned short valMarker, - unsigned short valSpan, - unsigned short valIndex, - su2double valTurboVelocity) final { - ExtAverageTurboVelocity[valMarker][valSpan][valIndex] = valTurboVelocity; - } - - /*! - * \brief Set the external average turbulent Nu at the boundary of interest. - * \param[in] val_marker - bound marker. - * \param[in] val_Span - value of the Span. - * \param[in] valNu - value to set. - */ - inline void SetExtAverageNu(unsigned short valMarker, - unsigned short valSpan, - su2double valNu) final { - ExtAverageNu[valMarker][valSpan] = valNu; - } - - /*! - * \brief Set the external average turbulent Kine at the boundary of interest. - * \param[in] val_marker - bound marker. - * \param[in] val_Span - value of the Span. - * \param[in] valKine - value to set. - */ - inline void SetExtAverageKine(unsigned short valMarker, - unsigned short valSpan, - su2double valKine) final { - ExtAverageKine[valMarker][valSpan] = valKine; - } - - /*! - * \brief Set the external average turbulent Omega at the boundary of interest. - * \param[in] val_marker - bound marker. - * \param[in] val_Span - value of the Span. - * \param[in] valOmega - value to set. - */ - inline void SetExtAverageOmega(unsigned short valMarker, - unsigned short valSpan, - su2double valOmega) final { - ExtAverageOmega[valMarker][valSpan] = valOmega; - } - /*! * \brief Provide the inlet density to check convergence of conservative mixing-plane. * \param[in] inMarkerTP - bound marker. diff --git a/SU2_CFD/include/solvers/CFVMFlowSolverBase.hpp b/SU2_CFD/include/solvers/CFVMFlowSolverBase.hpp index da14a8ef95ae..ae07bef84552 100644 --- a/SU2_CFD/include/solvers/CFVMFlowSolverBase.hpp +++ b/SU2_CFD/include/solvers/CFVMFlowSolverBase.hpp @@ -109,9 +109,19 @@ class CFVMFlowSolverBase : public CSolver { void allocate(int size); /*!< \brief Allocates arrays. */ - void setZero(int i); /*!< \brief Sets all values to zero at a particular index. */ - void setZero() { /*!< \brief Sets all values to zero for all indices. */ + /*!< \brief Sets all values to zero at a particular index. */ + void setZero(int i) { + CD[i] = CL[i] = CSF[i] = CEff[i] = 0.0; + CFx[i] = CFy[i] = CFz[i] = CMx[i] = 0.0; + CMy[i] = CMz[i] = CoPx[i] = CoPy[i] = 0.0; + CoPz[i] = CT[i] = CQ[i] = CMerit[i] = 0.0; + } + + /*!< \brief Sets all values to zero for all indices. */ + void setZero() { + SU2_OMP_FOR_STAT(OMP_MIN_SIZE / 2) for (int i = 0; i < _size; ++i) setZero(i); + END_SU2_OMP_FOR } AeroCoeffsArray(int size = 0) : _size(size) { @@ -327,25 +337,65 @@ class CFVMFlowSolverBase : public CSolver { /*! * \brief Compute the viscous contribution for a particular edge. - * \note The convective residual methods include a call to this for each edge, - * this allows convective and viscous loops to be "fused". + * \note The convective residual methods include a call to this for each edge, this allows convective and + * viscous loops to be "fused". Only the residual is applied here, the Jacobians are returned so that + * the caller can update the system matrix in a single operation together with the convective part + * (a requirement of quantized matrix storage). * \param[in] iEdge - Edge for which the flux and Jacobians are to be computed. * \param[in] geometry - Geometrical definition of the problem. * \param[in] solver_container - Container vector with all the solutions. * \param[in] numerics - Description of the numerical method. * \param[in] config - Definition of the particular problem. + * \return The viscous Jacobians (null for inviscid solvers). */ - inline virtual void Viscous_Residual(unsigned long iEdge, CGeometry *geometry, CSolver **solver_container, - CNumerics *numerics, CConfig *config) { } - void Viscous_Residual_impl(unsigned long iEdge, CGeometry *geometry, CSolver **solver_container, - CNumerics *numerics, CConfig *config); + inline virtual CNumerics::ResidualType<> Viscous_Residual(unsigned long iEdge, CGeometry *geometry, + CSolver **solver_container, CNumerics *numerics, + CConfig *config) { + return CNumerics::ResidualType<>(nullptr, nullptr, nullptr); + } + CNumerics::ResidualType<> Viscous_Residual_impl(unsigned long iEdge, CGeometry *geometry, + CSolver **solver_container, CNumerics *numerics, CConfig *config); using CSolver::Viscous_Residual; /*--- Silence warning ---*/ + /*! + * \brief Update the Jacobian for one edge with the fused convective and viscous contributions. + * \note Both contributions must be applied at once because in quantized mode the + * off-diagonal blocks of the matrix can only be overwritten, not accumulated. + * \param[in] iEdge - Edge index for the off-diagonal blocks. + * \param[in] iPoint, jPoint - Points connected by the edge (diagonal blocks). + * \param[in] conv - Convective residual/Jacobians (added to i, subtracted from j). + * \param[in] visc - Viscous residual/Jacobians (subtracted from i, added to j), may hold null Jacobians. + */ + inline void UpdateJacobian(unsigned long iEdge, unsigned long iPoint, unsigned long jPoint, + const CNumerics::ResidualType<>& conv, const CNumerics::ResidualType<>& visc) { + /*--- Lazy element-wise difference, presented with the [i][j] access the matrix expects. ---*/ + struct CJacobianDifference { + const su2double* const* conv; + const su2double* const* visc; + struct Row { + const su2double *c, *v; + su2double operator[](unsigned long j) const { return c[j] - v[j]; } + }; + Row operator[](unsigned long i) const { return {conv[i], visc[i]}; } + }; + if (visc.jacobian_i != nullptr) { + const CJacobianDifference jac_i{conv.jacobian_i, visc.jacobian_i}; + const CJacobianDifference jac_j{conv.jacobian_j, visc.jacobian_j}; + if (ReducerStrategy) Jacobian.SetBlocks(iEdge, jac_i, jac_j); + else Jacobian.UpdateBlocks(iEdge, iPoint, jPoint, jac_i, jac_j); + } else { + if (ReducerStrategy) Jacobian.SetBlocks(iEdge, conv.jacobian_i, conv.jacobian_j); + else Jacobian.UpdateBlocks(iEdge, iPoint, jPoint, conv.jacobian_i, conv.jacobian_j); + } + } + /*! * \brief Compute a suitable under-relaxation parameter to limit the change in the solution variables over a nonlinear * iteration for stability. */ - virtual void ComputeUnderRelaxationFactor(const CConfig* config); + virtual void ComputeUnderRelaxationFactor(const CConfig* config) { + SU2_MPI::Error("Not implemented for this solver.", CURRENT_FUNCTION); + } /*! * \brief General implementation to load a flow solution from a restart file. @@ -976,39 +1026,6 @@ class CFVMFlowSolverBase : public CSolver { } - /*! - * \brief Generic implementation to complete an implicit iteration, i.e. update the solution. - * \tparam compute_ur - Whether to use automatic under-relaxation for the update. - */ - template - void CompleteImplicitIteration_impl(CGeometry *geometry, CConfig *config) { - - if (compute_ur) ComputeUnderRelaxationFactor(config); - - /*--- Update solution with under-relaxation and communicate it. ---*/ - - if (!config->GetContinuous_Adjoint()) { - SU2_OMP_FOR_STAT(omp_chunk_size) - for (unsigned long iPoint = 0; iPoint < nPointDomain; iPoint++) { - for (unsigned short iVar = 0; iVar < nVar; iVar++) { - nodes->AddSolution(iPoint, iVar, nodes->GetUnderRelaxation(iPoint)*LinSysSol[iPoint*nVar+iVar]); - } - } - END_SU2_OMP_FOR - } - - for (unsigned short iPeriodic = 1; iPeriodic <= config->GetnMarker_Periodic()/2; iPeriodic++) { - InitiatePeriodicComms(geometry, config, iPeriodic, PERIODIC_IMPLICIT); - CompletePeriodicComms(geometry, config, iPeriodic, PERIODIC_IMPLICIT); - } - - InitiateComms(geometry, config, MPI_QUANTITIES::SOLUTION); - CompleteComms(geometry, config, MPI_QUANTITIES::SOLUTION); - - /*--- For verification cases, compute the global error metrics. ---*/ - ComputeVerificationError(geometry, config); - } - /*! * \brief Evaluate the vorticity and strain rate magnitude. */ @@ -1075,6 +1092,13 @@ class CFVMFlowSolverBase : public CSolver { */ void ImplicitEuler_Iteration(CGeometry *geometry, CSolver **solver_container, CConfig *config) final; + /*! + * \brief Complete an implicit iteration. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] config - Definition of the particular problem. + */ + void CompleteImplicitIteration(CGeometry *geometry, CSolver**, CConfig *config) final; + /*! * \brief Set the total residual adding the term that comes from the Dual Time Strategy. * \param[in] geometry - Geometrical definition of the problem. diff --git a/SU2_CFD/include/solvers/CFVMFlowSolverBase.inl b/SU2_CFD/include/solvers/CFVMFlowSolverBase.inl index 05ec25a0e56e..aea0dc1d6a26 100644 --- a/SU2_CFD/include/solvers/CFVMFlowSolverBase.inl +++ b/SU2_CFD/include/solvers/CFVMFlowSolverBase.inl @@ -74,14 +74,6 @@ CFVMFlowSolverBase::AeroCoeffsArray::~AeroCoeffsArray() { delete[] CMerit; } -template -void CFVMFlowSolverBase::AeroCoeffsArray::setZero(int i) { - CD[i] = CL[i] = CSF[i] = CEff[i] = 0.0; - CFx[i] = CFy[i] = CFz[i] = CMx[i] = 0.0; - CMy[i] = CMz[i] = CoPx[i] = CoPy[i] = 0.0; - CoPz[i] = CT[i] = CQ[i] = CMerit[i] = 0.0; -} - template void CFVMFlowSolverBase::Allocate(const CConfig& config) { SU2_ZONE_SCOPED @@ -317,7 +309,7 @@ void CFVMFlowSolverBase::HybridParallelInitialization(const CConfig& confi if (!coloring.empty()) { /*--- If the reducer strategy is used we are not constrained by group * size as we have no other edge loops in the Euler/NS solvers. ---*/ - auto groupSize = ReducerStrategy ? 1ul : geometry.GetEdgeColorGroupSize(); + auto groupSize = static_cast(ReducerStrategy ? 1ul : geometry.GetEdgeColorGroupSize()); auto nColor = coloring.getOuterSize(); EdgeColoring.reserve(nColor); @@ -362,9 +354,9 @@ void CFVMFlowSolverBase::HybridParallelInitialization(const CConfig& confi if (SU2_MPI::GetRank() == MASTER_NODE && numRanksUsingReducer != SU2_MPI::GetSize()) { cout << "Among the ranks that use edge coloring,\n" - << " the minimum efficiency is " << minColoredParallelEff << ",\n" - << " the maximum number of colors is " << maxColoredNumColors << ",\n" - << " the minimum edge color group size is " << minColoredEdgeColorGroupSize << "." << endl; + << " the minimum efficiency is " << minColoredParallelEff << ",\n" + << " the maximum number of colors is " << maxColoredNumColors << ",\n" + << " the minimum edge color group size is " << minColoredEdgeColorGroupSize << "." << endl; } } @@ -445,11 +437,11 @@ void CFVMFlowSolverBase::SetPrimitive_Limiter(CGeometry* geometry, const C } template -void CFVMFlowSolverBase::Viscous_Residual_impl(unsigned long iEdge, CGeometry *geometry, CSolver **solver_container, - CNumerics *numerics, CConfig *config) { +CNumerics::ResidualType<> CFVMFlowSolverBase::Viscous_Residual_impl(unsigned long iEdge, CGeometry *geometry, + CSolver **solver_container, + CNumerics *numerics, CConfig *config) { SU2_ZONE_SCOPED - const bool implicit = (config->GetKind_TimeIntScheme() == EULER_IMPLICIT); const bool tkeNeeded = (config->GetKind_Turb_Model() == TURB_MODEL::SST); const bool backscatter = config->GetSBSParam().StochasticBackscatter; const bool ideal_gas = (config->GetKind_FluidModel() == STANDARD_AIR) || @@ -518,17 +510,14 @@ void CFVMFlowSolverBase::Viscous_Residual_impl(unsigned long iEdge, CGeome if (ReducerStrategy) { EdgeFluxes.SubtractBlock(iEdge, residual); - if (implicit) - Jacobian.UpdateBlocksSub(iEdge, residual.jacobian_i, residual.jacobian_j); } else { LinSysRes.SubtractBlock(iPoint, residual); LinSysRes.AddBlock(jPoint, residual); - - if (implicit) - Jacobian.UpdateBlocksSub(iEdge, iPoint, jPoint, residual.jacobian_i, residual.jacobian_j); } + /*--- The Jacobians are applied by the caller, fused with the convective contribution. ---*/ + return residual; } template @@ -596,42 +585,33 @@ void CFVMFlowSolverBase::ComputeVerificationError(CGeometry* geometry, CCo } template -void CFVMFlowSolverBase::ComputeUnderRelaxationFactor(const CConfig* config) { +void CFVMFlowSolverBase::CompleteImplicitIteration(CGeometry *geometry, CSolver**, CConfig *config) { SU2_ZONE_SCOPED - /* Loop over the solution update given by relaxing the linear - system for this nonlinear iteration. */ - - const su2double allowableRatio = config->GetMaxUpdateFractionFlow(); + if constexpr (R == ENUM_REGIME::COMPRESSIBLE) ComputeUnderRelaxationFactor(config); - SU2_OMP_FOR_STAT(omp_chunk_size) - for (unsigned long iPoint = 0; iPoint < nPointDomain; iPoint++) { - su2double localUnderRelaxation = 1.0; + /*--- Update solution with under-relaxation and communicate it. ---*/ - for (unsigned short iVar = 0; iVar < nVar; iVar++) { - /* We impose a limit on the maximum percentage that the - density and energy can change over a nonlinear iteration. */ - - if ((iVar == 0) || (iVar == nVar - 1)) { - const unsigned long index = iPoint * nVar + iVar; - su2double ratio = fabs(LinSysSol[index]) / (fabs(nodes->GetSolution(iPoint, iVar)) + EPS); - if (ratio > allowableRatio) { - localUnderRelaxation = min(allowableRatio / ratio, localUnderRelaxation); - } + if (!config->GetContinuous_Adjoint()) { + SU2_OMP_FOR_STAT(omp_chunk_size) + for (unsigned long iPoint = 0; iPoint < nPointDomain; iPoint++) { + for (unsigned short iVar = 0; iVar < nVar; iVar++) { + nodes->AddSolution(iPoint, iVar, nodes->GetUnderRelaxation(iPoint) * LinSysSol(iPoint, iVar)); } } + END_SU2_OMP_FOR + } - /* Threshold the relaxation factor in the event that there is - a very small value. This helps avoid catastrophic crashes due - to non-realizable states by canceling the update. */ - - if (localUnderRelaxation < 1e-10) localUnderRelaxation = 0.0; + for (unsigned short iPeriodic = 1; iPeriodic <= config->GetnMarker_Periodic()/2; iPeriodic++) { + InitiatePeriodicComms(geometry, config, iPeriodic, PERIODIC_IMPLICIT); + CompletePeriodicComms(geometry, config, iPeriodic, PERIODIC_IMPLICIT); + } - /* Store the under-relaxation factor for this point. */ + InitiateComms(geometry, config, MPI_QUANTITIES::SOLUTION); + CompleteComms(geometry, config, MPI_QUANTITIES::SOLUTION); - nodes->SetUnderRelaxation(iPoint, localUnderRelaxation); - } - END_SU2_OMP_FOR + /*--- For verification cases, compute the global error metrics. ---*/ + ComputeVerificationError(geometry, config); } template @@ -1022,7 +1002,7 @@ void CFVMFlowSolverBase::LoadRestart_impl(CGeometry **geometry, CSolver ** * species solver does all the Pre-/Postprocessing. ---*/ if (config->GetKind_Turb_Model() == TURB_MODEL::NONE && config->GetKind_Species_Model() == SPECIES_MODEL::NONE) { - solver[MESH_0][FLOW_SOL]->Preprocessing(geometry[MESH_0], solver[MESH_0], config, MESH_0, NO_RK_ITER, RUNTIME_FLOW_SYS, false); + solver[MESH_0][FLOW_SOL]->Preprocessing(geometry[MESH_0], solver[MESH_0], config, MESH_0, NO_RK_ITER, RUNTIME_FLOW_SYS, true); } /*--- Interpolate the solution down to the coarse multigrid levels ---*/ @@ -1035,7 +1015,7 @@ void CFVMFlowSolverBase::LoadRestart_impl(CGeometry **geometry, CSolver ** if (config->GetKind_Turb_Model() == TURB_MODEL::NONE && config->GetKind_Species_Model() == SPECIES_MODEL::NONE) { - solver[iMesh][FLOW_SOL]->Preprocessing(geometry[iMesh], solver[iMesh], config, iMesh, NO_RK_ITER, RUNTIME_FLOW_SYS, false); + solver[iMesh][FLOW_SOL]->Preprocessing(geometry[iMesh], solver[iMesh], config, iMesh, NO_RK_ITER, RUNTIME_FLOW_SYS, true); } } @@ -1305,13 +1285,14 @@ void CFVMFlowSolverBase::BC_Sym_Plane(CGeometry* geometry, CSolve auto ModifyJacobian = [&](const unsigned long jPoint) { su2double jac[MAXNVAR * MAXNVAR], newJac[MAXNVAR * MAXNVAR]; - auto* block = Jacobian.GetBlock(iPoint, jPoint); - for (auto iVar = 0u; iVar < nVar * nVar; iVar++) jac[iVar] = block[iVar]; + const auto view = Jacobian.GetBlockView(iPoint, jPoint); + if (!view) return; + for (auto iVar = 0u; iVar < nVar; iVar++) + for (auto jVar = 0u; jVar < nVar; jVar++) jac[iVar * nVar + jVar] = view(iVar, jVar); CBlasStructure().gemm(nVar, nVar, nVar, mat, jac, newJac, config); - for (auto iVar = 0u; iVar < nVar * nVar; iVar++) - block[iVar] = SU2_TYPE::GetValue(newJac[iVar]); + Jacobian.SetBlock(iPoint, jPoint, newJac); }; ModifyJacobian(iPoint); for (size_t iNeigh = 0; iNeigh < geometry->nodes->GetnPoint(iPoint); ++iNeigh) { @@ -1604,6 +1585,8 @@ void CFVMFlowSolverBase::EdgeFluxResidual(const CGeometry *geometry, ErrorCounter = 0; END_SU2_OMP_MASTER + su2activevector* massFluxes = config->GetBounded_Scalar() ? &EdgeMassFluxes : nullptr; + /*--- For hybrid parallel AD, pause preaccumulation if there is shared reading of * variables, otherwise switch to the faster adjoint evaluation mode. ---*/ bool pausePreacc = false; @@ -1624,9 +1607,9 @@ void CFVMFlowSolverBase::EdgeFluxResidual(const CGeometry *geometry, } if (ReducerStrategy) { - edgeNumerics->ComputeFlux(iEdge, *config, *geometry, *nodes, UpdateType::REDUCTION, mask, EdgeFluxes, Jacobian); + edgeNumerics->ComputeFlux(iEdge, *config, *geometry, *nodes, UpdateType::REDUCTION, mask, EdgeFluxes, Jacobian, massFluxes); } else { - edgeNumerics->ComputeFlux(iEdge, *config, *geometry, *nodes, UpdateType::COLORING, mask, LinSysRes, Jacobian); + edgeNumerics->ComputeFlux(iEdge, *config, *geometry, *nodes, UpdateType::COLORING, mask, LinSysRes, Jacobian, massFluxes); } if (MGLevel == MESH_0) { for (auto j = 0ul; j < Double::Size; ++j) @@ -1848,81 +1831,408 @@ void CFVMFlowSolverBase::SetResidual_DualTime(CGeometry *geometry } -template -void CFVMFlowSolverBase::Pressure_Forces(const CGeometry* geometry, const CConfig* config) { - unsigned long iVertex, iPoint; - unsigned short iDim, iMarker, Boundary, Monitoring, iMarker_Monitoring; - su2double Pressure = 0.0, NFPressOF, RefPressure; - const su2double *Normal = nullptr, *Coord = nullptr; - string Marker_Tag, Monitoring_Tag; - su2double AxiFactor; - - su2double Alpha = config->GetAoA() * PI_NUMBER / 180.0; - su2double Beta = config->GetAoS() * PI_NUMBER / 180.0; - su2double RefArea = config->GetRefArea(); - su2double RefLength = config->GetRefLength(); - auto Origin = config->GetRefOriginMoment(0); - bool axisymmetric = config->GetAxisymmetric(); +/*--- Helpers shared by Pressure_Forces, Momentum_Forces and Friction_Forces. Free function + * templates rather than members: none need instance state, and the AeroCoeffs/AeroCoeffsArray + * types are simply deduced from the caller's arguments. ---*/ +namespace { - SetReferenceValues(*config); +/*! + * \brief Whether a marker's boundary kind is one of the momentum (inlet/outlet/actuator-disk/ + * engine) surfaces handled by Momentum_Forces, used both to gate accumulation into + * MntCoeff and to gate the later CEff/CMerit derivation from it. + * \param[in] Boundary - Boundary kind of the marker (config->GetMarker_All_KindBC(iMarker)). + */ +inline bool IsMomentumBoundary(unsigned short Boundary) { + return (Boundary == INLET_FLOW) || (Boundary == OUTLET_FLOW) || (Boundary == ACTDISK_INLET) || + (Boundary == ACTDISK_OUTLET) || (Boundary == ENGINE_INFLOW) || (Boundary == ENGINE_EXHAUST); +} - const su2double factor = 1.0 / AeroCoeffForceRef; +/*! + * \brief Find iMarker's index within the monitoring markers, and its reference origin if found. + * \param[in] config - Problem definition. + * \param[in] iMarker - Marker to look up. + * \param[in] Monitoring - config->GetMarker_All_Monitoring(iMarker). + * \param[in,out] Origin - Set to iMarker's reference origin if found, left unchanged otherwise. + * \return Index within the monitoring markers, or -1 if iMarker is not monitored, or not found + * among the monitoring markers. + */ +inline int FindMonitoringIndex(const CConfig* config, unsigned long iMarker, unsigned short Monitoring, + std::array& Origin) { + if (Monitoring != YES) return -1; + const string Marker_Tag = config->GetMarker_All_TagBound(iMarker); + const auto nMarker_Monitoring = static_cast(config->GetnMarker_Monitoring()); + for (int iMarker_Monitoring = 0; iMarker_Monitoring < nMarker_Monitoring; iMarker_Monitoring++) { + if (Marker_Tag == config->GetMarker_Monitoring_TagBound(iMarker_Monitoring)) { + Origin = config->GetRefOriginMoment(iMarker_Monitoring); + return iMarker_Monitoring; + } + } + return -1; +} - /*--- Reference pressure is always the far-field value. ---*/ +/*! + * \brief Fold one thread's partial force/moment coefficient contribution (accumulated over + * its share of a marker's vertices) into the per-marker, AllBound and per-surface + * aerodynamic coefficient totals, in a single critical section. Does not touch + * CEff/CMerit (nonlinear ratios), those are derived once from the fully-reduced totals + * after all threads have merged in. + * \param[in] iMarker - Marker index the contribution belongs to. + * \param[in] iMarker_Monitoring - If iMarker is monitored, index within the monitoring markers, -1 otherwise. + * \param[in] partial - This thread's partial coefficients (CD, CL, ..., CQ only). + * \param[in,out] coeffArray - Per-marker totals to update at iMarker. + * \param[in,out] allBoundCoeff - Totals over all boundaries to update. + * \param[in,out] surfaceCoeff - Per-monitoring-surface totals to update. + */ +template +void AddCoeffContribution(unsigned long iMarker, int iMarker_Monitoring, const AeroCoeffsT& partial, + AeroCoeffsArrayT& coeffArray, AeroCoeffsT& allBoundCoeff, AeroCoeffsArrayT& surfaceCoeff) { + SU2_OMP_CRITICAL { + coeffArray.CD[iMarker] += partial.CD; + coeffArray.CL[iMarker] += partial.CL; + coeffArray.CSF[iMarker] += partial.CSF; + coeffArray.CFx[iMarker] += partial.CFx; + coeffArray.CFy[iMarker] += partial.CFy; + coeffArray.CFz[iMarker] += partial.CFz; + coeffArray.CMx[iMarker] += partial.CMx; + coeffArray.CMy[iMarker] += partial.CMy; + coeffArray.CMz[iMarker] += partial.CMz; + coeffArray.CoPx[iMarker] += partial.CoPx; + coeffArray.CoPy[iMarker] += partial.CoPy; + coeffArray.CoPz[iMarker] += partial.CoPz; + coeffArray.CT[iMarker] += partial.CT; + coeffArray.CQ[iMarker] += partial.CQ; + + allBoundCoeff.CD += partial.CD; + allBoundCoeff.CL += partial.CL; + allBoundCoeff.CSF += partial.CSF; + allBoundCoeff.CFx += partial.CFx; + allBoundCoeff.CFy += partial.CFy; + allBoundCoeff.CFz += partial.CFz; + allBoundCoeff.CMx += partial.CMx; + allBoundCoeff.CMy += partial.CMy; + allBoundCoeff.CMz += partial.CMz; + allBoundCoeff.CoPx += partial.CoPx; + allBoundCoeff.CoPy += partial.CoPy; + allBoundCoeff.CoPz += partial.CoPz; + allBoundCoeff.CT += partial.CT; + allBoundCoeff.CQ += partial.CQ; + + /*--- Compute the coefficients per surface ---*/ + + if (iMarker_Monitoring >= 0) { + surfaceCoeff.CL[iMarker_Monitoring] += partial.CL; + surfaceCoeff.CD[iMarker_Monitoring] += partial.CD; + surfaceCoeff.CSF[iMarker_Monitoring] += partial.CSF; + surfaceCoeff.CFx[iMarker_Monitoring] += partial.CFx; + surfaceCoeff.CFy[iMarker_Monitoring] += partial.CFy; + surfaceCoeff.CFz[iMarker_Monitoring] += partial.CFz; + surfaceCoeff.CMx[iMarker_Monitoring] += partial.CMx; + surfaceCoeff.CMy[iMarker_Monitoring] += partial.CMy; + surfaceCoeff.CMz[iMarker_Monitoring] += partial.CMz; + } + } + END_SU2_OMP_CRITICAL +} + +/*! + * \brief Project summed force/moment components onto the wind axes (Alpha, Beta) to get the + * standard aerodynamic coefficients. Identical formulas are used by Pressure_Forces, + * Momentum_Forces and Friction_Forces, applied respectively to their inviscid, + * momentum and viscous force/moment sums. Does not set CEff/CMerit (nonlinear ratios, + * derived later from fully-reduced totals) nor CSF/CMx/CMy/CFz/CoPz in 2D (n/a). + * \param[in] nDim - Number of spatial dimensions (2 or 3). + * \param[in] CosAlpha, SinAlpha, CosBeta, SinBeta - sin/cos of the angle of attack and sideslip, + * precomputed once by the caller (this is called once per monitored marker, redundantly + * by every thread, so recomputing the trig functions here would not be free). + * \param[in] Force - Summed force components (size MAXNDIM). + * \param[in] Moment - Summed moment components about Origin (size MAXNDIM). + * \param[in] MomentX_Force, MomentY_Force, MomentZ_Force - Summed moment-of-force components + * about the coordinate axes, used for the center-of-pressure coordinates. + * \return The wind-axis aerodynamic coefficients (CD, CL, CSF, CFx..CFz, CMx..CMz, CoPx..CoPz, + * CT, CQ). AeroCoeffsT is explicit at the call site (it can't be deduced, since it is + * only the return type): ComputeAeroCoeffsFromForceMoment(...). + */ +template +AeroCoeffsT ComputeAeroCoeffsFromForceMoment(unsigned short nDim, su2double CosAlpha, su2double SinAlpha, + su2double CosBeta, su2double SinBeta, const su2double* Force, + const su2double* Moment, const su2double* MomentX_Force, + const su2double* MomentY_Force, const su2double* MomentZ_Force) { + AeroCoeffsT c; + + if (nDim == 2) { + c.CD = Force[0] * CosAlpha + Force[1] * SinAlpha; + c.CL = -Force[0] * SinAlpha + Force[1] * CosAlpha; + c.CMz = Moment[2]; + c.CoPx = MomentZ_Force[1]; + c.CoPy = -MomentZ_Force[0]; + c.CFx = Force[0]; + c.CFy = Force[1]; + c.CT = -c.CFx; + c.CQ = -c.CMz; + } + if (nDim == 3) { + c.CD = Force[0] * CosAlpha * CosBeta + Force[1] * SinBeta + Force[2] * SinAlpha * CosBeta; + c.CL = -Force[0] * SinAlpha + Force[2] * CosAlpha; + c.CSF = -Force[0] * SinBeta * CosAlpha + Force[1] * CosBeta - Force[2] * SinBeta * SinAlpha; + c.CMx = Moment[0]; + c.CMy = Moment[1]; + c.CMz = Moment[2]; + c.CoPx = -MomentY_Force[0]; + c.CoPz = MomentY_Force[2]; + c.CFx = Force[0]; + c.CFy = Force[1]; + c.CFz = Force[2]; + c.CT = -c.CFz; + c.CQ = -c.CMz; + } + + return c; +} + +/*! + * \brief MPI-sum a single value across ranks (identity if not built with MPI). + * \param[in] x - Value to reduce. + * \return Sum of x over all ranks. + */ +inline su2double MPIReduceSum(su2double x) { +#ifdef HAVE_MPI + su2double tmp = x; + x = 0.0; + SU2_MPI::Allreduce(&tmp, &x, 1, MPI_DOUBLE, MPI_SUM, SU2_MPI::GetComm()); +#endif + return x; +} + +/*! + * \brief MPI-sum an array of per-monitoring-surface values across ranks, in place + * (no-op if not built with MPI). + * \param[in,out] x - Array of size n to reduce in place. + * \param[in] n - Number of entries in x. + */ +inline void MPIReduceSumInPlace(su2double* x, int n) { +#ifdef HAVE_MPI + if (SU2_MPI::GetSize() == SINGLE_NODE) return; + static vector buffer; + buffer.resize(n); + SU2_MPI::Allreduce(x, buffer.data(), n, MPI_DOUBLE, MPI_SUM, SU2_MPI::GetComm()); + for (int i = 0; i < n; ++i) x[i] = buffer[i]; +#endif +} + +/*! + * \brief MPI-reduce an AllBound/Surface aerodynamic coefficient pair across ranks (no-op if + * not built with MPI, or if the comm level does not require it). + * \param[in] config - Definition of the particular problem. + * \param[in,out] allBoundCoeff - Totals over all boundaries to reduce. + * \param[in,out] surfaceCoeff - Per-monitoring-surface totals to reduce. + */ +template +void ReduceCoeffsMPI(const CConfig* config, AeroCoeffsT& allBoundCoeff, AeroCoeffsArrayT& surfaceCoeff) { +#ifdef HAVE_MPI + if (config->GetComm_Level() != COMM_FULL) return; + + /*--- Add AllBound information using all the nodes ---*/ + + allBoundCoeff.CD = MPIReduceSum(allBoundCoeff.CD); + allBoundCoeff.CL = MPIReduceSum(allBoundCoeff.CL); + allBoundCoeff.CSF = MPIReduceSum(allBoundCoeff.CSF); + allBoundCoeff.CEff = allBoundCoeff.CL / (allBoundCoeff.CD + EPS); + + allBoundCoeff.CMx = MPIReduceSum(allBoundCoeff.CMx); + allBoundCoeff.CMy = MPIReduceSum(allBoundCoeff.CMy); + allBoundCoeff.CMz = MPIReduceSum(allBoundCoeff.CMz); + + allBoundCoeff.CoPx = MPIReduceSum(allBoundCoeff.CoPx); + allBoundCoeff.CoPy = MPIReduceSum(allBoundCoeff.CoPy); + allBoundCoeff.CoPz = MPIReduceSum(allBoundCoeff.CoPz); + + allBoundCoeff.CFx = MPIReduceSum(allBoundCoeff.CFx); + allBoundCoeff.CFy = MPIReduceSum(allBoundCoeff.CFy); + allBoundCoeff.CFz = MPIReduceSum(allBoundCoeff.CFz); + + allBoundCoeff.CT = MPIReduceSum(allBoundCoeff.CT); + allBoundCoeff.CQ = MPIReduceSum(allBoundCoeff.CQ); + allBoundCoeff.CMerit = allBoundCoeff.CT / (allBoundCoeff.CQ + EPS); + + /*--- Add the forces on the surfaces using all the nodes ---*/ + + const int nMarkerMon = config->GetnMarker_Monitoring(); + + MPIReduceSumInPlace(surfaceCoeff.CL, nMarkerMon); + MPIReduceSumInPlace(surfaceCoeff.CD, nMarkerMon); + MPIReduceSumInPlace(surfaceCoeff.CSF, nMarkerMon); + + for (int iMarker_Monitoring = 0; iMarker_Monitoring < nMarkerMon; iMarker_Monitoring++) + surfaceCoeff.CEff[iMarker_Monitoring] = + surfaceCoeff.CL[iMarker_Monitoring] / (surfaceCoeff.CD[iMarker_Monitoring] + EPS); + + MPIReduceSumInPlace(surfaceCoeff.CFx, nMarkerMon); + MPIReduceSumInPlace(surfaceCoeff.CFy, nMarkerMon); + MPIReduceSumInPlace(surfaceCoeff.CFz, nMarkerMon); + + MPIReduceSumInPlace(surfaceCoeff.CMx, nMarkerMon); + MPIReduceSumInPlace(surfaceCoeff.CMy, nMarkerMon); + MPIReduceSumInPlace(surfaceCoeff.CMz, nMarkerMon); +#endif +} + +/*! + * \brief Merge an AllBound/Surface aerodynamic coefficient pair into the running total/ + * surfaceTotal grand totals. + * \param[in] config - Definition of the particular problem. + * \param[in] allBoundCoeff - Totals over all boundaries to merge in (already MPI-reduced). + * \param[in] surfaceCoeff - Per-monitoring-surface totals to merge in (already MPI-reduced). + * \param[in,out] total - Grand total to update (e.g. the solver's TotalCoeff). + * \param[in,out] surfaceTotal - Per-surface grand total to update (e.g. the solver's SurfaceCoeff). + * \param[in] overwrite - True to overwrite total/surfaceTotal (first contributor, i.e. + * Pressure_Forces, which also resets them to zero beforehand), false to add to them. + */ +template +void AccumulateTotalCoeffs(const CConfig* config, const AeroCoeffsT& allBoundCoeff, + const AeroCoeffsArrayT& surfaceCoeff, AeroCoeffsT& total, AeroCoeffsArrayT& surfaceTotal, + bool overwrite) { + auto Update = [overwrite](su2double& dst, su2double src) { + if (overwrite) dst = src; + else dst += src; + }; + + Update(total.CD, allBoundCoeff.CD); + Update(total.CL, allBoundCoeff.CL); + Update(total.CSF, allBoundCoeff.CSF); + total.CEff = total.CL / (total.CD + EPS); + Update(total.CFx, allBoundCoeff.CFx); + Update(total.CFy, allBoundCoeff.CFy); + Update(total.CFz, allBoundCoeff.CFz); + Update(total.CMx, allBoundCoeff.CMx); + Update(total.CMy, allBoundCoeff.CMy); + Update(total.CMz, allBoundCoeff.CMz); + Update(total.CoPx, allBoundCoeff.CoPx); + Update(total.CoPy, allBoundCoeff.CoPy); + Update(total.CoPz, allBoundCoeff.CoPz); + Update(total.CT, allBoundCoeff.CT); + Update(total.CQ, allBoundCoeff.CQ); + total.CMerit = total.CT / (total.CQ + EPS); + + /*--- Update the total coefficients per surface (note that all the nodes have the same value)---*/ - RefPressure = Pressure_Inf; + for (unsigned short iMarker_Monitoring = 0; iMarker_Monitoring < config->GetnMarker_Monitoring(); + iMarker_Monitoring++) { + Update(surfaceTotal.CL[iMarker_Monitoring], surfaceCoeff.CL[iMarker_Monitoring]); + Update(surfaceTotal.CD[iMarker_Monitoring], surfaceCoeff.CD[iMarker_Monitoring]); + Update(surfaceTotal.CSF[iMarker_Monitoring], surfaceCoeff.CSF[iMarker_Monitoring]); + surfaceTotal.CEff[iMarker_Monitoring] = + surfaceTotal.CL[iMarker_Monitoring] / (surfaceTotal.CD[iMarker_Monitoring] + EPS); + Update(surfaceTotal.CFx[iMarker_Monitoring], surfaceCoeff.CFx[iMarker_Monitoring]); + Update(surfaceTotal.CFy[iMarker_Monitoring], surfaceCoeff.CFy[iMarker_Monitoring]); + Update(surfaceTotal.CFz[iMarker_Monitoring], surfaceCoeff.CFz[iMarker_Monitoring]); + Update(surfaceTotal.CMx[iMarker_Monitoring], surfaceCoeff.CMx[iMarker_Monitoring]); + Update(surfaceTotal.CMy[iMarker_Monitoring], surfaceCoeff.CMy[iMarker_Monitoring]); + Update(surfaceTotal.CMz[iMarker_Monitoring], surfaceCoeff.CMz[iMarker_Monitoring]); + } +} + +/*! + * \brief Fold one vertex's Force/MomentDist/Coord into the running moment sums. Identical + * formulas are used by Pressure_Forces, Momentum_Forces and Friction_Forces, applied + * respectively to their inviscid, momentum and viscous force/moment sums. + * \param[in] nDim - Number of spatial dimensions (2 or 3). + * \param[in] RefLength - Reference length used to non-dimensionalize the moments. + * \param[in] Force, MomentDist, Coord - This vertex's force, moment arm and position (size MAXNDIM). + * \param[in,out] Moment - Running moment sum about Origin. + * \param[in,out] MomentX_Force, MomentY_Force, MomentZ_Force - Running moment-of-force sums about + * the coordinate axes, used for the center-of-pressure coordinates. + */ +void AccumulateMoment(unsigned short nDim, su2double RefLength, const su2double* Force, + const su2double* MomentDist, const su2double* Coord, su2double* Moment, + su2double* MomentX_Force, su2double* MomentY_Force, su2double* MomentZ_Force) { + if (nDim == 3) { + Moment[0] += (Force[2] * MomentDist[1] - Force[1] * MomentDist[2]) / RefLength; + MomentX_Force[1] += (-Force[1] * Coord[2]); + MomentX_Force[2] += (Force[2] * Coord[1]); + + Moment[1] += (Force[0] * MomentDist[2] - Force[2] * MomentDist[0]) / RefLength; + MomentY_Force[2] += (-Force[2] * Coord[0]); + MomentY_Force[0] += (Force[0] * Coord[2]); + } + Moment[2] += (Force[1] * MomentDist[0] - Force[0] * MomentDist[1]) / RefLength; + MomentZ_Force[0] += (-Force[0] * Coord[1]); + MomentZ_Force[1] += (Force[1] * Coord[0]); +} - /*-- Variables initialization ---*/ +} // namespace - TotalCoeff.setZero(); +template +void CFVMFlowSolverBase::Pressure_Forces(const CGeometry* geometry, const CConfig* config) { + SU2_ZONE_SCOPED - Total_CNearFieldOF = 0.0; - Total_Heat = 0.0; - Total_MaxHeat = 0.0; + const su2double Alpha = config->GetAoA() * PI_NUMBER / 180.0; + const su2double Beta = config->GetAoS() * PI_NUMBER / 180.0; + /*--- Precomputed once here since ComputeAeroCoeffsFromForceMoment is called once per monitored + * marker, redundantly by every thread (see below). ---*/ + const su2double CosAlpha = cos(Alpha), SinAlpha = sin(Alpha), CosBeta = cos(Beta), SinBeta = sin(Beta); + const su2double RefArea = config->GetRefArea(); + const su2double RefLength = config->GetRefLength(); + auto Origin = config->GetRefOriginMoment(0); + const bool axisymmetric = config->GetAxisymmetric(); - AllBoundInvCoeff.setZero(); + /*--- Variables initialization, and other writes to shared (possibly AD-active) state, + * are confined to the master thread, synchronized with a barrier so that the + * subsequent parallel loop over markers sees consistent, zeroed accumulators. ---*/ - AllBound_CNearFieldOF_Inv = 0.0; + BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS { + SetReferenceValues(*config); + Total_CNearFieldOF = 0.0; + Total_Heat = 0.0; + Total_MaxHeat = 0.0; + AllBound_CNearFieldOF_Inv = 0.0; + /*--- AeroCoeffs::setZero is not parallel. ---*/ + AllBoundInvCoeff.setZero(); + TotalCoeff.setZero(); + } + END_SU2_OMP_SAFE_GLOBAL_ACCESS SurfaceInvCoeff.setZero(); SurfaceCoeff.setZero(); + InvCoeff.setZero(); - /*--- Loop over the Euler and Navier-Stokes markers ---*/ + SU2_OMP_FOR_STAT(OMP_MIN_SIZE) + for (unsigned long iMarker = 0; iMarker < nMarker; iMarker++) CNearFieldOF_Inv[iMarker] = 0.0; + END_SU2_OMP_FOR - for (iMarker = 0; iMarker < nMarker; iMarker++) { - Boundary = config->GetMarker_All_KindBC(iMarker); - Monitoring = config->GetMarker_All_Monitoring(iMarker); + const su2double factor = 1.0 / AeroCoeffForceRef; - /*--- Obtain the origin for the moment computation for a particular marker ---*/ + /*--- Reference pressure is always the far-field value. ---*/ - if (Monitoring == YES) { - for (iMarker_Monitoring = 0; iMarker_Monitoring < config->GetnMarker_Monitoring(); iMarker_Monitoring++) { - Monitoring_Tag = config->GetMarker_Monitoring_TagBound(iMarker_Monitoring); - Marker_Tag = config->GetMarker_All_TagBound(iMarker); - if (Marker_Tag == Monitoring_Tag) Origin = config->GetRefOriginMoment(iMarker_Monitoring); - } - } + const su2double RefPressure = Pressure_Inf; + + /*--- Loop over the Euler and Navier-Stokes markers. Every thread runs this marker loop + * redundantly; only the per-vertex loop nested inside it is work-shared (SU2_OMP_FOR_STAT). + * Each thread accumulates its own partial force/moment sums and folds them into the shared + * per-marker/AllBound/Surface totals via AddCoeffContribution. CEff/CMerit are nonlinear + * ratios, so they are derived once after the loop, from the fully-reduced totals. ---*/ - if (config->GetSolid_Wall(iMarker) || (Boundary == NEARFIELD_BOUNDARY) || (Boundary == INLET_FLOW) || - (Boundary == OUTLET_FLOW) || (Boundary == ACTDISK_INLET) || (Boundary == ACTDISK_OUTLET) || - (Boundary == ENGINE_INFLOW) || (Boundary == ENGINE_EXHAUST)) { - /*--- Forces initialization at each Marker ---*/ + for (unsigned long iMarker = 0; iMarker < nMarker; iMarker++) { + const auto Boundary = config->GetMarker_All_KindBC(iMarker); + + if (config->GetSolid_Wall(iMarker) || (Boundary == NEARFIELD_BOUNDARY) || IsMomentumBoundary(Boundary)) { - InvCoeff.setZero(iMarker); + /*--- Obtain the origin for the moment computation for a particular marker ---*/ - CNearFieldOF_Inv[iMarker] = 0.0; + const auto Monitoring = config->GetMarker_All_Monitoring(iMarker); + const int iMarker_Monitoring = FindMonitoringIndex(config, iMarker, Monitoring, Origin); su2double ForceInviscid[MAXNDIM] = {0.0}, MomentInviscid[MAXNDIM] = {0.0}; su2double MomentX_Force[MAXNDIM] = {0.0}, MomentY_Force[MAXNDIM] = {0.0}, MomentZ_Force[MAXNDIM] = {0.0}; - NFPressOF = 0.0; + su2double NFPressOF = 0.0; /*--- Loop over the vertices to compute the forces ---*/ - for (iVertex = 0; iVertex < geometry->GetnVertex(iMarker); iVertex++) { - iPoint = geometry->vertex[iMarker][iVertex]->GetNode(); + SU2_OMP_FOR_(schedule(static, OMP_MIN_SIZE) SU2_NOWAIT) + for (unsigned long iVertex = 0; iVertex < geometry->GetnVertex(iMarker); iVertex++) { + const auto iPoint = geometry->vertex[iMarker][iVertex]->GetNode(); - Pressure = nodes->GetPressure(iPoint); + const su2double Pressure = nodes->GetPressure(iPoint); CPressure[iMarker][iVertex] = (Pressure - RefPressure) * factor * RefArea; @@ -1930,8 +2240,8 @@ void CFVMFlowSolverBase::Pressure_Forces(const CGeometry* geometr halo cells (for visualization purposes), but not the forces ---*/ if ((geometry->nodes->GetDomain(iPoint)) && (Monitoring == YES)) { - Normal = geometry->vertex[iMarker][iVertex]->GetNormal(); - Coord = geometry->nodes->GetCoord(iPoint); + const su2double* Normal = geometry->vertex[iMarker][iVertex]->GetNormal(); + const su2double* Coord = geometry->nodes->GetCoord(iPoint); /*--- Quadratic objective function for the near-field. This uses the infinity pressure regardless of Mach number. ---*/ @@ -1939,533 +2249,220 @@ void CFVMFlowSolverBase::Pressure_Forces(const CGeometry* geometr NFPressOF += 0.5 * (Pressure - Pressure_Inf) * (Pressure - Pressure_Inf) * Normal[nDim - 1]; su2double MomentDist[MAXNDIM] = {0.0}; - for (iDim = 0; iDim < nDim; iDim++) { - MomentDist[iDim] = Coord[iDim] - Origin[iDim]; - } + GeometryToolbox::Distance(nDim, Coord, Origin.data(), MomentDist); /*--- Axisymmetric simulations ---*/ - if (axisymmetric) - AxiFactor = 2.0 * PI_NUMBER * geometry->nodes->GetCoord(iPoint, 1); - else - AxiFactor = 1.0; + const su2double AxiFactor = axisymmetric ? su2double(2.0 * PI_NUMBER * geometry->nodes->GetCoord(iPoint, 1)) : su2double(1.0); /*--- Force computation, note the minus sign due to the orientation of the normal (outward) ---*/ su2double Force[MAXNDIM] = {0.0}; - for (iDim = 0; iDim < nDim; iDim++) { + for (unsigned short iDim = 0; iDim < nDim; iDim++) { Force[iDim] = -(Pressure - Pressure_Inf) * Normal[iDim] * factor * AxiFactor; ForceInviscid[iDim] += Force[iDim]; } /*--- Moment with respect to the reference axis ---*/ - if (nDim == 3) { - MomentInviscid[0] += (Force[2] * MomentDist[1] - Force[1] * MomentDist[2]) / RefLength; - MomentX_Force[1] += (-Force[1] * Coord[2]); - MomentX_Force[2] += (Force[2] * Coord[1]); - - MomentInviscid[1] += (Force[0] * MomentDist[2] - Force[2] * MomentDist[0]) / RefLength; - MomentY_Force[2] += (-Force[2] * Coord[0]); - MomentY_Force[0] += (Force[0] * Coord[2]); - } - MomentInviscid[2] += (Force[1] * MomentDist[0] - Force[0] * MomentDist[1]) / RefLength; - MomentZ_Force[0] += (-Force[0] * Coord[1]); - MomentZ_Force[1] += (Force[1] * Coord[0]); + AccumulateMoment(nDim, RefLength, Force, MomentDist, Coord, MomentInviscid, MomentX_Force, MomentY_Force, + MomentZ_Force); } } - - /*--- Project forces and store the non-dimensional coefficients ---*/ + END_SU2_OMP_FOR if (Monitoring == YES) { if (Boundary != NEARFIELD_BOUNDARY) { - if (nDim == 2) { - InvCoeff.CD[iMarker] = ForceInviscid[0] * cos(Alpha) + ForceInviscid[1] * sin(Alpha); - InvCoeff.CL[iMarker] = -ForceInviscid[0] * sin(Alpha) + ForceInviscid[1] * cos(Alpha); - InvCoeff.CEff[iMarker] = InvCoeff.CL[iMarker] / (InvCoeff.CD[iMarker] + EPS); - InvCoeff.CMz[iMarker] = MomentInviscid[2]; - InvCoeff.CoPx[iMarker] = MomentZ_Force[1]; - InvCoeff.CoPy[iMarker] = -MomentZ_Force[0]; - InvCoeff.CFx[iMarker] = ForceInviscid[0]; - InvCoeff.CFy[iMarker] = ForceInviscid[1]; - InvCoeff.CT[iMarker] = -InvCoeff.CFx[iMarker]; - InvCoeff.CQ[iMarker] = -InvCoeff.CMz[iMarker]; - InvCoeff.CMerit[iMarker] = InvCoeff.CT[iMarker] / (InvCoeff.CQ[iMarker] + EPS); - } - if (nDim == 3) { - InvCoeff.CD[iMarker] = ForceInviscid[0] * cos(Alpha) * cos(Beta) + ForceInviscid[1] * sin(Beta) + - ForceInviscid[2] * sin(Alpha) * cos(Beta); - InvCoeff.CL[iMarker] = -ForceInviscid[0] * sin(Alpha) + ForceInviscid[2] * cos(Alpha); - InvCoeff.CSF[iMarker] = -ForceInviscid[0] * sin(Beta) * cos(Alpha) + ForceInviscid[1] * cos(Beta) - - ForceInviscid[2] * sin(Beta) * sin(Alpha); - InvCoeff.CEff[iMarker] = InvCoeff.CL[iMarker] / (InvCoeff.CD[iMarker] + EPS); - InvCoeff.CMx[iMarker] = MomentInviscid[0]; - InvCoeff.CMy[iMarker] = MomentInviscid[1]; - InvCoeff.CMz[iMarker] = MomentInviscid[2]; - InvCoeff.CoPx[iMarker] = -MomentY_Force[0]; - InvCoeff.CoPz[iMarker] = MomentY_Force[2]; - InvCoeff.CFx[iMarker] = ForceInviscid[0]; - InvCoeff.CFy[iMarker] = ForceInviscid[1]; - InvCoeff.CFz[iMarker] = ForceInviscid[2]; - InvCoeff.CT[iMarker] = -InvCoeff.CFz[iMarker]; - InvCoeff.CQ[iMarker] = -InvCoeff.CMz[iMarker]; - InvCoeff.CMerit[iMarker] = InvCoeff.CT[iMarker] / (InvCoeff.CQ[iMarker] + EPS); - } + const auto partial = ComputeAeroCoeffsFromForceMoment( + nDim, CosAlpha, SinAlpha, CosBeta, SinBeta, ForceInviscid, MomentInviscid, MomentX_Force, MomentY_Force, + MomentZ_Force); - AllBoundInvCoeff.CD += InvCoeff.CD[iMarker]; - AllBoundInvCoeff.CL += InvCoeff.CL[iMarker]; - AllBoundInvCoeff.CSF += InvCoeff.CSF[iMarker]; - AllBoundInvCoeff.CEff = AllBoundInvCoeff.CL / (AllBoundInvCoeff.CD + EPS); - AllBoundInvCoeff.CMx += InvCoeff.CMx[iMarker]; - AllBoundInvCoeff.CMy += InvCoeff.CMy[iMarker]; - AllBoundInvCoeff.CMz += InvCoeff.CMz[iMarker]; - AllBoundInvCoeff.CoPx += InvCoeff.CoPx[iMarker]; - AllBoundInvCoeff.CoPy += InvCoeff.CoPy[iMarker]; - AllBoundInvCoeff.CoPz += InvCoeff.CoPz[iMarker]; - AllBoundInvCoeff.CFx += InvCoeff.CFx[iMarker]; - AllBoundInvCoeff.CFy += InvCoeff.CFy[iMarker]; - AllBoundInvCoeff.CFz += InvCoeff.CFz[iMarker]; - AllBoundInvCoeff.CT += InvCoeff.CT[iMarker]; - AllBoundInvCoeff.CQ += InvCoeff.CQ[iMarker]; - AllBoundInvCoeff.CMerit = AllBoundInvCoeff.CT / (AllBoundInvCoeff.CQ + EPS); - - /*--- Compute the coefficients per surface ---*/ - - for (iMarker_Monitoring = 0; iMarker_Monitoring < config->GetnMarker_Monitoring(); iMarker_Monitoring++) { - Monitoring_Tag = config->GetMarker_Monitoring_TagBound(iMarker_Monitoring); - Marker_Tag = config->GetMarker_All_TagBound(iMarker); - if (Marker_Tag == Monitoring_Tag) { - SurfaceInvCoeff.CL[iMarker_Monitoring] += InvCoeff.CL[iMarker]; - SurfaceInvCoeff.CD[iMarker_Monitoring] += InvCoeff.CD[iMarker]; - SurfaceInvCoeff.CSF[iMarker_Monitoring] += InvCoeff.CSF[iMarker]; - SurfaceInvCoeff.CEff[iMarker_Monitoring] = SurfaceInvCoeff.CL[iMarker_Monitoring] / (SurfaceInvCoeff.CD[iMarker_Monitoring] + EPS); - SurfaceInvCoeff.CFx[iMarker_Monitoring] += InvCoeff.CFx[iMarker]; - SurfaceInvCoeff.CFy[iMarker_Monitoring] += InvCoeff.CFy[iMarker]; - SurfaceInvCoeff.CFz[iMarker_Monitoring] += InvCoeff.CFz[iMarker]; - SurfaceInvCoeff.CMx[iMarker_Monitoring] += InvCoeff.CMx[iMarker]; - SurfaceInvCoeff.CMy[iMarker_Monitoring] += InvCoeff.CMy[iMarker]; - SurfaceInvCoeff.CMz[iMarker_Monitoring] += InvCoeff.CMz[iMarker]; - } + AddCoeffContribution(iMarker, iMarker_Monitoring, partial, InvCoeff, AllBoundInvCoeff, SurfaceInvCoeff); + } else { + /*--- At the Nearfield SU2 only cares about the pressure coeffient ---*/ + SU2_OMP_CRITICAL { + CNearFieldOF_Inv[iMarker] += NFPressOF; + AllBound_CNearFieldOF_Inv += NFPressOF; } - - } - - /*--- At the Nearfield SU2 only cares about the pressure coeffient ---*/ - - else { - CNearFieldOF_Inv[iMarker] = NFPressOF; - AllBound_CNearFieldOF_Inv += CNearFieldOF_Inv[iMarker]; + END_SU2_OMP_CRITICAL } } } } + /*--- For the SU2_NOWAIT in the vertex loop. ---*/ + SU2_OMP_BARRIER -#ifdef HAVE_MPI - - /*--- Add AllBound information using all the nodes ---*/ - - if (config->GetComm_Level() == COMM_FULL) { - auto Allreduce = [](su2double x) { - su2double tmp = x; - x = 0.0; - SU2_MPI::Allreduce(&tmp, &x, 1, MPI_DOUBLE, MPI_SUM, SU2_MPI::GetComm()); - return x; - }; - AllBoundInvCoeff.CD = Allreduce(AllBoundInvCoeff.CD); - AllBoundInvCoeff.CL = Allreduce(AllBoundInvCoeff.CL); - AllBoundInvCoeff.CSF = Allreduce(AllBoundInvCoeff.CSF); - AllBoundInvCoeff.CEff = AllBoundInvCoeff.CL / (AllBoundInvCoeff.CD + EPS); - - AllBoundInvCoeff.CMx = Allreduce(AllBoundInvCoeff.CMx); - AllBoundInvCoeff.CMy = Allreduce(AllBoundInvCoeff.CMy); - AllBoundInvCoeff.CMz = Allreduce(AllBoundInvCoeff.CMz); - - AllBoundInvCoeff.CoPx = Allreduce(AllBoundInvCoeff.CoPx); - AllBoundInvCoeff.CoPy = Allreduce(AllBoundInvCoeff.CoPy); - AllBoundInvCoeff.CoPz = Allreduce(AllBoundInvCoeff.CoPz); + /*--- Derive the (nonlinear) per-marker, AllBound and Surface ratio coefficients from the + * now fully-reduced totals. This must happen once, after every thread has finished + * folding its partial contributions above (guaranteed by the barrier at the start of + * the safe-global-access section). ---*/ - AllBoundInvCoeff.CFx = Allreduce(AllBoundInvCoeff.CFx); - AllBoundInvCoeff.CFy = Allreduce(AllBoundInvCoeff.CFy); - AllBoundInvCoeff.CFz = Allreduce(AllBoundInvCoeff.CFz); - - AllBoundInvCoeff.CT = Allreduce(AllBoundInvCoeff.CT); - AllBoundInvCoeff.CQ = Allreduce(AllBoundInvCoeff.CQ); - AllBoundInvCoeff.CMerit = AllBoundInvCoeff.CT / (AllBoundInvCoeff.CQ + EPS); - AllBound_CNearFieldOF_Inv = Allreduce(AllBound_CNearFieldOF_Inv); + SU2_OMP_FOR_(schedule(static, OMP_MIN_SIZE) SU2_NOWAIT) + for (unsigned long iMarker = 0; iMarker < nMarker; iMarker++) { + const auto Boundary = config->GetMarker_All_KindBC(iMarker); + const auto Monitoring = config->GetMarker_All_Monitoring(iMarker); + if (Monitoring == YES && Boundary != NEARFIELD_BOUNDARY) { + InvCoeff.CEff[iMarker] = InvCoeff.CL[iMarker] / (InvCoeff.CD[iMarker] + EPS); + InvCoeff.CMerit[iMarker] = InvCoeff.CT[iMarker] / (InvCoeff.CQ[iMarker] + EPS); + } } + END_SU2_OMP_FOR - /*--- Add the forces on the surfaces using all the nodes ---*/ - - if (config->GetComm_Level() == COMM_FULL) { - int nMarkerMon = config->GetnMarker_Monitoring(); - - /*--- Use the same buffer for all reductions. We could avoid the copy back into - * the original variable by swaping pointers, but it is safer this way... ---*/ - - su2double* buffer = new su2double[nMarkerMon]; - - auto Allreduce_inplace = [buffer](int size, su2double* x) { - SU2_MPI::Allreduce(x, buffer, size, MPI_DOUBLE, MPI_SUM, SU2_MPI::GetComm()); - for (int i = 0; i < size; ++i) x[i] = buffer[i]; - }; - - Allreduce_inplace(nMarkerMon, SurfaceInvCoeff.CL); - Allreduce_inplace(nMarkerMon, SurfaceInvCoeff.CD); - Allreduce_inplace(nMarkerMon, SurfaceInvCoeff.CSF); - - for (iMarker_Monitoring = 0; iMarker_Monitoring < nMarkerMon; iMarker_Monitoring++) - SurfaceInvCoeff.CEff[iMarker_Monitoring] = - SurfaceInvCoeff.CL[iMarker_Monitoring] / (SurfaceInvCoeff.CD[iMarker_Monitoring] + EPS); - - Allreduce_inplace(nMarkerMon, SurfaceInvCoeff.CFx); - Allreduce_inplace(nMarkerMon, SurfaceInvCoeff.CFy); - Allreduce_inplace(nMarkerMon, SurfaceInvCoeff.CFz); - - Allreduce_inplace(nMarkerMon, SurfaceInvCoeff.CMx); - Allreduce_inplace(nMarkerMon, SurfaceInvCoeff.CMy); - Allreduce_inplace(nMarkerMon, SurfaceInvCoeff.CMz); - - delete[] buffer; + SU2_OMP_FOR_(schedule(static, OMP_MIN_SIZE) SU2_NOWAIT) + for (unsigned short iMarker_Monitoring = 0; iMarker_Monitoring < config->GetnMarker_Monitoring(); + iMarker_Monitoring++) { + SurfaceInvCoeff.CEff[iMarker_Monitoring] = + SurfaceInvCoeff.CL[iMarker_Monitoring] / (SurfaceInvCoeff.CD[iMarker_Monitoring] + EPS); } + END_SU2_OMP_FOR -#endif + BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS { + AllBoundInvCoeff.CEff = AllBoundInvCoeff.CL / (AllBoundInvCoeff.CD + EPS); + AllBoundInvCoeff.CMerit = AllBoundInvCoeff.CT / (AllBoundInvCoeff.CQ + EPS); - /*--- Update the total coefficients (note that all the nodes have the same value) ---*/ - - TotalCoeff.CD = AllBoundInvCoeff.CD; - TotalCoeff.CL = AllBoundInvCoeff.CL; - TotalCoeff.CSF = AllBoundInvCoeff.CSF; - TotalCoeff.CEff = TotalCoeff.CL / (TotalCoeff.CD + EPS); - TotalCoeff.CFx = AllBoundInvCoeff.CFx; - TotalCoeff.CFy = AllBoundInvCoeff.CFy; - TotalCoeff.CFz = AllBoundInvCoeff.CFz; - TotalCoeff.CMx = AllBoundInvCoeff.CMx; - TotalCoeff.CMy = AllBoundInvCoeff.CMy; - TotalCoeff.CMz = AllBoundInvCoeff.CMz; - TotalCoeff.CoPx = AllBoundInvCoeff.CoPx; - TotalCoeff.CoPy = AllBoundInvCoeff.CoPy; - TotalCoeff.CoPz = AllBoundInvCoeff.CoPz; - TotalCoeff.CT = AllBoundInvCoeff.CT; - TotalCoeff.CQ = AllBoundInvCoeff.CQ; - TotalCoeff.CMerit = TotalCoeff.CT / (TotalCoeff.CQ + EPS); - Total_CNearFieldOF = AllBound_CNearFieldOF_Inv; + ReduceCoeffsMPI(config, AllBoundInvCoeff, SurfaceInvCoeff); - /*--- Update the total coefficients per surface (note that all the nodes have the same value)---*/ + /*--- AllBound_CNearFieldOF_Inv, not covered by ReduceCoeffsMPI, is reduced separately. ---*/ + if (config->GetComm_Level() == COMM_FULL) { + AllBound_CNearFieldOF_Inv = MPIReduceSum(AllBound_CNearFieldOF_Inv); + } - for (iMarker_Monitoring = 0; iMarker_Monitoring < config->GetnMarker_Monitoring(); iMarker_Monitoring++) { - SurfaceCoeff.CL[iMarker_Monitoring] = SurfaceInvCoeff.CL[iMarker_Monitoring]; - SurfaceCoeff.CD[iMarker_Monitoring] = SurfaceInvCoeff.CD[iMarker_Monitoring]; - SurfaceCoeff.CSF[iMarker_Monitoring] = SurfaceInvCoeff.CSF[iMarker_Monitoring]; - SurfaceCoeff.CEff[iMarker_Monitoring] = - SurfaceCoeff.CL[iMarker_Monitoring] / (SurfaceCoeff.CD[iMarker_Monitoring] + EPS); - SurfaceCoeff.CFx[iMarker_Monitoring] = SurfaceInvCoeff.CFx[iMarker_Monitoring]; - SurfaceCoeff.CFy[iMarker_Monitoring] = SurfaceInvCoeff.CFy[iMarker_Monitoring]; - SurfaceCoeff.CFz[iMarker_Monitoring] = SurfaceInvCoeff.CFz[iMarker_Monitoring]; - SurfaceCoeff.CMx[iMarker_Monitoring] = SurfaceInvCoeff.CMx[iMarker_Monitoring]; - SurfaceCoeff.CMy[iMarker_Monitoring] = SurfaceInvCoeff.CMy[iMarker_Monitoring]; - SurfaceCoeff.CMz[iMarker_Monitoring] = SurfaceInvCoeff.CMz[iMarker_Monitoring]; + AccumulateTotalCoeffs(config, AllBoundInvCoeff, SurfaceInvCoeff, TotalCoeff, SurfaceCoeff, /*overwrite=*/true); + Total_CNearFieldOF = AllBound_CNearFieldOF_Inv; } + END_SU2_OMP_SAFE_GLOBAL_ACCESS } template void CFVMFlowSolverBase::Momentum_Forces(const CGeometry* geometry, const CConfig* config) { - unsigned long iVertex, iPoint; - unsigned short iDim, iMarker, Boundary, Monitoring, iMarker_Monitoring; - su2double MassFlow, Density; - const su2double *Normal = nullptr, *Coord = nullptr; - string Marker_Tag, Monitoring_Tag; - su2double AxiFactor; - - su2double Alpha = config->GetAoA() * PI_NUMBER / 180.0; - su2double Beta = config->GetAoS() * PI_NUMBER / 180.0; - su2double RefLength = config->GetRefLength(); + SU2_ZONE_SCOPED + + const su2double Alpha = config->GetAoA() * PI_NUMBER / 180.0; + const su2double Beta = config->GetAoS() * PI_NUMBER / 180.0; + const su2double CosAlpha = cos(Alpha), SinAlpha = sin(Alpha), CosBeta = cos(Beta), SinBeta = sin(Beta); + const su2double RefLength = config->GetRefLength(); auto Origin = config->GetRefOriginMoment(0); - bool axisymmetric = config->GetAxisymmetric(); + const bool axisymmetric = config->GetAxisymmetric(); const su2double factor = 1.0 / AeroCoeffForceRef; - /*-- Variables initialization ---*/ - - AllBoundMntCoeff.setZero(); + SU2_OMP_SAFE_GLOBAL_ACCESS(AllBoundMntCoeff.setZero();) SurfaceMntCoeff.setZero(); + MntCoeff.setZero(); - /*--- Loop over the Inlet -Outlet Markers ---*/ - - for (iMarker = 0; iMarker < nMarker; iMarker++) { - Boundary = config->GetMarker_All_KindBC(iMarker); - Monitoring = config->GetMarker_All_Monitoring(iMarker); - - /*--- Obtain the origin for the moment computation for a particular marker ---*/ - - if (Monitoring == YES) { - for (iMarker_Monitoring = 0; iMarker_Monitoring < config->GetnMarker_Monitoring(); iMarker_Monitoring++) { - Monitoring_Tag = config->GetMarker_Monitoring_TagBound(iMarker_Monitoring); - Marker_Tag = config->GetMarker_All_TagBound(iMarker); - if (Marker_Tag == Monitoring_Tag) Origin = config->GetRefOriginMoment(iMarker_Monitoring); - } - } - - if ((Boundary == INLET_FLOW) || (Boundary == OUTLET_FLOW) || (Boundary == ACTDISK_INLET) || - (Boundary == ACTDISK_OUTLET) || (Boundary == ENGINE_INFLOW) || (Boundary == ENGINE_EXHAUST)) { - /*--- Forces initialization at each Marker ---*/ + /*--- Loop over the Inlet-Outlet Markers (see Pressure_Forces for how the parallel + * reduction over threads is organized). ---*/ - MntCoeff.setZero(iMarker); + for (unsigned long iMarker = 0; iMarker < nMarker; iMarker++) { + const auto Boundary = config->GetMarker_All_KindBC(iMarker); + if (!IsMomentumBoundary(Boundary)) continue; - su2double ForceMomentum[MAXNDIM] = {0.0}, MomentMomentum[MAXNDIM] = {0.0}; - su2double MomentX_Force[3] = {0.0}, MomentY_Force[3] = {0.0}, MomentZ_Force[3] = {0.0}; + const auto Monitoring = config->GetMarker_All_Monitoring(iMarker); - /*--- Loop over the vertices to compute the forces ---*/ + /*--- Obtain the origin for the moment computation for a particular marker ---*/ - for (iVertex = 0; iVertex < geometry->GetnVertex(iMarker); iVertex++) { - iPoint = geometry->vertex[iMarker][iVertex]->GetNode(); + const int iMarker_Monitoring = FindMonitoringIndex(config, iMarker, Monitoring, Origin); - /*--- Note that the pressure coefficient is computed at the - halo cells (for visualization purposes), but not the forces ---*/ + su2double ForceMomentum[MAXNDIM] = {0.0}, MomentMomentum[MAXNDIM] = {0.0}; + su2double MomentX_Force[3] = {0.0}, MomentY_Force[3] = {0.0}, MomentZ_Force[3] = {0.0}; - if ((geometry->nodes->GetDomain(iPoint)) && (Monitoring == YES)) { - Normal = geometry->vertex[iMarker][iVertex]->GetNormal(); - Coord = geometry->nodes->GetCoord(iPoint); - Density = nodes->GetDensity(iPoint); - MassFlow = 0.0; - su2double Velocity[MAXNDIM] = {0.0}, MomentDist[MAXNDIM] = {0.0}; - for (iDim = 0; iDim < nDim; iDim++) { - Velocity[iDim] = nodes->GetVelocity(iPoint, iDim); - MomentDist[iDim] = Coord[iDim] - Origin[iDim]; - MassFlow -= Normal[iDim] * Velocity[iDim] * Density; - } + /*--- Loop over the vertices to compute the forces (work-shared across threads, see + * Pressure_Forces for why the chunk size is computed and the barrier skipped). ---*/ - /*--- Axisymmetric simulations ---*/ + SU2_OMP_FOR_(schedule(static, OMP_MIN_SIZE) SU2_NOWAIT) + for (unsigned long iVertex = 0; iVertex < geometry->GetnVertex(iMarker); iVertex++) { + const auto iPoint = geometry->vertex[iMarker][iVertex]->GetNode(); - if (axisymmetric) - AxiFactor = 2.0 * PI_NUMBER * geometry->nodes->GetCoord(iPoint, 1); - else - AxiFactor = 1.0; + /*--- Note that the pressure coefficient is computed at the + halo cells (for visualization purposes), but not the forces ---*/ - /*--- Force computation, note the minus sign due to the - orientation of the normal (outward) ---*/ + if ((geometry->nodes->GetDomain(iPoint)) && (Monitoring == YES)) { + const su2double* Normal = geometry->vertex[iMarker][iVertex]->GetNormal(); + const su2double* Coord = geometry->nodes->GetCoord(iPoint); + const su2double Density = nodes->GetDensity(iPoint); + su2double MassFlow = 0.0; + su2double Velocity[MAXNDIM] = {0.0}, MomentDist[MAXNDIM] = {0.0}; + for (unsigned short iDim = 0; iDim < nDim; iDim++) { + Velocity[iDim] = nodes->GetVelocity(iPoint, iDim); + MomentDist[iDim] = Coord[iDim] - Origin[iDim]; + MassFlow -= Normal[iDim] * Velocity[iDim] * Density; + } - su2double Force[MAXNDIM] = {0.0}; - for (iDim = 0; iDim < nDim; iDim++) { - Force[iDim] = MassFlow * Velocity[iDim] * factor * AxiFactor; - ForceMomentum[iDim] += Force[iDim]; - } + /*--- Axisymmetric simulations ---*/ - /*--- Moment with respect to the reference axis ---*/ + const su2double AxiFactor = axisymmetric ? su2double(2.0 * PI_NUMBER * geometry->nodes->GetCoord(iPoint, 1)) : su2double(1.0); - if (nDim == 3) { - MomentMomentum[0] += (Force[2] * MomentDist[1] - Force[1] * MomentDist[2]) / RefLength; - MomentX_Force[1] += (-Force[1] * Coord[2]); - MomentX_Force[2] += (Force[2] * Coord[1]); + /*--- Force computation, note the minus sign due to the + orientation of the normal (outward) ---*/ - MomentMomentum[1] += (Force[0] * MomentDist[2] - Force[2] * MomentDist[0]) / RefLength; - MomentY_Force[2] += (-Force[2] * Coord[0]); - MomentY_Force[0] += (Force[0] * Coord[2]); - } - MomentMomentum[2] += (Force[1] * MomentDist[0] - Force[0] * MomentDist[1]) / RefLength; - MomentZ_Force[0] += (-Force[0] * Coord[1]); - MomentZ_Force[1] += (Force[1] * Coord[0]); + su2double Force[MAXNDIM] = {0.0}; + for (unsigned short iDim = 0; iDim < nDim; iDim++) { + Force[iDim] = MassFlow * Velocity[iDim] * factor * AxiFactor; + ForceMomentum[iDim] += Force[iDim]; } - } - /*--- Project forces and store the non-dimensional coefficients ---*/ - - if (Monitoring == YES) { - if (nDim == 2) { - MntCoeff.CD[iMarker] = ForceMomentum[0] * cos(Alpha) + ForceMomentum[1] * sin(Alpha); - MntCoeff.CL[iMarker] = -ForceMomentum[0] * sin(Alpha) + ForceMomentum[1] * cos(Alpha); - MntCoeff.CEff[iMarker] = MntCoeff.CL[iMarker] / (MntCoeff.CD[iMarker] + EPS); - MntCoeff.CFx[iMarker] = ForceMomentum[0]; - MntCoeff.CFy[iMarker] = ForceMomentum[1]; - MntCoeff.CMz[iMarker] = MomentMomentum[2]; - MntCoeff.CoPx[iMarker] = MomentZ_Force[1]; - MntCoeff.CoPy[iMarker] = -MomentZ_Force[0]; - MntCoeff.CT[iMarker] = -MntCoeff.CFx[iMarker]; - MntCoeff.CQ[iMarker] = -MntCoeff.CMz[iMarker]; - MntCoeff.CMerit[iMarker] = MntCoeff.CT[iMarker] / (MntCoeff.CQ[iMarker] + EPS); - } - if (nDim == 3) { - MntCoeff.CD[iMarker] = ForceMomentum[0] * cos(Alpha) * cos(Beta) + ForceMomentum[1] * sin(Beta) + - ForceMomentum[2] * sin(Alpha) * cos(Beta); - MntCoeff.CL[iMarker] = -ForceMomentum[0] * sin(Alpha) + ForceMomentum[2] * cos(Alpha); - MntCoeff.CSF[iMarker] = -ForceMomentum[0] * sin(Beta) * cos(Alpha) + ForceMomentum[1] * cos(Beta) - - ForceMomentum[2] * sin(Beta) * sin(Alpha); - MntCoeff.CEff[iMarker] = MntCoeff.CL[iMarker] / (MntCoeff.CD[iMarker] + EPS); - MntCoeff.CFx[iMarker] = ForceMomentum[0]; - MntCoeff.CFy[iMarker] = ForceMomentum[1]; - MntCoeff.CFz[iMarker] = ForceMomentum[2]; - MntCoeff.CMx[iMarker] = MomentMomentum[0]; - MntCoeff.CMy[iMarker] = MomentMomentum[1]; - MntCoeff.CMz[iMarker] = MomentMomentum[2]; - MntCoeff.CoPx[iMarker] = -MomentY_Force[0]; - MntCoeff.CoPz[iMarker] = MomentY_Force[2]; - MntCoeff.CT[iMarker] = -MntCoeff.CFz[iMarker]; - MntCoeff.CQ[iMarker] = -MntCoeff.CMz[iMarker]; - MntCoeff.CMerit[iMarker] = MntCoeff.CT[iMarker] / (MntCoeff.CQ[iMarker] + EPS); - } + /*--- Moment with respect to the reference axis ---*/ - AllBoundMntCoeff.CD += MntCoeff.CD[iMarker]; - AllBoundMntCoeff.CL += MntCoeff.CL[iMarker]; - AllBoundMntCoeff.CSF += MntCoeff.CSF[iMarker]; - AllBoundMntCoeff.CEff = AllBoundMntCoeff.CL / (AllBoundMntCoeff.CD + EPS); - AllBoundMntCoeff.CFx += MntCoeff.CFx[iMarker]; - AllBoundMntCoeff.CFy += MntCoeff.CFy[iMarker]; - AllBoundMntCoeff.CFz += MntCoeff.CFz[iMarker]; - AllBoundMntCoeff.CMx += MntCoeff.CMx[iMarker]; - AllBoundMntCoeff.CMy += MntCoeff.CMy[iMarker]; - AllBoundMntCoeff.CMx += MntCoeff.CMz[iMarker]; - AllBoundMntCoeff.CoPx += MntCoeff.CoPx[iMarker]; - AllBoundMntCoeff.CoPy += MntCoeff.CoPy[iMarker]; - AllBoundMntCoeff.CoPz += MntCoeff.CoPz[iMarker]; - AllBoundMntCoeff.CT += MntCoeff.CT[iMarker]; - AllBoundMntCoeff.CQ += MntCoeff.CQ[iMarker]; - AllBoundMntCoeff.CMerit += AllBoundMntCoeff.CT / (AllBoundMntCoeff.CQ + EPS); - - /*--- Compute the coefficients per surface ---*/ - - for (iMarker_Monitoring = 0; iMarker_Monitoring < config->GetnMarker_Monitoring(); iMarker_Monitoring++) { - Monitoring_Tag = config->GetMarker_Monitoring_TagBound(iMarker_Monitoring); - Marker_Tag = config->GetMarker_All_TagBound(iMarker); - if (Marker_Tag == Monitoring_Tag) { - SurfaceMntCoeff.CL[iMarker_Monitoring] += MntCoeff.CL[iMarker]; - SurfaceMntCoeff.CD[iMarker_Monitoring] += MntCoeff.CD[iMarker]; - SurfaceMntCoeff.CSF[iMarker_Monitoring] += MntCoeff.CSF[iMarker]; - SurfaceMntCoeff.CEff[iMarker_Monitoring] = SurfaceMntCoeff.CL[iMarker_Monitoring] / (SurfaceMntCoeff.CD[iMarker_Monitoring] + EPS); - SurfaceMntCoeff.CFx[iMarker_Monitoring] += MntCoeff.CFx[iMarker]; - SurfaceMntCoeff.CFy[iMarker_Monitoring] += MntCoeff.CFy[iMarker]; - SurfaceMntCoeff.CFz[iMarker_Monitoring] += MntCoeff.CFz[iMarker]; - SurfaceMntCoeff.CMx[iMarker_Monitoring] += MntCoeff.CMx[iMarker]; - SurfaceMntCoeff.CMy[iMarker_Monitoring] += MntCoeff.CMy[iMarker]; - SurfaceMntCoeff.CMz[iMarker_Monitoring] += MntCoeff.CMz[iMarker]; - } - } + AccumulateMoment(nDim, RefLength, Force, MomentDist, Coord, MomentMomentum, MomentX_Force, MomentY_Force, + MomentZ_Force); } } - } - -#ifdef HAVE_MPI - - /*--- Add AllBound information using all the nodes ---*/ - - if (config->GetComm_Level() == COMM_FULL) { - auto Allreduce = [](su2double x) { - su2double tmp = x; - x = 0.0; - SU2_MPI::Allreduce(&tmp, &x, 1, MPI_DOUBLE, MPI_SUM, SU2_MPI::GetComm()); - return x; - }; - - AllBoundMntCoeff.CD = Allreduce(AllBoundMntCoeff.CD); - AllBoundMntCoeff.CL = Allreduce(AllBoundMntCoeff.CL); - AllBoundMntCoeff.CSF = Allreduce(AllBoundMntCoeff.CSF); - AllBoundMntCoeff.CEff = AllBoundMntCoeff.CL / (AllBoundMntCoeff.CD + EPS); - - AllBoundMntCoeff.CFx = Allreduce(AllBoundMntCoeff.CFx); - AllBoundMntCoeff.CFy = Allreduce(AllBoundMntCoeff.CFy); - AllBoundMntCoeff.CFz = Allreduce(AllBoundMntCoeff.CFz); - - AllBoundMntCoeff.CMx = Allreduce(AllBoundMntCoeff.CMx); - AllBoundMntCoeff.CMy = Allreduce(AllBoundMntCoeff.CMy); - AllBoundMntCoeff.CMz = Allreduce(AllBoundMntCoeff.CMz); + END_SU2_OMP_FOR - AllBoundMntCoeff.CoPx = Allreduce(AllBoundMntCoeff.CoPx); - AllBoundMntCoeff.CoPy = Allreduce(AllBoundMntCoeff.CoPy); - AllBoundMntCoeff.CoPz = Allreduce(AllBoundMntCoeff.CoPz); + if (Monitoring == YES) { + const auto partial = ComputeAeroCoeffsFromForceMoment( + nDim, CosAlpha, SinAlpha, CosBeta, SinBeta, ForceMomentum, MomentMomentum, MomentX_Force, MomentY_Force, + MomentZ_Force); - AllBoundMntCoeff.CT = Allreduce(AllBoundMntCoeff.CT); - AllBoundMntCoeff.CQ = Allreduce(AllBoundMntCoeff.CQ); - AllBoundMntCoeff.CMerit = AllBoundMntCoeff.CT / (AllBoundMntCoeff.CQ + EPS); + AddCoeffContribution(iMarker, iMarker_Monitoring, partial, MntCoeff, AllBoundMntCoeff, SurfaceMntCoeff); + } } + /*--- For the SU2_NOWAIT in the vertex loop. ---*/ + SU2_OMP_BARRIER - /*--- Add the forces on the surfaces using all the nodes ---*/ - - if (config->GetComm_Level() == COMM_FULL) { - int nMarkerMon = config->GetnMarker_Monitoring(); - - /*--- Use the same buffer for all reductions. We could avoid the copy back into - * the original variable by swaping pointers, but it is safer this way... ---*/ - - su2double* buffer = new su2double[nMarkerMon]; - - auto Allreduce_inplace = [buffer](int size, su2double* x) { - SU2_MPI::Allreduce(x, buffer, size, MPI_DOUBLE, MPI_SUM, SU2_MPI::GetComm()); - for (int i = 0; i < size; ++i) x[i] = buffer[i]; - }; - - Allreduce_inplace(nMarkerMon, SurfaceMntCoeff.CL); - Allreduce_inplace(nMarkerMon, SurfaceMntCoeff.CD); - Allreduce_inplace(nMarkerMon, SurfaceMntCoeff.CSF); - - for (iMarker_Monitoring = 0; iMarker_Monitoring < nMarkerMon; iMarker_Monitoring++) - SurfaceMntCoeff.CEff[iMarker_Monitoring] = - SurfaceMntCoeff.CL[iMarker_Monitoring] / (SurfaceMntCoeff.CD[iMarker_Monitoring] + EPS); + /*--- Derive the ratio coefficients from the fully-reduced totals, once. ---*/ - Allreduce_inplace(nMarkerMon, SurfaceMntCoeff.CFx); - Allreduce_inplace(nMarkerMon, SurfaceMntCoeff.CFy); - Allreduce_inplace(nMarkerMon, SurfaceMntCoeff.CFz); - - Allreduce_inplace(nMarkerMon, SurfaceMntCoeff.CMx); - Allreduce_inplace(nMarkerMon, SurfaceMntCoeff.CMy); - Allreduce_inplace(nMarkerMon, SurfaceMntCoeff.CMz); - - delete[] buffer; + SU2_OMP_FOR_(schedule(static, OMP_MIN_SIZE) SU2_NOWAIT) + for (unsigned long iMarker = 0; iMarker < nMarker; iMarker++) { + const auto Boundary = config->GetMarker_All_KindBC(iMarker); + const auto Monitoring = config->GetMarker_All_Monitoring(iMarker); + if (Monitoring == YES && IsMomentumBoundary(Boundary)) { + MntCoeff.CEff[iMarker] = MntCoeff.CL[iMarker] / (MntCoeff.CD[iMarker] + EPS); + MntCoeff.CMerit[iMarker] = MntCoeff.CT[iMarker] / (MntCoeff.CQ[iMarker] + EPS); + } } + END_SU2_OMP_FOR -#endif - - /*--- Update the total coefficients (note that all the nodes have the same value) ---*/ - - TotalCoeff.CD += AllBoundMntCoeff.CD; - TotalCoeff.CL += AllBoundMntCoeff.CL; - TotalCoeff.CSF += AllBoundMntCoeff.CSF; - TotalCoeff.CEff = TotalCoeff.CL / (TotalCoeff.CD + EPS); - TotalCoeff.CFx += AllBoundMntCoeff.CFx; - TotalCoeff.CFy += AllBoundMntCoeff.CFy; - TotalCoeff.CFz += AllBoundMntCoeff.CFz; - TotalCoeff.CMx += AllBoundMntCoeff.CMx; - TotalCoeff.CMy += AllBoundMntCoeff.CMy; - TotalCoeff.CMz += AllBoundMntCoeff.CMz; - TotalCoeff.CoPx += AllBoundMntCoeff.CoPx; - TotalCoeff.CoPy += AllBoundMntCoeff.CoPy; - TotalCoeff.CoPz += AllBoundMntCoeff.CoPz; - TotalCoeff.CT += AllBoundMntCoeff.CT; - TotalCoeff.CQ += AllBoundMntCoeff.CQ; - TotalCoeff.CMerit = TotalCoeff.CT / (TotalCoeff.CQ + EPS); + SU2_OMP_FOR_(schedule(static, OMP_MIN_SIZE) SU2_NOWAIT) + for (unsigned short iMarker_Monitoring = 0; iMarker_Monitoring < config->GetnMarker_Monitoring(); + iMarker_Monitoring++) { + SurfaceMntCoeff.CEff[iMarker_Monitoring] = + SurfaceMntCoeff.CL[iMarker_Monitoring] / (SurfaceMntCoeff.CD[iMarker_Monitoring] + EPS); + } + END_SU2_OMP_FOR - /*--- Update the total coefficients per surface (note that all the nodes have the same value)---*/ + BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS { + AllBoundMntCoeff.CEff = AllBoundMntCoeff.CL / (AllBoundMntCoeff.CD + EPS); + AllBoundMntCoeff.CMerit = AllBoundMntCoeff.CT / (AllBoundMntCoeff.CQ + EPS); - for (iMarker_Monitoring = 0; iMarker_Monitoring < config->GetnMarker_Monitoring(); iMarker_Monitoring++) { - SurfaceCoeff.CL[iMarker_Monitoring] += SurfaceMntCoeff.CL[iMarker_Monitoring]; - SurfaceCoeff.CD[iMarker_Monitoring] += SurfaceMntCoeff.CD[iMarker_Monitoring]; - SurfaceCoeff.CSF[iMarker_Monitoring] += SurfaceMntCoeff.CSF[iMarker_Monitoring]; - SurfaceCoeff.CEff[iMarker_Monitoring] = - SurfaceCoeff.CL[iMarker_Monitoring] / (SurfaceCoeff.CD[iMarker_Monitoring] + EPS); - SurfaceCoeff.CFx[iMarker_Monitoring] += SurfaceMntCoeff.CFx[iMarker_Monitoring]; - SurfaceCoeff.CFy[iMarker_Monitoring] += SurfaceMntCoeff.CFy[iMarker_Monitoring]; - SurfaceCoeff.CFz[iMarker_Monitoring] += SurfaceMntCoeff.CFz[iMarker_Monitoring]; - SurfaceCoeff.CMx[iMarker_Monitoring] += SurfaceMntCoeff.CMx[iMarker_Monitoring]; - SurfaceCoeff.CMy[iMarker_Monitoring] += SurfaceMntCoeff.CMy[iMarker_Monitoring]; - SurfaceCoeff.CMz[iMarker_Monitoring] += SurfaceMntCoeff.CMz[iMarker_Monitoring]; + ReduceCoeffsMPI(config, AllBoundMntCoeff, SurfaceMntCoeff); + AccumulateTotalCoeffs(config, AllBoundMntCoeff, SurfaceMntCoeff, TotalCoeff, SurfaceCoeff, /*overwrite=*/false); } + END_SU2_OMP_SAFE_GLOBAL_ACCESS } template void CFVMFlowSolverBase::Friction_Forces(const CGeometry* geometry, const CConfig* config) { - /// TODO: Major cleanup needed. - + SU2_ZONE_SCOPED if (!config->GetViscous()) return; - unsigned long iVertex, iPoint, iPointNormal; - unsigned short iMarker, iMarker_Monitoring, iDim, jDim; - su2double Viscosity = 0.0, Area, Density = 0.0, FrictionVel, - UnitNormal[3] = {0.0}, TauElem[3] = {0.0}, Tau[3][3] = {{0.0}}, - thermal_conductivity, MaxNorm = 8.0, Grad_Vel[3][3] = {{0.0}}, Grad_Temp[3] = {0.0}, - Grad_Temp_ve[3] = {0.0}, AxiFactor; - const su2double *Coord = nullptr, *Coord_Normal = nullptr, *Normal = nullptr; + constexpr int MaxNorm = 8; const su2double minYPlus = config->GetwallModel_MinYPlus(); const su2double Alpha = config->GetAoA() * PI_NUMBER / 180.0; const su2double Beta = config->GetAoS() * PI_NUMBER / 180.0; + const su2double CosAlpha = cos(Alpha), SinAlpha = sin(Alpha), CosBeta = cos(Beta), SinBeta = sin(Beta); const su2double RefLength = config->GetRefLength(); const su2double RefHeatFlux = config->GetHeat_Flux_Ref(); const su2double RefTemperature = config->GetTemperature_Ref(); @@ -2482,93 +2479,126 @@ void CFVMFlowSolverBase::Friction_Forces(const CGeometry* geometr /*--- Variables initialization ---*/ - AllBoundViscCoeff.setZero(); - SurfaceViscCoeff.setZero(); + BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS { + AllBound_HF_Visc = 0.0; + AllBound_MaxHF_Visc = 0.0; + AllBoundViscCoeff.setZero(); + } + END_SU2_OMP_SAFE_GLOBAL_ACCESS - AllBound_HF_Visc = 0.0; - AllBound_MaxHF_Visc = 0.0; + SurfaceViscCoeff.setZero(); - for (iMarker_Monitoring = 0; iMarker_Monitoring < config->GetnMarker_Monitoring(); iMarker_Monitoring++) { + SU2_OMP_FOR_(schedule(static, OMP_MIN_SIZE) SU2_NOWAIT) + for (unsigned short iMarker_Monitoring = 0; iMarker_Monitoring < config->GetnMarker_Monitoring(); + iMarker_Monitoring++) { Surface_HF_Visc[iMarker_Monitoring] = 0.0; Surface_MaxHF_Visc[iMarker_Monitoring] = 0.0; } + END_SU2_OMP_FOR + + SU2_OMP_FOR_STAT(OMP_MIN_SIZE) + for (unsigned long iMarker = 0; iMarker < nMarker; iMarker++) { + if (!config->GetViscous_Wall(iMarker)) continue; + ViscCoeff.setZero(iMarker); + HF_Visc[iMarker] = 0.0; + MaxHF_Visc[iMarker] = 0.0; + } + END_SU2_OMP_FOR - /*--- Loop over the Navier-Stokes markers ---*/ + /*--- Loop over the Navier-Stokes markers (see Pressure_Forces for how the parallel + * reduction over threads is organized). The per-vertex loop below is the expensive + * part (stress-tensor and heat-flux evaluations) and is work-shared across threads. ---*/ - for (iMarker = 0; iMarker < nMarker; iMarker++) { + for (unsigned long iMarker = 0; iMarker < nMarker; iMarker++) { if (!config->GetViscous_Wall(iMarker)) continue; const auto Marker_Tag = config->GetMarker_All_TagBound(iMarker); - + const auto Monitoring = config->GetMarker_All_Monitoring(iMarker); const bool py_custom = config->GetMarker_All_PyCustom(iMarker); /*--- Obtain the origin for the moment computation for a particular marker ---*/ - const auto Monitoring = config->GetMarker_All_Monitoring(iMarker); - if (Monitoring == YES) { - for (iMarker_Monitoring = 0; iMarker_Monitoring < config->GetnMarker_Monitoring(); iMarker_Monitoring++) { - const auto Monitoring_Tag = config->GetMarker_Monitoring_TagBound(iMarker_Monitoring); - if (Marker_Tag == Monitoring_Tag) Origin = config->GetRefOriginMoment(iMarker_Monitoring); - } - } - - /*--- Forces initialization at each Marker ---*/ - - ViscCoeff.setZero(iMarker); - - HF_Visc[iMarker] = 0.0; - MaxHF_Visc[iMarker] = 0.0; + const int iMarker_Monitoring = FindMonitoringIndex(config, iMarker, Monitoring, Origin); su2double ForceViscous[MAXNDIM] = {0.0}, MomentViscous[MAXNDIM] = {0.0}; su2double MomentX_Force[MAXNDIM] = {0.0}, MomentY_Force[MAXNDIM] = {0.0}, MomentZ_Force[MAXNDIM] = {0.0}; + su2double HF_Visc_Local = 0.0, MaxHF_Visc_Local = 0.0; /* --- check if wall functions are used --- */ const bool wallfunctions = (config->GetWallFunction_Treatment(Marker_Tag) != WALL_FUNCTIONS::NONE); - /*--- Loop over the vertices to compute the forces ---*/ + /*--- Marker-level lookups hoisted out of the per-vertex loop below: GetWallRoughnessProperties, + * GetWall_HeatFlux, GetIsothermal_Temperature and GetCatalytic_Wall all scan over marker + * lists (some by string comparison), so evaluating them once per vertex instead of once per + * marker was a real cost for markers with many vertices. ---*/ - for (iVertex = 0; iVertex < geometry->nVertex[iMarker]; iVertex++) { - iPoint = geometry->vertex[iMarker][iVertex]->GetNode(); + WALL_TYPE WallType = WALL_TYPE::SMOOTH; + if (roughwall) { + su2double Roughness_Height; + tie(WallType, Roughness_Height) = config->GetWallRoughnessProperties(Marker_Tag); + } - Coord = geometry->nodes->GetCoord(iPoint); + const auto KindBC = config->GetMarker_All_KindBC(iMarker); + su2double Wall_HeatFlux_Value = 0.0, Twall = 0.0; + if (!nemo && !py_custom) { + if (KindBC == BC_TYPE::HEAT_FLUX) { + Wall_HeatFlux_Value = -config->GetWall_HeatFlux(Marker_Tag); + if (config->GetIntegrated_HeatFlux()) Wall_HeatFlux_Value /= geometry->GetSurfaceArea(config, iMarker); + } else if (KindBC == BC_TYPE::ISOTHERMAL) { + Twall = config->GetIsothermal_Temperature(Marker_Tag) / RefTemperature; + } + } + const bool catalytic = nemo && config->GetCatalytic_Wall(iMarker); - Normal = geometry->vertex[iMarker][iVertex]->GetNormal(); + /*--- Loop over the vertices to compute the forces (work-shared across threads, see + * Pressure_Forces for why the chunk size is computed and the barrier skipped). ---*/ - for (iDim = 0; iDim < nDim; iDim++) { - for (jDim = 0; jDim < nDim; jDim++) { - Grad_Vel[iDim][jDim] = nodes->GetGradient_Primitive(iPoint, prim_idx.Velocity() + iDim, jDim); - } + SU2_OMP_FOR_(schedule(static, OMP_MIN_SIZE) SU2_NOWAIT) + for (unsigned long iVertex = 0; iVertex < geometry->nVertex[iMarker]; iVertex++) { + const auto iPoint = geometry->vertex[iMarker][iVertex]->GetNode(); + + const su2double* Coord = geometry->nodes->GetCoord(iPoint); + + const su2double* Normal = geometry->vertex[iMarker][iVertex]->GetNormal(); + + /*--- One view covering the whole velocity-gradient block, instead of nDim*nDim separate + * single-element lookups (ComputeStressTensor/AddQCR accept any [][]-indexable type). ---*/ + const auto Grad_Vel = nodes->GetVelocityGradient(iPoint); + + su2double Grad_Temp[3] = {0.0}, Grad_Temp_ve[3] = {0.0}; + for (unsigned short iDim = 0; iDim < nDim; iDim++) { Grad_Temp[iDim] = nodes->GetGradient_Primitive(iPoint, prim_idx.Temperature(), iDim); if (nemo) Grad_Temp_ve[iDim] = nodes->GetGradient_Primitive(iPoint, prim_idx.Temperature_ve(), iDim); } - Viscosity = nodes->GetLaminarViscosity(iPoint); - if (roughwall) { - WALL_TYPE WallType; - su2double Roughness_Height; - tie(WallType, Roughness_Height) = config->GetWallRoughnessProperties(Marker_Tag); - if (WallType == WALL_TYPE::ROUGH) Viscosity += nodes->GetEddyViscosity(iPoint); + su2double Viscosity = nodes->GetLaminarViscosity(iPoint); + su2double EddyViscosity = 0.0; + if (WallType == WALL_TYPE::ROUGH) { + EddyViscosity = nodes->GetEddyViscosity(iPoint); + Viscosity += EddyViscosity; } - Density = nodes->GetDensity(iPoint); + const su2double Density = nodes->GetDensity(iPoint); - Area = GeometryToolbox::Norm(nDim, Normal); - for (iDim = 0; iDim < nDim; iDim++) { + const su2double Area = GeometryToolbox::Norm(nDim, Normal); + su2double UnitNormal[3] = {0.0}; + for (unsigned short iDim = 0; iDim < nDim; iDim++) { UnitNormal[iDim] = Normal[iDim] / Area; } /*--- Evaluate Tau ---*/ + su2double Tau[3][3] = {{0.0}}; CNumerics::ComputeStressTensor(nDim, Tau, Grad_Vel, Viscosity); /*--- If necessary evaluate the QCR contribution to Tau ---*/ - if (QCR) CNumerics::AddQCR(nDim, Grad_Vel, Tau); + if (QCR) CNumerics::AddQCR(nDim, Grad_Vel, Tau, EddyViscosity / Viscosity); /*--- Project Tau in each surface element ---*/ - for (iDim = 0; iDim < nDim; iDim++) { - TauElem[iDim] = 0.0; - for (jDim = 0; jDim < nDim; jDim++) { + su2double TauElem[3] = {0.0}; + for (unsigned short iDim = 0; iDim < nDim; iDim++) { + for (unsigned short jDim = 0; jDim < nDim; jDim++) { TauElem[iDim] += Tau[iDim][jDim] * UnitNormal[jDim]; } } @@ -2582,11 +2612,10 @@ void CFVMFlowSolverBase::Friction_Forces(const CGeometry* geometr WallShearStress[iMarker][iVertex] = GeometryToolbox::Norm(int(MAXNDIM), TauTangent); /*--- For wall functions, the wall stresses need to be scaled by the wallfunction stress Tau_Wall---*/ - su2double Tau_Wall, scale; if (wallfunctions && (YPlus[iMarker][iVertex] > minYPlus)){ - Tau_Wall = nodes->GetTau_Wall(iPoint); - scale = Tau_Wall / WallShearStress[iMarker][iVertex]; - for (iDim = 0; iDim < nDim; iDim++) { + const su2double Tau_Wall = nodes->GetTau_Wall(iPoint); + const su2double scale = Tau_Wall / WallShearStress[iMarker][iVertex]; + for (unsigned short iDim = 0; iDim < nDim; iDim++) { TauTangent[iDim] *= scale; TauElem[iDim] *= scale; } @@ -2594,48 +2623,39 @@ void CFVMFlowSolverBase::Friction_Forces(const CGeometry* geometr WallShearStress[iMarker][iVertex] = Tau_Wall; } - for (iDim = 0; iDim < nDim; iDim++) { + for (unsigned short iDim = 0; iDim < nDim; iDim++) { CSkinFriction[iMarker](iVertex,iDim) = TauTangent[iDim] * factorFric; } /*--- Compute non-dimensional velocity and y+ ---*/ - FrictionVel = sqrt(fabs(WallShearStress[iMarker][iVertex]) / Density); + const su2double FrictionVel = sqrt(fabs(WallShearStress[iMarker][iVertex]) / Density); if (!wallfunctions && MGLevel == MESH_0 && geometry->nodes->GetDomain(iPoint)) { // for CMultiGridGeometry and halos, the nearest neighbor distance is not set const su2double WallDistMod = geometry->vertex[iMarker][iVertex]->GetNearestNeighborDistance(); - YPlus[iMarker][iVertex] = WallDistMod * FrictionVel / (Viscosity / Density); + YPlus[iMarker][iVertex] = WallDistMod * FrictionVel * Density / Viscosity; } /*--- Compute total and maximum heat flux on the wall ---*/ if (!nemo) { + su2double thermal_conductivity = 0.0; if ((FlowRegime == ENUM_REGIME::COMPRESSIBLE) || (FlowRegime == ENUM_REGIME::INCOMPRESSIBLE)) { thermal_conductivity = nodes->GetThermalConductivity(iPoint); } - if (config->GetMarker_All_KindBC(iMarker) == BC_TYPE::HEAT_FLUX) { - if (py_custom) { - HeatFlux[iMarker][iVertex] = -geometry->GetCustomBoundaryHeatFlux(iMarker, iVertex); - } else { - HeatFlux[iMarker][iVertex] = -config->GetWall_HeatFlux(Marker_Tag); - if (config->GetIntegrated_HeatFlux()) { - HeatFlux[iMarker][iVertex] /= geometry->GetSurfaceArea(config, iMarker); - } - } - } else if (config->GetMarker_All_KindBC(iMarker) == BC_TYPE::ISOTHERMAL) { - su2double Twall = 0.0; - if (py_custom) { - Twall = geometry->GetCustomBoundaryTemperature(iMarker, iVertex) / RefTemperature; - } else { - Twall = config->GetIsothermal_Temperature(Marker_Tag) / RefTemperature; - } - iPointNormal = geometry->vertex[iMarker][iVertex]->GetNormal_Neighbor(); - Coord_Normal = geometry->nodes->GetCoord(iPointNormal); + if (KindBC == BC_TYPE::HEAT_FLUX) { + HeatFlux[iMarker][iVertex] = + py_custom ? -geometry->GetCustomBoundaryHeatFlux(iMarker, iVertex) : Wall_HeatFlux_Value; + } else if (KindBC == BC_TYPE::ISOTHERMAL) { + const su2double Twall_local = + py_custom ? geometry->GetCustomBoundaryTemperature(iMarker, iVertex) / RefTemperature : Twall; + const auto iPointNormal = geometry->vertex[iMarker][iVertex]->GetNormal_Neighbor(); + const su2double* Coord_Normal = geometry->nodes->GetCoord(iPointNormal); const su2double dist_ij = GeometryToolbox::NormalDistance(nDim, UnitNormal, Coord, Coord_Normal); const su2double There = nodes->GetTemperature(iPointNormal); - HeatFlux[iMarker][iVertex] = thermal_conductivity * (There - Twall) / dist_ij * RefHeatFlux; + HeatFlux[iMarker][iVertex] = thermal_conductivity * (There - Twall_local) / dist_ij * RefHeatFlux; } else { su2double dTdn = GeometryToolbox::DotProduct(nDim, Grad_Temp, UnitNormal); if (FlowRegime == ENUM_REGIME::INCOMPRESSIBLE && !energy) dTdn = 0.0; @@ -2653,8 +2673,7 @@ void CFVMFlowSolverBase::Friction_Forces(const CGeometry* geometr HeatFlux[iMarker][iVertex] = -(thermal_conductivity_tr*dTdn + thermal_conductivity_ve*dTvedn); /*--- Compute enthalpy transport to surface due to mass diffusion ---*/ - bool catalytic = config->GetCatalytic_Wall(iMarker); - if (catalytic){ + if (catalytic) { const auto nSpecies = config->GetnSpecies(); const auto& Grad_PrimVar = nodes->GetGradient_Primitive(iPoint); @@ -2682,15 +2701,12 @@ void CFVMFlowSolverBase::Friction_Forces(const CGeometry* geometr if ((geometry->nodes->GetDomain(iPoint)) && (Monitoring == YES)) { /*--- Axisymmetric simulations ---*/ - if (axisymmetric) - AxiFactor = 2.0 * PI_NUMBER * geometry->nodes->GetCoord(iPoint, 1); - else - AxiFactor = 1.0; + const su2double AxiFactor = axisymmetric ? su2double(2.0 * PI_NUMBER * geometry->nodes->GetCoord(iPoint, 1)) : su2double(1.0); /*--- Force computation ---*/ su2double Force[MAXNDIM] = {0.0}, MomentDist[MAXNDIM] = {0.0}; - for (iDim = 0; iDim < nDim; iDim++) { + for (unsigned short iDim = 0; iDim < nDim; iDim++) { Force[iDim] = TauElem[iDim] * Area * factor * AxiFactor; ForceViscous[iDim] += Force[iDim]; MomentDist[iDim] = Coord[iDim] - Origin[iDim]; @@ -2698,227 +2714,97 @@ void CFVMFlowSolverBase::Friction_Forces(const CGeometry* geometr /*--- Moment with respect to the reference axis ---*/ - if (nDim == 3) { - MomentViscous[0] += (Force[2] * MomentDist[1] - Force[1] * MomentDist[2]) / RefLength; - MomentX_Force[1] += (-Force[1] * Coord[2]); - MomentX_Force[2] += (Force[2] * Coord[1]); - - MomentViscous[1] += (Force[0] * MomentDist[2] - Force[2] * MomentDist[0]) / RefLength; - MomentY_Force[2] += (-Force[2] * Coord[0]); - MomentY_Force[0] += (Force[0] * Coord[2]); - } - MomentViscous[2] += (Force[1] * MomentDist[0] - Force[0] * MomentDist[1]) / RefLength; - MomentZ_Force[0] += (-Force[0] * Coord[1]); - MomentZ_Force[1] += (Force[1] * Coord[0]); + AccumulateMoment(nDim, RefLength, Force, MomentDist, Coord, MomentViscous, MomentX_Force, MomentY_Force, + MomentZ_Force); - HF_Visc[iMarker] += HeatFlux[iMarker][iVertex] * Area; - MaxHF_Visc[iMarker] += pow(HeatFlux[iMarker][iVertex], MaxNorm); + HF_Visc_Local += HeatFlux[iMarker][iVertex] * Area; + MaxHF_Visc_Local += pow(HeatFlux[iMarker][iVertex], MaxNorm); } } + END_SU2_OMP_FOR - /*--- Project forces and store the non-dimensional coefficients ---*/ + /*--- MaxHF_Visc_Local (and the shared accumulators it feeds below) are left un-rooted, + * i.e. still a raw sum of HeatFlux^MaxNorm; pow(., 1/MaxNorm) is taken once at the end, + * which is equivalent since pow(x^(1/n), n) == x. ---*/ if (Monitoring == YES) { - if (nDim == 2) { - ViscCoeff.CD[iMarker] = ForceViscous[0] * cos(Alpha) + ForceViscous[1] * sin(Alpha); - ViscCoeff.CL[iMarker] = -ForceViscous[0] * sin(Alpha) + ForceViscous[1] * cos(Alpha); - ViscCoeff.CEff[iMarker] = ViscCoeff.CL[iMarker] / (ViscCoeff.CD[iMarker] + EPS); - ViscCoeff.CFx[iMarker] = ForceViscous[0]; - ViscCoeff.CFy[iMarker] = ForceViscous[1]; - ViscCoeff.CMz[iMarker] = MomentViscous[2]; - ViscCoeff.CoPx[iMarker] = MomentZ_Force[1]; - ViscCoeff.CoPy[iMarker] = -MomentZ_Force[0]; - ViscCoeff.CT[iMarker] = -ViscCoeff.CFx[iMarker]; - ViscCoeff.CQ[iMarker] = -ViscCoeff.CMz[iMarker]; - ViscCoeff.CMerit[iMarker] = ViscCoeff.CT[iMarker] / (ViscCoeff.CQ[iMarker] + EPS); - MaxHF_Visc[iMarker] = pow(MaxHF_Visc[iMarker], 1.0 / MaxNorm); - } - if (nDim == 3) { - ViscCoeff.CD[iMarker] = ForceViscous[0] * cos(Alpha) * cos(Beta) + ForceViscous[1] * sin(Beta) + - ForceViscous[2] * sin(Alpha) * cos(Beta); - ViscCoeff.CL[iMarker] = -ForceViscous[0] * sin(Alpha) + ForceViscous[2] * cos(Alpha); - ViscCoeff.CSF[iMarker] = -ForceViscous[0] * sin(Beta) * cos(Alpha) + ForceViscous[1] * cos(Beta) - - ForceViscous[2] * sin(Beta) * sin(Alpha); - ViscCoeff.CEff[iMarker] = ViscCoeff.CL[iMarker] / (ViscCoeff.CD[iMarker] + EPS); - ViscCoeff.CFx[iMarker] = ForceViscous[0]; - ViscCoeff.CFy[iMarker] = ForceViscous[1]; - ViscCoeff.CFz[iMarker] = ForceViscous[2]; - ViscCoeff.CMx[iMarker] = MomentViscous[0]; - ViscCoeff.CMy[iMarker] = MomentViscous[1]; - ViscCoeff.CMz[iMarker] = MomentViscous[2]; - ViscCoeff.CoPx[iMarker] = -MomentY_Force[0]; - ViscCoeff.CoPz[iMarker] = MomentY_Force[2]; - ViscCoeff.CT[iMarker] = -ViscCoeff.CFz[iMarker]; - ViscCoeff.CQ[iMarker] = -ViscCoeff.CMz[iMarker]; - ViscCoeff.CMerit[iMarker] = ViscCoeff.CT[iMarker] / (ViscCoeff.CQ[iMarker] + EPS); - MaxHF_Visc[iMarker] = pow(MaxHF_Visc[iMarker], 1.0 / MaxNorm); - } + const auto partial = ComputeAeroCoeffsFromForceMoment( + nDim, CosAlpha, SinAlpha, CosBeta, SinBeta, ForceViscous, MomentViscous, MomentX_Force, MomentY_Force, + MomentZ_Force); + + AddCoeffContribution(iMarker, iMarker_Monitoring, partial, ViscCoeff, AllBoundViscCoeff, SurfaceViscCoeff); - AllBoundViscCoeff.CD += ViscCoeff.CD[iMarker]; - AllBoundViscCoeff.CL += ViscCoeff.CL[iMarker]; - AllBoundViscCoeff.CSF += ViscCoeff.CSF[iMarker]; - AllBoundViscCoeff.CFx += ViscCoeff.CFx[iMarker]; - AllBoundViscCoeff.CFy += ViscCoeff.CFy[iMarker]; - AllBoundViscCoeff.CFz += ViscCoeff.CFz[iMarker]; - AllBoundViscCoeff.CMx += ViscCoeff.CMx[iMarker]; - AllBoundViscCoeff.CMy += ViscCoeff.CMy[iMarker]; - AllBoundViscCoeff.CMz += ViscCoeff.CMz[iMarker]; - AllBoundViscCoeff.CoPx += ViscCoeff.CoPx[iMarker]; - AllBoundViscCoeff.CoPy += ViscCoeff.CoPy[iMarker]; - AllBoundViscCoeff.CoPz += ViscCoeff.CoPz[iMarker]; - AllBoundViscCoeff.CT += ViscCoeff.CT[iMarker]; - AllBoundViscCoeff.CQ += ViscCoeff.CQ[iMarker]; - AllBound_HF_Visc += HF_Visc[iMarker]; - AllBound_MaxHF_Visc += pow(MaxHF_Visc[iMarker], MaxNorm); - - /*--- Compute the coefficients per surface ---*/ - - for (iMarker_Monitoring = 0; iMarker_Monitoring < config->GetnMarker_Monitoring(); iMarker_Monitoring++) { - const auto Monitoring_Tag = config->GetMarker_Monitoring_TagBound(iMarker_Monitoring); - if (Marker_Tag == Monitoring_Tag) { - SurfaceViscCoeff.CL[iMarker_Monitoring] += ViscCoeff.CL[iMarker]; - SurfaceViscCoeff.CD[iMarker_Monitoring] += ViscCoeff.CD[iMarker]; - SurfaceViscCoeff.CSF[iMarker_Monitoring] += ViscCoeff.CSF[iMarker]; - SurfaceViscCoeff.CEff[iMarker_Monitoring] = SurfaceViscCoeff.CL[iMarker_Monitoring] / (SurfaceViscCoeff.CD[iMarker_Monitoring] + EPS); - SurfaceViscCoeff.CFx[iMarker_Monitoring] += ViscCoeff.CFx[iMarker]; - SurfaceViscCoeff.CFy[iMarker_Monitoring] += ViscCoeff.CFy[iMarker]; - SurfaceViscCoeff.CFz[iMarker_Monitoring] += ViscCoeff.CFz[iMarker]; - SurfaceViscCoeff.CMx[iMarker_Monitoring] += ViscCoeff.CMx[iMarker]; - SurfaceViscCoeff.CMy[iMarker_Monitoring] += ViscCoeff.CMy[iMarker]; - SurfaceViscCoeff.CMz[iMarker_Monitoring] += ViscCoeff.CMz[iMarker]; - Surface_HF_Visc[iMarker_Monitoring] += HF_Visc[iMarker]; - Surface_MaxHF_Visc[iMarker_Monitoring] += pow(MaxHF_Visc[iMarker], MaxNorm); + /*--- Heat flux, not covered by AddCoeffContribution, is folded in its own critical section. ---*/ + + SU2_OMP_CRITICAL { + HF_Visc[iMarker] += HF_Visc_Local; + AllBound_HF_Visc += HF_Visc_Local; + MaxHF_Visc[iMarker] += MaxHF_Visc_Local; + AllBound_MaxHF_Visc += MaxHF_Visc_Local; + + if (iMarker_Monitoring >= 0) { + Surface_HF_Visc[iMarker_Monitoring] += HF_Visc_Local; + Surface_MaxHF_Visc[iMarker_Monitoring] += MaxHF_Visc_Local; } } + END_SU2_OMP_CRITICAL } } + /*--- For the SU2_NOWAIT in the vertex loop. ---*/ + SU2_OMP_BARRIER - /*--- Update some global coeffients ---*/ - - AllBoundViscCoeff.CEff = AllBoundViscCoeff.CL / (AllBoundViscCoeff.CD + EPS); - AllBoundViscCoeff.CMerit = AllBoundViscCoeff.CT / (AllBoundViscCoeff.CQ + EPS); - -#ifdef HAVE_MPI - - /*--- Add AllBound information using all the nodes ---*/ - - if (config->GetComm_Level() == COMM_FULL) { - auto Allreduce = [](su2double x) { - su2double tmp = x; - x = 0.0; - SU2_MPI::Allreduce(&tmp, &x, 1, MPI_DOUBLE, MPI_SUM, SU2_MPI::GetComm()); - return x; - }; - AllBoundViscCoeff.CD = Allreduce(AllBoundViscCoeff.CD); - AllBoundViscCoeff.CL = Allreduce(AllBoundViscCoeff.CL); - AllBoundViscCoeff.CSF = Allreduce(AllBoundViscCoeff.CSF); - AllBoundViscCoeff.CEff = AllBoundViscCoeff.CL / (AllBoundViscCoeff.CD + EPS); - - AllBoundViscCoeff.CMx = Allreduce(AllBoundViscCoeff.CMx); - AllBoundViscCoeff.CMy = Allreduce(AllBoundViscCoeff.CMy); - AllBoundViscCoeff.CMz = Allreduce(AllBoundViscCoeff.CMz); - - AllBoundViscCoeff.CFx = Allreduce(AllBoundViscCoeff.CFx); - AllBoundViscCoeff.CFy = Allreduce(AllBoundViscCoeff.CFy); - AllBoundViscCoeff.CFz = Allreduce(AllBoundViscCoeff.CFz); + /*--- Derive the ratio coefficients, and root the (still raw) per-marker maximum heat flux, + * from the fully-reduced totals, once. Surface_MaxHF_Visc and AllBound_MaxHF_Visc are + * rooted later below, after the MPI reduction. ---*/ - AllBoundViscCoeff.CoPx = Allreduce(AllBoundViscCoeff.CoPx); - AllBoundViscCoeff.CoPy = Allreduce(AllBoundViscCoeff.CoPy); - AllBoundViscCoeff.CoPz = Allreduce(AllBoundViscCoeff.CoPz); - - AllBoundViscCoeff.CT = Allreduce(AllBoundViscCoeff.CT); - AllBoundViscCoeff.CQ = Allreduce(AllBoundViscCoeff.CQ); - AllBoundViscCoeff.CMerit = AllBoundViscCoeff.CT / (AllBoundViscCoeff.CQ + EPS); - - AllBound_HF_Visc = Allreduce(AllBound_HF_Visc); - AllBound_MaxHF_Visc = Allreduce(AllBound_MaxHF_Visc); + SU2_OMP_FOR_(schedule(static, OMP_MIN_SIZE) SU2_NOWAIT) + for (unsigned long iMarker = 0; iMarker < nMarker; iMarker++) { + if (!config->GetViscous_Wall(iMarker)) continue; + if (config->GetMarker_All_Monitoring(iMarker) == YES) { + ViscCoeff.CEff[iMarker] = ViscCoeff.CL[iMarker] / (ViscCoeff.CD[iMarker] + EPS); + ViscCoeff.CMerit[iMarker] = ViscCoeff.CT[iMarker] / (ViscCoeff.CQ[iMarker] + EPS); + MaxHF_Visc[iMarker] = pow(MaxHF_Visc[iMarker], 1.0 / MaxNorm); + } } + END_SU2_OMP_FOR - /*--- Add the forces on the surfaces using all the nodes ---*/ - - if (config->GetComm_Level() == COMM_FULL) { - int nMarkerMon = config->GetnMarker_Monitoring(); - - /*--- Use the same buffer for all reductions. We could avoid the copy back into - * the original variable by swaping pointers, but it is safer this way... ---*/ - - su2double* buffer = new su2double[nMarkerMon]; - - auto Allreduce_inplace = [buffer](int size, su2double* x) { - SU2_MPI::Allreduce(x, buffer, size, MPI_DOUBLE, MPI_SUM, SU2_MPI::GetComm()); - for (int i = 0; i < size; ++i) x[i] = buffer[i]; - }; - - Allreduce_inplace(nMarkerMon, SurfaceViscCoeff.CL); - Allreduce_inplace(nMarkerMon, SurfaceViscCoeff.CD); - Allreduce_inplace(nMarkerMon, SurfaceViscCoeff.CSF); - - for (iMarker_Monitoring = 0; iMarker_Monitoring < nMarkerMon; iMarker_Monitoring++) - SurfaceViscCoeff.CEff[iMarker_Monitoring] = - SurfaceViscCoeff.CL[iMarker_Monitoring] / (SurfaceViscCoeff.CD[iMarker_Monitoring] + EPS); + SU2_OMP_FOR_(schedule(static, OMP_MIN_SIZE) SU2_NOWAIT) + for (unsigned short iMarker_Monitoring = 0; iMarker_Monitoring < config->GetnMarker_Monitoring(); + iMarker_Monitoring++) { + SurfaceViscCoeff.CEff[iMarker_Monitoring] = + SurfaceViscCoeff.CL[iMarker_Monitoring] / (SurfaceViscCoeff.CD[iMarker_Monitoring] + EPS); + } + END_SU2_OMP_FOR - Allreduce_inplace(nMarkerMon, SurfaceViscCoeff.CFx); - Allreduce_inplace(nMarkerMon, SurfaceViscCoeff.CFy); - Allreduce_inplace(nMarkerMon, SurfaceViscCoeff.CFz); + BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS { + ReduceCoeffsMPI(config, AllBoundViscCoeff, SurfaceViscCoeff); - Allreduce_inplace(nMarkerMon, SurfaceViscCoeff.CMx); - Allreduce_inplace(nMarkerMon, SurfaceViscCoeff.CMy); - Allreduce_inplace(nMarkerMon, SurfaceViscCoeff.CMz); + /*--- HF_Visc/MaxHF_Visc, not covered by ReduceCoeffsMPI, are reduced separately. ---*/ + if (config->GetComm_Level() == COMM_FULL) { + AllBound_HF_Visc = MPIReduceSum(AllBound_HF_Visc); + AllBound_MaxHF_Visc = MPIReduceSum(AllBound_MaxHF_Visc); - Allreduce_inplace(nMarkerMon, Surface_HF_Visc.data()); - Allreduce_inplace(nMarkerMon, Surface_MaxHF_Visc.data()); + const int nMarkerMon = config->GetnMarker_Monitoring(); + MPIReduceSumInPlace(Surface_HF_Visc.data(), nMarkerMon); + MPIReduceSumInPlace(Surface_MaxHF_Visc.data(), nMarkerMon); + } - delete[] buffer; - } + /*--- Complete the calculation of maximum heat flux. ---*/ -#endif + for (auto& hf : Surface_MaxHF_Visc) { + hf = pow(hf, 1.0 / MaxNorm); + } + AllBound_MaxHF_Visc = pow(AllBound_MaxHF_Visc, 1.0 / MaxNorm); - /*--- Complete the calculation of maximum heat flux. ---*/ - - for (auto& hf : Surface_MaxHF_Visc) { - hf = pow(hf, 1.0 / MaxNorm); - } - AllBound_MaxHF_Visc = pow(AllBound_MaxHF_Visc, 1.0 / MaxNorm); - - /*--- Update the total coefficients (note that all the nodes have the same value)---*/ - - TotalCoeff.CD += AllBoundViscCoeff.CD; - TotalCoeff.CL += AllBoundViscCoeff.CL; - TotalCoeff.CSF += AllBoundViscCoeff.CSF; - TotalCoeff.CEff = TotalCoeff.CL / (TotalCoeff.CD + EPS); - TotalCoeff.CFx += AllBoundViscCoeff.CFx; - TotalCoeff.CFy += AllBoundViscCoeff.CFy; - TotalCoeff.CFz += AllBoundViscCoeff.CFz; - TotalCoeff.CMx += AllBoundViscCoeff.CMx; - TotalCoeff.CMy += AllBoundViscCoeff.CMy; - TotalCoeff.CMz += AllBoundViscCoeff.CMz; - TotalCoeff.CoPx += AllBoundViscCoeff.CoPx; - TotalCoeff.CoPy += AllBoundViscCoeff.CoPy; - TotalCoeff.CoPz += AllBoundViscCoeff.CoPz; - TotalCoeff.CT += AllBoundViscCoeff.CT; - TotalCoeff.CQ += AllBoundViscCoeff.CQ; - TotalCoeff.CMerit = AllBoundViscCoeff.CT / (AllBoundViscCoeff.CQ + EPS); - Total_Heat = AllBound_HF_Visc; - Total_MaxHeat = AllBound_MaxHF_Visc; + AccumulateTotalCoeffs(config, AllBoundViscCoeff, SurfaceViscCoeff, TotalCoeff, SurfaceCoeff, /*overwrite=*/false); + Total_Heat = AllBound_HF_Visc; + Total_MaxHeat = AllBound_MaxHF_Visc; - /*--- Update the total coefficients per surface (note that all the nodes have the same value)---*/ + /*--- Buffet_Monitoring is not thread-safe, hence confined to the master thread. ---*/ - for (iMarker_Monitoring = 0; iMarker_Monitoring < config->GetnMarker_Monitoring(); iMarker_Monitoring++) { - SurfaceCoeff.CL[iMarker_Monitoring] += SurfaceViscCoeff.CL[iMarker_Monitoring]; - SurfaceCoeff.CD[iMarker_Monitoring] += SurfaceViscCoeff.CD[iMarker_Monitoring]; - SurfaceCoeff.CSF[iMarker_Monitoring] += SurfaceViscCoeff.CSF[iMarker_Monitoring]; - SurfaceCoeff.CEff[iMarker_Monitoring] = - SurfaceCoeff.CL[iMarker_Monitoring] / (SurfaceCoeff.CD[iMarker_Monitoring] + EPS); - SurfaceCoeff.CFx[iMarker_Monitoring] += SurfaceViscCoeff.CFx[iMarker_Monitoring]; - SurfaceCoeff.CFy[iMarker_Monitoring] += SurfaceViscCoeff.CFy[iMarker_Monitoring]; - SurfaceCoeff.CFz[iMarker_Monitoring] += SurfaceViscCoeff.CFz[iMarker_Monitoring]; - SurfaceCoeff.CMx[iMarker_Monitoring] += SurfaceViscCoeff.CMx[iMarker_Monitoring]; - SurfaceCoeff.CMy[iMarker_Monitoring] += SurfaceViscCoeff.CMy[iMarker_Monitoring]; - SurfaceCoeff.CMz[iMarker_Monitoring] += SurfaceViscCoeff.CMz[iMarker_Monitoring]; + Buffet_Monitoring(geometry, config); } - - Buffet_Monitoring(geometry, config); - + END_SU2_OMP_SAFE_GLOBAL_ACCESS } template diff --git a/SU2_CFD/include/solvers/CHeatSolver.hpp b/SU2_CFD/include/solvers/CHeatSolver.hpp index a0cce12c57ca..1097b2cd06f8 100644 --- a/SU2_CFD/include/solvers/CHeatSolver.hpp +++ b/SU2_CFD/include/solvers/CHeatSolver.hpp @@ -93,46 +93,12 @@ class CHeatSolver final : public CScalarSolver { } } - /*! - * \brief Compute the viscous flux for the scalar equation at a particular edge. - * \param[in] iEdge - Edge for which we want to compute the flux - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] solver_container - Container vector with all the solutions. - * \param[in] numerics - Description of the numerical method. - * \param[in] config - Definition of the particular problem. - * \note Calls a generic implementation after defining a SolverSpecificNumerics object. - */ - inline void Viscous_Residual(const unsigned long iEdge, const CGeometry* geometry, CSolver** solver_container, - CNumerics* numerics, const CConfig* config) override { - const CVariable* flow_nodes = flow ? solver_container[FLOW_SOL]->GetNodes() : nullptr; - - const su2double const_diffusivity = config->GetThermalDiffusivity(); - const su2double pr_turb = config->GetPrandtl_Turb(); - - su2double thermal_diffusivity_i{}, thermal_diffusivity_j{}; - - /*--- Computes the thermal diffusivity to use in the viscous numerics. ---*/ - auto compute_thermal_diffusivity = [&](unsigned long iPoint, unsigned long jPoint) { - if (flow) { - thermal_diffusivity_i = flow_nodes->GetThermalConductivity(iPoint) / flow_nodes->GetSpecificHeatCp(iPoint) + - flow_nodes->GetEddyViscosity(iPoint) / pr_turb; - thermal_diffusivity_j = flow_nodes->GetThermalConductivity(jPoint) / flow_nodes->GetSpecificHeatCp(jPoint) + - flow_nodes->GetEddyViscosity(jPoint) / pr_turb; - numerics->SetDiffusionCoeff(&thermal_diffusivity_i, &thermal_diffusivity_j); - } else { - numerics->SetDiffusionCoeff(&const_diffusivity, &const_diffusivity); - } - }; - /*--- Compute residual and Jacobians. ---*/ - Viscous_Residual_impl(compute_thermal_diffusivity, iEdge, geometry, solver_container, numerics, config); - } - public: /*! * \brief Constructor of the class. */ - CHeatSolver(CGeometry *geometry, CConfig *config, unsigned short iMesh); + CHeatSolver(CGeometry *geometry, CConfig *config, const CSolver* flow_solver, unsigned short iMesh); /*! * \brief Restart residual and compute gradients. @@ -181,13 +147,16 @@ class CHeatSolver final : public CScalarSolver { unsigned short iMesh) override; /*! - * \brief Compute the viscous residuals for the turbulent equation. - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] solver_container - Container vector with all the solutions. - * \param[in] numerics_container - Description of the numerical method. - * \param[in] config - Definition of the particular problem. - * \param[in] iMesh - Index of the mesh in multigrid computations. - * \param[in] iRKStep - Current step of the Runge-Kutta iteration. + * \brief Resolve the compile-time parameters of CScalarFlux_Heat and run one of this solver's + * boundaries through the shared boundary flux pass. + * \param[in] opt - Flags of the boundary, from one of ScalarFluxOptions' named constructors. + */ + void BoundaryFlux(CGeometry* geometry, CSolver** solver_container, CConfig* config, const ScalarFluxOptions& opt, + unsigned short val_marker); + + /*! + * \brief Diffusion for solid conduction, called unconditionally unlike Upwind_Residual. A no-op + * for a fluid zone, where diffusion was already computed together with convection. */ void Viscous_Residual(CGeometry *geometry, CSolver **solver_container, @@ -273,6 +242,34 @@ class CHeatSolver final : public CScalarSolver { CConfig *config, unsigned short val_marker) override; + /*! + * \brief Impose the far-field boundary condition. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] solver_container - Container vector with all the solutions. + * \param[in] conv_numerics - Description of the numerical method. + * \param[in] visc_numerics - Description of the numerical method. + * \param[in] config - Definition of the particular problem. + * \param[in] val_marker - Surface marker where the boundary condition is applied. + */ + void BC_Far_Field(CGeometry *geometry, + CSolver **solver_container, + CNumerics *conv_numerics, + CNumerics *visc_numerics, + CConfig *config, + unsigned short val_marker) override; + + /*! + * \brief Impose the fluid interface (sliding mesh) boundary condition, via the + * CScalarFlux_Heat edge kernel. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] solver_container - Container vector with all the solutions. + * \param[in] conv_numerics - Unused, kept only for the boundary condition dispatch. + * \param[in] visc_numerics - Unused, kept only for the boundary condition dispatch. + * \param[in] config - Definition of the particular problem. + */ + void BC_Fluid_Interface(CGeometry *geometry, CSolver **solver_container, CNumerics *conv_numerics, + CNumerics *visc_numerics, CConfig *config) override; + /*! * \brief Impose the (received) conjugate heat variables. * \param[in] geometry - Geometrical definition of the problem. diff --git a/SU2_CFD/include/solvers/CIncEulerSolver.hpp b/SU2_CFD/include/solvers/CIncEulerSolver.hpp index 98c3d21a8f2a..fd5fbf986092 100644 --- a/SU2_CFD/include/solvers/CIncEulerSolver.hpp +++ b/SU2_CFD/include/solvers/CIncEulerSolver.hpp @@ -41,6 +41,12 @@ class CIncEulerSolver : public CFVMFlowSolverBase FluidModel; /*!< \brief fluid model used in the solver. */ StreamwisePeriodicValues SPvals, SPvalsUpdated; + bool pressure_based; + su2activevector alpha_p; + su2activevector pressureCorrection; + su2activematrix momentumCorrection; + su2activevector EdgeMassFluxCorrection; + /*! * \brief Preprocessing actions common to the Euler and NS solvers. * \param[in] geometry - Geometrical definition of the problem. @@ -86,6 +92,18 @@ class CIncEulerSolver : public CFVMFlowSolverBase Viscous_Residual(unsigned long iEdge, CGeometry *geometry, CSolver **solver_container, + CNumerics *numerics, CConfig *config) override; /*! * \brief Computes the wall shear stress (Tau_Wall) on the surface using a wall function. diff --git a/SU2_CFD/include/solvers/CNEMOEulerSolver.hpp b/SU2_CFD/include/solvers/CNEMOEulerSolver.hpp index ba68b9f9aef6..f1955c46a1aa 100644 --- a/SU2_CFD/include/solvers/CNEMOEulerSolver.hpp +++ b/SU2_CFD/include/solvers/CNEMOEulerSolver.hpp @@ -199,6 +199,19 @@ class CNEMOEulerSolver : public CFVMFlowSolverBase Viscous_Residual(unsigned long iEdge, CGeometry *geometry, CSolver **solver_container, + CNumerics *numerics, CConfig *config) override; /*! * \brief Computes the wall shear stress (Tau_Wall) on the surface using a wall function. diff --git a/SU2_CFD/include/solvers/CPoissonSolver.hpp b/SU2_CFD/include/solvers/CPoissonSolver.hpp new file mode 100644 index 000000000000..e00217e21e7a --- /dev/null +++ b/SU2_CFD/include/solvers/CPoissonSolver.hpp @@ -0,0 +1,227 @@ +/*! + * \file CPoissonSolver.hpp + * \brief Headers of the CPoissonSolver class + * \author T. Aalbers + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#pragma once + +#include "CScalarSolver.hpp" +#include "../variables/CPoissonVariable.hpp" + +/*! + * \class CPoissonSolver + * \brief Main class for defining the finite-volume poisson equation solver. + * \author T. Aalbers + * \version 8.5.0 "Harrier" + */ +class CPoissonSolver final : public CScalarSolver { +protected: + static constexpr size_t MAXNDIM = 3; /*!< \brief Max number of space dimensions, used in some static arrays. */ + static constexpr size_t MAXNVAR = 1; /*!< \brief Max number of variables, for static arrays. */ + +public: + + /* + * \overload + * \param[in] geometry - Geometrical definition of the problem + * \param[in] config - Definition of the particular problem + */ + CPoissonSolver(CGeometry *geometry, CConfig *config, unsigned short iMesh); + + /*! + * \brief Restart residual and compute gradients. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] solver_container - Container vector with all the solutions. + * \param[in] config - Definition of the particular problem. + * \param[in] iMesh - Index of the mesh in multigrid computations. + * \param[in] iRKStep - Current step of the Runge-Kutta iteration. + * \param[in] RunTime_EqSystem - System of equations which is going to be solved. + * \param[in] Output - boolean to determine whether to print output. + */ + void Preprocessing(CGeometry *geometry, + CSolver **solver_container, + CConfig *config, + unsigned short iMesh, + unsigned short iRKStep, + unsigned short RunTime_EqSystem, + bool Output) override; + + /*! + * \brief Correct the pressure and velocities for the flow solution + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] solver_container - Container vector with all the solutions. + * \param[in] config - Definition of the particular problem. + * \param[in] iMesh - Index of the mesh in multigrid computations. + */ + void Postprocessing(CGeometry *geometry, + CSolver **solver_container, + CConfig *config, + unsigned short iMesh) final; + + /*! + * \brief Compute the diffusion of the pressure correction, through the CScalarFlux_Poisson + * edge kernel; the equation has no convective term, see Upwind_Residual. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] solver_container - Container vector with all the solutions. + * \param[in] config - Definition of the particular problem. + * \param[in] iMesh - Index of the mesh in multigrid computations. + * \param[in] iRKStep - Current step of the Runge-Kutta iteration. + */ + void Viscous_Residual(CGeometry *geometry, + CSolver **solver_container, + CNumerics **numerics_container, + CConfig *config, + unsigned short iMesh, + unsigned short iRKStep) override; + + /*! + * \brief Source term computation. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] solver_container - Container vector with all the solutions. + * \param[in] numerics_container - Description of the numerical method. + * \param[in] config - Definition of the particular problem. + * \param[in] iMesh - Index of the mesh in multigrid computations. + */ + void Source_Residual(CGeometry *geometry, CSolver **solver_container, CNumerics **numerics_container, + CConfig *config, unsigned short iMesh) override; + + /*! + * \brief No upwind residual for poisson equation. + */ + void Upwind_Residual(CGeometry* geometry, CSolver** solver_container, CNumerics** numerics_container, + CConfig* config, unsigned short iMesh) override {} + + /*! + * \brief Update the solution using an implicit solver. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] solver_container - Container vector with all the solutions. + * \param[in] config - Definition of the particular problem. + */ + void ImplicitEuler_Iteration(CGeometry *geometry, CSolver **solver_container, CConfig *config) override; + + /*! + * \brief No time step as it is a stationary problem + */ + void SetTime_Step(CGeometry *geometry, + CSolver **solver_container, + CConfig *config, + unsigned short iMesh, + unsigned long Iteration) override {} + + /*! + * \brief No dual time stepping as there is no time stepping at all. + */ + void SetResidual_DualTime(CGeometry* geometry, CSolver** solver_container, CConfig* config, unsigned short iRKStep, + unsigned short iMesh, unsigned short RunTime_EqSystem) override {} + + /*! + * \brief The pressure correction is reset to zero every iteration (see Preprocessing), so it + * carries no state that a restart file needs to provide. + */ + void LoadRestart(CGeometry** geometry, CSolver*** solver, CConfig* config, int val_iter, + bool val_update_geo) override {} + + /*! + * \brief Compute the coefficients for the pressure correction equation based + * on the residuals from the solution of the momentum equation. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] solver_container - Container with all the solutions. + * \param[in] config - Definition of the particular problem. + * \param[in] periodic - Flag for periodic boundary conditions. + * \param[in] iMesh - Index of the mesh in multigrid computations. + */ + void SetMomCoeff(CGeometry *geometry, CSolver **solver_container, CConfig *config, bool periodic, unsigned short iMesh) final; + + + /*! + * \brief Compute the HbyA based on the momentum correction to be used in second PISO + * correction equation. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] solver_container - Container with all the solutions. + * \param[in] config - Definition of the particular problem. + * \param[in] iMesh - Index of the mesh in multigrid computations. + */ + void ComputeHbyA(CGeometry *geometry, CSolver **solver_container, CConfig *config, unsigned short iMesh) final; + + /*! + * \brief Impose a constant heat-flux condition at the wall. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] solver_container - Container vector with all the solutions. + * \param[in] conv_numerics - Description of the numerical method. + * \param[in] visc_numerics - Description of the numerical method. + * \param[in] config - Definition of the particular problem. + * \param[in] val_marker - Surface marker where the boundary condition is applied. + */ + void BC_HeatFlux_Wall(CGeometry *geometry, + CSolver **solver_container, + CNumerics *conv_numerics, + CNumerics *visc_numerics, + CConfig *config, + unsigned short val_marker) final; + + /*! + * \brief A virtual member. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] solver_container - Container vector with all the solutions. + * \param[in] conv_numerics - Description of the numerical method. + * \param[in] visc_numerics - Description of the numerical method. + * \param[in] config - Definition of the particular problem. + * \param[in] val_marker - Surface marker where the boundary condition is applied. + */ + void BC_Far_Field(CGeometry *geometry, CSolver **solver_container, CNumerics *conv_numerics, CNumerics *visc_numerics, CConfig *config, + unsigned short val_marker) final; + + /*! + * \brief Impose the inlet boundary condition. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] solver_container - Container vector with all the solutions. + * \param[in] conv_numerics - Description of the numerical method. + * \param[in] visc_numerics - Description of the numerical method. + * \param[in] config - Definition of the particular problem. + * \param[in] val_marker - Surface marker where the boundary condition is applied. + */ + void BC_Inlet(CGeometry *geometry, + CSolver **solver_container, + CNumerics *conv_numerics, + CNumerics *visc_numerics, + CConfig *config, + unsigned short val_marker) override; + /*! + * \brief Impose the outlet boundary condition. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] solver_container - Container vector with all the solutions. + * \param[in] conv_numerics - Description of the numerical method. + * \param[in] visc_numerics - Description of the numerical method. + * \param[in] config - Definition of the particular problem. + * \param[in] val_marker - Surface marker where the boundary condition is applied. + */ + void BC_Outlet(CGeometry *geometry, + CSolver **solver_container, + CNumerics *conv_numerics, + CNumerics *visc_numerics, + CConfig *config, + unsigned short val_marker) override; + +}; diff --git a/SU2_CFD/include/solvers/CScalarSolver.hpp b/SU2_CFD/include/solvers/CScalarSolver.hpp index 1ef6d597a94b..50ecf8a42c7f 100644 --- a/SU2_CFD/include/solvers/CScalarSolver.hpp +++ b/SU2_CFD/include/solvers/CScalarSolver.hpp @@ -30,11 +30,26 @@ #include "../../../Common/include/parallelization/omp_structure.hpp" #include "../../../Common/include/toolboxes/geometry_toolbox.hpp" +#include "../numerics/scalar/scalar_edge_flux.hpp" #include "../variables/CScalarVariable.hpp" +#include "../variables/CEulerVariable.hpp" #include "../variables/CFlowVariable.hpp" +#include "../variables/CGhostFlowVariable.hpp" +#include "../variables/CIncEulerVariable.hpp" #include "../variables/CPrimitiveIndices.hpp" #include "CSolver.hpp" +/*! + * \brief Carries a type through a value, so a runtime branch can hand a compile-time type to a + * generic lambda (its parameter deduces as CTypeTag, and the lambda recovers T as + * decltype(tag)::type). Standing in for a C++20 template lambda, which this project's + * C++17 baseline does not have. + */ +template +struct CTypeTag { + using type = T; +}; + /*! * \brief Main class for defining a scalar solver. * \tparam VariableType - Class of variable used by the solver inheriting from this template. @@ -58,6 +73,7 @@ class CScalarSolver : public CSolver { su2double Solution_Inf[MAXNVAR]; /*!< \brief Far-field solution. */ const bool Conservative; /*!< \brief Transported Variable is conservative. Solution has to be multiplied with rho. */ + const bool BoundedScalar; /*!< \brief Whether the derived solver uses the bounded-scalar convective scheme. */ const CPrimitiveIndices prim_idx; /*!< \brief Indices of the primitive flow variables. */ @@ -79,6 +95,17 @@ class CScalarSolver : public CSolver { CSysVector EdgeFluxes; /*!< \brief Flux across each edge. */ CSysVector EdgeFluxesDiff; /*!< \brief Flux difference between ij and ji for non-conservative discretisation. */ + /*--- Ghost states of the marker currently being processed by a boundary, indexed by vertex + * and sized to the largest marker; same container types as the interior ones, so the flux + * kernels read a boundary through the same accessors as an interior edge. Boundary loops run + * one marker at a time, parallel over its vertices, so the buffers are written and consumed + * before the next marker reaches them (see BoundaryFluxResidual). ---*/ + unique_ptr ghostNodes; /*!< \brief Allocated by the derived solver, whose VariableType constructor it alone knows how to call. */ + unique_ptr ghostFlowNodes; /*!< \brief Sized from the flow solver, see the constructor. */ + su2activematrix ghostNormal; /*!< \brief Outward normals, sign flipped from the vertex normals. */ + su2activematrix ghostCoord; /*!< \brief Reflected coordinates, read by the diffusion sites. */ + su2vector ghostSkip; /*!< \brief Whether a vertex contributes no flux, set by the fill pass. */ + /*! * \brief The highest level in the variable hierarchy this solver can safely use. */ @@ -89,159 +116,6 @@ class CScalarSolver : public CSolver { */ inline CVariable* GetBaseClassPointerToNodes() final { return nodes; } - /*! - * \brief Compute the viscous flux for the scalar equation at a particular edge. - * \tparam SolverSpecificNumericsFunc - lambda-function, that implements solver specific contributions to numerics. - * \note The functor has to implement (iPoint, jPoint) - * \param[in] iEdge - Edge for which we want to compute the flux - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] solver_container - Container vector with all the solutions. - * \param[in] numerics - Description of the numerical method. - * \param[in] config - Definition of the particular problem. - */ - template - FORCEINLINE void Viscous_Residual_impl(const SolverSpecificNumericsFunc& SolverSpecificNumerics, const unsigned long iEdge, - const CGeometry* geometry, CSolver** solver_container, CNumerics* numerics, - const CConfig* config) { - const bool implicit = (config->GetKind_TimeIntScheme() == EULER_IMPLICIT); - CFlowVariable* flowNodes = solver_container[FLOW_SOL] ? - su2staticcast_p(solver_container[FLOW_SOL]->GetNodes()) : nullptr; - - /*--- Points in edge ---*/ - - auto iPoint = geometry->edges->GetNode(iEdge, 0); - auto jPoint = geometry->edges->GetNode(iEdge, 1); - - /*--- Points coordinates, and normal vector ---*/ - - numerics->SetCoord(geometry->nodes->GetCoord(iPoint), geometry->nodes->GetCoord(jPoint)); - numerics->SetNormal(geometry->edges->GetNormal(iEdge)); - - /*--- Conservative variables w/o reconstruction ---*/ - - if (flowNodes) { - numerics->SetPrimitive(flowNodes->GetPrimitive(iPoint), flowNodes->GetPrimitive(jPoint)); - } - - /*--- Turbulent variables w/o reconstruction, and its gradients ---*/ - - numerics->SetScalarVar(nodes->GetSolution(iPoint), nodes->GetSolution(jPoint)); - numerics->SetScalarVarGradient(nodes->GetGradient(iPoint), nodes->GetGradient(jPoint)); - - /*--- Call Numerics contribution which are Solver-Specifc. Implemented in the caller: Viscous_Residual. ---*/ - - SolverSpecificNumerics(iPoint, jPoint); - - /*--- Compute residual, and Jacobians ---*/ - - auto residual = numerics->ComputeResidual(config); - - if (ReducerStrategy) { - EdgeFluxes.SubtractBlock(iEdge, residual); - if (implicit) Jacobian.UpdateBlocksSub(iEdge, residual.jacobian_i, residual.jacobian_j); - } else { - LinSysRes.SubtractBlock(iPoint, residual); - LinSysRes.AddBlock(jPoint, residual); - if (implicit) Jacobian.UpdateBlocksSub(iEdge, iPoint, jPoint, residual.jacobian_i, residual.jacobian_j); - } - } - - /*! - * \brief Compute the viscous flux for the turbulence equations at a particular edge for a non-conservative discretisation. - * \tparam SolverSpecificNumericsTemp - lambda-function, to implement solver specific contributions to numerics. - * \note The functor has to implement (iPoint, jPoint) - * \param[in] iEdge - Edge for which we want to compute the flux - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] solver_container - Container vector with all the solutions. - * \param[in] numerics - Description of the numerical method. - * \param[in] config - Definition of the particular problem. - */ - template - void Viscous_Residual_NonCons(const unsigned long iEdge, const CGeometry* geometry, CSolver** solver_container, - CNumerics* numerics, const CConfig* config, SolverSpecificNumericsFunc&& SolverSpecificNumerics) { - const bool implicit = (config->GetKind_TimeIntScheme() == EULER_IMPLICIT); - CFlowVariable* flowNodes = solver_container[FLOW_SOL] ? - su2staticcast_p(solver_container[FLOW_SOL]->GetNodes()) : nullptr; - - const auto iPoint = geometry->edges->GetNode(iEdge, 0); - const auto jPoint = geometry->edges->GetNode(iEdge, 1); - - /*--- Lambda function to compute the flux ---*/ - auto ComputeFlux = [&](unsigned long iPoint, unsigned long jPoint, const su2double* normal) { - numerics->SetCoord(geometry->nodes->GetCoord(iPoint),geometry->nodes->GetCoord(jPoint)); - numerics->SetNormal(normal); - - if (flowNodes) { - numerics->SetPrimitive(flowNodes->GetPrimitive(iPoint), flowNodes->GetPrimitive(jPoint)); - } - - /*--- Solver specific numerics contribution. ---*/ - SolverSpecificNumerics(iPoint, jPoint); - - numerics->SetScalarVar(nodes->GetSolution(iPoint), nodes->GetSolution(jPoint)); - numerics->SetScalarVarGradient(nodes->GetGradient(iPoint), nodes->GetGradient(jPoint)); - - return numerics->ComputeResidual(config); - }; - - /*--- Compute fluxes and jacobians i->j ---*/ - const su2double* normal = geometry->edges->GetNormal(iEdge); - auto residual_ij = ComputeFlux(iPoint, jPoint, normal); - - su2mixedfloat *Block_ii = nullptr, *Block_ij = nullptr, *Block_ji = nullptr, *Block_jj = nullptr; - if (implicit) { - Jacobian.GetBlocks(iEdge, iPoint, jPoint, Block_ii, Block_ij, Block_ji, Block_jj); - } - if (ReducerStrategy) { - EdgeFluxes.SubtractBlock(iEdge, residual_ij); - EdgeFluxesDiff.SetBlock(iEdge, residual_ij); - if (implicit) { - /*--- For the reducer strategy the Jacobians are averaged for simplicity. ---*/ - for (int iVar=0; iVari ---*/ - su2double flipped_normal[MAXNDIM]; - for (auto iDim = 0u; iDim < nDim; iDim++) flipped_normal[iDim] = -normal[iDim]; - - auto residual_ji = ComputeFlux(jPoint, iPoint, flipped_normal); - if (ReducerStrategy) { - EdgeFluxesDiff.AddBlock(iEdge, residual_ji); - if (implicit) { - for (int iVar=0; iVar. + * \param[in] opt - Loop invariant flags built by the caller from the current CConfig state. */ - inline virtual void Viscous_Residual(const unsigned long iEdge, const CGeometry* geometry, CSolver** solver_container, - CNumerics* numerics, const CConfig* config) { - /*--- Define an empty object for solver specific numerics contribution. In case there are none, this default - *--- implementation will be called ---*/ - auto SolverSpecificNumerics = [&](unsigned long iPoint, unsigned long jPoint) {}; + template + void EdgeFluxResidual(const CGeometry* geometry, CSolver** solver_container, const CConfig* config, + const ScalarFluxOptions& opt); - /*--- Now instantiate the generic implementation with the functor above. ---*/ + /*! + * \brief Write the four flow primitives the flux kernels read into one row of ghostFlowNodes. + * \param[in] iVertex - Vertex of the marker currently being processed. + * \param[in] V - Row of flow primitives to copy from (e.g. GetCharacPrimVar's or a sliding state's). + */ + inline void SetGhostPrimitives(unsigned long iVertex, const su2double* V) { + auto* ghostV = ghostFlowNodes->GetPrimitive(iVertex); + ghostV[prim_idx.Density()] = V[prim_idx.Density()]; + for (auto iDim = 0u; iDim < nDim; ++iDim) ghostV[prim_idx.Velocity() + iDim] = V[prim_idx.Velocity() + iDim]; + ghostV[prim_idx.LaminarViscosity()] = V[prim_idx.LaminarViscosity()]; + ghostV[prim_idx.EddyViscosity()] = V[prim_idx.EddyViscosity()]; + /*--- NEMO's primitive layout has no single thermal conductivity or specific heat, so its + * CIndices returns a sentinel for these two; only heat reads them, and NEMO rejects any + * scalar transport at configuration. ---*/ + if (prim_idx.ThermalConductivity() != std::numeric_limits::max()) { + ghostV[prim_idx.ThermalConductivity()] = V[prim_idx.ThermalConductivity()]; + ghostV[prim_idx.CpTotal()] = V[prim_idx.CpTotal()]; + } + } + + /*! + * \brief Generic boundary flux pass, run after a boundary's fill pass has written the ghost + * row, the outward normal and (for the diffusion sites) the ghost gradient of every + * vertex of the marker. The ghost point has no row, so only the contribution to the + * interior point is assembled. + * \tparam Scheme - Same model the interior loop uses, instantiated with muscl false. + */ + template + void BoundaryFluxResidual(const CGeometry* geometry, CSolver** solver_container, const CConfig* config, + const ScalarFluxOptions& opt, unsigned short val_marker); - Viscous_Residual_impl(SolverSpecificNumerics, iEdge, geometry, solver_container, numerics, config); + /*! + * \brief Generic fluid interface (sliding mesh) flux pass, shared by every model. The convective + * term is a per-donor weighted average, computed in the same pass that fills the ghost row + * of each donor; the diffusive term is computed once per vertex, after the donor loop, + * from the ghost state the last donor left behind. This does not fit the + * fill-pass-then-BoundaryFluxResidual shape the other boundaries use, so it drives the + * kernel directly. + * \tparam Scheme - Same model the interior loop uses, instantiated with muscl false. + * \param[in] fillGhostExtras - Functor (iVertex, iPoint) writing the auxiliary ghost fields the + * model's diffusion coefficients read, e.g. SST's blending function or the species + * mass diffusivities. Called once per vertex, before the diffusive flux. + */ + template + void FluidInterfaceFluxResidual(const CGeometry* geometry, CSolver** solver_container, const CConfig* config, + const ScalarFluxOptions& optConv, const ScalarFluxOptions& optVisc, + const GhostFunc& fillGhostExtras); + + /*! + * \brief Write the outward normal of one vertex into the ghost row and mark the vertex as + * contributing a flux. + * \note Vertex normals point into the domain, the flux convention needs them outward. + */ + inline void SetGhostGeometry(const CGeometry* geometry, unsigned short val_marker, unsigned long iVertex) { + for (auto iDim = 0u; iDim < nDim; ++iDim) + ghostNormal(iVertex, iDim) = -geometry->vertex[val_marker][iVertex]->GetNormal(iDim); + ghostSkip[iVertex] = false; + } + + /*! + * \brief Write what the diffusion term of a boundary reads beyond the ghost solution: the + * coordinate of the interior point reflected about the boundary, and the interior + * gradient mirrored into the ghost row. + * \param[in] iPoint - Interior point of the vertex. + * \param[in] jPoint - Point the interior one is reflected about, the vertex's normal neighbor. + */ + inline void SetGhostDiffusionState(const CGeometry* geometry, unsigned long iVertex, unsigned long iPoint, + unsigned long jPoint) { + su2double Coord_Reflected[MAXNDIM]; + GeometryToolbox::PointPointReflect(nDim, geometry->nodes->GetCoord(jPoint), geometry->nodes->GetCoord(iPoint), + Coord_Reflected); + for (auto iDim = 0u; iDim < nDim; ++iDim) ghostCoord(iVertex, iDim) = Coord_Reflected[iDim]; + + auto ghostGrad = ghostNodes->GetGradient(iVertex); + const auto interiorGrad = nodes->GetGradient(iPoint); + for (auto iVar = 0u; iVar < nVar; ++iVar) + for (auto iDim = 0u; iDim < nDim; ++iDim) ghostGrad(iVar, iDim) = interiorGrad(iVar, iDim); + } + + /*! + * \brief Resolve the compile-time parameters of a scalar flux kernel, the flow indices, the + * dimension and the equation count, from the runtime state, and call f with a CTypeTag of + * the resulting scheme type: f is a generic lambda, + * `[&](auto tag){ using Scheme = typename decltype(tag)::type; ... }`. + * \tparam Model - Model class template, e.g. CScalarFlux_SST, taking the four parameters of + * CUpwScalarBase: value type, flow indices, dimension and equation count. + * \tparam nVarList - Equation counts to instantiate. Dynamic matches any count, a static one is + * taken when it equals the solver's nVar; the counts are tried in the order given. + * \note NEMO is not one of the index branches: transported scalars are rejected for it at + * configuration. + */ + template