From c851a9edcfca2dae17cb4dc4bc388c6a3ab72214 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 19 Feb 2026 19:13:28 +0100 Subject: [PATCH 1/2] Add ruff formatter Ruff line length Remove Python isort conflicts with ruff Try fixing the line length nicer ruff syntax in pyproject.toml Remove E203 from checking Remove explicit cmd line args from pre-commit config --- .github/workflows/source.yml | 4 +++- .pre-commit-config.yaml | 19 +++++++++++++------ pyproject.toml | 3 +++ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.github/workflows/source.yml b/.github/workflows/source.yml index 0b73d25964..522c2e208c 100644 --- a/.github/workflows/source.yml +++ b/.github/workflows/source.yml @@ -20,7 +20,9 @@ jobs: - name: PEP8 run: | python3 -m pip install -U flake8 - python3 -m flake8 --exclude=thirdParty . + # E203: checks that ':' is not preceded by a space + # But the Ruff formatter has better logic for that + python3 -m flake8 --extend-ignore=E203 --exclude=thirdParty . static-analysis: runs-on: ubuntu-22.04 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 723bd7ad12..20c434435d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -85,13 +85,20 @@ repos: - id: pycln name: pycln (python) -# Sorts Python imports according to PEP8 -# https://www.python.org/dev/peps/pep-0008/#imports -- repo: https://github.com/pycqa/isort - rev: 8.0.1 +# # Sorts Python imports according to PEP8 +# # https://www.python.org/dev/peps/pep-0008/#imports +# - repo: https://github.com/pycqa/isort +# rev: 8.0.1 +# hooks: +# - id: isort +# name: isort (python) + +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.15.1 hooks: - - id: isort - name: isort (python) + - id: ruff-check + args: [ --fix, --exit-non-zero-on-fix ] + - id: ruff-format # Python: Flake8 (checks only, does this support auto-fixes?) #- repo: https://github.com/PyCQA/flake8 diff --git a/pyproject.toml b/pyproject.toml index ca1cd00c16..efa03aba47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,3 +7,6 @@ requires = [ "pybind11>=2.13.0,<3.0.0" ] build-backend = "setuptools.build_meta" + +[tool.ruff] +line-length = 79 From d82ad2204505f3d5f65b8897b9f7b393732dd3e4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Mar 2026 17:02:54 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/source/conf.py | 102 +- examples/10_streaming_read.py | 9 +- examples/10_streaming_write.py | 19 +- examples/11_particle_dataframe.py | 32 +- examples/13_write_dynamic_configuration.py | 32 +- examples/15_compression.py | 10 +- examples/2_read_serial.py | 32 +- examples/2a_read_thetaMode_serial.py | 14 +- examples/3_write_serial.py | 26 +- examples/3a_write_thetaMode_serial.py | 25 +- examples/3b_write_resizable_particles.py | 12 +- examples/4_read_parallel.py | 28 +- examples/5_write_parallel.py | 55 +- examples/7_extended_write_serial.py | 94 +- examples/9_particle_write_serial.py | 12 +- new_version.py | 31 +- setup.py | 236 ++-- share/openPMD/json_schema/check.py | 27 +- src/binding/python/openpmd_api/DaskArray.py | 13 +- .../python/openpmd_api/DaskDataFrame.py | 18 +- src/binding/python/openpmd_api/DataFrame.py | 50 +- src/binding/python/openpmd_api/__init__.py | 7 +- src/binding/python/openpmd_api/ls/__main__.py | 3 +- .../python/openpmd_api/pipe/__main__.py | 268 ++-- src/cli/pipe.py | 1 + test/python/unittest/API/APITest.py | 1223 ++++++++++------- test/python/unittest/Test.py | 4 +- .../unittest/TestUtilities/TestUtilities.py | 2 +- 28 files changed, 1365 insertions(+), 1020 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 1900fe0741..d4c25f8f23 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,7 +26,7 @@ # -- General configuration ------------------------------------------------ # RTD -on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +on_rtd = os.environ.get("READTHEDOCS", None) == "True" show_authors = True @@ -37,63 +37,67 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.mathjax', - 'breathe', - 'sphinx_copybutton', - 'sphinx_design', - 'sphinxcontrib.googleanalytics', - 'sphinxcontrib.programoutput', - 'sphinxcontrib.rsvgconverter', - 'matplotlib.sphinxext.plot_directive'] +extensions = [ + "sphinx.ext.mathjax", + "breathe", + "sphinx_copybutton", + "sphinx_design", + "sphinxcontrib.googleanalytics", + "sphinxcontrib.programoutput", + "sphinxcontrib.rsvgconverter", + "matplotlib.sphinxext.plot_directive", +] if not on_rtd: - extensions.append('sphinx.ext.githubpages') + extensions.append("sphinx.ext.githubpages") # Google Analytics googleanalytics_id = "G-3P1ZT3SQQ5" googleanalytics_enabled = True # breathe config -breathe_projects = {'openPMD-api': '../xml'} -breathe_default_project = 'openPMD-api' +breathe_projects = {"openPMD-api": "../xml"} +breathe_default_project = "openPMD-api" -subprocess.call('cd ..; doxygen;' - 'mkdir -p source/_static;' - 'cp -r doxyhtml source/_static/;' - 'cp openpmd-api-doxygen-web.tag.xml source/_static/doxyhtml/', - shell=True) +subprocess.call( + "cd ..; doxygen;" + "mkdir -p source/_static;" + "cp -r doxyhtml source/_static/;" + "cp openpmd-api-doxygen-web.tag.xml source/_static/doxyhtml/", + shell=True, +) if not on_rtd: html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # source_parsers = { - '.md': CommonMarkParser, + ".md": CommonMarkParser, } -source_suffix = ['.rst', '.md'] +source_suffix = [".rst", ".md"] # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = u'openPMD-api' -copyright = u'Documentation under CC-BY 4.0, The openPMD Community' -author = u'The openPMD Community' +project = "openPMD-api" +copyright = "Documentation under CC-BY 4.0, The openPMD Community" +author = "The openPMD Community" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = u'0.18.0' +version = "0.18.0" # The full version, including alpha/beta/rc tags. -release = u'0.18.0-dev' +release = "0.18.0-dev" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -108,7 +112,7 @@ exclude_patterns = [] # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'default' +pygments_style = "default" # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False @@ -116,12 +120,12 @@ # -- Options for HTML output ---------------------------------------------- -html_logo = 'openPMD.png' +html_logo = "openPMD.png" # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" numfig = True @@ -134,34 +138,31 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. -htmlhelp_basename = 'openPMD-apidoc' +htmlhelp_basename = "openPMD-apidoc" # -- Options for LaTeX output --------------------------------------------- -latex_logo = 'openPMD.png' +latex_logo = "openPMD.png" latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - 'papersize': 'a4paper', - + "papersize": "a4paper", # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - 'preamble': r'\setcounter{tocdepth}{2}', - + "preamble": r"\setcounter{tocdepth}{2}", # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -171,8 +172,13 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'openPMD-api.tex', u'openPMD-api Documentation', - u'The openPMD Community', 'manual'), + ( + master_doc, + "openPMD-api.tex", + "openPMD-api Documentation", + "The openPMD Community", + "manual", + ), ] @@ -181,8 +187,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'openPMD-api', u'openPMD-api Documentation', - [author], 1) + (master_doc, "openPMD-api", "openPMD-api Documentation", [author], 1) ] @@ -192,14 +197,19 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'openPMD-api', u'openPMD-api Documentation', - author, 'openPMD-api', - 'C++ and Python APIs for the openPMD meta-standard', - """ + ( + master_doc, + "openPMD-api", + "openPMD-api Documentation", + author, + "openPMD-api", + "C++ and Python APIs for the openPMD meta-standard", + """ The openPMD standard, short for open standard for particle-mesh data files is not a file format per se. It is a standard for meta data and naming schemes. openPMD provides naming and attribute conventions that allow to exchange particle and mesh based data from scientific simulations and experiments. - """), + """, + ), ] diff --git a/examples/10_streaming_read.py b/examples/10_streaming_read.py index ac7c9654cb..be081948bb 100755 --- a/examples/10_streaming_read.py +++ b/examples/10_streaming_read.py @@ -5,10 +5,11 @@ # pass-through for ADIOS2 engine parameters # https://adios2.readthedocs.io/en/latest/engines/engines.html -config = {'adios2': {'engine': {}, 'dataset': {}}} -config['adios2']['engine'] = {'parameters': - {'Threads': '4', 'DataTransport': 'WAN'}} -config['adios2']['dataset'] = {'operators': [{'type': 'bzip2'}]} +config = {"adios2": {"engine": {}, "dataset": {}}} +config["adios2"]["engine"] = { + "parameters": {"Threads": "4", "DataTransport": "WAN"} +} +config["adios2"]["dataset"] = {"operators": [{"type": "bzip2"}]} if __name__ == "__main__": # this block is for our CI, SST engine is not present on all systems diff --git a/examples/10_streaming_write.py b/examples/10_streaming_write.py index f840d7d1db..f151f9358b 100755 --- a/examples/10_streaming_write.py +++ b/examples/10_streaming_write.py @@ -6,10 +6,11 @@ # pass-through for ADIOS2 engine parameters # https://adios2.readthedocs.io/en/latest/engines/engines.html -config = {'adios2': {'engine': {}, 'dataset': {}}} -config['adios2']['engine'] = {'parameters': - {'Threads': '4', 'DataTransport': 'WAN'}} -config['adios2']['dataset'] = {'operators': [{'type': 'bzip2'}]} +config = {"adios2": {"engine": {}, "dataset": {}}} +config["adios2"]["engine"] = { + "parameters": {"Threads": "4", "DataTransport": "WAN"} +} +config["adios2"]["dataset"] = {"operators": [{"type": "bzip2"}]} if __name__ == "__main__": # this block is for our CI, SST engine is not present on all systems @@ -41,8 +42,9 @@ electronPositions.set_attribute("comment", "I'm a comment") length = 10 - local_data = np.arange(i * length, (i + 1) * length, - dtype=np.dtype("double")) + local_data = np.arange( + i * length, (i + 1) * length, dtype=np.dtype("double") + ) for dim in ["x", "y", "z"]: pos = electronPositions[dim] pos.reset_dataset(io.Dataset(local_data.dtype, [length])) @@ -58,12 +60,13 @@ temperature = iteration.meshes["temperature"] temperature.unit_dimension = {io.Unit_Dimension.theta: 1.0} temperature.axis_labels = ["x", "y"] - temperature.grid_spacing = [1., 1.] + temperature.grid_spacing = [1.0, 1.0] # temperature has no x,y,z components, so skip the last layer: temperature_dataset = temperature # let's say we are in a 3x3 mesh temperature_dataset.reset_dataset( - io.Dataset(np.dtype("double"), [3, 3])) + io.Dataset(np.dtype("double"), [3, 3]) + ) # temperature is constant temperature_dataset.make_constant(273.15) diff --git a/examples/11_particle_dataframe.py b/examples/11_particle_dataframe.py index a7e0d305ed..6ee401ad0d 100755 --- a/examples/11_particle_dataframe.py +++ b/examples/11_particle_dataframe.py @@ -6,6 +6,7 @@ Authors: Axel Huebl, Dmitry Ganyushin License: LGPLv3+ """ + import sys import numpy as np @@ -20,6 +21,7 @@ found_cudf = False try: import cudf + found_cudf = True except ImportError: print("cudf NOT found. Install RAPIDS for CUDA DataFrame example.") @@ -29,6 +31,7 @@ import dask import dask.array as da from dask.delayed import delayed + found_dask = True except ImportError: print("dask NOT found. Install dask to run the 2nd example.") @@ -66,7 +69,7 @@ # the default schedulers are local/threaded, not requiring much. # But multi-node, "distributed" and local "processes" need object # pickle capabilities, so we test this here: - dask.config.set(scheduler='processes') + dask.config.set(scheduler="processes") df = electrons.to_dask(attributes=["particleShape"]) print(df) @@ -79,9 +82,12 @@ print("={}".format(df["momentum_z"].mean().compute())) # example2: momentum histogram - h, bins = da.histogram(df["momentum_z"].to_dask_array(), bins=50, - range=[-8.0e-23, 8.0e-23], - weights=df["weighting"].to_dask_array()) + h, bins = da.histogram( + df["momentum_z"].to_dask_array(), + bins=50, + range=[-8.0e-23, 8.0e-23], + weights=df["weighting"].to_dask_array(), + ) print(h.compute()) # example3: longitudinal phase space (dask 2021.04.0+) @@ -89,10 +95,10 @@ z_max = df["position_z"].max().compute() z_pz, z_pz_bins = da.histogramdd( - df[['position_z', 'momentum_z']].to_dask_array(), + df[["position_z", "momentum_z"]].to_dask_array(), bins=[80, 80], range=[[z_min, z_max], [-8.0e-23, 8.0e-23]], - weights=df["weighting"].to_dask_array() + weights=df["weighting"].to_dask_array(), ) print(z_pz.compute()) @@ -114,9 +120,15 @@ Intensity = darr_x * darr_x + darr_y * darr_y + darr_z * darr_z Intensity_max = Intensity.max().compute() idx_max = da.argwhere(Intensity == Intensity_max).compute()[0] - pos_max = E.grid_unit_SI * 1.0e6 * ( - idx_max * E.grid_spacing + E.grid_global_offset) - print("maximum intensity I={} at index={} z={}mu".format( - Intensity_max, idx_max, pos_max[2])) + pos_max = ( + E.grid_unit_SI + * 1.0e6 + * (idx_max * E.grid_spacing + E.grid_global_offset) + ) + print( + "maximum intensity I={} at index={} z={}mu".format( + Intensity_max, idx_max, pos_max[2] + ) + ) s.close() diff --git a/examples/13_write_dynamic_configuration.py b/examples/13_write_dynamic_configuration.py index 0dc67a8e5c..98c8662e23 100644 --- a/examples/13_write_dynamic_configuration.py +++ b/examples/13_write_dynamic_configuration.py @@ -51,14 +51,15 @@ def main(): - if not io.variants['adios2']: + if not io.variants["adios2"]: # Example configuration below selects the ADIOS2 backend return # create a series and specify some global metadata # change the file extension to .json, .h5 or .bp for regular file writing - series = io.Series("../samples/dynamicConfig.bp", - io.Access_Type.create_linear, defaults) + series = io.Series( + "../samples/dynamicConfig.bp", io.Access_Type.create_linear, defaults + ) # now, write a number of iterations (or: snapshots, time steps) for i in range(10): @@ -81,8 +82,9 @@ def main(): electronPositions.set_attribute("comment", "I'm a comment") length = 10 - local_data = np.arange(i * length, (i + 1) * length, - dtype=np.dtype("double")) + local_data = np.arange( + i * length, (i + 1) * length, dtype=np.dtype("double") + ) for dim in ["x", "y", "z"]: pos = electronPositions[dim] pos.reset_dataset(io.Dataset(local_data.dtype, [length])) @@ -98,27 +100,15 @@ def main(): # we want different compression settings here, # so we override the defaults # let's use JSON this time - config = { - 'resizable': True, - 'adios2': { - 'dataset': { - 'operators': [] - } - } - } - config['adios2']['dataset'] = { - 'operators': { - 'type': 'zlib', - 'parameters': { - 'clevel': 9 - } - } + config = {"resizable": True, "adios2": {"dataset": {"operators": []}}} + config["adios2"]["dataset"] = { + "operators": {"type": "zlib", "parameters": {"clevel": 9}} } temperature = iteration.meshes["temperature"] temperature.unit_dimension = {io.Unit_Dimension.theta: 1.0} temperature.axis_labels = ["x", "y"] - temperature.grid_spacing = [1., 1.] + temperature.grid_spacing = [1.0, 1.0] # temperature has no x,y,z components, so skip the last layer: temperature_dataset = temperature # let's say we are in a 3x3 mesh diff --git a/examples/15_compression.py b/examples/15_compression.py index c7f8e0fe95..aa4c9e3dbd 100644 --- a/examples/15_compression.py +++ b/examples/15_compression.py @@ -200,7 +200,7 @@ def main(): "permanent_filters": { "type": "zlib", # mandatory parameter "aggression": 5, # optional, defaults to 1 - } + }, } }, } @@ -225,7 +225,7 @@ def main(): # optional parameter for filters identified by ID, # mandatory only for zlib (see above) "type": "by_id", - } + }, } }, } @@ -242,7 +242,7 @@ def main(): "permanent_filters": [ {"aggression": 5, "type": "zlib"}, {"flags": "mandatory", "id": "shuffle"}, - ] + ], } }, } @@ -263,7 +263,7 @@ def main(): "permanent_filters": { "type": "zlib", "aggression": 5, - } + }, }, }, { @@ -273,7 +273,7 @@ def main(): "permanent_filters": { "id": "fletcher32", "flags": "mandatory", - } + }, }, }, ] diff --git a/examples/2_read_serial.py b/examples/2_read_serial.py index 3632471757..866cf91e39 100644 --- a/examples/2_read_serial.py +++ b/examples/2_read_serial.py @@ -6,18 +6,20 @@ Authors: Axel Huebl License: LGPLv3+ """ + import openpmd_api as io if __name__ == "__main__": - series = io.Series("../samples/git-sample/data%T.h5", - io.Access.read_only, { - "defer_iteration_parsing": True - }) - print("Read a Series with openPMD standard version %s" % - series.openPMD) + series = io.Series( + "../samples/git-sample/data%T.h5", + io.Access.read_only, + {"defer_iteration_parsing": True}, + ) + print("Read a Series with openPMD standard version %s" % series.openPMD) - print("The Series contains {0} iterations:".format( - len(series.snapshots()))) + print( + "The Series contains {0} iterations:".format(len(series.snapshots())) + ) for i in series.snapshots(): print("\t {0}".format(i)) print("") @@ -28,8 +30,9 @@ for m in i.meshes: print("\t {0}".format(m)) print("") - print("Iteration 100 contains {0} particle species:".format( - len(i.particles))) + print( + "Iteration 100 contains {0} particle species:".format(len(i.particles)) + ) for ps in i.particles: print("\t {0}".format(ps)) print("With records:") @@ -40,22 +43,19 @@ electrons = i.particles["electrons"] charge = electrons["charge"] series.flush() - print("And the first electron particle has a charge {}" - .format(charge[0])) + print("And the first electron particle has a charge {}".format(charge[0])) print("") E_x = i.meshes["E"]["x"] shape = E_x.shape - print("Field E.x has shape {0} and datatype {1}".format( - shape, E_x.dtype)) + print("Field E.x has shape {0} and datatype {1}".format(shape, E_x.dtype)) chunk_data = E_x[1:3, 1:3, 1:2] # print("Queued the loading of a single chunk from disk, " # "ready to execute") series.flush() - print("Chunk has been read from disk\n" - "Read chunk contains:") + print("Chunk has been read from disk\nRead chunk contains:") print(chunk_data) # for row in range(2): # for col in range(2): diff --git a/examples/2a_read_thetaMode_serial.py b/examples/2a_read_thetaMode_serial.py index 3bef932c18..ecab5328a7 100644 --- a/examples/2a_read_thetaMode_serial.py +++ b/examples/2a_read_thetaMode_serial.py @@ -6,15 +6,17 @@ Authors: Axel Huebl License: LGPLv3+ """ + import openpmd_api as io if __name__ == "__main__": # The pattern %E instructs the openPMD-api to determine the file ending # automatically. It can also be given explicitly, e.g. `data%T.h5`. - series = io.Series("../samples/git-sample/thetaMode/data%T.h5", - io.Access.read_only, { - "defer_iteration_parsing": True - }) + series = io.Series( + "../samples/git-sample/thetaMode/data%T.h5", + io.Access.read_only, + {"defer_iteration_parsing": True}, + ) # with defer_iteration_parsing, open() must be called explicitly i = series.snapshots()[500].open() @@ -24,9 +26,9 @@ # read E_z in all modes E_z_raw = E_z_modes[:, :, :] # read E_z in mode_0 (one scalar field) - E_z_m0 = E_z_modes[0:1, 0:shape[1], 0:shape[2]] + E_z_m0 = E_z_modes[0:1, 0 : shape[1], 0 : shape[2]] # read E_z in mode_1 (two fields; skip mode_0 with one scalar field) - E_z_m1 = E_z_modes[1:3, 0:shape[1], 0:shape[2]] + E_z_m1 = E_z_modes[1:3, 0 : shape[1], 0 : shape[2]] series.flush() print(E_z_raw) # still mode-decomposed data, not too useful for users diff --git a/examples/3_write_serial.py b/examples/3_write_serial.py index 1781ef1171..bd1f9fdb44 100644 --- a/examples/3_write_serial.py +++ b/examples/3_write_serial.py @@ -6,6 +6,7 @@ Authors: Axel Huebl License: LGPLv3+ """ + import numpy as np import openpmd_api as io @@ -14,15 +15,17 @@ size = 3 # matrix dataset to write with values 0...size*size-1 - data = np.arange(size*size, dtype=np.double).reshape(3, 3) + data = np.arange(size * size, dtype=np.double).reshape(3, 3) - print("Set up a 2D square array ({0}x{1}) that will be written".format( - size, size)) + print( + "Set up a 2D square array ({0}x{1}) that will be written".format( + size, size + ) + ) # open file for writing series = io.Series( - "../samples/3_write_serial_py.h5", - io.Access.create_linear + "../samples/3_write_serial_py.h5", io.Access.create_linear ) print("Created an empty {0} Series".format(series.iteration_encoding)) @@ -32,8 +35,11 @@ dataset = io.Dataset(data.dtype, data.shape) - print("Created a Dataset of size {0}x{1} and Datatype {2}".format( - dataset.extent[0], dataset.extent[1], dataset.dtype)) + print( + "Created a Dataset of size {0}x{1} and Datatype {2}".format( + dataset.extent[0], dataset.extent[1], dataset.dtype + ) + ) rho.reset_dataset(dataset) print("Set the dataset properties for the scalar field rho in iteration 1") @@ -43,8 +49,10 @@ rho[()] = data - print("Stored the whole Dataset contents as a single chunk, " + - "ready to write content") + print( + "Stored the whole Dataset contents as a single chunk, " + + "ready to write content" + ) # The iteration can be closed in order to help free up resources. # The iteration's content will be flushed automatically. diff --git a/examples/3a_write_thetaMode_serial.py b/examples/3a_write_thetaMode_serial.py index ae0ba47de4..9c203dae00 100644 --- a/examples/3a_write_thetaMode_serial.py +++ b/examples/3a_write_thetaMode_serial.py @@ -6,27 +6,29 @@ Authors: Axel Huebl License: LGPLv3+ """ + import numpy as np import openpmd_api as io if __name__ == "__main__": # open file for writing series = io.Series( - "../samples/3_write_thetaMode_serial_py.h5", - io.Access.create_linear + "../samples/3_write_thetaMode_serial_py.h5", io.Access.create_linear ) # configure and setup geometry num_modes = 5 - num_fields = 1 + (num_modes-1) * 2 # the first mode is purely real + num_fields = 1 + (num_modes - 1) * 2 # the first mode is purely real N_r = 60 N_z = 200 # write values 0...size-1 - E_r_data = np.arange(num_fields*N_r*N_z, dtype=np.double) \ - .reshape(num_fields, N_r, N_z) - E_t_data = np.arange(num_fields*N_r*N_z, dtype=np.single) \ - .reshape(num_fields, N_r, N_z) + E_r_data = np.arange(num_fields * N_r * N_z, dtype=np.double).reshape( + num_fields, N_r, N_z + ) + E_t_data = np.arange(num_fields * N_r * N_z, dtype=np.single).reshape( + num_fields, N_r, N_z + ) geometry_parameters = "m={0};imag=+".format(num_modes) @@ -38,12 +40,11 @@ E.grid_unit_SI = 1.0 E.axis_labels = ["r", "z"] E.data_order = "C" - E.unit_dimension = {io.Unit_Dimension.I: 1.0, - io.Unit_Dimension.J: 2.0} + E.unit_dimension = {io.Unit_Dimension.I: 1.0, io.Unit_Dimension.J: 2.0} # write components: E_z, E_r, E_t E_z = E["z"] - E_z.unit_SI = 10. + E_z.unit_SI = 10.0 E_z.position = [0.0, 0.5] # (modes, r, z) see geometry_parameters E_z.reset_dataset(io.Dataset(io.Datatype.FLOAT, [num_fields, N_r, N_z])) @@ -51,13 +52,13 @@ # write all modes at once (otherwise iterate over modes and first index E_r = E["r"] - E_r.unit_SI = 10. + E_r.unit_SI = 10.0 E_r.position = [0.5, 0.0] E_r.reset_dataset(io.Dataset(E_r_data.dtype, E_r_data.shape)) E_r.store_chunk(E_r_data) E_t = E["t"] - E_t.unit_SI = 10. + E_t.unit_SI = 10.0 E_t.position = [0.0, 0.0] E_t.reset_dataset(io.Dataset(E_t_data.dtype, E_t_data.shape)) E_t.store_chunk(E_t_data) diff --git a/examples/3b_write_resizable_particles.py b/examples/3b_write_resizable_particles.py index 9978877982..ff7623b6ca 100644 --- a/examples/3b_write_resizable_particles.py +++ b/examples/3b_write_resizable_particles.py @@ -6,21 +6,21 @@ Authors: Axel Huebl License: LGPLv3+ """ + import numpy as np import openpmd_api as io if __name__ == "__main__": # open file for writing series = io.Series( - "../samples/3b_write_resizable_particles_py.h5", - io.Access.create + "../samples/3b_write_resizable_particles_py.h5", io.Access.create ) electrons = series.snapshots()[0].particles["electrons"] # our initial data to write - x = np.array([0., 1., 2., 3., 4.], dtype=np.double) - y = np.array([-2., -3., -4., -5., -6.], dtype=np.double) + x = np.array([0.0, 1.0, 2.0, 3.0, 4.0], dtype=np.double) + y = np.array([-2.0, -3.0, -4.0, -5.0, -6.0], dtype=np.double) # both x and y the same type, otherwise we use two distinct datasets dataset = io.Dataset(x.dtype, x.shape, '{ "resizable": true }') @@ -46,8 +46,8 @@ series.flush() # extend and append more particles - x = np.array([5., 6., 7.], dtype=np.double) - y = np.array([-7., -8., -9.], dtype=np.double) + x = np.array([5.0, 6.0, 7.0], dtype=np.double) + y = np.array([-7.0, -8.0, -9.0], dtype=np.double) offset += dataset.extent[0] dataset = io.Dataset([dataset.extent[0] + x.shape[0]]) diff --git a/examples/4_read_parallel.py b/examples/4_read_parallel.py index 91075e6d90..1f612c7d33 100644 --- a/examples/4_read_parallel.py +++ b/examples/4_read_parallel.py @@ -6,6 +6,7 @@ Authors: Axel Huebl License: LGPLv3+ """ + # IMPORTANT: include mpi4py FIRST # https://mpi4py.readthedocs.io/en/stable/mpi4py.run.html # on import: calls MPI_Init_thread() @@ -20,13 +21,11 @@ series = io.Series( "../samples/git-sample/data%T.h5", io.Access.read_only, - comm, { - "defer_iteration_parsing": True - } + comm, + {"defer_iteration_parsing": True}, ) if 0 == comm.rank: - print("Read a series in parallel with {} MPI ranks".format( - comm.size)) + print("Read a series in parallel with {} MPI ranks".format(comm.size)) # with defer_iteration_parsing, open() must be called explicitly # explicit use of open() is recommended for parallel applications @@ -38,8 +37,10 @@ chunk_data = E_x.load_chunk(chunk_offset, chunk_extent) if 0 == comm.rank: - print("Queued the loading of a single chunk per MPI rank from disk, " - "ready to execute") + print( + "Queued the loading of a single chunk per MPI rank from disk, " + "ready to execute" + ) # The iteration can be closed in order to help free up resources. # The iteration's content will be flushed automatically. @@ -53,11 +54,14 @@ print("Rank {} - Read chunk contains:".format(i)) for row in range(chunk_extent[0]): for col in range(chunk_extent[1]): - print("\t({}|{}|1)\t{:e}".format( - row + chunk_offset[0], - col + chunk_offset[1], - chunk_data[row, col, 0] - ), end='') + print( + "\t({}|{}|1)\t{:e}".format( + row + chunk_offset[0], + col + chunk_offset[1], + chunk_data[row, col, 0], + ), + end="", + ) print("") # this barrier is not necessary but structures the example output diff --git a/examples/5_write_parallel.py b/examples/5_write_parallel.py index f4ec41b8e2..445329717c 100644 --- a/examples/5_write_parallel.py +++ b/examples/5_write_parallel.py @@ -6,6 +6,7 @@ Authors: Axel Huebl License: LGPLv3+ """ + # IMPORTANT: include mpi4py FIRST # https://mpi4py.readthedocs.io/en/stable/mpi4py.run.html # on import: calls MPI_Init_thread() @@ -17,8 +18,10 @@ try: import adios2 from packaging import version - USE_JOINED_DIMENSION = \ - version.parse(adios2.__version__) >= version.parse('2.9.0') + + USE_JOINED_DIMENSION = version.parse(adios2.__version__) >= version.parse( + "2.9.0" + ) except ImportError: USE_JOINED_DIMENSION = False @@ -29,11 +32,14 @@ # global data set to write: [MPI_Size * 10, 300] # each rank writes a 10x300 slice with its MPI rank as values local_value = comm.rank - local_data = np.ones(10 * 300, - dtype=np.double).reshape(10, 300) * local_value + local_data = ( + np.ones(10 * 300, dtype=np.double).reshape(10, 300) * local_value + ) if 0 == comm.rank: - print("Set up a 2D array with 10x300 elements per MPI rank ({}x) " - "that will be written to disk".format(comm.size)) + print( + "Set up a 2D array with 10x300 elements per MPI rank ({}x) " + "that will be written to disk".format(comm.size) + ) # open file for writing series = io.Series( @@ -41,11 +47,14 @@ if USE_JOINED_DIMENSION else "../samples/5_parallel_write_py.h5", io.Access.create_linear, - comm + comm, ) if 0 == comm.rank: - print("Created an empty series in parallel with {} MPI ranks".format( - comm.size)) + print( + "Created an empty series in parallel with {} MPI ranks".format( + comm.size + ) + ) # In parallel contexts, it's important to explicitly open iterations. # However, we use Access mode CREATE_LINEAR, so the Series creates @@ -54,19 +63,27 @@ mymesh = series.snapshots()[1].meshes["mymesh"] # example 1D domain decomposition in first index - global_extent = [io.Dataset.JOINED_DIMENSION, 300] \ - if USE_JOINED_DIMENSION else [comm.size * 10, 300] + global_extent = ( + [io.Dataset.JOINED_DIMENSION, 300] + if USE_JOINED_DIMENSION + else [comm.size * 10, 300] + ) dataset = io.Dataset(local_data.dtype, global_extent) if 0 == comm.rank: - print("Prepared a Dataset of size {} and Datatype {}".format( - dataset.extent, dataset.dtype)) + print( + "Prepared a Dataset of size {} and Datatype {}".format( + dataset.extent, dataset.dtype + ) + ) mymesh.reset_dataset(dataset) if 0 == comm.rank: - print("Set the global Dataset properties for the scalar field " - "mymesh in iteration 1") + print( + "Set the global Dataset properties for the scalar field " + "mymesh in iteration 1" + ) # example shows a 1D domain decomposition in first index @@ -77,10 +94,12 @@ # or short: # mymesh[()] = local_data else: - mymesh[comm.rank*10:(comm.rank+1)*10, :] = local_data + mymesh[comm.rank * 10 : (comm.rank + 1) * 10, :] = local_data if 0 == comm.rank: - print("Registered a single chunk per MPI rank containing its " - "contribution, ready to write content to disk") + print( + "Registered a single chunk per MPI rank containing its " + "contribution, ready to write content to disk" + ) # The iteration can be closed in order to help free up resources. # The iteration's content will be flushed automatically. diff --git a/examples/7_extended_write_serial.py b/examples/7_extended_write_serial.py index 49f5283b59..d18a719236 100755 --- a/examples/7_extended_write_serial.py +++ b/examples/7_extended_write_serial.py @@ -6,19 +6,22 @@ Authors: Axel Huebl, Fabian Koller License: LGPLv3+ """ + import numpy as np -from openpmd_api import (Access, Dataset, Mesh_Record_Component, Series, - Unit_Dimension) +from openpmd_api import ( + Access, + Dataset, + Mesh_Record_Component, + Series, + Unit_Dimension, +) SCALAR = Mesh_Record_Component.SCALAR if __name__ == "__main__": # open file for writing - f = Series( - "working/directory/2D_simData_py.h5", - Access.create - ) + f = Series("working/directory/2D_simData_py.h5", Access.create) # all required openPMD attributes will be set to reasonable default values # (all ones, all zeros, empty strings,...) @@ -31,7 +34,7 @@ f.set_attribute( "custom_attribute_name", "This attribute is manually added and can contain about any datatype " - "you would want" + "you would want", ) # note that removing attributes required by the standard typically makes # the file unusable for post-processing @@ -49,15 +52,18 @@ # this is a reference to an iteration reference = f.snapshots()[1] - reference.comment = "Modifications to a reference will always be visible" \ - " in the output" + reference.comment = ( + "Modifications to a reference will always be visible in the output" + ) del reference # alternatively, a copy may be created and later re-assigned to # f.snapshots()[1] copy = f.snapshots()[1] # TODO .copy() - copy.comment = "Modifications to copies will only take effect after you " \ - "reassign the copy" + copy.comment = ( + "Modifications to copies will only take effect after you " + "reassign the copy" + ) f.snapshots()[1] = copy del copy @@ -69,7 +75,9 @@ # https://github.com/openPMD/openPMD-standard/blob/upcoming-1.0.1/STANDARD.md#scalar-vector-and-tensor-records # Meshes are specialized records cur_it.meshes["generic_2D_field"].unit_dimension = { - Unit_Dimension.L: -3, Unit_Dimension.M: 1} + Unit_Dimension.L: -3, + Unit_Dimension.M: 1, + } # as this is a reference, it modifies the original resource lowRez = cur_it.meshes["generic_2D_field"] @@ -84,13 +92,14 @@ electrons = cur_it.particles["electrons"] electrons.set_attribute( "NoteWorthyParticleSpeciesProperty", - "Observing this species was a blast.") + "Observing this species was a blast.", + ) electrons["displacement"].unit_dimension = {Unit_Dimension.M: 1} - electrons["displacement"]["x"].unit_SI = 1.e-6 + electrons["displacement"]["x"].unit_SI = 1.0e-6 del electrons["displacement"] - electrons["weighting"] \ - .reset_dataset(Dataset(np.dtype("float32"), extent=[1])) \ - .make_constant(1.e-5) + electrons["weighting"].reset_dataset( + Dataset(np.dtype("float32"), extent=[1]) + ).make_constant(1.0e-5) mesh = cur_it.meshes["lowRez_2D_field"] mesh.axis_labels = ["x", "y"] @@ -105,12 +114,7 @@ dataset_config = { "adios2": { "dataset": { - "operators": [{ - "type": "zlib", - "parameters": { - "clevel": 9 - } - }] + "operators": [{"type": "zlib", "parameters": {"clevel": 9}}] } } } @@ -130,15 +134,12 @@ dset = Dataset(np.dtype("uint64"), extent=[2]) electrons.particle_patches["numParticles"].reset_dataset(dset) - electrons.particle_patches["numParticlesOffset"]. \ - reset_dataset(dset) + electrons.particle_patches["numParticlesOffset"].reset_dataset(dset) dset = Dataset(partial_particlePos.dtype, extent=[2]) - electrons.particle_patches["offset"].unit_dimension = \ - {Unit_Dimension.L: 1} + electrons.particle_patches["offset"].unit_dimension = {Unit_Dimension.L: 1} electrons.particle_patches["offset"]["x"].reset_dataset(dset) - electrons.particle_patches["extent"].unit_dimension = \ - {Unit_Dimension.L: 1} + electrons.particle_patches["extent"].unit_dimension = {Unit_Dimension.L: 1} electrons.particle_patches["extent"]["x"].reset_dataset(dset) # at any point in time you may decide to dump already created output to @@ -148,10 +149,7 @@ # chunked writing of the final dataset at a time is supported # this loop writes one row at a time - mesh_x = np.array([ - [1, 3, 5, 7, 9], - [11, 13, 15, 17, 19] - ]) + mesh_x = np.array([[1, 3, 5, 7, 9], [11, 13, 15, 17, 19]]) particle_position = np.array([0.1, 0.2, 0.3, 0.4], dtype=np.float32) particle_position_offset = [0, 1, 2, 3] for i in [0, 1]: @@ -166,10 +164,10 @@ # resource is returned to the caller for idx in [0, 1]: - partial_particlePos[idx] = particle_position[idx + 2*i] - partial_particleOff[idx] = particle_position_offset[idx + 2*i] + partial_particlePos[idx] = particle_position[idx + 2 * i] + partial_particleOff[idx] = particle_position_offset[idx + 2 * i] - numParticlesOffset = 2*i + numParticlesOffset = 2 * i numParticles = 2 o = numParticlesOffset @@ -178,20 +176,28 @@ electrons["positionOffset"]["x"][o:u] = partial_particleOff electrons.particle_patches["numParticles"].store( - i, np.array([numParticles], dtype=np.uint64)) + i, np.array([numParticles], dtype=np.uint64) + ) electrons.particle_patches["numParticlesOffset"].store( - i, np.array([numParticlesOffset], dtype=np.uint64)) + i, np.array([numParticlesOffset], dtype=np.uint64) + ) electrons.particle_patches["offset"]["x"].store( i, - np.array([particle_position[numParticlesOffset]], - dtype=np.float32)) + np.array( + [particle_position[numParticlesOffset]], dtype=np.float32 + ), + ) electrons.particle_patches["extent"]["x"].store( i, - np.array([ - particle_position[numParticlesOffset + numParticles - 1] - - particle_position[numParticlesOffset] - ], dtype=np.float32)) + np.array( + [ + particle_position[numParticlesOffset + numParticles - 1] + - particle_position[numParticlesOffset] + ], + dtype=np.float32, + ), + ) mesh["y"].reset_dataset(d) mesh["y"].unit_SI = 4 diff --git a/examples/9_particle_write_serial.py b/examples/9_particle_write_serial.py index 257fea89fc..d22559c78f 100644 --- a/examples/9_particle_write_serial.py +++ b/examples/9_particle_write_serial.py @@ -6,6 +6,7 @@ Authors: Axel Huebl License: LGPLv3+ """ + from pathlib import Path import numpy as np @@ -30,22 +31,23 @@ electrons = cur_it.particles["electrons"] electrons.set_attribute( "Electrons... the necessary evil for ion acceleration! ", - "Just kidding.") + "Just kidding.", + ) n_particles = 234 # let's set a weird user-defined record this time electrons["displacement"].unit_dimension = {Unit_Dimension.M: 1} - electrons["displacement"].unit_SI = 1.e-6 + electrons["displacement"].unit_SI = 1.0e-6 dset = Dataset(np.dtype("float64"), extent=[n_particles]) electrons["displacement"].reset_dataset(dset) electrons["displacement"].make_constant(42.43) # don't like it anymore? remove it with: # del electrons["displacement"] - electrons["weighting"] \ - .reset_dataset(Dataset(np.dtype("float32"), extent=[n_particles])) \ - .make_constant(1.e-5) + electrons["weighting"].reset_dataset( + Dataset(np.dtype("float32"), extent=[n_particles]) + ).make_constant(1.0e-5) particlePos_x = np.random.rand(n_particles).astype(np.float32) particlePos_y = np.random.rand(n_particles).astype(np.float32) diff --git a/new_version.py b/new_version.py index a261c99add..8c16134bbb 100755 --- a/new_version.py +++ b/new_version.py @@ -61,8 +61,9 @@ break OLD_VERSION_TAG = "" -with open(str(REPO_DIR.joinpath("include/openPMD/version.hpp")), - encoding="utf-8") as f: +with open( + str(REPO_DIR.joinpath("include/openPMD/version.hpp")), encoding="utf-8" +) as f: for line in f: match = re.search(r'#define OPENPMDAPI_VERSION_LABEL "([^"]+)"', line) if match: @@ -72,9 +73,10 @@ OLD_VERSION_SUFFIX = f"(-{OLD_VERSION_TAG})?" if OLD_VERSION_TAG else "" # The order of the alternatives is important, since the Regex parser # should greedily include the old version suffix -OLD_VERSION_STR = \ - f"({re.escape(OLD_VERSION_STR_CMAKE)}{OLD_VERSION_SUFFIX})" + \ - f"|({re.escape(OLD_VERSION_STR_README)})" +OLD_VERSION_STR = ( + f"({re.escape(OLD_VERSION_STR_CMAKE)}{OLD_VERSION_SUFFIX})" + + f"|({re.escape(OLD_VERSION_STR_README)})" +) print(f"The old version is: {OLD_VERSION_STR}") print() @@ -117,8 +119,8 @@ def generic_replace(filename, previous, after): for file in [ - "docs/source/dev/linking.rst", - "README.md", + "docs/source/dev/linking.rst", + "README.md", ]: generic_replace(file, previous=OLD_VERSION_STR, after=VERSION_STR) @@ -137,14 +139,19 @@ def generic_replace(filename, previous, after): if match: PREVIOUS_PIP_VERSION = match.group(1) break -generic_replace("setup.py", - previous=PREVIOUS_PIP_VERSION, - after=VERSION_STR_SUFFIX_WITH_DOT) +generic_replace( + "setup.py", + previous=PREVIOUS_PIP_VERSION, + after=VERSION_STR_SUFFIX_WITH_DOT, +) generic_replace( ".github/workflows/windows.yml", previous=f"{PREVIOUS_PIP_VERSION}0?", - after=(f"{VERSION_STR_SUFFIX_WITH_DOT}0" - if SUFFIX else VERSION_STR_SUFFIX_WITH_DOT), + after=( + f"{VERSION_STR_SUFFIX_WITH_DOT}0" + if SUFFIX + else VERSION_STR_SUFFIX_WITH_DOT + ), ) generic_replace( "docs/source/conf.py", diff --git a/setup.py b/setup.py index 7d973046e5..67a9786da7 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ class CMakeExtension(Extension): - def __init__(self, name, sourcedir=''): + def __init__(self, name, sourcedir=""): Extension.__init__(self, name, sources=[]) self.sourcedir = os.path.abspath(sourcedir) @@ -19,27 +19,27 @@ def run(self): from packaging.version import parse try: - out = subprocess.check_output(['cmake', '--version']) + out = subprocess.check_output(["cmake", "--version"]) except OSError: raise RuntimeError( - "CMake 3.22.0+ must be installed to build the following " + - "extensions: " + - ", ".join(e.name for e in self.extensions)) - - cmake_version = parse(re.search( - r'version\s*([\d.]+)', - out.decode() - ).group(1)) - if cmake_version < parse('3.22.0'): + "CMake 3.22.0+ must be installed to build the following " + + "extensions: " + + ", ".join(e.name for e in self.extensions) + ) + + cmake_version = parse( + re.search(r"version\s*([\d.]+)", out.decode()).group(1) + ) + if cmake_version < parse("3.22.0"): raise RuntimeError("CMake >= 3.22.0 is required") for ext in self.extensions: self.build_extension(ext) def build_extension(self, ext): - extdir = os.path.abspath(os.path.dirname( - self.get_ext_fullpath(ext.name) - )) + extdir = os.path.abspath( + os.path.dirname(self.get_ext_fullpath(ext.name)) + ) # required for auto-detection of auxiliary "native" libs if not extdir.endswith(os.path.sep): extdir += os.path.sep @@ -49,168 +49,170 @@ def build_extension(self, ext): # Python: use the calling interpreter in CMake # https://cmake.org/cmake/help/latest/module/FindPython.html#hints # https://cmake.org/cmake/help/latest/command/find_package.html#config-mode-version-selection - '-DPython_ROOT_DIR=' + sys.prefix, - f'-DPython_FIND_VERSION={pyv.major}.{pyv.minor}.{pyv.micro}', - '-DPython_FIND_VERSION_EXACT=TRUE', - '-DPython_FIND_STRATEGY=LOCATION', - '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + - os.path.join(extdir, "openpmd_api"), + "-DPython_ROOT_DIR=" + sys.prefix, + f"-DPython_FIND_VERSION={pyv.major}.{pyv.minor}.{pyv.micro}", + "-DPython_FIND_VERSION_EXACT=TRUE", + "-DPython_FIND_STRATEGY=LOCATION", + "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=" + + os.path.join(extdir, "openpmd_api"), # '-DCMAKE_RUNTIME_OUTPUT_DIRECTORY=' + extdir, - '-DopenPMD_PYTHON_OUTPUT_DIRECTORY=' + extdir, - '-DopenPMD_USE_PYTHON:BOOL=ON', + "-DopenPMD_PYTHON_OUTPUT_DIRECTORY=" + extdir, + "-DopenPMD_USE_PYTHON:BOOL=ON", # variants - '-DopenPMD_USE_MPI:BOOL=' + openPMD_USE_MPI, + "-DopenPMD_USE_MPI:BOOL=" + openPMD_USE_MPI, # skip building cli tools, examples & tests # note: CLI tools provided as console scripts - '-DopenPMD_BUILD_CLI_TOOLS:BOOL=OFF', - '-DopenPMD_BUILD_EXAMPLES:BOOL=' + BUILD_EXAMPLES, - '-DopenPMD_BUILD_TESTING:BOOL=' + BUILD_TESTING, + "-DopenPMD_BUILD_CLI_TOOLS:BOOL=OFF", + "-DopenPMD_BUILD_EXAMPLES:BOOL=" + BUILD_EXAMPLES, + "-DopenPMD_BUILD_TESTING:BOOL=" + BUILD_TESTING, # static/shared libs - '-DopenPMD_BUILD_SHARED_LIBS:BOOL=' + BUILD_SHARED_LIBS, + "-DopenPMD_BUILD_SHARED_LIBS:BOOL=" + BUILD_SHARED_LIBS, # Unix: rpath to current dir when packaged # needed for shared (here non-default) builds - '-DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=ON', - '-DCMAKE_INSTALL_RPATH_USE_LINK_PATH:BOOL=OFF', + "-DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=ON", + "-DCMAKE_INSTALL_RPATH_USE_LINK_PATH:BOOL=OFF", # Windows: has no RPath concept, all `.dll`s must be in %PATH% # or same dir as calling executable ] if HDF5_USE_STATIC_LIBRARIES is not None: - cmake_args.append('-DHDF5_USE_STATIC_LIBRARIES:BOOL=' + - HDF5_USE_STATIC_LIBRARIES) + cmake_args.append( + "-DHDF5_USE_STATIC_LIBRARIES:BOOL=" + HDF5_USE_STATIC_LIBRARIES + ) if ZLIB_USE_STATIC_LIBS is not None: - cmake_args.append('-DZLIB_USE_STATIC_LIBS:BOOL=' + - ZLIB_USE_STATIC_LIBS) + cmake_args.append( + "-DZLIB_USE_STATIC_LIBS:BOOL=" + ZLIB_USE_STATIC_LIBS + ) if CMAKE_INTERPROCEDURAL_OPTIMIZATION is not None: - cmake_args.append('-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=' + - CMAKE_INTERPROCEDURAL_OPTIMIZATION) + cmake_args.append( + "-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=" + + CMAKE_INTERPROCEDURAL_OPTIMIZATION + ) if sys.platform == "darwin": - cmake_args.append('-DCMAKE_INSTALL_RPATH=@loader_path') + cmake_args.append("-DCMAKE_INSTALL_RPATH=@loader_path") else: # values: linux*, aix, freebsd, ... # just as well win32 & cygwin (although Windows has no RPaths) - cmake_args.append('-DCMAKE_INSTALL_RPATH=$ORIGIN') + cmake_args.append("-DCMAKE_INSTALL_RPATH=$ORIGIN") cmake_args += extra_cmake_args - cfg = 'Debug' if self.debug else 'Release' - build_args = ['--config', cfg] + cfg = "Debug" if self.debug else "Release" + build_args = ["--config", cfg] # Assumption: Windows builds are always multi-config (MSVC VS) if platform.system() == "Windows": cmake_args += [ - '-DopenPMD_BUILD_NO_CFG_SUBPATH:BOOL=ON', - '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format( - cfg.upper(), - os.path.join(extdir, "openpmd_api") - ) + "-DopenPMD_BUILD_NO_CFG_SUBPATH:BOOL=ON", + "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}".format( + cfg.upper(), os.path.join(extdir, "openpmd_api") + ), ] if sys.maxsize > 2**32: - cmake_args += ['-A', 'x64'] - build_args += ['--', '/m'] + cmake_args += ["-A", "x64"] + build_args += ["--", "/m"] else: - cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg] - build_args += ['--', '-j2'] + cmake_args += ["-DCMAKE_BUILD_TYPE=" + cfg] + build_args += ["--", "-j2"] env = os.environ.copy() - env['CXXFLAGS'] = '{} -DVERSION_INFO=\\"{}\\"'.format( - env.get('CXXFLAGS', ''), - self.distribution.get_version() + env["CXXFLAGS"] = '{} -DVERSION_INFO=\\"{}\\"'.format( + env.get("CXXFLAGS", ""), self.distribution.get_version() ) if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) subprocess.check_call( - ['cmake', ext.sourcedir] + cmake_args, - cwd=self.build_temp, - env=env + ["cmake", ext.sourcedir] + cmake_args, cwd=self.build_temp, env=env ) subprocess.check_call( - ['cmake', '--build', '.'] + build_args, - cwd=self.build_temp + ["cmake", "--build", "."] + build_args, cwd=self.build_temp ) # note that this does not call install; # we pick up artifacts directly from the build output dirs -with open('./README.md', encoding='utf-8') as f: +with open("./README.md", encoding="utf-8") as f: long_description = f.read() # Allow to control options via environment vars. # Work-around for https://github.com/pypa/setuptools/issues/1712 # note: changed default for SHARED, MPI, TESTING and EXAMPLES -openPMD_USE_MPI = os.environ.get('openPMD_USE_MPI', 'OFF') -HDF5_USE_STATIC_LIBRARIES = os.environ.get('HDF5_USE_STATIC_LIBRARIES', None) -ZLIB_USE_STATIC_LIBS = os.environ.get('ZLIB_USE_STATIC_LIBS', None) +openPMD_USE_MPI = os.environ.get("openPMD_USE_MPI", "OFF") +HDF5_USE_STATIC_LIBRARIES = os.environ.get("HDF5_USE_STATIC_LIBRARIES", None) +ZLIB_USE_STATIC_LIBS = os.environ.get("ZLIB_USE_STATIC_LIBS", None) # deprecated: backwards compatibility to <= 0.13.* -BUILD_SHARED_LIBS = os.environ.get('BUILD_SHARED_LIBS', 'OFF') -BUILD_TESTING = os.environ.get('BUILD_TESTING', 'OFF') -BUILD_EXAMPLES = os.environ.get('BUILD_EXAMPLES', 'OFF') +BUILD_SHARED_LIBS = os.environ.get("BUILD_SHARED_LIBS", "OFF") +BUILD_TESTING = os.environ.get("BUILD_TESTING", "OFF") +BUILD_EXAMPLES = os.environ.get("BUILD_EXAMPLES", "OFF") # end deprecated -BUILD_SHARED_LIBS = os.environ.get('openPMD_BUILD_SHARED_LIBS', - BUILD_SHARED_LIBS) -BUILD_TESTING = os.environ.get('openPMD_BUILD_TESTING', - BUILD_TESTING) -BUILD_EXAMPLES = os.environ.get('openPMD_BUILD_EXAMPLES', - BUILD_EXAMPLES) +BUILD_SHARED_LIBS = os.environ.get( + "openPMD_BUILD_SHARED_LIBS", BUILD_SHARED_LIBS +) +BUILD_TESTING = os.environ.get("openPMD_BUILD_TESTING", BUILD_TESTING) +BUILD_EXAMPLES = os.environ.get("openPMD_BUILD_EXAMPLES", BUILD_EXAMPLES) CMAKE_INTERPROCEDURAL_OPTIMIZATION = os.environ.get( - 'CMAKE_INTERPROCEDURAL_OPTIMIZATION', None) + "CMAKE_INTERPROCEDURAL_OPTIMIZATION", None +) # extra CMake arguments extra_cmake_args = [] for k, v in os.environ.items(): extra_cmake_args_prefix = "openPMD_CMAKE_" - if k.startswith(extra_cmake_args_prefix) and \ - len(k) > len(extra_cmake_args_prefix): - extra_cmake_args.append("-D{0}={1}".format( - k[len(extra_cmake_args_prefix):], - v)) + if k.startswith(extra_cmake_args_prefix) and len(k) > len( + extra_cmake_args_prefix + ): + extra_cmake_args.append( + "-D{0}={1}".format(k[len(extra_cmake_args_prefix) :], v) + ) # https://cmake.org/cmake/help/v3.0/command/if.html -if openPMD_USE_MPI.upper() in ['1', 'ON', 'TRUE', 'YES']: +if openPMD_USE_MPI.upper() in ["1", "ON", "TRUE", "YES"]: openPMD_USE_MPI = "ON" else: openPMD_USE_MPI = "OFF" # Get the package requirements from the requirements.txt file -with open('./requirements.txt') as f: - install_requires = [line.strip('\n') for line in f.readlines()] +with open("./requirements.txt") as f: + install_requires = [line.strip("\n") for line in f.readlines()] if openPMD_USE_MPI == "ON": - install_requires.append('mpi4py>=2.1.0') + install_requires.append("mpi4py>=2.1.0") # keyword reference: # https://packaging.python.org/guides/distributing-packages-using-setuptools setup( - name='openPMD-api', + name="openPMD-api", # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version='0.18.0.dev', - author='Axel Huebl, Franz Poeschel, Fabian Koller, Junmin Gu', - author_email='axelhuebl@lbl.gov, f.poeschel@hzdr.de', - maintainer='Axel Huebl', - maintainer_email='axelhuebl@lbl.gov', - description='C++ & Python API for Scientific I/O with openPMD', + version="0.18.0.dev", + author="Axel Huebl, Franz Poeschel, Fabian Koller, Junmin Gu", + author_email="axelhuebl@lbl.gov, f.poeschel@hzdr.de", + maintainer="Axel Huebl", + maintainer_email="axelhuebl@lbl.gov", + description="C++ & Python API for Scientific I/O with openPMD", long_description=long_description, - long_description_content_type='text/markdown', - keywords=('openPMD openscience hdf5 adios mpi hpc research ' - 'file-format file-handling'), - url='https://www.openPMD.org', + long_description_content_type="text/markdown", + keywords=( + "openPMD openscience hdf5 adios mpi hpc research " + "file-format file-handling" + ), + url="https://www.openPMD.org", project_urls={ - 'Documentation': 'https://openpmd-api.readthedocs.io', - 'Doxygen': 'https://www.openpmd.org/openPMD-api', - 'Reference': 'https://doi.org/10.14278/rodare.27', - 'Source': 'https://github.com/openPMD/openPMD-api', - 'Tracker': 'https://github.com/openPMD/openPMD-api/issues', + "Documentation": "https://openpmd-api.readthedocs.io", + "Doxygen": "https://www.openpmd.org/openPMD-api", + "Reference": "https://doi.org/10.14278/rodare.27", + "Source": "https://github.com/openPMD/openPMD-api", + "Tracker": "https://github.com/openPMD/openPMD-api/issues", }, - ext_modules=[CMakeExtension('openpmd_api_cxx')], + ext_modules=[CMakeExtension("openpmd_api_cxx")], cmdclass=dict(build_ext=CMakeBuild), # scripts=['openpmd-ls'], zip_safe=False, - python_requires='>=3.10', + python_requires=">=3.10", # tests_require=['pytest'], install_requires=install_requires, # see: src/bindings/python/cli entry_points={ - 'console_scripts': [ - 'openpmd-ls = openpmd_api.ls.__main__:main', - 'openpmd-pipe = openpmd_api.pipe.__main__:main' + "console_scripts": [ + "openpmd-ls = openpmd_api.ls.__main__:main", + "openpmd-pipe = openpmd_api.pipe.__main__:main", ] }, # we would like to use this mechanism, but pip / setuptools do not @@ -223,21 +225,23 @@ def build_extension(self, ext): # cmdclass={'test': PyTest}, # platforms='any', classifiers=[ - 'Development Status :: 4 - Beta', - 'Natural Language :: English', - 'Environment :: Console', - 'Intended Audience :: Science/Research', - 'Operating System :: OS Independent', - 'Topic :: Scientific/Engineering', - 'Topic :: Database :: Front-Ends', - 'Programming Language :: C++', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - 'Programming Language :: Python :: 3.14', - ('License :: OSI Approved :: ' - 'GNU Lesser General Public License v3 or later (LGPLv3+)'), + "Development Status :: 4 - Beta", + "Natural Language :: English", + "Environment :: Console", + "Intended Audience :: Science/Research", + "Operating System :: OS Independent", + "Topic :: Scientific/Engineering", + "Topic :: Database :: Front-Ends", + "Programming Language :: C++", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + ( + "License :: OSI Approved :: " + "GNU Lesser General Public License v3 or later (LGPLv3+)" + ), ], ) diff --git a/share/openPMD/json_schema/check.py b/share/openPMD/json_schema/check.py index 155743007d..e1229c5d14 100755 --- a/share/openPMD/json_schema/check.py +++ b/share/openPMD/json_schema/check.py @@ -37,19 +37,22 @@ def parse_args(program_name): Examples: {0} --help {0} --schema_root={1} -""".format(os.path.basename(program_name), script_path / "series.json")) +""".format(os.path.basename(program_name), script_path / "series.json"), + ) parser.add_argument( - '--schema_root', + "--schema_root", default=script_path, help="""\ Directory where to resolve JSON schema files to validate against. -""" +""", + ) + parser.add_argument( + "openpmd_file", + metavar="file", + nargs=1, + help="The file which to validate.", ) - parser.add_argument('openpmd_file', - metavar='file', - nargs=1, - help="The file which to validate.") return parser.parse_args() @@ -64,7 +67,8 @@ def retrieve_from_filesystem(uri): with open(filepath, "r") as referred: loaded_json = json.load(referred) return Resource.from_contents( - loaded_json, default_specification=DRAFT202012) + loaded_json, default_specification=DRAFT202012 + ) registry = Registry(retrieve=retrieve_from_filesystem) @@ -76,5 +80,8 @@ def retrieve_from_filesystem(uri): schema={"$ref": "./series.json"}, registry=registry, ) - print("File {} was validated successfully against schema {}.".format( - instance.name, args.schema_root)) + print( + "File {} was validated successfully against schema {}.".format( + instance.name, args.schema_root + ) + ) diff --git a/src/binding/python/openpmd_api/DaskArray.py b/src/binding/python/openpmd_api/DaskArray.py index 709b668be8..a6ce99fa0f 100644 --- a/src/binding/python/openpmd_api/DaskArray.py +++ b/src/binding/python/openpmd_api/DaskArray.py @@ -5,6 +5,7 @@ Authors: Axel Huebl, Pawel Ordyna License: LGPLv3+ """ + import math import numpy as np @@ -81,13 +82,15 @@ def record_component_to_daskarray(record_component, chunks=None): # Import dask here for a lazy import try: from dask.array import from_array + found_dask = True except ImportError: found_dask = False if not found_dask: - raise ImportError("dask NOT found. Install dask for Dask DataFrame " - "support.") + raise ImportError( + "dask NOT found. Install dask for Dask DataFrame support." + ) if chunks is None: # get optimal chunks chunks = record_component.available_chunks() @@ -97,9 +100,11 @@ def record_component_to_daskarray(record_component, chunks=None): # https://docs.dask.org/en/latest/array-api.html?highlight=from_array#other-functions # sorted and unique offsets_per_dim = list( - map(list, zip(*[chunk.offset for chunk in chunks]))) + map(list, zip(*[chunk.offset for chunk in chunks])) + ) offsets_sorted_unique_per_dim = [ - sorted(set(o)) for o in offsets_per_dim] + sorted(set(o)) for o in offsets_per_dim + ] # print("offsets_sorted_unique_per_dim=", # list(offsets_sorted_unique_per_dim)) diff --git a/src/binding/python/openpmd_api/DaskDataFrame.py b/src/binding/python/openpmd_api/DaskDataFrame.py index 6196e09071..17284dbd7d 100644 --- a/src/binding/python/openpmd_api/DaskDataFrame.py +++ b/src/binding/python/openpmd_api/DaskDataFrame.py @@ -5,11 +5,12 @@ Authors: Axel Huebl, Dmitry Ganyushin, John Kirkham License: LGPLv3+ """ + import numpy as np def read_chunk_to_df(species, chunk, attributes=None): - stride = np.s_[chunk.offset[0]:chunk.offset[0]+chunk.extent[0]] + stride = np.s_[chunk.offset[0] : chunk.offset[0] + chunk.extent[0]] return species.to_df(attributes=attributes, slice=stride) @@ -46,21 +47,25 @@ def particles_to_daskdataframe(particle_species, attributes=None): try: import dask.dataframe as dd from dask.delayed import delayed + found_dask = True except ImportError: found_dask = False try: import pandas # noqa + found_pandas = True except ImportError: found_pandas = False if not found_dask: - raise ImportError("dask NOT found. Install dask for Dask DataFrame " - "support.") + raise ImportError( + "dask NOT found. Install dask for Dask DataFrame support." + ) if not found_pandas: # catch this early: before delayed functions - raise ImportError("pandas NOT found. Install pandas for DataFrame " - "support.") + raise ImportError( + "pandas NOT found. Install pandas for DataFrame support." + ) # get optimal chunks: query first non-constant record component and # assume the same chunking applies for all of them @@ -88,7 +93,8 @@ def particles_to_daskdataframe(particle_species, attributes=None): dfs = [ delayed(read_chunk_to_df)( particle_species, chunk=chunk, attributes=attributes - ) for chunk in chunks + ) + for chunk in chunks ] df = dd.from_delayed(dfs) diff --git a/src/binding/python/openpmd_api/DataFrame.py b/src/binding/python/openpmd_api/DataFrame.py index fed3db81d0..46b5706387 100644 --- a/src/binding/python/openpmd_api/DataFrame.py +++ b/src/binding/python/openpmd_api/DataFrame.py @@ -5,15 +5,15 @@ Authors: Axel Huebl License: LGPLv3+ """ + import math import numpy as np -def particles_to_dataframe(particle_species, - *legacy_args, - attributes=None, - slice=None): +def particles_to_dataframe( + particle_species, *legacy_args, attributes=None, slice=None +): """ Load all records of a particle species into a Pandas DataFrame. @@ -54,24 +54,28 @@ def particles_to_dataframe(particle_species, if attributes is None and slice is None and len(legacy_args) == 1: slice = legacy_args[0] import warnings - warnings.warn("The to_df() argument order changed in " - "openPMD-api 0.17.0!\nThe slice " - "argument must be passed as a named argument.", - DeprecationWarning - ) + + warnings.warn( + "The to_df() argument order changed in " + "openPMD-api 0.17.0!\nThe slice " + "argument must be passed as a named argument.", + DeprecationWarning, + ) else: raise RuntimeError("to_df() does not support unnamed arguments!") # import pandas here for a lazy import try: import pandas as pd + found_pandas = True except ImportError: found_pandas = False if not found_pandas: - raise ImportError("pandas NOT found. Install pandas for DataFrame " - "support.") + raise ImportError( + "pandas NOT found. Install pandas for DataFrame support." + ) if slice is None: slice = np.s_[()] @@ -87,7 +91,8 @@ def particles_to_dataframe(particle_species, particle_species.series_flush() if not math.isclose(1.0, rc.unit_SI): columns[column_name] = np.multiply( - columns[column_name], rc.unit_SI) + columns[column_name], rc.unit_SI + ) df = pd.DataFrame(columns) @@ -137,13 +142,13 @@ def iterations_to_dataframe(series, species_name, attributes=None): try: import pandas as pd except ImportError: - raise ImportError("pandas NOT found. Install pandas for DataFrame " - "support.") + raise ImportError( + "pandas NOT found. Install pandas for DataFrame support." + ) df = pd.concat( ( - iteration - .particles[species_name] + iteration.particles[species_name] .to_df(attributes=attributes) .assign(iteration=i) for i, iteration in series.snapshots().items() @@ -190,20 +195,21 @@ def iterations_to_cudf(series, species_name, attributes=None): try: import pandas # noqa except ImportError: - raise ImportError("pandas NOT found. Install pandas for DataFrame " - "support.") + raise ImportError( + "pandas NOT found. Install pandas for DataFrame support." + ) # import cudf here for a lazy import try: import cudf except ImportError: - raise ImportError("cudf NOT found. Install RAPIDS for CUDA DataFrame " - "support.") + raise ImportError( + "cudf NOT found. Install RAPIDS for CUDA DataFrame support." + ) cdf = cudf.concat( ( cudf.from_pandas( - iteration - .particles[species_name] + iteration.particles[species_name] .to_df(attributes=attributes) .assign(iteration=i) ) diff --git a/src/binding/python/openpmd_api/__init__.py b/src/binding/python/openpmd_api/__init__.py index 09f21026f9..3d248f9683 100644 --- a/src/binding/python/openpmd_api/__init__.py +++ b/src/binding/python/openpmd_api/__init__.py @@ -1,8 +1,11 @@ from . import openpmd_api_cxx as cxx from .DaskArray import record_component_to_daskarray from .DaskDataFrame import particles_to_daskdataframe -from .DataFrame import (iterations_to_cudf, iterations_to_dataframe, - particles_to_dataframe) +from .DataFrame import ( + iterations_to_cudf, + iterations_to_dataframe, + particles_to_dataframe, +) from .openpmd_api_cxx import * # noqa __version__ = cxx.__version__ diff --git a/src/binding/python/openpmd_api/ls/__main__.py b/src/binding/python/openpmd_api/ls/__main__.py index 15d43875b9..959c37516f 100644 --- a/src/binding/python/openpmd_api/ls/__main__.py +++ b/src/binding/python/openpmd_api/ls/__main__.py @@ -8,13 +8,14 @@ Authors: Axel Huebl License: LGPLv3+ """ + import sys from ..openpmd_api_cxx import _ls_run def main(): - """ for usage documentation, call this with a --help argument """ + """for usage documentation, call this with a --help argument""" return _ls_run(sys.argv) diff --git a/src/binding/python/openpmd_api/pipe/__main__.py b/src/binding/python/openpmd_api/pipe/__main__.py index d86ae5c04e..33eab945f4 100644 --- a/src/binding/python/openpmd_api/pipe/__main__.py +++ b/src/binding/python/openpmd_api/pipe/__main__.py @@ -8,6 +8,7 @@ Authors: Franz Poeschel License: LGPLv3+ """ + import argparse import os # os.path.basename import re @@ -55,24 +56,29 @@ def parse_args(program_name): --outfile simData_%T.bp {0} --infile uncompressed.bp \\ --outfile compressed.bp --outconfig @compressionConfig.json -""".format(os.path.basename(program_name))) - - parser.add_argument('--infile', type=str, help='In file') - parser.add_argument('--outfile', type=str, help='Out file') - parser.add_argument('--inconfig', - type=str, - default='{}', - help='JSON config for the in file') - parser.add_argument('--outconfig', - type=str, - default='{}', - help='JSON config for the out file') +""".format(os.path.basename(program_name)), + ) + + parser.add_argument("--infile", type=str, help="In file") + parser.add_argument("--outfile", type=str, help="Out file") + parser.add_argument( + "--inconfig", + type=str, + default="{}", + help="JSON config for the in file", + ) + parser.add_argument( + "--outconfig", + type=str, + default="{}", + help="JSON config for the out file", + ) # MPI, default: Import mpi4py if available and openPMD is parallel, # but don't use if MPI size is 1 (this makes it easier to interact with # JSON, since that backend is unavailable in parallel) - if io.variants['mpi']: - parser.add_argument('--mpi', action='store_true') - parser.add_argument('--no-mpi', dest='mpi', action='store_false') + if io.variants["mpi"]: + parser.add_argument("--mpi", action="store_true") + parser.add_argument("--no-mpi", dest="mpi", action="store_false") parser.set_defaults(mpi=None) return parser.parse_args() @@ -80,19 +86,22 @@ def parse_args(program_name): args = parse_args(sys.argv[0]) # MPI is an optional dependency -if io.variants['mpi'] and (args.mpi is None or args.mpi): +if io.variants["mpi"] and (args.mpi is None or args.mpi): try: from mpi4py import MPI + HAVE_MPI = True except (ImportError, ModuleNotFoundError): if args.mpi: raise else: - print(""" + print( + """ openPMD-api was built with support for MPI, but mpi4py Python package was not found. Will continue in serial mode.""", - file=sys.stderr) + file=sys.stderr, + ) HAVE_MPI = False else: HAVE_MPI = False @@ -113,6 +122,7 @@ def __init__(self, source, dynamicView, offset, extent): self.offset = offset self.extent = extent + # Find below a couple of examples on how to define chunk distribution # strategies in Python by extending classes PartialStrategy or Strategy. # These strategies may then be used inside composing strategies @@ -136,7 +146,6 @@ def assign(self, assignment, ranks_in, ranks_out, my_rank, num_ranks): # Example how to implement a simple strategy in Python class LoadAll(io.Strategy): - def __init__(self): super().__init__() @@ -176,7 +185,7 @@ def assign(self, assignment, in_ranks, out_ranks, my_rank, num_ranks): def hosts_in_order(rank_assignment): already_seen = set() res = [] - for (_, hostname) in rank_assignment.items(): + for _, hostname in rank_assignment.items(): if hostname not in already_seen: already_seen.add(hostname) res.append(hostname) @@ -209,22 +218,26 @@ def hostname_to_hostgroup(ordered_hosts, granularity): # Creates `in_ranks` and `out_ranks` for the inner call, based on the # meta hosts created above def inner_rank_assignment( - outer_rank_assignment, hostname_to_hostgroup): + outer_rank_assignment, hostname_to_hostgroup + ): res = {} - for (rank, hostname) in outer_rank_assignment.items(): + for rank, hostname in outer_rank_assignment.items(): res[rank] = hostname_to_hostgroup[hostname] return res - self.in_ranks_inner = \ - inner_rank_assignment(in_ranks, in_hostname_to_hostgroup) + self.in_ranks_inner = inner_rank_assignment( + in_ranks, in_hostname_to_hostgroup + ) self.out_ranks_inner = inner_rank_assignment( out_ranks, out_hostname_to_hostgroup ) return self.inner_distribution.assign( assignment, - self.in_ranks_inner, self.out_ranks_inner, - my_rank, num_ranks + self.in_ranks_inner, + self.out_ranks_inner, + my_rank, + num_ranks, ) @@ -237,7 +250,8 @@ def __init__(self, inner_strategy): def assign(self, assignment, in_ranks, out_ranks, my_rank, num_ranks): res = self.inner_strategy.assign( - assignment, in_ranks, out_ranks, my_rank, num_ranks) + assignment, in_ranks, out_ranks, my_rank, num_ranks + ) for out_rank, assignment in res.items(): merged = assignment.merge_chunks_from_same_sourceID() assignment.clear() @@ -245,57 +259,64 @@ def assign(self, assignment, in_ranks, out_ranks, my_rank, num_ranks): for chunk in chunks: assignment.append( io.WrittenChunkInfo( - chunk.offset, chunk.extent, in_rank) + chunk.offset, chunk.extent, in_rank + ) ) return res -def distribution_strategy(dataset_extent, - strategy_identifier=None): +def distribution_strategy(dataset_extent, strategy_identifier=None): if strategy_identifier is None or not strategy_identifier: - if 'OPENPMD_CHUNK_DISTRIBUTION' in os.environ: + if "OPENPMD_CHUNK_DISTRIBUTION" in os.environ: strategy_identifier = os.environ[ - 'OPENPMD_CHUNK_DISTRIBUTION'].lower() + "OPENPMD_CHUNK_DISTRIBUTION" + ].lower() else: - strategy_identifier = 'hostname_binpacking_slicedataset' # default - match = re.search('hostname_(.*)_(.*)', strategy_identifier) + strategy_identifier = "hostname_binpacking_slicedataset" # default + match = re.search("hostname_(.*)_(.*)", strategy_identifier) if match is not None: inside_node = distribution_strategy( - dataset_extent, strategy_identifier=match.group(1)) + dataset_extent, strategy_identifier=match.group(1) + ) second_phase = distribution_strategy( - dataset_extent, - strategy_identifier=match.group(2)) + dataset_extent, strategy_identifier=match.group(2) + ) return io.FromPartialStrategy(io.ByHostname(inside_node), second_phase) - elif strategy_identifier == 'fan_in': - granularity = os.environ['OPENPMD_FAN_IN'] + elif strategy_identifier == "fan_in": + granularity = os.environ["OPENPMD_FAN_IN"] granularity = int(granularity) return IncreaseGranularity( - granularity, 1, - io.FromPartialStrategy(io.ByHostname(io.RoundRobin()), - io.DiscardingStrategy())) - elif strategy_identifier == 'all': + granularity, + 1, + io.FromPartialStrategy( + io.ByHostname(io.RoundRobin()), io.DiscardingStrategy() + ), + ) + elif strategy_identifier == "all": return io.FromPartialStrategy(IncreaseGranularity(5), LoadAll()) - elif strategy_identifier == 'roundrobin': + elif strategy_identifier == "roundrobin": return io.RoundRobin() - elif strategy_identifier == 'binpacking': + elif strategy_identifier == "binpacking": return io.BinPacking() - elif strategy_identifier == 'slicedataset': + elif strategy_identifier == "slicedataset": return io.ByCuboidSlice(io.OneDimensionalBlockSlicer(), dataset_extent) - elif strategy_identifier == 'fail': + elif strategy_identifier == "fail": return io.FailingStrategy() - elif strategy_identifier == 'discard': + elif strategy_identifier == "discard": return io.DiscardingStrategy() - elif strategy_identifier == 'blocksofsourceranks': + elif strategy_identifier == "blocksofsourceranks": return io.BlocksOfSourceRanks() else: - raise RuntimeError("Unknown distribution strategy: " + - strategy_identifier) + raise RuntimeError( + "Unknown distribution strategy: " + strategy_identifier + ) class pipe: """ Represents the configuration of one "pipe" pass. """ + def __init__(self, infile, outfile, inconfig, outconfig, comm): self.infile = infile self.outfile = outfile @@ -313,23 +334,27 @@ def run(self): if not HAVE_MPI or (args.mpi is None and self.comm.size == 1): print("Opening data source") sys.stdout.flush() - inseries = io.Series(self.infile, io.Access.read_linear, - self.inconfig) + inseries = io.Series( + self.infile, io.Access.read_linear, self.inconfig + ) print("Opening data sink") sys.stdout.flush() - outseries = io.Series(self.outfile, io.Access.create, - self.outconfig) + outseries = io.Series( + self.outfile, io.Access.create, self.outconfig + ) print("Opened input and output") sys.stdout.flush() else: print("Opening data source on rank {}.".format(self.comm.rank)) sys.stdout.flush() - inseries = io.Series(self.infile, io.Access.read_linear, self.comm, - self.inconfig) + inseries = io.Series( + self.infile, io.Access.read_linear, self.comm, self.inconfig + ) print("Opening data sink on rank {}.".format(self.comm.rank)) sys.stdout.flush() - outseries = io.Series(self.outfile, io.Access.create, self.comm, - self.outconfig) + outseries = io.Series( + self.outfile, io.Access.create, self.comm, self.outconfig + ) print("Opened input and output on rank {}.".format(self.comm.rank)) sys.stdout.flush() # In Linear read mode, global attributes are only present after calling @@ -343,25 +368,39 @@ def __copy(self, src, dest, current_path="/data/"): Copies data from src to dest. May represent any point in the openPMD hierarchy, but src and dest must both represent the same layer. """ - if (type(src) is not type(dest) - and not isinstance(src, io.IndexedIteration) - and not isinstance(dest, io.Iteration)): + if ( + type(src) is not type(dest) + and not isinstance(src, io.IndexedIteration) + and not isinstance(dest, io.Iteration) + ): raise RuntimeError( - "Internal error: Trying to copy mismatching types") + "Internal error: Trying to copy mismatching types" + ) attribute_dtypes = src.attribute_dtypes # The following attributes are written automatically by openPMD-api # and should not be manually overwritten here ignored_attributes = { - io.Series: - ["basePath", "iterationEncoding", "iterationFormat", "openPMD"], + io.Series: [ + "basePath", + "iterationEncoding", + "iterationFormat", + "openPMD", + ], io.Iteration: ["snapshot"], - io.Record_Component: ["value", "shape"] if isinstance( - src, io.Record_Component) and src.constant else [] + io.Record_Component: ["value", "shape"] + if isinstance(src, io.Record_Component) and src.constant + else [], } # filter the map for relevant openpmd object model types from itertools import chain - ignored_attributes = set(chain.from_iterable(value for ( - key, value) in ignored_attributes.items() if isinstance(src, key))) + + ignored_attributes = set( + chain.from_iterable( + value + for (key, value) in ignored_attributes.items() + if isinstance(src, key) + ) + ) for key in src.attributes: ignore_this_attribute = key in ignored_attributes @@ -371,29 +410,41 @@ def __copy(self, src, dest, current_path="/data/"): dest.set_attribute(key, attr, attr_type) container_types = [ - io.Mesh_Container, io.Particle_Container, io.ParticleSpecies, - io.Record, io.Mesh, io.Particle_Patches, io.Patch_Record + io.Mesh_Container, + io.Particle_Container, + io.ParticleSpecies, + io.Record, + io.Mesh, + io.Particle_Patches, + io.Patch_Record, ] - is_container = any([ - isinstance(src, container_type) - for container_type in container_types - ]) + is_container = any( + [ + isinstance(src, container_type) + for container_type in container_types + ] + ) if isinstance(src, io.Series): # main loop: read iterations of src, write to dest write_iterations = dest.write_iterations() for in_iteration in src.read_iterations(): if self.comm.rank == 0: - print("Iteration {0} contains {1} meshes:".format( - in_iteration.iteration_index, - len(in_iteration.meshes))) + print( + "Iteration {0} contains {1} meshes:".format( + in_iteration.iteration_index, + len(in_iteration.meshes), + ) + ) for m in in_iteration.meshes: print("\t {0}".format(m)) print("") print( "Iteration {0} contains {1} particle species:".format( in_iteration.iteration_index, - len(in_iteration.particles))) + len(in_iteration.particles), + ) + ) for ps in in_iteration.particles: print("\t {0}".format(ps)) print("With records:") @@ -408,18 +459,23 @@ def __copy(self, src, dest, current_path="/data/"): out_iteration = write_iterations[in_iteration.iteration_index] sys.stdout.flush() self.__copy( - in_iteration, out_iteration, - current_path + str(in_iteration.iteration_index) + "/") + in_iteration, + out_iteration, + current_path + str(in_iteration.iteration_index) + "/", + ) for deferred in self.loads: deferred.source.load_chunk( - deferred.dynamicView.current_buffer(), deferred.offset, - deferred.extent) + deferred.dynamicView.current_buffer(), + deferred.offset, + deferred.extent, + ) in_iteration.close() out_iteration.close() self.loads.clear() sys.stdout.flush() - elif isinstance(src, io.Record_Component) and (not is_container - or src.scalar): + elif isinstance(src, io.Record_Component) and ( + not is_container or src.scalar + ): shape = src.shape dtype = src.dtype dest.reset_dataset(io.Dataset(dtype, shape)) @@ -433,25 +489,40 @@ def __copy(self, src, dest, current_path="/data/"): chunk_table = src.available_chunks() # todo buffer the strategy strategy = distribution_strategy(shape) - my_chunks = strategy.assign(chunk_table, self.inranks, - self.outranks, - self.comm.rank, self.comm.size) - for chunk in my_chunks[ - self.comm.rank] if self.comm.rank in my_chunks else []: + my_chunks = strategy.assign( + chunk_table, + self.inranks, + self.outranks, + self.comm.rank, + self.comm.size, + ) + for chunk in ( + my_chunks[self.comm.rank] + if self.comm.rank in my_chunks + else [] + ): if debug: end = chunk.offset.copy() for i in range(len(end)): end[i] += chunk.extent[i] - print("{}\t{}/{}:\t{} -- {}".format( - current_path, self.comm.rank, self.comm.size, - chunk.offset, end)) + print( + "{}\t{}/{}:\t{} -- {}".format( + current_path, + self.comm.rank, + self.comm.size, + chunk.offset, + end, + ) + ) span = dest.store_chunk(chunk.offset, chunk.extent) self.loads.append( - deferred_load(src, span, chunk.offset, chunk.extent)) + deferred_load(src, span, chunk.offset, chunk.extent) + ) elif isinstance(src, io.Iteration): self.__copy(src.meshes, dest.meshes, current_path + "meshes/") - self.__copy(src.particles, dest.particles, - current_path + "particles/") + self.__copy( + src.particles, dest.particles, current_path + "particles/" + ) elif is_container: for key in src: self.__copy(src[key], dest[key], current_path + key + "/") @@ -469,8 +540,9 @@ def main(): communicator = MPI.COMM_WORLD else: communicator = FallbackMPICommunicator() - run_pipe = pipe(args.infile, args.outfile, args.inconfig, args.outconfig, - communicator) + run_pipe = pipe( + args.infile, args.outfile, args.inconfig, args.outconfig, communicator + ) run_pipe.run() diff --git a/src/cli/pipe.py b/src/cli/pipe.py index 1d2ec3a05f..9c1fde90e5 100755 --- a/src/cli/pipe.py +++ b/src/cli/pipe.py @@ -9,6 +9,7 @@ Authors: Franz Poeschel License: LGPLv3+ """ + import sys import openpmd_api.pipe.__main__ as pipe diff --git a/test/python/unittest/API/APITest.py b/test/python/unittest/API/APITest.py index c22551a074..9876e146b7 100644 --- a/test/python/unittest/API/APITest.py +++ b/test/python/unittest/API/APITest.py @@ -16,6 +16,7 @@ try: import numpy as np + found_numpy = True print("numpy version: ", np.__version__) except ImportError: @@ -25,36 +26,41 @@ from TestUtilities.TestUtilities import generateTestFilePath tested_file_extensions = [ - ext for ext in io.file_extensions + ext + for ext in io.file_extensions # TOML is relatively slow and it's just an adaptor for the JSON backend, # so it doesn't require full testing - if ext != 'sst' and ext != 'ssc' and ext != 'toml'] + if ext != "sst" and ext != "ssc" and ext != "toml" +] class APITest(unittest.TestCase): - """ Test class testing the openPMD python API (plus some IO). """ + """Test class testing the openPMD python API (plus some IO).""" @classmethod def setUpClass(cls): - """ Setting up the test class. """ + """Setting up the test class.""" pass @classmethod def tearDownClass(cls): - """ Tearing down the test class. """ + """Tearing down the test class.""" pass def setUp(self): - """ Setting up a test. """ + """Setting up a test.""" self.__files_to_remove = [] self.__dirs_to_remove = [] path_to_field_data = generateTestFilePath( - os.path.join("issue-sample", "no_particles", "data%T.h5")) + os.path.join("issue-sample", "no_particles", "data%T.h5") + ) path_to_particle_data = generateTestFilePath( - os.path.join("issue-sample", "no_fields", "data%T.h5")) + os.path.join("issue-sample", "no_fields", "data%T.h5") + ) path_to_data = generateTestFilePath( - os.path.join("git-sample", "data%T.h5")) + os.path.join("git-sample", "data%T.h5") + ) mode = io.Access.read_only self.__field_series = io.Series(path_to_field_data, mode) self.__particle_series = io.Series(path_to_particle_data, mode) @@ -70,7 +76,7 @@ def setUp(self): # assert io.__author__ != "" def tearDown(self): - """ Tearing down a test. """ + """Tearing down a test.""" for f in self.__files_to_remove: if os.path.isfile(f): os.remove(f) @@ -98,7 +104,8 @@ def refcountingCreateData(self): for dim in ["x", "y"]: component = E[dim] component.reset_dataset( - io.Dataset(np.dtype("float"), [10, 10])) + io.Dataset(np.dtype("float"), [10, 10]) + ) component[:, :] = np.reshape( np.arange(i * 100, (i + 1) * 100, dtype=np.dtype("float")), [10, 10], @@ -110,7 +117,8 @@ def refcountingCreateData(self): # Do not bother with a positionOffset position_offset = e["positionOffset"][dim] position_offset.reset_dataset( - io.Dataset(np.dtype("int"), [100])) + io.Dataset(np.dtype("int"), [100]) + ) position_offset.make_constant(0) position = e["position"][dim] @@ -127,11 +135,12 @@ def testRefCounting(self): pos_x = iteration.particles["e"]["position"]["x"] loaded = pos_x[:] read.flush() - self.assertTrue(np.allclose( - loaded, np.arange(0, 100, dtype=np.dtype("float")))) + self.assertTrue( + np.allclose(loaded, np.arange(0, 100, dtype=np.dtype("float"))) + ) def testFieldData(self): - """ Testing serial IO on a pure field dataset. """ + """Testing serial IO on a pure field dataset.""" # Get reference to series stored on test case. series = self.__field_series @@ -151,10 +160,13 @@ def testFieldData(self): self.assertEqual( i.meshes[m].unit_dimension, io.Unit_Dimension.as_array( - io.Unit_Dimension.as_map(i.meshes[m].unit_dimension))) + io.Unit_Dimension.as_map(i.meshes[m].unit_dimension) + ), + ) self.assertEqual( io.Unit_Dimension.as_maps(i.meshes[m].grid_unit_dimension), - [{io.Unit_Dimension.L: 1}, {io.Unit_Dimension.L: 1}]) + [{io.Unit_Dimension.L: 1}, {io.Unit_Dimension.L: 1}], + ) self.assertEqual(io.Unit_Dimension.from_index(0), io.Unit_Dimension.L) self.assertEqual(io.Unit_Dimension.L.as_index(), 0) for idx in range(7): @@ -165,7 +177,7 @@ def testFieldData(self): self.assertEqual(len(i.particles), 0) def testParticleData(self): - """ Testing serial IO on a pure particle dataset. """ + """Testing serial IO on a pure particle dataset.""" # Get reference to series stored on test case. series = self.__field_series @@ -185,10 +197,7 @@ def testParticleData(self): def attributeRoundTrip(self, file_ending): # write - series = io.Series( - "unittest_py_API." + file_ending, - io.Access.create - ) + series = io.Series("unittest_py_API." + file_ending, io.Access.create) # meta data series.set_software("nonsense") # with unspecified version @@ -197,7 +206,7 @@ def attributeRoundTrip(self, file_ending): series.machine = "testMachine" # write one of each supported types - series.set_attribute("char", 'c') # string + series.set_attribute("char", "c") # string series.set_attribute("pyint", 13) series.set_attribute("pyfloat", 3.1416) series.set_attribute("pystring", "howdy!") @@ -206,10 +215,42 @@ def attributeRoundTrip(self, file_ending): series.set_attribute("pybool", False) # array of ... - series.set_attribute("arr_pyint", (13, 26, 39, 52, )) - series.set_attribute("arr_pyfloat", (1.2, 3.4, 4.5, 5.6, )) - series.set_attribute("arr_pystring", ("x", "y", "z", "www", )) - series.set_attribute("arr_pybool", (False, True, True, False, )) + series.set_attribute( + "arr_pyint", + ( + 13, + 26, + 39, + 52, + ), + ) + series.set_attribute( + "arr_pyfloat", + ( + 1.2, + 3.4, + 4.5, + 5.6, + ), + ) + series.set_attribute( + "arr_pystring", + ( + "x", + "y", + "z", + "www", + ), + ) + series.set_attribute( + "arr_pybool", + ( + False, + True, + True, + False, + ), + ) # list of ... series.set_attribute("l_pyint", [13, 26, 39, 52]) series.set_attribute("l_pyfloat", [1.2, 3.4, 4.5, 5.6]) @@ -226,24 +267,67 @@ def attributeRoundTrip(self, file_ending): series.set_attribute("single", np.single(1.234)) series.set_attribute("double", np.double(1.234567)) series.set_attribute("longdouble", np.longdouble(1.23456789)) - series.set_attribute("csingle", np.complex64(1.+2.j)) - series.set_attribute("cdouble", np.complex128(3.+4.j)) + series.set_attribute("csingle", np.complex64(1.0 + 2.0j)) + series.set_attribute("cdouble", np.complex128(3.0 + 4.0j)) if file_ending not in ["bp", "bp4", "bp5"]: - series.set_attribute("clongdouble", np.clongdouble(5.+6.j)) + series.set_attribute("clongdouble", np.clongdouble(5.0 + 6.0j)) # array of ... - series.set_attribute("arr_int16", (np.int16(23), np.int16(26), )) - series.set_attribute("arr_int32", (np.int32(34), np.int32(37), )) - series.set_attribute("arr_int64", (np.int64(45), np.int64(48), )) - series.set_attribute("arr_uint16", - (np.uint16(23), np.uint16(26), )) - series.set_attribute("arr_uint32", - (np.uint32(34), np.uint32(37), )) - series.set_attribute("arr_uint64", - (np.uint64(45), np.uint64(48), )) - series.set_attribute("arr_single", - (np.single(5.6), np.single(5.9), )) - series.set_attribute("arr_double", - (np.double(6.7), np.double(7.1), )) + series.set_attribute( + "arr_int16", + ( + np.int16(23), + np.int16(26), + ), + ) + series.set_attribute( + "arr_int32", + ( + np.int32(34), + np.int32(37), + ), + ) + series.set_attribute( + "arr_int64", + ( + np.int64(45), + np.int64(48), + ), + ) + series.set_attribute( + "arr_uint16", + ( + np.uint16(23), + np.uint16(26), + ), + ) + series.set_attribute( + "arr_uint32", + ( + np.uint32(34), + np.uint32(37), + ), + ) + series.set_attribute( + "arr_uint64", + ( + np.uint64(45), + np.uint64(48), + ), + ) + series.set_attribute( + "arr_single", + ( + np.single(5.6), + np.single(5.9), + ), + ) + series.set_attribute( + "arr_double", + ( + np.double(6.7), + np.double(7.1), + ), + ) # list of ... series.set_attribute("l_int16", [np.int16(23), np.int16(26)]) series.set_attribute("l_int32", [np.int32(34), np.int32(37)]) @@ -253,8 +337,9 @@ def attributeRoundTrip(self, file_ending): series.set_attribute("l_uint64", [np.uint64(45), np.uint64(48)]) series.set_attribute("l_single", [np.single(5.6), np.single(5.9)]) series.set_attribute("l_double", [np.double(6.7), np.double(7.1)]) - series.set_attribute("l_longdouble", - [np.longdouble(7.8e9), np.longdouble(8.2e3)]) + series.set_attribute( + "l_longdouble", [np.longdouble(7.8e9), np.longdouble(8.2e3)] + ) # TODO: ComplexWarning: Casting complex values to real discards the # imaginary part # series.set_attribute("l_csingle", @@ -269,28 +354,37 @@ def attributeRoundTrip(self, file_ending): # np.clongfloat(8.2e3-9.1e3j)]) # numpy.array of ... - series.set_attribute("nparr_int16", - np.array([234, 567], dtype=np.int16)) - series.set_attribute("nparr_int32", - np.array([456, 789], dtype=np.int32)) - series.set_attribute("nparr_int64", - np.array([678, 901], dtype=np.int64)) - series.set_attribute("nparr_single", - np.array([1.2, 2.3], dtype=np.single)) - series.set_attribute("nparr_double", - np.array([4.5, 6.7], dtype=np.double)) - series.set_attribute("nparr_longdouble", - np.array([8.9, 7.6], dtype=np.longdouble)) - series.set_attribute("nparr_csingle", - np.array([1.2 - 0.3j, 2.3 + 4.2j], - dtype=np.complex64)) - series.set_attribute("nparr_cdouble", - np.array([4.5 + 1.1j, 6.7 - 2.2j], - dtype=np.complex128)) + series.set_attribute( + "nparr_int16", np.array([234, 567], dtype=np.int16) + ) + series.set_attribute( + "nparr_int32", np.array([456, 789], dtype=np.int32) + ) + series.set_attribute( + "nparr_int64", np.array([678, 901], dtype=np.int64) + ) + series.set_attribute( + "nparr_single", np.array([1.2, 2.3], dtype=np.single) + ) + series.set_attribute( + "nparr_double", np.array([4.5, 6.7], dtype=np.double) + ) + series.set_attribute( + "nparr_longdouble", np.array([8.9, 7.6], dtype=np.longdouble) + ) + series.set_attribute( + "nparr_csingle", + np.array([1.2 - 0.3j, 2.3 + 4.2j], dtype=np.complex64), + ) + series.set_attribute( + "nparr_cdouble", + np.array([4.5 + 1.1j, 6.7 - 2.2j], dtype=np.complex128), + ) if file_ending not in ["bp", "bp4", "bp5"]: - series.set_attribute("nparr_clongdouble", - np.array([8.9 + 7.8j, 7.6 + 9.2j], - dtype=np.clongdouble)) + series.set_attribute( + "nparr_clongdouble", + np.array([8.9 + 7.8j, 7.6 + 9.2j], dtype=np.clongdouble), + ) # c_types # TODO remove the .value and handle types directly? @@ -303,10 +397,12 @@ def attributeRoundTrip(self, file_ending): series.set_attribute("uint16_c", ctypes.c_uint16(5).value) series.set_attribute("uint32_c", ctypes.c_uint32(6).value) series.set_attribute("uint64_c", ctypes.c_uint64(7).value) - series.set_attribute("float_c", ctypes.c_float(8.e9).value) - series.set_attribute("double_c", ctypes.c_double(7.e289).value) + series.set_attribute("float_c", ctypes.c_float(8.0e9).value) + series.set_attribute("double_c", ctypes.c_double(7.0e289).value) # TODO init of > e304 ? - series.set_attribute("longdouble_c", ctypes.c_longdouble(6.e200).value) + series.set_attribute( + "longdouble_c", ctypes.c_longdouble(6.0e200).value + ) self.assertTrue(series) series.close() @@ -314,8 +410,7 @@ def attributeRoundTrip(self, file_ending): # read back series = io.Series( - "unittest_py_API." + file_ending, - io.Access.read_only + "unittest_py_API." + file_ending, io.Access.read_only ) self.assertEqual(series.software, "openPMD-api-python-tests") @@ -325,7 +420,7 @@ def attributeRoundTrip(self, file_ending): self.assertEqual(series.get_attribute("char"), "c") self.assertEqual(series.get_attribute("pystring"), "howdy!") self.assertEqual(series.get_attribute("pystring2"), "howdy, too!") - if file_ending == 'h5': + if file_ending == "h5": # A byte string b"hello" is always (really?) a vector of unsigned # chars. # HDF5 does not distinguish a platform char type, only explicitly @@ -340,10 +435,12 @@ def attributeRoundTrip(self, file_ending): except TypeError: self.assertEqual( series.get_attribute("pystring3"), - [c for c in "howdy, again!"]) + [c for c in "howdy, again!"], + ) else: - self.assertEqual(bytes(series.get_attribute("pystring3")), - b"howdy, again!") + self.assertEqual( + bytes(series.get_attribute("pystring3")), b"howdy, again!" + ) self.assertEqual(series.get_attribute("pyint"), 13) self.assertAlmostEqual(series.get_attribute("pyfloat"), 3.1416) self.assertEqual(series.get_attribute("pybool"), False) @@ -353,39 +450,58 @@ def attributeRoundTrip(self, file_ending): self.assertEqual(series.get_attribute("int32"), 43) self.assertEqual(series.get_attribute("int64"), 987654321) self.assertAlmostEqual(series.get_attribute("single"), 1.234) - self.assertAlmostEqual(series.get_attribute("double"), - 1.234567) - self.assertAlmostEqual(series.get_attribute("longdouble"), - 1.23456789) - np.testing.assert_almost_equal(series.get_attribute("csingle"), - np.complex64(1.+2.j)) - self.assertAlmostEqual(series.get_attribute("cdouble"), - 3.+4.j) + self.assertAlmostEqual(series.get_attribute("double"), 1.234567) + self.assertAlmostEqual( + series.get_attribute("longdouble"), 1.23456789 + ) + np.testing.assert_almost_equal( + series.get_attribute("csingle"), np.complex64(1.0 + 2.0j) + ) + self.assertAlmostEqual(series.get_attribute("cdouble"), 3.0 + 4.0j) if file_ending not in ["bp", "bp4", "bp5"]: - self.assertAlmostEqual(series.get_attribute("clongdouble"), - 5.+6.j) + self.assertAlmostEqual( + series.get_attribute("clongdouble"), 5.0 + 6.0j + ) # array of ... (will be returned as list) - self.assertListEqual(series.get_attribute("arr_int16"), - [np.int16(23), np.int16(26), ]) + self.assertListEqual( + series.get_attribute("arr_int16"), + [ + np.int16(23), + np.int16(26), + ], + ) # list of ... - self.assertListEqual(series.get_attribute("l_int16"), - [np.int16(23), np.int16(26)]) - self.assertListEqual(series.get_attribute("l_int32"), - [np.int32(34), np.int32(37)]) - self.assertListEqual(series.get_attribute("l_int64"), - [np.int64(45), np.int64(48)]) - self.assertListEqual(series.get_attribute("l_uint16"), - [np.uint16(23), np.uint16(26)]) - self.assertListEqual(series.get_attribute("l_uint32"), - [np.uint32(34), np.uint32(37)]) - self.assertListEqual(series.get_attribute("l_uint64"), - [np.uint64(45), np.uint64(48)]) + self.assertListEqual( + series.get_attribute("l_int16"), [np.int16(23), np.int16(26)] + ) + self.assertListEqual( + series.get_attribute("l_int32"), [np.int32(34), np.int32(37)] + ) + self.assertListEqual( + series.get_attribute("l_int64"), [np.int64(45), np.int64(48)] + ) + self.assertListEqual( + series.get_attribute("l_uint16"), + [np.uint16(23), np.uint16(26)], + ) + self.assertListEqual( + series.get_attribute("l_uint32"), + [np.uint32(34), np.uint32(37)], + ) + self.assertListEqual( + series.get_attribute("l_uint64"), + [np.uint64(45), np.uint64(48)], + ) # self.assertListEqual(series.get_attribute("l_single"), # [np.single(5.6), np.single(5.9)]) - self.assertListEqual(series.get_attribute("l_double"), - [np.double(6.7), np.double(7.1)]) - self.assertListEqual(series.get_attribute("l_longdouble"), - [np.longdouble(7.8e9), np.longdouble(8.2e3)]) + self.assertListEqual( + series.get_attribute("l_double"), + [np.double(6.7), np.double(7.1)], + ) + self.assertListEqual( + series.get_attribute("l_longdouble"), + [np.longdouble(7.8e9), np.longdouble(8.2e3)], + ) # TODO: l_csingle # self.assertListEqual(series.get_attribute("l_cdouble"), # [np.complex128(6.7 + 6.8j), @@ -396,30 +512,37 @@ def attributeRoundTrip(self, file_ending): # np.clongdouble(8.2e3 - 9.1e3j)]) # numpy.array of ... - self.assertListEqual(series.get_attribute("nparr_int16"), - [234, 567]) - self.assertListEqual(series.get_attribute("nparr_int32"), - [456, 789]) - self.assertListEqual(series.get_attribute("nparr_int64"), - [678, 901]) + self.assertListEqual( + series.get_attribute("nparr_int16"), [234, 567] + ) + self.assertListEqual( + series.get_attribute("nparr_int32"), [456, 789] + ) + self.assertListEqual( + series.get_attribute("nparr_int64"), [678, 901] + ) np.testing.assert_almost_equal( - series.get_attribute("nparr_single"), [1.2, 2.3]) + series.get_attribute("nparr_single"), [1.2, 2.3] + ) np.testing.assert_almost_equal( - series.get_attribute("nparr_double"), [4.5, 6.7]) + series.get_attribute("nparr_double"), [4.5, 6.7] + ) np.testing.assert_almost_equal( - series.get_attribute("nparr_longdouble"), [8.9, 7.6]) + series.get_attribute("nparr_longdouble"), [8.9, 7.6] + ) np.testing.assert_almost_equal( series.get_attribute("nparr_csingle"), - np.array([1.2 - 0.3j, 2.3 + 4.2j], - dtype=np.complex64)) + np.array([1.2 - 0.3j, 2.3 + 4.2j], dtype=np.complex64), + ) np.testing.assert_almost_equal( - series.get_attribute("nparr_cdouble"), - [4.5 + 1.1j, 6.7 - 2.2j]) + series.get_attribute("nparr_cdouble"), [4.5 + 1.1j, 6.7 - 2.2j] + ) # not in ADIOS2 if file_ending not in ["bp", "bp4", "bp5"]: np.testing.assert_almost_equal( series.get_attribute("nparr_clongdouble"), - [8.9 + 7.8j, 7.6 + 9.2j]) + [8.9 + 7.8j, 7.6 + 9.2j], + ) # TODO instead of returning lists, return all arrays as np.array? # self.assertEqual( # series.get_attribute("nparr_int16").dtype, np.int16) @@ -441,19 +564,21 @@ def attributeRoundTrip(self, file_ending): if file_ending != "json" and file_ending != "toml": try: c = chr(series.get_attribute("char_c")) - self.assertEqual(c, 'd') + self.assertEqual(c, "d") except TypeError: - self.assertEqual(series.get_attribute("char_c"), 'd') + self.assertEqual(series.get_attribute("char_c"), "d") self.assertEqual(series.get_attribute("int16_c"), 2) self.assertEqual(series.get_attribute("int32_c"), 3) self.assertEqual(series.get_attribute("int64_c"), 4) self.assertEqual(series.get_attribute("uint16_c"), 5) self.assertEqual(series.get_attribute("uint32_c"), 6) self.assertEqual(series.get_attribute("uint64_c"), 7) - self.assertAlmostEqual(series.get_attribute("float_c"), 8.e9) - self.assertAlmostEqual(series.get_attribute("double_c"), 7.e289) - self.assertAlmostEqual(series.get_attribute("longdouble_c"), - ctypes.c_longdouble(6.e200).value) + self.assertAlmostEqual(series.get_attribute("float_c"), 8.0e9) + self.assertAlmostEqual(series.get_attribute("double_c"), 7.0e289) + self.assertAlmostEqual( + series.get_attribute("longdouble_c"), + ctypes.c_longdouble(6.0e200).value, + ) # check listing API io.list_series(series) @@ -474,7 +599,8 @@ def testOpenPMD_2_0(self): E.grid_unit_dimension = [ {io.Unit_Dimension.L: 1}, {io.Unit_Dimension.L: 1}, - {io.Unit_Dimension.L: 1, io.Unit_Dimension.T: -1}] + {io.Unit_Dimension.L: 1, io.Unit_Dimension.T: -1}, + ] E.make_constant(17) B = meshes["B"] @@ -487,22 +613,31 @@ def testOpenPMD_2_0(self): write_2_0.close() read_2_0 = io.Series( - "../samples/openpmd_2_0.json", io.Access.read_only) + "../samples/openpmd_2_0.json", io.Access.read_only + ) meshes = read_2_0.iterations[100].meshes E = meshes["E"] self.assertEqual(E.grid_unit_SI, [1, 2, 3]) - self.assertEqual(E.grid_unit_dimension, io.Unit_Dimension.as_arrays([ - {io.Unit_Dimension.L: 1}, - {io.Unit_Dimension.L: 1}, - {io.Unit_Dimension.L: 1, io.Unit_Dimension.T: -1}])) + self.assertEqual( + E.grid_unit_dimension, + io.Unit_Dimension.as_arrays( + [ + {io.Unit_Dimension.L: 1}, + {io.Unit_Dimension.L: 1}, + {io.Unit_Dimension.L: 1, io.Unit_Dimension.T: -1}, + ] + ), + ) B = meshes["B"] # Will return a list due to openPMD standard being set to 2.0.0 self.assertEqual(B.grid_unit_SI, [3]) # If the attribute is not defined, the mesh is implicitly spatial - self.assertEqual(io.Unit_Dimension.as_maps(B.grid_unit_dimension), [ - {io.Unit_Dimension.L: 1} for _ in range(3)]) + self.assertEqual( + io.Unit_Dimension.as_maps(B.grid_unit_dimension), + [{io.Unit_Dimension.L: 1} for _ in range(3)], + ) read_2_0.close() write_1_1 = io.Series("../samples/openpmd_1_1.json", io.Access.create) @@ -514,13 +649,15 @@ def testOpenPMD_2_0(self): def unsupported_in_1_1(): E.grid_unit_SI = [1, 2, 3] + # self.assertRaises( # io.ErrorIllegalInOpenPMDStandard, unsupported_in_1_1) E.axis_labels = ["x", "y", "z"] E.grid_unit_dimension = [ {io.Unit_Dimension.L: 1}, {io.Unit_Dimension.L: 1}, - {io.Unit_Dimension.L: 1, io.Unit_Dimension.T: -1}] + {io.Unit_Dimension.L: 1, io.Unit_Dimension.T: -1}, + ] E.make_constant(17) B = meshes["B"] @@ -533,31 +670,39 @@ def unsupported_in_1_1(): write_1_1.close() read_1_1 = io.Series( - "../samples/openpmd_1_1.json", io.Access.read_only) + "../samples/openpmd_1_1.json", io.Access.read_only + ) meshes = read_1_1.iterations[100].meshes E = meshes["E"] # Will return a default value due to the failed attempt at setting # a list at write time self.assertEqual(E.grid_unit_SI, 1) - self.assertEqual(E.grid_unit_dimension, io.Unit_Dimension.as_arrays([ - {io.Unit_Dimension.L: 1}, - {io.Unit_Dimension.L: 1}, - {io.Unit_Dimension.L: 1, io.Unit_Dimension.T: -1}])) + self.assertEqual( + E.grid_unit_dimension, + io.Unit_Dimension.as_arrays( + [ + {io.Unit_Dimension.L: 1}, + {io.Unit_Dimension.L: 1}, + {io.Unit_Dimension.L: 1, io.Unit_Dimension.T: -1}, + ] + ), + ) B = meshes["B"] # Will return a scalar due to openPMD standard being set to 2.0.0 self.assertEqual(B.grid_unit_SI, 3) # If the attribute is not defined, the mesh is implicitly spatial - self.assertEqual(io.Unit_Dimension.as_maps(B.grid_unit_dimension), [ - {io.Unit_Dimension.L: 1} for _ in range(3)]) + self.assertEqual( + io.Unit_Dimension.as_maps(B.grid_unit_dimension), + [{io.Unit_Dimension.L: 1} for _ in range(3)], + ) read_1_1.close() def makeConstantRoundTrip(self, file_ending): # write series = io.Series( - "unittest_py_constant_API." + file_ending, - io.Access.create + "unittest_py_constant_API." + file_ending, io.Access.create ) ms = series.iterations[0].meshes @@ -578,10 +723,10 @@ def makeConstantRoundTrip(self, file_ending): ms["pybool"][SCALAR].make_constant(False) # just testing the data_order attribute - ms["char"].data_order = 'C' - ms["pyint"].data_order = 'F' - self.assertEqual(ms["char"].data_order, 'C') - self.assertEqual(ms["pyint"].data_order, 'F') + ms["char"].data_order = "C" + ms["pyint"].data_order = "F" + self.assertEqual(ms["char"].data_order, "C") + self.assertEqual(ms["pyint"].data_order, "F") # staggering meta data ms["pyint"][SCALAR].position = [0.25, 0.5] @@ -611,23 +756,28 @@ def makeConstantRoundTrip(self, file_ending): ms["single"][SCALAR].make_constant(np.single(1.234)) ms["double"][SCALAR].reset_dataset(DS(np.dtype("double"), extent)) ms["double"][SCALAR].make_constant(np.double(1.234567)) - ms["longdouble"][SCALAR].reset_dataset(DS(np.dtype("longdouble"), - extent)) + ms["longdouble"][SCALAR].reset_dataset( + DS(np.dtype("longdouble"), extent) + ) ms["longdouble"][SCALAR].make_constant(np.longdouble(1.23456789)) ms["complex64"][SCALAR].reset_dataset( - DS(np.dtype("complex64"), extent)) - ms["complex64"][SCALAR].make_constant( - np.complex64(1.234 + 2.345j)) + DS(np.dtype("complex64"), extent) + ) + ms["complex64"][SCALAR].make_constant(np.complex64(1.234 + 2.345j)) ms["complex128"][SCALAR].reset_dataset( - DS(np.dtype("complex128"), extent)) + DS(np.dtype("complex128"), extent) + ) ms["complex128"][SCALAR].make_constant( - np.complex128(1.234567 + 2.345678j)) + np.complex128(1.234567 + 2.345678j) + ) if file_ending not in ["bp", "bp4", "bp5"]: ms["clongdouble"][SCALAR].reset_dataset( - DS(np.dtype("clongdouble"), extent)) + DS(np.dtype("clongdouble"), extent) + ) ms["clongdouble"][SCALAR].make_constant( - np.clongdouble(1.23456789 + 2.34567890j)) + np.clongdouble(1.23456789 + 2.34567890j) + ) # flush and close file self.assertTrue(series) @@ -636,16 +786,15 @@ def makeConstantRoundTrip(self, file_ending): # read back series = io.Series( - "unittest_py_constant_API." + file_ending, - io.Access.read_only + "unittest_py_constant_API." + file_ending, io.Access.read_only ) ms = series.iterations[0].meshes o = [1, 2, 3] e = [1, 1, 1] - self.assertEqual(ms["char"].data_order, 'C') - self.assertEqual(ms["pyint"].data_order, 'F') + self.assertEqual(ms["char"].data_order, "C") + self.assertEqual(ms["pyint"].data_order, "F") self.assertTrue(ms["char"].scalar) self.assertTrue(ms["pyint"].scalar) @@ -658,17 +807,19 @@ def makeConstantRoundTrip(self, file_ending): self.assertTrue(ms["pybool"][SCALAR].constant) if found_numpy: - self.assertEqual(ms["char"][SCALAR].load_chunk(o, e), ord('c')) + self.assertEqual(ms["char"][SCALAR].load_chunk(o, e), ord("c")) self.assertEqual(ms["pyint"][SCALAR].load_chunk(o, e), 13) self.assertEqual(ms["pyfloat"][SCALAR].load_chunk(o, e), 3.1416) self.assertEqual(ms["pybool"][SCALAR].load_chunk(o, e), False) if found_numpy: # staggering meta data - np.testing.assert_allclose(ms["pyint"][SCALAR].position, - [0.25, 0.5]) - np.testing.assert_allclose(ms["pyfloat"][SCALAR].position, - [0.5, 0.75]) + np.testing.assert_allclose( + ms["pyint"][SCALAR].position, [0.25, 0.5] + ) + np.testing.assert_allclose( + ms["pyfloat"][SCALAR].position, [0.5, 0.75] + ) self.assertTrue(ms["int16"].scalar) self.assertTrue(ms["int32"].scalar) @@ -684,58 +835,95 @@ def makeConstantRoundTrip(self, file_ending): self.assertTrue(ms["uint64"][SCALAR].constant) self.assertTrue(ms["double"][SCALAR].constant) - self.assertTrue(ms["int16"][SCALAR].load_chunk(o, e).dtype == - np.dtype('int16')) - self.assertTrue(ms["int32"][SCALAR].load_chunk(o, e).dtype == - np.dtype('int32')) - self.assertTrue(ms["int64"][SCALAR].load_chunk(o, e).dtype == - np.dtype('int64')) - self.assertTrue(ms["uint16"][SCALAR].load_chunk(o, e).dtype == - np.dtype('uint16')) - self.assertTrue(ms["uint32"][SCALAR].load_chunk(o, e).dtype == - np.dtype('uint32')) - self.assertTrue(ms["uint64"][SCALAR].load_chunk(o, e).dtype == - np.dtype('uint64')) - self.assertTrue(ms["single"][SCALAR].load_chunk(o, e).dtype == - np.dtype('single')) - self.assertTrue(ms["double"][SCALAR].load_chunk(o, e).dtype == - np.dtype('double')) - self.assertTrue(ms["longdouble"][SCALAR].load_chunk(o, e).dtype - == np.dtype('longdouble')) - self.assertTrue(ms["complex64"][SCALAR].load_chunk(o, e).dtype - == np.dtype('complex64')) - self.assertTrue(ms["complex128"][SCALAR].load_chunk(o, e).dtype - == np.dtype('complex128')) + self.assertTrue( + ms["int16"][SCALAR].load_chunk(o, e).dtype == np.dtype("int16") + ) + self.assertTrue( + ms["int32"][SCALAR].load_chunk(o, e).dtype == np.dtype("int32") + ) + self.assertTrue( + ms["int64"][SCALAR].load_chunk(o, e).dtype == np.dtype("int64") + ) + self.assertTrue( + ms["uint16"][SCALAR].load_chunk(o, e).dtype + == np.dtype("uint16") + ) + self.assertTrue( + ms["uint32"][SCALAR].load_chunk(o, e).dtype + == np.dtype("uint32") + ) + self.assertTrue( + ms["uint64"][SCALAR].load_chunk(o, e).dtype + == np.dtype("uint64") + ) + self.assertTrue( + ms["single"][SCALAR].load_chunk(o, e).dtype + == np.dtype("single") + ) + self.assertTrue( + ms["double"][SCALAR].load_chunk(o, e).dtype + == np.dtype("double") + ) + self.assertTrue( + ms["longdouble"][SCALAR].load_chunk(o, e).dtype + == np.dtype("longdouble") + ) + self.assertTrue( + ms["complex64"][SCALAR].load_chunk(o, e).dtype + == np.dtype("complex64") + ) + self.assertTrue( + ms["complex128"][SCALAR].load_chunk(o, e).dtype + == np.dtype("complex128") + ) if file_ending not in ["bp", "bp4", "bp5"]: - self.assertTrue(ms["clongdouble"][SCALAR].load_chunk(o, e) - .dtype == np.dtype('clongdouble')) + self.assertTrue( + ms["clongdouble"][SCALAR].load_chunk(o, e).dtype + == np.dtype("clongdouble") + ) # FIXME: why does this even work w/o a flush() ? - self.assertEqual(ms["int16"][SCALAR].load_chunk(o, e), - np.int16(234)) - self.assertEqual(ms["int32"][SCALAR].load_chunk(o, e), - np.int32(43)) - self.assertEqual(ms["int64"][SCALAR].load_chunk(o, e), - np.int64(987654321)) - self.assertEqual(ms["uint16"][SCALAR].load_chunk(o, e), - np.uint16(134)) - self.assertEqual(ms["uint32"][SCALAR].load_chunk(o, e), - np.uint32(32)) - self.assertEqual(ms["uint64"][SCALAR].load_chunk(o, e), - np.uint64(9876543210)) - self.assertEqual(ms["single"][SCALAR].load_chunk(o, e), - np.single(1.234)) - self.assertEqual(ms["longdouble"][SCALAR].load_chunk(o, e), - np.longdouble(1.23456789)) - self.assertEqual(ms["double"][SCALAR].load_chunk(o, e), - np.double(1.234567)) - self.assertEqual(ms["complex64"][SCALAR].load_chunk(o, e), - np.complex64(1.234 + 2.345j)) - self.assertEqual(ms["complex128"][SCALAR].load_chunk(o, e), - np.complex128(1.234567 + 2.345678j)) + self.assertEqual( + ms["int16"][SCALAR].load_chunk(o, e), np.int16(234) + ) + self.assertEqual( + ms["int32"][SCALAR].load_chunk(o, e), np.int32(43) + ) + self.assertEqual( + ms["int64"][SCALAR].load_chunk(o, e), np.int64(987654321) + ) + self.assertEqual( + ms["uint16"][SCALAR].load_chunk(o, e), np.uint16(134) + ) + self.assertEqual( + ms["uint32"][SCALAR].load_chunk(o, e), np.uint32(32) + ) + self.assertEqual( + ms["uint64"][SCALAR].load_chunk(o, e), np.uint64(9876543210) + ) + self.assertEqual( + ms["single"][SCALAR].load_chunk(o, e), np.single(1.234) + ) + self.assertEqual( + ms["longdouble"][SCALAR].load_chunk(o, e), + np.longdouble(1.23456789), + ) + self.assertEqual( + ms["double"][SCALAR].load_chunk(o, e), np.double(1.234567) + ) + self.assertEqual( + ms["complex64"][SCALAR].load_chunk(o, e), + np.complex64(1.234 + 2.345j), + ) + self.assertEqual( + ms["complex128"][SCALAR].load_chunk(o, e), + np.complex128(1.234567 + 2.345678j), + ) if file_ending not in ["bp", "bp4", "bp5"]: - self.assertEqual(ms["clongdouble"][SCALAR].load_chunk(o, e), - np.clongdouble(1.23456789 + 2.34567890j)) + self.assertEqual( + ms["clongdouble"][SCALAR].load_chunk(o, e), + np.clongdouble(1.23456789 + 2.34567890j), + ) def testConstantRecords(self): for ext in tested_file_extensions: @@ -747,8 +935,7 @@ def makeDataRoundTrip(self, file_ending): # write series = io.Series( - "unittest_py_data_API." + file_ending, - io.Access.create + "unittest_py_data_API." + file_ending, io.Access.create ) it = series.iterations[0] @@ -763,21 +950,26 @@ def makeDataRoundTrip(self, file_ending): extent = [42, 24, 11] ms["complex64"][SCALAR].reset_dataset( - DS(np.dtype("complex64"), extent)) + DS(np.dtype("complex64"), extent) + ) ms["complex64"][SCALAR].store_chunk( - np.ones(extent, dtype=np.complex64) * - np.complex64(1.234 + 2.345j)) + np.ones(extent, dtype=np.complex64) * np.complex64(1.234 + 2.345j) + ) ms["complex128"][SCALAR].reset_dataset( - DS(np.dtype("complex128"), extent)) + DS(np.dtype("complex128"), extent) + ) ms["complex128"][SCALAR].store_chunk( - np.ones(extent, dtype=np.complex128) * - np.complex128(1.234567 + 2.345678j)) + np.ones(extent, dtype=np.complex128) + * np.complex128(1.234567 + 2.345678j) + ) if file_ending not in ["bp", "bp4", "bp5"]: ms["clongdouble"][SCALAR].reset_dataset( - DS(np.dtype("clongdouble"), extent)) + DS(np.dtype("clongdouble"), extent) + ) ms["clongdouble"][SCALAR].store_chunk( - np.ones(extent, dtype=np.clongdouble) * - np.clongdouble(1.23456789 + 2.34567890j)) + np.ones(extent, dtype=np.clongdouble) + * np.clongdouble(1.23456789 + 2.34567890j) + ) # flush and close file self.assertTrue(series) @@ -786,8 +978,7 @@ def makeDataRoundTrip(self, file_ending): # read back series = io.Series( - "unittest_py_data_API." + file_ending, - io.Access.read_only + "unittest_py_data_API." + file_ending, io.Access.read_only ) it = series.iterations[0] @@ -807,21 +998,17 @@ def makeDataRoundTrip(self, file_ending): if file_ending not in ["bp", "bp4", "bp5"]: dc256 = ms["clongdouble"][SCALAR].load_chunk(o, e) - self.assertTrue(dc64.dtype == np.dtype('complex64')) - self.assertTrue(dc128.dtype == np.dtype('complex128')) + self.assertTrue(dc64.dtype == np.dtype("complex64")) + self.assertTrue(dc128.dtype == np.dtype("complex128")) if file_ending not in ["bp", "bp4", "bp5"]: - self.assertTrue( - dc256.dtype == np.dtype('clongdouble')) + self.assertTrue(dc256.dtype == np.dtype("clongdouble")) series.flush() - self.assertEqual(dc64, - np.complex64(1.234 + 2.345j)) - self.assertEqual(dc128, - np.complex128(1.234567 + 2.345678j)) + self.assertEqual(dc64, np.complex64(1.234 + 2.345j)) + self.assertEqual(dc128, np.complex128(1.234567 + 2.345678j)) if file_ending not in ["bp", "bp4", "bp5"]: - self.assertEqual(dc256, - np.clongdouble(1.23456789 + 2.34567890j)) + self.assertEqual(dc256, np.clongdouble(1.23456789 + 2.34567890j)) def testDataRoundTrip(self): for ext in tested_file_extensions: @@ -830,8 +1017,7 @@ def testDataRoundTrip(self): def makeEmptyRoundTrip(self, file_ending): # write series = io.Series( - "unittest_py_empty_API." + file_ending, - io.Access_Type.create + "unittest_py_empty_API." + file_ending, io.Access_Type.create ) ms = series.iterations[0].meshes @@ -869,97 +1055,45 @@ def makeEmptyRoundTrip(self, file_ending): # read back series = io.Series( - "unittest_py_empty_API." + file_ending, - io.Access_Type.read_only + "unittest_py_empty_API." + file_ending, io.Access_Type.read_only ) ms = series.iterations[0].meshes + self.assertEqual(ms["CHAR"][SCALAR].shape, [0 for _ in range(1)]) + self.assertEqual(ms["UCHAR"][SCALAR].shape, [0 for _ in range(2)]) + self.assertEqual(ms["SHORT"][SCALAR].shape, [0 for _ in range(3)]) + self.assertEqual(ms["INT"][SCALAR].shape, [0 for _ in range(4)]) + self.assertEqual(ms["LONG"][SCALAR].shape, [0 for _ in range(5)]) + self.assertEqual(ms["LONGLONG"][SCALAR].shape, [0 for _ in range(6)]) + self.assertEqual(ms["USHORT"][SCALAR].shape, [0 for _ in range(7)]) + self.assertEqual(ms["UINT"][SCALAR].shape, [0 for _ in range(8)]) + self.assertEqual(ms["ULONG"][SCALAR].shape, [0 for _ in range(9)]) + self.assertEqual(ms["ULONGLONG"][SCALAR].shape, [0 for _ in range(10)]) + self.assertEqual(ms["FLOAT"][SCALAR].shape, [0 for _ in range(11)]) + self.assertEqual(ms["DOUBLE"][SCALAR].shape, [0 for _ in range(12)]) self.assertEqual( - ms["CHAR"][SCALAR].shape, - [0 for _ in range(1)] - ) - self.assertEqual( - ms["UCHAR"][SCALAR].shape, - [0 for _ in range(2)] - ) - self.assertEqual( - ms["SHORT"][SCALAR].shape, - [0 for _ in range(3)] - ) - self.assertEqual( - ms["INT"][SCALAR].shape, - [0 for _ in range(4)] - ) - self.assertEqual( - ms["LONG"][SCALAR].shape, - [0 for _ in range(5)] - ) - self.assertEqual( - ms["LONGLONG"][SCALAR].shape, - [0 for _ in range(6)] - ) - self.assertEqual( - ms["USHORT"][SCALAR].shape, - [0 for _ in range(7)] - ) - self.assertEqual( - ms["UINT"][SCALAR].shape, - [0 for _ in range(8)] - ) - self.assertEqual( - ms["ULONG"][SCALAR].shape, - [0 for _ in range(9)] - ) - self.assertEqual( - ms["ULONGLONG"][SCALAR].shape, - [0 for _ in range(10)] - ) - self.assertEqual( - ms["FLOAT"][SCALAR].shape, - [0 for _ in range(11)] - ) - self.assertEqual( - ms["DOUBLE"][SCALAR].shape, - [0 for _ in range(12)] - ) - self.assertEqual( - ms["LONG_DOUBLE"][SCALAR].shape, - [0 for _ in range(13)] + ms["LONG_DOUBLE"][SCALAR].shape, [0 for _ in range(13)] ) if found_numpy: + self.assertEqual(ms["int16"][SCALAR].shape, [0 for _ in range(14)]) + self.assertEqual(ms["int32"][SCALAR].shape, [0 for _ in range(15)]) + self.assertEqual(ms["int64"][SCALAR].shape, [0 for _ in range(16)]) self.assertEqual( - ms["int16"][SCALAR].shape, - [0 for _ in range(14)] - ) - self.assertEqual( - ms["int32"][SCALAR].shape, - [0 for _ in range(15)] - ) - self.assertEqual( - ms["int64"][SCALAR].shape, - [0 for _ in range(16)] - ) - self.assertEqual( - ms["uint16"][SCALAR].shape, - [0 for _ in range(17)] + ms["uint16"][SCALAR].shape, [0 for _ in range(17)] ) self.assertEqual( - ms["uint32"][SCALAR].shape, - [0 for _ in range(18)] + ms["uint32"][SCALAR].shape, [0 for _ in range(18)] ) self.assertEqual( - ms["uint64"][SCALAR].shape, - [0 for _ in range(19)] + ms["uint64"][SCALAR].shape, [0 for _ in range(19)] ) self.assertEqual( - ms["single"][SCALAR].shape, - [0 for _ in range(20)] + ms["single"][SCALAR].shape, [0 for _ in range(20)] ) self.assertEqual( - ms["np_double"][SCALAR].shape, - [0 for _ in range(21)] + ms["np_double"][SCALAR].shape, [0 for _ in range(21)] ) # test datatypes for fixed-sized types only @@ -972,20 +1106,17 @@ def makeEmptyRoundTrip(self, file_ending): self.assertTrue(ms["uint64"][SCALAR].dtype == np.dtype("uint64")) self.assertTrue(ms["single"][SCALAR].dtype == np.dtype("single")) self.assertTrue( - ms["np_double"][SCALAR].dtype == np.dtype("double")) + ms["np_double"][SCALAR].dtype == np.dtype("double") + ) def testEmptyRecords(self): - backend_filesupport = { - 'json': 'json', - 'hdf5': 'h5', - 'adios2': 'bp' - } + backend_filesupport = {"json": "json", "hdf5": "h5", "adios2": "bp"} for b in io.variants: if io.variants[b] is True and b in backend_filesupport: self.makeEmptyRoundTrip(backend_filesupport[b]) def testData(self): - """ Test IO on data containing particles and meshes.""" + """Test IO on data containing particles and meshes.""" # Get series. series = self.__series @@ -1032,41 +1163,80 @@ def testData(self): assert pos_y.dtype == np.double assert w.dtype == np.double - self.assertSequenceEqual(pos_y.shape, [270625, ]) - self.assertSequenceEqual(w.shape, [270625, ]) + self.assertSequenceEqual( + pos_y.shape, + [ + 270625, + ], + ) + self.assertSequenceEqual( + w.shape, + [ + 270625, + ], + ) if found_numpy: self.assertEqual(pos_y.dtype, np.float64) self.assertEqual(w.dtype, np.float64) - y_data = pos_y.load_chunk([200000, ], [10, ]) - w_data = w.load_chunk([200000, ], [10, ]) + y_data = pos_y.load_chunk( + [ + 200000, + ], + [ + 10, + ], + ) + w_data = w.load_chunk( + [ + 200000, + ], + [ + 10, + ], + ) electrons.series_flush() - self.assertSequenceEqual(y_data.shape, [10, ]) - self.assertSequenceEqual(w_data.shape, [10, ]) + self.assertSequenceEqual( + y_data.shape, + [ + 10, + ], + ) + self.assertSequenceEqual( + w_data.shape, + [ + 10, + ], + ) self.assertEqual(y_data.dtype, np.float64) self.assertEqual(w_data.dtype, np.float64) np.testing.assert_allclose( y_data, - [-9.60001131e-06, -8.80004967e-06, -8.00007455e-06, - -7.20008487e-06, -6.40007232e-06, -5.60002710e-06, - -4.79993871e-06, -3.99980648e-06, -3.19964406e-06, - -2.39947455e-06] - ) - np.testing.assert_allclose( - w_data, - np.ones((10,)) * 1600000. + [ + -9.60001131e-06, + -8.80004967e-06, + -8.00007455e-06, + -7.20008487e-06, + -6.40007232e-06, + -5.60002710e-06, + -4.79993871e-06, + -3.99980648e-06, + -3.19964406e-06, + -2.39947455e-06, + ], ) + np.testing.assert_allclose(w_data, np.ones((10,)) * 1600000.0) E_x = E["x"] shape = E_x.shape if found_numpy: - np.testing.assert_allclose(E.unit_dimension, - [1., 1., -3., -1., 0., 0., 0.]) - np.testing.assert_allclose(E_x.position, - [0.5, 0., 0.]) + np.testing.assert_allclose( + E.unit_dimension, [1.0, 1.0, -3.0, -1.0, 0.0, 0.0, 0.0] + ) + np.testing.assert_allclose(E_x.position, [0.5, 0.0, 0.0]) self.assertAlmostEqual(E_x.unit_SI, 1.0) self.assertSequenceEqual(shape, [26, 26, 201]) @@ -1085,19 +1255,13 @@ def testData(self): np.testing.assert_allclose( chunk_data, [ - [ - [6.26273197e7], - [2.70402498e8] - ], - [ - [-1.89238617e8], - [-1.66413019e8] - ] - ] + [[6.26273197e7], [2.70402498e8]], + [[-1.89238617e8], [-1.66413019e8]], + ], ) def testPickle(self): - """ test pickling of any attributable, especially record components.""" + """test pickling of any attributable, especially record components.""" import pickle # Get series. @@ -1156,7 +1320,8 @@ def testPickle(self): pos_y = pickle.loads(pickled_pos_y) w = pickle.loads(pickled_w) print( - f"This is E_x.position of the unpickled object:\n{E_x.position}\n") + f"This is E_x.position of the unpickled object:\n{E_x.position}\n" + ) self.assertIsInstance(series, io.Series) self.assertIsInstance(i, io.Iteration) @@ -1189,21 +1354,17 @@ def testPickle(self): # get particle data if found_numpy: - np.testing.assert_allclose(E.unit_dimension, - [1., 1., -3., -1., 0., 0., 0.]) - np.testing.assert_allclose(E_x.position, - [0.5, 0., 0.]) + np.testing.assert_allclose( + E.unit_dimension, [1.0, 1.0, -3.0, -1.0, 0.0, 0.0, 0.0] + ) + np.testing.assert_allclose(E_x.position, [0.5, 0.0, 0.0]) # indirectly accessed record component after pickle - np.testing.assert_allclose(data_indir, - data) + np.testing.assert_allclose(data_indir, data) # indirectly accessed record component after pickle - np.testing.assert_allclose(data_pos_y_indir1, - data_pos_y) - np.testing.assert_allclose(data_pos_y_indir2, - data_pos_y) + np.testing.assert_allclose(data_pos_y_indir1, data_pos_y) + np.testing.assert_allclose(data_pos_y_indir2, data_pos_y) # original data access vs. pickled access - np.testing.assert_allclose(data_pos_y_org, - data_pos_y) + np.testing.assert_allclose(data_pos_y_org, data_pos_y) self.assertAlmostEqual(E_x.unit_SI, 1.0) self.assertSequenceEqual(E_x.shape, [26, 26, 201]) @@ -1222,19 +1383,13 @@ def testPickle(self): np.testing.assert_allclose( chunk_data, [ - [ - [6.26273197e7], - [2.70402498e8] - ], - [ - [-1.89238617e8], - [-1.66413019e8] - ] - ] + [[6.26273197e7], [2.70402498e8]], + [[-1.89238617e8], [-1.66413019e8]], + ], ) def testLoadSeries(self): - """ Test loading an openPMD series from hdf5.""" + """Test loading an openPMD series from hdf5.""" # Get series. series = self.__series @@ -1243,7 +1398,7 @@ def testLoadSeries(self): self.assertEqual(series.openPMD, "1.1.0") def testListSeries(self): - """ Test print()-ing and openPMD series from hdf5.""" + """Test print()-ing and openPMD series from hdf5.""" # Get series. series = self.__series @@ -1256,7 +1411,7 @@ def testListSeries(self): print(io.list_series.__doc__) def testSliceRead(self): - """ Testing sliced read on record components. """ + """Testing sliced read on record components.""" # Get series. series = self.__series @@ -1288,16 +1443,32 @@ def testSliceRead(self): if not found_numpy: return - np.testing.assert_allclose(electrons["position"].unit_dimension, - [1., 0., 0., 0., 0., 0., 0.]) + np.testing.assert_allclose( + electrons["position"].unit_dimension, + [1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ) offset = [4, 5, 9] extent = [4, 2, 3] E_x_data = E_x.load_chunk(offset, extent) E_x_data_slice = E_x[4:8, 5:7, 9:12] - y_data = pos_y.load_chunk([200000, ], [10, ]) - w_data = w.load_chunk([200000, ], [10, ]) + y_data = pos_y.load_chunk( + [ + 200000, + ], + [ + 10, + ], + ) + w_data = w.load_chunk( + [ + 200000, + ], + [ + 10, + ], + ) y_data_slice = pos_y[200000:200010] w_data_slice = w[200000:200010] series.flush() @@ -1309,52 +1480,43 @@ def testSliceRead(self): self.assertEqual(y_data.dtype, y_data_slice.dtype) self.assertEqual(w_data.dtype, w_data_slice.dtype) - np.testing.assert_allclose( - E_x_data, - E_x_data_slice - ) - np.testing.assert_allclose( - y_data, - y_data_slice - ) - np.testing.assert_allclose( - w_data, - w_data_slice - ) + np.testing.assert_allclose(E_x_data, E_x_data_slice) + np.testing.assert_allclose(y_data, y_data_slice) + np.testing.assert_allclose(w_data, w_data_slice) # more exotic syntax # https://docs.scipy.org/doc/numpy-1.15.0/reference/arrays.indexing.html # - [x]: [M, L:LARGE, K:LARGE] over-select upper range # (is numpy-allowed: crop to max range) - d1 = pos_y[0:pos_y.shape[0]+10] - d2 = pos_y[0:pos_y.shape[0]] + d1 = pos_y[0 : pos_y.shape[0] + 10] + d2 = pos_y[0 : pos_y.shape[0]] series.flush() np.testing.assert_array_equal(d1.shape, d2.shape) np.testing.assert_allclose(d1, d2) # - [x]: [M, L, -K] negative indexes - d1 = E_x[4:8, 2:3, E_x.shape[2]-5] + d1 = E_x[4:8, 2:3, E_x.shape[2] - 5] d2 = E_x[4:8, 2:3, -4] series.flush() np.testing.assert_allclose(d1, d2) - d1 = E_x[4:8, 2:3, E_x.shape[2]-4:] + d1 = E_x[4:8, 2:3, E_x.shape[2] - 4 :] d2 = E_x[4:8, 2:3, -4:] series.flush() np.testing.assert_allclose(d1, d2) - d1 = E_x[4:8, E_x.shape[1]-3:E_x.shape[1], E_x.shape[2]-4:] + d1 = E_x[4:8, E_x.shape[1] - 3 : E_x.shape[1], E_x.shape[2] - 4 :] d2 = E_x[4:8, -3:, -4:] series.flush() np.testing.assert_allclose(d1, d2) - d1 = pos_y[0:pos_y.shape[0]-1] + d1 = pos_y[0 : pos_y.shape[0] - 1] d2 = pos_y[0:-1] series.flush() np.testing.assert_allclose(d1, d2) - d1 = w[0:w.shape[0]-2] + d1 = w[0 : w.shape[0] - 2] d2 = w[0:-2] series.flush() np.testing.assert_allclose(d1, d2) @@ -1394,17 +1556,17 @@ def testSliceRead(self): np.testing.assert_array_equal(d3.shape, [E_x.shape[0], 1, 1]) d1 = E_x[5, 6, :] - d2 = E_x[5, 6, 0:E_x.shape[2]] + d2 = E_x[5, 6, 0 : E_x.shape[2]] series.flush() np.testing.assert_allclose(d1, d2) d1 = pos_y[:] - d2 = pos_y[0:pos_y.shape[0]] + d2 = pos_y[0 : pos_y.shape[0]] series.flush() np.testing.assert_allclose(d1, d2) d1 = w[:] - d2 = w[0:w.shape[0]] + d2 = w[0 : w.shape[0]] series.flush() np.testing.assert_allclose(d1, d2) @@ -1451,7 +1613,7 @@ def testSliceRead(self): # - [x]: [()] all from all dimensions d1 = pos_y[()] - d2 = pos_y[0:pos_y.shape[0]] + d2 = pos_y[0 : pos_y.shape[0]] series.flush() np.testing.assert_allclose(d1, d2) @@ -1510,23 +1672,23 @@ def testSliceRead(self): d1 = w[w.shape[0]] # cropped to upper range - d1 = E_x[10:E_x.shape[0]+2, 0, 0] - d2 = pos_y[10:pos_y.shape[0]+3] + d1 = E_x[10 : E_x.shape[0] + 2, 0, 0] + d2 = pos_y[10 : pos_y.shape[0] + 3] self.assertEqual(d1.ndim, 1) self.assertEqual(d2.ndim, 1) - self.assertEqual(d1.shape[0], E_x.shape[0]-10) - self.assertEqual(d2.shape[0], pos_y.shape[0]-10) + self.assertEqual(d1.shape[0], E_x.shape[0] - 10) + self.assertEqual(d2.shape[0], pos_y.shape[0] - 10) # meta-data should have been accessible already series.flush() # negative index out-of-range checks with self.assertRaises(IndexError): - d1 = E_x[-E_x.shape[0]-1, 0, 0] + d1 = E_x[-E_x.shape[0] - 1, 0, 0] with self.assertRaises(IndexError): - d1 = E_x[0, -E_x.shape[1]-1, 0] + d1 = E_x[0, -E_x.shape[1] - 1, 0] with self.assertRaises(IndexError): - d1 = E_x[0, 0, -E_x.shape[2]-1] + d1 = E_x[0, 0, -E_x.shape[2] - 1] # - [x] too many indices passed for axes with self.assertRaises(IndexError): @@ -1564,15 +1726,14 @@ def testSliceWrite(self): self.backend_write_slices(ext) def backend_write_slices(self, file_ending): - """ Testing sliced write on record components. """ + """Testing sliced write on record components.""" if not found_numpy: return # get series series = io.Series( - "unittest_py_slice_API." + file_ending, - io.Access.create + "unittest_py_slice_API." + file_ending, io.Access.create ) i = series.iterations[0] @@ -1595,7 +1756,7 @@ def backend_write_slices(self, file_ending): # get a mesh record component rho = i.meshes["rho"][io.Record_Component.SCALAR] - rho.position = [0., 0.] # Yee staggered + rho.position = [0.0, 0.0] # Yee staggered rho.reset_dataset(io.Dataset(data.dtype, data.shape)) @@ -1635,7 +1796,7 @@ def backend_write_slices(self, file_ending): series.flush() def testIterations(self): - """ Test querying a series' iterations and loop over them. """ + """Test querying a series' iterations and loop over them.""" # Get series. series = self.__series @@ -1653,7 +1814,7 @@ def testIterations(self): self.assertIsInstance(i, io.Iteration) def testMeshes(self): - """ Test querying a mesh. """ + """Test querying a mesh.""" # Get series. series = self.__series @@ -1668,7 +1829,7 @@ def testMeshes(self): self.assertIsInstance(i.meshes[m], io.Mesh) def testParticles(self): - """ Test querying a particle species. """ + """Test querying a particle species.""" # Get series. series = self.__series @@ -1682,17 +1843,24 @@ def testParticles(self): self.assertIsInstance(i.particles[ps], io.ParticleSpecies) def testDatatype(self): - """ Test Datatype. """ + """Test Datatype.""" data_type = io.Datatype(1) del data_type def testDataset(self): - """ Test Dataset. """ + """Test Dataset.""" data_type = io.Datatype.LONG extent = [1, 1, 1] obj = io.Dataset(data_type, extent) if found_numpy: - d = np.array((1, 1, 1, ), dtype="l") + d = np.array( + ( + 1, + 1, + 1, + ), + dtype="l", + ) obj2 = io.Dataset(d.dtype, d.shape) assert data_type == io.determine_datatype(d.dtype) assert obj2.dtype == obj.dtype @@ -1700,12 +1868,12 @@ def testDataset(self): del obj def testGeometry(self): - """ Test Geometry. """ + """Test Geometry.""" obj = io.Geometry(0) del obj def testIteration(self): - """ Test Iteration. """ + """Test Iteration.""" self.assertRaises(TypeError, io.Iteration) iteration = self.__particle_series.iterations[400] @@ -1720,21 +1888,21 @@ def testIteration(self): # TODO verify change is reflected in original iteration object def testIteration_Encoding(self): - """ Test Iteration_Encoding. """ + """Test Iteration_Encoding.""" obj = io.Iteration_Encoding(1) del obj def testMesh(self): - """ Test Mesh. """ + """Test Mesh.""" self.assertRaises(TypeError, io.Mesh) - mesh = self.__series.iterations[100].meshes['E'] + mesh = self.__series.iterations[100].meshes["E"] copy_mesh = io.Mesh(mesh) - self.assertEqual(mesh.data_order, 'C') + self.assertEqual(mesh.data_order, "C") self.assertIsInstance(copy_mesh, io.Mesh) def testMesh_Container(self): - """ Test Mesh_Container. """ + """Test Mesh_Container.""" self.assertRaises(TypeError, io.Mesh_Container) def backend_particle_patches(self, file_ending): @@ -1743,12 +1911,13 @@ def backend_particle_patches(self, file_ending): DS = io.Dataset SCALAR = io.Record_Component.SCALAR - extent = [123, ] + extent = [ + 123, + ] num_patches = 2 series = io.Series( - "unittest_py_particle_patches." + file_ending, - io.Access.create + "unittest_py_particle_patches." + file_ending, io.Access.create ) e = series.iterations[42].particles["electrons"] @@ -1759,13 +1928,29 @@ def backend_particle_patches(self, file_ending): x.store_chunk(np.arange(extent[0], dtype=np.single)) o = e["positionOffset"][r] o.reset_dataset(DS(np.dtype("uint64"), extent)) - o.store_chunk(np.arange(extent[0], dtype=np.uint64), [0, ], extent) + o.store_chunk( + np.arange(extent[0], dtype=np.uint64), + [ + 0, + ], + extent, + ) - dset = DS(np.dtype("uint64"), [num_patches, ]) + dset = DS( + np.dtype("uint64"), + [ + num_patches, + ], + ) e.particle_patches["numParticles"][SCALAR].reset_dataset(dset) e.particle_patches["numParticlesOffset"][SCALAR].reset_dataset(dset) - dset = DS(np.dtype("single"), [num_patches, ]) + dset = DS( + np.dtype("single"), + [ + num_patches, + ], + ) e.particle_patches["offset"]["x"].reset_dataset(dset) e.particle_patches["offset"]["y"].reset_dataset(dset) e.particle_patches["extent"]["x"].reset_dataset(dset) @@ -1774,19 +1959,19 @@ def backend_particle_patches(self, file_ending): # patch 0 (decomposed in x) e.particle_patches["numParticles"][SCALAR].store(0, np.uint64(10)) e.particle_patches["numParticlesOffset"][SCALAR].store(0, np.uint64(0)) - e.particle_patches["offset"]["x"].store(0, np.single(0.)) - e.particle_patches["offset"]["y"].store(0, np.single(0.)) - e.particle_patches["extent"]["x"].store(0, np.single(10.)) - e.particle_patches["extent"]["y"].store(0, np.single(123.)) + e.particle_patches["offset"]["x"].store(0, np.single(0.0)) + e.particle_patches["offset"]["y"].store(0, np.single(0.0)) + e.particle_patches["extent"]["x"].store(0, np.single(10.0)) + e.particle_patches["extent"]["y"].store(0, np.single(123.0)) # patch 1 (decomposed in x) - e.particle_patches["numParticles"][SCALAR].store( - 1, np.uint64(113)) + e.particle_patches["numParticles"][SCALAR].store(1, np.uint64(113)) e.particle_patches["numParticlesOffset"][SCALAR].store( - 1, np.uint64(10)) - e.particle_patches["offset"]["x"].store(1, np.single(10.)) - e.particle_patches["offset"]["y"].store(1, np.single(0.)) - e.particle_patches["extent"]["x"].store(1, np.single(113.)) - e.particle_patches["extent"]["y"].store(1, np.single(123.)) + 1, np.uint64(10) + ) + e.particle_patches["offset"]["x"].store(1, np.single(10.0)) + e.particle_patches["offset"]["y"].store(1, np.single(0.0)) + e.particle_patches["extent"]["x"].store(1, np.single(113.0)) + e.particle_patches["extent"]["y"].store(1, np.single(123.0)) # read back self.assertTrue(series) @@ -1794,14 +1979,14 @@ def backend_particle_patches(self, file_ending): self.assertFalse(series) series = io.Series( - "unittest_py_particle_patches." + file_ending, - io.Access.read_only + "unittest_py_particle_patches." + file_ending, io.Access.read_only ) e = series.iterations[42].particles["electrons"] numParticles = e.particle_patches["numParticles"][SCALAR].load() - numParticlesOffset = e.particle_patches["numParticlesOffset"][SCALAR].\ - load() + numParticlesOffset = e.particle_patches["numParticlesOffset"][ + SCALAR + ].load() extent_x = e.particle_patches["extent"]["x"].load() extent_y = e.particle_patches["extent"]["y"].load() offset_x = e.particle_patches["offset"]["x"].load() @@ -1810,17 +1995,15 @@ def backend_particle_patches(self, file_ending): series.flush() np.testing.assert_almost_equal( - numParticles, np.array([10, 113], np.uint64)) - np.testing.assert_almost_equal( - numParticlesOffset, np.array([0, 10], np.uint64)) - np.testing.assert_almost_equal( - extent_x, [10., 113.]) - np.testing.assert_almost_equal( - extent_y, [123., 123.]) - np.testing.assert_almost_equal( - offset_x, [0., 10.]) + numParticles, np.array([10, 113], np.uint64) + ) np.testing.assert_almost_equal( - offset_y, [0., 0.]) + numParticlesOffset, np.array([0, 10], np.uint64) + ) + np.testing.assert_almost_equal(extent_x, [10.0, 113.0]) + np.testing.assert_almost_equal(extent_y, [123.0, 123.0]) + np.testing.assert_almost_equal(offset_x, [0.0, 10.0]) + np.testing.assert_almost_equal(offset_y, [0.0, 0.0]) def testParticlePatches(self): self.assertRaises(TypeError, io.Particle_Patches) @@ -1829,23 +2012,23 @@ def testParticlePatches(self): self.backend_particle_patches(ext) def testParticleSpecies(self): - """ Test ParticleSpecies. """ + """Test ParticleSpecies.""" self.assertRaises(TypeError, io.ParticleSpecies) def testParticle_Container(self): - """ Test Particle_Container. """ + """Test Particle_Container.""" self.assertRaises(TypeError, io.Particle_Container) def testRecord(self): - """ Test Record. """ + """Test Record.""" # Has only copy constructor. self.assertRaises(TypeError, io.Record) # Get a record. - electrons = self.__series.iterations[400].particles['electrons'] - position = electrons['position'] + electrons = self.__series.iterations[400].particles["electrons"] + position = electrons["position"] self.assertIsInstance(position, io.Record) - x = position['x'] + x = position["x"] self.assertIsInstance(x, io.Record_Component) # Copy. @@ -1858,11 +2041,11 @@ def testRecord(self): # io.Record_Component) def testRecord_Component(self): - """ Test Record_Component. """ + """Test Record_Component.""" self.assertRaises(TypeError, io.Record_Component) def testFieldRecord(self): - """ Test querying for a non-scalar field record. """ + """Test querying for a non-scalar field record.""" E = self.__series.iterations[100].meshes["E"] Ex = E["x"] @@ -1873,7 +2056,7 @@ def makeCloseIterationRoundTrip(self, file_ending): # write series = io.Series( "../samples/unittest_closeIteration_%T." + file_ending, - io.Access_Type.create + io.Access_Type.create, ) DS = io.Dataset data = np.array([2, 4, 6, 8], dtype=np.dtype("int")) @@ -1887,7 +2070,7 @@ def makeCloseIterationRoundTrip(self, file_ending): read = io.Series( "../samples/unittest_closeIteration_%T." + file_ending, - io.Access_Type.read_only + io.Access_Type.read_only, ) it0 = read.iterations[0] E_x = it0.meshes["E"]["x"] @@ -1909,7 +2092,7 @@ def makeCloseIterationRoundTrip(self, file_ending): read = io.Series( "../samples/unittest_closeIteration_%T." + file_ending, - io.Access_Type.read_only + io.Access_Type.read_only, ) it1 = read.iterations[1] E_x = it1.meshes["E"]["x"] @@ -1931,17 +2114,12 @@ def makeIteratorRoundTrip(self, backend, file_ending): # write jsonConfig = { "defer_iteration_parsing": True, - "adios2": { - "engine": { - "type": "bp4", - "usesteps": True - } - } + "adios2": {"engine": {"type": "bp4", "usesteps": True}}, } series = io.Series( "../samples/unittest_serialIterator." + file_ending, io.Access_Type.create, - jsonConfig + jsonConfig, ) DS = io.Dataset data = np.array([2, 4, 6, 8], dtype=np.dtype("int")) @@ -1972,7 +2150,7 @@ def makeIteratorRoundTrip(self, backend, file_ending): read = io.Series( "../samples/unittest_serialIterator." + file_ending, io.Access_Type.read_only, - jsonConfig + jsonConfig, ) for it in read.read_iterations(): lastIterationIndex = it.iteration_index @@ -1994,11 +2172,7 @@ def makeIteratorRoundTrip(self, backend, file_ending): self.assertEqual(lastIterationIndex, 9) def testIterator(self): - backend_filesupport = { - 'json': 'json', - 'hdf5': 'h5', - 'adios2': 'bp' - } + backend_filesupport = {"json": "json", "hdf5": "h5", "adios2": "bp"} for b in io.variants: if io.variants[b] is True and b in backend_filesupport: self.makeIteratorRoundTrip(b, backend_filesupport[b]) @@ -2007,16 +2181,14 @@ def makeAvailableChunksRoundTrip(self, ext): if ext == "h5" or ext == "toml": return name = "../samples/available_chunks_python." + ext - write = io.Series( - name, - io.Access_Type.create - ) + write = io.Series(name, io.Access_Type.create) DS = io.Dataset E_x = write.iterations[0].meshes["E"]["x"] E_x.reset_dataset(DS(np.dtype("int"), [10, 4])) data = np.array( - [[2, 4, 6, 8], [10, 12, 14, 16]], dtype=np.dtype("int")) + [[2, 4, 6, 8], [10, 12, 14, 16]], dtype=np.dtype("int") + ) E_x.store_chunk(data, [1, 0], [2, 4]) data2 = np.array([[2, 4], [6, 8], [10, 12]], dtype=np.dtype("int")) E_x.store_chunk(data2, [4, 2], [3, 2]) @@ -2031,7 +2203,7 @@ def makeAvailableChunksRoundTrip(self, ext): read = io.Series( name, io.Access_Type.read_only, - options={"defer_iteration_parsing": True} + options={"defer_iteration_parsing": True}, ) read.iterations[0].open() @@ -2056,15 +2228,15 @@ def testAvailableChunks(self): def writeFromTemporaryStore(self, E_x): if found_numpy: - E_x.store_chunk(np.array([[4, 5, 6]], dtype=np.dtype("int")), - [1, 0]) + E_x.store_chunk( + np.array([[4, 5, 6]], dtype=np.dtype("int")), [1, 0] + ) data = np.array([[1, 2, 3]], dtype=np.dtype("int")) E_x.store_chunk(data) data2 = np.array([[7, 8, 9]], dtype=np.dtype("int")) - E_x.store_chunk(np.repeat(data2, 198, axis=0), - [2, 0]) + E_x.store_chunk(np.repeat(data2, 198, axis=0), [2, 0]) def loadToTemporaryStore(self, r_E_x): # not catching the return value shall not result in a use-after-free: @@ -2076,10 +2248,7 @@ def loadToTemporaryStore(self, r_E_x): def writeFromTemporary(self, ext): name = "../samples/write_from_temporary_python." + ext - write = io.Series( - name, - io.Access_Type.create - ) + write = io.Series(name, io.Access_Type.create) DS = io.Dataset E_x = write.iterations[0].meshes["E"]["x"] @@ -2091,13 +2260,10 @@ def writeFromTemporary(self, ext): write.close() self.assertFalse(write) - read = io.Series( - name, - io.Access_Type.read_only - ) + read = io.Series(name, io.Access_Type.read_only) r_E_x = read.iterations[0].meshes["E"]["x"] - if read.backend == 'ADIOS2': + if read.backend == "ADIOS2": self.assertEqual(len(r_E_x.available_chunks()), 3) else: self.assertEqual(len(r_E_x.available_chunks()), 1) @@ -2110,8 +2276,9 @@ def writeFromTemporary(self, ext): if found_numpy: np.testing.assert_allclose( r_d[:3, :], - np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], - dtype=np.dtype("int")) + np.array( + [[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.dtype("int") + ), ) def testWriteFromTemporary(self): @@ -2126,8 +2293,8 @@ def testJsonConfigADIOS2(self): "unused": "parameter", "parameters": { "BufferGrowthFactor": "2.0", - "Profile": "On" - } + "Profile": "On", + }, }, "unused": "as well", "dataset": { @@ -2136,11 +2303,11 @@ def testJsonConfigADIOS2(self): "type": "blosc", "parameters": { "clevel": "1", - "doshuffle": "BLOSC_BITSHUFFLE" - } + "doshuffle": "BLOSC_BITSHUFFLE", + }, } ] - } + }, } } @@ -2154,20 +2321,21 @@ def testJsonConfigADIOS2(self): "type": "blosc", "parameters": { "clevel": "3", - "doshuffle": "BLOSC_BITSHUFFLE" - } + "doshuffle": "BLOSC_BITSHUFFLE", + }, } - ] - } + ], + }, } } - if not io.variants['adios2']: + if not io.variants["adios2"]: return series = io.Series( "../samples/unittest_jsonConfiguredBP3.bp", io.Access_Type.create, - global_config) + global_config, + ) DS = io.Dataset data = np.array(range(1000), dtype=np.dtype("double")) @@ -2187,7 +2355,8 @@ def testJsonConfigADIOS2(self): read = io.Series( "../samples/unittest_jsonConfiguredBP3.bp", io.Access_Type.read_only, - global_config) + global_config, + ) E_x = read.iterations[0].meshes["E"]["x"] chunk_x = E_x.load_chunk([0], [1000]) @@ -2201,7 +2370,7 @@ def testJsonConfigADIOS2(self): self.assertEqual(chunk_y[i], i) def testError(self): - if 'test_throw' in io.__dict__: + if "test_throw" in io.__dict__: with self.assertRaises(io.ErrorOperationUnsupportedInBackend): io.test_throw("test description") with self.assertRaises(io.Error): @@ -2212,8 +2381,9 @@ def testCustomGeometries(self): DT = io.Datatype sample_data = np.ones([10], dtype=np.int_) - write = io.Series("../samples/custom_geometries_python.json", - io.Access.create) + write = io.Series( + "../samples/custom_geometries_python.json", io.Access.create + ) E = write.iterations[0].meshes["E"] E.set_attribute("geometry", "other:customGeometry") E_x = E["x"] @@ -2242,8 +2412,9 @@ def testCustomGeometries(self): write.close() self.assertFalse(write) - read = io.Series("../samples/custom_geometries_python.json", - io.Access.read_only) + read = io.Series( + "../samples/custom_geometries_python.json", io.Access.read_only + ) E = read.iterations[0].meshes["E"] self.assertEqual(E.get_attribute("geometry"), "other:customGeometry") @@ -2256,11 +2427,13 @@ def testCustomGeometries(self): self.assertEqual(B.geometry_string, "other:customGeometry") e_energyDensity = read.iterations[0].meshes["e_energyDensity"] - self.assertEqual(e_energyDensity.get_attribute("geometry"), - "other:customGeometry") + self.assertEqual( + e_energyDensity.get_attribute("geometry"), "other:customGeometry" + ) self.assertEqual(e_energyDensity.geometry, io.Geometry.other) - self.assertEqual(e_energyDensity.geometry_string, - "other:customGeometry") + self.assertEqual( + e_energyDensity.geometry_string, "other:customGeometry" + ) e_chargeDensity = read.iterations[0].meshes["e_chargeDensity"] self.assertEqual(e_chargeDensity.get_attribute("geometry"), "other") @@ -2274,7 +2447,7 @@ def testSeriesConstructors(self): cfg = {"iteration_encoding": "variable_based"} cfg_as_string = json.dumps(cfg) cfg_as_file = "../samples/cfg.json" - with open(cfg_as_file, 'w') as f: + with open(cfg_as_file, "w") as f: json.dump(cfg, f) cfg_as_filepath = Path(cfg_as_file) @@ -2316,8 +2489,9 @@ def testScalarHdf5Fields(self): # Now turn E_x into a scalar with h5py.File(file, "r+") as f: E = f["data"]["0"]["meshes"]["E"] - reapply_attributes = \ - {key: val for key, val in E["x"].attrs.items()} + reapply_attributes = { + key: val for key, val in E["x"].attrs.items() + } del E["x"] E["x"] = 44 for key, val in reapply_attributes.items(): @@ -2335,12 +2509,13 @@ def testScalarHdf5Fields(self): series_read_write.close() series_read_again = io.Series(file, io.Access.read_only) - loaded_from_scalar = \ - series_read_again.iterations[0].meshes["E"]["x"][:] + loaded_from_scalar = series_read_again.iterations[0].meshes["E"]["x"][ + : + ] series_read_again.flush() self.assertEqual(loaded_from_scalar, np.array([45])) series_read_again.close() -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/test/python/unittest/Test.py b/test/python/unittest/Test.py index b194f7d77e..ad66fc10de 100644 --- a/test/python/unittest/Test.py +++ b/test/python/unittest/Test.py @@ -18,6 +18,7 @@ def suite(): test_loader = unittest.TestLoader() env_name = "OPENPMD_PYTHON_TEST_PREFIX" from os import environ + if env_name in environ: test_loader.testMethodPrefix = environ[env_name] suites = [ @@ -30,11 +31,10 @@ def suite(): # Run the top level suite and return a success status code. # This enables running an automated git-bisect. if __name__ == "__main__": - result = unittest.TextTestRunner(verbosity=2).run(suite()) if result.wasSuccessful(): - print('---> OK <---') + print("---> OK <---") sys.exit(0) sys.exit(1) diff --git a/test/python/unittest/TestUtilities/TestUtilities.py b/test/python/unittest/TestUtilities/TestUtilities.py index e03e5f6492..ffa4c63092 100644 --- a/test/python/unittest/TestUtilities/TestUtilities.py +++ b/test/python/unittest/TestUtilities/TestUtilities.py @@ -17,7 +17,7 @@ def generateTestFilePath(file_name): @return : The absolute path to ../TestFiles/ . """ - test_files_dir = path.join('..', 'samples', file_name) + test_files_dir = path.join("..", "samples", file_name) return test_files_dir # this_path = path.abspath(path.dirname(__file__))