From 412069ecb126597ba4c5c50d68284699409cbbbd Mon Sep 17 00:00:00 2001 From: Roy Stegeman Date: Fri, 15 Nov 2024 16:52:43 +0000 Subject: [PATCH 1/4] address deprecation of ruamel.yaml functions --- conda-recipe/meta.yaml | 2 +- pyproject.toml | 2 +- src/reportengine/compat.py | 13 ------------- src/reportengine/configparser.py | 9 +++++---- src/reportengine/report.py | 14 +++++--------- src/reportengine/templateparser.py | 8 +++++--- src/reportengine/tests/test_app.py | 7 +++++-- src/reportengine/tests/test_configparser.py | 7 +++++-- 8 files changed, 27 insertions(+), 35 deletions(-) delete mode 100644 src/reportengine/compat.py diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index a663664..bb6b424 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -17,7 +17,7 @@ requirements: run: - python - jinja2 - - ruamel_yaml =0.15 + - ruamel.yaml >=0.18 - matplotlib - pandas >=1 - pygments diff --git a/pyproject.toml b/pyproject.toml index c5574d9..7957ccc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ classifiers = [ description-file="README.md" requires = [ "jinja2", - "ruamel_yaml<0.18", # the code is not compatible with ruamel 0.18 + "ruamel.yaml>=0.18", "matplotlib", "pandas", "pygments", diff --git a/src/reportengine/compat.py b/src/reportengine/compat.py deleted file mode 100644 index 55eef0e..0000000 --- a/src/reportengine/compat.py +++ /dev/null @@ -1,13 +0,0 @@ -""" -compat.py - -Fix library compatibility issues. -""" - -#To learn about the mess with yaml see this: -#https://bihttps://bitbucket.org/ruamel/yaml/issues/28/consider-making-ruamelyaml-available-as-atbucket.org/ruamel/yaml/issues/28/consider-making-ruamelyaml-available-as-a -try: - import ruamel_yaml as yaml -except ImportError: - import ruamel.yaml as yaml - diff --git a/src/reportengine/configparser.py b/src/reportengine/configparser.py index 6e3bddd..accf9c9 100644 --- a/src/reportengine/configparser.py +++ b/src/reportengine/configparser.py @@ -13,14 +13,15 @@ import json from copy import copy +import ruamel.yaml -from reportengine.compat import yaml from reportengine import namespaces from reportengine.utils import ChainMap, get_classmembers from reportengine import templateparser from reportengine.baseexceptions import ErrorWithAlternatives, AsInputError log = logging.getLogger(__name__) +yaml=ruamel.yaml.YAML(typ='safe') _config_token = 'parse_' _produce_token = 'produce_' @@ -800,10 +801,10 @@ def __contains__(self, item): @classmethod def from_yaml(cls, o, *args, **kwargs): try: - return cls(yaml.round_trip_load(o), *args, **kwargs) - except yaml.error.YAMLError as e: + return cls(yaml.load(o), *args, **kwargs) + except ruamel.yaml.YAMLError as e: raise ConfigError(f"Failed to parse yaml file: {e}") def dump_lockfile(self): with open(self.environment.input_folder/"lockfile.yaml", "w+") as f: - yaml.dump(self.lockfile, stream=f, Dumper=yaml.RoundTripDumper) + yaml.dump(self.lockfile, stream=f) diff --git a/src/reportengine/report.py b/src/reportengine/report.py index 956c8d7..eb0043e 100644 --- a/src/reportengine/report.py +++ b/src/reportengine/report.py @@ -39,9 +39,9 @@ from collections import UserList import pathlib +import dask.distributed import jinja2 -from reportengine.compat import yaml - +import ruamel.yaml from . import configparser from . resourcebuilder import target_map @@ -53,9 +53,10 @@ from . import filefinder from . import floatformatting -import dask.distributed log = logging.getLogger(__name__) +yaml=ruamel.yaml.YAML(typ='safe') + __all__ = ('report', 'Config') @@ -213,12 +214,7 @@ def meta_file(output_path, meta:(dict, type(None))=None): path = output_path/fname with open(path, 'w') as f: f.write('\n') - #Using round_trip_dump is important here because the input dict may - #be a recursive commented map, which yaml.dump (or safe_dumo) doesn't - #know how to - #process correctly. - yaml.round_trip_dump(meta, f, explicit_start=True, explicit_end=True, - default_flow_style=False) + yaml.dump(meta, f) return fname def pandoc_template(*, templatename='report.template', output_path): diff --git a/src/reportengine/templateparser.py b/src/reportengine/templateparser.py index 68fb5a1..fd27daf 100644 --- a/src/reportengine/templateparser.py +++ b/src/reportengine/templateparser.py @@ -9,10 +9,12 @@ from collections import namedtuple import logging -from reportengine.compat import yaml +import ruamel.yaml + from reportengine.targets import FuzzyTarget log = logging.getLogger(__name__) +yaml=ruamel.yaml.YAML(typ='safe') #TODO: Do a real tokenizer/lexer/parser? Would avoid having r'\s*?' @@ -43,8 +45,8 @@ def parse_assignments(args): k = m.group(1) vstring = m.group(2) try: - v = yaml.safe_load(vstring) - except yaml.YamlError: + v = yaml.load(vstring) + except ruamel.yaml.YAMLError: raise ValueError(f"Couldn't process assignment value '{vstring}'") res.append((k, v)) else: diff --git a/src/reportengine/tests/test_app.py b/src/reportengine/tests/test_app.py index e518a53..42ea2f9 100644 --- a/src/reportengine/tests/test_app.py +++ b/src/reportengine/tests/test_app.py @@ -5,10 +5,13 @@ @author: Zahari Kassabov """ import pytest +import ruamel.yaml from reportengine import app from reportengine.tests.utils import tmp -from reportengine.compat import yaml + +yaml=ruamel.yaml.YAML(typ='safe') + runcard =\ """ @@ -58,7 +61,7 @@ def test_app_runs(tmp): #Test meta round trip with open(output_path/'meta.yaml') as f: - meta = yaml.safe_load(f) + meta = yaml.load(f) assert meta['author'] == "Zahari Kassabov" assert meta['keywords'] == ["test", "debug"] diff --git a/src/reportengine/tests/test_configparser.py b/src/reportengine/tests/test_configparser.py index 7071074..8693884 100644 --- a/src/reportengine/tests/test_configparser.py +++ b/src/reportengine/tests/test_configparser.py @@ -8,12 +8,15 @@ from collections import OrderedDict import unittest -from reportengine.compat import yaml +import ruamel.yaml + from reportengine.utils import ChainMap from reportengine import namespaces from reportengine.configparser import (Config, BadInputType, element_of, named_element_of, ConfigError) +yaml=ruamel.yaml.YAML(typ='safe') + class BaseConfig(Config): @element_of('ys') @@ -193,7 +196,7 @@ def test_rewrite_actions(): c = BaseConfig(inp) r = c.parse_actions_(inp['actions_']) suggested_yaml = c._rewrite_old_actions(r) - newacts = yaml.safe_load(suggested_yaml) + newacts = yaml.load(suggested_yaml) newr = c.parse_actions_(newacts['actions_']) assert newr == r From 3c9cd0a8c26cebdc8308f17fe9a27a5b6fc68a5f Mon Sep 17 00:00:00 2001 From: Roy Stegeman Date: Fri, 15 Nov 2024 16:55:29 +0000 Subject: [PATCH 2/4] reduce ruamel yaml pin from 0.18 to 0.15 --- conda-recipe/meta.yaml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index bb6b424..5966a22 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -17,7 +17,7 @@ requirements: run: - python - jinja2 - - ruamel.yaml >=0.18 + - ruamel.yaml >=0.15 - matplotlib - pandas >=1 - pygments diff --git a/pyproject.toml b/pyproject.toml index 7957ccc..5436703 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ classifiers = [ description-file="README.md" requires = [ "jinja2", - "ruamel.yaml>=0.18", + "ruamel.yaml>=0.15", "matplotlib", "pandas", "pygments", From 5660c29286f2bb353794b6754d8ebdbdf90c5f63 Mon Sep 17 00:00:00 2001 From: Roy Stegeman Date: Wed, 4 Dec 2024 13:55:11 +0000 Subject: [PATCH 3/4] update ruamel parsers to be same as with old version of ruamel --- src/reportengine/configparser.py | 11 +++++------ src/reportengine/report.py | 13 ++++++++----- src/reportengine/templateparser.py | 8 ++++---- src/reportengine/tests/test_app.py | 6 ++---- src/reportengine/tests/test_configparser.py | 7 ++----- src/reportengine/utils.py | 4 ++++ 6 files changed, 25 insertions(+), 24 deletions(-) diff --git a/src/reportengine/configparser.py b/src/reportengine/configparser.py index accf9c9..bce1482 100644 --- a/src/reportengine/configparser.py +++ b/src/reportengine/configparser.py @@ -13,15 +13,14 @@ import json from copy import copy -import ruamel.yaml +from ruamel.yaml import YAMLError from reportengine import namespaces -from reportengine.utils import ChainMap, get_classmembers +from reportengine.utils import ChainMap, get_classmembers, yaml_rt from reportengine import templateparser from reportengine.baseexceptions import ErrorWithAlternatives, AsInputError log = logging.getLogger(__name__) -yaml=ruamel.yaml.YAML(typ='safe') _config_token = 'parse_' _produce_token = 'produce_' @@ -801,10 +800,10 @@ def __contains__(self, item): @classmethod def from_yaml(cls, o, *args, **kwargs): try: - return cls(yaml.load(o), *args, **kwargs) - except ruamel.yaml.YAMLError as e: + return cls(yaml_rt.load(o), *args, **kwargs) + except YAMLError as e: raise ConfigError(f"Failed to parse yaml file: {e}") def dump_lockfile(self): with open(self.environment.input_folder/"lockfile.yaml", "w+") as f: - yaml.dump(self.lockfile, stream=f) + yaml_rt.dump(self.lockfile, stream=f) diff --git a/src/reportengine/report.py b/src/reportengine/report.py index eb0043e..8f8c317 100644 --- a/src/reportengine/report.py +++ b/src/reportengine/report.py @@ -31,7 +31,6 @@ """ from __future__ import generator_stop -import os import os.path as osp import logging import subprocess @@ -41,7 +40,6 @@ import dask.distributed import jinja2 -import ruamel.yaml from . import configparser from . resourcebuilder import target_map @@ -52,10 +50,9 @@ from . import styles from . import filefinder from . import floatformatting - +from . utils import yaml_rt log = logging.getLogger(__name__) -yaml=ruamel.yaml.YAML(typ='safe') __all__ = ('report', 'Config') @@ -214,7 +211,13 @@ def meta_file(output_path, meta:(dict, type(None))=None): path = output_path/fname with open(path, 'w') as f: f.write('\n') - yaml.dump(meta, f) + #Using round_trip_dump is important here because the input dict may be a + #recursive commented map, which yaml.dump (or safe_dumo) doesn't know + #how to process correctly. + yaml_rt.explicit_start=True + yaml_rt.explicit_end=True + yaml_rt.default_flow_style=False + yaml_rt.dump(meta, f) return fname def pandoc_template(*, templatename='report.template', output_path): diff --git a/src/reportengine/templateparser.py b/src/reportengine/templateparser.py index fd27daf..27ba91d 100644 --- a/src/reportengine/templateparser.py +++ b/src/reportengine/templateparser.py @@ -9,12 +9,12 @@ from collections import namedtuple import logging -import ruamel.yaml +from ruamel.yaml import YAMLError +from reportengine.utils import yaml_safe from reportengine.targets import FuzzyTarget log = logging.getLogger(__name__) -yaml=ruamel.yaml.YAML(typ='safe') #TODO: Do a real tokenizer/lexer/parser? Would avoid having r'\s*?' @@ -45,8 +45,8 @@ def parse_assignments(args): k = m.group(1) vstring = m.group(2) try: - v = yaml.load(vstring) - except ruamel.yaml.YAMLError: + v = yaml_safe.load(vstring) + except YAMLError: raise ValueError(f"Couldn't process assignment value '{vstring}'") res.append((k, v)) else: diff --git a/src/reportengine/tests/test_app.py b/src/reportengine/tests/test_app.py index 42ea2f9..f825d9b 100644 --- a/src/reportengine/tests/test_app.py +++ b/src/reportengine/tests/test_app.py @@ -8,9 +8,7 @@ import ruamel.yaml from reportengine import app -from reportengine.tests.utils import tmp - -yaml=ruamel.yaml.YAML(typ='safe') +from reportengine.utils import yaml_safe runcard =\ @@ -61,7 +59,7 @@ def test_app_runs(tmp): #Test meta round trip with open(output_path/'meta.yaml') as f: - meta = yaml.load(f) + meta = yaml_safe.load(f) assert meta['author'] == "Zahari Kassabov" assert meta['keywords'] == ["test", "debug"] diff --git a/src/reportengine/tests/test_configparser.py b/src/reportengine/tests/test_configparser.py index 8693884..2986990 100644 --- a/src/reportengine/tests/test_configparser.py +++ b/src/reportengine/tests/test_configparser.py @@ -8,14 +8,11 @@ from collections import OrderedDict import unittest -import ruamel.yaml - -from reportengine.utils import ChainMap +from reportengine.utils import ChainMap, yaml_safe from reportengine import namespaces from reportengine.configparser import (Config, BadInputType, element_of, named_element_of, ConfigError) -yaml=ruamel.yaml.YAML(typ='safe') class BaseConfig(Config): @@ -196,7 +193,7 @@ def test_rewrite_actions(): c = BaseConfig(inp) r = c.parse_actions_(inp['actions_']) suggested_yaml = c._rewrite_old_actions(r) - newacts = yaml.load(suggested_yaml) + newacts = yaml_safe.load(suggested_yaml) newr = c.parse_actions_(newacts['actions_']) assert newr == r diff --git a/src/reportengine/utils.py b/src/reportengine/utils.py index fa9bae8..a2d5e5a 100644 --- a/src/reportengine/utils.py +++ b/src/reportengine/utils.py @@ -9,6 +9,10 @@ import re import importlib.util import pathlib +from ruamel.yaml import YAML + +yaml_rt = YAML(typ="rt") +yaml_safe = YAML(typ="safe") #TODO: Support metaclass attributes? def get_classmembers(cls, *, predicate=None): From 64d779143056fac6aa8c7b54472b10688a51d948 Mon Sep 17 00:00:00 2001 From: Roy Stegeman Date: Wed, 4 Dec 2024 14:15:50 +0000 Subject: [PATCH 4/4] fix test --- src/reportengine/report.py | 6 +++--- src/reportengine/tests/test_app.py | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/reportengine/report.py b/src/reportengine/report.py index 8f8c317..9ca5eb9 100644 --- a/src/reportengine/report.py +++ b/src/reportengine/report.py @@ -31,7 +31,7 @@ """ from __future__ import generator_stop -import os.path as osp +import os import logging import subprocess import shutil @@ -61,7 +61,7 @@ def _process_template_text(source, *, filename=None): if filename: #PY36 - log.debug("Processing template %s" % osp.abspath(str(filename))) + log.debug("Processing template %s" % os.path.abspath(str(filename))) root = {} d = root @@ -109,7 +109,7 @@ class JinjaEnv(jinja2.Environment): def preprocess(self, source, name=None, filename=None): if filename: - log.debug("Processing template %s" % osp.abspath(filename)) + log.debug("Processing template %s" % os.path.abspath(filename)) root = {} d = root diff --git a/src/reportengine/tests/test_app.py b/src/reportengine/tests/test_app.py index f825d9b..602d160 100644 --- a/src/reportengine/tests/test_app.py +++ b/src/reportengine/tests/test_app.py @@ -5,11 +5,10 @@ @author: Zahari Kassabov """ import pytest -import ruamel.yaml from reportengine import app from reportengine.utils import yaml_safe - +from reportengine.tests.utils import tmp runcard =\ """