From 5d8e22c44d91b51ac51e91587f3ee973bf881970 Mon Sep 17 00:00:00 2001 From: Dan Rose Date: Wed, 22 Apr 2020 17:52:29 -0500 Subject: [PATCH 1/3] Use pickle instead of repr/ast.literal_eval This fixes a crash with objects like`SpecifierSet` --- .../package_identification/python_setup_py.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/colcon_python_setup_py/package_identification/python_setup_py.py b/colcon_python_setup_py/package_identification/python_setup_py.py index f37b63a..6ec5e88 100644 --- a/colcon_python_setup_py/package_identification/python_setup_py.py +++ b/colcon_python_setup_py/package_identification/python_setup_py.py @@ -1,9 +1,9 @@ # Copyright 2016-2018 Dirk Thomas # Licensed under the Apache License, Version 2.0 -import ast import distutils.core import os +import pickle from pathlib import Path import runpy try: @@ -134,11 +134,11 @@ def get_setup_arguments(setup_py): setuptools.setup = setuptools_setup except NameError: pass - # filter out any data which doesn't work with ast.literal_eval + # filter out any data which doesn't serialize for key, value in list(data.items()): try: - ast.literal_eval(repr(value)) - except SyntaxError: + pickle.dumps(value) + except pickle.PicklingError: del data[key] return data @@ -237,9 +237,9 @@ def get_setup_arguments_with_context(setup_py, env): cmd = [sys.executable, '-c', ';'.join(code_lines)] result = subprocess.run( cmd, stdout=subprocess.PIPE, env=env, check=True) - output = result.stdout.decode('utf-8') + output = result.stdout - return ast.literal_eval(output) + return pickle.loads(output) _setup_information_cache = {} @@ -269,6 +269,7 @@ def get_setup_information(setup_py, *, env=None): def _get_setup_information(setup_py, *, env=None): code_lines = [ 'import sys', + 'import pickle', 'from distutils.core import run_setup', 'dist = run_setup(' @@ -293,7 +294,7 @@ def _get_setup_information(setup_py, *, env=None): # skip values with custom type OrderedSet " if k not in ('license_files', 'provides_extras')}", - "sys.stdout.buffer.write(repr(data).encode('utf-8'))"] + 'pickle.dump(data, sys.stdout)'] # invoke distutils.core.run_setup() in a separate interpreter cmd = [ @@ -301,6 +302,6 @@ def _get_setup_information(setup_py, *, env=None): result = subprocess.run( cmd, stdout=subprocess.PIPE, cwd=os.path.abspath(str(setup_py.parent)), check=True, env=env) - output = result.stdout.decode('utf-8') + output = result.stdout - return ast.literal_eval(output) + return pickle.loads(output) From 9066cc8d82c569a10c15e95e4d6fa9e5c955a233 Mon Sep 17 00:00:00 2001 From: Dan Rose Date: Fri, 24 Apr 2020 13:32:21 -0500 Subject: [PATCH 2/3] fix my code --- .../package_identification/python_setup_py.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/colcon_python_setup_py/package_identification/python_setup_py.py b/colcon_python_setup_py/package_identification/python_setup_py.py index 6ec5e88..0a93105 100644 --- a/colcon_python_setup_py/package_identification/python_setup_py.py +++ b/colcon_python_setup_py/package_identification/python_setup_py.py @@ -226,12 +226,14 @@ def get_setup_arguments_with_context(setup_py, env): pkg_path = str(pkg_path).replace(os.sep, os.altsep) setup_py = str(setup_py).replace(os.sep, os.altsep) code_lines = [ + 'import pickle', 'import sys', "sys.path.insert(0, '%s')" % pkg_path, 'from colcon_python_setup_py.package_identification.python_setup_py' ' import get_setup_arguments', - "output = repr(get_setup_arguments('%s'))" % setup_py, - "sys.stdout.buffer.write(output.encode('utf-8'))"] + "output = get_setup_arguments('%s')" % setup_py, + 'pickle.dump(output, sys.stdout.buffer)' + ] # invoke get_setup_arguments() in a separate interpreter cmd = [sys.executable, '-c', ';'.join(code_lines)] @@ -294,7 +296,7 @@ def _get_setup_information(setup_py, *, env=None): # skip values with custom type OrderedSet " if k not in ('license_files', 'provides_extras')}", - 'pickle.dump(data, sys.stdout)'] + 'pickle.dump(data, sys.stdout.buffer)'] # invoke distutils.core.run_setup() in a separate interpreter cmd = [ From 126e4bdbab045ba6aefa1db986ebbb743b680d0b Mon Sep 17 00:00:00 2001 From: Dan Rose Date: Wed, 22 Apr 2020 18:23:48 -0500 Subject: [PATCH 3/3] Pass Distribution object back from subprocess Upstream change removes the need for much of our workaround: https://github.com/pypa/setuptools/pull/1890 I do not remove turning the object into a dict, as it would change the public signature of `get_setup_information`. --- .../package_identification/python_setup_py.py | 27 +++++-------------- setup.cfg | 3 ++- test/spell_check.words | 1 - 3 files changed, 8 insertions(+), 23 deletions(-) diff --git a/colcon_python_setup_py/package_identification/python_setup_py.py b/colcon_python_setup_py/package_identification/python_setup_py.py index 0a93105..3109fe0 100644 --- a/colcon_python_setup_py/package_identification/python_setup_py.py +++ b/colcon_python_setup_py/package_identification/python_setup_py.py @@ -277,26 +277,7 @@ def _get_setup_information(setup_py, *, env=None): 'dist = run_setup(' " 'setup.py', script_args=('--dry-run',), stop_after='config')", - "skip_keys = ('cmdclass', 'distclass', 'ext_modules', 'metadata')", - 'data = {' - ' key: value for key, value in dist.__dict__.items() ' - ' if (' - # skip private properties - " not key.startswith('_') and " - # skip methods - ' not callable(value) and ' - # skip objects whose representation can't be evaluated - ' key not in skip_keys and ' - # skip display options since they have no value, using metadata instead - ' key not in dist.display_option_names' - ' )' - '}', - "data['metadata'] = {" - ' k: v for k, v in dist.metadata.__dict__.items() ' - # skip values with custom type OrderedSet - " if k not in ('license_files', 'provides_extras')}", - - 'pickle.dump(data, sys.stdout.buffer)'] + 'pickle.dump(dist, sys.stdout.buffer)'] # invoke distutils.core.run_setup() in a separate interpreter cmd = [ @@ -305,5 +286,9 @@ def _get_setup_information(setup_py, *, env=None): cmd, stdout=subprocess.PIPE, cwd=os.path.abspath(str(setup_py.parent)), check=True, env=env) output = result.stdout + dist = pickle.loads(output) - return pickle.loads(output) + # turn into a dict for backwards compatibility + dist_dict = dist.__dict__.copy() + dist_dict['metadata'] = dist.metadata.__dict__ + return dist_dict diff --git a/setup.cfg b/setup.cfg index c7b2f4d..740e2c7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,7 +27,8 @@ keywords = colcon [options] install_requires = colcon-core>=0.3.10 - setuptools + # DistributionMetadata not picklable in these versions https://github.com/pypa/setuptools/issues/1888 + setuptools !=42.*,!=43.*,!=44.*,!=45.* packages = find: tests_require = flake8>=3.6.0 diff --git a/test/spell_check.words b/test/spell_check.words index 75c65a6..d4d0cc5 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -1,7 +1,6 @@ apache chdir colcon -distclass hashable iterdir lstrip