From 6d7ec2f6d8313b50e73a2d573218f2509dc0523a Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 30 Apr 2026 18:13:50 +0200 Subject: [PATCH 1/5] Add preserve-dir --- dev/archery/archery/cli.py | 36 +++++++++++++++++------------ dev/archery/archery/utils/tmpdir.py | 7 ++++-- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/dev/archery/archery/cli.py b/dev/archery/archery/cli.py index e70cc3874f22..c61e0b7c3a94 100644 --- a/dev/archery/archery/cli.py +++ b/dev/archery/archery/cli.py @@ -305,6 +305,11 @@ def check_language(ctx, param, value): click.option("--preserve", type=BOOL, default=False, show_default=True, is_flag=True, help="Preserve workspace for investigation."), + click.option("--preserve-dir", metavar="", + type=click.Path(file_okay=False, resolve_path=True), + default=None, + help="Parent directory in which to create the preserved " + "workspace. Has no effect without --preserve."), click.option("--output", metavar="", type=click.File("w", encoding="utf8"), default=None, help="Capture output result into file."), @@ -347,12 +352,12 @@ def benchmark_filter_options(cmd): default="WORKSPACE", required=False) @benchmark_common_options @click.pass_context -def benchmark_list(ctx, rev_or_path, src, preserve, output, cmake_extras, - java_home, java_options, build_extras, benchmark_extras, - cpp_benchmark_extras, language, **kwargs): +def benchmark_list(ctx, rev_or_path, src, preserve, preserve_dir, output, + cmake_extras, java_home, java_options, build_extras, + benchmark_extras, cpp_benchmark_extras, language, **kwargs): """ List benchmark suite. """ - with tmpdir(preserve=preserve) as root: + with tmpdir(preserve=preserve, preserve_dir=preserve_dir) as root: logger.debug(f"Running benchmark {rev_or_path}") if language == "cpp": @@ -392,10 +397,11 @@ def benchmark_list(ctx, rev_or_path, src, preserve, output, cmake_extras, "Currently only supported for language=cpp. " "[default: use runner-specific defaults]")) @click.pass_context -def benchmark_run(ctx, rev_or_path, src, preserve, output, cmake_extras, - java_home, java_options, build_extras, benchmark_extras, - language, suite_filter, benchmark_filter, repetitions, - repetition_min_time, cpp_benchmark_extras, **kwargs): +def benchmark_run(ctx, rev_or_path, src, preserve, preserve_dir, output, + cmake_extras, java_home, java_options, build_extras, + benchmark_extras, language, suite_filter, benchmark_filter, + repetitions, repetition_min_time, cpp_benchmark_extras, + **kwargs): """ Run benchmark suite. This command will run the benchmark suite for a single build. This is @@ -433,7 +439,7 @@ def benchmark_run(ctx, rev_or_path, src, preserve, output, cmake_extras, \b archery benchmark run --output=run.json """ - with tmpdir(preserve=preserve) as root: + with tmpdir(preserve=preserve, preserve_dir=preserve_dir) as root: logger.debug(f"Running benchmark {rev_or_path}") if language == "cpp": @@ -486,11 +492,11 @@ def benchmark_run(ctx, rev_or_path, src, preserve, output, cmake_extras, @click.argument("baseline", metavar="[]]", default="origin/HEAD", required=False) @click.pass_context -def benchmark_diff(ctx, src, preserve, output, language, cmake_extras, - suite_filter, benchmark_filter, repetitions, no_counters, - java_home, java_options, build_extras, benchmark_extras, - cpp_benchmark_extras, threshold, contender, baseline, - **kwargs): +def benchmark_diff(ctx, src, preserve, preserve_dir, output, language, + cmake_extras, suite_filter, benchmark_filter, repetitions, + no_counters, java_home, java_options, build_extras, + benchmark_extras, cpp_benchmark_extras, threshold, + contender, baseline, **kwargs): """Compare (diff) benchmark runs. This command acts like git-diff but for benchmark results. @@ -564,7 +570,7 @@ def benchmark_diff(ctx, src, preserve, output, language, cmake_extras, # This should not recompute the benchmark from run.json archery --quiet benchmark diff WORKSPACE run.json > result.json """ - with tmpdir(preserve=preserve) as root: + with tmpdir(preserve=preserve, preserve_dir=preserve_dir) as root: logger.debug(f"Comparing {contender} (contender) with {baseline} (baseline)") if language == "cpp": diff --git a/dev/archery/archery/utils/tmpdir.py b/dev/archery/archery/utils/tmpdir.py index 07d7355c87fb..abd9e34877ab 100644 --- a/dev/archery/archery/utils/tmpdir.py +++ b/dev/archery/archery/utils/tmpdir.py @@ -15,14 +15,17 @@ # specific language governing permissions and limitations # under the License. +import os from contextlib import contextmanager from tempfile import mkdtemp, TemporaryDirectory @contextmanager -def tmpdir(preserve=False, prefix="arrow-archery-"): +def tmpdir(preserve=False, prefix="arrow-archery-", preserve_dir=None): if preserve: - yield mkdtemp(prefix=prefix) + if preserve_dir is not None: + os.makedirs(preserve_dir, exist_ok=True) + yield mkdtemp(prefix=prefix, dir=preserve_dir) else: with TemporaryDirectory(prefix=prefix) as tmp: yield tmp From 8e34e12f489dd023a40420e9b22b65ca22eb3dad Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 30 Apr 2026 18:34:11 +0200 Subject: [PATCH 2/5] Manage run build subfolder --- dev/archery/archery/benchmark/runner.py | 52 +++++++++++++++++-------- dev/archery/archery/utils/tmpdir.py | 9 +++-- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/dev/archery/archery/benchmark/runner.py b/dev/archery/archery/benchmark/runner.py index 571a5e58393b..ab38ab9cb737 100644 --- a/dev/archery/archery/benchmark/runner.py +++ b/dev/archery/archery/benchmark/runner.py @@ -18,6 +18,7 @@ import glob import json import os +import random import re from .core import BenchmarkSuite @@ -26,8 +27,22 @@ from ..lang.cpp import CppCMakeDefinition, CppConfiguration from ..lang.java import JavaMavenDefinition, JavaConfiguration from ..utils.cmake import CMakeBuild +from ..utils.git import git from ..utils.maven import MavenBuild from ..utils.logger import logger +from ..utils.source import ArrowSources + + +def _rev_or_path_dirname(src, rev_or_path): + if rev_or_path == ArrowSources.WORKSPACE: + return rev_or_path + try: + sha = git.rev_parse(rev_or_path, git_dir=src.path) + if isinstance(sha, bytes): + sha = sha.decode("ascii") + return sha + except Exception: + return rev_or_path.replace("/", "_") def regex_filter(re_expr): @@ -108,10 +123,11 @@ def __repr__(self): class CppBenchmarkRunner(BenchmarkRunner): """ Run suites from a CMakeBuild. """ - def __init__(self, build, benchmark_extras, **kwargs): + def __init__(self, build, benchmark_extras, run_id=None, **kwargs): """ Initialize a CppBenchmarkRunner. """ self.build = build self.benchmark_extras = benchmark_extras + self.run_id = run_id super().__init__(**kwargs) @staticmethod @@ -217,19 +233,22 @@ def from_rev_or_path(src, root, rev_or_path, cmake_conf, **kwargs): build = CMakeBuild.from_path(rev_or_path) return CppBenchmarkRunner(build, **kwargs) else: - # Revisions can references remote via the `/` character, ensure - # that the revision is path friendly - path_rev = rev_or_path.replace("/", "_") + path_rev = _rev_or_path_dirname(src, rev_or_path) root_rev = os.path.join(root, path_rev) - os.mkdir(root_rev) + os.makedirs(root_rev, exist_ok=True) clone_dir = os.path.join(root_rev, "arrow") - # Possibly checkout the sources at given revision, no need to - # perform cleanup on cloned repository as root_rev is reclaimed. - src_rev, _ = src.at_revision(rev_or_path, clone_dir) + if os.path.isdir(clone_dir): + src_rev = ArrowSources(clone_dir) + else: + src_rev, _ = src.at_revision(rev_or_path, clone_dir) cmake_def = CppCMakeDefinition(src_rev.cpp, cmake_conf) - build_dir = os.path.join(root_rev, "build") - return CppBenchmarkRunner(cmake_def.build(build_dir), **kwargs) + run_root = os.path.join(root_rev, "build", "run") + os.makedirs(run_root, exist_ok=True) + run_id = f"{random.randrange(16**8):08x}" + build_dir = os.path.join(run_root, run_id) + build = cmake_def.build(build_dir) + return CppBenchmarkRunner(build, run_id=run_id, **kwargs) class JavaBenchmarkRunner(BenchmarkRunner): @@ -310,16 +329,15 @@ def from_rev_or_path(src, root, rev_or_path, maven_conf, **kwargs): maven_def = JavaMavenDefinition(rev_or_path, maven_conf) return JavaBenchmarkRunner(maven_def.build(rev_or_path), **kwargs) else: - # Revisions can references remote via the `/` character, ensure - # that the revision is path friendly - path_rev = rev_or_path.replace("/", "_") + path_rev = _rev_or_path_dirname(src, rev_or_path) root_rev = os.path.join(root, path_rev) - os.mkdir(root_rev) + os.makedirs(root_rev, exist_ok=True) clone_dir = os.path.join(root_rev, "arrow") - # Possibly checkout the sources at given revision, no need to - # perform cleanup on cloned repository as root_rev is reclaimed. - src_rev, _ = src.at_revision(rev_or_path, clone_dir) + if os.path.isdir(clone_dir): + src_rev = ArrowSources(clone_dir) + else: + src_rev, _ = src.at_revision(rev_or_path, clone_dir) maven_def = JavaMavenDefinition(src_rev.java, maven_conf) build_dir = os.path.join(root_rev, "arrow/java") return JavaBenchmarkRunner(maven_def.build(build_dir), **kwargs) diff --git a/dev/archery/archery/utils/tmpdir.py b/dev/archery/archery/utils/tmpdir.py index abd9e34877ab..06eb9a6ba627 100644 --- a/dev/archery/archery/utils/tmpdir.py +++ b/dev/archery/archery/utils/tmpdir.py @@ -22,10 +22,11 @@ @contextmanager def tmpdir(preserve=False, prefix="arrow-archery-", preserve_dir=None): - if preserve: - if preserve_dir is not None: - os.makedirs(preserve_dir, exist_ok=True) - yield mkdtemp(prefix=prefix, dir=preserve_dir) + if preserve and preserve_dir is not None: + os.makedirs(preserve_dir, exist_ok=True) + yield preserve_dir + elif preserve: + yield mkdtemp(prefix=prefix) else: with TemporaryDirectory(prefix=prefix) as tmp: yield tmp From a26b4470dac6c296ed8da6f7f78e23953d2e4b26 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Tue, 26 May 2026 17:04:35 +0200 Subject: [PATCH 3/5] Add result dir --- dev/archery/archery/benchmark/runner.py | 10 ++++++++-- dev/archery/archery/cli.py | 5 +++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/dev/archery/archery/benchmark/runner.py b/dev/archery/archery/benchmark/runner.py index ab38ab9cb737..1b57a0212265 100644 --- a/dev/archery/archery/benchmark/runner.py +++ b/dev/archery/archery/benchmark/runner.py @@ -62,6 +62,7 @@ def __init__(self, suite_filter=None, benchmark_filter=None, self.benchmark_filter = benchmark_filter self.repetitions = repetitions self.repetition_min_time = repetition_min_time + self.results_dir = None @property def suites(self): @@ -123,11 +124,13 @@ def __repr__(self): class CppBenchmarkRunner(BenchmarkRunner): """ Run suites from a CMakeBuild. """ - def __init__(self, build, benchmark_extras, run_id=None, **kwargs): + def __init__(self, build, benchmark_extras, run_id=None, + results_dir=None, **kwargs): """ Initialize a CppBenchmarkRunner. """ self.build = build self.benchmark_extras = benchmark_extras self.run_id = run_id + self.results_dir = results_dir super().__init__(**kwargs) @staticmethod @@ -248,7 +251,10 @@ def from_rev_or_path(src, root, rev_or_path, cmake_conf, **kwargs): run_id = f"{random.randrange(16**8):08x}" build_dir = os.path.join(run_root, run_id) build = cmake_def.build(build_dir) - return CppBenchmarkRunner(build, run_id=run_id, **kwargs) + results_dir = os.path.join(root_rev, "bench", "run", run_id) + os.makedirs(results_dir, exist_ok=True) + return CppBenchmarkRunner(build, run_id=run_id, + results_dir=results_dir, **kwargs) class JavaBenchmarkRunner(BenchmarkRunner): diff --git a/dev/archery/archery/cli.py b/dev/archery/archery/cli.py index c61e0b7c3a94..0bf3ca6d315f 100644 --- a/dev/archery/archery/cli.py +++ b/dev/archery/archery/cli.py @@ -471,6 +471,11 @@ def benchmark_run(ctx, rev_or_path, src, preserve, preserve_dir, output, # when asked to JSON-serialize the results, so produce a JSON # output even when none is requested. json_out = json.dumps(runner_base, cls=JsonEncoder) + if runner_base.results_dir is not None: + results_path = os.path.join(runner_base.results_dir, + "benchmark.json") + with open(results_path, "w") as f: + f.write(json_out) if output is not None: output.write(json_out) From 4284861a355fea4835d57f2e4db02a73cbb9224f Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Wed, 27 May 2026 15:20:11 +0200 Subject: [PATCH 4/5] Save basic information --- dev/archery/archery/benchmark/runner.py | 2 +- dev/archery/archery/cli.py | 1077 ++++++++++++++++------- 2 files changed, 737 insertions(+), 342 deletions(-) diff --git a/dev/archery/archery/benchmark/runner.py b/dev/archery/archery/benchmark/runner.py index 1b57a0212265..ba0f239e3bd1 100644 --- a/dev/archery/archery/benchmark/runner.py +++ b/dev/archery/archery/benchmark/runner.py @@ -127,11 +127,11 @@ class CppBenchmarkRunner(BenchmarkRunner): def __init__(self, build, benchmark_extras, run_id=None, results_dir=None, **kwargs): """ Initialize a CppBenchmarkRunner. """ + super().__init__(**kwargs) self.build = build self.benchmark_extras = benchmark_extras self.run_id = run_id self.results_dir = results_dir - super().__init__(**kwargs) @staticmethod def default_configuration(**kwargs): diff --git a/dev/archery/archery/cli.py b/dev/archery/archery/cli.py index 0bf3ca6d315f..75b96e1f9153 100644 --- a/dev/archery/archery/cli.py +++ b/dev/archery/archery/cli.py @@ -19,8 +19,10 @@ import click import json import logging +import datetime import os import pathlib +import platform import sys from .benchmark.codec import JsonEncoder @@ -41,17 +43,54 @@ BOOL = ArrowBool() +def _run_metadata(ctx): + uname = platform.uname() + return { + "time": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "cmd": { + "full": " ".join(sys.argv), + "params": ctx.params, + }, + "machine_info": { + "platform": platform.platform(), + "system": uname.system, + "release": uname.release, + "version": uname.version, + "machine": platform.machine(), + "processor": platform.processor(), + "architecture": platform.architecture()[0], + "logical_cores": os.cpu_count(), + }, + } + + @click.group(context_settings={"help_option_names": ["-h", "--help"]}) -@click.option("--debug", type=BOOL, is_flag=True, default=False, - envvar='ARCHERY_DEBUG', - help="Increase logging with debugging output.") -@click.option("--pdb", type=BOOL, is_flag=True, default=False, - help="Invoke pdb on uncaught exception.") -@click.option("-q", "--quiet", type=BOOL, is_flag=True, default=False, - help="Silence executed commands.") +@click.option( + "--debug", + type=BOOL, + is_flag=True, + default=False, + envvar="ARCHERY_DEBUG", + help="Increase logging with debugging output.", +) +@click.option( + "--pdb", + type=BOOL, + is_flag=True, + default=False, + help="Invoke pdb on uncaught exception.", +) +@click.option( + "-q", + "--quiet", + type=BOOL, + is_flag=True, + default=False, + help="Silence executed commands.", +) @click.pass_context def archery(ctx, debug, pdb, quiet): - """ Apache Arrow developer utilities. + """Apache Arrow developer utilities. See sub-commands help with `archery --help`. @@ -63,23 +102,23 @@ def archery(ctx, debug, pdb, quiet): if debug: logger.setLevel(logging.DEBUG) - ctx.obj['debug'] = debug + ctx.obj["debug"] = debug if pdb: import pdb + sys.excepthook = lambda t, v, e: pdb.pm() build_dir_type = click.Path(dir_okay=True, file_okay=False, resolve_path=True) # Supported build types -build_type = click.Choice(["debug", "relwithdebinfo", "release"], - case_sensitive=False) +build_type = click.Choice(["debug", "relwithdebinfo", "release"], case_sensitive=False) # Supported warn levels -warn_level_type = click.Choice(["everything", "checkin", "production"], - case_sensitive=False) +warn_level_type = click.Choice( + ["everything", "checkin", "production"], case_sensitive=False +) -simd_level = click.Choice(["NONE", "SSE4_2", "AVX2", "AVX512"], - case_sensitive=True) +simd_level = click.Choice(["NONE", "SSE4_2", "AVX2", "AVX512"], case_sensitive=True) def cpp_toolchain_options(cmd): @@ -87,17 +126,22 @@ def cpp_toolchain_options(cmd): click.option("--cc", metavar="", help="C compiler."), click.option("--cxx", metavar="", help="C++ compiler."), click.option("--cxx-flags", help="C++ compiler flags."), - click.option("--cpp-package-prefix", - help=("Value to pass for ARROW_PACKAGE_PREFIX and " - "use ARROW_DEPENDENCY_SOURCE=SYSTEM")) + click.option( + "--cpp-package-prefix", + help=( + "Value to pass for ARROW_PACKAGE_PREFIX and " + "use ARROW_DEPENDENCY_SOURCE=SYSTEM" + ), + ), ] return _apply_options(cmd, options) def java_toolchain_options(cmd): options = [ - click.option("--java-home", metavar="", - help="Path to Java Developers Kit."), + click.option( + "--java-home", metavar="", help="Path to Java Developers Kit." + ), click.option("--java-options", help="java compiler options."), ] return _apply_options(cmd, options) @@ -110,107 +154,183 @@ def _apply_options(cmd, options): @archery.command(short_help="Initialize an Arrow C++ build") -@click.option("--src", metavar="", default=None, - callback=validate_arrow_sources, - help="Specify Arrow source directory") +@click.option( + "--src", + metavar="", + default=None, + callback=validate_arrow_sources, + help="Specify Arrow source directory", +) # toolchain @cpp_toolchain_options -@click.option("--build-type", default=None, type=build_type, - help="CMake's CMAKE_BUILD_TYPE") -@click.option("--build-static", default=True, type=BOOL, - help="Build static libraries") -@click.option("--build-shared", default=True, type=BOOL, - help="Build shared libraries") -@click.option("--build-unity", default=True, type=BOOL, - help="Use CMAKE_UNITY_BUILD") -@click.option("--warn-level", default="production", type=warn_level_type, - help="Controls compiler warnings -W(no-)error.") -@click.option("--simd-level", default="DEFAULT", type=simd_level, - help="Toggles ARROW_SIMD_LEVEL option.") +@click.option( + "--build-type", default=None, type=build_type, help="CMake's CMAKE_BUILD_TYPE" +) +@click.option("--build-static", default=True, type=BOOL, help="Build static libraries") +@click.option("--build-shared", default=True, type=BOOL, help="Build shared libraries") +@click.option("--build-unity", default=True, type=BOOL, help="Use CMAKE_UNITY_BUILD") +@click.option( + "--warn-level", + default="production", + type=warn_level_type, + help="Controls compiler warnings -W(no-)error.", +) +@click.option( + "--simd-level", + default="DEFAULT", + type=simd_level, + help="Toggles ARROW_SIMD_LEVEL option.", +) # Tests and benchmarks -@click.option("--with-tests", default=True, type=BOOL, - help="Build with tests.") -@click.option("--with-benchmarks", default=None, type=BOOL, - help="Build with benchmarks.") -@click.option("--with-examples", default=None, type=BOOL, - help="Build with examples.") -@click.option("--with-integration", default=None, type=BOOL, - help="Build with integration test executables.") +@click.option("--with-tests", default=True, type=BOOL, help="Build with tests.") +@click.option( + "--with-benchmarks", default=None, type=BOOL, help="Build with benchmarks." +) +@click.option("--with-examples", default=None, type=BOOL, help="Build with examples.") +@click.option( + "--with-integration", + default=None, + type=BOOL, + help="Build with integration test executables.", +) # Static checks -@click.option("--use-asan", default=None, type=BOOL, - help="Toggle ARROW_USE_ASAN sanitizer.") -@click.option("--use-tsan", default=None, type=BOOL, - help="Toggle ARROW_USE_TSAN sanitizer.") -@click.option("--use-ubsan", default=None, type=BOOL, - help="Toggle ARROW_USE_UBSAN sanitizer.") -@click.option("--with-fuzzing", default=None, type=BOOL, - help="Toggle ARROW_FUZZING.") +@click.option( + "--use-asan", default=None, type=BOOL, help="Toggle ARROW_USE_ASAN sanitizer." +) +@click.option( + "--use-tsan", default=None, type=BOOL, help="Toggle ARROW_USE_TSAN sanitizer." +) +@click.option( + "--use-ubsan", default=None, type=BOOL, help="Toggle ARROW_USE_UBSAN sanitizer." +) +@click.option("--with-fuzzing", default=None, type=BOOL, help="Toggle ARROW_FUZZING.") # Components -@click.option("--with-compute", default=None, type=BOOL, - help="Build the Arrow compute module.") -@click.option("--with-csv", default=None, type=BOOL, - help="Build the Arrow CSV parser module.") -@click.option("--with-cuda", default=None, type=BOOL, - help="Build the Arrow CUDA extensions.") -@click.option("--with-dataset", default=None, type=BOOL, - help="Build the Arrow dataset module.") -@click.option("--with-filesystem", default=None, type=BOOL, - help="Build the Arrow filesystem layer.") -@click.option("--with-flight", default=None, type=BOOL, - help="Build with Flight rpc support.") -@click.option("--with-gandiva", default=None, type=BOOL, - help="Build with Gandiva expression compiler support.") -@click.option("--with-gcs", default=None, type=BOOL, - help="Build Arrow with Google Cloud Storage (GCS) support.") -@click.option("--with-hdfs", default=None, type=BOOL, - help="Build the Arrow HDFS bridge.") -@click.option("--with-hiveserver2", default=None, type=BOOL, - help="Build the HiveServer2 client and arrow adapter.") -@click.option("--with-ipc", default=None, type=BOOL, - help="Build the Arrow IPC extensions.") -@click.option("--with-json", default=None, type=BOOL, - help="Build the Arrow JSON parser module.") -@click.option("--with-mimalloc", default=None, type=BOOL, - help="Build the Arrow mimalloc based allocator.") -@click.option("--with-parquet", default=None, type=BOOL, - help="Build with Parquet file support.") -@click.option("--with-python", default=None, type=BOOL, - help="Build the Arrow CPython extensions.") -@click.option("--with-r", default=None, type=BOOL, - help="Build the Arrow R extensions. This is not a CMake option, " - "it will toggle required options") -@click.option("--with-s3", default=None, type=BOOL, - help="Build Arrow with S3 support.") +@click.option( + "--with-compute", default=None, type=BOOL, help="Build the Arrow compute module." +) +@click.option( + "--with-csv", default=None, type=BOOL, help="Build the Arrow CSV parser module." +) +@click.option( + "--with-cuda", default=None, type=BOOL, help="Build the Arrow CUDA extensions." +) +@click.option( + "--with-dataset", default=None, type=BOOL, help="Build the Arrow dataset module." +) +@click.option( + "--with-filesystem", + default=None, + type=BOOL, + help="Build the Arrow filesystem layer.", +) +@click.option( + "--with-flight", default=None, type=BOOL, help="Build with Flight rpc support." +) +@click.option( + "--with-gandiva", + default=None, + type=BOOL, + help="Build with Gandiva expression compiler support.", +) +@click.option( + "--with-gcs", + default=None, + type=BOOL, + help="Build Arrow with Google Cloud Storage (GCS) support.", +) +@click.option( + "--with-hdfs", default=None, type=BOOL, help="Build the Arrow HDFS bridge." +) +@click.option( + "--with-hiveserver2", + default=None, + type=BOOL, + help="Build the HiveServer2 client and arrow adapter.", +) +@click.option( + "--with-ipc", default=None, type=BOOL, help="Build the Arrow IPC extensions." +) +@click.option( + "--with-json", default=None, type=BOOL, help="Build the Arrow JSON parser module." +) +@click.option( + "--with-mimalloc", + default=None, + type=BOOL, + help="Build the Arrow mimalloc based allocator.", +) +@click.option( + "--with-parquet", default=None, type=BOOL, help="Build with Parquet file support." +) +@click.option( + "--with-python", default=None, type=BOOL, help="Build the Arrow CPython extensions." +) +@click.option( + "--with-r", + default=None, + type=BOOL, + help="Build the Arrow R extensions. This is not a CMake option, " + "it will toggle required options", +) +@click.option("--with-s3", default=None, type=BOOL, help="Build Arrow with S3 support.") # Compressions -@click.option("--with-brotli", default=None, type=BOOL, - help="Build Arrow with brotli compression.") -@click.option("--with-bz2", default=None, type=BOOL, - help="Build Arrow with bz2 compression.") -@click.option("--with-lz4", default=None, type=BOOL, - help="Build Arrow with lz4 compression.") -@click.option("--with-snappy", default=None, type=BOOL, - help="Build Arrow with snappy compression.") -@click.option("--with-zlib", default=None, type=BOOL, - help="Build Arrow with zlib compression.") -@click.option("--with-zstd", default=None, type=BOOL, - help="Build Arrow with zstd compression.") +@click.option( + "--with-brotli", + default=None, + type=BOOL, + help="Build Arrow with brotli compression.", +) +@click.option( + "--with-bz2", default=None, type=BOOL, help="Build Arrow with bz2 compression." +) +@click.option( + "--with-lz4", default=None, type=BOOL, help="Build Arrow with lz4 compression." +) +@click.option( + "--with-snappy", + default=None, + type=BOOL, + help="Build Arrow with snappy compression.", +) +@click.option( + "--with-zlib", default=None, type=BOOL, help="Build Arrow with zlib compression." +) +@click.option( + "--with-zstd", default=None, type=BOOL, help="Build Arrow with zstd compression." +) # CMake extra feature -@click.option("--cmake-extras", type=str, multiple=True, - help="Extra flags/options to pass to cmake invocation. " - "Can be stacked") -@click.option("--install-prefix", type=str, - help="Destination directory where files are installed. Expand to" - "CMAKE_INSTALL_PREFIX. Defaults to to $CONDA_PREFIX if the" - "variable exists.") +@click.option( + "--cmake-extras", + type=str, + multiple=True, + help="Extra flags/options to pass to cmake invocation. Can be stacked", +) +@click.option( + "--install-prefix", + type=str, + help="Destination directory where files are installed. Expand to" + "CMAKE_INSTALL_PREFIX. Defaults to to $CONDA_PREFIX if the" + "variable exists.", +) # misc -@click.option("-f", "--force", type=BOOL, is_flag=True, default=False, - help="Delete existing build directory if found.") -@click.option("--targets", type=str, multiple=True, - help="Generator targets to run. Can be stacked.") +@click.option( + "-f", + "--force", + type=BOOL, + is_flag=True, + default=False, + help="Delete existing build directory if found.", +) +@click.option( + "--targets", + type=str, + multiple=True, + help="Generator targets to run. Can be stacked.", +) @click.argument("build_dir", type=build_dir_type) @click.pass_context def build(ctx, src, build_dir, force, targets, **kwargs): - """ Initialize a C++ build directory. + """Initialize a C++ build directory. The build command creates a directory initialized with Arrow's cpp source cmake and configuration. It can also optionally invoke the generator to @@ -246,19 +366,31 @@ def build(ctx, src, build_dir, force, targets, **kwargs): def _flatten_numpydoc_rules(rules): flattened = [] for rule in rules: - flattened.extend(filter(None, rule.split(','))) + flattened.extend(filter(None, rule.split(","))) return flattened @archery.command(short_help="Lint python docstring with NumpyDoc") -@click.argument('symbols', nargs=-1) -@click.option("--src", metavar="", default=None, - callback=validate_arrow_sources, - help="Specify Arrow source directory") -@click.option("--allow-rule", "-a", multiple=True, - help="Allow only these rules (can be comma-separated)") -@click.option("--disallow-rule", "-d", multiple=True, - help="Disallow these rules (can be comma-separated)") +@click.argument("symbols", nargs=-1) +@click.option( + "--src", + metavar="", + default=None, + callback=validate_arrow_sources, + help="Specify Arrow source directory", +) +@click.option( + "--allow-rule", + "-a", + multiple=True, + help="Allow only these rules (can be comma-separated)", +) +@click.option( + "--disallow-rule", + "-d", + multiple=True, + help="Disallow these rules (can be comma-separated)", +) def numpydoc(src, symbols, allow_rule, disallow_rule): """ Pass list of modules or symbols as arguments to restrict the validation. @@ -271,11 +403,13 @@ def numpydoc(src, symbols, allow_rule, disallow_rule): archery numpydoc pyarrow.csv pyarrow.json pyarrow.parquet archery numpydoc pyarrow.array """ - disallow_rule = disallow_rule or {'GL01', 'SA01', 'EX01', 'ES01'} + disallow_rule = disallow_rule or {"GL01", "SA01", "EX01", "ES01"} try: results = python_numpydoc( - symbols, allow_rules=_flatten_numpydoc_rules(allow_rule), - disallow_rules=_flatten_numpydoc_rules(disallow_rule)) + symbols, + allow_rules=_flatten_numpydoc_rules(allow_rule), + disallow_rules=_flatten_numpydoc_rules(disallow_rule), + ) for result in results: result.ok() except LintValidationException: @@ -285,7 +419,7 @@ def numpydoc(src, symbols, allow_rule, disallow_rule): @archery.group() @click.pass_context def benchmark(ctx): - """ Arrow benchmarking. + """Arrow benchmarking. Use the diff sub-command to benchmark revisions, and/or build directories. """ @@ -299,35 +433,74 @@ def check_language(ctx, param, value): return value options = [ - click.option("--src", metavar="", show_default=True, - default=None, callback=validate_arrow_sources, - help="Specify Arrow source directory"), - click.option("--preserve", type=BOOL, default=False, show_default=True, - is_flag=True, - help="Preserve workspace for investigation."), - click.option("--preserve-dir", metavar="", - type=click.Path(file_okay=False, resolve_path=True), - default=None, - help="Parent directory in which to create the preserved " - "workspace. Has no effect without --preserve."), - click.option("--output", metavar="", - type=click.File("w", encoding="utf8"), default=None, - help="Capture output result into file."), - click.option("--language", metavar="", type=str, default="cpp", - show_default=True, callback=check_language, - help="Specify target language for the benchmark"), - click.option("--build-extras", type=str, multiple=True, - help="Extra flags/options to pass to mvn build. " - "Can be stacked. For language=java"), - click.option("--benchmark-extras", type=str, multiple=True, - help="Extra flags/options to pass to mvn benchmark. " - "Can be stacked. For language=java"), - click.option("--cmake-extras", type=str, multiple=True, - help="Extra flags/options to pass to cmake invocation. " - "Can be stacked. For language=cpp"), - click.option("--cpp-benchmark-extras", type=str, multiple=True, - help="Extra flags/options to pass to C++ benchmark executables. " - "Can be stacked. For language=cpp"), + click.option( + "--src", + metavar="", + show_default=True, + default=None, + callback=validate_arrow_sources, + help="Specify Arrow source directory", + ), + click.option( + "--preserve", + type=BOOL, + default=False, + show_default=True, + is_flag=True, + help="Preserve workspace for investigation.", + ), + click.option( + "--preserve-dir", + metavar="", + type=click.Path(file_okay=False, resolve_path=True), + default=None, + help="Parent directory in which to create the preserved " + "workspace. Has no effect without --preserve.", + ), + click.option( + "--output", + metavar="", + type=click.File("w", encoding="utf8"), + default=None, + help="Capture output result into file.", + ), + click.option( + "--language", + metavar="", + type=str, + default="cpp", + show_default=True, + callback=check_language, + help="Specify target language for the benchmark", + ), + click.option( + "--build-extras", + type=str, + multiple=True, + help="Extra flags/options to pass to mvn build. " + "Can be stacked. For language=java", + ), + click.option( + "--benchmark-extras", + type=str, + multiple=True, + help="Extra flags/options to pass to mvn benchmark. " + "Can be stacked. For language=java", + ), + click.option( + "--cmake-extras", + type=str, + multiple=True, + help="Extra flags/options to pass to cmake invocation. " + "Can be stacked. For language=cpp", + ), + click.option( + "--cpp-benchmark-extras", + type=str, + multiple=True, + help="Extra flags/options to pass to C++ benchmark executables. " + "Can be stacked. For language=cpp", + ), ] cmd = java_toolchain_options(cmd) @@ -337,72 +510,128 @@ def check_language(ctx, param, value): def benchmark_filter_options(cmd): options = [ - click.option("--suite-filter", metavar="", show_default=True, - type=str, default=None, - help="Regex filtering benchmark suites."), - click.option("--benchmark-filter", metavar="", - show_default=True, type=str, default=None, - help="Regex filtering benchmarks.") + click.option( + "--suite-filter", + metavar="", + show_default=True, + type=str, + default=None, + help="Regex filtering benchmark suites.", + ), + click.option( + "--benchmark-filter", + metavar="", + show_default=True, + type=str, + default=None, + help="Regex filtering benchmarks.", + ), ] return _apply_options(cmd, options) @benchmark.command(name="list", short_help="List benchmark suite") -@click.argument("rev_or_path", metavar="[]", - default="WORKSPACE", required=False) +@click.argument( + "rev_or_path", metavar="[]", default="WORKSPACE", required=False +) @benchmark_common_options @click.pass_context -def benchmark_list(ctx, rev_or_path, src, preserve, preserve_dir, output, - cmake_extras, java_home, java_options, build_extras, - benchmark_extras, cpp_benchmark_extras, language, **kwargs): - """ List benchmark suite. - """ +def benchmark_list( + ctx, + rev_or_path, + src, + preserve, + preserve_dir, + output, + cmake_extras, + java_home, + java_options, + build_extras, + benchmark_extras, + cpp_benchmark_extras, + language, + **kwargs, +): + """List benchmark suite.""" with tmpdir(preserve=preserve, preserve_dir=preserve_dir) as root: logger.debug(f"Running benchmark {rev_or_path}") if language == "cpp": conf = CppBenchmarkRunner.default_configuration( - cmake_extras=cmake_extras, **kwargs) + cmake_extras=cmake_extras, **kwargs + ) runner_base = CppBenchmarkRunner.from_rev_or_path( - src, root, rev_or_path, conf, - benchmark_extras=cpp_benchmark_extras) + src, root, rev_or_path, conf, benchmark_extras=cpp_benchmark_extras + ) elif language == "java": - for key in {'cpp_package_prefix', 'cxx_flags', 'cxx', 'cc'}: + for key in {"cpp_package_prefix", "cxx_flags", "cxx", "cc"}: del kwargs[key] conf = JavaBenchmarkRunner.default_configuration( - java_home=java_home, java_options=java_options, - build_extras=build_extras, benchmark_extras=benchmark_extras, - **kwargs) + java_home=java_home, + java_options=java_options, + build_extras=build_extras, + benchmark_extras=benchmark_extras, + **kwargs, + ) runner_base = JavaBenchmarkRunner.from_rev_or_path( - src, root, rev_or_path, conf) + src, root, rev_or_path, conf + ) for b in runner_base.list_benchmarks: click.echo(b, file=output or sys.stdout) @benchmark.command(name="run", short_help="Run benchmark suite") -@click.argument("rev_or_path", metavar="[]", - default="WORKSPACE", required=False) +@click.argument( + "rev_or_path", metavar="[]", default="WORKSPACE", required=False +) @benchmark_common_options @benchmark_filter_options -@click.option("--repetitions", type=int, default=-1, - help=("Number of repetitions of each benchmark. Increasing " - "may improve result precision. " - "[default: 1 for cpp, 5 for java]")) -@click.option("--repetition-min-time", type=float, default=None, - help=("Minimum duration of each repetition in seconds. " - "Currently only supported for language=cpp. " - "[default: use runner-specific defaults]")) +@click.option( + "--repetitions", + type=int, + default=-1, + help=( + "Number of repetitions of each benchmark. Increasing " + "may improve result precision. " + "[default: 1 for cpp, 5 for java]" + ), +) +@click.option( + "--repetition-min-time", + type=float, + default=None, + help=( + "Minimum duration of each repetition in seconds. " + "Currently only supported for language=cpp. " + "[default: use runner-specific defaults]" + ), +) @click.pass_context -def benchmark_run(ctx, rev_or_path, src, preserve, preserve_dir, output, - cmake_extras, java_home, java_options, build_extras, - benchmark_extras, language, suite_filter, benchmark_filter, - repetitions, repetition_min_time, cpp_benchmark_extras, - **kwargs): - """ Run benchmark suite. +def benchmark_run( + ctx, + rev_or_path, + src, + preserve, + preserve_dir, + output, + cmake_extras, + java_home, + java_options, + build_extras, + benchmark_extras, + language, + suite_filter, + benchmark_filter, + repetitions, + repetition_min_time, + cpp_benchmark_extras, + **kwargs, +): + """Run benchmark suite. This command will run the benchmark suite for a single build. This is used to capture (and/or publish) the results. @@ -444,38 +673,54 @@ def benchmark_run(ctx, rev_or_path, src, preserve, preserve_dir, output, if language == "cpp": conf = CppBenchmarkRunner.default_configuration( - cmake_extras=cmake_extras, **kwargs) + cmake_extras=cmake_extras, **kwargs + ) repetitions = repetitions if repetitions != -1 else 1 runner_base = CppBenchmarkRunner.from_rev_or_path( - src, root, rev_or_path, conf, - repetitions=repetitions, repetition_min_time=repetition_min_time, - suite_filter=suite_filter, benchmark_filter=benchmark_filter, - benchmark_extras=cpp_benchmark_extras) + src, + root, + rev_or_path, + conf, + repetitions=repetitions, + repetition_min_time=repetition_min_time, + suite_filter=suite_filter, + benchmark_filter=benchmark_filter, + benchmark_extras=cpp_benchmark_extras, + ) elif language == "java": - for key in {'cpp_package_prefix', 'cxx_flags', 'cxx', 'cc'}: + for key in {"cpp_package_prefix", "cxx_flags", "cxx", "cc"}: del kwargs[key] conf = JavaBenchmarkRunner.default_configuration( - java_home=java_home, java_options=java_options, - build_extras=build_extras, benchmark_extras=benchmark_extras, - **kwargs) + java_home=java_home, + java_options=java_options, + build_extras=build_extras, + benchmark_extras=benchmark_extras, + **kwargs, + ) repetitions = repetitions if repetitions != -1 else 5 runner_base = JavaBenchmarkRunner.from_rev_or_path( - src, root, rev_or_path, conf, + src, + root, + rev_or_path, + conf, repetitions=repetitions, - benchmark_filter=benchmark_filter) + benchmark_filter=benchmark_filter, + ) # XXX for some reason, the benchmark runner only does its work # when asked to JSON-serialize the results, so produce a JSON # output even when none is requested. json_out = json.dumps(runner_base, cls=JsonEncoder) if runner_base.results_dir is not None: - results_path = os.path.join(runner_base.results_dir, - "benchmark.json") + results_path = os.path.join(runner_base.results_dir, "benchmark.json") with open(results_path, "w") as f: f.write(json_out) + metadata_path = os.path.join(runner_base.results_dir, "metadata.json") + with open(metadata_path, "w") as f: + json.dump(_run_metadata(ctx), f, indent=2, default=str) if output is not None: output.write(json_out) @@ -483,25 +728,60 @@ def benchmark_run(ctx, rev_or_path, src, preserve, preserve_dir, output, @benchmark.command(name="diff", short_help="Compare benchmark suites") @benchmark_common_options @benchmark_filter_options -@click.option("--threshold", type=float, default=DEFAULT_THRESHOLD, - show_default=True, - help="Regression failure threshold in percentage.") -@click.option("--repetitions", type=int, default=1, show_default=True, - help=("Number of repetitions of each benchmark. Increasing " - "may improve result precision. " - "[default: 1 for cpp, 5 for java")) -@click.option("--no-counters", type=BOOL, default=False, is_flag=True, - help="Hide counters field in diff report.") -@click.argument("contender", metavar="[", - default=ArrowSources.WORKSPACE, required=False) -@click.argument("baseline", metavar="[]]", default="origin/HEAD", - required=False) +@click.option( + "--threshold", + type=float, + default=DEFAULT_THRESHOLD, + show_default=True, + help="Regression failure threshold in percentage.", +) +@click.option( + "--repetitions", + type=int, + default=1, + show_default=True, + help=( + "Number of repetitions of each benchmark. Increasing " + "may improve result precision. " + "[default: 1 for cpp, 5 for java" + ), +) +@click.option( + "--no-counters", + type=BOOL, + default=False, + is_flag=True, + help="Hide counters field in diff report.", +) +@click.argument( + "contender", metavar="[", default=ArrowSources.WORKSPACE, required=False +) +@click.argument( + "baseline", metavar="[]]", default="origin/HEAD", required=False +) @click.pass_context -def benchmark_diff(ctx, src, preserve, preserve_dir, output, language, - cmake_extras, suite_filter, benchmark_filter, repetitions, - no_counters, java_home, java_options, build_extras, - benchmark_extras, cpp_benchmark_extras, threshold, - contender, baseline, **kwargs): +def benchmark_diff( + ctx, + src, + preserve, + preserve_dir, + output, + language, + cmake_extras, + suite_filter, + benchmark_filter, + repetitions, + no_counters, + java_home, + java_options, + build_extras, + benchmark_extras, + cpp_benchmark_extras, + threshold, + contender, + baseline, + **kwargs, +): """Compare (diff) benchmark runs. This command acts like git-diff but for benchmark results. @@ -580,47 +860,68 @@ def benchmark_diff(ctx, src, preserve, preserve_dir, output, language, if language == "cpp": conf = CppBenchmarkRunner.default_configuration( - cmake_extras=cmake_extras, **kwargs) + cmake_extras=cmake_extras, **kwargs + ) repetitions = repetitions if repetitions != -1 else 1 runner_cont = CppBenchmarkRunner.from_rev_or_path( - src, root, contender, conf, + src, + root, + contender, + conf, repetitions=repetitions, suite_filter=suite_filter, benchmark_filter=benchmark_filter, - benchmark_extras=cpp_benchmark_extras) + benchmark_extras=cpp_benchmark_extras, + ) runner_base = CppBenchmarkRunner.from_rev_or_path( - src, root, baseline, conf, + src, + root, + baseline, + conf, repetitions=repetitions, suite_filter=suite_filter, benchmark_filter=benchmark_filter, - benchmark_extras=cpp_benchmark_extras) + benchmark_extras=cpp_benchmark_extras, + ) elif language == "java": - for key in {'cpp_package_prefix', 'cxx_flags', 'cxx', 'cc'}: + for key in {"cpp_package_prefix", "cxx_flags", "cxx", "cc"}: del kwargs[key] conf = JavaBenchmarkRunner.default_configuration( - java_home=java_home, java_options=java_options, - build_extras=build_extras, benchmark_extras=benchmark_extras, - **kwargs) + java_home=java_home, + java_options=java_options, + build_extras=build_extras, + benchmark_extras=benchmark_extras, + **kwargs, + ) repetitions = repetitions if repetitions != -1 else 5 runner_cont = JavaBenchmarkRunner.from_rev_or_path( - src, root, contender, conf, + src, + root, + contender, + conf, repetitions=repetitions, - benchmark_filter=benchmark_filter) + benchmark_filter=benchmark_filter, + ) runner_base = JavaBenchmarkRunner.from_rev_or_path( - src, root, baseline, conf, + src, + root, + baseline, + conf, repetitions=repetitions, - benchmark_filter=benchmark_filter) + benchmark_filter=benchmark_filter, + ) runner_comp = RunnerComparator(runner_cont, runner_base, threshold) # TODO(kszucs): test that the output is properly formatted jsonlines comparisons_json = _get_comparisons_as_json(runner_comp.comparisons) ren_counters = language == "java" - formatted = _format_comparisons_with_pandas(comparisons_json, - no_counters, ren_counters) + formatted = _format_comparisons_with_pandas( + comparisons_json, no_counters, ren_counters + ) print(formatted, file=output or sys.stdout) @@ -633,38 +934,42 @@ def _get_comparisons_as_json(comparisons): return buf.getvalue() -def _format_comparisons_with_pandas(comparisons_json, no_counters, - ren_counters): +def _format_comparisons_with_pandas(comparisons_json, no_counters, ren_counters): pd = _import_pandas() df = pd.read_json(StringIO(comparisons_json), lines=True) # parse change % so we can sort by it - df['change %'] = df.pop('change').str[:-1].map(float) - first_regression = len(df) - df['regression'].sum() + df["change %"] = df.pop("change").str[:-1].map(float) + first_regression = len(df) - df["regression"].sum() - fields = ['benchmark', 'baseline', 'contender', 'change %'] + fields = ["benchmark", "baseline", "contender", "change %"] if not no_counters: - fields += ['counters'] + fields += ["counters"] df = df[fields] if ren_counters: - df = df.rename(columns={'counters': 'configurations'}) - df = df.sort_values(by='change %', ascending=False) + df = df.rename(columns={"counters": "configurations"}) + df = df.sort_values(by="change %", ascending=False) def labelled(title, df): if len(df) == 0: - return '' - title += f': ({len(df)})' + return "" + title += f": ({len(df)})" df_str = df.to_string(index=False) - bar = '-' * df_str.index('\n') - return '\n'.join([bar, title, bar, df_str]) + bar = "-" * df_str.index("\n") + return "\n".join([bar, title, bar, df_str]) - return '\n\n'.join([labelled('Non-regressions', df[:first_regression]), - labelled('Regressions', df[first_regression:])]) + return "\n\n".join( + [ + labelled("Non-regressions", df[:first_regression]), + labelled("Regressions", df[first_regression:]), + ] + ) # ---------------------------------------------------------------------- # Integration testing + def _set_default(opt, default): if opt is None: return default @@ -672,65 +977,136 @@ def _set_default(opt, default): @archery.command(short_help="Execute protocol and Flight integration tests") -@click.option('--with-all', is_flag=True, default=False, - help=('Include all known implementations by default ' - 'in integration tests')) -@click.option('--random-seed', type=int, default=12345, - help="Seed for PRNG when generating test data") -@click.option('--with-cpp', type=bool, default=False, - help='Include C++ in integration tests', - envvar="ARCHERY_INTEGRATION_WITH_CPP") -@click.option('--with-dotnet', type=bool, default=False, - help='Include .NET in integration tests', - envvar="ARCHERY_INTEGRATION_WITH_DOTNET") -@click.option('--with-java', type=bool, default=False, - help='Include Java in integration tests', - envvar="ARCHERY_INTEGRATION_WITH_JAVA") -@click.option('--with-js', type=bool, default=False, - help='Include JavaScript in integration tests', - envvar="ARCHERY_INTEGRATION_WITH_JS") -@click.option('--with-go', type=bool, default=False, - help='Include Go in integration tests', - envvar="ARCHERY_INTEGRATION_WITH_GO") -@click.option('--with-nanoarrow', type=bool, default=False, - help='Include nanoarrow in integration tests', - envvar="ARCHERY_INTEGRATION_WITH_NANOARROW") -@click.option('--with-ruby', type=bool, default=False, - help='Include Ruby in integration tests', - envvar="ARCHERY_INTEGRATION_WITH_RUBY") -@click.option('--with-rust', type=bool, default=False, - help='Include Rust in integration tests', - envvar="ARCHERY_INTEGRATION_WITH_RUST") -@click.option('--target-implementations', default='', - help=('Target implementations in this integration tests'), - envvar="ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS") -@click.option('--write-generated-json', default="", - help='Generate test JSON to indicated path') -@click.option('--write-gold-files', default="", - help='Generate gold files to indicated path') -@click.option('--run-ipc', is_flag=True, default=False, - help='Run IPC integration tests') -@click.option('--run-flight', is_flag=True, default=False, - help='Run Flight integration tests') -@click.option('--run-c-data', is_flag=True, default=False, - help='Run C Data Interface integration tests') -@click.option('--debug', is_flag=True, default=False, - help='Run executables in debug mode as relevant') -@click.option('--serial', is_flag=True, default=False, - help='Run tests serially, rather than in parallel') -@click.option('--tempdir', default=None, - help=('Directory to use for writing ' - 'integration test temporary files')) -@click.option('stop_on_error', '-x', '--stop-on-error', - is_flag=True, default=False, - help='Stop on first error') -@click.option('--gold-dirs', multiple=True, - help="gold integration test file paths") -@click.option('-k', '--match', - help=("Substring for test names to include in run, " - "e.g. -k primitive")) -def integration(with_all=False, random_seed=12345, write_generated_json="", - write_gold_files="", **args): +@click.option( + "--with-all", + is_flag=True, + default=False, + help=("Include all known implementations by default in integration tests"), +) +@click.option( + "--random-seed", + type=int, + default=12345, + help="Seed for PRNG when generating test data", +) +@click.option( + "--with-cpp", + type=bool, + default=False, + help="Include C++ in integration tests", + envvar="ARCHERY_INTEGRATION_WITH_CPP", +) +@click.option( + "--with-dotnet", + type=bool, + default=False, + help="Include .NET in integration tests", + envvar="ARCHERY_INTEGRATION_WITH_DOTNET", +) +@click.option( + "--with-java", + type=bool, + default=False, + help="Include Java in integration tests", + envvar="ARCHERY_INTEGRATION_WITH_JAVA", +) +@click.option( + "--with-js", + type=bool, + default=False, + help="Include JavaScript in integration tests", + envvar="ARCHERY_INTEGRATION_WITH_JS", +) +@click.option( + "--with-go", + type=bool, + default=False, + help="Include Go in integration tests", + envvar="ARCHERY_INTEGRATION_WITH_GO", +) +@click.option( + "--with-nanoarrow", + type=bool, + default=False, + help="Include nanoarrow in integration tests", + envvar="ARCHERY_INTEGRATION_WITH_NANOARROW", +) +@click.option( + "--with-ruby", + type=bool, + default=False, + help="Include Ruby in integration tests", + envvar="ARCHERY_INTEGRATION_WITH_RUBY", +) +@click.option( + "--with-rust", + type=bool, + default=False, + help="Include Rust in integration tests", + envvar="ARCHERY_INTEGRATION_WITH_RUST", +) +@click.option( + "--target-implementations", + default="", + help=("Target implementations in this integration tests"), + envvar="ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS", +) +@click.option( + "--write-generated-json", default="", help="Generate test JSON to indicated path" +) +@click.option( + "--write-gold-files", default="", help="Generate gold files to indicated path" +) +@click.option( + "--run-ipc", is_flag=True, default=False, help="Run IPC integration tests" +) +@click.option( + "--run-flight", is_flag=True, default=False, help="Run Flight integration tests" +) +@click.option( + "--run-c-data", + is_flag=True, + default=False, + help="Run C Data Interface integration tests", +) +@click.option( + "--debug", + is_flag=True, + default=False, + help="Run executables in debug mode as relevant", +) +@click.option( + "--serial", + is_flag=True, + default=False, + help="Run tests serially, rather than in parallel", +) +@click.option( + "--tempdir", + default=None, + help=("Directory to use for writing integration test temporary files"), +) +@click.option( + "stop_on_error", + "-x", + "--stop-on-error", + is_flag=True, + default=False, + help="Stop on first error", +) +@click.option("--gold-dirs", multiple=True, help="gold integration test file paths") +@click.option( + "-k", + "--match", + help=("Substring for test names to include in run, e.g. -k primitive"), +) +def integration( + with_all=False, + random_seed=12345, + write_generated_json="", + write_gold_files="", + **args, +): """If you don't specify the "--target-implementations" option nor the "ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS" environment variable, test patterns are product of all specified @@ -789,8 +1165,7 @@ def integration(with_all=False, random_seed=12345, write_generated_json="", """ - from .integration.datagen import ( - get_generated_json_files, generate_gold_files) + from .integration.datagen import get_generated_json_files, generate_gold_files from .integration.runner import run_all_tests, select_testers import numpy as np @@ -800,11 +1175,11 @@ def integration(with_all=False, random_seed=12345, write_generated_json="", # Make runs involving data generation deterministic np.random.seed(random_seed) - formats = ['ipc', 'flight', 'c_data'] + formats = ["ipc", "flight", "c_data"] enabled_formats = 0 for fmt in formats: - param = f'run_{fmt}' + param = f"run_{fmt}" enabled_formats += args[param] testers, other_testers = select_testers(**args) @@ -815,41 +1190,54 @@ def integration(with_all=False, random_seed=12345, write_generated_json="", elif write_gold_files: if len(testers) != 1 or len(other_testers) != 0: raise click.UsageError( - "Need exactly one implementation to generate gold files; try --help") + "Need exactly one implementation to generate gold files; try --help" + ) generate_gold_files(testers[0], write_gold_files) else: if enabled_formats == 0: raise click.UsageError( "Need to enable at least one format to test " - "(IPC, Flight, C Data Interface); try --help") + "(IPC, Flight, C Data Interface); try --help" + ) if len(testers) == 0: raise click.UsageError( - "Need to enable at least one implementation to test; try --help") + "Need to enable at least one implementation to test; try --help" + ) run_all_tests(testers, other_testers, **args) @archery.command() -@click.option('--arrow-token', envvar=['GH_TOKEN', 'ARROW_GITHUB_TOKEN'], - help='OAuth token for responding comment in the arrow repo') -@click.option('--committers-file', '-c', type=click.File('r', encoding='utf8')) -@click.option('--event-name', '-n', required=True) -@click.option('--event-payload', '-p', type=click.File('r', encoding='utf8'), - default='-', required=True) +@click.option( + "--arrow-token", + envvar=["GH_TOKEN", "ARROW_GITHUB_TOKEN"], + help="OAuth token for responding comment in the arrow repo", +) +@click.option("--committers-file", "-c", type=click.File("r", encoding="utf8")) +@click.option("--event-name", "-n", required=True) +@click.option( + "--event-payload", + "-p", + type=click.File("r", encoding="utf8"), + default="-", + required=True, +) def trigger_bot(arrow_token, committers_file, event_name, event_payload): from .bot import CommentBot, PullRequestWorkflowBot, actions from ruamel.yaml import YAML event_payload = json.loads(event_payload.read()) - if 'comment' in event_name: - bot = CommentBot(name='github-actions', handler=actions, token=arrow_token) + if "comment" in event_name: + bot = CommentBot(name="github-actions", handler=actions, token=arrow_token) bot.handle(event_name, event_payload) else: committers = None if committers_file: - committers = [committer['alias'] - for committer in YAML().load(committers_file)] - bot = PullRequestWorkflowBot(event_name, event_payload, token=arrow_token, - committers=committers) + committers = [ + committer["alias"] for committer in YAML().load(committers_file) + ] + bot = PullRequestWorkflowBot( + event_name, event_payload, token=arrow_token, committers=committers + ) bot.handle() @@ -864,10 +1252,16 @@ def linking(obj): @linking.command("check-dependencies") @click.argument("paths", nargs=-1) -@click.option("--allow", "-a", "allowed", multiple=True, - help="Name of the allowed libraries") -@click.option("--disallow", "-d", "disallowed", multiple=True, - help="Name of the disallowed libraries") +@click.option( + "--allow", "-a", "allowed", multiple=True, help="Name of the allowed libraries" +) +@click.option( + "--disallow", + "-d", + "disallowed", + multiple=True, + help="Name of the disallowed libraries", +) @click.pass_obj def linking_check_dependencies(obj, allowed, disallowed, paths): from .linking import check_dynamic_library_dependencies, DependencyError @@ -875,20 +1269,21 @@ def linking_check_dependencies(obj, allowed, disallowed, paths): allowed, disallowed = set(allowed), set(disallowed) try: for path in map(pathlib.Path, paths): - check_dynamic_library_dependencies(path, allowed=allowed, - disallowed=disallowed) + check_dynamic_library_dependencies( + path, allowed=allowed, disallowed=disallowed + ) except DependencyError as e: raise click.ClickException(str(e)) -add_optional_command("docker", module=".docker.cli", function="docker", - parent=archery) -add_optional_command("release", module=".release.cli", function="release", - parent=archery) -add_optional_command("crossbow", module=".crossbow.cli", function="crossbow", - parent=archery) -add_optional_command("ci", module=".ci.cli", function="ci", - parent=archery) +add_optional_command("docker", module=".docker.cli", function="docker", parent=archery) +add_optional_command( + "release", module=".release.cli", function="release", parent=archery +) +add_optional_command( + "crossbow", module=".crossbow.cli", function="crossbow", parent=archery +) +add_optional_command("ci", module=".ci.cli", function="ci", parent=archery) if __name__ == "__main__": From ad8a3ad70fd7734507496b03595eda52c0a167ff Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Wed, 27 May 2026 17:03:13 +0200 Subject: [PATCH 5/5] Revert formatting --- dev/archery/archery/cli.py | 1056 ++++++++++++------------------------ 1 file changed, 344 insertions(+), 712 deletions(-) diff --git a/dev/archery/archery/cli.py b/dev/archery/archery/cli.py index 75b96e1f9153..5a3e2b347f72 100644 --- a/dev/archery/archery/cli.py +++ b/dev/archery/archery/cli.py @@ -17,9 +17,9 @@ from io import StringIO import click +import datetime import json import logging -import datetime import os import pathlib import platform @@ -65,32 +65,16 @@ def _run_metadata(ctx): @click.group(context_settings={"help_option_names": ["-h", "--help"]}) -@click.option( - "--debug", - type=BOOL, - is_flag=True, - default=False, - envvar="ARCHERY_DEBUG", - help="Increase logging with debugging output.", -) -@click.option( - "--pdb", - type=BOOL, - is_flag=True, - default=False, - help="Invoke pdb on uncaught exception.", -) -@click.option( - "-q", - "--quiet", - type=BOOL, - is_flag=True, - default=False, - help="Silence executed commands.", -) +@click.option("--debug", type=BOOL, is_flag=True, default=False, + envvar='ARCHERY_DEBUG', + help="Increase logging with debugging output.") +@click.option("--pdb", type=BOOL, is_flag=True, default=False, + help="Invoke pdb on uncaught exception.") +@click.option("-q", "--quiet", type=BOOL, is_flag=True, default=False, + help="Silence executed commands.") @click.pass_context def archery(ctx, debug, pdb, quiet): - """Apache Arrow developer utilities. + """ Apache Arrow developer utilities. See sub-commands help with `archery --help`. @@ -102,23 +86,23 @@ def archery(ctx, debug, pdb, quiet): if debug: logger.setLevel(logging.DEBUG) - ctx.obj["debug"] = debug + ctx.obj['debug'] = debug if pdb: import pdb - sys.excepthook = lambda t, v, e: pdb.pm() build_dir_type = click.Path(dir_okay=True, file_okay=False, resolve_path=True) # Supported build types -build_type = click.Choice(["debug", "relwithdebinfo", "release"], case_sensitive=False) +build_type = click.Choice(["debug", "relwithdebinfo", "release"], + case_sensitive=False) # Supported warn levels -warn_level_type = click.Choice( - ["everything", "checkin", "production"], case_sensitive=False -) +warn_level_type = click.Choice(["everything", "checkin", "production"], + case_sensitive=False) -simd_level = click.Choice(["NONE", "SSE4_2", "AVX2", "AVX512"], case_sensitive=True) +simd_level = click.Choice(["NONE", "SSE4_2", "AVX2", "AVX512"], + case_sensitive=True) def cpp_toolchain_options(cmd): @@ -126,22 +110,17 @@ def cpp_toolchain_options(cmd): click.option("--cc", metavar="", help="C compiler."), click.option("--cxx", metavar="", help="C++ compiler."), click.option("--cxx-flags", help="C++ compiler flags."), - click.option( - "--cpp-package-prefix", - help=( - "Value to pass for ARROW_PACKAGE_PREFIX and " - "use ARROW_DEPENDENCY_SOURCE=SYSTEM" - ), - ), + click.option("--cpp-package-prefix", + help=("Value to pass for ARROW_PACKAGE_PREFIX and " + "use ARROW_DEPENDENCY_SOURCE=SYSTEM")) ] return _apply_options(cmd, options) def java_toolchain_options(cmd): options = [ - click.option( - "--java-home", metavar="", help="Path to Java Developers Kit." - ), + click.option("--java-home", metavar="", + help="Path to Java Developers Kit."), click.option("--java-options", help="java compiler options."), ] return _apply_options(cmd, options) @@ -154,183 +133,107 @@ def _apply_options(cmd, options): @archery.command(short_help="Initialize an Arrow C++ build") -@click.option( - "--src", - metavar="", - default=None, - callback=validate_arrow_sources, - help="Specify Arrow source directory", -) +@click.option("--src", metavar="", default=None, + callback=validate_arrow_sources, + help="Specify Arrow source directory") # toolchain @cpp_toolchain_options -@click.option( - "--build-type", default=None, type=build_type, help="CMake's CMAKE_BUILD_TYPE" -) -@click.option("--build-static", default=True, type=BOOL, help="Build static libraries") -@click.option("--build-shared", default=True, type=BOOL, help="Build shared libraries") -@click.option("--build-unity", default=True, type=BOOL, help="Use CMAKE_UNITY_BUILD") -@click.option( - "--warn-level", - default="production", - type=warn_level_type, - help="Controls compiler warnings -W(no-)error.", -) -@click.option( - "--simd-level", - default="DEFAULT", - type=simd_level, - help="Toggles ARROW_SIMD_LEVEL option.", -) +@click.option("--build-type", default=None, type=build_type, + help="CMake's CMAKE_BUILD_TYPE") +@click.option("--build-static", default=True, type=BOOL, + help="Build static libraries") +@click.option("--build-shared", default=True, type=BOOL, + help="Build shared libraries") +@click.option("--build-unity", default=True, type=BOOL, + help="Use CMAKE_UNITY_BUILD") +@click.option("--warn-level", default="production", type=warn_level_type, + help="Controls compiler warnings -W(no-)error.") +@click.option("--simd-level", default="DEFAULT", type=simd_level, + help="Toggles ARROW_SIMD_LEVEL option.") # Tests and benchmarks -@click.option("--with-tests", default=True, type=BOOL, help="Build with tests.") -@click.option( - "--with-benchmarks", default=None, type=BOOL, help="Build with benchmarks." -) -@click.option("--with-examples", default=None, type=BOOL, help="Build with examples.") -@click.option( - "--with-integration", - default=None, - type=BOOL, - help="Build with integration test executables.", -) +@click.option("--with-tests", default=True, type=BOOL, + help="Build with tests.") +@click.option("--with-benchmarks", default=None, type=BOOL, + help="Build with benchmarks.") +@click.option("--with-examples", default=None, type=BOOL, + help="Build with examples.") +@click.option("--with-integration", default=None, type=BOOL, + help="Build with integration test executables.") # Static checks -@click.option( - "--use-asan", default=None, type=BOOL, help="Toggle ARROW_USE_ASAN sanitizer." -) -@click.option( - "--use-tsan", default=None, type=BOOL, help="Toggle ARROW_USE_TSAN sanitizer." -) -@click.option( - "--use-ubsan", default=None, type=BOOL, help="Toggle ARROW_USE_UBSAN sanitizer." -) -@click.option("--with-fuzzing", default=None, type=BOOL, help="Toggle ARROW_FUZZING.") +@click.option("--use-asan", default=None, type=BOOL, + help="Toggle ARROW_USE_ASAN sanitizer.") +@click.option("--use-tsan", default=None, type=BOOL, + help="Toggle ARROW_USE_TSAN sanitizer.") +@click.option("--use-ubsan", default=None, type=BOOL, + help="Toggle ARROW_USE_UBSAN sanitizer.") +@click.option("--with-fuzzing", default=None, type=BOOL, + help="Toggle ARROW_FUZZING.") # Components -@click.option( - "--with-compute", default=None, type=BOOL, help="Build the Arrow compute module." -) -@click.option( - "--with-csv", default=None, type=BOOL, help="Build the Arrow CSV parser module." -) -@click.option( - "--with-cuda", default=None, type=BOOL, help="Build the Arrow CUDA extensions." -) -@click.option( - "--with-dataset", default=None, type=BOOL, help="Build the Arrow dataset module." -) -@click.option( - "--with-filesystem", - default=None, - type=BOOL, - help="Build the Arrow filesystem layer.", -) -@click.option( - "--with-flight", default=None, type=BOOL, help="Build with Flight rpc support." -) -@click.option( - "--with-gandiva", - default=None, - type=BOOL, - help="Build with Gandiva expression compiler support.", -) -@click.option( - "--with-gcs", - default=None, - type=BOOL, - help="Build Arrow with Google Cloud Storage (GCS) support.", -) -@click.option( - "--with-hdfs", default=None, type=BOOL, help="Build the Arrow HDFS bridge." -) -@click.option( - "--with-hiveserver2", - default=None, - type=BOOL, - help="Build the HiveServer2 client and arrow adapter.", -) -@click.option( - "--with-ipc", default=None, type=BOOL, help="Build the Arrow IPC extensions." -) -@click.option( - "--with-json", default=None, type=BOOL, help="Build the Arrow JSON parser module." -) -@click.option( - "--with-mimalloc", - default=None, - type=BOOL, - help="Build the Arrow mimalloc based allocator.", -) -@click.option( - "--with-parquet", default=None, type=BOOL, help="Build with Parquet file support." -) -@click.option( - "--with-python", default=None, type=BOOL, help="Build the Arrow CPython extensions." -) -@click.option( - "--with-r", - default=None, - type=BOOL, - help="Build the Arrow R extensions. This is not a CMake option, " - "it will toggle required options", -) -@click.option("--with-s3", default=None, type=BOOL, help="Build Arrow with S3 support.") +@click.option("--with-compute", default=None, type=BOOL, + help="Build the Arrow compute module.") +@click.option("--with-csv", default=None, type=BOOL, + help="Build the Arrow CSV parser module.") +@click.option("--with-cuda", default=None, type=BOOL, + help="Build the Arrow CUDA extensions.") +@click.option("--with-dataset", default=None, type=BOOL, + help="Build the Arrow dataset module.") +@click.option("--with-filesystem", default=None, type=BOOL, + help="Build the Arrow filesystem layer.") +@click.option("--with-flight", default=None, type=BOOL, + help="Build with Flight rpc support.") +@click.option("--with-gandiva", default=None, type=BOOL, + help="Build with Gandiva expression compiler support.") +@click.option("--with-gcs", default=None, type=BOOL, + help="Build Arrow with Google Cloud Storage (GCS) support.") +@click.option("--with-hdfs", default=None, type=BOOL, + help="Build the Arrow HDFS bridge.") +@click.option("--with-hiveserver2", default=None, type=BOOL, + help="Build the HiveServer2 client and arrow adapter.") +@click.option("--with-ipc", default=None, type=BOOL, + help="Build the Arrow IPC extensions.") +@click.option("--with-json", default=None, type=BOOL, + help="Build the Arrow JSON parser module.") +@click.option("--with-mimalloc", default=None, type=BOOL, + help="Build the Arrow mimalloc based allocator.") +@click.option("--with-parquet", default=None, type=BOOL, + help="Build with Parquet file support.") +@click.option("--with-python", default=None, type=BOOL, + help="Build the Arrow CPython extensions.") +@click.option("--with-r", default=None, type=BOOL, + help="Build the Arrow R extensions. This is not a CMake option, " + "it will toggle required options") +@click.option("--with-s3", default=None, type=BOOL, + help="Build Arrow with S3 support.") # Compressions -@click.option( - "--with-brotli", - default=None, - type=BOOL, - help="Build Arrow with brotli compression.", -) -@click.option( - "--with-bz2", default=None, type=BOOL, help="Build Arrow with bz2 compression." -) -@click.option( - "--with-lz4", default=None, type=BOOL, help="Build Arrow with lz4 compression." -) -@click.option( - "--with-snappy", - default=None, - type=BOOL, - help="Build Arrow with snappy compression.", -) -@click.option( - "--with-zlib", default=None, type=BOOL, help="Build Arrow with zlib compression." -) -@click.option( - "--with-zstd", default=None, type=BOOL, help="Build Arrow with zstd compression." -) +@click.option("--with-brotli", default=None, type=BOOL, + help="Build Arrow with brotli compression.") +@click.option("--with-bz2", default=None, type=BOOL, + help="Build Arrow with bz2 compression.") +@click.option("--with-lz4", default=None, type=BOOL, + help="Build Arrow with lz4 compression.") +@click.option("--with-snappy", default=None, type=BOOL, + help="Build Arrow with snappy compression.") +@click.option("--with-zlib", default=None, type=BOOL, + help="Build Arrow with zlib compression.") +@click.option("--with-zstd", default=None, type=BOOL, + help="Build Arrow with zstd compression.") # CMake extra feature -@click.option( - "--cmake-extras", - type=str, - multiple=True, - help="Extra flags/options to pass to cmake invocation. Can be stacked", -) -@click.option( - "--install-prefix", - type=str, - help="Destination directory where files are installed. Expand to" - "CMAKE_INSTALL_PREFIX. Defaults to to $CONDA_PREFIX if the" - "variable exists.", -) +@click.option("--cmake-extras", type=str, multiple=True, + help="Extra flags/options to pass to cmake invocation. " + "Can be stacked") +@click.option("--install-prefix", type=str, + help="Destination directory where files are installed. Expand to" + "CMAKE_INSTALL_PREFIX. Defaults to to $CONDA_PREFIX if the" + "variable exists.") # misc -@click.option( - "-f", - "--force", - type=BOOL, - is_flag=True, - default=False, - help="Delete existing build directory if found.", -) -@click.option( - "--targets", - type=str, - multiple=True, - help="Generator targets to run. Can be stacked.", -) +@click.option("-f", "--force", type=BOOL, is_flag=True, default=False, + help="Delete existing build directory if found.") +@click.option("--targets", type=str, multiple=True, + help="Generator targets to run. Can be stacked.") @click.argument("build_dir", type=build_dir_type) @click.pass_context def build(ctx, src, build_dir, force, targets, **kwargs): - """Initialize a C++ build directory. + """ Initialize a C++ build directory. The build command creates a directory initialized with Arrow's cpp source cmake and configuration. It can also optionally invoke the generator to @@ -366,31 +269,19 @@ def build(ctx, src, build_dir, force, targets, **kwargs): def _flatten_numpydoc_rules(rules): flattened = [] for rule in rules: - flattened.extend(filter(None, rule.split(","))) + flattened.extend(filter(None, rule.split(','))) return flattened @archery.command(short_help="Lint python docstring with NumpyDoc") -@click.argument("symbols", nargs=-1) -@click.option( - "--src", - metavar="", - default=None, - callback=validate_arrow_sources, - help="Specify Arrow source directory", -) -@click.option( - "--allow-rule", - "-a", - multiple=True, - help="Allow only these rules (can be comma-separated)", -) -@click.option( - "--disallow-rule", - "-d", - multiple=True, - help="Disallow these rules (can be comma-separated)", -) +@click.argument('symbols', nargs=-1) +@click.option("--src", metavar="", default=None, + callback=validate_arrow_sources, + help="Specify Arrow source directory") +@click.option("--allow-rule", "-a", multiple=True, + help="Allow only these rules (can be comma-separated)") +@click.option("--disallow-rule", "-d", multiple=True, + help="Disallow these rules (can be comma-separated)") def numpydoc(src, symbols, allow_rule, disallow_rule): """ Pass list of modules or symbols as arguments to restrict the validation. @@ -403,13 +294,11 @@ def numpydoc(src, symbols, allow_rule, disallow_rule): archery numpydoc pyarrow.csv pyarrow.json pyarrow.parquet archery numpydoc pyarrow.array """ - disallow_rule = disallow_rule or {"GL01", "SA01", "EX01", "ES01"} + disallow_rule = disallow_rule or {'GL01', 'SA01', 'EX01', 'ES01'} try: results = python_numpydoc( - symbols, - allow_rules=_flatten_numpydoc_rules(allow_rule), - disallow_rules=_flatten_numpydoc_rules(disallow_rule), - ) + symbols, allow_rules=_flatten_numpydoc_rules(allow_rule), + disallow_rules=_flatten_numpydoc_rules(disallow_rule)) for result in results: result.ok() except LintValidationException: @@ -419,7 +308,7 @@ def numpydoc(src, symbols, allow_rule, disallow_rule): @archery.group() @click.pass_context def benchmark(ctx): - """Arrow benchmarking. + """ Arrow benchmarking. Use the diff sub-command to benchmark revisions, and/or build directories. """ @@ -433,74 +322,35 @@ def check_language(ctx, param, value): return value options = [ - click.option( - "--src", - metavar="", - show_default=True, - default=None, - callback=validate_arrow_sources, - help="Specify Arrow source directory", - ), - click.option( - "--preserve", - type=BOOL, - default=False, - show_default=True, - is_flag=True, - help="Preserve workspace for investigation.", - ), - click.option( - "--preserve-dir", - metavar="", - type=click.Path(file_okay=False, resolve_path=True), - default=None, - help="Parent directory in which to create the preserved " - "workspace. Has no effect without --preserve.", - ), - click.option( - "--output", - metavar="", - type=click.File("w", encoding="utf8"), - default=None, - help="Capture output result into file.", - ), - click.option( - "--language", - metavar="", - type=str, - default="cpp", - show_default=True, - callback=check_language, - help="Specify target language for the benchmark", - ), - click.option( - "--build-extras", - type=str, - multiple=True, - help="Extra flags/options to pass to mvn build. " - "Can be stacked. For language=java", - ), - click.option( - "--benchmark-extras", - type=str, - multiple=True, - help="Extra flags/options to pass to mvn benchmark. " - "Can be stacked. For language=java", - ), - click.option( - "--cmake-extras", - type=str, - multiple=True, - help="Extra flags/options to pass to cmake invocation. " - "Can be stacked. For language=cpp", - ), - click.option( - "--cpp-benchmark-extras", - type=str, - multiple=True, - help="Extra flags/options to pass to C++ benchmark executables. " - "Can be stacked. For language=cpp", - ), + click.option("--src", metavar="", show_default=True, + default=None, callback=validate_arrow_sources, + help="Specify Arrow source directory"), + click.option("--preserve", type=BOOL, default=False, show_default=True, + is_flag=True, + help="Preserve workspace for investigation."), + click.option("--preserve-dir", metavar="", + type=click.Path(file_okay=False, resolve_path=True), + default=None, + help="Parent directory in which to create the preserved " + "workspace. Has no effect without --preserve."), + click.option("--output", metavar="", + type=click.File("w", encoding="utf8"), default=None, + help="Capture output result into file."), + click.option("--language", metavar="", type=str, default="cpp", + show_default=True, callback=check_language, + help="Specify target language for the benchmark"), + click.option("--build-extras", type=str, multiple=True, + help="Extra flags/options to pass to mvn build. " + "Can be stacked. For language=java"), + click.option("--benchmark-extras", type=str, multiple=True, + help="Extra flags/options to pass to mvn benchmark. " + "Can be stacked. For language=java"), + click.option("--cmake-extras", type=str, multiple=True, + help="Extra flags/options to pass to cmake invocation. " + "Can be stacked. For language=cpp"), + click.option("--cpp-benchmark-extras", type=str, multiple=True, + help="Extra flags/options to pass to C++ benchmark executables. " + "Can be stacked. For language=cpp"), ] cmd = java_toolchain_options(cmd) @@ -510,128 +360,72 @@ def check_language(ctx, param, value): def benchmark_filter_options(cmd): options = [ - click.option( - "--suite-filter", - metavar="", - show_default=True, - type=str, - default=None, - help="Regex filtering benchmark suites.", - ), - click.option( - "--benchmark-filter", - metavar="", - show_default=True, - type=str, - default=None, - help="Regex filtering benchmarks.", - ), + click.option("--suite-filter", metavar="", show_default=True, + type=str, default=None, + help="Regex filtering benchmark suites."), + click.option("--benchmark-filter", metavar="", + show_default=True, type=str, default=None, + help="Regex filtering benchmarks.") ] return _apply_options(cmd, options) @benchmark.command(name="list", short_help="List benchmark suite") -@click.argument( - "rev_or_path", metavar="[]", default="WORKSPACE", required=False -) +@click.argument("rev_or_path", metavar="[]", + default="WORKSPACE", required=False) @benchmark_common_options @click.pass_context -def benchmark_list( - ctx, - rev_or_path, - src, - preserve, - preserve_dir, - output, - cmake_extras, - java_home, - java_options, - build_extras, - benchmark_extras, - cpp_benchmark_extras, - language, - **kwargs, -): - """List benchmark suite.""" +def benchmark_list(ctx, rev_or_path, src, preserve, preserve_dir, output, + cmake_extras, java_home, java_options, build_extras, + benchmark_extras, cpp_benchmark_extras, language, **kwargs): + """ List benchmark suite. + """ with tmpdir(preserve=preserve, preserve_dir=preserve_dir) as root: logger.debug(f"Running benchmark {rev_or_path}") if language == "cpp": conf = CppBenchmarkRunner.default_configuration( - cmake_extras=cmake_extras, **kwargs - ) + cmake_extras=cmake_extras, **kwargs) runner_base = CppBenchmarkRunner.from_rev_or_path( - src, root, rev_or_path, conf, benchmark_extras=cpp_benchmark_extras - ) + src, root, rev_or_path, conf, + benchmark_extras=cpp_benchmark_extras) elif language == "java": - for key in {"cpp_package_prefix", "cxx_flags", "cxx", "cc"}: + for key in {'cpp_package_prefix', 'cxx_flags', 'cxx', 'cc'}: del kwargs[key] conf = JavaBenchmarkRunner.default_configuration( - java_home=java_home, - java_options=java_options, - build_extras=build_extras, - benchmark_extras=benchmark_extras, - **kwargs, - ) + java_home=java_home, java_options=java_options, + build_extras=build_extras, benchmark_extras=benchmark_extras, + **kwargs) runner_base = JavaBenchmarkRunner.from_rev_or_path( - src, root, rev_or_path, conf - ) + src, root, rev_or_path, conf) for b in runner_base.list_benchmarks: click.echo(b, file=output or sys.stdout) @benchmark.command(name="run", short_help="Run benchmark suite") -@click.argument( - "rev_or_path", metavar="[]", default="WORKSPACE", required=False -) +@click.argument("rev_or_path", metavar="[]", + default="WORKSPACE", required=False) @benchmark_common_options @benchmark_filter_options -@click.option( - "--repetitions", - type=int, - default=-1, - help=( - "Number of repetitions of each benchmark. Increasing " - "may improve result precision. " - "[default: 1 for cpp, 5 for java]" - ), -) -@click.option( - "--repetition-min-time", - type=float, - default=None, - help=( - "Minimum duration of each repetition in seconds. " - "Currently only supported for language=cpp. " - "[default: use runner-specific defaults]" - ), -) +@click.option("--repetitions", type=int, default=-1, + help=("Number of repetitions of each benchmark. Increasing " + "may improve result precision. " + "[default: 1 for cpp, 5 for java]")) +@click.option("--repetition-min-time", type=float, default=None, + help=("Minimum duration of each repetition in seconds. " + "Currently only supported for language=cpp. " + "[default: use runner-specific defaults]")) @click.pass_context -def benchmark_run( - ctx, - rev_or_path, - src, - preserve, - preserve_dir, - output, - cmake_extras, - java_home, - java_options, - build_extras, - benchmark_extras, - language, - suite_filter, - benchmark_filter, - repetitions, - repetition_min_time, - cpp_benchmark_extras, - **kwargs, -): - """Run benchmark suite. +def benchmark_run(ctx, rev_or_path, src, preserve, preserve_dir, output, + cmake_extras, java_home, java_options, build_extras, + benchmark_extras, language, suite_filter, benchmark_filter, + repetitions, repetition_min_time, cpp_benchmark_extras, + **kwargs): + """ Run benchmark suite. This command will run the benchmark suite for a single build. This is used to capture (and/or publish) the results. @@ -673,52 +467,40 @@ def benchmark_run( if language == "cpp": conf = CppBenchmarkRunner.default_configuration( - cmake_extras=cmake_extras, **kwargs - ) + cmake_extras=cmake_extras, **kwargs) repetitions = repetitions if repetitions != -1 else 1 runner_base = CppBenchmarkRunner.from_rev_or_path( - src, - root, - rev_or_path, - conf, - repetitions=repetitions, - repetition_min_time=repetition_min_time, - suite_filter=suite_filter, - benchmark_filter=benchmark_filter, - benchmark_extras=cpp_benchmark_extras, - ) + src, root, rev_or_path, conf, + repetitions=repetitions, repetition_min_time=repetition_min_time, + suite_filter=suite_filter, benchmark_filter=benchmark_filter, + benchmark_extras=cpp_benchmark_extras) elif language == "java": - for key in {"cpp_package_prefix", "cxx_flags", "cxx", "cc"}: + for key in {'cpp_package_prefix', 'cxx_flags', 'cxx', 'cc'}: del kwargs[key] conf = JavaBenchmarkRunner.default_configuration( - java_home=java_home, - java_options=java_options, - build_extras=build_extras, - benchmark_extras=benchmark_extras, - **kwargs, - ) + java_home=java_home, java_options=java_options, + build_extras=build_extras, benchmark_extras=benchmark_extras, + **kwargs) repetitions = repetitions if repetitions != -1 else 5 runner_base = JavaBenchmarkRunner.from_rev_or_path( - src, - root, - rev_or_path, - conf, + src, root, rev_or_path, conf, repetitions=repetitions, - benchmark_filter=benchmark_filter, - ) + benchmark_filter=benchmark_filter) # XXX for some reason, the benchmark runner only does its work # when asked to JSON-serialize the results, so produce a JSON # output even when none is requested. json_out = json.dumps(runner_base, cls=JsonEncoder) if runner_base.results_dir is not None: - results_path = os.path.join(runner_base.results_dir, "benchmark.json") + results_path = os.path.join(runner_base.results_dir, + "benchmark.json") with open(results_path, "w") as f: f.write(json_out) - metadata_path = os.path.join(runner_base.results_dir, "metadata.json") + metadata_path = os.path.join(runner_base.results_dir, + "metadata.json") with open(metadata_path, "w") as f: json.dump(_run_metadata(ctx), f, indent=2, default=str) if output is not None: @@ -728,60 +510,25 @@ def benchmark_run( @benchmark.command(name="diff", short_help="Compare benchmark suites") @benchmark_common_options @benchmark_filter_options -@click.option( - "--threshold", - type=float, - default=DEFAULT_THRESHOLD, - show_default=True, - help="Regression failure threshold in percentage.", -) -@click.option( - "--repetitions", - type=int, - default=1, - show_default=True, - help=( - "Number of repetitions of each benchmark. Increasing " - "may improve result precision. " - "[default: 1 for cpp, 5 for java" - ), -) -@click.option( - "--no-counters", - type=BOOL, - default=False, - is_flag=True, - help="Hide counters field in diff report.", -) -@click.argument( - "contender", metavar="[", default=ArrowSources.WORKSPACE, required=False -) -@click.argument( - "baseline", metavar="[]]", default="origin/HEAD", required=False -) +@click.option("--threshold", type=float, default=DEFAULT_THRESHOLD, + show_default=True, + help="Regression failure threshold in percentage.") +@click.option("--repetitions", type=int, default=1, show_default=True, + help=("Number of repetitions of each benchmark. Increasing " + "may improve result precision. " + "[default: 1 for cpp, 5 for java")) +@click.option("--no-counters", type=BOOL, default=False, is_flag=True, + help="Hide counters field in diff report.") +@click.argument("contender", metavar="[", + default=ArrowSources.WORKSPACE, required=False) +@click.argument("baseline", metavar="[]]", default="origin/HEAD", + required=False) @click.pass_context -def benchmark_diff( - ctx, - src, - preserve, - preserve_dir, - output, - language, - cmake_extras, - suite_filter, - benchmark_filter, - repetitions, - no_counters, - java_home, - java_options, - build_extras, - benchmark_extras, - cpp_benchmark_extras, - threshold, - contender, - baseline, - **kwargs, -): +def benchmark_diff(ctx, src, preserve, preserve_dir, output, language, + cmake_extras, suite_filter, benchmark_filter, repetitions, + no_counters, java_home, java_options, build_extras, + benchmark_extras, cpp_benchmark_extras, threshold, + contender, baseline, **kwargs): """Compare (diff) benchmark runs. This command acts like git-diff but for benchmark results. @@ -860,68 +607,47 @@ def benchmark_diff( if language == "cpp": conf = CppBenchmarkRunner.default_configuration( - cmake_extras=cmake_extras, **kwargs - ) + cmake_extras=cmake_extras, **kwargs) repetitions = repetitions if repetitions != -1 else 1 runner_cont = CppBenchmarkRunner.from_rev_or_path( - src, - root, - contender, - conf, + src, root, contender, conf, repetitions=repetitions, suite_filter=suite_filter, benchmark_filter=benchmark_filter, - benchmark_extras=cpp_benchmark_extras, - ) + benchmark_extras=cpp_benchmark_extras) runner_base = CppBenchmarkRunner.from_rev_or_path( - src, - root, - baseline, - conf, + src, root, baseline, conf, repetitions=repetitions, suite_filter=suite_filter, benchmark_filter=benchmark_filter, - benchmark_extras=cpp_benchmark_extras, - ) + benchmark_extras=cpp_benchmark_extras) elif language == "java": - for key in {"cpp_package_prefix", "cxx_flags", "cxx", "cc"}: + for key in {'cpp_package_prefix', 'cxx_flags', 'cxx', 'cc'}: del kwargs[key] conf = JavaBenchmarkRunner.default_configuration( - java_home=java_home, - java_options=java_options, - build_extras=build_extras, - benchmark_extras=benchmark_extras, - **kwargs, - ) + java_home=java_home, java_options=java_options, + build_extras=build_extras, benchmark_extras=benchmark_extras, + **kwargs) repetitions = repetitions if repetitions != -1 else 5 runner_cont = JavaBenchmarkRunner.from_rev_or_path( - src, - root, - contender, - conf, + src, root, contender, conf, repetitions=repetitions, - benchmark_filter=benchmark_filter, - ) + benchmark_filter=benchmark_filter) runner_base = JavaBenchmarkRunner.from_rev_or_path( - src, - root, - baseline, - conf, + src, root, baseline, conf, repetitions=repetitions, - benchmark_filter=benchmark_filter, - ) + benchmark_filter=benchmark_filter) runner_comp = RunnerComparator(runner_cont, runner_base, threshold) # TODO(kszucs): test that the output is properly formatted jsonlines comparisons_json = _get_comparisons_as_json(runner_comp.comparisons) ren_counters = language == "java" - formatted = _format_comparisons_with_pandas( - comparisons_json, no_counters, ren_counters - ) + formatted = _format_comparisons_with_pandas(comparisons_json, + no_counters, ren_counters) print(formatted, file=output or sys.stdout) @@ -934,42 +660,38 @@ def _get_comparisons_as_json(comparisons): return buf.getvalue() -def _format_comparisons_with_pandas(comparisons_json, no_counters, ren_counters): +def _format_comparisons_with_pandas(comparisons_json, no_counters, + ren_counters): pd = _import_pandas() df = pd.read_json(StringIO(comparisons_json), lines=True) # parse change % so we can sort by it - df["change %"] = df.pop("change").str[:-1].map(float) - first_regression = len(df) - df["regression"].sum() + df['change %'] = df.pop('change').str[:-1].map(float) + first_regression = len(df) - df['regression'].sum() - fields = ["benchmark", "baseline", "contender", "change %"] + fields = ['benchmark', 'baseline', 'contender', 'change %'] if not no_counters: - fields += ["counters"] + fields += ['counters'] df = df[fields] if ren_counters: - df = df.rename(columns={"counters": "configurations"}) - df = df.sort_values(by="change %", ascending=False) + df = df.rename(columns={'counters': 'configurations'}) + df = df.sort_values(by='change %', ascending=False) def labelled(title, df): if len(df) == 0: - return "" - title += f": ({len(df)})" + return '' + title += f': ({len(df)})' df_str = df.to_string(index=False) - bar = "-" * df_str.index("\n") - return "\n".join([bar, title, bar, df_str]) + bar = '-' * df_str.index('\n') + return '\n'.join([bar, title, bar, df_str]) - return "\n\n".join( - [ - labelled("Non-regressions", df[:first_regression]), - labelled("Regressions", df[first_regression:]), - ] - ) + return '\n\n'.join([labelled('Non-regressions', df[:first_regression]), + labelled('Regressions', df[first_regression:])]) # ---------------------------------------------------------------------- # Integration testing - def _set_default(opt, default): if opt is None: return default @@ -977,136 +699,65 @@ def _set_default(opt, default): @archery.command(short_help="Execute protocol and Flight integration tests") -@click.option( - "--with-all", - is_flag=True, - default=False, - help=("Include all known implementations by default in integration tests"), -) -@click.option( - "--random-seed", - type=int, - default=12345, - help="Seed for PRNG when generating test data", -) -@click.option( - "--with-cpp", - type=bool, - default=False, - help="Include C++ in integration tests", - envvar="ARCHERY_INTEGRATION_WITH_CPP", -) -@click.option( - "--with-dotnet", - type=bool, - default=False, - help="Include .NET in integration tests", - envvar="ARCHERY_INTEGRATION_WITH_DOTNET", -) -@click.option( - "--with-java", - type=bool, - default=False, - help="Include Java in integration tests", - envvar="ARCHERY_INTEGRATION_WITH_JAVA", -) -@click.option( - "--with-js", - type=bool, - default=False, - help="Include JavaScript in integration tests", - envvar="ARCHERY_INTEGRATION_WITH_JS", -) -@click.option( - "--with-go", - type=bool, - default=False, - help="Include Go in integration tests", - envvar="ARCHERY_INTEGRATION_WITH_GO", -) -@click.option( - "--with-nanoarrow", - type=bool, - default=False, - help="Include nanoarrow in integration tests", - envvar="ARCHERY_INTEGRATION_WITH_NANOARROW", -) -@click.option( - "--with-ruby", - type=bool, - default=False, - help="Include Ruby in integration tests", - envvar="ARCHERY_INTEGRATION_WITH_RUBY", -) -@click.option( - "--with-rust", - type=bool, - default=False, - help="Include Rust in integration tests", - envvar="ARCHERY_INTEGRATION_WITH_RUST", -) -@click.option( - "--target-implementations", - default="", - help=("Target implementations in this integration tests"), - envvar="ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS", -) -@click.option( - "--write-generated-json", default="", help="Generate test JSON to indicated path" -) -@click.option( - "--write-gold-files", default="", help="Generate gold files to indicated path" -) -@click.option( - "--run-ipc", is_flag=True, default=False, help="Run IPC integration tests" -) -@click.option( - "--run-flight", is_flag=True, default=False, help="Run Flight integration tests" -) -@click.option( - "--run-c-data", - is_flag=True, - default=False, - help="Run C Data Interface integration tests", -) -@click.option( - "--debug", - is_flag=True, - default=False, - help="Run executables in debug mode as relevant", -) -@click.option( - "--serial", - is_flag=True, - default=False, - help="Run tests serially, rather than in parallel", -) -@click.option( - "--tempdir", - default=None, - help=("Directory to use for writing integration test temporary files"), -) -@click.option( - "stop_on_error", - "-x", - "--stop-on-error", - is_flag=True, - default=False, - help="Stop on first error", -) -@click.option("--gold-dirs", multiple=True, help="gold integration test file paths") -@click.option( - "-k", - "--match", - help=("Substring for test names to include in run, e.g. -k primitive"), -) -def integration( - with_all=False, - random_seed=12345, - write_generated_json="", - write_gold_files="", - **args, -): +@click.option('--with-all', is_flag=True, default=False, + help=('Include all known implementations by default ' + 'in integration tests')) +@click.option('--random-seed', type=int, default=12345, + help="Seed for PRNG when generating test data") +@click.option('--with-cpp', type=bool, default=False, + help='Include C++ in integration tests', + envvar="ARCHERY_INTEGRATION_WITH_CPP") +@click.option('--with-dotnet', type=bool, default=False, + help='Include .NET in integration tests', + envvar="ARCHERY_INTEGRATION_WITH_DOTNET") +@click.option('--with-java', type=bool, default=False, + help='Include Java in integration tests', + envvar="ARCHERY_INTEGRATION_WITH_JAVA") +@click.option('--with-js', type=bool, default=False, + help='Include JavaScript in integration tests', + envvar="ARCHERY_INTEGRATION_WITH_JS") +@click.option('--with-go', type=bool, default=False, + help='Include Go in integration tests', + envvar="ARCHERY_INTEGRATION_WITH_GO") +@click.option('--with-nanoarrow', type=bool, default=False, + help='Include nanoarrow in integration tests', + envvar="ARCHERY_INTEGRATION_WITH_NANOARROW") +@click.option('--with-ruby', type=bool, default=False, + help='Include Ruby in integration tests', + envvar="ARCHERY_INTEGRATION_WITH_RUBY") +@click.option('--with-rust', type=bool, default=False, + help='Include Rust in integration tests', + envvar="ARCHERY_INTEGRATION_WITH_RUST") +@click.option('--target-implementations', default='', + help=('Target implementations in this integration tests'), + envvar="ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS") +@click.option('--write-generated-json', default="", + help='Generate test JSON to indicated path') +@click.option('--write-gold-files', default="", + help='Generate gold files to indicated path') +@click.option('--run-ipc', is_flag=True, default=False, + help='Run IPC integration tests') +@click.option('--run-flight', is_flag=True, default=False, + help='Run Flight integration tests') +@click.option('--run-c-data', is_flag=True, default=False, + help='Run C Data Interface integration tests') +@click.option('--debug', is_flag=True, default=False, + help='Run executables in debug mode as relevant') +@click.option('--serial', is_flag=True, default=False, + help='Run tests serially, rather than in parallel') +@click.option('--tempdir', default=None, + help=('Directory to use for writing ' + 'integration test temporary files')) +@click.option('stop_on_error', '-x', '--stop-on-error', + is_flag=True, default=False, + help='Stop on first error') +@click.option('--gold-dirs', multiple=True, + help="gold integration test file paths") +@click.option('-k', '--match', + help=("Substring for test names to include in run, " + "e.g. -k primitive")) +def integration(with_all=False, random_seed=12345, write_generated_json="", + write_gold_files="", **args): """If you don't specify the "--target-implementations" option nor the "ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS" environment variable, test patterns are product of all specified @@ -1165,7 +816,8 @@ def integration( """ - from .integration.datagen import get_generated_json_files, generate_gold_files + from .integration.datagen import ( + get_generated_json_files, generate_gold_files) from .integration.runner import run_all_tests, select_testers import numpy as np @@ -1175,11 +827,11 @@ def integration( # Make runs involving data generation deterministic np.random.seed(random_seed) - formats = ["ipc", "flight", "c_data"] + formats = ['ipc', 'flight', 'c_data'] enabled_formats = 0 for fmt in formats: - param = f"run_{fmt}" + param = f'run_{fmt}' enabled_formats += args[param] testers, other_testers = select_testers(**args) @@ -1190,54 +842,41 @@ def integration( elif write_gold_files: if len(testers) != 1 or len(other_testers) != 0: raise click.UsageError( - "Need exactly one implementation to generate gold files; try --help" - ) + "Need exactly one implementation to generate gold files; try --help") generate_gold_files(testers[0], write_gold_files) else: if enabled_formats == 0: raise click.UsageError( "Need to enable at least one format to test " - "(IPC, Flight, C Data Interface); try --help" - ) + "(IPC, Flight, C Data Interface); try --help") if len(testers) == 0: raise click.UsageError( - "Need to enable at least one implementation to test; try --help" - ) + "Need to enable at least one implementation to test; try --help") run_all_tests(testers, other_testers, **args) @archery.command() -@click.option( - "--arrow-token", - envvar=["GH_TOKEN", "ARROW_GITHUB_TOKEN"], - help="OAuth token for responding comment in the arrow repo", -) -@click.option("--committers-file", "-c", type=click.File("r", encoding="utf8")) -@click.option("--event-name", "-n", required=True) -@click.option( - "--event-payload", - "-p", - type=click.File("r", encoding="utf8"), - default="-", - required=True, -) +@click.option('--arrow-token', envvar=['GH_TOKEN', 'ARROW_GITHUB_TOKEN'], + help='OAuth token for responding comment in the arrow repo') +@click.option('--committers-file', '-c', type=click.File('r', encoding='utf8')) +@click.option('--event-name', '-n', required=True) +@click.option('--event-payload', '-p', type=click.File('r', encoding='utf8'), + default='-', required=True) def trigger_bot(arrow_token, committers_file, event_name, event_payload): from .bot import CommentBot, PullRequestWorkflowBot, actions from ruamel.yaml import YAML event_payload = json.loads(event_payload.read()) - if "comment" in event_name: - bot = CommentBot(name="github-actions", handler=actions, token=arrow_token) + if 'comment' in event_name: + bot = CommentBot(name='github-actions', handler=actions, token=arrow_token) bot.handle(event_name, event_payload) else: committers = None if committers_file: - committers = [ - committer["alias"] for committer in YAML().load(committers_file) - ] - bot = PullRequestWorkflowBot( - event_name, event_payload, token=arrow_token, committers=committers - ) + committers = [committer['alias'] + for committer in YAML().load(committers_file)] + bot = PullRequestWorkflowBot(event_name, event_payload, token=arrow_token, + committers=committers) bot.handle() @@ -1252,16 +891,10 @@ def linking(obj): @linking.command("check-dependencies") @click.argument("paths", nargs=-1) -@click.option( - "--allow", "-a", "allowed", multiple=True, help="Name of the allowed libraries" -) -@click.option( - "--disallow", - "-d", - "disallowed", - multiple=True, - help="Name of the disallowed libraries", -) +@click.option("--allow", "-a", "allowed", multiple=True, + help="Name of the allowed libraries") +@click.option("--disallow", "-d", "disallowed", multiple=True, + help="Name of the disallowed libraries") @click.pass_obj def linking_check_dependencies(obj, allowed, disallowed, paths): from .linking import check_dynamic_library_dependencies, DependencyError @@ -1269,21 +902,20 @@ def linking_check_dependencies(obj, allowed, disallowed, paths): allowed, disallowed = set(allowed), set(disallowed) try: for path in map(pathlib.Path, paths): - check_dynamic_library_dependencies( - path, allowed=allowed, disallowed=disallowed - ) + check_dynamic_library_dependencies(path, allowed=allowed, + disallowed=disallowed) except DependencyError as e: raise click.ClickException(str(e)) -add_optional_command("docker", module=".docker.cli", function="docker", parent=archery) -add_optional_command( - "release", module=".release.cli", function="release", parent=archery -) -add_optional_command( - "crossbow", module=".crossbow.cli", function="crossbow", parent=archery -) -add_optional_command("ci", module=".ci.cli", function="ci", parent=archery) +add_optional_command("docker", module=".docker.cli", function="docker", + parent=archery) +add_optional_command("release", module=".release.cli", function="release", + parent=archery) +add_optional_command("crossbow", module=".crossbow.cli", function="crossbow", + parent=archery) +add_optional_command("ci", module=".ci.cli", function="ci", + parent=archery) if __name__ == "__main__":