Merge pull request #21683 from github/tausbn/python-add-extractor-pack-build-script

Python: Add `create-extractor-pack.sh` for Python

Author: tausbn

python/scripts/create-extractor-pack.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+#
+# Build a local Python extractor pack from source.
+#
+# Usage with the CodeQL CLI (run from the repository root):
+#
+#   codeql database create <db> -l python -s <src> --search-path .
+#   codeql test run --search-path . python/ql/test/<test-dir>
+#
+set -eux
+
+if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+  platform="linux64"
+elif [[ "$OSTYPE" == "darwin"* ]]; then
+  platform="osx64"
+else
+  echo "Unknown OS"
+  exit 1
+fi
+
+cd "$(dirname "$0")/.."
+
+# Build the tsg-python Rust binary
+(cd extractor/tsg-python && cargo build --release)
+tsg_bin="extractor/tsg-python/target/release/tsg-python"
+
+# Generate python3src.zip from the Python extractor source.
+# make_zips.py creates the zip in the source directory and then copies it to the
+# given output directory. We use a temporary directory to avoid a same-file copy
+# error, then move the zip back.
+tmpdir=$(mktemp -d)
+trap 'rm -rf "$tmpdir"' EXIT
+(cd extractor && python3 make_zips.py "$tmpdir")
+cp "$tmpdir/python3src.zip" extractor/python3src.zip
+
+# Assemble the extractor pack
+rm -rf extractor-pack
+mkdir -p extractor-pack/tools/${platform}
+
+# Root-level metadata and schema files
+cp codeql-extractor.yml extractor-pack/
+cp ql/lib/semmlecode.python.dbscheme extractor-pack/
+cp ql/lib/semmlecode.python.dbscheme.stats extractor-pack/
+
+# Python extractor engine files (into tools/)
+cp extractor/python_tracer.py extractor-pack/tools/
+cp extractor/index.py extractor-pack/tools/
+cp extractor/setup.py extractor-pack/tools/
+cp extractor/convert_setup.py extractor-pack/tools/
+cp extractor/get_venv_lib.py extractor-pack/tools/
+cp extractor/imp.py extractor-pack/tools/
+cp extractor/LICENSE-PSF.md extractor-pack/tools/
+cp extractor/python3src.zip extractor-pack/tools/
+cp -r extractor/data extractor-pack/tools/
+
+# Shell tool scripts (autobuild, pre-finalize, lgtm-scripts)
+cp tools/autobuild.sh extractor-pack/tools/
+cp tools/autobuild.cmd extractor-pack/tools/
+cp tools/pre-finalize.sh extractor-pack/tools/
+cp tools/pre-finalize.cmd extractor-pack/tools/
+cp -r tools/lgtm-scripts extractor-pack/tools/
+
+# Downgrades
+cp -r downgrades extractor-pack/
+
+# Platform-specific Rust binary
+cp "${tsg_bin}" extractor-pack/tools/${platform}/tsg-python