diff --git a/python/scripts/create-extractor-pack.sh b/python/scripts/create-extractor-pack.sh new file mode 100755 index 000000000000..ae9248f6ccfc --- /dev/null +++ b/python/scripts/create-extractor-pack.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# +# Build a local Python extractor pack from source. +# +# Usage with the CodeQL CLI (run from the repository root): +# +# codeql database create -l python -s --search-path . +# codeql test run --search-path . python/ql/test/ +# +set -eux + +if [[ "$OSTYPE" == "linux-gnu"* ]]; then + platform="linux64" +elif [[ "$OSTYPE" == "darwin"* ]]; then + platform="osx64" +else + echo "Unknown OS" + exit 1 +fi + +cd "$(dirname "$0")/.." + +# Build the tsg-python Rust binary +(cd extractor/tsg-python && cargo build --release) +tsg_bin="extractor/tsg-python/target/release/tsg-python" + +# Generate python3src.zip from the Python extractor source. +# make_zips.py creates the zip in the source directory and then copies it to the +# given output directory. We use a temporary directory to avoid a same-file copy +# error, then move the zip back. +tmpdir=$(mktemp -d) +trap 'rm -rf "$tmpdir"' EXIT +(cd extractor && python3 make_zips.py "$tmpdir") +cp "$tmpdir/python3src.zip" extractor/python3src.zip + +# Assemble the extractor pack +rm -rf extractor-pack +mkdir -p extractor-pack/tools/${platform} + +# Root-level metadata and schema files +cp codeql-extractor.yml extractor-pack/ +cp ql/lib/semmlecode.python.dbscheme extractor-pack/ +cp ql/lib/semmlecode.python.dbscheme.stats extractor-pack/ + +# Python extractor engine files (into tools/) +cp extractor/python_tracer.py extractor-pack/tools/ +cp extractor/index.py extractor-pack/tools/ +cp extractor/setup.py extractor-pack/tools/ +cp extractor/convert_setup.py extractor-pack/tools/ +cp extractor/get_venv_lib.py extractor-pack/tools/ +cp extractor/imp.py extractor-pack/tools/ +cp extractor/LICENSE-PSF.md extractor-pack/tools/ +cp extractor/python3src.zip extractor-pack/tools/ +cp -r extractor/data extractor-pack/tools/ + +# Shell tool scripts (autobuild, pre-finalize, lgtm-scripts) +cp tools/autobuild.sh extractor-pack/tools/ +cp tools/autobuild.cmd extractor-pack/tools/ +cp tools/pre-finalize.sh extractor-pack/tools/ +cp tools/pre-finalize.cmd extractor-pack/tools/ +cp -r tools/lgtm-scripts extractor-pack/tools/ + +# Downgrades +cp -r downgrades extractor-pack/ + +# Platform-specific Rust binary +cp "${tsg_bin}" extractor-pack/tools/${platform}/tsg-python