Release 2026.3 (#512)

.gitignore

@@ -35,6 +35,7 @@ MANIFEST
 # Environments
 .env
 .venv
+docker/.venv
 env/
 venv/
 ENV/

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 default_language_version:
   python: python3.11
 
-exclude: 'examples|lib|test/fixtures/.*\.json'
+exclude: ^(examples/|lib/|tools/|test/fixtures/)
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks

IDEAS.mk

@@ -26,7 +26,7 @@ export EXTRACT_INFO_CMAKE CFLAGS
 GIT = git -C ${TRANSLATION_DIR}
 
 TEST_FILES := $(realpath $(wildcard test_vectors/*.json))
-TARGETS ?= $(shell find build-ninja -maxdepth 1 -type f -executable -exec basename {} \; | cut -d. -f1 | sed -e "s/^lib//gi")
+TARGETS ?= $(shell [ -d build-ninja ] && find build-ninja -maxdepth 1 -type f -executable -exec basename {} \; | cut -d. -f1 | sed -e "s/^lib//gi")
 ifeq (${TARGETS},)
 ifeq ($(filter cmake clean,$(MAKECMDGOALS)),)
 $(error No TARGETS found! You need to run cmake!)
@@ -39,8 +39,7 @@ endif
 cmake: build-ninja/cmake.log
 
 build-ninja/cmake.log: test_case/CMakeLists.txt ${EXTRACT_INFO_CMAKE}
-	uv run python -m ideas.cmake source_dir=test_case \
-      build_dir=build-ninja
+	uv run python -m ideas.cmake source_dir=test_case build_dir=build-ninja
 	@touch $@
 
 build-ninja/CMakeCache.txt: build-ninja/cmake.log
@@ -53,50 +52,58 @@ init: $(patsubst %,${TRANSLATION_DIR}/%/init,${TARGETS}) ;
 ${TRANSLATION_DIR}/%/init: ${TRANSLATION_DIR}/%/src/lib.c | build-ninja/lib%.so.type ;
 ${TRANSLATION_DIR}/%/init: ${TRANSLATION_DIR}/%/src/main.c | build-ninja/%.type ;
 
-# FIXME: It would be really nice if we could just check out a branch here if the repo already exists and start from there
-.PRECIOUS: ${TRANSLATION_DIR}/.git/config
-${TRANSLATION_DIR}/.git/config:
+# initialize workspace
+.PRECIOUS: ${TRANSLATION_DIR}/Cargo.toml
+${TRANSLATION_DIR}/Cargo.toml:
 	@mkdir -p ${TRANSLATION_DIR}
 	${GIT} init --quiet --initial-branch=main
 	echo "Cargo.lock\ntarget/\n*.log\n*.jsonl" > ${TRANSLATION_DIR}/.gitignore
 	${GIT} add .gitignore
 	${GIT} commit --quiet --all --message "Initial commit"
-
-.PRECIOUS: ${TRANSLATION_DIR}/Cargo.toml
-${TRANSLATION_DIR}/Cargo.toml: | ${TRANSLATION_DIR}/.git/config
 	echo -n "[workspace]\nresolver = \"3\"" > $@
 	${GIT} add Cargo.toml
 	${GIT} commit --quiet --all --message "Created cargo workspace"
 
+# initialize translated crate for each C target
+.PRECIOUS: ${TRANSLATION_DIR}/%/Cargo.toml
+${TRANSLATION_DIR}/%/Cargo.toml: | ${TRANSLATION_DIR}/Cargo.toml build-ninja/lib%.so.type
+	uv run python -m ideas.init.crate crate_type=lib vcs=git \
+                            hydra.output_subdir=.init \
+                            hydra.run.dir=${TRANSLATION_DIR}/$*
+
+.PRECIOUS: ${TRANSLATION_DIR}/%/Cargo.toml
+${TRANSLATION_DIR}/%/Cargo.toml: | ${TRANSLATION_DIR}/Cargo.toml build-ninja/%.type
+	uv run python -m ideas.init.crate crate_type=bin vcs=git \
+                            hydra.output_subdir=.init \
+                            hydra.run.dir=${TRANSLATION_DIR}/$*
+
+# consolidate each C target
 .PRECIOUS: ${TRANSLATION_DIR}/%/src/lib.c
-${TRANSLATION_DIR}/%/Cargo.toml ${TRANSLATION_DIR}/%/src/lib.c &: | ${TRANSLATION_DIR}/Cargo.toml build-ninja/lib%.so.type
-	uv run python -m ideas.init filename=build-ninja/compile_commands.json \
-                            crate_type=lib \
+${TRANSLATION_DIR}/%/src/lib.c: | ${TRANSLATION_DIR}/%/Cargo.toml build-ninja/lib%.so.type
+	uv run python -m ideas.init.consolidate filename=build-ninja/compile_commands.json \
+                            cargo_toml=${TRANSLATION_DIR}/$*/Cargo.toml \
                             export_symbols=build-ninja/lib$*.so.symbols \
                             source_priority=build-ninja/lib$*.so.sources \
-                            vcs=git \
-                            hydra.output_subdir=.init \
+                            hydra.output_subdir=.init.consolidate \
                             hydra.run.dir=${TRANSLATION_DIR}/$*
 
 .PRECIOUS: ${TRANSLATION_DIR}/%/src/main.c
-${TRANSLATION_DIR}/%/Cargo.toml ${TRANSLATION_DIR}/%/src/main.c &: | ${TRANSLATION_DIR}/Cargo.toml build-ninja/%.type
-	uv run python -m ideas.init filename=build-ninja/compile_commands.json \
-                            crate_type=bin \
+${TRANSLATION_DIR}/%/src/main.c: | ${TRANSLATION_DIR}/%/Cargo.toml build-ninja/%.type
+	uv run python -m ideas.init.consolidate filename=build-ninja/compile_commands.json \
+                            cargo_toml=${TRANSLATION_DIR}/$*/Cargo.toml \
                             export_symbols=build-ninja/$*.symbols \
                             source_priority=build-ninja/$*.sources \
-                            vcs=git \
-                            hydra.output_subdir=.init \
+                            hydra.output_subdir=.init.consolidate \
                             hydra.run.dir=${TRANSLATION_DIR}/$*
 
-
 # translate
 .PHONY: translate
 translate: $(patsubst %,${TRANSLATION_DIR}/%/translate,${TARGETS}) ;
 ${TRANSLATION_DIR}/%/translate: ${TRANSLATION_DIR}/%/src/lib.rs | build-ninja/lib%.so.type ;
 ${TRANSLATION_DIR}/%/translate: ${TRANSLATION_DIR}/%/src/main.rs | build-ninja/%.type ;
 
 .PRECIOUS: ${TRANSLATION_DIR}/%/src/lib.rs
-${TRANSLATION_DIR}/%/src/lib.rs: ${TRANSLATION_DIR}/%/src/lib.c ${TRANSLATION_DIR}/%/tests/test_cases.rs | build-ninja/compile_commands.json build-ninja/lib%.so.symbols build-ninja/lib%.so.sources
+${TRANSLATION_DIR}/%/src/lib.rs: ${TRANSLATION_DIR}/%/src/lib.c | build-ninja/compile_commands.json build-ninja/lib%.so.symbols build-ninja/lib%.so.sources
 	-uv run python -m ideas.translate model.name=${PROVIDER}/${MODEL} \
                                  filename=${TRANSLATION_DIR}/$*/src/lib.c \
                                  vcs=git \
@@ -105,7 +112,7 @@ ${TRANSLATION_DIR}/%/src/lib.rs: ${TRANSLATION_DIR}/%/src/lib.c ${TRANSLATION_DI
                                  hydra.run.dir=${TRANSLATION_DIR}/$* ${TRANSLATE_ARGS}
 
 .PRECIOUS: ${TRANSLATION_DIR}/%/src/main.rs
-${TRANSLATION_DIR}/%/src/main.rs: ${TRANSLATION_DIR}/%/src/main.c ${TRANSLATION_DIR}/%/tests/test_cases.rs | build-ninja/compile_commands.json build-ninja/%.symbols build-ninja/%.sources
+${TRANSLATION_DIR}/%/src/main.rs: ${TRANSLATION_DIR}/%/src/main.c | build-ninja/compile_commands.json build-ninja/%.symbols build-ninja/%.sources
 	-uv run python -m ideas.translate model.name=${PROVIDER}/${MODEL} \
                                  filename=${TRANSLATION_DIR}/$*/src/main.c \
                                  vcs=git \
@@ -162,12 +169,17 @@ ${TRANSLATION_DIR}/%/cargo_test.log: ${TRANSLATION_DIR}/%/build.log ${TRANSLATIO
     fi \
 
 .PRECIOUS: ${TRANSLATION_DIR}/%/tests/test_cases.rs
+${TRANSLATION_DIR}/%/tests/test_cases.rs: | ${TEST_FILES} ${TRANSLATION_DIR}/%/Cargo.toml runner/Cargo.toml build-ninja/lib%.so.type
+	-uv run python -m ideas.convert_tests runner_manifest=$(realpath runner/Cargo.toml) \
+                                     crate_manifest=$(realpath ${TRANSLATION_DIR}/$*/Cargo.toml) \
+                                     template=${MAKEFILE_DIR}/tools/rust_tests/lib_testing.rs \
+                                     output=${TRANSLATION_DIR}/$*/tests/test_cases.rs \
+                                     'test_vectors=[$(shell echo "$(TEST_FILES)" | tr ' ' ',')]'
+
 ${TRANSLATION_DIR}/%/tests/test_cases.rs: | ${TEST_FILES} ${TRANSLATION_DIR}/%/Cargo.toml build-ninja/%.type
-	@mkdir -p $(@D)
-	-uv run python -m ideas.convert_tests --crate_manifest $(realpath ${TRANSLATION_DIR}/$*/Cargo.toml) \
-                                     ${TEST_FILES} | rustfmt > $@
-	${GIT} add $*/Cargo.toml $*/tests/test_cases.rs
-	${GIT} commit --quiet --message "Converted \`$*\` test vectors"
+	-uv run python -m ideas.convert_tests crate_manifest=$(realpath ${TRANSLATION_DIR}/$*/Cargo.toml) \
+                                     output=${TRANSLATION_DIR}/$*/tests/test_cases.rs \
+                                     'test_vectors=[$(shell echo "$(TEST_FILES)" | tr ' ' ',')]'
 
 .PRECIOUS: test_vectors/%.json
 test_vectors/%.json:

Makefile

@@ -19,15 +19,11 @@ VLLM_VERSION ?= 0.13.0
 VLLM_ARGS ?= --tensor-parallel-size 8 --enable-expert-parallel --max-num-seqs 32 --max-model-len 128k## Args to pass to vllm serve
 TRANSLATION_DIR ?= translation.$(shell git rev-parse HEAD)## Directory to put IDEAS translation
 TRANSLATE_ARGS ?= ## Args to pass to IDEAS translation
-TESTGEN_DIR ?= testgen.$(shell git rev-parse HEAD)## Directory to put IDEAS test generation
-TESTGEN_ARGS ?= ## Args to pass to IDEAS test generation
 RUSTFLAGS ?= -Awarnings## Flags to build Rust translation
 VERBOSE ?= 0## Whether to output failed/partial projects in summaries
 
-AFL_TAG = aflplusplus/aflplusplus:stable
-
 # Pass these variables to IDEAS.mk
-export MODEL BASE_URL TRANSLATION_DIR TESTGEN_DIR RUSTFLAGS
+export MODEL BASE_URL TRANSLATION_DIR RUSTFLAGS
 
 EXAMPLES ?= $(sort $(patsubst %/test_case,%,$(shell find ${EXAMPLES_DIR} -maxdepth 3 -name test_case -type d)))## List of examples to run on
 ifeq ($(EXAMPLES),)
@@ -37,8 +33,43 @@ endif
 
 all: help ;
 
+.PHONY: docker/build
+docker/build:## Build translation Docker image
+docker/build: docker/docker_build.log
+
+.PRECIOUS: docker/docker_build.log
+docker/docker_build.log: docker/ideas.Dockerfile
+	rm -rf docker/.venv
+	cd docker && \
+	docker build --build-arg USER_UID=$(shell id -u) \
+                 --build-arg USER_GID=$(shell id -g) \
+                 -f ideas.Dockerfile -t ideas-$(shell id -u) . && \
+	docker images --quiet ideas:latest 2>&1 | tee $(notdir $@)
+
+.PHONY: docker
+docker:## Mount translation Docker image
+docker: docker/docker_build.log
+	mkdir -p docker/.venv
+	test -f $(HOME)/.bashrc || touch $(HOME)/.bashrc
+	docker run -it --rm -v ".:/home/user/IDEAS" \
+                    -v "./docker/.venv:/home/user/IDEAS/.venv" \
+                    -v "$(HOME)/.bashrc:/home/user/.bashrc:ro" \
+                    ideas-$(shell id -u) bash
+
+
+.PHONY: docker/build_measurements
+docker/build_measurements:## Build measurement Docker images
+docker/build_measurements: ${PIPELINE_DIR}/evaluate_unsafe_usage/unsafety.Dockerfile \
+       ${PIPELINE_DIR}/idiomaticity/idiomaticity_measurements.Dockerfile
+	docker build -t ${PIPELINE_TAG}/unsafety \
+                 -f ${PIPELINE_DIR}/evaluate_unsafe_usage/unsafety.Dockerfile \
+                 ${PIPELINE_DIR}/evaluate_unsafe_usage/
+	docker build -t ${PIPELINE_TAG}/idiomaticity \
+                 -f ${PIPELINE_DIR}/idiomaticity/idiomaticity_measurements.Dockerfile \
+                 ${PIPELINE_DIR}/idiomaticity/
+
 .PHONY: install
-install: install-uv install-rust install-deno ## Install uv, Rust, and Deno
+install: install-uv install-rust ## Install uv and Rust
 
 .PHONY: install-uv
 install-uv:## Install uv@0.9.22
@@ -48,26 +79,18 @@ install-uv:## Install uv@0.9.22
 install-rust:## Install Rust@1.88.0
 	curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain 1.88.0
 
-.PHONY: install-deno
-install-deno:## Install Deno, which is required by dspy.PythonInterpreter()
-	curl -fsSL https://deno.land/install.sh | sh
-
 .PHONY: install-clang
 install-clang:## Install Clang-21, must be sudo
 	wget https://apt.llvm.org/llvm.sh
 	chmod +x llvm.sh
 	-./llvm.sh 21 all
 	rm ./llvm.sh
 
-.PHONY: test
-test:## Run pytest
-	uv run pytest
-
 .PHONY: serve
 serve:## Start vLLM server
 	uv run --no-project --python 3.11 --with vllm==${VLLM_VERSION} vllm serve ${MODEL} --revision ${REVISION} --host ${HOST} --port ${PORT} --dtype auto ${VLLM_ARGS}
 
-kill:## Kill all vLLM serves
+kill:## Kill all vLLM servers
 	pkill -f "^uv run --no-project --python 3.11 --with vllm==${VLLM_VERSION} vllm serve" -u ${USER}
 
 .PHONY: FORCE
@@ -119,14 +142,6 @@ examples/%/wrapper: FORCE
 	-@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) wrapper
 
 
-.PHONY: examples/add_test_vectors
-examples/add_test_vectors:## Build all translated examples
-examples/add_test_vectors: $(addsuffix /add_test_vectors,${EXAMPLES})
-examples/%/add_test_vectors:## Build specific translated example
-examples/%/add_test_vectors: FORCE
-	-@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) cmake
-	-@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) add_test_vectors
-
 .PHONY: examples/build
 examples/build:## Build all translated examples
 examples/build: $(addsuffix /build,${EXAMPLES})
@@ -186,19 +201,12 @@ examples/%/clean:## Clean specific example
 examples/%/clean: FORCE
 	-@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) clean
 
-# update tests
-.PHONY: examples/update_tests
-examples/update_tests:## Update test cases to use TRANSLATION_DIR test cases for all examples
-examples/update_tests: $(addsuffix /update_tests,${EXAMPLES})
-examples/%/update_tests:## Update specific test cases to use TRANSLATION_DIR test cases
-examples/%/update_tests: FORCE
-	-@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) cmake
-	-@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) update_tests
-
-# clean
+# Global clean
 clean:
-	rm -rf examples
-	git checkout HEAD examples
+	rm -rf docker/.venv
+	rm -rf docker/docker_build.log
+	rm -rf ${PIPELINE_DIR} examples
+	git checkout HEAD ${PIPELINE_DIR} examples
 
 # help
 RESET := \033[0;0m
@@ -219,5 +227,5 @@ help:
      | awk 'BEGIN { FS=" [:?!+]?= |##" } ; \
           { printf "  ${YELLOW_COL}%-30s${RESET}%s ${GREY_COL}(default: %s)${RESET}\n", $$1, $$3, $$2}'
 	@echo ""
-	@echo "Examples:"
-	@echo "  make examples/test TRANSLATION_DIR=test_case ${GREY_COL}# Build and run tests on C examples ${RESET}"
+	@echo "Example:"
+	@echo "  make examples/test TRANSLATION_DIR=test_case ${GREY_COL}# Translate, build, and run tests on C examples ${RESET}"