From 5bd00ca84be605e9d4dc1c1314d08f4090bd37ea Mon Sep 17 00:00:00 2001 From: Andreea Alexandru Date: Mon, 6 Jul 2026 11:42:44 -0400 Subject: [PATCH 1/2] clean arguments, enable num_runs, supress warnings, add mini-workload labels --- README.md | 3 +- harness/cleartext_impl.py | 2 +- harness/generate_dataset.py | 2 +- harness/run_submission.py | 118 ++++++++++++++++++---------------- harness/utils.py | 7 +- harness/verify_result.py | 14 ++-- submission/Cargo.toml | 3 + submission/cbs_lib/Cargo.toml | 3 + 8 files changed, 82 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index 04430f7..986128f 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ An example run is provided below. ```console $ python3 harness/run_submission.py -h -usage: run_submission.py [-h] [--num_runs NUM_RUNS] [--seed SEED] [--clrtxt CLRTXT] {0,1,2,3} +usage: run_submission.py [-h] [--num_runs NUM_RUNS] [--seed SEED] {0,1,2,3} Run the AES transciphering FHE benchmark. @@ -57,7 +57,6 @@ options: -h, --help show this help message and exit --num_runs NUM_RUNS Number of times to run steps 4-9 (default: 1) --seed SEED Random seed for dataset and query generation - --clrtxt CLRTXT Specify with 1 if to rerun the cleartext computation $ python3 ./harness/run_submission.py 2 --seed 3 --num_runs 2 diff --git a/harness/cleartext_impl.py b/harness/cleartext_impl.py index 8d89a8f..b28acec 100644 --- a/harness/cleartext_impl.py +++ b/harness/cleartext_impl.py @@ -22,7 +22,7 @@ def main(): - __, params, __, __, __, __ = parse_submission_arguments('Generate dataset for FHE benchmark.') + __, params, __, __, __ = parse_submission_arguments('Generate dataset for FHE benchmark.') DATASET_PATH = params.datadir() / f"db.txt" DATASET_ENC_PATH = params.datadir() / f"db.hex" AES_KEY_PATH = params.datadir() / f"aes_key.hex" diff --git a/harness/generate_dataset.py b/harness/generate_dataset.py index ae48264..78e8498 100644 --- a/harness/generate_dataset.py +++ b/harness/generate_dataset.py @@ -15,7 +15,7 @@ from utils import parse_submission_arguments def main(): - __, params, seed, __, __, __ = parse_submission_arguments('Generate dataset for FHE benchmark.') + __, params, seed, __, __ = parse_submission_arguments('Generate dataset for FHE benchmark.') DATASET_PATH = params.datadir() / f"db.txt" AES_KEY_PATH = params.datadir() / f"aes_key.hex" IV_PATH = params.datadir() / f"aes_iv.hex" diff --git a/harness/run_submission.py b/harness/run_submission.py index 90abb93..619efd5 100755 --- a/harness/run_submission.py +++ b/harness/run_submission.py @@ -23,7 +23,7 @@ def main(): # 0. Prepare running # Get the arguments - size, params, seed, num_runs, clrtxt, mini_workload = utils.parse_submission_arguments('Run the add-two-values FHE benchmark.') + size, params, seed, num_runs, mini_workload = utils.parse_submission_arguments('Run the AES-transciphering FHE benchmark.') test = instance_name(size) print(f"\n[harness] Running submission for {test} dataset") @@ -83,60 +83,68 @@ def main(): subprocess.run([exec_dir/"server_preprocess_dataset", str(size)], check=True) utils.log_step(6, "(Encrypted) dataset preprocessing") - # 7. Server side: Run aes_decryption - subprocess.run([exec_dir/"server_encrypted_aes_decryption", str(size)], check=True) - utils.log_step(7, "Encrypted aes decryption") - utils.log_size(io_dir / "ciphertext_aes_download", "Encrypted results") - - # 8. Server side: Run the encrypted processing run exec_dir/server_encrypted_compute - subprocess.run([exec_dir/"server_encrypted_compute", str(size)], check=True) - utils.log_step(8, "Encrypted computation of mini workload") - utils.log_size(io_dir / "ciphertexts_download", "Encrypted results") - - # 9. Client-side: decrypt - subprocess.run([exec_dir/"client_decrypt_decode_aes_decryption", str(size)], check=True) - utils.log_step(9, "Result decryption") - - # 10. Client-side: post-process - subprocess.run([exec_dir/"client_postprocess_aes_decryption", str(size)], check=True) - utils.log_step(10, "Result postprocessing") - - # 11. Client-side: decrypt - subprocess.run([exec_dir/"client_decrypt_decode", str(size)], check=True) - utils.log_step(11, "Result decryption") - - # 12. Client-side: post-process - subprocess.run([exec_dir/"client_postprocess", str(size)], check=True) - utils.log_step(12, "Result postprocessing") - - # 13. Verify aes_decryption result - aes_expected_file = params.datadir() / "expected_aes.txt" - aes_result_file = io_dir / "result_aes.txt" - - if not aes_result_file.exists(): - print(f"Error: Result file {aes_result_file} not found") - sys.exit(1) - - subprocess.run(["python3", harness_dir/"verify_aes_decryption.py", - str(aes_expected_file), str(aes_result_file)], check=False) - - # 14. Verify the final result - expected_file = params.datadir() / "max_value.txt" - if mini_workload == 1: - expected_file = params.datadir() / "inner_product.txt" - result_file = io_dir / "result.txt" - - if not result_file.exists(): - print(f"Error: Result file {result_file} not found") - sys.exit(1) - - subprocess.run(["python3", harness_dir/"verify_result.py", - str(expected_file), str(result_file)], check=False) - - # 15. Store measurements - run_path = params.measuredir() / f"results.json" - run_path.parent.mkdir(parents=True, exist_ok=True) - utils.save_run(run_path) + # Run steps 7-14 multiple times if requested. The computation is deterministic, so + # the results are identical every run; only the timings vary. + for run in range(num_runs): + if num_runs > 1: + print(f"\n [harness] Run {run+1} of {num_runs}") + + # 7. Server side: Run aes_decryption + subprocess.run([exec_dir/"server_encrypted_aes_decryption", str(size)], check=True) + utils.log_step(7, "Encrypted aes decryption") + utils.log_size(io_dir / "ciphertext_aes_download", "Encrypted results") + + # 8. Server side: Run the encrypted processing run exec_dir/server_encrypted_compute + subprocess.run([exec_dir/"server_encrypted_compute", str(size)], check=True) + utils.log_step(8, "Encrypted computation of mini workload") + utils.log_size(io_dir / "ciphertexts_download", "Encrypted results") + + # 9. Client-side: decrypt + subprocess.run([exec_dir/"client_decrypt_decode_aes_decryption", str(size)], check=True) + utils.log_step(9, "Result decryption") + + # 10. Client-side: post-process + subprocess.run([exec_dir/"client_postprocess_aes_decryption", str(size)], check=True) + utils.log_step(10, "Result postprocessing") + + # 11. Client-side: decrypt + subprocess.run([exec_dir/"client_decrypt_decode", str(size)], check=True) + utils.log_step(11, "Result decryption") + + # 12. Client-side: post-process + subprocess.run([exec_dir/"client_postprocess", str(size)], check=True) + utils.log_step(12, "Result postprocessing") + + # 13. Verify aes_decryption result + aes_expected_file = params.datadir() / "expected_aes.txt" + aes_result_file = io_dir / "result_aes.txt" + + if not aes_result_file.exists(): + print(f"Error: Result file {aes_result_file} not found") + sys.exit(1) + + subprocess.run(["python3", harness_dir/"verify_aes_decryption.py", + str(aes_expected_file), str(aes_result_file)], check=False) + + # 14. Verify the final result + expected_file = params.datadir() / "max_value.txt" + workload_label = "MAX" + if mini_workload == 1: + expected_file = params.datadir() / "inner_product.txt" + workload_label = "IP" + result_file = io_dir / "result.txt" + + if not result_file.exists(): + print(f"Error: Result file {result_file} not found") + sys.exit(1) + + subprocess.run(["python3", harness_dir/"verify_result.py", + str(expected_file), str(result_file), workload_label], check=False) + + # 15. Store this run's measurements + run_path = params.measuredir() / f"results-{run+1}.json" + run_path.parent.mkdir(parents=True, exist_ok=True) + utils.save_run(run_path) print(f"\nAll steps completed for the {instance_name(size)} dataset!") diff --git a/harness/utils.py b/harness/utils.py index 4123418..76e81b8 100644 --- a/harness/utils.py +++ b/harness/utils.py @@ -27,7 +27,7 @@ # Global variable to store measured sizes _bandwidth = {} -def parse_submission_arguments(workload: str) -> Tuple[int, InstanceParams, int, int, int, int]: +def parse_submission_arguments(workload: str) -> Tuple[int, InstanceParams, int, int, int]: """ Get the arguments of the submission. Populate arguments as needed for the workload. """ @@ -39,8 +39,6 @@ def parse_submission_arguments(workload: str) -> Tuple[int, InstanceParams, int, help='Number of times to run steps 4-9 (default: 1)') parser.add_argument('--seed', type=int, help='Random seed for dataset and query generation') - parser.add_argument('--clrtxt', type=int, - help='Specify with 1 if to rerun the cleartext computation') parser.add_argument('--mini_workload', type=int, default=0, help='Specify 0 for mini workload = max and 1 for mini workload = dot product.') @@ -48,12 +46,11 @@ def parse_submission_arguments(workload: str) -> Tuple[int, InstanceParams, int, size = args.size seed = args.seed num_runs = args.num_runs - clrtxt = args.clrtxt mini_workload = args.mini_workload # Use params.py to get instance parameters params = InstanceParams(size) - return size, params, seed, num_runs, clrtxt, mini_workload + return size, params, seed, num_runs, mini_workload def ensure_directories(rootdir: Path): """ Check that the current directory has sub-directories diff --git a/harness/verify_result.py b/harness/verify_result.py index 5794f1f..d396f91 100644 --- a/harness/verify_result.py +++ b/harness/verify_result.py @@ -16,16 +16,18 @@ def main(): """ - Usage: python3 verify_result.py + Usage: python3 verify_result.py [label] Returns exit-code 0 if equal, 1 otherwise. - Prints a message so the caller can log it. + Prints a message so the caller can log it. The optional label (e.g. MAX or + IP) identifies which mini workload was verified. """ - if len(sys.argv) != 3: - sys.exit("Usage: verify_result.py ") + if len(sys.argv) not in (3, 4): + sys.exit("Usage: verify_result.py [label]") expected_file = Path(sys.argv[1]) result_file = Path(sys.argv[2]) + label = f" {sys.argv[3]}" if len(sys.argv) == 4 else "" try: exp = list(map(int, expected_file.read_text().split())) @@ -35,10 +37,10 @@ def main(): sys.exit(1) if exp == res: - print(f"[harness] PASS (expected={exp}, got={res})") + print(f"[harness] PASS{label} (expected={exp}, got={res})") sys.exit(0) else: - print(f"[harness] FAIL (expected={exp}, got={res})") + print(f"[harness] FAIL{label} (expected={exp}, got={res})") sys.exit(1) if __name__ == "__main__": diff --git a/submission/Cargo.toml b/submission/Cargo.toml index 0fc2e7e..dff7b3c 100644 --- a/submission/Cargo.toml +++ b/submission/Cargo.toml @@ -3,6 +3,9 @@ name = "submission" version = "0.1.0" edition = "2021" +[lints.rust] +warnings = "allow" + [dependencies] bincode = "1.3" rand = { version = "0.9.1" } diff --git a/submission/cbs_lib/Cargo.toml b/submission/cbs_lib/Cargo.toml index 0bd9651..508fc6e 100644 --- a/submission/cbs_lib/Cargo.toml +++ b/submission/cbs_lib/Cargo.toml @@ -3,6 +3,9 @@ name = "auto-base-conv" version = "0.1.0" edition = "2021" +[lints.rust] +warnings = "allow" + [dependencies] tfhe = { version = "0.5.3", features = ["boolean", "shortint", "x86_64-unix"] } From be5fbbe9c00b07f7de0a7aa3580462edb9fad57a Mon Sep 17 00:00:00 2001 From: Andreea Alexandru Date: Mon, 6 Jul 2026 12:29:37 -0400 Subject: [PATCH 2/2] updated harness output, added measurements,updated README --- .gitignore | 1 - README.md | 135 ++++++++++++++++-------------- harness/run_submission.py | 3 +- harness/utils.py | 60 +++++++++---- harness/verify_aes_decryption.py | 5 +- harness/verify_result.py | 5 +- measurements/README.md | 9 ++ measurements/small/results-1.json | 21 +++++ measurements/small/results-2.json | 21 +++++ measurements/small/results-3.json | 21 +++++ measurements/toy/results-1.json | 21 +++++ measurements/toy/results-2.json | 21 +++++ measurements/toy/results-3.json | 21 +++++ 13 files changed, 258 insertions(+), 86 deletions(-) create mode 100644 measurements/README.md create mode 100644 measurements/small/results-1.json create mode 100644 measurements/small/results-2.json create mode 100644 measurements/small/results-3.json create mode 100644 measurements/toy/results-1.json create mode 100644 measurements/toy/results-2.json create mode 100644 measurements/toy/results-3.json diff --git a/.gitignore b/.gitignore index 41b789b..e1b45b7 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ */build/ io/ third_party/ -measurements/ harness/__pycache__/ submission/target/ diff --git a/README.md b/README.md index 986128f..ea3fa1b 100644 --- a/README.md +++ b/README.md @@ -1,64 +1,63 @@ -# FHE Benchmarking template +# AES Transciphering FHE Benchmark ## Cloning the workload ```console -git clone https://github.com/fhe-benchmarking/sample-workload -cd sample-workload +git clone https://github.com/code-perspective/temp-fhe-transciphering +cd temp-fhe-transciphering ``` ## Dependencies -The harness requires python and some corresponding packages specified in `requirements.txt`. +The harness is written in Python and requires a few packages listed in `requirements.txt`. ```console python3 -m venv virtualenv source ./virtualenv/bin/activate pip3 install -r requirements.txt ``` -In this template, the library for the encrypted computations is OpenFHE v1.3.1. -If other libraries are required, the developer should include the relevant instructions for installing. +In this template, the submission (the encrypted computation) is written in Rust and uses +the [tfhe-rs](https://github.com/zama-ai/tfhe-rs) library (v0.5.3) together with the +local `auto-base-conv` crate under `submission/cbs_lib`. -This template assumes the correct version of OpenFHE is installed locally at `/third-party/openfhe`. - -### Installing OpenFHE - -The installation steps for OpenFHE are described [here](https://openfhe-development.readthedocs.io/en/latest/sphinx_rsts/intro/installation/installation.html). - -If OpenFHE -is installed at a different location, that location should be specified using the `-CMAKE_PREFIX_PATH` variable in [`build_task.sh`](https://github.com/fhe-benchmarking/sample-workload/blob/baa3654cdc4a78ce331f49bda1ff48465a9c575d/scripts/build_task.sh#L24). -(In the case of a system-wide installation at `/usr/local/`, unset the `-CMAKE_PREFIX_PATH` variable.) - -For users who want to do a local fresh install, they should run `get_openfhe.sh`, which -is designed to install the specified version of OpenFHE at the `third-party` subdirectory in the current directory. -By default, `build_task.sh` looks for the library at this location. See more instructions in `submission/CMakeLists.txt` if -`build_task.sh` does not succeed. - -```console -./scripts/get_openfhe.sh -``` +Building requires a Rust toolchain (`cargo`/`rustc`). You do not need to install it manually: +[`scripts/build_task.sh`](scripts/build_task.sh) installs the Rust toolchain via `rustup` +if it is not already present, and then builds the submission with `cargo build --release`. +The harness invokes this script automatically, so the first run may take a while as the +dependencies are compiled. ## Running the "AES transciphering" workload -The goal of this workload is to homomorphically transcipher a number of AES blocks into FHE ciphertexts encrypting the same plaintexts. We assess the "quality" of the resulting FHE ciphertexts by evaluating two mini-workloads after the transciphering: 1. Computing the maximum between the input message parsed as 16-bit unsigned integers; and 2. Computing the inner product modulo 2^16 of the first half of the input message parsed as 16-bit unsigned integers and the second half. +The goal of this workload is to homomorphically transcipher a number of AES blocks into FHE +ciphertexts encrypting the same plaintexts. We assess the "quality" of the resulting FHE +ciphertexts by evaluating two mini-workloads after the transciphering: 1. Computing the +maximum between the input message parsed as 16-bit unsigned integers; and 2. Computing the +inner product modulo 2^16 of the first half of the input message parsed as 16-bit unsigned +integers and the second half. -An example run is provided below. +The mini-workload is selected with `--mini_workload` (`0` for the maximum, `1` +for the inner product). An example run is provided below. ```console $ python3 harness/run_submission.py -h -usage: run_submission.py [-h] [--num_runs NUM_RUNS] [--seed SEED] {0,1,2,3} +usage: run_submission.py [-h] [--num_runs NUM_RUNS] [--seed SEED] + [--mini_workload MINI_WORKLOAD] + {0,1,2,3} -Run the AES transciphering FHE benchmark. +Run the AES-transciphering FHE benchmark. positional arguments: - {0,1,2,3} Instance size (0-toy/1-small/2-medium) + {0,1,2,3} Instance size (0-toy/1-small/2-medium/3-large) options: - -h, --help show this help message and exit - --num_runs NUM_RUNS Number of times to run steps 4-9 (default: 1) - --seed SEED Random seed for dataset and query generation + -h, --help show this help message and exit + --num_runs NUM_RUNS Number of times to run steps 4-9 (default: 1) + --seed SEED Random seed for dataset generation + --mini_workload MINI_WORKLOAD + Specify 0 for mini workload = max and 1 for mini + workload = dot product. -$ python3 ./harness/run_submission.py 2 --seed 3 --num_runs 2 +$ python3 ./harness/run_submission.py 1 --seed 3 --num_runs 2 --mini_workload 0 [...] @@ -75,59 +74,69 @@ deactivate ```bash Each submission to the workload in the FHE benchmarking should have the following directory structure: [root] / -| ├─datasets/ # Holds cleartext data -| | ├─ toy/ # each instance-size in in a separate subdirectory +| ├─datasets/ # Holds cleartext data (plaintext db, AES key/IV, AES-encrypted db, expected outputs) +| | ├─ toy/ # each instance-size in a separate subdirectory | | ├─ small/ | | ├─ medium/ +| | ├─ large/ | ├─docs/ # Documentation (beyond the top-level README.md) | ├─harness/ # Scripts to generate data, run workload, check results -| ├─build/ # Handle installing dependencies and building the project +| ├─scripts/ # Handle installing dependencies and building the project (build_task.sh) | ├─submission/ # The implementation, this is what the submitters modify -| | └─ README.md # likely also a src/, include/ subdirectories, CMakeLists.txt, etc. +| | ├─ cbs_lib/ # The `auto-base-conv` Rust crate used by the submission +| | ├─ src/ # Rust sources for the client_*/server_* executables +| | └─ Cargo.toml # Rust package manifest | ├─io/ # Directory to hold the I/O between client & server parts | | ├─ toy/ # The reference implementation has subdirectories | | ├─ public_keys/ # holds the public evaluation keys -| | ├─ ciphertexts_download/ # holds the ciphertexts to be downloaded by the client -| | ├─ ciphertexts_upload/ # holds the ciphertexts (or other data except keys) to be uploaded by the client -| | ├─ intermediate/ # internal information to be passed around the functions -| | └─ secret_key/ # holds the secret key +| | ├─ secret_keys/ # holds the secret key(s) +| | ├─ ciphertexts_upload/ # holds the ciphertexts (or other data except keys) to be uploaded by the client +| | ├─ ciphertexts_download/ # holds the mini-workload ciphertexts to be downloaded by the client +| | ├─ ciphertext_aes_download/ # holds the transciphered (AES-decrypted) ciphertexts to be downloaded by the client +| | └─ intermediate/ # internal information to be passed around the functions | | ├─ small/ | | … | | ├─ medium/ | | … -| ├─measurements/ # Holds json files with the results for each run -| | ├─ toy/ # each instance-size in in a separate subdirectory +| ├─measurements/ # Holds json files (results-.json) with the results for each run +| | ├─ toy/ # each instance-size in a separate subdirectory | | ├─ small/ | | ├─ medium/ ``` ## Description of stages -A submitter can edit any of the `client_*` / `server_*` sources in `/submission`. +A submitter can edit any of the `client_*` / `server_*` sources in `/submission`. Moreover, for the particular parameters related to a workload, the submitter can modify the params files. -If the current description of the files are inaccurate, the stage names in `run_submission` can be also +If the current description of the files are inaccurate, the stage names in `run_submission` can be also modified. The current stages are the following, targeted to a client-server scenario. -The order in which they are happening in `run_submission` assumes an initialization step which is -database-dependent and run only once, and potentially multiple runs for multiple queries. -Each file can take as argument the test case size. - - -| Stage executables | Description | -|----------------------------------|-------------| -| `client_key_generation` | Generate all key material and cryptographic context at the client. -| `client_preprocess_dataset` | (Optional) Any in the clear computations the client wants to apply over the dataset/model. -| `client_preprocess_query` | (Optional) Any in the clear computations the client wants to apply over the query/input. -| `client_encode_encrypt_db` | (Optional) Plaintext encoding and encryption of the dataset/model at the client. -| `client_encode_encrypt_query` | Plaintext encoding and encryption of the query/input at the client. -| `server_preprocess_dataset` | (Optional) Any in the clear or encrypted computations the server wants to apply over the dataset/model. -| `server_encrypted_compute` | The computation the server applies to achieve the workload solution over encrypted daa. -| `client_decrypt_decode` | Decryption and plaintext decoding of the result at the client. -| `client_postprocess`: | Any in the clear computation that the client wants to apply on the decrypted result. - +The order in which they happen in `run_submission` assumes an initialization phase which is +dataset-dependent and run only once, followed by the encrypted computation, which can be repeated +`--num_runs` times to average out run-to-run variability in the measured latency. (The computation is +deterministic, so the results are identical across runs; only the timings vary.) +Each executable takes the test-case size as its argument. + + +| Stage executables | Description | +|---------------------------------------|-------------| +| `client_preprocess` | Any in-the-clear computation the client wants to apply over the dataset before encryption. +| `client_key_generation` | Generate all key material and cryptographic context at the client. +| `client_encode_encrypt` | Plaintext encoding and encryption of the AES key at the client. +| `server_preprocess_dataset` | (Optional) Any in-the-clear or encrypted computation the server applies over the AES-encrypted dataset. +| `server_encrypted_aes_decryption` | Homomorphically decrypt the AES ciphertexts, producing FHE ciphertexts of the same plaintexts (the transciphering). +| `server_encrypted_compute` | Evaluate the selected mini-workload (maximum or inner product) over the transciphered ciphertexts. +| `client_decrypt_decode_aes_decryption`| Decryption and plaintext decoding of the transciphered result at the client. +| `client_postprocess_aes_decryption` | Any in-the-clear computation over the decrypted transciphering result (written to `result_aes.txt`). +| `client_decrypt_decode` | Decryption and plaintext decoding of the mini-workload result at the client. +| `client_postprocess` | Any in-the-clear computation over the decrypted mini-workload result (written to `result.txt`). + +The harness additionally runs the Python helpers `generate_dataset.py` (generate and AES-encrypt the +dataset), `cleartext_impl.py` (cleartext reference), and `verify_aes_decryption.py` / `verify_result.py` +(correctness oracles). The outer python script measures the runtime of each stage. The current stage separation structure requires reading and writing to files more times than minimally necessary. For a more granular runtime measuring, which would account for the extra overhead described above, we encourage -submitters to separate and print in a log the individual times for reads/writes and computations inside each stage. +submitters to separate and print in a log the individual times for reads/writes and computations inside each stage. diff --git a/harness/run_submission.py b/harness/run_submission.py index 619efd5..b611fe9 100755 --- a/harness/run_submission.py +++ b/harness/run_submission.py @@ -144,7 +144,8 @@ def main(): # 15. Store this run's measurements run_path = params.measuredir() / f"results-{run+1}.json" run_path.parent.mkdir(parents=True, exist_ok=True) - utils.save_run(run_path) + submission_report_path = io_dir / "server_reported_steps.json" + utils.save_run(run_path, submission_report_path) print(f"\nAll steps completed for the {instance_name(size)} dataset!") diff --git a/harness/utils.py b/harness/utils.py index 76e81b8..7d915e6 100644 --- a/harness/utils.py +++ b/harness/utils.py @@ -11,12 +11,13 @@ """ import sys +import platform import subprocess import argparse import json from datetime import datetime from pathlib import Path -from params import InstanceParams, TOY, MEDIUM +from params import InstanceParams, TOY, LARGE from typing import Tuple # Global variable to track the last timestamp @@ -33,12 +34,12 @@ def parse_submission_arguments(workload: str) -> Tuple[int, InstanceParams, int, """ # Parse arguments using argparse parser = argparse.ArgumentParser(description=workload) - parser.add_argument('size', type=int, choices=range(TOY, MEDIUM+1), + parser.add_argument('size', type=int, choices=range(TOY, LARGE+1), help='Instance size (0-toy/1-small/2-medium/3-large)') parser.add_argument('--num_runs', type=int, default=1, help='Number of times to run steps 4-9 (default: 1)') parser.add_argument('--seed', type=int, - help='Random seed for dataset and query generation') + help='Random seed for dataset generation') parser.add_argument('--mini_workload', type=int, default=0, help='Specify 0 for mini workload = max and 1 for mini workload = dot product.') @@ -77,9 +78,10 @@ class TextFormat: YELLOW = "\033[33m" BLUE = "\033[34m" RED = "\033[31m" + PURPLE = "\033[35m" RESET = "\033[0m" -def log_step(step_num: int, step_name: str, start: bool = False): +def log_step(step_num: float, step_name: str, start: bool = False): """ Helper function to print timestamp after each step with second precision """ @@ -101,38 +103,62 @@ def log_step(step_num: int, step_name: str, start: bool = False): _last_timestamp = now if (not start): - print(f"{timestamp} [harness] {step_num}: {step_name} completed{elapsed_str}") + print(f"{TextFormat.BLUE}{timestamp} [harness] {step_num}: {step_name} completed{elapsed_str}{TextFormat.RESET}") _timestampsStr[step_name] = f"{round(elapsed_seconds, 4)}s" _timestamps[step_name] = elapsed_seconds def log_size(path: Path, object_name: str, flag: bool = False, previous: int = 0): + """Measure the size of a directory or file on disk""" global _bandwidth - size = int(subprocess.run(["du", "-sb", path], check=True, + + if not path.exists(): + size = 0 + elif platform.system() == "Darwin": # macOS + # Use -s for summary and multiply by 1024 since macOS du reports in 1K blocks by default + result = subprocess.run(["du", "-sk", path], check=True, + capture_output=True, text=True) + size = int(result.stdout.split()[0]) * 1024 + else: + # Linux/other systems support -b flag + size = int(subprocess.run(["du", "-sb", path], check=True, capture_output=True, text=True).stdout.split()[0]) - if(flag): + if flag: size -= previous - - print(" [harness]", object_name, "size:", human_readable_size(size)) + + print(f"{TextFormat.YELLOW} [harness] {object_name} size: {human_readable_size(size)}{TextFormat.RESET}") _bandwidth[object_name] = human_readable_size(size) return size def human_readable_size(n: int): + """Pretty print for size in bytes""" + n_float : float = n for unit in ["B","K","M","G","T"]: - if n < 1024: - return f"{n:.1f}{unit}" - n /= 1024 - return f"{n:.1f}P" + if n_float < 1024: + return f"{n_float:.1f}{unit}" + n_float /= 1024 + return f"{n_float:.1f}P" -def save_run(path: Path): +def save_run(path: Path, submission_report_path: Path): + """Save the timing from the current run to disk""" global _timestamps global _timestampsStr global _bandwidth + _timestampsStr["Total"] = f"{round(sum(_timestamps.values()), 4)}s" + + _timestampsRemote = {} + if submission_report_path.exists(): + with open(submission_report_path, "r") as f: + server_reported_times = json.load(f) + for step_name, time_str in server_reported_times.items(): + _timestampsRemote[step_name] = f"{time_str}s" + print(f"{TextFormat.PURPLE} [submission] {step_name}: {time_str}s{TextFormat.RESET}") + json.dump({ - "total_latency_s": round(sum(_timestamps.values()), 4), - "per_stage": _timestampsStr, - "bandwidth": _bandwidth, + "Timing": _timestampsStr, + "Bandwidth": _bandwidth, + "Server Reported": _timestampsRemote, }, open(path,"w"), indent=2) print("[total latency]", f"{round(sum(_timestamps.values()), 4)}s") diff --git a/harness/verify_aes_decryption.py b/harness/verify_aes_decryption.py index e22658b..771f58b 100644 --- a/harness/verify_aes_decryption.py +++ b/harness/verify_aes_decryption.py @@ -12,6 +12,7 @@ import sys from pathlib import Path +from utils import TextFormat def find_mismatches(list1, list2): """ @@ -60,11 +61,11 @@ def main(): sys.exit(1) if exp == res: - print(f"[harness] PASS AES Decryption") + print(f"{TextFormat.GREEN} [harness] PASS AES Decryption{TextFormat.RESET}") sys.exit(0) else: mismatches = find_mismatches(exp, res) - print(f"[harness] FAIL AES Decryption (find_mismatches): {mismatches}") + print(f"{TextFormat.RED} [harness] FAIL AES Decryption (find_mismatches): {mismatches}{TextFormat.RESET}") sys.exit(1) if __name__ == "__main__": diff --git a/harness/verify_result.py b/harness/verify_result.py index d396f91..66fcaf0 100644 --- a/harness/verify_result.py +++ b/harness/verify_result.py @@ -12,6 +12,7 @@ import sys from pathlib import Path +from utils import TextFormat def main(): @@ -37,10 +38,10 @@ def main(): sys.exit(1) if exp == res: - print(f"[harness] PASS{label} (expected={exp}, got={res})") + print(f"{TextFormat.GREEN} [harness] PASS{label} (expected={exp}, got={res}){TextFormat.RESET}") sys.exit(0) else: - print(f"[harness] FAIL{label} (expected={exp}, got={res})") + print(f"{TextFormat.RED} [harness] FAIL{label} (expected={exp}, got={res}){TextFormat.RESET}") sys.exit(1) if __name__ == "__main__": diff --git a/measurements/README.md b/measurements/README.md new file mode 100644 index 0000000..59dda24 --- /dev/null +++ b/measurements/README.md @@ -0,0 +1,9 @@ +# Measurements + +When running `python ./harness/run_submission.py `, it will generate measurement files in a sub-directory under this directory. +Specifically, the sub-directories that it uses are `toy`, `small`, `medium` and `large`, one per instance size. +If it is run with argument `--num_runs ` it will generate `` measurement files called `results-1.json`, ..., `results-.json`, all in the same sub-directory. + +Before submitting your implementation, run the `run_submission.py` script with argument `--num_runs 3` for each instance size (and each mini-workload you want to submit, selected with `--mini_workload`: `0` for the maximum, `1` for the inner product). Then commit all these results files to your fork, the average of these three runs will be the numbers reported for your submission. + +## Results for the reference implementation \ No newline at end of file diff --git a/measurements/small/results-1.json b/measurements/small/results-1.json new file mode 100644 index 0000000..436bfb3 --- /dev/null +++ b/measurements/small/results-1.json @@ -0,0 +1,21 @@ +{ + "Timing": { + "Dataset generation, AES Key generation and message encryption with AES": "0.1525s", + "Cleartext implementation": "0.0353s", + "Client side preprocessing": "0.001s", + "FHE Key Generation": "2.3615s", + "AES key encoding and encryption": "0.2025s", + "(Encrypted) dataset preprocessing": "0.003s", + "Encrypted aes decryption": "108.2187s", + "Encrypted computation of mini workload": "90.1992s", + "Result decryption": "0.0013s", + "Result postprocessing": "0.0008s", + "Total": "201.1758s" + }, + "Bandwidth": { + "Public and evaluation keys": "115.6M", + "Encrypted aes-encrypted dataset": "24.3M", + "Encrypted results": "260.2K" + }, + "Server Reported": {} +} \ No newline at end of file diff --git a/measurements/small/results-2.json b/measurements/small/results-2.json new file mode 100644 index 0000000..7d1a5cc --- /dev/null +++ b/measurements/small/results-2.json @@ -0,0 +1,21 @@ +{ + "Timing": { + "Dataset generation, AES Key generation and message encryption with AES": "0.1525s", + "Cleartext implementation": "0.0353s", + "Client side preprocessing": "0.001s", + "FHE Key Generation": "2.3615s", + "AES key encoding and encryption": "0.2025s", + "(Encrypted) dataset preprocessing": "0.003s", + "Encrypted aes decryption": "109.9047s", + "Encrypted computation of mini workload": "92.7831s", + "Result decryption": "0.0021s", + "Result postprocessing": "0.0016s", + "Total": "205.4475s" + }, + "Bandwidth": { + "Public and evaluation keys": "115.6M", + "Encrypted aes-encrypted dataset": "24.3M", + "Encrypted results": "260.2K" + }, + "Server Reported": {} +} \ No newline at end of file diff --git a/measurements/small/results-3.json b/measurements/small/results-3.json new file mode 100644 index 0000000..6c6bef9 --- /dev/null +++ b/measurements/small/results-3.json @@ -0,0 +1,21 @@ +{ + "Timing": { + "Dataset generation, AES Key generation and message encryption with AES": "0.1525s", + "Cleartext implementation": "0.0353s", + "Client side preprocessing": "0.001s", + "FHE Key Generation": "2.3615s", + "AES key encoding and encryption": "0.2025s", + "(Encrypted) dataset preprocessing": "0.003s", + "Encrypted aes decryption": "109.6406s", + "Encrypted computation of mini workload": "90.9087s", + "Result decryption": "0.0014s", + "Result postprocessing": "0.0009s", + "Total": "203.3075s" + }, + "Bandwidth": { + "Public and evaluation keys": "115.6M", + "Encrypted aes-encrypted dataset": "24.3M", + "Encrypted results": "260.2K" + }, + "Server Reported": {} +} \ No newline at end of file diff --git a/measurements/toy/results-1.json b/measurements/toy/results-1.json new file mode 100644 index 0000000..532559e --- /dev/null +++ b/measurements/toy/results-1.json @@ -0,0 +1,21 @@ +{ + "Timing": { + "Dataset generation, AES Key generation and message encryption with AES": "0.163s", + "Cleartext implementation": "0.0311s", + "Client side preprocessing": "0.0012s", + "FHE Key Generation": "2.3041s", + "AES key encoding and encryption": "0.0945s", + "(Encrypted) dataset preprocessing": "0.0032s", + "Encrypted aes decryption": "25.5735s", + "Encrypted computation of mini workload": "5.4911s", + "Result decryption": "0.0012s", + "Result postprocessing": "0.0009s", + "Total": "33.6638s" + }, + "Bandwidth": { + "Public and evaluation keys": "115.6M", + "Encrypted aes-encrypted dataset": "27.8M", + "Encrypted results": "260.2K" + }, + "Server Reported": {} +} \ No newline at end of file diff --git a/measurements/toy/results-2.json b/measurements/toy/results-2.json new file mode 100644 index 0000000..9ac6c0a --- /dev/null +++ b/measurements/toy/results-2.json @@ -0,0 +1,21 @@ +{ + "Timing": { + "Dataset generation, AES Key generation and message encryption with AES": "0.163s", + "Cleartext implementation": "0.0311s", + "Client side preprocessing": "0.0012s", + "FHE Key Generation": "2.3041s", + "AES key encoding and encryption": "0.0945s", + "(Encrypted) dataset preprocessing": "0.0032s", + "Encrypted aes decryption": "25.6947s", + "Encrypted computation of mini workload": "5.7111s", + "Result decryption": "0.0016s", + "Result postprocessing": "0.0015s", + "Total": "34.0059s" + }, + "Bandwidth": { + "Public and evaluation keys": "115.6M", + "Encrypted aes-encrypted dataset": "27.8M", + "Encrypted results": "260.2K" + }, + "Server Reported": {} +} \ No newline at end of file diff --git a/measurements/toy/results-3.json b/measurements/toy/results-3.json new file mode 100644 index 0000000..86593a5 --- /dev/null +++ b/measurements/toy/results-3.json @@ -0,0 +1,21 @@ +{ + "Timing": { + "Dataset generation, AES Key generation and message encryption with AES": "0.163s", + "Cleartext implementation": "0.0311s", + "Client side preprocessing": "0.0012s", + "FHE Key Generation": "2.3041s", + "AES key encoding and encryption": "0.0945s", + "(Encrypted) dataset preprocessing": "0.0032s", + "Encrypted aes decryption": "25.4697s", + "Encrypted computation of mini workload": "5.4048s", + "Result decryption": "0.0011s", + "Result postprocessing": "0.0009s", + "Total": "33.4735s" + }, + "Bandwidth": { + "Public and evaluation keys": "115.6M", + "Encrypted aes-encrypted dataset": "27.8M", + "Encrypted results": "260.2K" + }, + "Server Reported": {} +} \ No newline at end of file