Enable LIT Testing

.dockerignore

@@ -1,7 +1,7 @@
 *
-!.git
 !cmake/
 !config/
+!external/
 !include/
 !lib/
 !test/

.github/workflows/ci.yml

@@ -47,7 +47,7 @@ jobs:
     continue-on-error: false
     steps:
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           fetch-depth: 0
           submodules: recursive
@@ -60,7 +60,11 @@ jobs:
       - name: Install Phasar Dependencies
         shell: bash
         run: |
-          ./utils/InstallAptDependencies.sh --llvm-version ${{ matrix.llvm-major-version }} --noninteractive tzdata ccache ${{ matrix.compiler[2] }} ${{ matrix.extra_dependencies }}
+          ./utils/InstallAptDependencies.sh \
+            --llvm-version ${{ matrix.llvm-major-version }} \
+            --noninteractive \
+            tzdata ccache ${{ matrix.compiler[2] }} ${{ matrix.extra_dependencies }} \
+            llvm-${{ matrix.llvm-major-version }}-tools # For lit, FileCheck
 
       - name: Building Phasar in ${{ matrix.build }} with ${{ matrix.compiler[0] }}
         env:
@@ -84,7 +88,7 @@ jobs:
       - name: Run Unittests
         shell: bash
         run: |
-          cmake --build ./build --target check-phasar-unittests
+          cmake --build ./build --target check-phasar-unittests check-phasar-cli
 
       - name: Install PhASAR and Build Examples
         if: matrix.build == 'DebugLibdeps' # Circumvent conflicting ASAn flags

.github/workflows/docker.yml

@@ -18,7 +18,10 @@ jobs:
       contents: read
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+          submodules: recursive
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
       - name: Log in to the Container registry

.gitignore

@@ -142,3 +142,6 @@ unittests/PhasarLLVM/ifdside/Problems/IFDSTaintAnalysisTest
 unittests/PhasarLLVM/Pointer/LLVMTypeHierarchyTest
 unittests/Utils/LLVMShorthandsTest
 unittests/Utils/PAMMTest
+
+# LLVM lit test times files
+*.lit_test_times.txt

CMakeLists.txt

@@ -161,6 +161,7 @@ if(PHASAR_ALLOW_LTO_IN_RELEASE_BUILD)
 endif()
 
 option(PHASAR_BUILD_UNITTESTS "Build all tests (default is ON)" ${PHASAR_BUILD_OPTIONAL_TARGETS_DEFAULT})
+option(PHASAR_ENABLE_INTEGRATIONTESTS "Enable LIT integration tests (default is ON)" ${PHASAR_BUILD_OPTIONAL_TARGETS_DEFAULT})
 
 option(PHASAR_BUILD_OPENSSL_TS_UNITTESTS "Build OPENSSL typestate tests (require OpenSSL, default is OFF)" OFF)
 

Dockerfile

@@ -3,7 +3,7 @@ FROM "$baseimage" AS build
 
 RUN --mount=type=bind,source=./utils/InstallAptDependencies.sh,target=/InstallAptDependencies.sh \
   set -eux; \
-  ./InstallAptDependencies.sh --noninteractive tzdata clang-20 libclang-rt-20-dev clang-tools-20
+  ./InstallAptDependencies.sh --noninteractive tzdata clang-20 libclang-rt-20-dev clang-tools-20 llvm-16-tools llvm-20-tools
 
 ENV CC=/usr/bin/clang-20 \
     CXX=/usr/bin/clang++-20
@@ -14,7 +14,10 @@ ARG RUN_TESTS=OFF
 RUN --mount=type=bind,source=.,target=/usr/src/phasar,rw \
   set -eux; \
   cd /usr/src/phasar; \
-  git submodule update --init; \
+  if find external -mindepth 1 -maxdepth 1 -type d -empty | grep -q .; then \
+    echo "external/* submodules are not checked out; run 'git submodule update --init --recursive' on the host before building"; \
+    exit 1; \
+  fi; \
   cmake -S . -B cmake-build/Release \
     -DCMAKE_BUILD_TYPE=Release \
     -DPHASAR_TARGET_ARCH="" \
@@ -23,10 +26,11 @@ RUN --mount=type=bind,source=.,target=/usr/src/phasar,rw \
     -DPHASAR_USE_Z3=ON \
     -DPHASAR_BUILD_UNITTESTS=$RUN_TESTS \
     -DPHASAR_BUILD_IR=$RUN_TESTS \
+    -DPHASAR_ENABLE_INTEGRATIONTESTS=$RUN_TESTS \
     -DPHASAR_BUILD_OPENSSL_TS_UNITTESTS=OFF \
     -G Ninja; \
   ninja -C cmake-build/Release install; \
-  [ "${RUN_TESTS}" = "ON" ] && ctest --test-dir cmake-build/Release --output-on-failure || true; \
+  [ "${RUN_TESTS}" = "OFF" ] || cmake --build ./cmake-build/Release --target check-phasar-unittests check-phasar-cli ; \
   phasar-cli --version
 
 ENTRYPOINT [ "phasar-cli" ]

include/phasar/PhasarLLVM/DataFlow/IfdsIde/Problems/IDEExtendedTaintAnalysis.h

@@ -45,6 +45,12 @@ struct IDEExtendedTaintAnalysisDomain : public LLVMAnalysisDomainDefault {
   /// sanitizer on the current path, Bottom means sanitized on all paths.
   using l_t = XTaint::EdgeDomain;
 };
+
+template <>
+struct DefaultAnalysisPrinterSelector<IDEExtendedTaintAnalysisDomain>
+    : type_identity<
+          DefaultLLVMAnalysisPrinter<IDEExtendedTaintAnalysisDomain>> {};
+
 namespace XTaint {
 
 /// \brief An IDE-based taint analysis that uses k-limited field-access paths to

include/phasar/PhasarLLVM/Utils/LLVMAnalysisPrinter.h

@@ -67,6 +67,11 @@ class DefaultLLVMAnalysisPrinter
   }
 
   void doOnFinalize(llvm::raw_ostream &OS) override {
+    if (DbgResultsEntries.empty() && NonDbgResultsEntries.empty()) {
+      OS << "No leaks found!\n";
+      return;
+    }
+
     size_t Ctr = 0;
     for (const auto &[DbgLoc, Results] : DbgResultsEntries) {
       OS << '#' << ++Ctr << ": ";

lib/CMakeLists.txt

@@ -45,7 +45,6 @@ set(PHASAR_LINK_LIBS
     phasar_llvm_typehierarchy
     phasar_llvm_utils
 
-    phasar_pass
     phasar_passes
     phasar_pointer
     phasar_taintconfig

test/.clang-format

@@ -0,0 +1 @@
+DisableFormat: true

test/README.md

@@ -0,0 +1,2 @@
+Run lit tests with the following command from the phasar source directory:
+lit build/test/llvm_test_code/[FOLDER_OF_INTEREST]/

test/llvm_test_code/CMakeLists.txt

@@ -4,3 +4,50 @@ subdirlist(subdirs ${CMAKE_CURRENT_SOURCE_DIR})
 foreach(subdir ${subdirs})
   add_subdirectory(${subdir})
 endforeach(subdir)
+
+
+if (PHASAR_ENABLE_INTEGRATIONTESTS)
+  include(AddLLVM)
+
+  find_program(LLVM_EXTERNAL_LIT_DEFAULT
+    NAMES lit-22 lit-21 lit-20 lit-16 llvm-lit lit lit.py
+    HINTS "${LLVM_TOOLS_BINARY_DIR}" "${LLVM_INSTALL_PREFIX}/build/utils/lit"
+    REQUIRED
+  )
+
+  set(LLVM_EXTERNAL_LIT "${LLVM_EXTERNAL_LIT_DEFAULT}"
+    CACHE STRING "Command used to spawn lit"
+    FORCE
+  )
+
+  get_filename_component(PSR_LIT_REALPATH ${LLVM_EXTERNAL_LIT} REALPATH)
+  get_filename_component(PSR_LIT_DIR ${PSR_LIT_REALPATH} DIRECTORY)
+  find_program(PSR_FILECHECK
+    NAMES FileCheck-22 FileCheck-21 FileCheck-20 FileCheck
+    HINTS "${PSR_LIT_DIR}" "${LLVM_TOOLS_BINARY_DIR}"
+    REQUIRED
+  )
+
+  # Note: PSR_FILECHECK_DIR configured into lit.site.cfg.py
+  get_filename_component(PSR_FILECHECK_REALPATH ${PSR_FILECHECK} REALPATH)
+  get_filename_component(PSR_FILECHECK_DIR ${PSR_FILECHECK_REALPATH} DIRECTORY)
+
+  if(NOT clang OR NOT clangcpp)
+    message(FATAL_ERROR "Require clang and clang++")
+  endif()
+  get_filename_component(PSR_CLANG_REALPATH ${clang} REALPATH)
+  get_filename_component(PSR_CLANGPP_REALPATH ${clangcpp} REALPATH)
+
+  configure_lit_site_cfg(
+    ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
+    ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
+    MAIN_CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
+  )
+
+  add_lit_testsuite(check-phasar-cli "Running the phasar-cli lit test suite"
+    # Note: Currently, we only have LIT-tests for taint_analysis and xtaint
+    ${CMAKE_CURRENT_BINARY_DIR}/taint_analysis
+    ${CMAKE_CURRENT_BINARY_DIR}/xtaint
+    DEPENDS phasar-cli LLFileGeneration
+  )
+endif(PHASAR_ENABLE_INTEGRATIONTESTS)

test/llvm_test_code/lit.cfg.py

@@ -0,0 +1,35 @@
+import lit.formats
+import os
+
+config.name = "phasar-cli"
+config.test_format = lit.formats.ShTest(True)
+config.suffixes = ['.c', '.cpp']
+
+config.test_source_root = os.path.dirname(__file__)
+
+# Only files with a RUN: line are actual lit tests; everything else under
+# taint_analysis/ and xtaint/ is not converted yet and would otherwise show
+# up as "Unresolved".
+config.excludes = ['Output']
+for root, _, files in os.walk(config.test_source_root):
+  for f in files:
+    if not f.endswith(tuple(config.suffixes)):
+      continue
+    with open(os.path.join(root, f)) as handle:
+      if 'RUN:' not in handle.read():
+        config.excludes.append(f)
+
+config.test_exec_root = config.phasar_obj_root
+
+config.substitutions.append(
+    ('%phasar-cli', os.path.join(config.phasar_cli_dir, 'phasar-cli')))
+config.substitutions.append(
+    ('%llvm_test_code', config.phasar_ll_dir))
+config.substitutions.append(
+    ('%config', config.phasar_config_dir))
+config.substitutions.append(('%clang', config.clang))
+config.substitutions.append(('%clangpp', config.clangpp))
+
+config.environment["PATH"] = os.pathsep.join(
+    [config.phasar_filecheck_dir, config.environment["PATH"]]
+)

test/llvm_test_code/lit.site.cfg.py.in

@@ -0,0 +1,13 @@
+import lit.llvm
+
+config.phasar_config_dir = "@PHASAR_SRC_DIR@/config"
+config.phasar_obj_root = "@CMAKE_CURRENT_BINARY_DIR@"
+config.phasar_cli_dir = "@PHASAR_BINARY_DIR@/tools/phasar-cli/"
+config.phasar_ll_dir = "@PHASAR_BINARY_DIR@/test/llvm_test_code/"
+config.phasar_filecheck_dir = "@PSR_FILECHECK_DIR@"
+config.clang = "@PSR_CLANG_REALPATH@"
+config.clangpp = "@PSR_CLANGPP_REALPATH@"
+
+lit.llvm.initialize(lit_config, config)
+
+lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg.py")

test/llvm_test_code/taint_analysis/CMakeLists.txt

@@ -41,4 +41,5 @@ foreach(TEST_SRC ${NoMem2regSources})
   generate_ll_file(FILE ${TEST_SRC} DEBUG)
 endforeach(TEST_SRC)
 
+add_subdirectory(double_free)
 add_subdirectory(dummy_source_sink)

test/llvm_test_code/taint_analysis/double_free/CMakeLists.txt

@@ -0,0 +1,33 @@
+set(NoMem2regSources
+  df_01.c
+  df_02.c
+  df_03.c
+  df_04_1.c
+  df_04.c
+  df_05.c
+  df_06.c
+  df_07_1.c
+  df_07.c
+  df_08.c
+  df_09.c
+  df_10.c
+  df_11.c
+  df_12.c
+  df_13.c
+  df_14.c
+  df_15.c
+  df_16.c
+  df_17.c
+  df_18.c
+  df_19.c
+  df_20.c
+  df_21.c
+  df_22.c
+  df_ctx_01.c
+  df_ctx_02.c
+  df_ctx_03.c
+)
+
+foreach(TEST_SRC ${NoMem2regSources})
+  generate_ll_file(FILE ${TEST_SRC} DEBUG)
+endforeach(TEST_SRC)

test/llvm_test_code/taint_analysis/double_free/df_01.c

@@ -0,0 +1,22 @@
+#include <stdlib.h>
+int main() {
+  int *foo = (int *)malloc(32);
+  free(foo);
+  free(foo); // vulnerability
+  return 0;
+}
+
+// RUN: %phasar-cli --data-flow-analysis=ide-xtaint --module %llvm_test_code/taint_analysis/double_free/df_01_c_dbg.ll --analysis-config %config/double-free-config.json | FileCheck %s -check-prefix=xtaint
+// xtaint: /taint_analysis/double_free/df_01.c:5:3:
+
+// RUN: %phasar-cli --data-flow-analysis=ifds-taint --module %llvm_test_code/taint_analysis/double_free/df_01_c_dbg.ll --analysis-config %config/double-free-config.json | FileCheck %s -check-prefix=ifds-taint
+// ifds-taint: /taint_analysis/double_free/df_01.c:5:3:
+
+// RUN: %phasar-cli --data-flow-analysis=ifds-fieldsens-taint --module %llvm_test_code/taint_analysis/double_free/df_01_c_dbg.ll --analysis-config %config/double-free-config.json | FileCheck %s -check-prefix=ifds-fieldsens-taint
+// ifds-fieldsens-taint: /taint_analysis/double_free/df_01.c:5:3:
+
+// RUN: %phasar-cli --data-flow-analysis=monoifds-taint --module %llvm_test_code/taint_analysis/double_free/df_01_c_dbg.ll --analysis-config %config/double-free-config.json | FileCheck %s -check-prefix=monoifds-taint
+// monoifds-taint: /taint_analysis/double_free/df_01.c:5:3:
+
+// RUN: %phasar-cli --data-flow-analysis=sparse-ifds-taint --module %llvm_test_code/taint_analysis/double_free/df_01_c_dbg.ll --analysis-config %config/double-free-config.json | FileCheck %s -check-prefix=sparse-ifds-taint
+// sparse-ifds-taint: /taint_analysis/double_free/df_01.c:5:3:

test/llvm_test_code/taint_analysis/double_free/df_01_standalone.c

@@ -0,0 +1,15 @@
+#include <stdlib.h>
+int main() {
+  int *foo = (int *)malloc(32);
+  free(foo);
+  free(foo); // vulnerability
+  return 0;
+}
+
+
+// This file is not integrated in the CMakeLists.txt; below run-commands compile it on-the-fly
+
+// RUN: %clang %s -S -emit-llvm -g -w -o %t
+// RUN: %phasar-cli --data-flow-analysis=ide-xtaint --module %t --analysis-config %config/double-free-config.json | FileCheck %s
+
+// CHECK: taint_analysis/double_free/df_01_standalone.c:5:3:

test/llvm_test_code/taint_analysis/double_free/df_02.c

@@ -0,0 +1,25 @@
+#include <stdlib.h>
+void v() {
+  int *foo = (int *)malloc(32);
+  free(foo);
+  free(foo); // vulnerability
+}
+int main() {
+  v();
+  return 0;
+}
+
+// RUN: %phasar-cli --data-flow-analysis=ide-xtaint --module %llvm_test_code/taint_analysis/double_free/df_02_c_dbg.ll --analysis-config %config/double-free-config.json | FileCheck %s -check-prefix=xtaint
+// xtaint: /taint_analysis/double_free/df_02.c:5:3:
+
+// RUN: %phasar-cli --data-flow-analysis=ifds-taint --module %llvm_test_code/taint_analysis/double_free/df_02_c_dbg.ll --analysis-config %config/double-free-config.json | FileCheck %s -check-prefix=ifds-taint
+// ifds-taint: /taint_analysis/double_free/df_02.c:5:3:
+
+// RUN: %phasar-cli --data-flow-analysis=ifds-fieldsens-taint --module %llvm_test_code/taint_analysis/double_free/df_02_c_dbg.ll --analysis-config %config/double-free-config.json | FileCheck %s -check-prefix=ifds-fieldsens-taint
+// ifds-fieldsens-taint: /taint_analysis/double_free/df_02.c:5:3:
+
+// RUN: %phasar-cli --data-flow-analysis=monoifds-taint --module %llvm_test_code/taint_analysis/double_free/df_02_c_dbg.ll --analysis-config %config/double-free-config.json | FileCheck %s -check-prefix=monoifds-taint
+// monoifds-taint: /taint_analysis/double_free/df_02.c:5:3:
+
+// RUN: %phasar-cli --data-flow-analysis=sparse-ifds-taint --module %llvm_test_code/taint_analysis/double_free/df_02_c_dbg.ll --analysis-config %config/double-free-config.json | FileCheck %s -check-prefix=sparse-ifds-taint
+// sparse-ifds-taint: /taint_analysis/double_free/df_02.c:5:3:

test/llvm_test_code/taint_analysis/double_free/df_03.c

@@ -0,0 +1,25 @@
+#include <stdlib.h>
+void v(int *foo) {
+  free(foo);
+  free(foo); // vulnerability
+}
+int main() {
+  int *foo = (int *)malloc(32);
+  v(foo);
+  return 0;
+}
+
+// RUN: %phasar-cli --data-flow-analysis=ide-xtaint --module %llvm_test_code/taint_analysis/double_free/df_03_c_dbg.ll --analysis-config %config/double-free-config.json | FileCheck %s -check-prefix=xtaint
+// xtaint: /taint_analysis/double_free/df_03.c:4:3:
+
+// RUN: %phasar-cli --data-flow-analysis=ifds-taint --module %llvm_test_code/taint_analysis/double_free/df_03_c_dbg.ll --analysis-config %config/double-free-config.json | FileCheck %s -check-prefix=ifds-taint
+// ifds-taint: /taint_analysis/double_free/df_03.c:4:3:
+
+// RUN: %phasar-cli --data-flow-analysis=ifds-fieldsens-taint --module %llvm_test_code/taint_analysis/double_free/df_03_c_dbg.ll --analysis-config %config/double-free-config.json | FileCheck %s -check-prefix=ifds-fieldsens-taint
+// ifds-fieldsens-taint: /taint_analysis/double_free/df_03.c:4:3:
+
+// RUN: %phasar-cli --data-flow-analysis=monoifds-taint --module %llvm_test_code/taint_analysis/double_free/df_03_c_dbg.ll --analysis-config %config/double-free-config.json | FileCheck %s -check-prefix=monoifds-taint
+// monoifds-taint: /taint_analysis/double_free/df_03.c:4:3:
+
+// RUN: %phasar-cli --data-flow-analysis=sparse-ifds-taint --module %llvm_test_code/taint_analysis/double_free/df_03_c_dbg.ll --analysis-config %config/double-free-config.json | FileCheck %s -check-prefix=sparse-ifds-taint
+// sparse-ifds-taint: /taint_analysis/double_free/df_03.c:4:3:

test/llvm_test_code/taint_analysis/double_free/df_04.c

@@ -0,0 +1,25 @@
+#include <stdlib.h>
+void v(int *foo) {
+  free(foo); // vulnerability
+}
+int main() {
+  int *foo = (int *)malloc(32);
+  free(foo);
+  v(foo);
+  return 0;
+}
+
+// RUN: %phasar-cli --data-flow-analysis=ide-xtaint --module %llvm_test_code/taint_analysis/double_free/df_04_c_dbg.ll --analysis-config %config/double-free-config.json | FileCheck %s -check-prefix=xtaint
+// xtaint: /taint_analysis/double_free/df_04.c:3:3:
+
+// RUN: %phasar-cli --data-flow-analysis=ifds-taint --module %llvm_test_code/taint_analysis/double_free/df_04_c_dbg.ll --analysis-config %config/double-free-config.json | FileCheck %s -check-prefix=ifds-taint
+// ifds-taint: /taint_analysis/double_free/df_04.c:3:3:
+
+// RUN: %phasar-cli --data-flow-analysis=ifds-fieldsens-taint --module %llvm_test_code/taint_analysis/double_free/df_04_c_dbg.ll --analysis-config %config/double-free-config.json | FileCheck %s -check-prefix=ifds-fieldsens-taint
+// ifds-fieldsens-taint: /taint_analysis/double_free/df_04.c:3:3:
+
+// RUN: %phasar-cli --data-flow-analysis=monoifds-taint --module %llvm_test_code/taint_analysis/double_free/df_04_c_dbg.ll --analysis-config %config/double-free-config.json | FileCheck %s -check-prefix=ifds-fieldsens-taint
+// monoifds-taint: /taint_analysis/double_free/df_04.c:3:3:
+
+// RUN: %phasar-cli --data-flow-analysis=sparse-ifds-taint --module %llvm_test_code/taint_analysis/double_free/df_04_c_dbg.ll --analysis-config %config/double-free-config.json | FileCheck %s -check-prefix=sparse-ifds-taint
+// sparse-ifds-taint: /taint_analysis/double_free/df_04.c:3:3: