Prototype 2: Evaluating alternatives to YAJL for JSON processing

.gitmodules

@@ -10,3 +10,9 @@
 [submodule "others/mbedtls"]
 	path = others/mbedtls
 	url = https://github.com/Mbed-TLS/mbedtls.git
+[submodule "others/simdjson"]
+	path = others/simdjson
+	url = https://github.com/simdjson/simdjson.git
+[submodule "others/jsoncons"]
+	path = others/jsoncons
+	url = https://github.com/danielaparker/jsoncons.git

README.md

@@ -253,6 +253,12 @@ It is recommended to use assertions where applicable, and to enable them with
 
 The source tree includes a Benchmark tool that can help measure library performance. The tool is located in the `test/benchmark/` directory. The build process also creates the binary here, so you will have the tool after the compilation is finished.
 
+To compare the JSON regression suite across both supported JSON backends with separate build directories, run:
+
+```shell
+$ ./test/run-json-backend-matrix.sh --jobs 4
+```
+
 To run, just type:
 
 ```shell
@@ -323,6 +329,8 @@ Note that the tool does not call the last phase (logging).
 
 Please remember to reset `basic_rules.conf` if you want to try with a different ruleset.
 
+The benchmark directory also includes `json_benchmark`, which targets JSON request-body processing with fixed scenario classes such as large objects, deep nesting, numeric payloads and UTF-8-heavy strings.
+
 ## Reporting Issues
 
 If you are facing a configuration issue or something is not working as you

build/msc_find_lib.m4

@@ -10,11 +10,11 @@ dnl Sets and AC_SUBSTs:
 dnl   ${NAME}_CFLAGS, ${NAME}_LDADD, ${NAME}_LDFLAGS,
 dnl   ${NAME}_VERSION, ${NAME}_DISPLAY, ${NAME}_FOUND (0/1/2)
 dnl
-dnl NAME         - Variable prefix (e.g., YAJL, CURL, LIBXML2)
+dnl NAME         - Variable prefix (e.g., CURL, LIBXML2, LMDB)
 dnl PKG_NAMES    - Space-separated pkg-config names to try
-dnl HEADER       - Header file to look for (e.g., yajl/yajl_parse.h)
+dnl HEADER       - Header file to look for (e.g., libxml/parser.h)
 dnl LIB_NAMES    - Space-separated library names for -l flags
-dnl EXTRA_CFLAGS - Additional CFLAGS when found (e.g., -DWITH_YAJL)
+dnl EXTRA_CFLAGS - Additional CFLAGS when found (e.g., -DWITH_LIBXML2)
 dnl MIN_VERSION  - Optional minimum version for pkg-config check
 dnl WITH_NAME    - Optional --with-X name if different from lowercased NAME
 
@@ -208,7 +208,7 @@ if test "${_msc_header_dir}" = "."; then
         _msc_check_inc_path="$4"
     fi
 else
-    # Header with subdirectory (e.g., "yajl/yajl_parse.h")
+    # Header with subdirectory (e.g., "libxml/parser.h")
     if test -e "$4/include/$2"; then
         _msc_check_inc_path="$4/include"
     elif test -e "$4/$2"; then

build/win32/CMakeLists.txt

@@ -7,6 +7,8 @@ option(WITH_LUA     "Include LUA support"     ON)
 option(WITH_LIBXML2 "Include LibXML2 support" ON)
 option(WITH_MAXMIND "Include MaxMind support" ON)
 option(WITH_CURL    "Include CURL support"    ON)
+set(JSON_BACKEND "simdjson" CACHE STRING "Select internal JSON backend (simdjson or jsoncons)")
+set_property(CACHE JSON_BACKEND PROPERTY STRINGS simdjson jsoncons)
 
 option(USE_ASAN     "Build with Address Sanitizer" OFF)
 
@@ -51,6 +53,8 @@ target_compile_definitions(libinjection PRIVATE LIBINJECTION_VERSION="${LIBINJEC
 project(mbedcrypto C)
 
 set(MBEDTLS_DIR ${BASE_DIR}/others/mbedtls)
+set(SIMDJSON_DIR ${BASE_DIR}/others/simdjson/singleheader)
+set(JSONCONS_DIR ${BASE_DIR}/others/jsoncons/include)
 
 add_library(mbedcrypto STATIC ${MBEDTLS_DIR}/library/base64.c ${MBEDTLS_DIR}/library/sha1.c ${MBEDTLS_DIR}/library/md5.c ${MBEDTLS_DIR}/library/platform_util.c ${MBEDTLS_DIR}/library/constant_time.c)
 
@@ -87,7 +91,25 @@ set(PACKAGE_VERSION "${PROJECT_VERSION}")
 set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}")
 set(PACKAGE_TARNAME "${PACKAGE_NAME}")
 
-set(HAVE_YAJL     1)    # should always be one, mandatory dependency
+if(NOT JSON_BACKEND STREQUAL "simdjson" AND NOT JSON_BACKEND STREQUAL "jsoncons")
+  message(FATAL_ERROR "Unsupported JSON_BACKEND '${JSON_BACKEND}'. Use simdjson or jsoncons.")
+endif()
+
+unset(MSC_JSON_BACKEND_SIMDJSON)
+unset(MSC_JSON_BACKEND_JSONCONS)
+set(JSON_BACKEND_SIMDJSON_SOURCE ${BASE_DIR}/src/request_body_processor/json_backend_simdjson.cc)
+set(JSON_BACKEND_JSONCONS_SOURCE ${BASE_DIR}/src/request_body_processor/json_backend_jsoncons.cc)
+
+if(JSON_BACKEND STREQUAL "simdjson")
+  set(MSC_JSON_BACKEND_SIMDJSON 1)
+  set(JSON_BACKEND_SOURCES ${JSON_BACKEND_SIMDJSON_SOURCE} ${SIMDJSON_DIR}/simdjson.cpp)
+  set(JSON_BACKEND_INCLUDE_DIR ${SIMDJSON_DIR})
+else()
+  set(MSC_JSON_BACKEND_JSONCONS 1)
+  set(JSON_BACKEND_SOURCES ${JSON_BACKEND_JSONCONS_SOURCE})
+  set(JSON_BACKEND_INCLUDE_DIR ${JSONCONS_DIR})
+endif()
+
 set(HAVE_GEOIP    0)    # should always be zero, no conan package available
 set(HAVE_SSDEEP   0)    # should always be zero, no conan package available
 
@@ -119,7 +141,6 @@ macro(include_package package flag)
   endif()
 endmacro()
 
-include_package(yajl HAVE_YAJL)
 include_package(libxml2 HAVE_LIBXML2)
 include_package(lua HAVE_LUA)
 include_package(CURL HAVE_CURL)
@@ -133,11 +154,13 @@ include_package(maxminddb HAVE_MAXMIND)
 set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
 
 file(GLOB_RECURSE libModSecuritySources ${BASE_DIR}/src/*.cc)
+list(REMOVE_ITEM libModSecuritySources ${JSON_BACKEND_SIMDJSON_SOURCE} ${JSON_BACKEND_JSONCONS_SOURCE})
+list(APPEND libModSecuritySources ${JSON_BACKEND_SOURCES})
 
 add_library(libModSecurity SHARED ${libModSecuritySources})
 
 target_compile_definitions(libModSecurity PRIVATE WITH_PCRE2)
-target_include_directories(libModSecurity PRIVATE ${BASE_DIR} ${BASE_DIR}/headers ${BASE_DIR}/others ${MBEDTLS_DIR}/include)
+target_include_directories(libModSecurity PRIVATE ${BASE_DIR} ${BASE_DIR}/headers ${BASE_DIR}/others ${MBEDTLS_DIR}/include ${JSON_BACKEND_INCLUDE_DIR})
 target_link_libraries(libModSecurity PRIVATE pcre2::pcre2 libinjection mbedcrypto Poco::Poco Iphlpapi.lib)
 
 macro(add_package_dependency project compile_definition link_library flag)
@@ -147,7 +170,6 @@ macro(add_package_dependency project compile_definition link_library flag)
   endif()
 endmacro()
 
-add_package_dependency(libModSecurity WITH_YAJL yajl::yajl HAVE_YAJL)
 add_package_dependency(libModSecurity WITH_LIBXML2 LibXml2::LibXml2 HAVE_LIBXML2)
 add_package_dependency(libModSecurity WITH_LUA lua::lua HAVE_LUA)
 if(HAVE_LUA)
@@ -164,9 +186,8 @@ project(libModSecurityTests)
 
 function(setTestTargetProperties executable)
   target_compile_definitions(${executable} PRIVATE WITH_PCRE2)
-  target_include_directories(${executable} PRIVATE ${BASE_DIR} ${BASE_DIR}/headers)
+  target_include_directories(${executable} PRIVATE ${BASE_DIR} ${BASE_DIR}/headers ${JSONCONS_DIR})
   target_link_libraries(${executable} PRIVATE libModSecurity pcre2::pcre2 dirent::dirent)
-  add_package_dependency(${executable} WITH_YAJL yajl::yajl HAVE_YAJL)
 endfunction()
 
 # unit tests

build/win32/conanfile.txt

@@ -1,5 +1,4 @@
 [requires]
-yajl/2.1.0
 pcre2/10.42
 libxml2/2.12.6
 lua/5.4.6

build/win32/config.h.cmake

@@ -57,12 +57,15 @@
 /* Define if SSDEEP is available */
 #cmakedefine HAVE_SSDEEP
 
-/* Define if YAJL is available */
-#cmakedefine HAVE_YAJL
-
 /* Define if libcurl is available */
 #cmakedefine HAVE_CURL
 
+/* Define if jsoncons is the selected internal JSON backend */
+#cmakedefine MSC_JSON_BACKEND_JSONCONS
+
+/* Define if simdjson is the selected internal JSON backend */
+#cmakedefine MSC_JSON_BACKEND_SIMDJSON
+
 /* Name of package */
 #define PACKAGE "@PACKAGE_NAME@"
 
@@ -89,4 +92,4 @@
 #cmakedefine STDC_HEADERS
 #endif
 
-#endif // ndef MODSECURITY_CONFIG_H
+#endif // ndef MODSECURITY_CONFIG_H

configure.ac

@@ -27,7 +27,7 @@ m4_define([msc_version_with_patchlevel],
     [msc_version_major.msc_version_minor.msc_version_patchlevel])
 
 m4_define([msc_version_git],
-    [m4_esyscmd_s(git describe)])
+    [m4_esyscmd_s(git describe --tags --always 2>/dev/null || echo unknown)])
 
 m4_define([msc_version_info],
     [msc_version_c_plus_a:msc_version_patchlevel:msc_version_minor])
@@ -62,6 +62,23 @@ PKG_PROG_PKG_CONFIG
 # Set C++ standard version and check if compiler supports it.
 AX_CXX_COMPILE_STDCXX(17, noext, mandatory)
 
+AC_ARG_WITH([json-backend],
+    [AS_HELP_STRING([--with-json-backend=BACKEND],
+        [Select internal JSON backend: simdjson or jsoncons [default=simdjson]])],
+    [json_backend="$withval"],
+    [json_backend="simdjson"])
+
+case "$json_backend" in
+    simdjson|jsoncons)
+        ;;
+    *)
+        AC_MSG_ERROR([Unsupported JSON backend '$json_backend'. Use simdjson or jsoncons.])
+        ;;
+esac
+
+JSON_BACKEND="$json_backend"
+AC_SUBST([JSON_BACKEND])
+
 # Check for libinjection
 if ! test -f "${srcdir}/others/libinjection/src/libinjection_html5.c"; then
 AC_MSG_ERROR([\
@@ -80,7 +97,7 @@ AC_MSG_ERROR([\
    ])
 fi
 # Libinjection version
-AC_DEFUN([LIBINJECTION_VERSION], m4_esyscmd_s(cd "others/libinjection" && git describe && cd ../..))
+AC_DEFUN([LIBINJECTION_VERSION], m4_esyscmd_s(cd "others/libinjection" && (git describe --tags --always 2>/dev/null || echo unknown) && cd ../..))
 AC_SUBST([LIBINJECTION_VERSION])
 
 # Check for Mbed TLS
@@ -101,16 +118,68 @@ AC_MSG_ERROR([\
    ])
 fi
 # Mbed TLS version
-AC_DEFUN([MBEDTLS_VERSION], m4_esyscmd_s(cd "others/mbedtls" && git describe && cd ../..))
+AC_DEFUN([MBEDTLS_VERSION], m4_esyscmd_s(cd "others/mbedtls" && (git describe --tags --always 2>/dev/null || echo unknown) && cd ../..))
 
-# SecLang test version
-AC_DEFUN([SECLANG_TEST_VERSION], m4_esyscmd_s(cd "test/test-cases/secrules-language-tests" && git log -1 --format="%h" --abbrev-commit && cd ../../..))
+# Check for selected JSON backend
+if test "x$json_backend" = "xsimdjson"; then
+if ! test -f "${srcdir}/others/simdjson/singleheader/simdjson.h" || \
+   ! test -f "${srcdir}/others/simdjson/singleheader/simdjson.cpp"; then
+AC_MSG_ERROR([\
+
+
+  simdjson was not found within ModSecurity source directory.
+
+  simdjson code is available as part of ModSecurity source code in a format
+  of a git-submodule. git-submodule allow us to specify the correct version of
+  simdjson and still uses the simdjson repository to download it.
+
+  You can download simdjson using git:
+
+     $ git submodule update --init --recursive
+
+   ])
+fi
+JSON_BACKEND_VERSION=`cd "${srcdir}/others/simdjson" && git describe --tags --always 2>/dev/null || echo unknown`
+AC_DEFINE([MSC_JSON_BACKEND_SIMDJSON], [1],
+    [Define if simdjson is the selected internal JSON backend])
+elif test "x$json_backend" = "xjsoncons"; then
+if ! test -d "${srcdir}/others/jsoncons/include" || \
+   ! test -f "${srcdir}/others/jsoncons/include/jsoncons/json.hpp"; then
+AC_MSG_ERROR([\
+
+
+  jsoncons was not found within ModSecurity source directory.
+
+  jsoncons code is available as part of ModSecurity source code in a format
+  of a git-submodule. git-submodule allow us to specify the correct version of
+  jsoncons and still uses the jsoncons repository to download it.
+
+  You can download jsoncons using git:
+
+     $ git submodule update --init --recursive
 
+   ])
+fi
+JSON_BACKEND_VERSION=`cd "${srcdir}/others/jsoncons" && git describe --tags --always 2>/dev/null || echo unknown`
+AC_DEFINE([MSC_JSON_BACKEND_JSONCONS], [1],
+    [Define if jsoncons is the selected internal JSON backend])
+fi
+AC_SUBST([JSON_BACKEND_VERSION])
+
+AC_ARG_ENABLE([json-audit-instrumentation],
+    [AS_HELP_STRING([--enable-json-audit-instrumentation],
+        [Enable optional JSON audit instrumentation for benchmark builds [default=no]])],
+    [enable_json_audit_instrumentation="$enableval"],
+    [enable_json_audit_instrumentation="no"])
 
-# Check for yajl
-PROG_YAJL
+AS_CASE([$enable_json_audit_instrumentation],
+    [yes], [AC_DEFINE([MSC_JSON_AUDIT_INSTRUMENTATION], [1],
+                [Define if optional JSON audit instrumentation is enabled])],
+    [no], [],
+    [AC_MSG_ERROR([Unsupported value '$enable_json_audit_instrumentation' for --enable-json-audit-instrumentation. Use yes or no.])])
 
-AM_CONDITIONAL([YAJL_VERSION], [test "$YAJL_VERSION" != ""])
+# SecLang test version
+AC_DEFUN([SECLANG_TEST_VERSION], m4_esyscmd_s(cd "test/test-cases/secrules-language-tests" && git log -1 --format="%h" --abbrev-commit && cd ../../..))
 
 # Check for LibGeoIP
 PROG_GEOIP
@@ -306,14 +375,7 @@ fi
 
 
 # Decide if we want to build the tests or not.
-buildTestUtilities=false
-if test "x$YAJL_FOUND" = "x1"; then
-    # Regression tests will not be able to run without the logging support.
-    # But we still have the unit tests.
-    # if test "$debugLogs" = "true"; then
-        buildTestUtilities=true
-    # fi
-fi
+buildTestUtilities=true
 
 
 AM_CONDITIONAL([TEST_UTILITIES], [test $buildTestUtilities = true])
@@ -328,6 +390,8 @@ fi
 AM_CONDITIONAL([EXAMPLES], [test $buildExamples = true])
 AM_CONDITIONAL([BUILD_PARSER], [test $buildParser = true])
 AM_CONDITIONAL([USE_MUTEX_ON_PM], [test $mutexPm = true])
+AM_CONDITIONAL([JSON_BACKEND_SIMDJSON], [test "x$json_backend" = "xsimdjson"])
+AM_CONDITIONAL([JSON_BACKEND_JSONCONS], [test "x$json_backend" = "xjsoncons"])
 
 
 # General link options
@@ -422,6 +486,8 @@ AS_ECHO_N("   + libInjection                                  ....")
 echo LIBINJECTION_VERSION
 AS_ECHO_N("   + Mbed TLS                                      ....")
 echo MBEDTLS_VERSION
+AS_ECHO_N("   + JSON backend                                  ....")
+echo "$JSON_BACKEND ($JSON_BACKEND_VERSION)"
 AS_ECHO_N("   + SecLang tests                                 ....")
 echo SECLANG_TEST_VERSION
 
@@ -451,7 +517,6 @@ if test "x$GEOIP_FOUND" = "x2" && test "x$MAXMIND_FOUND" = "x2"; then
 fi
 
 MSC_STATUS_LIB([LibCURL                                      ], [CURL])
-MSC_STATUS_LIB([YAJL                                         ], [YAJL])
 MSC_STATUS_LIB([LMDB                                         ], [LMDB])
 MSC_STATUS_LIB([LibXML2                                      ], [LIBXML2])
 MSC_STATUS_LIB([SSDEEP                                       ], [SSDEEP])
@@ -532,4 +597,3 @@ if test "$aflFuzzer" = "true"; then
     echo "  $ export CC=afl-clang-fast "
     echo " "
 fi
-

examples/multiprocess_c/Makefile.am

@@ -19,15 +19,12 @@ multi_LDFLAGS = \
 	-lstdc++ \
 	$(LUA_LDFLAGS) \
 	$(SSDEEP_LDFLAGS) \
-	$(MAXMIND_LDFLAGS) \
-	$(YAJL_LDFLAGS)
+	$(MAXMIND_LDFLAGS)
 
 multi_CFLAGS = \
-	-I$(top_builddir)/headers \
+	-I$(top_srcdir)/headers \
 	-I$(top_builddir) \
 	$(GLOBAL_CFLAGS)
 
 MAINTAINERCLEANFILES = \
 	Makefile.in
-
-