Test embedded for CI

.github/workflows/cpp-bindings.yml

@@ -6,7 +6,7 @@ on:
       - ".github/workflows/cpp-bindings.yml"
       - "include/**"
       - "src/**"
-      - "cpp/**"
+      - "test/cpp/**"
       - "*akefile*"
     branches:
       - main
@@ -40,6 +40,6 @@ jobs:
       - name: Compile prism
         run: bundle exec rake compile
       - name: Compile tests
-        run: ${{ matrix.compiler }} ${{ matrix.standard }} ${{ matrix.exceptions }} -Wall -Wextra -pedantic -Werror -o ./cpp_test cpp/*_test.cpp build/static/*.o -Iinclude
+        run: ${{ matrix.compiler }} ${{ matrix.standard }} ${{ matrix.exceptions }} -Wall -Wextra -pedantic -Werror -o ./cpp_test test/cpp/*_test.cpp build/static/*.o -Iinclude
       - name: Run
         run: ./cpp_test

.github/workflows/main.yml

@@ -176,6 +176,38 @@ jobs:
       env:
         PRISM_BUILD_MINIMAL: "1"
 
+  build-arm-none-eabi:
+    strategy:
+      fail-fast: false
+      matrix:
+        cflags:
+          - ""
+          - "-DPRISM_BUILD_MINIMAL"
+    runs-on: ubuntu-24.04
+    env:
+      CC: arm-none-eabi-gcc
+      AR: arm-none-eabi-ar
+      CFLAGS: -mcpu=cortex-m33 -mthumb ${{ matrix.cflags }}
+    steps:
+    - uses: actions/checkout@v7
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ruby
+        bundler-cache: true
+    - name: Install toolchain
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y gcc-arm-none-eabi libnewlib-arm-none-eabi qemu-user
+    - name: Generate templates
+      run: bundle exec rake templates
+    - name: Build static library
+      run: make -j$(nproc) static
+    - name: Build smoke test
+      run: $CC $CFLAGS -O2 -std=c99 -Wall -Werror -Wextra -Wpedantic -Iinclude test/embedded/main.c build/libprism.a --specs=rdimon.specs -o build/embedded.elf
+    - name: Run smoke test
+      run: qemu-arm -cpu cortex-m33 build/embedded.elf
+
   build-with-various-compilers:
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -31,7 +31,7 @@ a.out
 out.svg
 
 /ext/prism/api_node.c
-/fuzz/output/
+/test/fuzz/output/
 /gemfiles/typecheck/bin/
 /include/prism/ast.h
 /include/prism/ast.hpp

Makefile

@@ -6,7 +6,7 @@ Q1 = $(V:1=)
 Q = $(Q1:0=@)
 ECHO1 = $(V:1=@ :)
 ECHO = $(ECHO1:0=@ echo)
-FUZZ_OUTPUT_DIR = $(CURDIR)/fuzz/output
+FUZZ_OUTPUT_DIR = $(CURDIR)/test/fuzz/output
 
 SOEXT ?= $(shell ruby -e 'puts RbConfig::CONFIG["SOEXT"]')
 
@@ -94,24 +94,24 @@ build/static/%.o: src/%.c Makefile $(HEADERS)
 	$(Q) $(MAKEDIRS) $(@D)
 	$(Q) $(CC) $(DEBUG_FLAGS) $(CPPFLAGS) $(CFLAGS) -c -o $@ $<
 
-build/fuzz.%: $(SOURCES) fuzz/%.c fuzz/fuzz.c
+build/fuzz.%: $(SOURCES) test/fuzz/%.c test/fuzz/fuzz.c
 	$(ECHO) "building $* fuzzer"
 	$(Q) $(MAKEDIRS) $(@D)
 	$(ECHO) "building main fuzz binary"
 	$(Q) afl-clang-lto $(DEBUG_FLAGS) $(CPPFLAGS) $(CFLAGS) $(FUZZ_FLAGS) -O0 -fsanitize=fuzzer,address -ggdb3 -std=c99 -Iinclude -o $@ $^
 	$(ECHO) "building cmplog binary"
 	$(Q) AFL_LLVM_CMPLOG=1 afl-clang-lto $(DEBUG_FLAGS) $(CPPFLAGS) $(CFLAGS) $(FUZZ_FLAGS) -O0 -fsanitize=fuzzer,address -ggdb3 -std=c99 -Iinclude -o $@.cmplog $^
 
-build/fuzz.heisenbug.%: $(SOURCES) fuzz/%.c fuzz/heisenbug.c
+build/fuzz.heisenbug.%: $(SOURCES) test/fuzz/%.c test/fuzz/heisenbug.c
 	$(Q) afl-clang-lto $(DEBUG_FLAGS) $(CPPFLAGS) $(CFLAGS) $(FUZZ_FLAGS) -O0 -fsanitize=fuzzer,address -ggdb3 -std=c99 -Iinclude -o $@ $^
 
 fuzz-debug:
 	$(ECHO) "entering debug shell"
-	$(Q) docker run -it --rm -e HISTFILE=/prism/fuzz/output/.bash_history -v $(CURDIR):/prism -v $(FUZZ_OUTPUT_DIR):/fuzz_output prism/fuzz
+	$(Q) docker run -it --rm -e HISTFILE=/prism/test/fuzz/output/.bash_history -v $(CURDIR):/prism -v $(FUZZ_OUTPUT_DIR):/fuzz_output prism/fuzz
 
-fuzz-docker-build: fuzz/docker/Dockerfile
+fuzz-docker-build: test/fuzz/docker/Dockerfile
 	$(ECHO) "building docker image"
-	$(Q) docker build -t prism/fuzz fuzz/docker/
+	$(Q) docker build -t prism/fuzz test/fuzz/docker/
 
 fuzz-run-%: FORCE fuzz-docker-build
 	$(ECHO) "generating templates"
@@ -120,11 +120,11 @@ fuzz-run-%: FORCE fuzz-docker-build
 	$(Q) docker run --rm -v $(CURDIR):/prism prism/fuzz /bin/bash -c "FUZZ_FLAGS=\"$(FUZZ_FLAGS)\" make build/fuzz.$*"
 	$(ECHO) "starting AFL++ run"
 	$(Q) $(MAKEDIRS) $(FUZZ_OUTPUT_DIR)/$*
-	$(Q) docker run -it --rm -v $(CURDIR):/prism -v $(FUZZ_OUTPUT_DIR):/fuzz_output prism/fuzz /bin/bash -c "./fuzz/$*.sh /fuzz_output/$*"
+	$(Q) docker run -it --rm -v $(CURDIR):/prism -v $(FUZZ_OUTPUT_DIR):/fuzz_output prism/fuzz /bin/bash -c "./test/fuzz/$*.sh /fuzz_output/$*"
 FORCE:
 
 fuzz-clean:
-	$(Q) $(RMALL) fuzz/output
+	$(Q) $(RMALL) test/fuzz/output
 
 clean:
 	$(Q) $(RMALL) build

README.md

@@ -49,7 +49,6 @@ The repository contains the infrastructure for both a shared library (libprism)
 │   └── prism
 │       ├── extconf.rb    configuration to generate the Makefile for the native extension
 │       └── extension.c   the native extension that interacts with libprism
-├── fuzz                  files related to fuzz testing
 ├── gemfiles              gemfiles used by different Ruby versions in CI
 ├── include
 │   ├── prism             header files for the shared library
@@ -75,6 +74,9 @@ The repository contains the infrastructure for both a shared library (libprism)
 ├── templates             contains ERB templates generated by templates/template.rb
 │   └── template.rb       generates code from the nodes and tokens configured by config.yml
 └── test
+    ├── cpp               tests for the C++ bindings
+    ├── embedded          smoke test for bare-metal targets
+    ├── fuzz              files related to fuzz testing
     └── prism
         ├── fixtures      Ruby code used for testing
         └── snapshots     snapshots of generated syntax trees corresponding to fixtures

docs/fuzzing.md

@@ -1,9 +1,9 @@
 # Fuzzing
 
-We use fuzzing to test the various entrypoints to the library. The fuzzer we use is [AFL++](https://aflplus.plus). All files related to fuzzing live within the `fuzz` directory, which has the following structure:
+We use fuzzing to test the various entrypoints to the library. The fuzzer we use is [AFL++](https://aflplus.plus). All files related to fuzzing live within the `test/fuzz` directory, which has the following structure:
 
 ```
-fuzz
+test/fuzz
 ├── corpus
 │   └── parse             fuzzing corpus for parsing (a symlink to our fixtures)
 ├── dict                  a AFL++ dictionary containing various tokens
@@ -57,20 +57,20 @@ make build/fuzz.heisenbug.parse
 Triaging crashes and hangs is easier when the inputs are as short as possible. In the fuzz container, an entire crash or hang directory can be minimized using
 
 ```
-./fuzz/tools/minimize.sh <directory>
+./test/fuzz/tools/minimize.sh <directory>
 ```
 
 e.g.
 ```
-./fuzz/tools/minimize.sh fuzz/output/parse/default/crashes
+./test/fuzz/tools/minimize.sh test/fuzz/output/parse/default/crashes
 ```
 
 This may take a long time. In the crash/hang directory, for each input file there will appear a minimized version with the extension `.min` appended.
 
 Backtraces for crashes (not hangs) can be generated en masse with
 
 ```
-./fuzz/tools/backtrace.sh <directory>
+./test/fuzz/tools/backtrace.sh <directory>
 ```
 
 Files with basename equal to the input file name with extension `.bt` will be created e.g.

rakelib/check_manifest.rake

@@ -14,9 +14,7 @@ task check_manifest: :templates do
     autom4te.cache
     bin
     build
-    cpp
     doc
-    fuzz
     gemfiles
     javascript
     java

src/errors_format.c

@@ -8,10 +8,6 @@
 #include "prism/internal/parser.h"
 
 #include <assert.h>
-// Include sys/types.h before inttypes.h to work around an issue with certain
-// versions of GCC and newlib which causes omission of the 64-bit PRI macros
-// (see 61ebe5123; the removal of defines.h dropped it).
-#include <sys/types.h>
 #include <inttypes.h>
 #include <stdbool.h>
 #include <stdlib.h>

src/integer.c

@@ -3,12 +3,13 @@
 #include "prism/internal/allocator.h"
 #include "prism/internal/buffer.h"
 
-#include <assert.h>
-// Include sys/types.h before inttypes.h to work around an issue with certain
-// versions of GCC and newlib which causes omission of PRIu64 and the other
-// 64-bit macros (see 61ebe5123; the removal of defines.h dropped it).
+/* Include sys/types.h before inttypes.h to work around an issue with certain
+ * versions of GCC and newlib which causes omission of PRIu64 and the other
+ * 64-bit macros. */
 #include <sys/types.h>
 #include <inttypes.h>
+
+#include <assert.h>
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdlib.h>

src/static_literals.c

@@ -12,6 +12,7 @@
 #include "prism/internal/xxh3.h"
 
 #include <assert.h>
+#include <inttypes.h>
 #include <math.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -534,7 +535,7 @@ pm_static_literal_inspect(pm_buffer_t *buffer, const pm_parser_t *parser, const
             break;
         }
         case PM_SOURCE_LINE_NODE:
-            pm_buffer_append_format(buffer, "%d", pm_line_offset_list_line_column(&parser->line_offsets, node->location.start, parser->start_line).line);
+            pm_buffer_append_format(buffer, "%" PRIi32, pm_line_offset_list_line_column(&parser->line_offsets, node->location.start, parser->start_line).line);
             break;
         case PM_STRING_NODE: {
             const pm_string_t *unescaped = &((const pm_string_node_t *) node)->unescaped;

test/embedded/main.c

@@ -0,0 +1,55 @@
+#include "prism.h"
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+int
+main(void) {
+    static const char source[] = "def foo = bar(0x1FFFFFFFFFFFFFFFF)\n";
+
+    pm_arena_t *arena = pm_arena_new();
+    pm_parser_t *parser = pm_parser_new(arena, (const uint8_t *) source, sizeof(source) - 1, NULL);
+    pm_node_t *node = pm_parse(parser);
+    int status = EXIT_SUCCESS;
+
+    if (PM_NODE_TYPE(node) != PM_PROGRAM_NODE) {
+        printf("expected a program node, got type %d\n", (int) PM_NODE_TYPE(node));
+        status = EXIT_FAILURE;
+    }
+
+    if (pm_parser_errors_size(parser) != 0) {
+        printf("expected no errors, got %zu\n", pm_parser_errors_size(parser));
+        status = EXIT_FAILURE;
+    }
+
+#ifndef PRISM_EXCLUDE_PRETTYPRINT
+    pm_buffer_t *buffer = pm_buffer_new();
+    pm_prettyprint(buffer, parser, node);
+
+    const char *haystack = pm_buffer_value(buffer);
+    const char *haystack_end = haystack + pm_buffer_length(buffer);
+    const char *needle = "36893488147419103231";
+    size_t needle_length = strlen(needle);
+
+    bool found = false;
+
+    for (const char *cursor = haystack; !found && (cursor = memchr(cursor, needle[0], (size_t) (haystack_end - cursor))) != NULL; cursor++) {
+        found = (size_t) (haystack_end - cursor) >= needle_length && memcmp(cursor, needle, needle_length) == 0;
+    }
+
+    if (!found) {
+        fprintf(stderr, "expected the pretty-printed tree to contain %s\n", needle);
+        status = EXIT_FAILURE;
+    }
+
+    pm_buffer_free(buffer);
+#endif
+
+    pm_parser_free(parser);
+    pm_arena_free(arena);
+
+    if (status == EXIT_SUCCESS) printf("ok\n");
+    return status;
+}

test/fuzz/corpus/parse

@@ -0,0 +1 @@
+../../prism/fixtures

test/fuzz/parse.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+OUTPUT_DIR=$1
+
+screen -S grammar_fuzzer -d -m  /bin/bash -c "AFL_CUSTOM_MUTATOR_LIBRARY=/usr/local/lib/libgrammarmutator-ruby.so AFL_CUSTOM_MUTATOR_ONLY=1 afl-fuzz -i test/fuzz/corpus/parse -o $OUTPUT_DIR -S grammar -- ./build/fuzz.parse"
+
+screen -S main_fuzzer -m /bin/bash -c "afl-fuzz -x ./test/fuzz/dict -G 500 -c ./build/fuzz.parse.cmplog -i ./test/fuzz/corpus/parse -M main_parse -o $OUTPUT_DIR ./build/fuzz.parse || read -n 1"
+
+exec /bin/bash