diff --git a/CHANGELOG.md b/CHANGELOG.md index d802877fba..60b6e7e00c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +**Features**: + +- Add `on_crashed_last_run` callback for inspecting crash envelopes from previous runs. ([#1985](https://github.com/getsentry/sentry-native/pull/1985)) + **Fixes**: - Native: store daemon logs, minidumps, crash envelopes, and scratch files in `.run` directories so they are cleaned up with the run instead of accumulating in the database root. Minidumps can still be retained with `cache_keep`, which stores `.dmp` sidecars alongside cached envelopes. ([#1976](https://github.com/getsentry/sentry-native/pull/1976)) diff --git a/examples/example.c b/examples/example.c index 005a8f36dd..382be4749b 100644 --- a/examples/example.c +++ b/examples/example.c @@ -172,6 +172,16 @@ on_crash_callback( return event; } +static void +on_crashed_last_run_callback(const sentry_envelope_t *envelope, void *user_data) +{ + (void)user_data; + const char *event_id = sentry_value_as_string( + sentry_envelope_get_header(envelope, "event_id")); + printf("CRASHED_LAST_RUN:%s\n", event_id ? event_id : ""); + fflush(stdout); +} + static sentry_value_t restart_on_crash( const sentry_ucontext_t *uctx, sentry_value_t event, void *user_data) @@ -771,6 +781,11 @@ main(int argc, char **argv) sentry_options_set_on_crash(options, on_crash_callback, NULL); } + if (has_arg(argc, argv, "on-crashed-last-run")) { + sentry_options_set_on_crashed_last_run( + options, on_crashed_last_run_callback, NULL); + } + if (has_arg(argc, argv, "discarding-on-crash")) { sentry_options_set_on_crash( options, discarding_on_crash_callback, NULL); diff --git a/include/sentry.h b/include/sentry.h index 271f2d0006..6f5719a890 100644 --- a/include/sentry.h +++ b/include/sentry.h @@ -1257,6 +1257,33 @@ typedef sentry_value_t (*sentry_crash_function_t)( SENTRY_API void sentry_options_set_on_crash( sentry_options_t *opts, sentry_crash_function_t func, void *data); +/** + * Type of the `on_crashed_last_run` callback. + * + * The callback is invoked synchronously during `sentry_init` for every + * available envelope associated with a crash in a previous run. This can + * include multiple crashes in multi-process and early-startup crash scenarios. + * + * The callback does not take ownership of `envelope`. The envelope is only + * valid for the duration of the callback and must not be freed. Since + * `sentry_init` has not completed yet, the callback must not call SDK functions + * that require an initialized SDK. + * + * Unlike `on_crash`, this callback runs in a healthy process and does not need + * to be signal-safe. + */ +typedef void (*sentry_crashed_last_run_function_t)( + const sentry_envelope_t *envelope, void *user_data); + +/** + * Sets the `on_crashed_last_run` callback. + * + * The native backend requires this option to be configured in the crashed run + * so its out-of-process daemon retains the crash envelope for the next launch. + */ +SENTRY_API void sentry_options_set_on_crashed_last_run(sentry_options_t *opts, + sentry_crashed_last_run_function_t func, void *user_data); + /** * Sets the DSN. */ diff --git a/src/backends/native/sentry_crash_context.h b/src/backends/native/sentry_crash_context.h index 5a51c06ad1..86b267b221 100644 --- a/src/backends/native/sentry_crash_context.h +++ b/src/backends/native/sentry_crash_context.h @@ -291,6 +291,7 @@ typedef struct { // ms int cache_keep; // sentry_cache_keep_t bool require_user_consent; + bool has_on_crashed_last_run; bool enable_large_attachments; bool http_retry; uint64_t shutdown_timeout; diff --git a/src/backends/native/sentry_crash_daemon.c b/src/backends/native/sentry_crash_daemon.c index 7522074174..404ef0f6ee 100644 --- a/src/backends/native/sentry_crash_daemon.c +++ b/src/backends/native/sentry_crash_daemon.c @@ -4275,17 +4275,18 @@ sentry__process_crash(const sentry_options_t *options, sentry_crash_ipc_t *ipc) crash_captured = true; } - // Clean up temporary envelope file (keep minidump for - // inspection/debugging) + // Keep the original crash envelope for the callback on the next launch. + if (!ctx->has_on_crashed_last_run) { #if defined(SENTRY_PLATFORM_UNIX) - unlink(envelope_path); + unlink(envelope_path); #elif defined(SENTRY_PLATFORM_WINDOWS) - wchar_t *wenvelope_unlink = sentry__string_to_wstr(envelope_path); - if (wenvelope_unlink) { - _wunlink(wenvelope_unlink); - sentry_free(wenvelope_unlink); - } + wchar_t *wenvelope_unlink = sentry__string_to_wstr(envelope_path); + if (wenvelope_unlink) { + _wunlink(wenvelope_unlink); + sentry_free(wenvelope_unlink); + } #endif + } cleanup: // Send the staged session-replay envelope same-session, enriched from the @@ -4334,7 +4335,9 @@ remove_pending_run_envelopes(const sentry_path_t *run_path) const sentry_path_t *file; while (it && (file = sentry__pathiter_next(it)) != NULL) { if (sentry__path_is_file(file) && !sentry__path_is_symlink(file) - && sentry__path_ends_with(file, ".envelope")) { + && sentry__path_ends_with(file, ".envelope") + && !sentry__path_filename_matches( + file, "__sentry-crash.envelope")) { sentry__path_remove(file); } } diff --git a/src/backends/sentry_backend_breakpad.cpp b/src/backends/sentry_backend_breakpad.cpp index cef8f75383..f2ea63f9ba 100644 --- a/src/backends/sentry_backend_breakpad.cpp +++ b/src/backends/sentry_backend_breakpad.cpp @@ -140,6 +140,7 @@ breakpad_backend_callback(const google_breakpad::MinidumpDescriptor &descriptor, sentry_value_t transaction = sentry__trace_finish(SENTRY_SPAN_STATUS_ABORTED); + sentry_uuid_t event_id = sentry_uuid_nil(); bool should_handle = true; @@ -184,6 +185,9 @@ breakpad_backend_callback(const google_breakpad::MinidumpDescriptor &descriptor, sentry_envelope_t *envelope = sentry__prepare_event( options, event, nullptr, !options->on_crash_func, nullptr); + if (envelope) { + event_id = sentry__envelope_get_event_id(envelope); + } sentry_session_t *session = sentry__end_current_session_with_status( SENTRY_SESSION_STATUS_CRASHED); sentry__envelope_add_session(envelope, session); @@ -274,6 +278,10 @@ breakpad_backend_callback(const google_breakpad::MinidumpDescriptor &descriptor, // after capturing the crash event, try to dump all the in-flight // data of the previous transports sentry__transport_dump_queue(options->transport, options->run); + if (!sentry_uuid_is_nil(&event_id) + && !sentry__run_write_crash_marker(options->run, &event_id)) { + SENTRY_SIGNAL_SAFE_LOG("WARN writing run crash marker failed"); + } // and restore the old transport } SENTRY_SIGNAL_SAFE_LOG("INFO crash has been captured"); diff --git a/src/backends/sentry_backend_crashpad.cpp b/src/backends/sentry_backend_crashpad.cpp index 9f851a4b8a..8bf00f8dbe 100644 --- a/src/backends/sentry_backend_crashpad.cpp +++ b/src/backends/sentry_backend_crashpad.cpp @@ -603,8 +603,8 @@ report_attachments_dir(const crashpad::CrashReportDatabase::Report &report, return attachments_dir; } -// Converts a completed crashpad report into a sentry envelope by reading the -// event, breadcrumbs, and attachments from the report's attachments directory. +// Converts a crashpad report into a sentry envelope by reading the event, +// breadcrumbs, and attachments from the report's attachments directory. static sentry_envelope_t * report_to_envelope(const crashpad::CrashReportDatabase::Report &report, const sentry_options_t *options) @@ -687,8 +687,75 @@ report_to_envelope(const crashpad::CrashReportDatabase::Report &report, return envelope; } +static bool +crashpad_backend_process_old_run(sentry_backend_t *backend, + const sentry_options_t *options, const sentry_path_t *run_path) +{ + if (!options->on_crashed_last_run_func) { + return true; + } + + auto *state = static_cast(backend->data); + if (!state || !state->db) { + return false; + } + + sentry_path_t *event_path + = sentry__path_join_str(run_path, "__sentry-event"); + if (!event_path) { + return false; + } + sentry_value_t event = read_msgpack_file(event_path); + sentry_uuid_t event_id + = sentry__value_as_uuid(sentry_value_get_by_key(event, "event_id")); + sentry_value_decref(event); + if (sentry_uuid_is_nil(&event_id)) { + sentry__path_free(event_path); + return true; + } + + char event_id_str[37]; + sentry_uuid_as_string(&event_id, event_id_str); + crashpad::UUID report_id; + if (!report_id.InitializeFromString(event_id_str)) { + sentry__path_free(event_path); + return true; + } + + crashpad::CrashReportDatabase::Report report; + crashpad::CrashReportDatabase::OperationStatus status + = state->db->LookUpCrashReport(report_id, &report); + if (status == crashpad::CrashReportDatabase::kReportNotFound) { + sentry__path_free(event_path); + return true; + } + if (status != crashpad::CrashReportDatabase::kNoError) { + sentry__path_free(event_path); + return false; + } + + sentry_envelope_t *envelope = report_to_envelope(report, options); + if (!envelope || !sentry__envelope_materialize(envelope)) { + sentry_envelope_free(envelope); + sentry__path_free(event_path); + return true; + } + + // remove before invoking to prevent repeated callbacks + bool removed = sentry__path_remove(event_path) == 0; + sentry__path_free(event_path); + if (!removed) { + sentry_envelope_free(envelope); + return false; + } + options->on_crashed_last_run_func( + envelope, options->on_crashed_last_run_data); + sentry_envelope_free(envelope); + return true; +} + // Caches completed crashpad reports as sentry envelopes and removes them from -// the crashpad database. Called during startup before the handler is started. +// the crashpad database. static void process_completed_reports( crashpad_state_t *state, const sentry_options_t *options) @@ -876,7 +943,6 @@ crashpad_backend_startup( // Initialize database first, flushing the consent later on as part of // `sentry_init` will persist the upload flag. data->db = crashpad::CrashReportDatabase::Initialize(database).release(); - process_completed_reports(data, options); data->client = new (std::nothrow) crashpad::CrashpadClient; char *minidump_url = sentry__dsn_get_minidump_url(options->dsn, options->user_agent); @@ -1119,6 +1185,7 @@ crashpad_backend_prune_database(sentry_backend_t *backend) // When offline caching is enabled, the user has full control over these // parameters via the cache_max_* options. SENTRY_WITH_OPTIONS (options) { + process_completed_reports(data, options); if (options->cache_keep) { max_age = options->cache_max_age; max_size = options->cache_max_size; @@ -1223,6 +1290,7 @@ sentry__backend_new(void) backend->add_breadcrumb_func = crashpad_backend_add_breadcrumb; backend->user_consent_changed_func = crashpad_backend_user_consent_changed; backend->get_last_crash_func = crashpad_backend_last_crash; + backend->process_old_run_func = crashpad_backend_process_old_run; backend->prune_database_func = crashpad_backend_prune_database; #if defined(SENTRY_PLATFORM_WINDOWS) || defined(SENTRY_PLATFORM_LINUX) \ || defined(SENTRY_PLATFORM_MACOS) diff --git a/src/backends/sentry_backend_inproc.c b/src/backends/sentry_backend_inproc.c index 86854ef6dc..f997c773c9 100644 --- a/src/backends/sentry_backend_inproc.c +++ b/src/backends/sentry_backend_inproc.c @@ -1075,6 +1075,7 @@ process_ucontext_deferred(const sentry_ucontext_t *uctx, sentry_value_t transaction = sentry__trace_finish(SENTRY_SPAN_STATUS_ABORTED); + sentry_uuid_t event_id = sentry_uuid_nil(); if (options->on_crash_func && !skip_hooks) { SENTRY_DEBUG("invoking `on_crash` hook"); @@ -1102,6 +1103,9 @@ process_ucontext_deferred(const sentry_ucontext_t *uctx, sentry_envelope_t *envelope = sentry__prepare_event(options, event, NULL, !options->on_crash_func && !skip_hooks, NULL); + if (envelope) { + event_id = sentry__envelope_get_event_id(envelope); + } sentry_session_t *session = sentry__end_current_session_with_status( SENTRY_SESSION_STATUS_CRASHED); sentry__envelope_add_session(envelope, session); @@ -1156,6 +1160,10 @@ process_ucontext_deferred(const sentry_ucontext_t *uctx, // after capturing the crash event, dump all the envelopes to disk sentry__transport_dump_queue(options->transport, options->run); + if (!sentry_uuid_is_nil(&event_id) + && !sentry__run_write_crash_marker(options->run, &event_id)) { + SENTRY_SIGNAL_SAFE_LOG("WARN writing run crash marker failed"); + } // Use signal-safe logging here since this may run in signal handler // context (fallback path) where stdio functions are not safe. diff --git a/src/backends/sentry_backend_native.c b/src/backends/sentry_backend_native.c index 20f7091b9c..495860a281 100644 --- a/src/backends/sentry_backend_native.c +++ b/src/backends/sentry_backend_native.c @@ -181,6 +181,41 @@ typedef struct { volatile long crashed; } native_backend_state_t; +static bool +native_backend_process_old_run(sentry_backend_t *backend, + const sentry_options_t *options, const sentry_path_t *run_path) +{ + (void)backend; + + sentry_pathiter_t *it = sentry__path_iter_directory(run_path); + const sentry_path_t *file; + while (it && (file = sentry__pathiter_next(it)) != NULL) { + if (!sentry__path_is_file(file) || sentry__path_is_symlink(file) + || !sentry__path_filename_matches( + file, "__sentry-crash.envelope")) { + continue; + } + + sentry_envelope_t *envelope = options->on_crashed_last_run_func + ? sentry__envelope_from_path(file) + : NULL; + bool materialized = envelope && sentry__envelope_materialize(envelope); + // remove before invoking to prevent repeated callbacks + if (sentry__path_remove(file) != 0) { + sentry_envelope_free(envelope); + sentry__pathiter_free(it); + return false; + } + if (materialized) { + options->on_crashed_last_run_func( + envelope, options->on_crashed_last_run_data); + } + sentry_envelope_free(envelope); + } + sentry__pathiter_free(it); + return true; +} + static int native_backend_startup( sentry_backend_t *backend, const sentry_options_t *options) @@ -309,6 +344,7 @@ native_backend_startup( ctx->session_replay_duration = options->session_replay_duration; ctx->cache_keep = (int)options->cache_keep; ctx->require_user_consent = options->require_user_consent; + ctx->has_on_crashed_last_run = options->on_crashed_last_run_func != NULL; ctx->enable_large_attachments = options->enable_large_attachments; ctx->http_retry = options->http_retry; ctx->shutdown_timeout = options->shutdown_timeout; @@ -1131,6 +1167,7 @@ sentry__backend_new(void) backend->add_breadcrumb_func = native_backend_add_breadcrumb; backend->add_attachment_func = native_backend_add_attachment; backend->user_consent_changed_func = native_backend_user_consent_changed; + backend->process_old_run_func = native_backend_process_old_run; backend->can_capture_after_shutdown = false; return backend; diff --git a/src/sentry_backend.h b/src/sentry_backend.h index a55fc507a4..7687d8a280 100644 --- a/src/sentry_backend.h +++ b/src/sentry_backend.h @@ -3,6 +3,7 @@ #include "sentry_boot.h" +#include "sentry_path.h" #include "sentry_scope.h" /** @@ -24,6 +25,9 @@ struct sentry_backend_s { const sentry_options_t *options); void (*user_consent_changed_func)(sentry_backend_t *); uint64_t (*get_last_crash_func)(sentry_backend_t *); + // called with the run file lock held; false retains the run + bool (*process_old_run_func)(sentry_backend_t *, + const sentry_options_t *options, const sentry_path_t *run_path); void (*prune_database_func)(sentry_backend_t *); void (*add_attachment_func)(sentry_backend_t *, sentry_attachment_t *); void (*remove_attachment_func)(sentry_backend_t *, sentry_attachment_t *); diff --git a/src/sentry_database.c b/src/sentry_database.c index ea145de1f6..39edcd750a 100644 --- a/src/sentry_database.c +++ b/src/sentry_database.c @@ -1,6 +1,7 @@ #include "sentry_database.h" #include "sentry_alloc.h" #include "sentry_attachment.h" +#include "sentry_backend.h" #include "sentry_client_report.h" #include "sentry_envelope.h" #include "sentry_json.h" @@ -618,6 +619,46 @@ sentry__run_clear_session(const sentry_run_t *run) return !rv; } +static sentry_path_t * +run_crash_marker_path( + const sentry_path_t *run_path, const sentry_uuid_t *event_id) +{ + if (!run_path || !event_id || sentry_uuid_is_nil(event_id)) { + return NULL; + } + char *filename = sentry__uuid_as_filename(event_id, ".crash"); + if (!filename) { + return NULL; + } + sentry_path_t *path = sentry__path_join_str(run_path, filename); + sentry_free(filename); + return path; +} + +bool +sentry__run_write_crash_marker( + const sentry_run_t *run, const sentry_uuid_t *event_id) +{ + if (!run || !event_id || sentry_uuid_is_nil(event_id)) { + return false; + } + + char *filename = sentry__uuid_as_filename(event_id, ".envelope"); + sentry_path_t *envelope_path + = filename ? sentry__path_join_str(run->run_path, filename) : NULL; + sentry_free(filename); + if (!envelope_path || !sentry__path_is_file(envelope_path)) { + sentry__path_free(envelope_path); + return false; + } + sentry__path_free(envelope_path); + + sentry_path_t *marker_path = run_crash_marker_path(run->run_path, event_id); + int rv = marker_path ? sentry__path_touch(marker_path) : 1; + sentry__path_free(marker_path); + return rv == 0; +} + void sentry__process_run_envelopes( const sentry_options_t *options, const sentry_path_t *run_path) @@ -626,11 +667,24 @@ sentry__process_run_envelopes( const sentry_path_t *file; while (it && (file = sentry__pathiter_next(it)) != NULL) { if (!sentry__path_is_file(file) || sentry__path_is_symlink(file) - || !sentry__path_ends_with(file, ".envelope")) { + || !sentry__path_ends_with(file, ".envelope") + || sentry__path_filename_matches(file, "__sentry-crash.envelope")) { continue; } sentry_envelope_t *envelope = sentry__envelope_from_path(file); if (envelope) { + sentry_uuid_t event_id = sentry__envelope_get_event_id(envelope); + sentry_path_t *marker = run_crash_marker_path(run_path, &event_id); + // remove before invoking to prevent repeated callbacks + if (marker && sentry__path_is_file(marker) + && sentry__path_remove(marker) == 0) { + if (options->on_crashed_last_run_func + && sentry__envelope_materialize(envelope)) { + options->on_crashed_last_run_func( + envelope, options->on_crashed_last_run_data); + } + } + sentry__path_free(marker); sentry__capture_envelope(options->transport, envelope, options); } sentry__path_remove(file); @@ -716,12 +770,20 @@ sentry__process_old_runs(const sentry_options_t *options, uint64_t last_crash) sentry__path_free(daemon_lockfile); } + bool processed = true; + if (options->backend && options->backend->process_old_run_func) { + processed = options->backend->process_old_run_func( + options->backend, options, run_dir); + } sentry__process_run_envelopes(options, run_dir); sentry_pathiter_t *run_iter = sentry__path_iter_directory(run_dir); const sentry_path_t *file; while (run_iter && (file = sentry__pathiter_next(run_iter)) != NULL) { - if (sentry__path_filename_matches(file, "session.json")) { + if (sentry__path_ends_with(file, ".crash")) { + // handled by sentry__process_run_envelopes above + continue; + } else if (sentry__path_filename_matches(file, "session.json")) { if (!session_envelope) { session_envelope = sentry__envelope_new(); } @@ -759,13 +821,16 @@ sentry__process_old_runs(const sentry_options_t *options, uint64_t last_crash) session_num = 0; } } + sentry__path_remove(file); + } else if (processed) { + sentry__path_remove(file); } - - sentry__path_remove(file); } sentry__pathiter_free(run_iter); - sentry__path_remove_all(run_dir); + if (processed) { + sentry__path_remove_all(run_dir); + } if (daemon_lock) { sentry__filelock_free(daemon_lock); } diff --git a/src/sentry_database.h b/src/sentry_database.h index 83d4e5e6e2..ccd85434db 100644 --- a/src/sentry_database.h +++ b/src/sentry_database.h @@ -174,6 +174,13 @@ void sentry__process_old_runs( void sentry__process_run_envelopes( const sentry_options_t *options, const sentry_path_t *run_path); +/** + * Writes `.crash` into `run` after verifying that the matching + * `.envelope` exists. + */ +bool sentry__run_write_crash_marker( + const sentry_run_t *run, const sentry_uuid_t *event_id); + /** * Parses a cache filename in either form: * - `.envelope` sets `*ts_out = 0`, `*count_out = -1`. diff --git a/src/sentry_options.c b/src/sentry_options.c index 26cd3e9d1a..e718c60596 100644 --- a/src/sentry_options.c +++ b/src/sentry_options.c @@ -231,6 +231,14 @@ sentry_options_set_on_crash( opts->on_crash_data = user_data; } +void +sentry_options_set_on_crashed_last_run(sentry_options_t *opts, + sentry_crashed_last_run_function_t func, void *user_data) +{ + opts->on_crashed_last_run_func = func; + opts->on_crashed_last_run_data = user_data; +} + void sentry_options_set_before_transaction( sentry_options_t *opts, sentry_transaction_function_t func, void *user_data) diff --git a/src/sentry_options.h b/src/sentry_options.h index 75d52220ad..02f1ddc81d 100644 --- a/src/sentry_options.h +++ b/src/sentry_options.h @@ -66,6 +66,8 @@ struct sentry_options_s { void *before_send_data; sentry_crash_function_t on_crash_func; void *on_crash_data; + sentry_crashed_last_run_function_t on_crashed_last_run_func; + void *on_crashed_last_run_data; sentry_transaction_function_t before_transaction_func; void *before_transaction_data; sentry_before_send_log_function_t before_send_log_func; diff --git a/tests/test_integration_crashpad.py b/tests/test_integration_crashpad.py index cb2b0aa69b..de3d2ebb2e 100644 --- a/tests/test_integration_crashpad.py +++ b/tests/test_integration_crashpad.py @@ -65,6 +65,46 @@ def test_crashpad_capture(cmake, httpserver): assert len(httpserver.log) == 2 +def test_crashpad_on_crashed_last_run(cmake): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "crashpad"}) + args = ["log", "on-crashed-last-run"] + + run( + tmp_path, + "sentry_example", + ["log", "crash"], + expect_failure=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + assert not list((tmp_path / ".sentry-native").glob("*.run/*.crash")) + + restarted = run( + tmp_path, + "sentry_example", + [*args, "no-setup"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + callbacks = [ + line + for line in restarted.stdout.splitlines() + if line.startswith(b"CRASHED_LAST_RUN:") + ] + assert len(callbacks) == 1 + assert len(callbacks[0].partition(b":")[2]) == 36 + + restarted_again = run( + tmp_path, + "sentry_example", + [*args, "no-setup"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert b"CRASHED_LAST_RUN:" not in restarted_again.stdout + + def _setup_crashpad_proxy_test(cmake, httpserver, proxy): if proxy: proxy_process, port = start_mitmdump(proxy) diff --git a/tests/test_integration_http.py b/tests/test_integration_http.py index 18c2129b6b..8f778cfd55 100644 --- a/tests/test_integration_http.py +++ b/tests/test_integration_http.py @@ -1048,6 +1048,62 @@ def test_native_crash_http(cmake, httpserver): assert_attachment(envelope) +@pytest.mark.parametrize( + "backend", + [ + "inproc", + pytest.param( + "breakpad", + marks=pytest.mark.skipif( + not has_breakpad or is_qemu, reason="test needs breakpad backend" + ), + ), + ], +) +def test_on_crashed_last_run(cmake, backend): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": backend}) + args = ["log", "on-crashed-last-run"] + + run( + tmp_path, + "sentry_example", + [*args, "crash"], + expect_failure=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + run_dirs = list((tmp_path / ".sentry-native").glob("*.run")) + assert len(run_dirs) == 1 + markers = list(run_dirs[0].glob("*.crash")) + assert len(markers) == 1 + assert (run_dirs[0] / f"{markers[0].stem}.envelope").is_file() + + restarted = run( + tmp_path, + "sentry_example", + [*args, "no-setup"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + callbacks = [ + line + for line in restarted.stdout.splitlines() + if line.startswith(b"CRASHED_LAST_RUN:") + ] + assert len(callbacks) == 1 + assert callbacks[0] == f"CRASHED_LAST_RUN:{markers[0].stem}".encode() + + restarted_again = run( + tmp_path, + "sentry_example", + [*args, "no-setup"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert b"CRASHED_LAST_RUN:" not in restarted_again.stdout + + @pytest.mark.parametrize( "backend", [ diff --git a/tests/test_integration_native.py b/tests/test_integration_native.py index 4cae17405e..706939e146 100644 --- a/tests/test_integration_native.py +++ b/tests/test_integration_native.py @@ -74,6 +74,67 @@ def test_native_capture_crash(cmake, httpserver): assert_native_crash(envelope) +def test_native_on_crashed_last_run(cmake, httpserver): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "native"}) + httpserver.expect_oneshot_request("/api/123456/envelope/").respond_with_data("OK") + env = dict(os.environ, SENTRY_DSN=make_dsn(httpserver)) + args = ["log", "on-crashed-last-run"] + + with httpserver.wait(timeout=10) as waiting: + run_crash( + tmp_path, + "sentry_example", + [*args, "crash"], + env=env, + wait_for_daemon=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert waiting.result + assert len(httpserver.log) == 1 + + crash_envelope = Envelope.deserialize(httpserver.log[0][0].get_data()) + assert_native_crash(crash_envelope) + event_id = crash_envelope.headers["event_id"] + + db_dir = tmp_path / ".sentry-native" + run_dirs = list(db_dir.glob("*.run")) + assert len(run_dirs) == 1 + assert {path.name for path in run_dirs[0].glob("*.envelope")} == { + "__sentry-crash.envelope" + } + assert not list(run_dirs[0].glob("*.crash")) + + restarted = run( + tmp_path, + "sentry_example", + [*args, "no-setup"], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + callbacks = [ + line + for line in restarted.stdout.splitlines() + if line.startswith(b"CRASHED_LAST_RUN:") + ] + assert callbacks == [f"CRASHED_LAST_RUN:{event_id}".encode()] + assert len(httpserver.log) == 1 + assert not list(db_dir.glob("*.run")) + assert not list(db_dir.glob("*.run*.lock")) + + restarted_again = run( + tmp_path, + "sentry_example", + [*args, "no-setup"], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert b"CRASHED_LAST_RUN:" not in restarted_again.stdout + assert len(httpserver.log) == 1 + + @pytest.mark.skipif( sys.platform != "win32" or bool(os.environ.get("TEST_MINGW")), reason="WER crash tests are only available in MSVC Windows builds", diff --git a/tests/unit/test_cache.c b/tests/unit/test_cache.c index 8d078a5b0f..dade035914 100644 --- a/tests/unit/test_cache.c +++ b/tests/unit/test_cache.c @@ -1,9 +1,12 @@ +#include "sentry_alloc.h" +#include "sentry_backend.h" #include "sentry_core.h" #include "sentry_database.h" #include "sentry_envelope.h" #include "sentry_options.h" #include "sentry_path.h" #include "sentry_retry.h" +#include "sentry_session.h" #include "sentry_string.h" #include "sentry_testsupport.h" #include "sentry_uuid.h" @@ -41,6 +44,101 @@ set_file_mtime(const sentry_path_t *path, time_t mtime) #endif } +typedef struct { + size_t count; + size_t materialized_count; + sentry_uuid_t event_ids[4]; +} crashed_last_run_state_t; + +static void +record_crashed_last_run(const sentry_envelope_t *envelope, void *user_data) +{ + crashed_last_run_state_t *state = user_data; + sentry_value_t event_id = sentry_envelope_get_header(envelope, "event_id"); + if (state->count < 4) { + state->event_ids[state->count] + = sentry_uuid_from_string(sentry_value_as_string(event_id)); + } + if (!sentry_value_is_null(event_id) + && !sentry_value_is_null(sentry_envelope_get_event(envelope))) { + state->materialized_count++; + } + state->count++; +} + +static void +count_sent_envelopes(sentry_envelope_t *envelope, void *user_data) +{ + size_t *count = user_data; + (*count)++; + sentry_envelope_free(envelope); +} + +#if !defined(SENTRY_PLATFORM_NX) && !defined(SENTRY_PLATFORM_PS) +typedef struct { + size_t attempts; +} old_run_retry_state_t; + +static bool +retry_old_run(sentry_backend_t *backend, const sentry_options_t *options, + const sentry_path_t *run_path) +{ + (void)options; + (void)run_path; + old_run_retry_state_t *state = backend->data; + return ++state->attempts > 1; +} +#endif + +static sentry_path_t * +write_event_envelope(const sentry_path_t *dir, const sentry_uuid_t *event_id) +{ + if (sentry__path_create_dir_all(dir) != 0) { + return NULL; + } + sentry_envelope_t *envelope = sentry__envelope_new(); + sentry__envelope_add_event( + envelope, sentry__value_new_event_with_id(event_id)); + char *filename = sentry__uuid_as_filename(event_id, ".envelope"); + sentry_path_t *path + = filename ? sentry__path_join_str(dir, filename) : NULL; + sentry_free(filename); + int rv = path ? sentry_envelope_write_to_path(envelope, path) : 1; + sentry_envelope_free(envelope); + if (rv != 0) { + sentry__path_free(path); + return NULL; + } + return path; +} + +static sentry_path_t * +write_run_crash_marker( + const sentry_path_t *run_path, const sentry_uuid_t *event_id) +{ + char *filename = sentry__uuid_as_filename(event_id, ".crash"); + sentry_path_t *path + = filename ? sentry__path_join_str(run_path, filename) : NULL; + sentry_free(filename); + if (!path || sentry__path_touch(path) != 0) { + sentry__path_free(path); + return NULL; + } + return path; +} + +static bool +state_has_event_id( + const crashed_last_run_state_t *state, const sentry_uuid_t *event_id) +{ + for (size_t i = 0; i < state->count && i < 4; i++) { + if (memcmp(&state->event_ids[i], event_id, sizeof(*event_id)) == 0) { + return true; + } + } + return false; +} + SENTRY_TEST(cache_keep) { #if defined(SENTRY_PLATFORM_NX) || defined(SENTRY_PLATFORM_PS) @@ -98,6 +196,233 @@ SENTRY_TEST(cache_keep) sentry_close(); } +SENTRY_TEST(old_run_retry) +{ +#if defined(SENTRY_PLATFORM_NX) || defined(SENTRY_PLATFORM_PS) + SKIP_TEST(); +#else + SENTRY_TEST_OPTIONS_NEW(options); + TEST_ASSERT(sentry__path_remove_all(options->database_path) == 0); + TEST_ASSERT(sentry__path_create_dir_all(options->database_path) == 0); + + options->run = sentry__run_new(options->database_path); + TEST_ASSERT(!!options->run); + sentry_run_t *old_run = sentry__run_new(options->database_path); + TEST_ASSERT(!!old_run); + sentry__filelock_unlock(old_run->lock); + + sentry_uuid_t event_id = sentry_uuid_new_v4(); + sentry_path_t *queued_envelope + = write_event_envelope(old_run->run_path, &event_id); + sentry_path_t *backend_state + = sentry__path_join_str(old_run->run_path, "backend-state"); + TEST_ASSERT(!!queued_envelope && !!backend_state); + TEST_ASSERT(sentry__path_touch(backend_state) == 0); + + static const char session_json[] + = "{\"init\":true,\"sid\":\"00000000-0000-4000-8000-000000000001\"," + "\"status\":\"ok\",\"errors\":0,\"started\":\"2020-01-01T00:00:00Z\"," + "\"duration\":0,\"attrs\":{\"release\":\"test@1.0.0\"," + "\"environment\":\"production\"}}"; + sentry_session_t *session + = sentry__session_from_json(session_json, sizeof(session_json) - 1); + TEST_ASSERT(!!session); + TEST_ASSERT(sentry__run_write_session(old_run, session)); + sentry__session_free(session); + + size_t sent_envelopes = 0; + sentry_transport_t *transport = sentry_transport_new(count_sent_envelopes); + TEST_ASSERT(!!transport); + sentry_transport_set_state(transport, &sent_envelopes); + sentry_options_set_transport(options, transport); + + old_run_retry_state_t retry_state = { 0 }; + sentry_backend_t *backend = SENTRY_MAKE(sentry_backend_t); + TEST_ASSERT(!!backend); + backend->data = &retry_state; + backend->process_old_run_func = retry_old_run; + sentry_options_set_backend(options, backend); + + sentry__process_old_runs(options, 0); + + TEST_CHECK_INT_EQUAL(retry_state.attempts, 1); + TEST_CHECK_INT_EQUAL(sent_envelopes, 2); + TEST_CHECK(sentry__path_is_dir(old_run->run_path)); + TEST_CHECK(sentry__path_is_file(backend_state)); + TEST_CHECK(!sentry__path_is_file(queued_envelope)); + TEST_CHECK(!sentry__path_is_file(old_run->session_path)); + + sentry__process_old_runs(options, 0); + + TEST_CHECK_INT_EQUAL(retry_state.attempts, 2); + TEST_CHECK_INT_EQUAL(sent_envelopes, 2); + TEST_CHECK(!sentry__path_is_dir(old_run->run_path)); + + sentry__path_free(backend_state); + sentry__path_free(queued_envelope); + sentry__run_free(old_run); + sentry__run_clean(options->run, true); + sentry_options_free(options); +#endif +} + +SENTRY_TEST(on_crashed_last_run) +{ + crashed_last_run_state_t state = { 0 }; + SENTRY_TEST_OPTIONS_NEW(options); + sentry_options_set_transport(options, NULL); + sentry_options_set_on_crashed_last_run( + options, record_crashed_last_run, &state); + TEST_CHECK_INT_EQUAL(sentry_init(options), 0); + + sentry_path_t *old_run1 + = sentry__path_join_str(options->database_path, "first.run"); + sentry_path_t *old_run2 + = sentry__path_join_str(options->database_path, "second.run"); + TEST_ASSERT(!!old_run1 && !!old_run2); + sentry__path_remove_all(old_run1); + sentry__path_remove_all(old_run2); + + sentry_uuid_t crash1 = sentry_uuid_new_v4(); + sentry_uuid_t crash2 = sentry_uuid_new_v4(); + sentry_uuid_t normal = sentry_uuid_new_v4(); + sentry_path_t *crash1_envelope = write_event_envelope(old_run1, &crash1); + sentry_path_t *normal_envelope = write_event_envelope(old_run1, &normal); + sentry_path_t *crash2_envelope = write_event_envelope(old_run2, &crash2); + sentry_path_t *crash1_marker = write_run_crash_marker(old_run1, &crash1); + sentry_path_t *crash2_marker = write_run_crash_marker(old_run2, &crash2); + TEST_ASSERT(!!crash1_envelope && !!normal_envelope && !!crash2_envelope); + TEST_ASSERT(!!crash1_marker && !!crash2_marker); + + sentry__process_old_runs(options, 0); + TEST_CHECK_INT_EQUAL(state.count, 2); + TEST_CHECK_INT_EQUAL(state.materialized_count, 2); + TEST_CHECK(state_has_event_id(&state, &crash1)); + TEST_CHECK(state_has_event_id(&state, &crash2)); + TEST_CHECK(!state_has_event_id(&state, &normal)); + TEST_CHECK(!sentry__path_is_dir(old_run1)); + TEST_CHECK(!sentry__path_is_dir(old_run2)); + + sentry__process_old_runs(options, 0); + TEST_CHECK_INT_EQUAL(state.count, 2); + + sentry__path_free(crash1_marker); + sentry__path_free(crash2_marker); + sentry__path_free(crash1_envelope); + sentry__path_free(normal_envelope); + sentry__path_free(crash2_envelope); + sentry__path_free(old_run1); + sentry__path_free(old_run2); + sentry_close(); +} + +SENTRY_TEST(on_crashed_last_run_cache) +{ + crashed_last_run_state_t state = { 0 }; + SENTRY_TEST_OPTIONS_NEW(options); + sentry_options_set_transport(options, NULL); + sentry_options_set_on_crashed_last_run( + options, record_crashed_last_run, &state); + TEST_CHECK_INT_EQUAL(sentry_init(options), 0); + sentry__path_remove_all(options->run->cache_path); + + sentry_uuid_t event_id = sentry_uuid_new_v4(); + TEST_CHECK(!sentry__run_write_crash_marker(options->run, &event_id)); + + sentry_path_t *old_run + = sentry__path_join_str(options->database_path, "old.run"); + sentry__path_remove_all(old_run); + sentry_path_t *crash_path = write_event_envelope(old_run, &event_id); + sentry_path_t *marker_path = write_run_crash_marker(old_run, &event_id); + + sentry_envelope_t *envelope = sentry__envelope_new(); + sentry__envelope_add_event( + envelope, sentry__value_new_event_with_id(&event_id)); + TEST_CHECK(sentry__run_write_cache(options->run, envelope, -1)); + sentry_envelope_free(envelope); + + char *cache_filename = sentry__uuid_as_filename(&event_id, ".envelope"); + sentry_path_t *cache_path + = sentry__path_join_str(options->run->cache_path, cache_filename); + sentry_free(cache_filename); + TEST_ASSERT(!!old_run && !!crash_path && !!marker_path && !!cache_path); + TEST_CHECK(sentry__path_is_file(crash_path)); + TEST_CHECK(sentry__path_is_file(marker_path)); + TEST_CHECK(sentry__path_is_file(cache_path)); + + sentry__process_old_runs(options, 0); + TEST_CHECK_INT_EQUAL(state.count, 1); + TEST_CHECK_INT_EQUAL(state.materialized_count, 1); + TEST_CHECK(state_has_event_id(&state, &event_id)); + TEST_CHECK(!sentry__path_is_file(crash_path)); + TEST_CHECK(!sentry__path_is_file(marker_path)); + TEST_CHECK(sentry__path_is_file(cache_path)); + + sentry__process_old_runs(options, 0); + TEST_CHECK_INT_EQUAL(state.count, 1); + + sentry__path_remove_all(options->run->cache_path); + sentry__path_free(old_run); + sentry__path_free(crash_path); + sentry__path_free(marker_path); + sentry__path_free(cache_path); + sentry_close(); +} + +SENTRY_TEST(callback_envelope_is_not_resent_without_backend) +{ +#if defined(SENTRY_PLATFORM_NX) || defined(SENTRY_PLATFORM_PS) + SKIP_TEST(); +#else + SENTRY_TEST_OPTIONS_NEW(options); + TEST_ASSERT(sentry__path_remove_all(options->database_path) == 0); + TEST_ASSERT(sentry__path_create_dir_all(options->database_path) == 0); + + options->run = sentry__run_new(options->database_path); + TEST_ASSERT(!!options->run); + sentry_run_t *old_run = sentry__run_new(options->database_path); + TEST_ASSERT(!!old_run); + sentry__filelock_unlock(old_run->lock); + + sentry_path_t *callback_path + = sentry__path_join_str(old_run->run_path, "__sentry-crash.envelope"); + TEST_ASSERT(!!callback_path); + sentry_envelope_t *callback_envelope = sentry__envelope_new(); + TEST_ASSERT(!!callback_envelope); + sentry_uuid_t callback_id = sentry_uuid_new_v4(); + sentry__envelope_add_event( + callback_envelope, sentry__value_new_event_with_id(&callback_id)); + TEST_ASSERT( + sentry_envelope_write_to_path(callback_envelope, callback_path) == 0); + sentry_envelope_free(callback_envelope); + + sentry_uuid_t queued_id = sentry_uuid_new_v4(); + sentry_path_t *queued_path + = write_event_envelope(old_run->run_path, &queued_id); + TEST_ASSERT(!!queued_path); + + size_t sent_envelopes = 0; + sentry_transport_t *transport = sentry_transport_new(count_sent_envelopes); + TEST_ASSERT(!!transport); + sentry_transport_set_state(transport, &sent_envelopes); + sentry_options_set_transport(options, transport); + sentry_options_set_backend(options, NULL); + + sentry__process_old_runs(options, 0); + + TEST_CHECK_INT_EQUAL(sent_envelopes, 1); + TEST_CHECK(!sentry__path_is_dir(old_run->run_path)); + TEST_CHECK(!sentry__path_is_file(callback_path)); + TEST_CHECK(!sentry__path_is_file(queued_path)); + + sentry__path_free(queued_path); + sentry__path_free(callback_path); + sentry__run_free(old_run); + sentry__run_clean(options->run, true); + sentry_options_free(options); +#endif +} + SENTRY_TEST(cache_max_size) { #if defined(SENTRY_PLATFORM_NX) || defined(SENTRY_PLATFORM_PS) diff --git a/tests/unit/test_native_backend.c b/tests/unit/test_native_backend.c index 89ab18c54b..59ad85cbd1 100644 --- a/tests/unit/test_native_backend.c +++ b/tests/unit/test_native_backend.c @@ -6,6 +6,7 @@ */ #include "sentry_database.h" +#include "sentry_envelope.h" #include "sentry_options.h" #include "sentry_path.h" #include "sentry_testsupport.h" @@ -15,6 +16,21 @@ // Include native backend headers # include "../../src/backends/native/minidump/sentry_minidump_format.h" # include "../../src/backends/native/sentry_crash_context.h" + +static void +noop_crashed_last_run(const sentry_envelope_t *envelope, void *user_data) +{ + (void)envelope; + (void)user_data; +} + +static void +count_sent_envelopes(sentry_envelope_t *envelope, void *state) +{ + size_t *count = state; + (*count)++; + sentry_envelope_free(envelope); +} #endif #if defined(SENTRY_PLATFORM_LINUX) || defined(SENTRY_PLATFORM_ANDROID) @@ -111,6 +127,58 @@ SENTRY_TEST(daemon_run_blocks_old_run_cleanup) sentry_options_free(options); } +SENTRY_TEST(corrupt_crash_envelope_does_not_block_old_run) +{ +#ifdef SENTRY_BACKEND_NATIVE + SENTRY_TEST_OPTIONS_NEW(options); + TEST_ASSERT(sentry__path_remove_all(options->database_path) == 0); + TEST_ASSERT(sentry__path_create_dir_all(options->database_path) == 0); + + options->run = sentry__run_new(options->database_path); + TEST_ASSERT(!!options->run); + sentry_run_t *old_run = sentry__run_new(options->database_path); + TEST_ASSERT(!!old_run); + sentry__filelock_unlock(old_run->lock); + + sentry_path_t *crash_envelope + = sentry__path_join_str(old_run->run_path, "__sentry-crash.envelope"); + sentry_path_t *queued_envelope + = sentry__path_join_str(old_run->run_path, "queued.envelope"); + TEST_ASSERT(!!crash_envelope && !!queued_envelope); + TEST_ASSERT(sentry__path_write_buffer(crash_envelope, "garbage", 7) == 0); + + sentry_envelope_t *envelope = sentry__envelope_new(); + TEST_ASSERT(!!envelope); + sentry__envelope_add_event(envelope, + sentry_value_new_message_event(SENTRY_LEVEL_ERROR, NULL, "queued")); + TEST_ASSERT(sentry_envelope_write_to_path(envelope, queued_envelope) == 0); + sentry_envelope_free(envelope); + + size_t sent_envelopes = 0; + sentry_transport_t *transport = sentry_transport_new(count_sent_envelopes); + TEST_ASSERT(!!transport); + sentry_transport_set_state(transport, &sent_envelopes); + sentry_options_set_transport(options, transport); + sentry_options_set_on_crashed_last_run( + options, noop_crashed_last_run, NULL); + + sentry__process_old_runs(options, 0); + + TEST_CHECK_INT_EQUAL(sent_envelopes, 1); + TEST_CHECK(!sentry__path_is_dir(old_run->run_path)); + TEST_CHECK(!sentry__path_is_file(crash_envelope)); + TEST_CHECK(!sentry__path_is_file(queued_envelope)); + + sentry__path_free(queued_envelope); + sentry__path_free(crash_envelope); + sentry__run_free(old_run); + sentry__run_clean(options->run, true); + sentry_options_free(options); +#else + SKIP_TEST(); +#endif +} + /** * Test minidump header structure size and alignment */ @@ -520,6 +588,8 @@ SENTRY_TEST(crash_context_options_propagation) sentry_options_set_proxy(options, "http://myproxy:3128"); sentry_options_set_shutdown_timeout(options, 12345); sentry_options_set_system_crash_reporter_enabled(options, true); + sentry_options_set_on_crashed_last_run( + options, noop_crashed_last_run, NULL); sentry_options_set_crash_upload_mode( options, SENTRY_CRASH_UPLOAD_MODE_ASYNC); sentry_options_set_transfer_timeout(options, 45000); @@ -554,6 +624,7 @@ SENTRY_TEST(crash_context_options_propagation) } ctx->shutdown_timeout = options->shutdown_timeout; ctx->system_crash_reporter_enabled = options->system_crash_reporter_enabled; + ctx->has_on_crashed_last_run = options->on_crashed_last_run_func != NULL; ctx->crash_upload_mode = options->crash_upload_mode; ctx->transfer_timeout = options->transfer_timeout; # ifdef SENTRY_PLATFORM_WINDOWS @@ -567,6 +638,7 @@ SENTRY_TEST(crash_context_options_propagation) TEST_CHECK(ctx->user_agent[0] != '\0'); TEST_CHECK_UINT64_EQUAL(ctx->shutdown_timeout, 12345); TEST_CHECK(ctx->system_crash_reporter_enabled); + TEST_CHECK(ctx->has_on_crashed_last_run); TEST_CHECK_INT_EQUAL( ctx->crash_upload_mode, SENTRY_CRASH_UPLOAD_MODE_ASYNC); TEST_CHECK_UINT64_EQUAL(ctx->transfer_timeout, 45000); diff --git a/tests/unit/tests.inc b/tests/unit/tests.inc index fe24891b61..272fbc61cb 100644 --- a/tests/unit/tests.inc +++ b/tests/unit/tests.inc @@ -79,6 +79,7 @@ XX(cache_remove_siblings) XX(cache_symlink_run) XX(cache_write_minidump) XX(cache_write_raw_with_minidump) +XX(callback_envelope_is_not_resent_without_backend) XX(capture_minidump_basic) XX(capture_minidump_discard) XX(capture_minidump_invalid_path) @@ -105,6 +106,7 @@ XX(concurrent_uninit) XX(cond_wait_timeout_overflow) XX(cond_wake_all) XX(continuation_no_baggage_uses_sdk_dsc) +XX(corrupt_crash_envelope_does_not_block_old_run) XX(count_sampled_events) XX(crash_context_handler_path_propagation) XX(crash_context_init) @@ -241,6 +243,9 @@ XX(mpack_newlines) XX(mpack_removed_tags) XX(multiple_inits) XX(multiple_transactions) +XX(old_run_retry) +XX(on_crashed_last_run) +XX(on_crashed_last_run_cache) XX(options_crash_reporting_mode_clamp) XX(options_crash_reporting_mode_default) XX(options_crash_reporting_mode_set_get)