From 268b2f22551a122720a8ce061bc7f8130c81e778 Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Tue, 8 Sep 2026 13:42:03 +0100 Subject: [PATCH 1/2] ROX-37216: use local container ID caching --- .github/workflows/collector.yml | 2 +- collector/lib/CollectorService.cpp | 2 +- collector/lib/CollectorStats.h | 3 + collector/lib/NetworkSignalHandler.cpp | 7 +- collector/lib/NetworkSignalHandler.h | 6 +- collector/lib/Process.cpp | 10 +- collector/lib/Process.h | 2 +- collector/lib/ProcessSignalFormatter.cpp | 25 ++- collector/lib/ProcessSignalFormatter.h | 7 +- collector/lib/ProcessSignalHandler.h | 5 +- collector/lib/Utility.cpp | 75 ++----- collector/lib/Utility.h | 9 - .../lib/system-inspector/ContainerIDCache.cpp | 98 +++++++++ .../lib/system-inspector/ContainerIDCache.h | 42 ++++ .../ContainerIDFilterCheck.cpp | 74 +++++++ .../system-inspector/ContainerIDFilterCheck.h | 33 +++ collector/lib/system-inspector/Service.cpp | 63 ++---- collector/lib/system-inspector/Service.h | 5 +- collector/test/ProcessSignalFormatterTest.cpp | 58 ++++-- collector/test/SystemInspectorServiceTest.cpp | 194 ++++++++++++++++-- 20 files changed, 555 insertions(+), 165 deletions(-) create mode 100644 collector/lib/system-inspector/ContainerIDCache.cpp create mode 100644 collector/lib/system-inspector/ContainerIDCache.h create mode 100644 collector/lib/system-inspector/ContainerIDFilterCheck.cpp create mode 100644 collector/lib/system-inspector/ContainerIDFilterCheck.h diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml index 562c9e1499..ba3bb6d56d 100644 --- a/.github/workflows/collector.yml +++ b/.github/workflows/collector.yml @@ -74,7 +74,7 @@ jobs: rhacs_eng_image: ${{ env.RHACS_ENG_IMAGE }} collector_image: ${{ inputs.collector-image }} collector_tag: ${{ inputs.collector-tag }} - debug_mode: ${{ github.event_name == 'pull_request' }} + debug_mode: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ci-release-build') }} driver_version: ${DRIVER_VERSION} EOF diff --git a/collector/lib/CollectorService.cpp b/collector/lib/CollectorService.cpp index 1d6243d591..4c7330d879 100644 --- a/collector/lib/CollectorService.cpp +++ b/collector/lib/CollectorService.cpp @@ -53,7 +53,7 @@ CollectorService::CollectorService(CollectorConfig& config, std::atomic(system_inspector_.GetInspector(), conn_tracker_, system_inspector_.GetUserspaceStats()); + auto network_signal_handler = std::make_unique(system_inspector_.GetInspector(), conn_tracker_, system_inspector_.GetUserspaceStats(), system_inspector_.GetContainerIDCache()); network_signal_handler->SetCollectConnectionStatus(config_.CollectConnectionStatus()); network_signal_handler->SetTrackSendRecv(config_.TrackingSendRecv()); system_inspector_.AddSignalHandler(std::move(network_signal_handler)); diff --git a/collector/lib/CollectorStats.h b/collector/lib/CollectorStats.h index 3a660c37b3..9f01296aa3 100644 --- a/collector/lib/CollectorStats.h +++ b/collector/lib/CollectorStats.h @@ -32,6 +32,9 @@ X(process_lineage_string_total) \ X(process_info_hit) \ X(process_info_miss) \ + X(container_id_cache_hits) \ + X(container_id_cache_misses) \ + X(container_id_cache_empty_extractions) \ X(rate_limit_flushing_counts) \ X(procfs_could_not_open_fd_dir) \ X(procfs_could_not_open_proc_dir) \ diff --git a/collector/lib/NetworkSignalHandler.cpp b/collector/lib/NetworkSignalHandler.cpp index c109470a7f..ad510503d3 100644 --- a/collector/lib/NetworkSignalHandler.cpp +++ b/collector/lib/NetworkSignalHandler.cpp @@ -6,6 +6,7 @@ #include "EventMap.h" #include "Utility.h" +#include "system-inspector/ContainerIDCache.h" #include "system-inspector/EventExtractor.h" namespace collector { @@ -43,8 +44,8 @@ EventMap modifiers = { } // namespace -NetworkSignalHandler::NetworkSignalHandler(sinsp* inspector, std::shared_ptr conn_tracker, system_inspector::Stats* stats) - : event_extractor_(std::make_unique()), conn_tracker_(std::move(conn_tracker)), stats_(stats), collect_connection_status_(true), track_send_recv_(false) { +NetworkSignalHandler::NetworkSignalHandler(sinsp* inspector, std::shared_ptr conn_tracker, system_inspector::Stats* stats, system_inspector::ContainerIDCache* container_id_cache) + : event_extractor_(std::make_unique()), conn_tracker_(std::move(conn_tracker)), stats_(stats), container_id_cache_(container_id_cache), collect_connection_status_(true), track_send_recv_(false) { event_extractor_->Init(inspector); } @@ -152,7 +153,7 @@ std::optional NetworkSignalHandler::GetConnection(sinsp_evt* evt) { const Endpoint* local = is_server ? &server : &client; const Endpoint* remote = is_server ? &client : &server; - auto container_id = GetContainerID(evt); + auto container_id = container_id_cache_->Get(*evt); if (container_id.empty()) { return std::nullopt; } diff --git a/collector/lib/NetworkSignalHandler.h b/collector/lib/NetworkSignalHandler.h index 36e10d3d60..fc5790d32d 100644 --- a/collector/lib/NetworkSignalHandler.h +++ b/collector/lib/NetworkSignalHandler.h @@ -13,12 +13,13 @@ class sinsp_evt; namespace collector { namespace system_inspector { +class ContainerIDCache; class EventExtractor; -} +} // namespace system_inspector class NetworkSignalHandler final : public SignalHandler { public: - explicit NetworkSignalHandler(sinsp* inspector, std::shared_ptr conn_tracker, system_inspector::Stats* stats); + explicit NetworkSignalHandler(sinsp* inspector, std::shared_ptr conn_tracker, system_inspector::Stats* stats, system_inspector::ContainerIDCache* container_id_cache); ~NetworkSignalHandler() override; std::string GetName() override { return "NetworkSignalHandler"; } @@ -35,6 +36,7 @@ class NetworkSignalHandler final : public SignalHandler { std::unique_ptr event_extractor_; std::shared_ptr conn_tracker_; system_inspector::Stats* stats_; + system_inspector::ContainerIDCache* container_id_cache_; bool collect_connection_status_; bool track_send_recv_; diff --git a/collector/lib/Process.cpp b/collector/lib/Process.cpp index 4f6a305b51..489511cc69 100644 --- a/collector/lib/Process.cpp +++ b/collector/lib/Process.cpp @@ -32,10 +32,14 @@ const std::shared_ptr ProcessStore::Fetch(uint64_t pid) { std::string Process::container_id() const { WaitForProcessInfo(); + // system_inspector_threadinfo_ is assigned by ProcessInfoResolved() on the + // inspector thread; hold the lock across the read and the extraction. + std::lock_guard lock(process_info_mutex_); if (system_inspector_threadinfo_) { - auto id = GetContainerID(*system_inspector_threadinfo_); - if (!id.empty()) { - return id; + for (const auto& cgroup : system_inspector_threadinfo_->cgroups()) { + if (const auto id = ExtractContainerIDFromCgroup(cgroup.second)) { + return std::string(*id); + } } } diff --git a/collector/lib/Process.h b/collector/lib/Process.h index 8fb346984d..9f828b36a8 100644 --- a/collector/lib/Process.h +++ b/collector/lib/Process.h @@ -18,7 +18,7 @@ class Service; namespace collector::system_inspector { class Service; -} +} // namespace collector::system_inspector namespace collector { /* A Process object store used to deduplicate process information. diff --git a/collector/lib/ProcessSignalFormatter.cpp b/collector/lib/ProcessSignalFormatter.cpp index 17a4fa8e9f..bddd537466 100644 --- a/collector/lib/ProcessSignalFormatter.cpp +++ b/collector/lib/ProcessSignalFormatter.cpp @@ -10,6 +10,7 @@ #include "EventMap.h" #include "Logging.h" #include "Utility.h" +#include "system-inspector/ContainerIDCache.h" #include "system-inspector/EventExtractor.h" namespace collector { @@ -58,16 +59,24 @@ std::string extract_proc_args(sinsp_threadinfo* tinfo) { ProcessSignalFormatter::ProcessSignalFormatter( sinsp* inspector, - const CollectorConfig& config) : event_names_(EventNames::GetInstance()), - inspector_(inspector), - event_extractor_(std::make_unique()), - config_(config) { + const CollectorConfig& config, + system_inspector::ContainerIDCache& container_id_cache) + : event_names_(EventNames::GetInstance()), + inspector_(inspector), + event_extractor_(std::make_unique()), + container_id_cache_(container_id_cache), + config_(config) { event_extractor_->Init(inspector); } ProcessSignalFormatter::~ProcessSignalFormatter() {} const SignalStreamMessage* ProcessSignalFormatter::ToProtoMessage(sinsp_evt* event) { + if (event->get_thread_info() == nullptr) { + CLOG_THROTTLED(INFO, std::chrono::seconds(60)) << "Dropping process event without thread information"; + return nullptr; + } + if (process_signals[event->get_type()] == ProcessSignalType::UNKNOWN_PROCESS_TYPE) { return nullptr; } @@ -176,7 +185,7 @@ ProcessSignal* ProcessSignalFormatter::CreateProcessSignal(sinsp_evt* event) { signal->set_allocated_time(timestamp); // set container_id - auto container_id = GetContainerID(event); + auto container_id = container_id_cache_.Get(*event); if (!container_id.empty()) { signal->set_container_id(container_id); } @@ -242,7 +251,7 @@ ProcessSignal* ProcessSignalFormatter::CreateProcessSignal(sinsp_threadinfo* tin signal->set_allocated_time(timestamp); // set container_id - signal->set_container_id(GetContainerID(*tinfo)); + signal->set_container_id(container_id_cache_.Get(*tinfo)); // set process lineage std::vector lineage; @@ -266,7 +275,7 @@ std::string ProcessSignalFormatter::ProcessDetails(sinsp_evt* event) { std::stringstream ss; const std::string* path = event_extractor_->get_exepath(event); const std::string* name = event_extractor_->get_comm(event); - auto container_id = GetContainerID(event); + std::string container_id = container_id_cache_.Get(*event); const char* args = event_extractor_->get_proc_args(event); const int64_t* pid = event_extractor_->get_pid(event); @@ -348,7 +357,7 @@ void ProcessSignalFormatter::GetProcessLineage(sinsp_threadinfo* tinfo, // all platforms. // if (pt->m_vpid == 0) { - if (GetContainerID(*pt).empty()) { + if (!container_id_cache_.Exists(*pt)) { return false; } } else if (pt->m_pid == pt->m_vpid) { diff --git a/collector/lib/ProcessSignalFormatter.h b/collector/lib/ProcessSignalFormatter.h index ceeeb98dea..eec4eef89d 100644 --- a/collector/lib/ProcessSignalFormatter.h +++ b/collector/lib/ProcessSignalFormatter.h @@ -19,14 +19,16 @@ class sinsp_threadinfo; namespace collector { namespace system_inspector { class EventExtractor; -} +class ContainerIDCache; +} // namespace system_inspector } // namespace collector namespace collector { class ProcessSignalFormatter : public ProtoSignalFormatter { public: - ProcessSignalFormatter(sinsp* inspector, const CollectorConfig& config); + ProcessSignalFormatter( + sinsp* inspector, const CollectorConfig& config, system_inspector::ContainerIDCache& container_id_cache); ~ProcessSignalFormatter(); using Signal = v1::Signal; @@ -56,6 +58,7 @@ class ProcessSignalFormatter : public ProtoSignalFormatter event_extractor_; + system_inspector::ContainerIDCache& container_id_cache_; const CollectorConfig& config_; }; diff --git a/collector/lib/ProcessSignalHandler.h b/collector/lib/ProcessSignalHandler.h index b6c2797a17..04aa8a4fb9 100644 --- a/collector/lib/ProcessSignalHandler.h +++ b/collector/lib/ProcessSignalHandler.h @@ -23,9 +23,10 @@ class ProcessSignalHandler : public SignalHandler { sinsp* inspector, ISignalServiceClient* client, system_inspector::Stats* stats, - const CollectorConfig& config) + const CollectorConfig& config, + system_inspector::ContainerIDCache& container_id_cache) : client_(client), - formatter_(inspector, config), + formatter_(inspector, config, container_id_cache), stats_(stats), config_(config) {} diff --git a/collector/lib/Utility.cpp b/collector/lib/Utility.cpp index 20e021d99e..22fd7a5530 100644 --- a/collector/lib/Utility.cpp +++ b/collector/lib/Utility.cpp @@ -15,6 +15,8 @@ extern "C" { #include } +#include +#include #include #include @@ -57,29 +59,9 @@ const char* SignalName(int signum) { } } -std::string GetContainerID(const sinsp_threadinfo& tinfo) { - for (const auto& [subsys, cgroup_path] : tinfo.cgroups()) { - if (auto id = ExtractContainerIDFromCgroup(cgroup_path)) { - return std::string(*id); - } - } - return {}; -} - -std::string GetContainerID(sinsp_evt* event) { - if (!event) { - return {}; - } - sinsp_threadinfo* tinfo = event->get_thread_info(); - if (!tinfo) { - return {}; - } - return GetContainerID(*tinfo); -} - std::ostream& operator<<(std::ostream& os, const sinsp_threadinfo* t) { if (t) { - os << "Container: \"" << GetContainerID(*t) << "\", Name: " << t->m_comm << ", PID: " << t->m_pid << ", Args: " << t->m_exe; + os << "Name: " << t->m_comm << ", PID: " << t->m_pid << ", Args: " << t->m_exe; } else { os << "NULL\n"; } @@ -195,50 +177,31 @@ void TryUnlink(const char* path) { } } -const static unsigned int CONTAINER_ID_LENGTH = 64; -const static unsigned int SHORT_CONTAINER_ID_LENGTH = 12; - -// IsContainerID returns whether the given string view represents a container ID. -bool IsContainerID(std::string_view str) { - if (str.size() != CONTAINER_ID_LENGTH) { - return false; - } - - return std::all_of(str.begin(), str.end(), [](char c) -> bool { - return std::isxdigit(c) != 0; - }); -} - std::optional ExtractContainerIDFromCgroup(std::string_view cgroup) { - if (cgroup.size() < CONTAINER_ID_LENGTH + 1) { - return {}; - } - - auto scope = cgroup.rfind(".scope"); + constexpr size_t kContainerIDLength = 64; + constexpr size_t kShortContainerIDLength = 12; + const auto scope = cgroup.rfind(".scope"); if (scope != std::string_view::npos) { - cgroup.remove_suffix(cgroup.length() - scope); - if (cgroup.size() < CONTAINER_ID_LENGTH + 1) { - return {}; - } + cgroup.remove_suffix(cgroup.size() - scope); } - - auto container_id_part = cgroup.substr(cgroup.size() - (CONTAINER_ID_LENGTH + 1)); - if (container_id_part[0] != '/' && container_id_part[0] != '-' && container_id_part[0] != ':') { + if (cgroup.size() < kContainerIDLength + 1) { return {}; } - - cgroup.remove_suffix(CONTAINER_ID_LENGTH + 1); - // conmon runs as its own container, we ignore it. - if (cgroup.find("-conmon", cgroup.size() - StrLen("-conmon")) != std::string_view::npos) { + const auto id_start = cgroup.size() - kContainerIDLength; + const char separator = cgroup[id_start - 1]; + if (separator != '/' && separator != '-' && separator != ':') { return {}; } - - container_id_part.remove_prefix(1); - - if (!IsContainerID(container_id_part)) { + const std::string_view parent = cgroup.substr(0, id_start - 1); + constexpr std::string_view kConmonSuffix = "-conmon"; + if (parent.size() >= kConmonSuffix.size() && parent.substr(parent.size() - kConmonSuffix.size()) == kConmonSuffix) { + return {}; + } + const std::string_view id = cgroup.substr(id_start); + if (!std::all_of(id.begin(), id.end(), [](char c) { return std::isxdigit(static_cast(c)); })) { return {}; } - return std::make_optional(container_id_part.substr(0, SHORT_CONTAINER_ID_LENGTH)); + return id.substr(0, kShortContainerIDLength); } std::optional SanitizedUTF8(std::string_view str) { diff --git a/collector/lib/Utility.h b/collector/lib/Utility.h index f9ba73126b..04be8cd480 100644 --- a/collector/lib/Utility.h +++ b/collector/lib/Utility.h @@ -14,7 +14,6 @@ // forward declarations class sinsp_threadinfo; -class sinsp_evt; namespace collector { @@ -66,14 +65,6 @@ std::string Str(Args&&... args) { std::ostream& operator<<(std::ostream& os, const sinsp_threadinfo* t); -// Extract container ID from a threadinfo's cgroups. -// Returns an empty string if no container ID found. -std::string GetContainerID(const sinsp_threadinfo& tinfo); - -// Extract container ID from an event's thread info cgroups. -// Returns an empty string if no container ID found. -std::string GetContainerID(sinsp_evt* event); - // UUIDStr returns UUID in string format. const char* UUIDStr(); diff --git a/collector/lib/system-inspector/ContainerIDCache.cpp b/collector/lib/system-inspector/ContainerIDCache.cpp new file mode 100644 index 0000000000..ec6e668a22 --- /dev/null +++ b/collector/lib/system-inspector/ContainerIDCache.cpp @@ -0,0 +1,98 @@ +#include "ContainerIDCache.h" + +#include + +#include +#include +#include + +#include "CollectorStats.h" +#include "Utility.h" + +namespace collector::system_inspector { + +std::string ContainerIDCache::Cache(const sinsp_threadinfo& tinfo) const { + std::string container_id; + for (const auto& cgroup : tinfo.cgroups()) { + if (const auto id = ExtractContainerIDFromCgroup(cgroup.second)) { + container_id = *id; + break; + } + } + + // Only cache confirmed container IDs. An empty extraction can be transient + // (cgroups are populated when libsinsp (re)creates or refreshes the thread + // info, e.g. from an event that permits a /proc query or on clone/execve + // exit parsing), so empty lookups must recompute. A thread's container ID + // otherwise never changes during its lifetime. + if (!container_id.empty()) { + std::lock_guard lock(mutex_); + // A reused tid gets a new clone timestamp, so a stale entry can never + // validate as a hit; it misses and recomputes. + entries_[tinfo.m_tid] = {tinfo.m_clone_ts, container_id}; + } else { + // Separates threads without a container ID (expected, e.g. host + // processes) from genuine cache misses, so a hit ratio is computable. + COUNTER_INC(CollectorStats::container_id_cache_empty_extractions); + } + return container_id; +} + +void ContainerIDCache::Initialize(sinsp& inspector) { + inspector.m_thread_manager->get_threads()->loop([this](sinsp_threadinfo& tinfo) { + Cache(tinfo); + return true; + }); +} + +// Prune has a single caller, the event loop under the service's libsinsp +// mutex, so last_prune_us_ needs no atomic. +void ContainerIDCache::Prune(sinsp& inspector, uint64_t now_us) { + if (now_us - last_prune_us_ < 60'000'000) { + return; + } + last_prune_us_ = now_us; + + std::unordered_set live_tids; + inspector.m_thread_manager->get_threads()->loop([&live_tids](sinsp_threadinfo& tinfo) { + live_tids.insert(tinfo.m_tid); + return true; + }); + + std::lock_guard lock(mutex_); + for (auto it = entries_.begin(); it != entries_.end();) { + if (live_tids.count(it->first) == 0) { + it = entries_.erase(it); + } else { + ++it; + } + } +} + +std::string ContainerIDCache::Get(const sinsp_threadinfo& tinfo) const { + { + std::lock_guard lock(mutex_); + const auto entry = entries_.find(tinfo.m_tid); + if (entry != entries_.end() && entry->second.clone_ts == tinfo.m_clone_ts) { + COUNTER_INC(CollectorStats::container_id_cache_hits); + return entry->second.container_id; + } + COUNTER_INC(CollectorStats::container_id_cache_misses); + } + + return Cache(tinfo); +} + +bool ContainerIDCache::Exists(const sinsp_threadinfo& tinfo) const { + return !Get(tinfo).empty(); +} + +std::string ContainerIDCache::Get(sinsp_evt& evt) const { + auto* tinfo = evt.get_thread_info(); + if (tinfo == nullptr) { + return {}; + } + return Get(*tinfo); +} + +} // namespace collector::system_inspector diff --git a/collector/lib/system-inspector/ContainerIDCache.h b/collector/lib/system-inspector/ContainerIDCache.h new file mode 100644 index 0000000000..9d36063ae2 --- /dev/null +++ b/collector/lib/system-inspector/ContainerIDCache.h @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include +#include + +class sinsp; +class sinsp_evt; +class sinsp_threadinfo; + +namespace collector::system_inspector { + +// Caches container IDs extracted from thread cgroups, keyed by thread ID and +// validated by clone timestamp. Only non-empty extractions are cached: cgroups +// can be populated after a thread is first seen (e.g. a dropped clone event, +// with the cgroups arriving on the next execve), so empty lookups recompute +// until a container ID is found. +class ContainerIDCache final { + public: + void Initialize(sinsp& inspector); + void Prune(sinsp& inspector, uint64_t now_us); + std::string Get(const sinsp_threadinfo& tinfo) const; + // Returns whether a thread has a container ID, populating the cache on a miss. + bool Exists(const sinsp_threadinfo& tinfo) const; + // Returns an empty string if the event carries no thread information. + std::string Get(sinsp_evt& evt) const; + + private: + struct Entry { + uint64_t clone_ts; + std::string container_id; + }; + + std::string Cache(const sinsp_threadinfo& tinfo) const; + + mutable std::mutex mutex_; + mutable std::unordered_map entries_; + uint64_t last_prune_us_ = 0; +}; + +} // namespace collector::system_inspector diff --git a/collector/lib/system-inspector/ContainerIDFilterCheck.cpp b/collector/lib/system-inspector/ContainerIDFilterCheck.cpp new file mode 100644 index 0000000000..d99bed0e10 --- /dev/null +++ b/collector/lib/system-inspector/ContainerIDFilterCheck.cpp @@ -0,0 +1,74 @@ +#include "ContainerIDFilterCheck.h" + +#include +#include +#include + +#include "ContainerIDCache.h" + +namespace collector::system_inspector { + +std::unique_ptr CompileFilter( + sinsp& inspector, const ContainerIDCache* container_id_cache, const std::string& expression) { + sinsp_filter_check_list filter_list; + filter_list.add_filter_check(inspector.new_generic_filtercheck()); + filter_list.add_filter_check(std::make_unique(container_id_cache)); + auto filter_factory = std::make_shared(&inspector, filter_list); + sinsp_filter_compiler filter_compiler(filter_factory, expression); + return filter_compiler.compile(); +} + +namespace { + +constexpr char kHostContainerID[] = "host"; + +const filtercheck_field_info kFields[] = { + {PT_CHARBUF, EPF_NONE, PF_NA, "container.id", "Container", "Cached container ID for the event thread"}, +}; + +} // namespace + +ContainerIDFilterCheck::ContainerIDFilterCheck(const ContainerIDCache* container_id_cache) + : container_id_cache_(container_id_cache) { + static const filter_check_info info = { + "container", + "Container fields", + "Container fields", + sizeof(kFields) / sizeof(kFields[0]), + kFields, + filter_check_info::FL_NONE, + }; + m_info = &info; +} + +std::unique_ptr ContainerIDFilterCheck::allocate_new() { + return std::make_unique(container_id_cache_); +} + +uint8_t* ContainerIDFilterCheck::extract_single(sinsp_evt* event, uint32_t* len, bool) { + *len = 0; + if (event == nullptr || m_field_id != 0) { + return nullptr; + } + + sinsp_threadinfo* tinfo = event->get_thread_info(); + if (tinfo == nullptr) { + return nullptr; + } + + result_ = container_id_cache_->Get(*tinfo); + if (!result_.empty()) { + *len = result_.size(); + return reinterpret_cast(result_.data()); + } + + // Match the bundled container filter: an empty ID identifies the host only + // when the process is outside a PID namespace. + if (!tinfo->is_in_pid_namespace()) { + *len = sizeof(kHostContainerID) - 1; + return reinterpret_cast(const_cast(kHostContainerID)); + } + return reinterpret_cast(result_.data()); +} + +} // namespace collector::system_inspector diff --git a/collector/lib/system-inspector/ContainerIDFilterCheck.h b/collector/lib/system-inspector/ContainerIDFilterCheck.h new file mode 100644 index 0000000000..7a3fdc760d --- /dev/null +++ b/collector/lib/system-inspector/ContainerIDFilterCheck.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include +#include + +#include + +class sinsp; +class sinsp_filter; + +namespace collector::system_inspector { + +class ContainerIDCache; + +std::unique_ptr CompileFilter( + sinsp& inspector, const ContainerIDCache* container_id_cache, const std::string& expression); + +class ContainerIDFilterCheck final : public sinsp_filter_check { + public: + explicit ContainerIDFilterCheck(const ContainerIDCache* container_id_cache); + + std::unique_ptr allocate_new() override; + + protected: + uint8_t* extract_single(sinsp_evt* event, uint32_t* len, bool sanitize_strings) override; + + private: + const ContainerIDCache* container_id_cache_; + std::string result_; +}; + +} // namespace collector::system_inspector diff --git a/collector/lib/system-inspector/Service.cpp b/collector/lib/system-inspector/Service.cpp index 9985bdce68..a3a9e45286 100644 --- a/collector/lib/system-inspector/Service.cpp +++ b/collector/lib/system-inspector/Service.cpp @@ -6,7 +6,9 @@ #include +#include "libsinsp/filter.h" #include "libsinsp/parsers.h" +#include "libsinsp/plugin.h" #include "libsinsp/sinsp.h" #include @@ -14,6 +16,8 @@ #include "CollectionMethod.h" #include "CollectorException.h" #include "CollectorStats.h" +#include "ContainerIDCache.h" +#include "ContainerIDFilterCheck.h" #include "EventExtractor.h" #include "EventNames.h" #include "HostInfo.h" @@ -33,6 +37,7 @@ Service::~Service() = default; Service::Service(const CollectorConfig& config) : inspector_(std::make_unique(true)), + container_id_cache_(std::make_unique()), default_formatter_(std::make_unique( inspector_.get(), DEFAULT_OUTPUT_STR, @@ -58,36 +63,6 @@ Service::Service(const CollectorConfig& config) inspector_->get_parser()->set_track_connection_status(true); } - // Filter out host processes to avoid flooding Sensor with events it - // cannot associate with a container. The filter has two clauses: - // - // 1. pid != vpid — In a PID namespace (the common container case), - // the kernel PID differs from the virtual PID visible inside the - // container. The val() transformer makes the parser treat - // proc.vpid as a field reference instead of a literal string. - // - // 2. cgroup regex — Catches containers that share the host PID - // namespace (hostPID: true), where pid == vpid despite the - // process running inside a container. Container runtimes always - // place container processes in a cgroup whose path ends with the - // 64-hex-character container ID, so matching that pattern - // identifies containerised processes regardless of PID namespace - // configuration. This mirrors the cgroup-based container ID - // extraction in ExtractContainerIDFromCgroup(). - // - // The memory cgroup is used because on cgroups v2 with systemd, - // the memory controller is reliably delegated to the container's - // leaf cgroup (where the path contains the container ID), while - // cpuset is often only available at a higher level in the - // hierarchy. The trailing (/.*) accounts for additional path - // components some runtimes append (e.g. podman adds /container). - // - // The 'or' short-circuits: the regex only evaluates for events where - // the PID check fails, so the performance cost is negligible. - inspector_->set_filter( - "proc.pid != val(proc.vpid)" - " or thread.cgroup.memory regex \".*[/:-][0-9a-f]{64}(\\\\.scope)?(/.*)?\""); - // The self-check handlers should only operate during start up, // so they are added to the handler list first, so they have access // to self-check events before the network and process handlers have @@ -103,7 +78,8 @@ Service::Service(const CollectorConfig& config) AddSignalHandler(std::make_unique(inspector_.get(), signal_client_.get(), &userspace_stats_, - config)); + config, + *container_id_cache_)); if (signal_handlers_.size() == 2) { // self-check handlers do not count towards this check, because they @@ -118,6 +94,12 @@ bool Service::InitKernel(const CollectorConfig& config) { CLOG(ERROR) << "Failed to setup " << config.GetCollectionMethod() << " driver."; return false; } + container_id_cache_->Initialize(*inspector_); + + constexpr char kSinspFilterExpression[] = "proc.pid != val(proc.vpid) or container.id != host"; + inspector_->set_filter( + CompileFilter(*inspector_, container_id_cache_.get(), kSinspFilterExpression), + kSinspFilterExpression); return true; } @@ -131,6 +113,7 @@ sinsp_evt* Service::GetNext() { if (res != SCAP_SUCCESS || event == nullptr) { return nullptr; } + container_id_cache_->Prune(*inspector_, NowMicros()); #ifdef TRACE_SINSP_EVENTS // Do not allow to change sinsp events tracing at runtime, as the output @@ -174,7 +157,10 @@ sinsp_evt* Service::GetNext() { bool Service::FilterEvent(sinsp_evt* event) { const auto* tinfo = event->get_thread_info(); - return FilterEvent(tinfo); + // Exclude host processes that pass the sinsp filter. Requiring a cached + // container ID also rejects runtime helpers whose cgroups do not resolve to + // a container ID, without maintaining a list of helper names. + return tinfo != nullptr && !container_id_cache_->Get(*tinfo).empty() && FilterEvent(tinfo); } bool Service::FilterEvent(const sinsp_threadinfo* tinfo) { @@ -182,17 +168,6 @@ bool Service::FilterEvent(const sinsp_threadinfo* tinfo) { return false; } - // Exclude host processes that leak through the sinsp filter. - // The sinsp filter uses a cgroup regex to catch containers in - // the host PID namespace, but this can also match container - // runtime helpers (crun, runc, conmon, podman) that run on the - // host within cgroup paths containing container IDs. Checking - // GetContainerID catches all such cases without maintaining a - // list of runtime helper names. - if (GetContainerID(*tinfo).empty()) { - return false; - } - std::string_view exepath_sv{tinfo->m_exepath}; auto marker = exepath_sv.rfind(':'); if (marker != std::string_view::npos) { @@ -300,7 +275,7 @@ bool Service::SendExistingProcesses(SignalHandler* handler) { } return threads->loop([&](sinsp_threadinfo& tinfo) { - if (!GetContainerID(tinfo).empty() && tinfo.is_main_thread()) { + if (tinfo.is_main_thread() && !container_id_cache_->Get(tinfo).empty()) { auto result = handler->HandleExistingProcess(&tinfo); if (result == SignalHandler::ERROR || result == SignalHandler::NEEDS_REFRESH) { CLOG(WARNING) << "Failed to write existing process signal: " << &tinfo; diff --git a/collector/lib/system-inspector/Service.h b/collector/lib/system-inspector/Service.h index 1f2398c648..6ac96e1506 100644 --- a/collector/lib/system-inspector/Service.h +++ b/collector/lib/system-inspector/Service.h @@ -21,6 +21,7 @@ class sinsp_threadinfo; namespace collector::system_inspector { +class ContainerIDCache; class Service : public SystemInspector { public: Service(const Service&) = delete; @@ -43,6 +44,7 @@ class Service : public SystemInspector { void GetProcessInformation(uint64_t pid, ProcessInfoCallbackRef callback); sinsp* GetInspector() { return inspector_.get(); } + ContainerIDCache* GetContainerIDCache() { return container_id_cache_.get(); } Stats* GetUserspaceStats() { return &userspace_stats_; } void AddSignalHandler(std::unique_ptr signal_handler); @@ -61,13 +63,14 @@ class Service : public SystemInspector { }; sinsp_evt* GetNext(); - static bool FilterEvent(sinsp_evt* event); + bool FilterEvent(sinsp_evt* event); static bool FilterEvent(const sinsp_threadinfo* tinfo); bool SendExistingProcesses(SignalHandler* handler); mutable std::mutex libsinsp_mutex_; std::unique_ptr inspector_; + std::unique_ptr container_id_cache_; std::unique_ptr default_formatter_; std::unique_ptr signal_client_; std::vector signal_handlers_; diff --git a/collector/test/ProcessSignalFormatterTest.cpp b/collector/test/ProcessSignalFormatterTest.cpp index 233c3bfed3..c3fbbb6012 100644 --- a/collector/test/ProcessSignalFormatterTest.cpp +++ b/collector/test/ProcessSignalFormatterTest.cpp @@ -7,6 +7,7 @@ #include "ProcessSignalFormatter.h" #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "system-inspector/ContainerIDCache.h" namespace collector { @@ -26,8 +27,9 @@ TEST(ProcessSignalFormatterTest, NoProcessTest) { sinsp* inspector = NULL; CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector, config); + ProcessSignalFormatter processSignalFormatter(inspector, config, container_id_cache); sinsp_threadinfo* tinfo = NULL; std::vector lineage; @@ -51,8 +53,9 @@ TEST(ProcessSignalFormatterTest, ProcessWithoutParentTest) { std::unique_ptr inspector(new sinsp()); CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 0; @@ -86,8 +89,9 @@ TEST(ProcessSignalFormatterTest, ProcessWithParentTest) { std::unique_ptr inspector(new sinsp()); CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 3; @@ -131,8 +135,9 @@ TEST(ProcessSignalFormatterTest, ProcessWithParentWithPid0Test) { std::unique_ptr inspector(new sinsp()); CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 0; @@ -171,8 +176,9 @@ TEST(ProcessSignalFormatterTest, ProcessWithParentWithSameNameTest) { std::unique_ptr inspector(new sinsp()); CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 3; @@ -216,8 +222,9 @@ TEST(ProcessSignalFormatterTest, ProcessWithTwoParentsTest) { std::unique_ptr inspector(new sinsp()); CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 3; @@ -275,8 +282,9 @@ TEST(ProcessSignalFormatterTest, ProcessWithTwoParentsWithTheSameNameTest) { std::unique_ptr inspector(new sinsp()); CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 3; @@ -331,8 +339,9 @@ TEST(ProcessSignalFormatterTest, ProcessCollapseParentChildWithSameNameTest) { std::unique_ptr inspector(new sinsp()); CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 3; @@ -396,8 +405,9 @@ TEST(ProcessSignalFormatterTest, ProcessCollapseParentChildWithSameName2Test) { std::unique_ptr inspector(new sinsp()); CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 3; @@ -464,8 +474,9 @@ TEST(ProcessSignalFormatterTest, ProcessWithUnrelatedProcessTest) { std::unique_ptr inspector(new sinsp()); CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 3; @@ -532,8 +543,9 @@ TEST(ProcessSignalFormatterTest, CountTwoCounterCallsTest) { std::unique_ptr inspector(new sinsp()); CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 1; @@ -583,8 +595,9 @@ TEST(ProcessSignalFormatterTest, Rox3377ProcessLineageWithNoVPidTest) { std::unique_ptr inspector(new sinsp()); CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 3; @@ -638,8 +651,9 @@ TEST(ProcessSignalFormatterTest, Rox3377ProcessLineageWithNoVPidTest) { TEST(ProcessSignalFormatterTest, ProcessArguments) { std::unique_ptr inspector(new sinsp()); MockCollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 3; @@ -663,12 +677,28 @@ TEST(ProcessSignalFormatterTest, ProcessArguments) { EXPECT_FALSE(signal->args().empty()); } +TEST(ProcessSignalFormatterTest, EventWithoutThreadInfo) { + std::unique_ptr inspector(new sinsp()); + CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); + + sinsp_evt evt(inspector.get()); + scap_evt s_evt{}; + s_evt.type = PPME_SYSCALL_EXECVE_19_X; + s_evt.tid = -1; + evt.set_scap_evt(&s_evt); + + EXPECT_EQ(processSignalFormatter.ToProtoMessage(&evt), nullptr); +} + TEST(ProcessSignalFormatterTest, NoProcessArguments) { std::unique_ptr inspector(new sinsp()); MockCollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; config.SetDisableProcessArguments(true); - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 3; diff --git a/collector/test/SystemInspectorServiceTest.cpp b/collector/test/SystemInspectorServiceTest.cpp index c2745cdbbc..e0bc0de4f1 100644 --- a/collector/test/SystemInspectorServiceTest.cpp +++ b/collector/test/SystemInspectorServiceTest.cpp @@ -1,40 +1,67 @@ +#include #include +#include "CollectorStats.h" #include "gtest/gtest.h" +#include "system-inspector/ContainerIDCache.h" +#include "system-inspector/ContainerIDFilterCheck.h" #include "system-inspector/Service.h" namespace collector::system_inspector { +TEST(SystemInspectorServiceTest, CleanUpBeforeDestruction) { + CollectorConfig config; + { + Service service(config); + service.CleanUp(); + } +} + TEST(SystemInspectorServiceTest, FilterEvent) { std::unique_ptr inspector(new sinsp()); + ContainerIDCache container_id_cache; + auto filter = CompileFilter(*inspector, &container_id_cache, "container.id != host"); const auto& factory = inspector->get_threadinfo_factory(); - // A container cgroup path with a 64-hex-char container ID. - const std::string container_cgroup = "/kubepods/burstable/pod123/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - sinsp_threadinfo::cgroups_t container_cgroups = {{"memory", container_cgroup}}; - auto regular_process = factory.create(); + regular_process->m_tid = 1; regular_process->m_exepath = "/bin/busybox"; regular_process->m_comm = "sleep"; - regular_process->set_cgroups(container_cgroups); + regular_process->set_cgroups({"cpu=/docker/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}); + ASSERT_EQ(container_id_cache.Get(*regular_process), "aaaaaaaaaaaa"); auto runc_process = factory.create(); + runc_process->m_tid = 2; runc_process->m_exepath = "runc"; runc_process->m_comm = "6"; + auto host_process = factory.create(); + host_process->m_tid = 3; + host_process->m_exepath = "/usr/bin/bash"; + host_process->m_comm = "bash"; + + auto pid_namespace_process = factory.create(); + pid_namespace_process->m_tid = 42; + pid_namespace_process->m_vtid = 1; + + auto uncached_container_process = factory.create(); + uncached_container_process->m_tid = 43; + uncached_container_process->set_cgroups({"cpu=/docker/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}); + auto proc_self_process = factory.create(); - proc_self_process->m_exepath = "/proc/self/exe"; - proc_self_process->m_comm = "6"; - proc_self_process->set_cgroups(container_cgroups); + proc_self_process->m_tid = 4; + proc_self_process->m_exepath = "container:/proc/self/exe"; auto memfd_process = factory.create(); - memfd_process->m_exepath = "memfd:runc_cloned:/proc/self/exe"; - memfd_process->m_comm = "6"; - memfd_process->set_cgroups(container_cgroups); + memfd_process->m_tid = 5; + memfd_process->m_exepath = "/memfd:payload (deleted)"; - auto host_process = factory.create(); - host_process->m_exepath = "/usr/bin/bash"; - host_process->m_comm = "bash"; + auto runc_cloned_process = factory.create(); + runc_cloned_process->m_tid = 6; + runc_cloned_process->m_exepath = "memfd:runc_cloned:/proc/self/exe"; + runc_cloned_process->set_cgroups({"cpu=/docker/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}); + + sinsp_evt event(inspector.get()); struct test_t { const sinsp_threadinfo* tinfo; @@ -43,16 +70,147 @@ TEST(SystemInspectorServiceTest, FilterEvent) { }; std::vector tests{ {regular_process.get(), true, "regular container process"}, - {runc_process.get(), false, "runc (no container ID)"}, - {proc_self_process.get(), false, "/proc/self exe path"}, - {memfd_process.get(), false, "memfd /proc/self exe path"}, - {host_process.get(), false, "host process (no container ID)"}, + {runc_process.get(), true, "runc process"}, + {host_process.get(), true, "host process"}, + {proc_self_process.get(), false, "proc self executable"}, + {memfd_process.get(), true, "memfd executable"}, + {runc_cloned_process.get(), false, "runc-cloned memfd /proc/self exe path"}, }; + // The static FilterEvent only enforces the exepath rule; the container-ID + // rejection lives in the production instance guard asserted further below. for (const auto& t : tests) { ASSERT_EQ(system_inspector::Service::FilterEvent(t.tinfo), t.expected) << "Failed for: " << t.name; } + + event.set_tinfo(regular_process.get()); + EXPECT_TRUE(filter->run(&event)); + event.set_tinfo(runc_process.get()); + EXPECT_FALSE(filter->run(&event)); + event.set_tinfo(host_process.get()); + EXPECT_FALSE(filter->run(&event)); + event.set_tinfo(pid_namespace_process.get()); + EXPECT_TRUE(filter->run(&event)); + + // Service::FilterEvent(sinsp_evt*) is the production guard run by GetNext: + // an event passes only when its thread resolves to a non-empty cached + // container ID (rejecting host processes, runtime helpers, and + // PID-namespace threads without a container ID) and survives the exepath + // rule asserted above. The event's thread info belongs to the test + // inspector; the service consults its own cache, which populates lazily on + // the first lookup. + CollectorConfig config; + Service service(config); + event.set_tinfo(host_process.get()); + EXPECT_FALSE(service.FilterEvent(&event)); + event.set_tinfo(runc_process.get()); + EXPECT_FALSE(service.FilterEvent(&event)); + event.set_tinfo(pid_namespace_process.get()); + EXPECT_FALSE(service.FilterEvent(&event)); + event.set_tinfo(regular_process.get()); + EXPECT_TRUE(service.FilterEvent(&event)); + event.set_tinfo(runc_cloned_process.get()); + EXPECT_FALSE(service.FilterEvent(&event)); + service.CleanUp(); + + EXPECT_EQ(container_id_cache.Get(*uncached_container_process), "aaaaaaaaaaaa"); +} + +TEST(SystemInspectorServiceTest, ContainerIDCacheLookupStats) { + std::unique_ptr inspector(new sinsp()); + ContainerIDCache container_id_cache; + COUNTER_ZERO(CollectorStats::container_id_cache_hits); + COUNTER_ZERO(CollectorStats::container_id_cache_misses); + COUNTER_ZERO(CollectorStats::container_id_cache_empty_extractions); + auto tinfo = inspector->get_threadinfo_factory().create(); + tinfo->m_tid = 42; + tinfo->m_clone_ts = 100; + tinfo->set_cgroups({"cpu=/docker/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}); + + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + + ++tinfo->m_clone_ts; + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + + // Threads without a container ID are never cached, so their lookups + // recompute (and count as misses) on every event. + auto host_tinfo = inspector->get_threadinfo_factory().create(); + host_tinfo->m_tid = 43; + EXPECT_EQ(container_id_cache.Get(*host_tinfo), ""); + EXPECT_EQ(container_id_cache.Get(*host_tinfo), ""); + + EXPECT_EQ(CollectorStats::GetOrCreate().GetCounter(CollectorStats::container_id_cache_hits), 1); + EXPECT_EQ(CollectorStats::GetOrCreate().GetCounter(CollectorStats::container_id_cache_misses), 4); + EXPECT_EQ(CollectorStats::GetOrCreate().GetCounter(CollectorStats::container_id_cache_empty_extractions), 2); + COUNTER_ZERO(CollectorStats::container_id_cache_hits); + COUNTER_ZERO(CollectorStats::container_id_cache_misses); + COUNTER_ZERO(CollectorStats::container_id_cache_empty_extractions); +} + +TEST(SystemInspectorServiceTest, ContainerIDCacheLateCgroups) { + std::unique_ptr inspector(new sinsp()); + ContainerIDCache container_id_cache; + auto tinfo = inspector->get_threadinfo_factory().create(); + tinfo->m_tid = 42; + tinfo->m_clone_ts = 100; + + // A thread first seen before its cgroups are known (e.g. a dropped clone + // event) has no container ID to cache. + EXPECT_EQ(container_id_cache.Get(*tinfo), ""); + + // When the cgroups arrive (libsinsp populates them when it (re)creates or + // refreshes the thread info, e.g. from an event permitting a /proc query or + // clone/execve exit parsing), the next lookup must pick up the container ID + // instead of pinning the empty result. + tinfo->set_cgroups({"cpu=/docker/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}); + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); +} + +TEST(SystemInspectorServiceTest, ContainerIDCachePrune) { + std::unique_ptr inspector(new sinsp()); + ContainerIDCache container_id_cache; + COUNTER_ZERO(CollectorStats::container_id_cache_hits); + COUNTER_ZERO(CollectorStats::container_id_cache_misses); + COUNTER_ZERO(CollectorStats::container_id_cache_empty_extractions); + auto tinfo = inspector->get_threadinfo_factory().create(); + tinfo->m_pid = 42; + tinfo->m_tid = 42; + tinfo->m_ptid = -1; + tinfo->m_clone_ts = 100; + tinfo->set_cgroups({"cpu=/docker/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}); + + // The thread is not in the thread table: the first lookup extracts and + // caches the container ID. + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + + // The first prune (last_prune_us_ starts at 0, so 61s clears the 60s rate + // limit) evicts the entry, because the tid is not in the thread table. + container_id_cache.Prune(*inspector, 61'000'000); + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + + // Only 30s since the last prune: rate-limited no-op, the entry survives. + container_id_cache.Prune(*inspector, 91'000'000); + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + + // A thread present in the thread table counts as live. + inspector->m_thread_manager->add_thread(std::move(tinfo), false); + + // 61s since the last real prune: prunes, but the tid is now live, so the + // entry survives. + container_id_cache.Prune(*inspector, 122'000'000); + EXPECT_EQ(container_id_cache.Get(*inspector->m_thread_manager->find_thread(42, true)), "aaaaaaaaaaaa"); + + EXPECT_EQ(CollectorStats::GetOrCreate().GetCounter(CollectorStats::container_id_cache_hits), 4); + EXPECT_EQ(CollectorStats::GetOrCreate().GetCounter(CollectorStats::container_id_cache_misses), 2); + EXPECT_EQ(CollectorStats::GetOrCreate().GetCounter(CollectorStats::container_id_cache_empty_extractions), 0); + COUNTER_ZERO(CollectorStats::container_id_cache_hits); + COUNTER_ZERO(CollectorStats::container_id_cache_misses); + COUNTER_ZERO(CollectorStats::container_id_cache_empty_extractions); } } // namespace collector::system_inspector From 7d714bc6816a058b4d598c442a1655db89fb0985 Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Fri, 6 Nov 2026 10:00:31 +0000 Subject: [PATCH 2/2] PR review fixes --- collector/lib/CollectorConfig.cpp | 4 ++ collector/lib/CollectorConfig.h | 6 ++ collector/lib/CollectorStatsExporter.cpp | 3 + collector/lib/Process.cpp | 46 +++++++-------- collector/lib/Process.h | 5 +- collector/lib/ProcessSignalFormatter.cpp | 4 +- collector/lib/Utility.cpp | 27 +++++++-- .../lib/system-inspector/ContainerIDCache.cpp | 20 +++++-- .../lib/system-inspector/ContainerIDCache.h | 16 ++++-- .../ContainerIDFilterCheck.cpp | 4 -- .../system-inspector/ContainerIDFilterCheck.h | 6 +- collector/lib/system-inspector/Service.cpp | 23 +++++--- collector/test/SystemInspectorServiceTest.cpp | 57 ++++++++++++++++++- collector/test/UtilityTest.cpp | 8 +++ docs/references.md | 5 ++ 15 files changed, 179 insertions(+), 55 deletions(-) diff --git a/collector/lib/CollectorConfig.cpp b/collector/lib/CollectorConfig.cpp index 294e25bb4d..a93d63d1d2 100644 --- a/collector/lib/CollectorConfig.cpp +++ b/collector/lib/CollectorConfig.cpp @@ -82,6 +82,9 @@ PathEnvVar tls_client_cert_path("ROX_COLLECTOR_TLS_CLIENT_CERT"); PathEnvVar tls_client_key_path("ROX_COLLECTOR_TLS_CLIENT_KEY"); BoolEnvVar disable_process_arguments("ROX_COLLECTOR_NO_PROCESS_ARGUMENTS", false); + +// Interval between container ID cache prunes, in seconds. +IntEnvVar container_id_cache_prune_interval("ROX_COLLECTOR_CONTAINER_ID_CACHE_PRUNE_INTERVAL", 60); } // namespace constexpr bool CollectorConfig::kTurnOffScrape; @@ -112,6 +115,7 @@ void CollectorConfig::InitCollectorConfig(CollectorArgs* args) { enable_introspection_ = enable_introspection.value(); track_send_recv_ = track_send_recv.value(); disable_process_arguments_ = disable_process_arguments.value(); + container_id_cache_prune_interval_ = container_id_cache_prune_interval.value(); for (const auto& syscall : kSyscalls) { syscalls_.emplace_back(syscall); diff --git a/collector/lib/CollectorConfig.h b/collector/lib/CollectorConfig.h index 2a5244c29e..a29b661ed7 100644 --- a/collector/lib/CollectorConfig.h +++ b/collector/lib/CollectorConfig.h @@ -160,6 +160,7 @@ class CollectorConfig { unsigned int GetSinspBufferSize() const; unsigned int GetSinspTotalBufferSize() const { return sinsp_total_buffer_size_; } unsigned int GetSinspThreadCacheSize() const { return sinsp_thread_cache_size_; } + unsigned int GetContainerIDCachePruneInterval() const { return container_id_cache_prune_interval_; } bool DisableProcessArguments() const { return disable_process_arguments_; } static std::pair CheckConfiguration(const char* config, Json::Value* root); @@ -241,6 +242,11 @@ class CollectorConfig { // is 2^17 (131072) and twice as large. unsigned int sinsp_thread_cache_size_ = 32768; + // Interval between container ID cache prunes, in seconds. Pruning syncs + // the cache with the sinsp thread table, whose own purging interval this + // deliberately mirrors. + unsigned int container_id_cache_prune_interval_ = 60; + std::optional tls_config_; std::optional runtime_config_; diff --git a/collector/lib/CollectorStatsExporter.cpp b/collector/lib/CollectorStatsExporter.cpp index ee628c8eb6..4fc3b07060 100644 --- a/collector/lib/CollectorStatsExporter.cpp +++ b/collector/lib/CollectorStatsExporter.cpp @@ -9,6 +9,7 @@ #include "Logging.h" #include "Utility.h" #include "prometheus/gauge.h" +#include "system-inspector/ContainerIDCache.h" #include "system-inspector/Service.h" namespace collector { @@ -58,6 +59,7 @@ void CollectorStatsExporter::run() { auto& preemptions = collectorEventCounters.Add({{"type", "preemptions"}}); auto& grpcSendFailures = collectorEventCounters.Add({{"type", "grpcSendFailures"}}); auto& threadTableSize = collectorEventCounters.Add({{"type", "threadCacheSize"}}); + auto& containerIDCacheSize = collectorEventCounters.Add({{"type", "containerIDCacheSize"}}); auto& processSent = collectorEventCounters.Add({{"type", "processSent"}}); auto& processSendFailures = collectorEventCounters.Add({{"type", "processSendFailures"}}); @@ -170,6 +172,7 @@ void CollectorStatsExporter::run() { ringbufferDrops.Set(stats.nDropsBuffer); preemptions.Set(stats.nPreemptions); threadTableSize.Set(stats.nThreadCacheSize); + containerIDCacheSize.Set(system_inspector_->GetContainerIDCache()->Size()); if (config_->EnableDetailedMetrics()) { uint64_t nUserspace = 0; diff --git a/collector/lib/Process.cpp b/collector/lib/Process.cpp index 489511cc69..88e1ae1d3c 100644 --- a/collector/lib/Process.cpp +++ b/collector/lib/Process.cpp @@ -30,13 +30,9 @@ const std::shared_ptr ProcessStore::Fetch(uint64_t pid) { } std::string Process::container_id() const { - WaitForProcessInfo(); - - // system_inspector_threadinfo_ is assigned by ProcessInfoResolved() on the - // inspector thread; hold the lock across the read and the extraction. - std::lock_guard lock(process_info_mutex_); - if (system_inspector_threadinfo_) { - for (const auto& cgroup : system_inspector_threadinfo_->cgroups()) { + const auto process_info = WaitForProcessInfo(); + if (process_info) { + for (const auto& cgroup : process_info->cgroups()) { if (const auto id = ExtractContainerIDFromCgroup(cgroup.second)) { return std::string(*id); } @@ -47,50 +43,50 @@ std::string Process::container_id() const { } std::string Process::comm() const { - WaitForProcessInfo(); + const auto process_info = WaitForProcessInfo(); - if (system_inspector_threadinfo_) { - return system_inspector_threadinfo_->get_comm(); + if (process_info) { + return process_info->get_comm(); } return NOT_AVAILABLE; } std::string Process::exe() const { - WaitForProcessInfo(); + const auto process_info = WaitForProcessInfo(); - if (system_inspector_threadinfo_) { - return system_inspector_threadinfo_->get_exe(); + if (process_info) { + return process_info->get_exe(); } return NOT_AVAILABLE; } std::string Process::exe_path() const { - WaitForProcessInfo(); + const auto process_info = WaitForProcessInfo(); - if (system_inspector_threadinfo_) { - return system_inspector_threadinfo_->get_exepath(); + if (process_info) { + return process_info->get_exepath(); } return NOT_AVAILABLE; } std::string Process::args() const { - WaitForProcessInfo(); + const auto process_info = WaitForProcessInfo(); - if (!system_inspector_threadinfo_) { + if (!process_info) { return NOT_AVAILABLE; } - if (system_inspector_threadinfo_->m_args.empty()) { + if (process_info->m_args.empty()) { return ""; } std::ostringstream args; - for (auto it = system_inspector_threadinfo_->m_args.begin(); it != system_inspector_threadinfo_->m_args.end();) { + for (auto it = process_info->m_args.begin(); it != process_info->m_args.end();) { args << *it++; - if (it != system_inspector_threadinfo_->m_args.end()) { + if (it != process_info->m_args.end()) { args << " "; } } @@ -120,7 +116,7 @@ Process::~Process() { } void Process::ProcessInfoResolved(std::shared_ptr process_info) { - std::unique_lock lock(process_info_mutex_); + std::lock_guard lock(process_info_mutex_); if (process_info) { CLOG(DEBUG) << "Process-info resolved. PID: " << pid() << " Exe: " + process_info->m_exe; @@ -134,7 +130,7 @@ void Process::ProcessInfoResolved(std::shared_ptr process_info process_info_condition_.notify_all(); } -void Process::WaitForProcessInfo() const { +std::shared_ptr Process::WaitForProcessInfo() const { std::unique_lock lock(process_info_mutex_); COUNTER_ADD( @@ -150,6 +146,10 @@ void Process::WaitForProcessInfo() const { CLOG_IF(std::cv_status::timeout == status, ERROR) << "Timed-out waiting for process-info. PID: " << pid(); } + + // Copy the shared pointer while holding the mutex; callers can then safely + // use the snapshot after this function releases the lock. + return system_inspector_threadinfo_; } std::ostream& operator<<(std::ostream& os, const IProcess& process) { diff --git a/collector/lib/Process.h b/collector/lib/Process.h index 9f828b36a8..a9ec4c488d 100644 --- a/collector/lib/Process.h +++ b/collector/lib/Process.h @@ -91,8 +91,9 @@ class Process : public IProcess { // entry-point when system inspector resolved the requested process info void ProcessInfoResolved(std::shared_ptr process_info); - // block until process information is available, or timeout - void WaitForProcessInfo() const; + // Block until process information is available or times out, then return a + // synchronized snapshot of the thread-info pointer. + std::shared_ptr WaitForProcessInfo() const; }; std::ostream& operator<<(std::ostream& os, const IProcess& process); diff --git a/collector/lib/ProcessSignalFormatter.cpp b/collector/lib/ProcessSignalFormatter.cpp index bddd537466..b003095efd 100644 --- a/collector/lib/ProcessSignalFormatter.cpp +++ b/collector/lib/ProcessSignalFormatter.cpp @@ -105,7 +105,9 @@ const SignalStreamMessage* ProcessSignalFormatter::ToProtoMessage(sinsp_evt* eve const SignalStreamMessage* ProcessSignalFormatter::ToProtoMessage(sinsp_threadinfo* tinfo) { Reset(); if (!ValidateProcessDetails(tinfo)) { - CLOG(INFO) << "Dropping process event: " << tinfo; + const auto container_id = container_id_cache_.Get(*tinfo); + CLOG(INFO) << "Dropping process event: Container: " + << (container_id.empty() ? "null" : container_id) << ", " << tinfo; return nullptr; } diff --git a/collector/lib/Utility.cpp b/collector/lib/Utility.cpp index 22fd7a5530..207254d854 100644 --- a/collector/lib/Utility.cpp +++ b/collector/lib/Utility.cpp @@ -177,9 +177,27 @@ void TryUnlink(const char* path) { } } +namespace { + +constexpr size_t kContainerIDLength = 64; +constexpr size_t kShortContainerIDLength = 12; + +bool IsContainerID(std::string_view id) { + if (id.size() != kContainerIDLength) { + return false; + } + + return std::all_of(id.begin(), id.end(), [](char c) { return std::isxdigit(static_cast(c)); }); +} + +} // namespace + std::optional ExtractContainerIDFromCgroup(std::string_view cgroup) { - constexpr size_t kContainerIDLength = 64; - constexpr size_t kShortContainerIDLength = 12; + // Reject too-short paths before searching for an optional systemd suffix. + if (cgroup.size() < kContainerIDLength + 1) { + return {}; + } + const auto scope = cgroup.rfind(".scope"); if (scope != std::string_view::npos) { cgroup.remove_suffix(cgroup.size() - scope); @@ -194,11 +212,12 @@ std::optional ExtractContainerIDFromCgroup(std::string_view cg } const std::string_view parent = cgroup.substr(0, id_start - 1); constexpr std::string_view kConmonSuffix = "-conmon"; - if (parent.size() >= kConmonSuffix.size() && parent.substr(parent.size() - kConmonSuffix.size()) == kConmonSuffix) { + if (parent.size() >= kConmonSuffix.size() && + parent.find(kConmonSuffix, parent.size() - kConmonSuffix.size()) != std::string_view::npos) { return {}; } const std::string_view id = cgroup.substr(id_start); - if (!std::all_of(id.begin(), id.end(), [](char c) { return std::isxdigit(static_cast(c)); })) { + if (!IsContainerID(id)) { return {}; } return id.substr(0, kShortContainerIDLength); diff --git a/collector/lib/system-inspector/ContainerIDCache.cpp b/collector/lib/system-inspector/ContainerIDCache.cpp index ec6e668a22..232337c7af 100644 --- a/collector/lib/system-inspector/ContainerIDCache.cpp +++ b/collector/lib/system-inspector/ContainerIDCache.cpp @@ -11,6 +11,9 @@ namespace collector::system_inspector { +ContainerIDCache::ContainerIDCache(uint64_t prune_interval_s) + : prune_interval_us_(prune_interval_s * 1'000'000) {} + std::string ContainerIDCache::Cache(const sinsp_threadinfo& tinfo) const { std::string container_id; for (const auto& cgroup : tinfo.cgroups()) { @@ -23,13 +26,14 @@ std::string ContainerIDCache::Cache(const sinsp_threadinfo& tinfo) const { // Only cache confirmed container IDs. An empty extraction can be transient // (cgroups are populated when libsinsp (re)creates or refreshes the thread // info, e.g. from an event that permits a /proc query or on clone/execve - // exit parsing), so empty lookups must recompute. A thread's container ID - // otherwise never changes during its lifetime. + // exit parsing), so empty lookups must recompute. The cache key also tracks + // the last exec timestamp because libsinsp may refresh cgroups on exec + // without changing the thread's clone timestamp. if (!container_id.empty()) { std::lock_guard lock(mutex_); // A reused tid gets a new clone timestamp, so a stale entry can never // validate as a hit; it misses and recomputes. - entries_[tinfo.m_tid] = {tinfo.m_clone_ts, container_id}; + entries_[tinfo.m_tid] = {tinfo.m_clone_ts, tinfo.m_lastexec_ts, container_id}; } else { // Separates threads without a container ID (expected, e.g. host // processes) from genuine cache misses, so a hit ratio is computable. @@ -48,7 +52,7 @@ void ContainerIDCache::Initialize(sinsp& inspector) { // Prune has a single caller, the event loop under the service's libsinsp // mutex, so last_prune_us_ needs no atomic. void ContainerIDCache::Prune(sinsp& inspector, uint64_t now_us) { - if (now_us - last_prune_us_ < 60'000'000) { + if (now_us - last_prune_us_ < prune_interval_us_) { return; } last_prune_us_ = now_us; @@ -73,7 +77,8 @@ std::string ContainerIDCache::Get(const sinsp_threadinfo& tinfo) const { { std::lock_guard lock(mutex_); const auto entry = entries_.find(tinfo.m_tid); - if (entry != entries_.end() && entry->second.clone_ts == tinfo.m_clone_ts) { + if (entry != entries_.end() && entry->second.clone_ts == tinfo.m_clone_ts && + entry->second.last_exec_ts == tinfo.m_lastexec_ts) { COUNTER_INC(CollectorStats::container_id_cache_hits); return entry->second.container_id; } @@ -83,6 +88,11 @@ std::string ContainerIDCache::Get(const sinsp_threadinfo& tinfo) const { return Cache(tinfo); } +std::size_t ContainerIDCache::Size() const { + std::lock_guard lock(mutex_); + return entries_.size(); +} + bool ContainerIDCache::Exists(const sinsp_threadinfo& tinfo) const { return !Get(tinfo).empty(); } diff --git a/collector/lib/system-inspector/ContainerIDCache.h b/collector/lib/system-inspector/ContainerIDCache.h index 9d36063ae2..ce7a481dcb 100644 --- a/collector/lib/system-inspector/ContainerIDCache.h +++ b/collector/lib/system-inspector/ContainerIDCache.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -12,15 +13,19 @@ class sinsp_threadinfo; namespace collector::system_inspector { // Caches container IDs extracted from thread cgroups, keyed by thread ID and -// validated by clone timestamp. Only non-empty extractions are cached: cgroups -// can be populated after a thread is first seen (e.g. a dropped clone event, -// with the cgroups arriving on the next execve), so empty lookups recompute -// until a container ID is found. +// validated by clone and last-exec timestamps. Only non-empty extractions are +// cached: cgroups can be populated after a thread is first seen (e.g. a dropped +// clone event, with the cgroups arriving on the next execve), so empty lookups +// recompute until a container ID is found. class ContainerIDCache final { public: + // prune_interval_s: how often, in seconds, the cache is pruned of entries + // whose thread is no longer present in the sinsp thread table. + explicit ContainerIDCache(uint64_t prune_interval_s = 60); void Initialize(sinsp& inspector); void Prune(sinsp& inspector, uint64_t now_us); std::string Get(const sinsp_threadinfo& tinfo) const; + std::size_t Size() const; // Returns whether a thread has a container ID, populating the cache on a miss. bool Exists(const sinsp_threadinfo& tinfo) const; // Returns an empty string if the event carries no thread information. @@ -29,6 +34,8 @@ class ContainerIDCache final { private: struct Entry { uint64_t clone_ts; + // libsinsp refreshes cgroups on exec, which can change the container ID. + uint64_t last_exec_ts; std::string container_id; }; @@ -36,6 +43,7 @@ class ContainerIDCache final { mutable std::mutex mutex_; mutable std::unordered_map entries_; + uint64_t prune_interval_us_; uint64_t last_prune_us_ = 0; }; diff --git a/collector/lib/system-inspector/ContainerIDFilterCheck.cpp b/collector/lib/system-inspector/ContainerIDFilterCheck.cpp index d99bed0e10..005ea0724c 100644 --- a/collector/lib/system-inspector/ContainerIDFilterCheck.cpp +++ b/collector/lib/system-inspector/ContainerIDFilterCheck.cpp @@ -41,10 +41,6 @@ ContainerIDFilterCheck::ContainerIDFilterCheck(const ContainerIDCache* container m_info = &info; } -std::unique_ptr ContainerIDFilterCheck::allocate_new() { - return std::make_unique(container_id_cache_); -} - uint8_t* ContainerIDFilterCheck::extract_single(sinsp_evt* event, uint32_t* len, bool) { *len = 0; if (event == nullptr || m_field_id != 0) { diff --git a/collector/lib/system-inspector/ContainerIDFilterCheck.h b/collector/lib/system-inspector/ContainerIDFilterCheck.h index 7a3fdc760d..e18bea70a6 100644 --- a/collector/lib/system-inspector/ContainerIDFilterCheck.h +++ b/collector/lib/system-inspector/ContainerIDFilterCheck.h @@ -20,13 +20,17 @@ class ContainerIDFilterCheck final : public sinsp_filter_check { public: explicit ContainerIDFilterCheck(const ContainerIDCache* container_id_cache); - std::unique_ptr allocate_new() override; + std::unique_ptr allocate_new() override { + return std::make_unique(container_id_cache_); + } protected: uint8_t* extract_single(sinsp_evt* event, uint32_t* len, bool sanitize_strings) override; private: const ContainerIDCache* container_id_cache_; + // Owns the storage returned by extract_single; libsinsp shallow-caches the + // pointer for the current event, so this must remain member storage. std::string result_; }; diff --git a/collector/lib/system-inspector/Service.cpp b/collector/lib/system-inspector/Service.cpp index a3a9e45286..c1090ac53c 100644 --- a/collector/lib/system-inspector/Service.cpp +++ b/collector/lib/system-inspector/Service.cpp @@ -37,7 +37,7 @@ Service::~Service() = default; Service::Service(const CollectorConfig& config) : inspector_(std::make_unique(true)), - container_id_cache_(std::make_unique()), + container_id_cache_(std::make_unique(config.GetContainerIDCachePruneInterval())), default_formatter_(std::make_unique( inspector_.get(), DEFAULT_OUTPUT_STR, @@ -160,7 +160,7 @@ bool Service::FilterEvent(sinsp_evt* event) { // Exclude host processes that pass the sinsp filter. Requiring a cached // container ID also rejects runtime helpers whose cgroups do not resolve to // a container ID, without maintaining a list of helper names. - return tinfo != nullptr && !container_id_cache_->Get(*tinfo).empty() && FilterEvent(tinfo); + return tinfo != nullptr && container_id_cache_->Exists(*tinfo) && FilterEvent(tinfo); } bool Service::FilterEvent(const sinsp_threadinfo* tinfo) { @@ -275,14 +275,19 @@ bool Service::SendExistingProcesses(SignalHandler* handler) { } return threads->loop([&](sinsp_threadinfo& tinfo) { - if (tinfo.is_main_thread() && !container_id_cache_->Get(tinfo).empty()) { - auto result = handler->HandleExistingProcess(&tinfo); - if (result == SignalHandler::ERROR || result == SignalHandler::NEEDS_REFRESH) { - CLOG(WARNING) << "Failed to write existing process signal: " << &tinfo; - return false; - } - CLOG(DEBUG) << "Found existing process: " << &tinfo; + if (!tinfo.is_main_thread()) { + return true; + } + const auto container_id = container_id_cache_->Get(tinfo); + if (container_id.empty()) { + return true; + } + auto result = handler->HandleExistingProcess(&tinfo); + if (result == SignalHandler::ERROR || result == SignalHandler::NEEDS_REFRESH) { + CLOG(WARNING) << "Failed to write existing process signal: Container: " << container_id << ", " << &tinfo; + return false; } + CLOG(DEBUG) << "Found existing process: Container: " << container_id << ", " << &tinfo; return true; }); } diff --git a/collector/test/SystemInspectorServiceTest.cpp b/collector/test/SystemInspectorServiceTest.cpp index e0bc0de4f1..8bf2868b73 100644 --- a/collector/test/SystemInspectorServiceTest.cpp +++ b/collector/test/SystemInspectorServiceTest.cpp @@ -112,6 +112,12 @@ TEST(SystemInspectorServiceTest, FilterEvent) { EXPECT_TRUE(service.FilterEvent(&event)); event.set_tinfo(runc_cloned_process.get()); EXPECT_FALSE(service.FilterEvent(&event)); + // Neither carries a container ID, so the service guard rejects them before + // the exepath rule is consulted. + event.set_tinfo(proc_self_process.get()); + EXPECT_FALSE(service.FilterEvent(&event)); + event.set_tinfo(memfd_process.get()); + EXPECT_FALSE(service.FilterEvent(&event)); service.CleanUp(); EXPECT_EQ(container_id_cache.Get(*uncached_container_process), "aaaaaaaaaaaa"); @@ -169,9 +175,31 @@ TEST(SystemInspectorServiceTest, ContainerIDCacheLateCgroups) { EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); } +TEST(SystemInspectorServiceTest, ContainerIDCacheRefreshesAfterExec) { + std::unique_ptr inspector(new sinsp()); + ContainerIDCache container_id_cache; + auto tinfo = inspector->get_threadinfo_factory().create(); + tinfo->m_tid = 42; + tinfo->m_clone_ts = 100; + tinfo->m_lastexec_ts = 200; + tinfo->set_cgroups({"cpu=/docker/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}); + + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + + // Exec parsing can refresh a thread's cgroups without changing its TID or + // clone timestamp. The cache must not keep returning the previous ID. + tinfo->set_cgroups({"cpu=/docker/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}); + ++tinfo->m_lastexec_ts; + + EXPECT_EQ(container_id_cache.Get(*tinfo), "bbbbbbbbbbbb"); + EXPECT_EQ(container_id_cache.Get(*tinfo), "bbbbbbbbbbbb"); +} + TEST(SystemInspectorServiceTest, ContainerIDCachePrune) { std::unique_ptr inspector(new sinsp()); ContainerIDCache container_id_cache; + EXPECT_EQ(container_id_cache.Size(), 0); COUNTER_ZERO(CollectorStats::container_id_cache_hits); COUNTER_ZERO(CollectorStats::container_id_cache_misses); COUNTER_ZERO(CollectorStats::container_id_cache_empty_extractions); @@ -185,12 +213,16 @@ TEST(SystemInspectorServiceTest, ContainerIDCachePrune) { // The thread is not in the thread table: the first lookup extracts and // caches the container ID. EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + EXPECT_EQ(container_id_cache.Size(), 1); EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); - // The first prune (last_prune_us_ starts at 0, so 61s clears the 60s rate - // limit) evicts the entry, because the tid is not in the thread table. + // The first prune (last_prune_us_ starts at 0, so 61s clears the default + // 60s rate limit) evicts the entry, because the tid is not in the thread + // table. container_id_cache.Prune(*inspector, 61'000'000); + EXPECT_EQ(container_id_cache.Size(), 0); EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + EXPECT_EQ(container_id_cache.Size(), 1); EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); // Only 30s since the last prune: rate-limited no-op, the entry survives. @@ -213,4 +245,25 @@ TEST(SystemInspectorServiceTest, ContainerIDCachePrune) { COUNTER_ZERO(CollectorStats::container_id_cache_empty_extractions); } +TEST(SystemInspectorServiceTest, ContainerIDCachePruneInterval) { + std::unique_ptr inspector(new sinsp()); + ContainerIDCache container_id_cache(5); + auto tinfo = inspector->get_threadinfo_factory().create(); + tinfo->m_tid = 42; + tinfo->m_clone_ts = 100; + tinfo->set_cgroups({"cpu=/docker/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}); + + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + EXPECT_EQ(container_id_cache.Size(), 1); + + // Only 4s since the last prune: rate-limited no-op, the entry survives. + container_id_cache.Prune(*inspector, 4'000'000); + EXPECT_EQ(container_id_cache.Size(), 1); + + // 6s clears the configured 5s rate limit and evicts the entry, because the + // tid is not in the thread table. + container_id_cache.Prune(*inspector, 6'000'000); + EXPECT_EQ(container_id_cache.Size(), 0); +} + } // namespace collector::system_inspector diff --git a/collector/test/UtilityTest.cpp b/collector/test/UtilityTest.cpp index 14df61f69c..3340ea92ee 100644 --- a/collector/test/UtilityTest.cpp +++ b/collector/test/UtilityTest.cpp @@ -103,6 +103,11 @@ TEST(ExtractContainerIDFromCgroupTest, TestExtractContainerIDFromCgroup) { "/kubepods-burstable-podbd12dd3393227d950605a2444b13c27a.slice:cri-containerd:d52db56a9c80d536a91354c0951c061187ca46249e64865a12703003d8f42366", "d52db56a9c80", }, + // minimal path: separator plus ID, no parent + { + "/0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "0123456789ab", + }, // conmon { "/machine.slice/libpod-conmon-b6ce30d02945df4bbf8e8b7193b2c56ebb3cd10227dd7e59d7f7cdc2cfa2a307.scope", @@ -119,6 +124,9 @@ TEST(ExtractContainerIDFromCgroupTest, TestExtractContainerIDFromCgroup) { auto short_container_id = ExtractContainerIDFromCgroup(c.input); EXPECT_EQ(short_container_id, c.expected_output); } + + const std::string invalid_id_cgroup = "/docker/" + std::string(63, 'a') + static_cast(0xff); + EXPECT_FALSE(ExtractContainerIDFromCgroup(invalid_id_cgroup).has_value()); } TEST(SanitizeUTF8Test, TestSanitizeUTF8_Invalid) { diff --git a/docs/references.md b/docs/references.md index daa5fc4787..eecfc4cb14 100644 --- a/docs/references.md +++ b/docs/references.md @@ -79,6 +79,11 @@ workloads it's the main part of memory consumption, this value effectively translates into the upper limit for memory usage. Note, that Falco puts it's own upper limit on top of that, which is 2^17. +* `ROX_COLLECTOR_CONTAINER_ID_CACHE_PRUNE_INTERVAL`: How often, in seconds, +Collector prunes its container ID cache of entries belonging to threads that +are no longer present in the sinsp thread table. The default is 60 seconds, +matching the thread table's own purging interval. + * `ROX_COLLECTOR_ENABLE_DETAILED_METRICS`: Specified whether to expose per-syscall metrics. This information could be useful for troubleshooting, but under normal functioning is quite verbose. The default is true.