diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml index 562c9e1499..ba3bb6d56d 100644 --- a/.github/workflows/collector.yml +++ b/.github/workflows/collector.yml @@ -74,7 +74,7 @@ jobs: rhacs_eng_image: ${{ env.RHACS_ENG_IMAGE }} collector_image: ${{ inputs.collector-image }} collector_tag: ${{ inputs.collector-tag }} - debug_mode: ${{ github.event_name == 'pull_request' }} + debug_mode: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ci-release-build') }} driver_version: ${DRIVER_VERSION} EOF diff --git a/collector/lib/CollectorConfig.cpp b/collector/lib/CollectorConfig.cpp index 294e25bb4d..a93d63d1d2 100644 --- a/collector/lib/CollectorConfig.cpp +++ b/collector/lib/CollectorConfig.cpp @@ -82,6 +82,9 @@ PathEnvVar tls_client_cert_path("ROX_COLLECTOR_TLS_CLIENT_CERT"); PathEnvVar tls_client_key_path("ROX_COLLECTOR_TLS_CLIENT_KEY"); BoolEnvVar disable_process_arguments("ROX_COLLECTOR_NO_PROCESS_ARGUMENTS", false); + +// Interval between container ID cache prunes, in seconds. +IntEnvVar container_id_cache_prune_interval("ROX_COLLECTOR_CONTAINER_ID_CACHE_PRUNE_INTERVAL", 60); } // namespace constexpr bool CollectorConfig::kTurnOffScrape; @@ -112,6 +115,7 @@ void CollectorConfig::InitCollectorConfig(CollectorArgs* args) { enable_introspection_ = enable_introspection.value(); track_send_recv_ = track_send_recv.value(); disable_process_arguments_ = disable_process_arguments.value(); + container_id_cache_prune_interval_ = container_id_cache_prune_interval.value(); for (const auto& syscall : kSyscalls) { syscalls_.emplace_back(syscall); diff --git a/collector/lib/CollectorConfig.h b/collector/lib/CollectorConfig.h index 2a5244c29e..a29b661ed7 100644 --- a/collector/lib/CollectorConfig.h +++ b/collector/lib/CollectorConfig.h @@ -160,6 +160,7 @@ class CollectorConfig { unsigned int GetSinspBufferSize() const; unsigned int GetSinspTotalBufferSize() const { return sinsp_total_buffer_size_; } unsigned int GetSinspThreadCacheSize() const { return sinsp_thread_cache_size_; } + unsigned int GetContainerIDCachePruneInterval() const { return container_id_cache_prune_interval_; } bool DisableProcessArguments() const { return disable_process_arguments_; } static std::pair CheckConfiguration(const char* config, Json::Value* root); @@ -241,6 +242,11 @@ class CollectorConfig { // is 2^17 (131072) and twice as large. unsigned int sinsp_thread_cache_size_ = 32768; + // Interval between container ID cache prunes, in seconds. Pruning syncs + // the cache with the sinsp thread table, whose own purging interval this + // deliberately mirrors. + unsigned int container_id_cache_prune_interval_ = 60; + std::optional tls_config_; std::optional runtime_config_; diff --git a/collector/lib/CollectorService.cpp b/collector/lib/CollectorService.cpp index 1d6243d591..4c7330d879 100644 --- a/collector/lib/CollectorService.cpp +++ b/collector/lib/CollectorService.cpp @@ -53,7 +53,7 @@ CollectorService::CollectorService(CollectorConfig& config, std::atomic(system_inspector_.GetInspector(), conn_tracker_, system_inspector_.GetUserspaceStats()); + auto network_signal_handler = std::make_unique(system_inspector_.GetInspector(), conn_tracker_, system_inspector_.GetUserspaceStats(), system_inspector_.GetContainerIDCache()); network_signal_handler->SetCollectConnectionStatus(config_.CollectConnectionStatus()); network_signal_handler->SetTrackSendRecv(config_.TrackingSendRecv()); system_inspector_.AddSignalHandler(std::move(network_signal_handler)); diff --git a/collector/lib/CollectorStats.h b/collector/lib/CollectorStats.h index 3a660c37b3..9f01296aa3 100644 --- a/collector/lib/CollectorStats.h +++ b/collector/lib/CollectorStats.h @@ -32,6 +32,9 @@ X(process_lineage_string_total) \ X(process_info_hit) \ X(process_info_miss) \ + X(container_id_cache_hits) \ + X(container_id_cache_misses) \ + X(container_id_cache_empty_extractions) \ X(rate_limit_flushing_counts) \ X(procfs_could_not_open_fd_dir) \ X(procfs_could_not_open_proc_dir) \ diff --git a/collector/lib/CollectorStatsExporter.cpp b/collector/lib/CollectorStatsExporter.cpp index ee628c8eb6..4fc3b07060 100644 --- a/collector/lib/CollectorStatsExporter.cpp +++ b/collector/lib/CollectorStatsExporter.cpp @@ -9,6 +9,7 @@ #include "Logging.h" #include "Utility.h" #include "prometheus/gauge.h" +#include "system-inspector/ContainerIDCache.h" #include "system-inspector/Service.h" namespace collector { @@ -58,6 +59,7 @@ void CollectorStatsExporter::run() { auto& preemptions = collectorEventCounters.Add({{"type", "preemptions"}}); auto& grpcSendFailures = collectorEventCounters.Add({{"type", "grpcSendFailures"}}); auto& threadTableSize = collectorEventCounters.Add({{"type", "threadCacheSize"}}); + auto& containerIDCacheSize = collectorEventCounters.Add({{"type", "containerIDCacheSize"}}); auto& processSent = collectorEventCounters.Add({{"type", "processSent"}}); auto& processSendFailures = collectorEventCounters.Add({{"type", "processSendFailures"}}); @@ -170,6 +172,7 @@ void CollectorStatsExporter::run() { ringbufferDrops.Set(stats.nDropsBuffer); preemptions.Set(stats.nPreemptions); threadTableSize.Set(stats.nThreadCacheSize); + containerIDCacheSize.Set(system_inspector_->GetContainerIDCache()->Size()); if (config_->EnableDetailedMetrics()) { uint64_t nUserspace = 0; diff --git a/collector/lib/NetworkSignalHandler.cpp b/collector/lib/NetworkSignalHandler.cpp index c109470a7f..ad510503d3 100644 --- a/collector/lib/NetworkSignalHandler.cpp +++ b/collector/lib/NetworkSignalHandler.cpp @@ -6,6 +6,7 @@ #include "EventMap.h" #include "Utility.h" +#include "system-inspector/ContainerIDCache.h" #include "system-inspector/EventExtractor.h" namespace collector { @@ -43,8 +44,8 @@ EventMap modifiers = { } // namespace -NetworkSignalHandler::NetworkSignalHandler(sinsp* inspector, std::shared_ptr conn_tracker, system_inspector::Stats* stats) - : event_extractor_(std::make_unique()), conn_tracker_(std::move(conn_tracker)), stats_(stats), collect_connection_status_(true), track_send_recv_(false) { +NetworkSignalHandler::NetworkSignalHandler(sinsp* inspector, std::shared_ptr conn_tracker, system_inspector::Stats* stats, system_inspector::ContainerIDCache* container_id_cache) + : event_extractor_(std::make_unique()), conn_tracker_(std::move(conn_tracker)), stats_(stats), container_id_cache_(container_id_cache), collect_connection_status_(true), track_send_recv_(false) { event_extractor_->Init(inspector); } @@ -152,7 +153,7 @@ std::optional NetworkSignalHandler::GetConnection(sinsp_evt* evt) { const Endpoint* local = is_server ? &server : &client; const Endpoint* remote = is_server ? &client : &server; - auto container_id = GetContainerID(evt); + auto container_id = container_id_cache_->Get(*evt); if (container_id.empty()) { return std::nullopt; } diff --git a/collector/lib/NetworkSignalHandler.h b/collector/lib/NetworkSignalHandler.h index 36e10d3d60..fc5790d32d 100644 --- a/collector/lib/NetworkSignalHandler.h +++ b/collector/lib/NetworkSignalHandler.h @@ -13,12 +13,13 @@ class sinsp_evt; namespace collector { namespace system_inspector { +class ContainerIDCache; class EventExtractor; -} +} // namespace system_inspector class NetworkSignalHandler final : public SignalHandler { public: - explicit NetworkSignalHandler(sinsp* inspector, std::shared_ptr conn_tracker, system_inspector::Stats* stats); + explicit NetworkSignalHandler(sinsp* inspector, std::shared_ptr conn_tracker, system_inspector::Stats* stats, system_inspector::ContainerIDCache* container_id_cache); ~NetworkSignalHandler() override; std::string GetName() override { return "NetworkSignalHandler"; } @@ -35,6 +36,7 @@ class NetworkSignalHandler final : public SignalHandler { std::unique_ptr event_extractor_; std::shared_ptr conn_tracker_; system_inspector::Stats* stats_; + system_inspector::ContainerIDCache* container_id_cache_; bool collect_connection_status_; bool track_send_recv_; diff --git a/collector/lib/Process.cpp b/collector/lib/Process.cpp index 4f6a305b51..88e1ae1d3c 100644 --- a/collector/lib/Process.cpp +++ b/collector/lib/Process.cpp @@ -30,12 +30,12 @@ const std::shared_ptr ProcessStore::Fetch(uint64_t pid) { } std::string Process::container_id() const { - WaitForProcessInfo(); - - if (system_inspector_threadinfo_) { - auto id = GetContainerID(*system_inspector_threadinfo_); - if (!id.empty()) { - return id; + const auto process_info = WaitForProcessInfo(); + if (process_info) { + for (const auto& cgroup : process_info->cgroups()) { + if (const auto id = ExtractContainerIDFromCgroup(cgroup.second)) { + return std::string(*id); + } } } @@ -43,50 +43,50 @@ std::string Process::container_id() const { } std::string Process::comm() const { - WaitForProcessInfo(); + const auto process_info = WaitForProcessInfo(); - if (system_inspector_threadinfo_) { - return system_inspector_threadinfo_->get_comm(); + if (process_info) { + return process_info->get_comm(); } return NOT_AVAILABLE; } std::string Process::exe() const { - WaitForProcessInfo(); + const auto process_info = WaitForProcessInfo(); - if (system_inspector_threadinfo_) { - return system_inspector_threadinfo_->get_exe(); + if (process_info) { + return process_info->get_exe(); } return NOT_AVAILABLE; } std::string Process::exe_path() const { - WaitForProcessInfo(); + const auto process_info = WaitForProcessInfo(); - if (system_inspector_threadinfo_) { - return system_inspector_threadinfo_->get_exepath(); + if (process_info) { + return process_info->get_exepath(); } return NOT_AVAILABLE; } std::string Process::args() const { - WaitForProcessInfo(); + const auto process_info = WaitForProcessInfo(); - if (!system_inspector_threadinfo_) { + if (!process_info) { return NOT_AVAILABLE; } - if (system_inspector_threadinfo_->m_args.empty()) { + if (process_info->m_args.empty()) { return ""; } std::ostringstream args; - for (auto it = system_inspector_threadinfo_->m_args.begin(); it != system_inspector_threadinfo_->m_args.end();) { + for (auto it = process_info->m_args.begin(); it != process_info->m_args.end();) { args << *it++; - if (it != system_inspector_threadinfo_->m_args.end()) { + if (it != process_info->m_args.end()) { args << " "; } } @@ -116,7 +116,7 @@ Process::~Process() { } void Process::ProcessInfoResolved(std::shared_ptr process_info) { - std::unique_lock lock(process_info_mutex_); + std::lock_guard lock(process_info_mutex_); if (process_info) { CLOG(DEBUG) << "Process-info resolved. PID: " << pid() << " Exe: " + process_info->m_exe; @@ -130,7 +130,7 @@ void Process::ProcessInfoResolved(std::shared_ptr process_info process_info_condition_.notify_all(); } -void Process::WaitForProcessInfo() const { +std::shared_ptr Process::WaitForProcessInfo() const { std::unique_lock lock(process_info_mutex_); COUNTER_ADD( @@ -146,6 +146,10 @@ void Process::WaitForProcessInfo() const { CLOG_IF(std::cv_status::timeout == status, ERROR) << "Timed-out waiting for process-info. PID: " << pid(); } + + // Copy the shared pointer while holding the mutex; callers can then safely + // use the snapshot after this function releases the lock. + return system_inspector_threadinfo_; } std::ostream& operator<<(std::ostream& os, const IProcess& process) { diff --git a/collector/lib/Process.h b/collector/lib/Process.h index 8fb346984d..a9ec4c488d 100644 --- a/collector/lib/Process.h +++ b/collector/lib/Process.h @@ -18,7 +18,7 @@ class Service; namespace collector::system_inspector { class Service; -} +} // namespace collector::system_inspector namespace collector { /* A Process object store used to deduplicate process information. @@ -91,8 +91,9 @@ class Process : public IProcess { // entry-point when system inspector resolved the requested process info void ProcessInfoResolved(std::shared_ptr process_info); - // block until process information is available, or timeout - void WaitForProcessInfo() const; + // Block until process information is available or times out, then return a + // synchronized snapshot of the thread-info pointer. + std::shared_ptr WaitForProcessInfo() const; }; std::ostream& operator<<(std::ostream& os, const IProcess& process); diff --git a/collector/lib/ProcessSignalFormatter.cpp b/collector/lib/ProcessSignalFormatter.cpp index 17a4fa8e9f..b003095efd 100644 --- a/collector/lib/ProcessSignalFormatter.cpp +++ b/collector/lib/ProcessSignalFormatter.cpp @@ -10,6 +10,7 @@ #include "EventMap.h" #include "Logging.h" #include "Utility.h" +#include "system-inspector/ContainerIDCache.h" #include "system-inspector/EventExtractor.h" namespace collector { @@ -58,16 +59,24 @@ std::string extract_proc_args(sinsp_threadinfo* tinfo) { ProcessSignalFormatter::ProcessSignalFormatter( sinsp* inspector, - const CollectorConfig& config) : event_names_(EventNames::GetInstance()), - inspector_(inspector), - event_extractor_(std::make_unique()), - config_(config) { + const CollectorConfig& config, + system_inspector::ContainerIDCache& container_id_cache) + : event_names_(EventNames::GetInstance()), + inspector_(inspector), + event_extractor_(std::make_unique()), + container_id_cache_(container_id_cache), + config_(config) { event_extractor_->Init(inspector); } ProcessSignalFormatter::~ProcessSignalFormatter() {} const SignalStreamMessage* ProcessSignalFormatter::ToProtoMessage(sinsp_evt* event) { + if (event->get_thread_info() == nullptr) { + CLOG_THROTTLED(INFO, std::chrono::seconds(60)) << "Dropping process event without thread information"; + return nullptr; + } + if (process_signals[event->get_type()] == ProcessSignalType::UNKNOWN_PROCESS_TYPE) { return nullptr; } @@ -96,7 +105,9 @@ const SignalStreamMessage* ProcessSignalFormatter::ToProtoMessage(sinsp_evt* eve const SignalStreamMessage* ProcessSignalFormatter::ToProtoMessage(sinsp_threadinfo* tinfo) { Reset(); if (!ValidateProcessDetails(tinfo)) { - CLOG(INFO) << "Dropping process event: " << tinfo; + const auto container_id = container_id_cache_.Get(*tinfo); + CLOG(INFO) << "Dropping process event: Container: " + << (container_id.empty() ? "null" : container_id) << ", " << tinfo; return nullptr; } @@ -176,7 +187,7 @@ ProcessSignal* ProcessSignalFormatter::CreateProcessSignal(sinsp_evt* event) { signal->set_allocated_time(timestamp); // set container_id - auto container_id = GetContainerID(event); + auto container_id = container_id_cache_.Get(*event); if (!container_id.empty()) { signal->set_container_id(container_id); } @@ -242,7 +253,7 @@ ProcessSignal* ProcessSignalFormatter::CreateProcessSignal(sinsp_threadinfo* tin signal->set_allocated_time(timestamp); // set container_id - signal->set_container_id(GetContainerID(*tinfo)); + signal->set_container_id(container_id_cache_.Get(*tinfo)); // set process lineage std::vector lineage; @@ -266,7 +277,7 @@ std::string ProcessSignalFormatter::ProcessDetails(sinsp_evt* event) { std::stringstream ss; const std::string* path = event_extractor_->get_exepath(event); const std::string* name = event_extractor_->get_comm(event); - auto container_id = GetContainerID(event); + std::string container_id = container_id_cache_.Get(*event); const char* args = event_extractor_->get_proc_args(event); const int64_t* pid = event_extractor_->get_pid(event); @@ -348,7 +359,7 @@ void ProcessSignalFormatter::GetProcessLineage(sinsp_threadinfo* tinfo, // all platforms. // if (pt->m_vpid == 0) { - if (GetContainerID(*pt).empty()) { + if (!container_id_cache_.Exists(*pt)) { return false; } } else if (pt->m_pid == pt->m_vpid) { diff --git a/collector/lib/ProcessSignalFormatter.h b/collector/lib/ProcessSignalFormatter.h index ceeeb98dea..eec4eef89d 100644 --- a/collector/lib/ProcessSignalFormatter.h +++ b/collector/lib/ProcessSignalFormatter.h @@ -19,14 +19,16 @@ class sinsp_threadinfo; namespace collector { namespace system_inspector { class EventExtractor; -} +class ContainerIDCache; +} // namespace system_inspector } // namespace collector namespace collector { class ProcessSignalFormatter : public ProtoSignalFormatter { public: - ProcessSignalFormatter(sinsp* inspector, const CollectorConfig& config); + ProcessSignalFormatter( + sinsp* inspector, const CollectorConfig& config, system_inspector::ContainerIDCache& container_id_cache); ~ProcessSignalFormatter(); using Signal = v1::Signal; @@ -56,6 +58,7 @@ class ProcessSignalFormatter : public ProtoSignalFormatter event_extractor_; + system_inspector::ContainerIDCache& container_id_cache_; const CollectorConfig& config_; }; diff --git a/collector/lib/ProcessSignalHandler.h b/collector/lib/ProcessSignalHandler.h index b6c2797a17..04aa8a4fb9 100644 --- a/collector/lib/ProcessSignalHandler.h +++ b/collector/lib/ProcessSignalHandler.h @@ -23,9 +23,10 @@ class ProcessSignalHandler : public SignalHandler { sinsp* inspector, ISignalServiceClient* client, system_inspector::Stats* stats, - const CollectorConfig& config) + const CollectorConfig& config, + system_inspector::ContainerIDCache& container_id_cache) : client_(client), - formatter_(inspector, config), + formatter_(inspector, config, container_id_cache), stats_(stats), config_(config) {} diff --git a/collector/lib/Utility.cpp b/collector/lib/Utility.cpp index 20e021d99e..207254d854 100644 --- a/collector/lib/Utility.cpp +++ b/collector/lib/Utility.cpp @@ -15,6 +15,8 @@ extern "C" { #include } +#include +#include #include #include @@ -57,29 +59,9 @@ const char* SignalName(int signum) { } } -std::string GetContainerID(const sinsp_threadinfo& tinfo) { - for (const auto& [subsys, cgroup_path] : tinfo.cgroups()) { - if (auto id = ExtractContainerIDFromCgroup(cgroup_path)) { - return std::string(*id); - } - } - return {}; -} - -std::string GetContainerID(sinsp_evt* event) { - if (!event) { - return {}; - } - sinsp_threadinfo* tinfo = event->get_thread_info(); - if (!tinfo) { - return {}; - } - return GetContainerID(*tinfo); -} - std::ostream& operator<<(std::ostream& os, const sinsp_threadinfo* t) { if (t) { - os << "Container: \"" << GetContainerID(*t) << "\", Name: " << t->m_comm << ", PID: " << t->m_pid << ", Args: " << t->m_exe; + os << "Name: " << t->m_comm << ", PID: " << t->m_pid << ", Args: " << t->m_exe; } else { os << "NULL\n"; } @@ -195,50 +177,50 @@ void TryUnlink(const char* path) { } } -const static unsigned int CONTAINER_ID_LENGTH = 64; -const static unsigned int SHORT_CONTAINER_ID_LENGTH = 12; +namespace { -// IsContainerID returns whether the given string view represents a container ID. -bool IsContainerID(std::string_view str) { - if (str.size() != CONTAINER_ID_LENGTH) { +constexpr size_t kContainerIDLength = 64; +constexpr size_t kShortContainerIDLength = 12; + +bool IsContainerID(std::string_view id) { + if (id.size() != kContainerIDLength) { return false; } - return std::all_of(str.begin(), str.end(), [](char c) -> bool { - return std::isxdigit(c) != 0; - }); + return std::all_of(id.begin(), id.end(), [](char c) { return std::isxdigit(static_cast(c)); }); } +} // namespace + std::optional ExtractContainerIDFromCgroup(std::string_view cgroup) { - if (cgroup.size() < CONTAINER_ID_LENGTH + 1) { + // Reject too-short paths before searching for an optional systemd suffix. + if (cgroup.size() < kContainerIDLength + 1) { return {}; } - auto scope = cgroup.rfind(".scope"); + const auto scope = cgroup.rfind(".scope"); if (scope != std::string_view::npos) { - cgroup.remove_suffix(cgroup.length() - scope); - if (cgroup.size() < CONTAINER_ID_LENGTH + 1) { - return {}; - } + cgroup.remove_suffix(cgroup.size() - scope); } - - auto container_id_part = cgroup.substr(cgroup.size() - (CONTAINER_ID_LENGTH + 1)); - if (container_id_part[0] != '/' && container_id_part[0] != '-' && container_id_part[0] != ':') { + if (cgroup.size() < kContainerIDLength + 1) { return {}; } - - cgroup.remove_suffix(CONTAINER_ID_LENGTH + 1); - // conmon runs as its own container, we ignore it. - if (cgroup.find("-conmon", cgroup.size() - StrLen("-conmon")) != std::string_view::npos) { + const auto id_start = cgroup.size() - kContainerIDLength; + const char separator = cgroup[id_start - 1]; + if (separator != '/' && separator != '-' && separator != ':') { return {}; } - - container_id_part.remove_prefix(1); - - if (!IsContainerID(container_id_part)) { + const std::string_view parent = cgroup.substr(0, id_start - 1); + constexpr std::string_view kConmonSuffix = "-conmon"; + if (parent.size() >= kConmonSuffix.size() && + parent.find(kConmonSuffix, parent.size() - kConmonSuffix.size()) != std::string_view::npos) { + return {}; + } + const std::string_view id = cgroup.substr(id_start); + if (!IsContainerID(id)) { return {}; } - return std::make_optional(container_id_part.substr(0, SHORT_CONTAINER_ID_LENGTH)); + return id.substr(0, kShortContainerIDLength); } std::optional SanitizedUTF8(std::string_view str) { diff --git a/collector/lib/Utility.h b/collector/lib/Utility.h index f9ba73126b..04be8cd480 100644 --- a/collector/lib/Utility.h +++ b/collector/lib/Utility.h @@ -14,7 +14,6 @@ // forward declarations class sinsp_threadinfo; -class sinsp_evt; namespace collector { @@ -66,14 +65,6 @@ std::string Str(Args&&... args) { std::ostream& operator<<(std::ostream& os, const sinsp_threadinfo* t); -// Extract container ID from a threadinfo's cgroups. -// Returns an empty string if no container ID found. -std::string GetContainerID(const sinsp_threadinfo& tinfo); - -// Extract container ID from an event's thread info cgroups. -// Returns an empty string if no container ID found. -std::string GetContainerID(sinsp_evt* event); - // UUIDStr returns UUID in string format. const char* UUIDStr(); diff --git a/collector/lib/system-inspector/ContainerIDCache.cpp b/collector/lib/system-inspector/ContainerIDCache.cpp new file mode 100644 index 0000000000..232337c7af --- /dev/null +++ b/collector/lib/system-inspector/ContainerIDCache.cpp @@ -0,0 +1,108 @@ +#include "ContainerIDCache.h" + +#include + +#include +#include +#include + +#include "CollectorStats.h" +#include "Utility.h" + +namespace collector::system_inspector { + +ContainerIDCache::ContainerIDCache(uint64_t prune_interval_s) + : prune_interval_us_(prune_interval_s * 1'000'000) {} + +std::string ContainerIDCache::Cache(const sinsp_threadinfo& tinfo) const { + std::string container_id; + for (const auto& cgroup : tinfo.cgroups()) { + if (const auto id = ExtractContainerIDFromCgroup(cgroup.second)) { + container_id = *id; + break; + } + } + + // Only cache confirmed container IDs. An empty extraction can be transient + // (cgroups are populated when libsinsp (re)creates or refreshes the thread + // info, e.g. from an event that permits a /proc query or on clone/execve + // exit parsing), so empty lookups must recompute. The cache key also tracks + // the last exec timestamp because libsinsp may refresh cgroups on exec + // without changing the thread's clone timestamp. + if (!container_id.empty()) { + std::lock_guard lock(mutex_); + // A reused tid gets a new clone timestamp, so a stale entry can never + // validate as a hit; it misses and recomputes. + entries_[tinfo.m_tid] = {tinfo.m_clone_ts, tinfo.m_lastexec_ts, container_id}; + } else { + // Separates threads without a container ID (expected, e.g. host + // processes) from genuine cache misses, so a hit ratio is computable. + COUNTER_INC(CollectorStats::container_id_cache_empty_extractions); + } + return container_id; +} + +void ContainerIDCache::Initialize(sinsp& inspector) { + inspector.m_thread_manager->get_threads()->loop([this](sinsp_threadinfo& tinfo) { + Cache(tinfo); + return true; + }); +} + +// Prune has a single caller, the event loop under the service's libsinsp +// mutex, so last_prune_us_ needs no atomic. +void ContainerIDCache::Prune(sinsp& inspector, uint64_t now_us) { + if (now_us - last_prune_us_ < prune_interval_us_) { + return; + } + last_prune_us_ = now_us; + + std::unordered_set live_tids; + inspector.m_thread_manager->get_threads()->loop([&live_tids](sinsp_threadinfo& tinfo) { + live_tids.insert(tinfo.m_tid); + return true; + }); + + std::lock_guard lock(mutex_); + for (auto it = entries_.begin(); it != entries_.end();) { + if (live_tids.count(it->first) == 0) { + it = entries_.erase(it); + } else { + ++it; + } + } +} + +std::string ContainerIDCache::Get(const sinsp_threadinfo& tinfo) const { + { + std::lock_guard lock(mutex_); + const auto entry = entries_.find(tinfo.m_tid); + if (entry != entries_.end() && entry->second.clone_ts == tinfo.m_clone_ts && + entry->second.last_exec_ts == tinfo.m_lastexec_ts) { + COUNTER_INC(CollectorStats::container_id_cache_hits); + return entry->second.container_id; + } + COUNTER_INC(CollectorStats::container_id_cache_misses); + } + + return Cache(tinfo); +} + +std::size_t ContainerIDCache::Size() const { + std::lock_guard lock(mutex_); + return entries_.size(); +} + +bool ContainerIDCache::Exists(const sinsp_threadinfo& tinfo) const { + return !Get(tinfo).empty(); +} + +std::string ContainerIDCache::Get(sinsp_evt& evt) const { + auto* tinfo = evt.get_thread_info(); + if (tinfo == nullptr) { + return {}; + } + return Get(*tinfo); +} + +} // namespace collector::system_inspector diff --git a/collector/lib/system-inspector/ContainerIDCache.h b/collector/lib/system-inspector/ContainerIDCache.h new file mode 100644 index 0000000000..ce7a481dcb --- /dev/null +++ b/collector/lib/system-inspector/ContainerIDCache.h @@ -0,0 +1,50 @@ +#pragma once + +#include +#include +#include +#include +#include + +class sinsp; +class sinsp_evt; +class sinsp_threadinfo; + +namespace collector::system_inspector { + +// Caches container IDs extracted from thread cgroups, keyed by thread ID and +// validated by clone and last-exec timestamps. Only non-empty extractions are +// cached: cgroups can be populated after a thread is first seen (e.g. a dropped +// clone event, with the cgroups arriving on the next execve), so empty lookups +// recompute until a container ID is found. +class ContainerIDCache final { + public: + // prune_interval_s: how often, in seconds, the cache is pruned of entries + // whose thread is no longer present in the sinsp thread table. + explicit ContainerIDCache(uint64_t prune_interval_s = 60); + void Initialize(sinsp& inspector); + void Prune(sinsp& inspector, uint64_t now_us); + std::string Get(const sinsp_threadinfo& tinfo) const; + std::size_t Size() const; + // Returns whether a thread has a container ID, populating the cache on a miss. + bool Exists(const sinsp_threadinfo& tinfo) const; + // Returns an empty string if the event carries no thread information. + std::string Get(sinsp_evt& evt) const; + + private: + struct Entry { + uint64_t clone_ts; + // libsinsp refreshes cgroups on exec, which can change the container ID. + uint64_t last_exec_ts; + std::string container_id; + }; + + std::string Cache(const sinsp_threadinfo& tinfo) const; + + mutable std::mutex mutex_; + mutable std::unordered_map entries_; + uint64_t prune_interval_us_; + uint64_t last_prune_us_ = 0; +}; + +} // namespace collector::system_inspector diff --git a/collector/lib/system-inspector/ContainerIDFilterCheck.cpp b/collector/lib/system-inspector/ContainerIDFilterCheck.cpp new file mode 100644 index 0000000000..005ea0724c --- /dev/null +++ b/collector/lib/system-inspector/ContainerIDFilterCheck.cpp @@ -0,0 +1,70 @@ +#include "ContainerIDFilterCheck.h" + +#include +#include +#include + +#include "ContainerIDCache.h" + +namespace collector::system_inspector { + +std::unique_ptr CompileFilter( + sinsp& inspector, const ContainerIDCache* container_id_cache, const std::string& expression) { + sinsp_filter_check_list filter_list; + filter_list.add_filter_check(inspector.new_generic_filtercheck()); + filter_list.add_filter_check(std::make_unique(container_id_cache)); + auto filter_factory = std::make_shared(&inspector, filter_list); + sinsp_filter_compiler filter_compiler(filter_factory, expression); + return filter_compiler.compile(); +} + +namespace { + +constexpr char kHostContainerID[] = "host"; + +const filtercheck_field_info kFields[] = { + {PT_CHARBUF, EPF_NONE, PF_NA, "container.id", "Container", "Cached container ID for the event thread"}, +}; + +} // namespace + +ContainerIDFilterCheck::ContainerIDFilterCheck(const ContainerIDCache* container_id_cache) + : container_id_cache_(container_id_cache) { + static const filter_check_info info = { + "container", + "Container fields", + "Container fields", + sizeof(kFields) / sizeof(kFields[0]), + kFields, + filter_check_info::FL_NONE, + }; + m_info = &info; +} + +uint8_t* ContainerIDFilterCheck::extract_single(sinsp_evt* event, uint32_t* len, bool) { + *len = 0; + if (event == nullptr || m_field_id != 0) { + return nullptr; + } + + sinsp_threadinfo* tinfo = event->get_thread_info(); + if (tinfo == nullptr) { + return nullptr; + } + + result_ = container_id_cache_->Get(*tinfo); + if (!result_.empty()) { + *len = result_.size(); + return reinterpret_cast(result_.data()); + } + + // Match the bundled container filter: an empty ID identifies the host only + // when the process is outside a PID namespace. + if (!tinfo->is_in_pid_namespace()) { + *len = sizeof(kHostContainerID) - 1; + return reinterpret_cast(const_cast(kHostContainerID)); + } + return reinterpret_cast(result_.data()); +} + +} // namespace collector::system_inspector diff --git a/collector/lib/system-inspector/ContainerIDFilterCheck.h b/collector/lib/system-inspector/ContainerIDFilterCheck.h new file mode 100644 index 0000000000..e18bea70a6 --- /dev/null +++ b/collector/lib/system-inspector/ContainerIDFilterCheck.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include + +#include + +class sinsp; +class sinsp_filter; + +namespace collector::system_inspector { + +class ContainerIDCache; + +std::unique_ptr CompileFilter( + sinsp& inspector, const ContainerIDCache* container_id_cache, const std::string& expression); + +class ContainerIDFilterCheck final : public sinsp_filter_check { + public: + explicit ContainerIDFilterCheck(const ContainerIDCache* container_id_cache); + + std::unique_ptr allocate_new() override { + return std::make_unique(container_id_cache_); + } + + protected: + uint8_t* extract_single(sinsp_evt* event, uint32_t* len, bool sanitize_strings) override; + + private: + const ContainerIDCache* container_id_cache_; + // Owns the storage returned by extract_single; libsinsp shallow-caches the + // pointer for the current event, so this must remain member storage. + std::string result_; +}; + +} // namespace collector::system_inspector diff --git a/collector/lib/system-inspector/Service.cpp b/collector/lib/system-inspector/Service.cpp index 9985bdce68..c1090ac53c 100644 --- a/collector/lib/system-inspector/Service.cpp +++ b/collector/lib/system-inspector/Service.cpp @@ -6,7 +6,9 @@ #include +#include "libsinsp/filter.h" #include "libsinsp/parsers.h" +#include "libsinsp/plugin.h" #include "libsinsp/sinsp.h" #include @@ -14,6 +16,8 @@ #include "CollectionMethod.h" #include "CollectorException.h" #include "CollectorStats.h" +#include "ContainerIDCache.h" +#include "ContainerIDFilterCheck.h" #include "EventExtractor.h" #include "EventNames.h" #include "HostInfo.h" @@ -33,6 +37,7 @@ Service::~Service() = default; Service::Service(const CollectorConfig& config) : inspector_(std::make_unique(true)), + container_id_cache_(std::make_unique(config.GetContainerIDCachePruneInterval())), default_formatter_(std::make_unique( inspector_.get(), DEFAULT_OUTPUT_STR, @@ -58,36 +63,6 @@ Service::Service(const CollectorConfig& config) inspector_->get_parser()->set_track_connection_status(true); } - // Filter out host processes to avoid flooding Sensor with events it - // cannot associate with a container. The filter has two clauses: - // - // 1. pid != vpid — In a PID namespace (the common container case), - // the kernel PID differs from the virtual PID visible inside the - // container. The val() transformer makes the parser treat - // proc.vpid as a field reference instead of a literal string. - // - // 2. cgroup regex — Catches containers that share the host PID - // namespace (hostPID: true), where pid == vpid despite the - // process running inside a container. Container runtimes always - // place container processes in a cgroup whose path ends with the - // 64-hex-character container ID, so matching that pattern - // identifies containerised processes regardless of PID namespace - // configuration. This mirrors the cgroup-based container ID - // extraction in ExtractContainerIDFromCgroup(). - // - // The memory cgroup is used because on cgroups v2 with systemd, - // the memory controller is reliably delegated to the container's - // leaf cgroup (where the path contains the container ID), while - // cpuset is often only available at a higher level in the - // hierarchy. The trailing (/.*) accounts for additional path - // components some runtimes append (e.g. podman adds /container). - // - // The 'or' short-circuits: the regex only evaluates for events where - // the PID check fails, so the performance cost is negligible. - inspector_->set_filter( - "proc.pid != val(proc.vpid)" - " or thread.cgroup.memory regex \".*[/:-][0-9a-f]{64}(\\\\.scope)?(/.*)?\""); - // The self-check handlers should only operate during start up, // so they are added to the handler list first, so they have access // to self-check events before the network and process handlers have @@ -103,7 +78,8 @@ Service::Service(const CollectorConfig& config) AddSignalHandler(std::make_unique(inspector_.get(), signal_client_.get(), &userspace_stats_, - config)); + config, + *container_id_cache_)); if (signal_handlers_.size() == 2) { // self-check handlers do not count towards this check, because they @@ -118,6 +94,12 @@ bool Service::InitKernel(const CollectorConfig& config) { CLOG(ERROR) << "Failed to setup " << config.GetCollectionMethod() << " driver."; return false; } + container_id_cache_->Initialize(*inspector_); + + constexpr char kSinspFilterExpression[] = "proc.pid != val(proc.vpid) or container.id != host"; + inspector_->set_filter( + CompileFilter(*inspector_, container_id_cache_.get(), kSinspFilterExpression), + kSinspFilterExpression); return true; } @@ -131,6 +113,7 @@ sinsp_evt* Service::GetNext() { if (res != SCAP_SUCCESS || event == nullptr) { return nullptr; } + container_id_cache_->Prune(*inspector_, NowMicros()); #ifdef TRACE_SINSP_EVENTS // Do not allow to change sinsp events tracing at runtime, as the output @@ -174,7 +157,10 @@ sinsp_evt* Service::GetNext() { bool Service::FilterEvent(sinsp_evt* event) { const auto* tinfo = event->get_thread_info(); - return FilterEvent(tinfo); + // Exclude host processes that pass the sinsp filter. Requiring a cached + // container ID also rejects runtime helpers whose cgroups do not resolve to + // a container ID, without maintaining a list of helper names. + return tinfo != nullptr && container_id_cache_->Exists(*tinfo) && FilterEvent(tinfo); } bool Service::FilterEvent(const sinsp_threadinfo* tinfo) { @@ -182,17 +168,6 @@ bool Service::FilterEvent(const sinsp_threadinfo* tinfo) { return false; } - // Exclude host processes that leak through the sinsp filter. - // The sinsp filter uses a cgroup regex to catch containers in - // the host PID namespace, but this can also match container - // runtime helpers (crun, runc, conmon, podman) that run on the - // host within cgroup paths containing container IDs. Checking - // GetContainerID catches all such cases without maintaining a - // list of runtime helper names. - if (GetContainerID(*tinfo).empty()) { - return false; - } - std::string_view exepath_sv{tinfo->m_exepath}; auto marker = exepath_sv.rfind(':'); if (marker != std::string_view::npos) { @@ -300,14 +275,19 @@ bool Service::SendExistingProcesses(SignalHandler* handler) { } return threads->loop([&](sinsp_threadinfo& tinfo) { - if (!GetContainerID(tinfo).empty() && tinfo.is_main_thread()) { - auto result = handler->HandleExistingProcess(&tinfo); - if (result == SignalHandler::ERROR || result == SignalHandler::NEEDS_REFRESH) { - CLOG(WARNING) << "Failed to write existing process signal: " << &tinfo; - return false; - } - CLOG(DEBUG) << "Found existing process: " << &tinfo; + if (!tinfo.is_main_thread()) { + return true; + } + const auto container_id = container_id_cache_->Get(tinfo); + if (container_id.empty()) { + return true; + } + auto result = handler->HandleExistingProcess(&tinfo); + if (result == SignalHandler::ERROR || result == SignalHandler::NEEDS_REFRESH) { + CLOG(WARNING) << "Failed to write existing process signal: Container: " << container_id << ", " << &tinfo; + return false; } + CLOG(DEBUG) << "Found existing process: Container: " << container_id << ", " << &tinfo; return true; }); } diff --git a/collector/lib/system-inspector/Service.h b/collector/lib/system-inspector/Service.h index 1f2398c648..6ac96e1506 100644 --- a/collector/lib/system-inspector/Service.h +++ b/collector/lib/system-inspector/Service.h @@ -21,6 +21,7 @@ class sinsp_threadinfo; namespace collector::system_inspector { +class ContainerIDCache; class Service : public SystemInspector { public: Service(const Service&) = delete; @@ -43,6 +44,7 @@ class Service : public SystemInspector { void GetProcessInformation(uint64_t pid, ProcessInfoCallbackRef callback); sinsp* GetInspector() { return inspector_.get(); } + ContainerIDCache* GetContainerIDCache() { return container_id_cache_.get(); } Stats* GetUserspaceStats() { return &userspace_stats_; } void AddSignalHandler(std::unique_ptr signal_handler); @@ -61,13 +63,14 @@ class Service : public SystemInspector { }; sinsp_evt* GetNext(); - static bool FilterEvent(sinsp_evt* event); + bool FilterEvent(sinsp_evt* event); static bool FilterEvent(const sinsp_threadinfo* tinfo); bool SendExistingProcesses(SignalHandler* handler); mutable std::mutex libsinsp_mutex_; std::unique_ptr inspector_; + std::unique_ptr container_id_cache_; std::unique_ptr default_formatter_; std::unique_ptr signal_client_; std::vector signal_handlers_; diff --git a/collector/test/ProcessSignalFormatterTest.cpp b/collector/test/ProcessSignalFormatterTest.cpp index 233c3bfed3..c3fbbb6012 100644 --- a/collector/test/ProcessSignalFormatterTest.cpp +++ b/collector/test/ProcessSignalFormatterTest.cpp @@ -7,6 +7,7 @@ #include "ProcessSignalFormatter.h" #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "system-inspector/ContainerIDCache.h" namespace collector { @@ -26,8 +27,9 @@ TEST(ProcessSignalFormatterTest, NoProcessTest) { sinsp* inspector = NULL; CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector, config); + ProcessSignalFormatter processSignalFormatter(inspector, config, container_id_cache); sinsp_threadinfo* tinfo = NULL; std::vector lineage; @@ -51,8 +53,9 @@ TEST(ProcessSignalFormatterTest, ProcessWithoutParentTest) { std::unique_ptr inspector(new sinsp()); CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 0; @@ -86,8 +89,9 @@ TEST(ProcessSignalFormatterTest, ProcessWithParentTest) { std::unique_ptr inspector(new sinsp()); CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 3; @@ -131,8 +135,9 @@ TEST(ProcessSignalFormatterTest, ProcessWithParentWithPid0Test) { std::unique_ptr inspector(new sinsp()); CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 0; @@ -171,8 +176,9 @@ TEST(ProcessSignalFormatterTest, ProcessWithParentWithSameNameTest) { std::unique_ptr inspector(new sinsp()); CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 3; @@ -216,8 +222,9 @@ TEST(ProcessSignalFormatterTest, ProcessWithTwoParentsTest) { std::unique_ptr inspector(new sinsp()); CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 3; @@ -275,8 +282,9 @@ TEST(ProcessSignalFormatterTest, ProcessWithTwoParentsWithTheSameNameTest) { std::unique_ptr inspector(new sinsp()); CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 3; @@ -331,8 +339,9 @@ TEST(ProcessSignalFormatterTest, ProcessCollapseParentChildWithSameNameTest) { std::unique_ptr inspector(new sinsp()); CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 3; @@ -396,8 +405,9 @@ TEST(ProcessSignalFormatterTest, ProcessCollapseParentChildWithSameName2Test) { std::unique_ptr inspector(new sinsp()); CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 3; @@ -464,8 +474,9 @@ TEST(ProcessSignalFormatterTest, ProcessWithUnrelatedProcessTest) { std::unique_ptr inspector(new sinsp()); CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 3; @@ -532,8 +543,9 @@ TEST(ProcessSignalFormatterTest, CountTwoCounterCallsTest) { std::unique_ptr inspector(new sinsp()); CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 1; @@ -583,8 +595,9 @@ TEST(ProcessSignalFormatterTest, Rox3377ProcessLineageWithNoVPidTest) { std::unique_ptr inspector(new sinsp()); CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 3; @@ -638,8 +651,9 @@ TEST(ProcessSignalFormatterTest, Rox3377ProcessLineageWithNoVPidTest) { TEST(ProcessSignalFormatterTest, ProcessArguments) { std::unique_ptr inspector(new sinsp()); MockCollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 3; @@ -663,12 +677,28 @@ TEST(ProcessSignalFormatterTest, ProcessArguments) { EXPECT_FALSE(signal->args().empty()); } +TEST(ProcessSignalFormatterTest, EventWithoutThreadInfo) { + std::unique_ptr inspector(new sinsp()); + CollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); + + sinsp_evt evt(inspector.get()); + scap_evt s_evt{}; + s_evt.type = PPME_SYSCALL_EXECVE_19_X; + s_evt.tid = -1; + evt.set_scap_evt(&s_evt); + + EXPECT_EQ(processSignalFormatter.ToProtoMessage(&evt), nullptr); +} + TEST(ProcessSignalFormatterTest, NoProcessArguments) { std::unique_ptr inspector(new sinsp()); MockCollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; config.SetDisableProcessArguments(true); - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 3; diff --git a/collector/test/SystemInspectorServiceTest.cpp b/collector/test/SystemInspectorServiceTest.cpp index c2745cdbbc..8bf2868b73 100644 --- a/collector/test/SystemInspectorServiceTest.cpp +++ b/collector/test/SystemInspectorServiceTest.cpp @@ -1,40 +1,67 @@ +#include #include +#include "CollectorStats.h" #include "gtest/gtest.h" +#include "system-inspector/ContainerIDCache.h" +#include "system-inspector/ContainerIDFilterCheck.h" #include "system-inspector/Service.h" namespace collector::system_inspector { +TEST(SystemInspectorServiceTest, CleanUpBeforeDestruction) { + CollectorConfig config; + { + Service service(config); + service.CleanUp(); + } +} + TEST(SystemInspectorServiceTest, FilterEvent) { std::unique_ptr inspector(new sinsp()); + ContainerIDCache container_id_cache; + auto filter = CompileFilter(*inspector, &container_id_cache, "container.id != host"); const auto& factory = inspector->get_threadinfo_factory(); - // A container cgroup path with a 64-hex-char container ID. - const std::string container_cgroup = "/kubepods/burstable/pod123/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - sinsp_threadinfo::cgroups_t container_cgroups = {{"memory", container_cgroup}}; - auto regular_process = factory.create(); + regular_process->m_tid = 1; regular_process->m_exepath = "/bin/busybox"; regular_process->m_comm = "sleep"; - regular_process->set_cgroups(container_cgroups); + regular_process->set_cgroups({"cpu=/docker/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}); + ASSERT_EQ(container_id_cache.Get(*regular_process), "aaaaaaaaaaaa"); auto runc_process = factory.create(); + runc_process->m_tid = 2; runc_process->m_exepath = "runc"; runc_process->m_comm = "6"; + auto host_process = factory.create(); + host_process->m_tid = 3; + host_process->m_exepath = "/usr/bin/bash"; + host_process->m_comm = "bash"; + + auto pid_namespace_process = factory.create(); + pid_namespace_process->m_tid = 42; + pid_namespace_process->m_vtid = 1; + + auto uncached_container_process = factory.create(); + uncached_container_process->m_tid = 43; + uncached_container_process->set_cgroups({"cpu=/docker/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}); + auto proc_self_process = factory.create(); - proc_self_process->m_exepath = "/proc/self/exe"; - proc_self_process->m_comm = "6"; - proc_self_process->set_cgroups(container_cgroups); + proc_self_process->m_tid = 4; + proc_self_process->m_exepath = "container:/proc/self/exe"; auto memfd_process = factory.create(); - memfd_process->m_exepath = "memfd:runc_cloned:/proc/self/exe"; - memfd_process->m_comm = "6"; - memfd_process->set_cgroups(container_cgroups); + memfd_process->m_tid = 5; + memfd_process->m_exepath = "/memfd:payload (deleted)"; - auto host_process = factory.create(); - host_process->m_exepath = "/usr/bin/bash"; - host_process->m_comm = "bash"; + auto runc_cloned_process = factory.create(); + runc_cloned_process->m_tid = 6; + runc_cloned_process->m_exepath = "memfd:runc_cloned:/proc/self/exe"; + runc_cloned_process->set_cgroups({"cpu=/docker/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}); + + sinsp_evt event(inspector.get()); struct test_t { const sinsp_threadinfo* tinfo; @@ -43,16 +70,200 @@ TEST(SystemInspectorServiceTest, FilterEvent) { }; std::vector tests{ {regular_process.get(), true, "regular container process"}, - {runc_process.get(), false, "runc (no container ID)"}, - {proc_self_process.get(), false, "/proc/self exe path"}, - {memfd_process.get(), false, "memfd /proc/self exe path"}, - {host_process.get(), false, "host process (no container ID)"}, + {runc_process.get(), true, "runc process"}, + {host_process.get(), true, "host process"}, + {proc_self_process.get(), false, "proc self executable"}, + {memfd_process.get(), true, "memfd executable"}, + {runc_cloned_process.get(), false, "runc-cloned memfd /proc/self exe path"}, }; + // The static FilterEvent only enforces the exepath rule; the container-ID + // rejection lives in the production instance guard asserted further below. for (const auto& t : tests) { ASSERT_EQ(system_inspector::Service::FilterEvent(t.tinfo), t.expected) << "Failed for: " << t.name; } + + event.set_tinfo(regular_process.get()); + EXPECT_TRUE(filter->run(&event)); + event.set_tinfo(runc_process.get()); + EXPECT_FALSE(filter->run(&event)); + event.set_tinfo(host_process.get()); + EXPECT_FALSE(filter->run(&event)); + event.set_tinfo(pid_namespace_process.get()); + EXPECT_TRUE(filter->run(&event)); + + // Service::FilterEvent(sinsp_evt*) is the production guard run by GetNext: + // an event passes only when its thread resolves to a non-empty cached + // container ID (rejecting host processes, runtime helpers, and + // PID-namespace threads without a container ID) and survives the exepath + // rule asserted above. The event's thread info belongs to the test + // inspector; the service consults its own cache, which populates lazily on + // the first lookup. + CollectorConfig config; + Service service(config); + event.set_tinfo(host_process.get()); + EXPECT_FALSE(service.FilterEvent(&event)); + event.set_tinfo(runc_process.get()); + EXPECT_FALSE(service.FilterEvent(&event)); + event.set_tinfo(pid_namespace_process.get()); + EXPECT_FALSE(service.FilterEvent(&event)); + event.set_tinfo(regular_process.get()); + EXPECT_TRUE(service.FilterEvent(&event)); + event.set_tinfo(runc_cloned_process.get()); + EXPECT_FALSE(service.FilterEvent(&event)); + // Neither carries a container ID, so the service guard rejects them before + // the exepath rule is consulted. + event.set_tinfo(proc_self_process.get()); + EXPECT_FALSE(service.FilterEvent(&event)); + event.set_tinfo(memfd_process.get()); + EXPECT_FALSE(service.FilterEvent(&event)); + service.CleanUp(); + + EXPECT_EQ(container_id_cache.Get(*uncached_container_process), "aaaaaaaaaaaa"); +} + +TEST(SystemInspectorServiceTest, ContainerIDCacheLookupStats) { + std::unique_ptr inspector(new sinsp()); + ContainerIDCache container_id_cache; + COUNTER_ZERO(CollectorStats::container_id_cache_hits); + COUNTER_ZERO(CollectorStats::container_id_cache_misses); + COUNTER_ZERO(CollectorStats::container_id_cache_empty_extractions); + auto tinfo = inspector->get_threadinfo_factory().create(); + tinfo->m_tid = 42; + tinfo->m_clone_ts = 100; + tinfo->set_cgroups({"cpu=/docker/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}); + + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + + ++tinfo->m_clone_ts; + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + + // Threads without a container ID are never cached, so their lookups + // recompute (and count as misses) on every event. + auto host_tinfo = inspector->get_threadinfo_factory().create(); + host_tinfo->m_tid = 43; + EXPECT_EQ(container_id_cache.Get(*host_tinfo), ""); + EXPECT_EQ(container_id_cache.Get(*host_tinfo), ""); + + EXPECT_EQ(CollectorStats::GetOrCreate().GetCounter(CollectorStats::container_id_cache_hits), 1); + EXPECT_EQ(CollectorStats::GetOrCreate().GetCounter(CollectorStats::container_id_cache_misses), 4); + EXPECT_EQ(CollectorStats::GetOrCreate().GetCounter(CollectorStats::container_id_cache_empty_extractions), 2); + COUNTER_ZERO(CollectorStats::container_id_cache_hits); + COUNTER_ZERO(CollectorStats::container_id_cache_misses); + COUNTER_ZERO(CollectorStats::container_id_cache_empty_extractions); +} + +TEST(SystemInspectorServiceTest, ContainerIDCacheLateCgroups) { + std::unique_ptr inspector(new sinsp()); + ContainerIDCache container_id_cache; + auto tinfo = inspector->get_threadinfo_factory().create(); + tinfo->m_tid = 42; + tinfo->m_clone_ts = 100; + + // A thread first seen before its cgroups are known (e.g. a dropped clone + // event) has no container ID to cache. + EXPECT_EQ(container_id_cache.Get(*tinfo), ""); + + // When the cgroups arrive (libsinsp populates them when it (re)creates or + // refreshes the thread info, e.g. from an event permitting a /proc query or + // clone/execve exit parsing), the next lookup must pick up the container ID + // instead of pinning the empty result. + tinfo->set_cgroups({"cpu=/docker/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}); + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); +} + +TEST(SystemInspectorServiceTest, ContainerIDCacheRefreshesAfterExec) { + std::unique_ptr inspector(new sinsp()); + ContainerIDCache container_id_cache; + auto tinfo = inspector->get_threadinfo_factory().create(); + tinfo->m_tid = 42; + tinfo->m_clone_ts = 100; + tinfo->m_lastexec_ts = 200; + tinfo->set_cgroups({"cpu=/docker/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}); + + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + + // Exec parsing can refresh a thread's cgroups without changing its TID or + // clone timestamp. The cache must not keep returning the previous ID. + tinfo->set_cgroups({"cpu=/docker/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}); + ++tinfo->m_lastexec_ts; + + EXPECT_EQ(container_id_cache.Get(*tinfo), "bbbbbbbbbbbb"); + EXPECT_EQ(container_id_cache.Get(*tinfo), "bbbbbbbbbbbb"); +} + +TEST(SystemInspectorServiceTest, ContainerIDCachePrune) { + std::unique_ptr inspector(new sinsp()); + ContainerIDCache container_id_cache; + EXPECT_EQ(container_id_cache.Size(), 0); + COUNTER_ZERO(CollectorStats::container_id_cache_hits); + COUNTER_ZERO(CollectorStats::container_id_cache_misses); + COUNTER_ZERO(CollectorStats::container_id_cache_empty_extractions); + auto tinfo = inspector->get_threadinfo_factory().create(); + tinfo->m_pid = 42; + tinfo->m_tid = 42; + tinfo->m_ptid = -1; + tinfo->m_clone_ts = 100; + tinfo->set_cgroups({"cpu=/docker/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}); + + // The thread is not in the thread table: the first lookup extracts and + // caches the container ID. + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + EXPECT_EQ(container_id_cache.Size(), 1); + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + + // The first prune (last_prune_us_ starts at 0, so 61s clears the default + // 60s rate limit) evicts the entry, because the tid is not in the thread + // table. + container_id_cache.Prune(*inspector, 61'000'000); + EXPECT_EQ(container_id_cache.Size(), 0); + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + EXPECT_EQ(container_id_cache.Size(), 1); + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + + // Only 30s since the last prune: rate-limited no-op, the entry survives. + container_id_cache.Prune(*inspector, 91'000'000); + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + + // A thread present in the thread table counts as live. + inspector->m_thread_manager->add_thread(std::move(tinfo), false); + + // 61s since the last real prune: prunes, but the tid is now live, so the + // entry survives. + container_id_cache.Prune(*inspector, 122'000'000); + EXPECT_EQ(container_id_cache.Get(*inspector->m_thread_manager->find_thread(42, true)), "aaaaaaaaaaaa"); + + EXPECT_EQ(CollectorStats::GetOrCreate().GetCounter(CollectorStats::container_id_cache_hits), 4); + EXPECT_EQ(CollectorStats::GetOrCreate().GetCounter(CollectorStats::container_id_cache_misses), 2); + EXPECT_EQ(CollectorStats::GetOrCreate().GetCounter(CollectorStats::container_id_cache_empty_extractions), 0); + COUNTER_ZERO(CollectorStats::container_id_cache_hits); + COUNTER_ZERO(CollectorStats::container_id_cache_misses); + COUNTER_ZERO(CollectorStats::container_id_cache_empty_extractions); +} + +TEST(SystemInspectorServiceTest, ContainerIDCachePruneInterval) { + std::unique_ptr inspector(new sinsp()); + ContainerIDCache container_id_cache(5); + auto tinfo = inspector->get_threadinfo_factory().create(); + tinfo->m_tid = 42; + tinfo->m_clone_ts = 100; + tinfo->set_cgroups({"cpu=/docker/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}); + + EXPECT_EQ(container_id_cache.Get(*tinfo), "aaaaaaaaaaaa"); + EXPECT_EQ(container_id_cache.Size(), 1); + + // Only 4s since the last prune: rate-limited no-op, the entry survives. + container_id_cache.Prune(*inspector, 4'000'000); + EXPECT_EQ(container_id_cache.Size(), 1); + + // 6s clears the configured 5s rate limit and evicts the entry, because the + // tid is not in the thread table. + container_id_cache.Prune(*inspector, 6'000'000); + EXPECT_EQ(container_id_cache.Size(), 0); } } // namespace collector::system_inspector diff --git a/collector/test/UtilityTest.cpp b/collector/test/UtilityTest.cpp index 14df61f69c..3340ea92ee 100644 --- a/collector/test/UtilityTest.cpp +++ b/collector/test/UtilityTest.cpp @@ -103,6 +103,11 @@ TEST(ExtractContainerIDFromCgroupTest, TestExtractContainerIDFromCgroup) { "/kubepods-burstable-podbd12dd3393227d950605a2444b13c27a.slice:cri-containerd:d52db56a9c80d536a91354c0951c061187ca46249e64865a12703003d8f42366", "d52db56a9c80", }, + // minimal path: separator plus ID, no parent + { + "/0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "0123456789ab", + }, // conmon { "/machine.slice/libpod-conmon-b6ce30d02945df4bbf8e8b7193b2c56ebb3cd10227dd7e59d7f7cdc2cfa2a307.scope", @@ -119,6 +124,9 @@ TEST(ExtractContainerIDFromCgroupTest, TestExtractContainerIDFromCgroup) { auto short_container_id = ExtractContainerIDFromCgroup(c.input); EXPECT_EQ(short_container_id, c.expected_output); } + + const std::string invalid_id_cgroup = "/docker/" + std::string(63, 'a') + static_cast(0xff); + EXPECT_FALSE(ExtractContainerIDFromCgroup(invalid_id_cgroup).has_value()); } TEST(SanitizeUTF8Test, TestSanitizeUTF8_Invalid) { diff --git a/docs/references.md b/docs/references.md index daa5fc4787..eecfc4cb14 100644 --- a/docs/references.md +++ b/docs/references.md @@ -79,6 +79,11 @@ workloads it's the main part of memory consumption, this value effectively translates into the upper limit for memory usage. Note, that Falco puts it's own upper limit on top of that, which is 2^17. +* `ROX_COLLECTOR_CONTAINER_ID_CACHE_PRUNE_INTERVAL`: How often, in seconds, +Collector prunes its container ID cache of entries belonging to threads that +are no longer present in the sinsp thread table. The default is 60 seconds, +matching the thread table's own purging interval. + * `ROX_COLLECTOR_ENABLE_DETAILED_METRICS`: Specified whether to expose per-syscall metrics. This information could be useful for troubleshooting, but under normal functioning is quite verbose. The default is true.