Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/collector.yml
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ jobs:
rhacs_eng_image: ${{ env.RHACS_ENG_IMAGE }}
collector_image: ${{ inputs.collector-image }}
collector_tag: ${{ inputs.collector-tag }}
debug_mode: ${{ github.event_name == 'pull_request' }}
debug_mode: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ci-release-build') }}
driver_version: ${DRIVER_VERSION}
EOF

Expand Down
4 changes: 4 additions & 0 deletions collector/lib/CollectorConfig.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,9 @@ PathEnvVar tls_client_cert_path("ROX_COLLECTOR_TLS_CLIENT_CERT");
PathEnvVar tls_client_key_path("ROX_COLLECTOR_TLS_CLIENT_KEY");

BoolEnvVar disable_process_arguments("ROX_COLLECTOR_NO_PROCESS_ARGUMENTS", false);

// Interval between container ID cache prunes, in seconds.
IntEnvVar container_id_cache_prune_interval("ROX_COLLECTOR_CONTAINER_ID_CACHE_PRUNE_INTERVAL", 60);
} // namespace

constexpr bool CollectorConfig::kTurnOffScrape;
Expand Down Expand Up @@ -112,6 +115,7 @@ void CollectorConfig::InitCollectorConfig(CollectorArgs* args) {
enable_introspection_ = enable_introspection.value();
track_send_recv_ = track_send_recv.value();
disable_process_arguments_ = disable_process_arguments.value();
container_id_cache_prune_interval_ = container_id_cache_prune_interval.value();

for (const auto& syscall : kSyscalls) {
syscalls_.emplace_back(syscall);
Expand Down
6 changes: 6 additions & 0 deletions collector/lib/CollectorConfig.h
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@ class CollectorConfig {
unsigned int GetSinspBufferSize() const;
unsigned int GetSinspTotalBufferSize() const { return sinsp_total_buffer_size_; }
unsigned int GetSinspThreadCacheSize() const { return sinsp_thread_cache_size_; }
unsigned int GetContainerIDCachePruneInterval() const { return container_id_cache_prune_interval_; }
bool DisableProcessArguments() const { return disable_process_arguments_; }

static std::pair<option::ArgStatus, std::string> CheckConfiguration(const char* config, Json::Value* root);
Expand Down Expand Up @@ -241,6 +242,11 @@ class CollectorConfig {
// is 2^17 (131072) and twice as large.
unsigned int sinsp_thread_cache_size_ = 32768;

// Interval between container ID cache prunes, in seconds. Pruning syncs
// the cache with the sinsp thread table, whose own purging interval this
// deliberately mirrors.
unsigned int container_id_cache_prune_interval_ = 60;

std::optional<TlsConfig> tls_config_;

std::optional<sensor::CollectorConfig> runtime_config_;
Expand Down
2 changes: 1 addition & 1 deletion collector/lib/CollectorService.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ CollectorService::CollectorService(CollectorConfig& config, std::atomic<ControlV
&system_inspector_,
exporter_.GetRegistry().get());

auto network_signal_handler = std::make_unique<NetworkSignalHandler>(system_inspector_.GetInspector(), conn_tracker_, system_inspector_.GetUserspaceStats());
auto network_signal_handler = std::make_unique<NetworkSignalHandler>(system_inspector_.GetInspector(), conn_tracker_, system_inspector_.GetUserspaceStats(), system_inspector_.GetContainerIDCache());
network_signal_handler->SetCollectConnectionStatus(config_.CollectConnectionStatus());
network_signal_handler->SetTrackSendRecv(config_.TrackingSendRecv());
system_inspector_.AddSignalHandler(std::move(network_signal_handler));
Expand Down
3 changes: 3 additions & 0 deletions collector/lib/CollectorStats.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@
X(process_lineage_string_total) \
X(process_info_hit) \
X(process_info_miss) \
X(container_id_cache_hits) \
X(container_id_cache_misses) \
X(container_id_cache_empty_extractions) \
X(rate_limit_flushing_counts) \
X(procfs_could_not_open_fd_dir) \
X(procfs_could_not_open_proc_dir) \
Expand Down
3 changes: 3 additions & 0 deletions collector/lib/CollectorStatsExporter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include "Logging.h"
#include "Utility.h"
#include "prometheus/gauge.h"
#include "system-inspector/ContainerIDCache.h"
#include "system-inspector/Service.h"

namespace collector {
Expand Down Expand Up @@ -58,6 +59,7 @@ void CollectorStatsExporter::run() {
auto& preemptions = collectorEventCounters.Add({{"type", "preemptions"}});
auto& grpcSendFailures = collectorEventCounters.Add({{"type", "grpcSendFailures"}});
auto& threadTableSize = collectorEventCounters.Add({{"type", "threadCacheSize"}});
auto& containerIDCacheSize = collectorEventCounters.Add({{"type", "containerIDCacheSize"}});

auto& processSent = collectorEventCounters.Add({{"type", "processSent"}});
auto& processSendFailures = collectorEventCounters.Add({{"type", "processSendFailures"}});
Expand Down Expand Up @@ -170,6 +172,7 @@ void CollectorStatsExporter::run() {
ringbufferDrops.Set(stats.nDropsBuffer);
preemptions.Set(stats.nPreemptions);
threadTableSize.Set(stats.nThreadCacheSize);
containerIDCacheSize.Set(system_inspector_->GetContainerIDCache()->Size());

if (config_->EnableDetailedMetrics()) {
uint64_t nUserspace = 0;
Expand Down
7 changes: 4 additions & 3 deletions collector/lib/NetworkSignalHandler.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

#include "EventMap.h"
#include "Utility.h"
#include "system-inspector/ContainerIDCache.h"
#include "system-inspector/EventExtractor.h"

namespace collector {
Expand Down Expand Up @@ -43,8 +44,8 @@ EventMap<Modifier> modifiers = {

} // namespace

NetworkSignalHandler::NetworkSignalHandler(sinsp* inspector, std::shared_ptr<ConnectionTracker> conn_tracker, system_inspector::Stats* stats)
: event_extractor_(std::make_unique<system_inspector::EventExtractor>()), conn_tracker_(std::move(conn_tracker)), stats_(stats), collect_connection_status_(true), track_send_recv_(false) {
NetworkSignalHandler::NetworkSignalHandler(sinsp* inspector, std::shared_ptr<ConnectionTracker> conn_tracker, system_inspector::Stats* stats, system_inspector::ContainerIDCache* container_id_cache)
: event_extractor_(std::make_unique<system_inspector::EventExtractor>()), conn_tracker_(std::move(conn_tracker)), stats_(stats), container_id_cache_(container_id_cache), collect_connection_status_(true), track_send_recv_(false) {
event_extractor_->Init(inspector);
}

Expand Down Expand Up @@ -152,7 +153,7 @@ std::optional<Connection> NetworkSignalHandler::GetConnection(sinsp_evt* evt) {
const Endpoint* local = is_server ? &server : &client;
const Endpoint* remote = is_server ? &client : &server;

auto container_id = GetContainerID(evt);
auto container_id = container_id_cache_->Get(*evt);
if (container_id.empty()) {
return std::nullopt;
}
Expand Down
6 changes: 4 additions & 2 deletions collector/lib/NetworkSignalHandler.h
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,13 @@ class sinsp_evt;

namespace collector {
namespace system_inspector {
class ContainerIDCache;
class EventExtractor;
}
} // namespace system_inspector

class NetworkSignalHandler final : public SignalHandler {
public:
explicit NetworkSignalHandler(sinsp* inspector, std::shared_ptr<ConnectionTracker> conn_tracker, system_inspector::Stats* stats);
explicit NetworkSignalHandler(sinsp* inspector, std::shared_ptr<ConnectionTracker> conn_tracker, system_inspector::Stats* stats, system_inspector::ContainerIDCache* container_id_cache);
~NetworkSignalHandler() override;

std::string GetName() override { return "NetworkSignalHandler"; }
Expand All @@ -35,6 +36,7 @@ class NetworkSignalHandler final : public SignalHandler {
std::unique_ptr<system_inspector::EventExtractor> event_extractor_;
std::shared_ptr<ConnectionTracker> conn_tracker_;
system_inspector::Stats* stats_;
system_inspector::ContainerIDCache* container_id_cache_;

bool collect_connection_status_;
bool track_send_recv_;
Expand Down
48 changes: 26 additions & 22 deletions collector/lib/Process.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -30,63 +30,63 @@ const std::shared_ptr<IProcess> ProcessStore::Fetch(uint64_t pid) {
}

std::string Process::container_id() const {
WaitForProcessInfo();

if (system_inspector_threadinfo_) {
auto id = GetContainerID(*system_inspector_threadinfo_);
if (!id.empty()) {
return id;
const auto process_info = WaitForProcessInfo();
if (process_info) {
for (const auto& cgroup : process_info->cgroups()) {
if (const auto id = ExtractContainerIDFromCgroup(cgroup.second)) {
return std::string(*id);
}
}
}

return NOT_AVAILABLE;
}

std::string Process::comm() const {
WaitForProcessInfo();
const auto process_info = WaitForProcessInfo();

if (system_inspector_threadinfo_) {
return system_inspector_threadinfo_->get_comm();
if (process_info) {
return process_info->get_comm();
}

return NOT_AVAILABLE;
}

std::string Process::exe() const {
WaitForProcessInfo();
const auto process_info = WaitForProcessInfo();

if (system_inspector_threadinfo_) {
return system_inspector_threadinfo_->get_exe();
if (process_info) {
return process_info->get_exe();
}

return NOT_AVAILABLE;
}

std::string Process::exe_path() const {
WaitForProcessInfo();
const auto process_info = WaitForProcessInfo();

if (system_inspector_threadinfo_) {
return system_inspector_threadinfo_->get_exepath();
if (process_info) {
return process_info->get_exepath();
}

return NOT_AVAILABLE;
}

std::string Process::args() const {
WaitForProcessInfo();
const auto process_info = WaitForProcessInfo();

if (!system_inspector_threadinfo_) {
if (!process_info) {
return NOT_AVAILABLE;
}

if (system_inspector_threadinfo_->m_args.empty()) {
if (process_info->m_args.empty()) {
return "";
}

std::ostringstream args;
for (auto it = system_inspector_threadinfo_->m_args.begin(); it != system_inspector_threadinfo_->m_args.end();) {
for (auto it = process_info->m_args.begin(); it != process_info->m_args.end();) {
args << *it++;
if (it != system_inspector_threadinfo_->m_args.end()) {
if (it != process_info->m_args.end()) {
args << " ";
}
}
Expand Down Expand Up @@ -116,7 +116,7 @@ Process::~Process() {
}

void Process::ProcessInfoResolved(std::shared_ptr<sinsp_threadinfo> process_info) {
std::unique_lock<std::mutex> lock(process_info_mutex_);
std::lock_guard<std::mutex> lock(process_info_mutex_);

if (process_info) {
CLOG(DEBUG) << "Process-info resolved. PID: " << pid() << " Exe: " + process_info->m_exe;
Expand All @@ -130,7 +130,7 @@ void Process::ProcessInfoResolved(std::shared_ptr<sinsp_threadinfo> process_info
process_info_condition_.notify_all();
}

void Process::WaitForProcessInfo() const {
std::shared_ptr<sinsp_threadinfo> Process::WaitForProcessInfo() const {
std::unique_lock<std::mutex> lock(process_info_mutex_);

COUNTER_ADD(
Expand All @@ -146,6 +146,10 @@ void Process::WaitForProcessInfo() const {

CLOG_IF(std::cv_status::timeout == status, ERROR) << "Timed-out waiting for process-info. PID: " << pid();
}

// Copy the shared pointer while holding the mutex; callers can then safely
// use the snapshot after this function releases the lock.
return system_inspector_threadinfo_;
Comment on lines +150 to +152

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm a bit confused by this comment, system_inspector_threadinfo_ is a shared_ptr owned by the Process object, copying it here just means the internal atomic counter will be incremented to 2 instead of 1, the underlying type is still a threadinfo that can be muted, it just can't be fully removed. If we want to make the underlying threadinfo fully immutable (like a snapshot), then we need to wrap that in a mutex somehow, but that will be a lot of work on sinsp.h.

What am I missing?

}

std::ostream& operator<<(std::ostream& os, const IProcess& process) {
Expand Down
7 changes: 4 additions & 3 deletions collector/lib/Process.h
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ class Service;

namespace collector::system_inspector {
class Service;
}
} // namespace collector::system_inspector

namespace collector {
/* A Process object store used to deduplicate process information.
Expand Down Expand Up @@ -91,8 +91,9 @@ class Process : public IProcess {
// entry-point when system inspector resolved the requested process info
void ProcessInfoResolved(std::shared_ptr<sinsp_threadinfo> process_info);

// block until process information is available, or timeout
void WaitForProcessInfo() const;
// Block until process information is available or times out, then return a
// synchronized snapshot of the thread-info pointer.
std::shared_ptr<sinsp_threadinfo> WaitForProcessInfo() const;
};

std::ostream& operator<<(std::ostream& os, const IProcess& process);
Expand Down
29 changes: 20 additions & 9 deletions collector/lib/ProcessSignalFormatter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
#include "EventMap.h"
#include "Logging.h"
#include "Utility.h"
#include "system-inspector/ContainerIDCache.h"
#include "system-inspector/EventExtractor.h"

namespace collector {
Expand Down Expand Up @@ -58,16 +59,24 @@ std::string extract_proc_args(sinsp_threadinfo* tinfo) {

ProcessSignalFormatter::ProcessSignalFormatter(
sinsp* inspector,
const CollectorConfig& config) : event_names_(EventNames::GetInstance()),
inspector_(inspector),
event_extractor_(std::make_unique<system_inspector::EventExtractor>()),
config_(config) {
const CollectorConfig& config,
system_inspector::ContainerIDCache& container_id_cache)
: event_names_(EventNames::GetInstance()),
inspector_(inspector),
event_extractor_(std::make_unique<system_inspector::EventExtractor>()),
container_id_cache_(container_id_cache),
config_(config) {
event_extractor_->Init(inspector);
}

ProcessSignalFormatter::~ProcessSignalFormatter() {}

const SignalStreamMessage* ProcessSignalFormatter::ToProtoMessage(sinsp_evt* event) {
if (event->get_thread_info() == nullptr) {
CLOG_THROTTLED(INFO, std::chrono::seconds(60)) << "Dropping process event without thread information";
return nullptr;
}

if (process_signals[event->get_type()] == ProcessSignalType::UNKNOWN_PROCESS_TYPE) {
return nullptr;
}
Expand Down Expand Up @@ -96,7 +105,9 @@ const SignalStreamMessage* ProcessSignalFormatter::ToProtoMessage(sinsp_evt* eve
const SignalStreamMessage* ProcessSignalFormatter::ToProtoMessage(sinsp_threadinfo* tinfo) {
Reset();
if (!ValidateProcessDetails(tinfo)) {
CLOG(INFO) << "Dropping process event: " << tinfo;
const auto container_id = container_id_cache_.Get(*tinfo);
CLOG(INFO) << "Dropping process event: Container: "
<< (container_id.empty() ? "null" : container_id) << ", " << tinfo;
return nullptr;
}

Expand Down Expand Up @@ -176,7 +187,7 @@ ProcessSignal* ProcessSignalFormatter::CreateProcessSignal(sinsp_evt* event) {
signal->set_allocated_time(timestamp);

// set container_id
auto container_id = GetContainerID(event);
auto container_id = container_id_cache_.Get(*event);
if (!container_id.empty()) {
signal->set_container_id(container_id);
}
Expand Down Expand Up @@ -242,7 +253,7 @@ ProcessSignal* ProcessSignalFormatter::CreateProcessSignal(sinsp_threadinfo* tin
signal->set_allocated_time(timestamp);

// set container_id
signal->set_container_id(GetContainerID(*tinfo));
signal->set_container_id(container_id_cache_.Get(*tinfo));

// set process lineage
std::vector<LineageInfo> lineage;
Expand All @@ -266,7 +277,7 @@ std::string ProcessSignalFormatter::ProcessDetails(sinsp_evt* event) {
std::stringstream ss;
const std::string* path = event_extractor_->get_exepath(event);
const std::string* name = event_extractor_->get_comm(event);
auto container_id = GetContainerID(event);
std::string container_id = container_id_cache_.Get(*event);
const char* args = event_extractor_->get_proc_args(event);
const int64_t* pid = event_extractor_->get_pid(event);

Expand Down Expand Up @@ -348,7 +359,7 @@ void ProcessSignalFormatter::GetProcessLineage(sinsp_threadinfo* tinfo,
// all platforms.
//
if (pt->m_vpid == 0) {
if (GetContainerID(*pt).empty()) {
if (!container_id_cache_.Exists(*pt)) {
return false;
}
} else if (pt->m_pid == pt->m_vpid) {
Expand Down
7 changes: 5 additions & 2 deletions collector/lib/ProcessSignalFormatter.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,16 @@ class sinsp_threadinfo;
namespace collector {
namespace system_inspector {
class EventExtractor;
}
class ContainerIDCache;
} // namespace system_inspector
} // namespace collector

namespace collector {

class ProcessSignalFormatter : public ProtoSignalFormatter<sensor::SignalStreamMessage> {
public:
ProcessSignalFormatter(sinsp* inspector, const CollectorConfig& config);
ProcessSignalFormatter(
sinsp* inspector, const CollectorConfig& config, system_inspector::ContainerIDCache& container_id_cache);
~ProcessSignalFormatter();

using Signal = v1::Signal;
Expand Down Expand Up @@ -56,6 +58,7 @@ class ProcessSignalFormatter : public ProtoSignalFormatter<sensor::SignalStreamM
const EventNames& event_names_;
sinsp* inspector_;
std::unique_ptr<system_inspector::EventExtractor> event_extractor_;
system_inspector::ContainerIDCache& container_id_cache_;

const CollectorConfig& config_;
};
Expand Down
5 changes: 3 additions & 2 deletions collector/lib/ProcessSignalHandler.h
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,10 @@ class ProcessSignalHandler : public SignalHandler {
sinsp* inspector,
ISignalServiceClient* client,
system_inspector::Stats* stats,
const CollectorConfig& config)
const CollectorConfig& config,
system_inspector::ContainerIDCache& container_id_cache)
: client_(client),
formatter_(inspector, config),
formatter_(inspector, config, container_id_cache),
stats_(stats),
config_(config) {}

Expand Down
Loading
Loading