From 2edf7e922bae4ed3fc935cf134876a7e8c946ede Mon Sep 17 00:00:00 2001 From: Richard Zowalla Date: Mon, 16 Nov 2026 10:11:11 +0100 Subject: [PATCH] #2336 StatusMetricsBolt: report the status counts as a HashMap The counts are kept in a ConcurrentHashMap, which Storm does not register with Kryo, so a V1 metrics consumer in another worker could not receive them and the status_metrics executor died. The gauge now returns a HashMap copy, as in 3.5.1. --- .../opensearch/metrics/StatusMetricsBolt.java | 12 +++- .../metrics/StatusMetricsBoltTest.java | 56 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBoltTest.java diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java index c74184c22..d3d20eb89 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java @@ -17,6 +17,7 @@ package org.apache.stormcrawler.opensearch.metrics; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; @@ -84,7 +85,7 @@ public void prepare( } CrawlerMetrics.registerGauge( - context, stormConf, "status.count", () -> latestStatusCounts, freqStats); + context, stormConf, "status.count", this::statusCounts, freqStats); counters = new StatusCounter[6]; @@ -96,6 +97,15 @@ public void prepare( counters[5] = new StatusCounter("TOTAL"); } + /** + * The counts reported by the {@code status.count} metric. A V1 metrics consumer in another + * worker gets them through Kryo, which Storm only sets up for {@link HashMap}, not for the + * {@link ConcurrentHashMap} the counts are kept in. + */ + Map statusCounts() { + return new HashMap<>(latestStatusCounts); + } + @Override public Map getComponentConfiguration() { Config conf = new Config(); diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBoltTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBoltTest.java new file mode 100644 index 000000000..40f21ecee --- /dev/null +++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBoltTest.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.opensearch.metrics; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.io.Output; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.storm.serialization.SerializationFactory; +import org.apache.storm.utils.Utils; +import org.junit.jupiter.api.Test; + +class StatusMetricsBoltTest { + + /** Kryo as Storm sets it up for a topology that registers no classes of its own. */ + private static Kryo stormKryo() { + return SerializationFactory.getKryo(Utils.readDefaultConfig()); + } + + @Test + void statusCountsCanBeSentToAMetricsConsumerInAnotherWorker() { + final Kryo kryo = stormKryo(); + assertDoesNotThrow( + () -> + kryo.writeClassAndObject( + new Output(1024), new StatusMetricsBolt().statusCounts())); + } + + @Test + void theMapTheCountsAreKeptInCannotBe() { + // why statusCounts() hands out a copy rather than the map itself + final Kryo kryo = stormKryo(); + assertThrows( + IllegalArgumentException.class, + () -> + kryo.writeClassAndObject( + new Output(1024), new ConcurrentHashMap())); + } +}