From b5ca4942847e655549ad2af165254029648ad86c Mon Sep 17 00:00:00 2001 From: Richard Zowalla Date: Mon, 16 Nov 2026 18:56:52 +0100 Subject: [PATCH] #2343 CollectionMetric: report the measurements as an ArrayList The measurements were reported as a LinkedList, which Storm does not register with Kryo, so a V1 metrics consumer in another worker could not receive them and the spout executor died. --- .../stormcrawler/util/CollectionMetric.java | 11 +++- .../util/CollectionMetricTest.java | 65 +++++++++++++++++++ 2 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 core/src/test/java/org/apache/stormcrawler/util/CollectionMetricTest.java diff --git a/core/src/main/java/org/apache/stormcrawler/util/CollectionMetric.java b/core/src/main/java/org/apache/stormcrawler/util/CollectionMetric.java index 6e24e90f8..ac714b47b 100644 --- a/core/src/main/java/org/apache/stormcrawler/util/CollectionMetric.java +++ b/core/src/main/java/org/apache/stormcrawler/util/CollectionMetric.java @@ -17,13 +17,13 @@ package org.apache.stormcrawler.util; -import java.util.LinkedList; +import java.util.ArrayList; import java.util.List; import org.apache.storm.metric.api.IMetric; public class CollectionMetric implements IMetric { - private List measurements = new LinkedList<>(); + private List measurements = new ArrayList<>(); public void addMeasurement(long l) { synchronized (measurements) { @@ -31,10 +31,15 @@ public void addMeasurement(long l) { } } + /** + * Returns the measurements taken since the last call as an {@link ArrayList}: a V1 metrics + * consumer in another worker gets them through Kryo, which Storm sets up for {@link ArrayList} + * but not for other lists. + */ @Override public Object getValueAndReset() { synchronized (measurements) { - LinkedList copy = new LinkedList<>(measurements); + ArrayList copy = new ArrayList<>(measurements); measurements.clear(); return copy; } diff --git a/core/src/test/java/org/apache/stormcrawler/util/CollectionMetricTest.java b/core/src/test/java/org/apache/stormcrawler/util/CollectionMetricTest.java new file mode 100644 index 000000000..c39e929c2 --- /dev/null +++ b/core/src/test/java/org/apache/stormcrawler/util/CollectionMetricTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.util; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.io.Output; +import java.util.LinkedList; +import java.util.List; +import org.apache.storm.serialization.SerializationFactory; +import org.apache.storm.utils.Utils; +import org.junit.jupiter.api.Test; + +class CollectionMetricTest { + + /** Kryo as Storm sets it up for a topology that registers no classes of its own. */ + private static Kryo stormKryo() { + return SerializationFactory.getKryo(Utils.readDefaultConfig()); + } + + @Test + void returnsTheMeasurementsSinceTheLastCall() { + final CollectionMetric metric = new CollectionMetric(); + metric.addMeasurement(3); + metric.addMeasurement(5); + assertEquals(List.of(3L, 5L), metric.getValueAndReset()); + assertEquals(List.of(), metric.getValueAndReset()); + } + + @Test + void measurementsCanBeSentToAMetricsConsumerInAnotherWorker() { + final CollectionMetric metric = new CollectionMetric(); + metric.addMeasurement(3); + final Kryo kryo = stormKryo(); + assertDoesNotThrow( + () -> kryo.writeClassAndObject(new Output(1024), metric.getValueAndReset())); + } + + @Test + void linkedListCannotBeSent() { + // why getValueAndReset() copies the measurements into an ArrayList + final Kryo kryo = stormKryo(); + assertThrows( + IllegalArgumentException.class, + () -> kryo.writeClassAndObject(new Output(1024), new LinkedList())); + } +}