From 3b3a7b98c3c485929ae03e8a6c32bf19bf3744d8 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Fri, 6 Feb 2026 15:32:25 -0500 Subject: [PATCH 01/21] SOLR-18099: CollapsingQParserPlugin extensibility --- .../solr/search/CollapsingQParserPlugin.java | 594 +++++++++--------- 1 file changed, 282 insertions(+), 312 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index 9f127f7d4600..53f7d0d92acd 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -293,15 +293,15 @@ public static GroupHeadSelector build(final SolrParams localParams) { public static class CollapsingPostFilter extends ExtendedQueryBase implements PostFilter { - private String collapseField; - private final GroupHeadSelector groupHeadSelector; - private final SortSpec sortSpec; // may be null, parsed at most once from groupHeadSelector - public String hint; - private boolean needsScores = true; - private boolean needsScores4Collapsing = false; - private NullPolicy nullPolicy; + protected final String collapseField; + protected final GroupHeadSelector groupHeadSelector; + protected final SortSpec sortSpec; // may be null, parsed at most once from groupHeadSelector + public final String hint; + protected final boolean needsScores; + protected final boolean needsScores4Collapsing; + protected final NullPolicy nullPolicy; + private final int size; private Set boosted; // ordered by "priority" - private int size; public String getField() { return this.collapseField; @@ -449,9 +449,6 @@ public CollapsingPostFilter( @SuppressWarnings({"unchecked"}) public DelegatingCollector getFilterCollector(IndexSearcher indexSearcher) { try { - - SolrIndexSearcher searcher = (SolrIndexSearcher) indexSearcher; - CollectorFactory collectorFactory = new CollectorFactory(); // Deal with boosted docs. // We have to deal with it here rather then the constructor because // because the QueryElevationComponent runs after the Queries are constructed. @@ -467,23 +464,210 @@ public DelegatingCollector getFilterCollector(IndexSearcher indexSearcher) { this.boosted = (Set) context.get(QueryElevationComponent.BOOSTED); } + SolrIndexSearcher searcher = (SolrIndexSearcher) indexSearcher; boostDocsMap = QueryElevationComponent.getBoostDocs(searcher, this.boosted, context); - return collectorFactory.getCollector( - this.collapseField, - this.groupHeadSelector, - this.sortSpec, - this.nullPolicy.getCode(), - this.hint, - this.needsScores4Collapsing, - this.needsScores, - this.size, - boostDocsMap, - searcher); + return this.getCollector(boostDocsMap, searcher); } catch (IOException e) { throw new RuntimeException(e); } } + + /** + * @see #isNumericCollapsible + */ + private static final EnumSet NUMERIC_COLLAPSIBLE_TYPES = + EnumSet.of(NumberType.INTEGER, NumberType.FLOAT); + + private boolean isNumericCollapsible(FieldType collapseFieldType) { + return NUMERIC_COLLAPSIBLE_TYPES.contains(collapseFieldType.getNumberType()); + } + + protected DelegatingCollector getCollector(IntIntHashMap boostDocs, SolrIndexSearcher searcher) + throws IOException { + + DocValuesProducer docValuesProducer = null; + FunctionQuery funcQuery = null; + + // block collapsing logic is much simpler and uses less memory, but is only viable in specific + // situations + final boolean blockCollapse = + (("_root_".equals(collapseField) || HINT_BLOCK.equals(hint)) + // because we currently handle all min/max cases using + // AbstractBlockSortSpecCollector, we can't handle functions wrapping cscore() + // (for the same reason cscore() isn't supported in 'sort' local param) + && (!CollapseScore.wantsCScore(groupHeadSelector.selectorText)) + // + && NullPolicy.COLLAPSE.getCode() != nullPolicy.getCode()); + if (HINT_BLOCK.equals(hint) && !blockCollapse) { + log.debug( + "Query specifies hint={} but other local params prevent the use block based collapse", + HINT_BLOCK); + } + + FieldType collapseFieldType = searcher.getSchema().getField(collapseField).getType(); + + if (collapseFieldType instanceof StrField) { + // if we are using blockCollapse, then there is no need to bother with TOP_FC + if (HINT_TOP_FC.equals(hint) && !blockCollapse) { + @SuppressWarnings("resource") + final LeafReader uninvertingReader = getTopFieldCacheReader(searcher, collapseField); + + docValuesProducer = + new EmptyDocValuesProducer() { + @Override + public SortedDocValues getSorted(FieldInfo ignored) throws IOException { + SortedDocValues values = uninvertingReader.getSortedDocValues(collapseField); + if (values != null) { + return values; + } else { + return DocValues.emptySorted(); + } + } + }; + } else { + docValuesProducer = + new EmptyDocValuesProducer() { + @Override + public SortedDocValues getSorted(FieldInfo ignored) throws IOException { + return DocValues.getSorted(searcher.getSlowAtomicReader(), collapseField); + } + }; + } + } else { + if (HINT_TOP_FC.equals(hint)) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "top_fc hint is only supported when collapsing on String Fields"); + } + } + + FieldType minMaxFieldType = null; + if (GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type)) { + final String text = groupHeadSelector.selectorText; + if (!text.contains("(")) { + minMaxFieldType = searcher.getSchema().getField(text).getType(); + } else { + SolrParams params = new ModifiableSolrParams(); + try (SolrQueryRequest request = SolrQueryRequest.wrapSearcher(searcher, params)) { + FunctionQParser functionQParser = new FunctionQParser(text, null, params, request); + funcQuery = (FunctionQuery) functionQParser.parse(); + } catch (SyntaxError e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); + } + } + } + + int maxDoc = searcher.maxDoc(); + int leafCount = searcher.getTopReaderContext().leaves().size(); + + SolrRequestInfo req = SolrRequestInfo.getRequestInfo(); + boolean collectElevatedDocsWhenCollapsing = + req != null + && req.getReq().getParams().getBool(COLLECT_ELEVATED_DOCS_WHEN_COLLAPSING, true); + + if (GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type)) { + + if (collapseFieldType instanceof StrField) { + if (blockCollapse) { + return new BlockOrdScoreCollector(collapseField, nullPolicy.getCode(), boostDocs); + } + return new OrdScoreCollector( + maxDoc, + leafCount, + docValuesProducer, + nullPolicy.getCode(), + boostDocs, + searcher, + collectElevatedDocsWhenCollapsing); + + } else if (isNumericCollapsible(collapseFieldType)) { + if (blockCollapse) { + return new BlockIntScoreCollector(collapseField, nullPolicy.getCode(), boostDocs); + } + + return new IntScoreCollector( + maxDoc, + leafCount, + nullPolicy.getCode(), + size, + collapseField, + boostDocs, + searcher, + collectElevatedDocsWhenCollapsing); + + } else { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Collapsing field should be of either String, Int or Float type"); + } + + } else { // min, max, sort, etc.. something other then just "score" + + if (collapseFieldType instanceof StrField) { + if (blockCollapse) { + // NOTE: for now we don't worry about whether this is a sortSpec of min/max + // groupHeadSelector, we use a "sort spec' based block collector unless/until there is + // some (performance?) reason to specialize + return new BlockOrdSortSpecCollector( + collapseField, + nullPolicy.getCode(), + boostDocs, + BlockOrdSortSpecCollector.getSort(groupHeadSelector, sortSpec, funcQuery, searcher), + needsScores || needsScores4Collapsing); + } + + return new OrdFieldValueCollector( + maxDoc, + leafCount, + docValuesProducer, + nullPolicy.getCode(), + groupHeadSelector, + sortSpec, + needsScores4Collapsing, + needsScores, + minMaxFieldType, + boostDocs, + funcQuery, + searcher, + collectElevatedDocsWhenCollapsing); + + } else if (isNumericCollapsible(collapseFieldType)) { + + if (blockCollapse) { + // NOTE: for now we don't worry about whether this is a sortSpec of min/max + // groupHeadSelector, we use a "sort spec' based block collector unless/until there is + // some (performance?) reason to specialize + return new BlockIntSortSpecCollector( + collapseField, + nullPolicy.getCode(), + boostDocs, + BlockOrdSortSpecCollector.getSort(groupHeadSelector, sortSpec, funcQuery, searcher), + needsScores || needsScores4Collapsing); + } + + return new IntFieldValueCollector( + maxDoc, + size, + leafCount, + nullPolicy.getCode(), + collapseField, + groupHeadSelector, + sortSpec, + needsScores4Collapsing, + needsScores, + minMaxFieldType, + boostDocs, + funcQuery, + searcher, + collectElevatedDocsWhenCollapsing); + } else { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Collapsing field should be of either String, Int or Float type"); + } + } + } } /** @@ -1001,26 +1185,26 @@ public void complete() throws IOException { * * @lucene.internal */ - static class OrdFieldValueCollector extends DelegatingCollector { + protected static class OrdFieldValueCollector extends DelegatingCollector { private LeafReaderContext[] contexts; private DocValuesProducer collapseValuesProducer; - private SortedDocValues collapseValues; + protected SortedDocValues collapseValues; protected OrdinalMap ordinalMap; protected SortedDocValues segmentValues; protected LongValues segmentOrdinalMap; protected MultiDocValues.MultiSortedDocValues multiSortedDocValues; - private int maxDoc; - private int nullPolicy; + protected int maxDoc; + protected int nullPolicy; private OrdFieldValueStrategy collapseStrategy; - private boolean needsScores4Collapsing; - private boolean needsScores; + protected boolean needsScores4Collapsing; + protected boolean needsScores; private boolean collectElevatedDocsWhenCollapsing; - private final BoostedDocsCollector boostedDocsCollector; + protected final BoostedDocsCollector boostedDocsCollector; public OrdFieldValueCollector( int maxDoc, @@ -1056,36 +1240,48 @@ public OrdFieldValueCollector( this.boostedDocsCollector = BoostedDocsCollector.build(boostDocsMap); - int valueCount = collapseValues.getValueCount(); this.nullPolicy = nullPolicy; this.needsScores4Collapsing = needsScores4Collapsing; this.needsScores = needsScores; + this.collapseStrategy = + createCollapseStrategy( + maxDoc, groupHeadSelector, sortSpec, fieldType, funcQuery, searcher); + } + + protected OrdFieldValueStrategy createCollapseStrategy( + int maxDoc, + GroupHeadSelector groupHeadSelector, + SortSpec sortSpec, + FieldType fieldType, + FunctionQuery funcQuery, + IndexSearcher searcher) + throws IOException { + int valueCount = collapseValues.getValueCount(); + if (null != sortSpec) { - this.collapseStrategy = - new OrdSortSpecStrategy( - maxDoc, - nullPolicy, - valueCount, - groupHeadSelector, - this.needsScores4Collapsing, - this.needsScores, - boostedDocsCollector, - sortSpec, - searcher, - collapseValues); + return new OrdSortSpecStrategy( + maxDoc, + nullPolicy, + valueCount, + groupHeadSelector, + this.needsScores4Collapsing, + this.needsScores, + boostedDocsCollector, + sortSpec, + searcher, + collapseValues); } else if (funcQuery != null) { - this.collapseStrategy = - new OrdValueSourceStrategy( - maxDoc, - nullPolicy, - valueCount, - groupHeadSelector, - this.needsScores4Collapsing, - this.needsScores, - boostedDocsCollector, - funcQuery, - searcher, - collapseValues); + return new OrdValueSourceStrategy( + maxDoc, + nullPolicy, + valueCount, + groupHeadSelector, + this.needsScores4Collapsing, + this.needsScores, + boostedDocsCollector, + funcQuery, + searcher, + collapseValues); } else { NumberType numType = fieldType.getNumberType(); if (null == numType) { @@ -1093,53 +1289,35 @@ public OrdFieldValueCollector( SolrException.ErrorCode.BAD_REQUEST, "min/max must be either Int/Long/Float based field types"); } - switch (numType) { - case INTEGER: - { - this.collapseStrategy = - new OrdIntStrategy( - maxDoc, - nullPolicy, - valueCount, - groupHeadSelector, - this.needsScores, - boostedDocsCollector, - collapseValues); - break; - } - case FLOAT: - { - this.collapseStrategy = - new OrdFloatStrategy( - maxDoc, - nullPolicy, - valueCount, - groupHeadSelector, - this.needsScores, - boostedDocsCollector, - collapseValues); - break; - } - case LONG: - { - this.collapseStrategy = - new OrdLongStrategy( - maxDoc, - nullPolicy, - valueCount, - groupHeadSelector, - this.needsScores, - boostedDocsCollector, - collapseValues); - break; - } - default: - { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - "min/max must be either Int/Long/Float field types"); - } - } + return switch (numType) { + case INTEGER -> new OrdIntStrategy( + maxDoc, + nullPolicy, + valueCount, + groupHeadSelector, + this.needsScores, + boostedDocsCollector, + collapseValues); + case FLOAT -> new OrdFloatStrategy( + maxDoc, + nullPolicy, + valueCount, + groupHeadSelector, + this.needsScores, + boostedDocsCollector, + collapseValues); + case LONG -> new OrdLongStrategy( + maxDoc, + nullPolicy, + valueCount, + groupHeadSelector, + this.needsScores, + boostedDocsCollector, + collapseValues); + default -> throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "min/max must be either Int/Long/Float field types"); + }; } } @@ -2039,214 +2217,6 @@ public void collect(int contextDoc) throws IOException { } } - private static class CollectorFactory { - /** - * @see #isNumericCollapsible - */ - private static final EnumSet NUMERIC_COLLAPSIBLE_TYPES = - EnumSet.of(NumberType.INTEGER, NumberType.FLOAT); - - private boolean isNumericCollapsible(FieldType collapseFieldType) { - return NUMERIC_COLLAPSIBLE_TYPES.contains(collapseFieldType.getNumberType()); - } - - public DelegatingCollector getCollector( - String collapseField, - GroupHeadSelector groupHeadSelector, - SortSpec sortSpec, - int nullPolicy, - String hint, - boolean needsScores4Collapsing, - boolean needsScores, - int size, - IntIntHashMap boostDocs, - SolrIndexSearcher searcher) - throws IOException { - - DocValuesProducer docValuesProducer = null; - FunctionQuery funcQuery = null; - - // block collapsing logic is much simpler and uses less memory, but is only viable in specific - // situations - final boolean blockCollapse = - (("_root_".equals(collapseField) || HINT_BLOCK.equals(hint)) - // because we currently handle all min/max cases using - // AbstractBlockSortSpecCollector, we can't handle functions wrapping cscore() - // (for the same reason cscore() isn't supported in 'sort' local param) - && (!CollapseScore.wantsCScore(groupHeadSelector.selectorText)) - // - && NullPolicy.COLLAPSE.getCode() != nullPolicy); - if (HINT_BLOCK.equals(hint) && !blockCollapse) { - log.debug( - "Query specifies hint={} but other local params prevent the use block based collapse", - HINT_BLOCK); - } - - FieldType collapseFieldType = searcher.getSchema().getField(collapseField).getType(); - - if (collapseFieldType instanceof StrField) { - // if we are using blockCollapse, then there is no need to bother with TOP_FC - if (HINT_TOP_FC.equals(hint) && !blockCollapse) { - @SuppressWarnings("resource") - final LeafReader uninvertingReader = getTopFieldCacheReader(searcher, collapseField); - - docValuesProducer = - new EmptyDocValuesProducer() { - @Override - public SortedDocValues getSorted(FieldInfo ignored) throws IOException { - SortedDocValues values = uninvertingReader.getSortedDocValues(collapseField); - if (values != null) { - return values; - } else { - return DocValues.emptySorted(); - } - } - }; - } else { - docValuesProducer = - new EmptyDocValuesProducer() { - @Override - public SortedDocValues getSorted(FieldInfo ignored) throws IOException { - return DocValues.getSorted(searcher.getSlowAtomicReader(), collapseField); - } - }; - } - } else { - if (HINT_TOP_FC.equals(hint)) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - "top_fc hint is only supported when collapsing on String Fields"); - } - } - - FieldType minMaxFieldType = null; - if (GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type)) { - final String text = groupHeadSelector.selectorText; - if (!text.contains("(")) { - minMaxFieldType = searcher.getSchema().getField(text).getType(); - } else { - SolrParams params = new ModifiableSolrParams(); - try (SolrQueryRequest request = SolrQueryRequest.wrapSearcher(searcher, params)) { - FunctionQParser functionQParser = new FunctionQParser(text, null, params, request); - funcQuery = (FunctionQuery) functionQParser.parse(); - } catch (SyntaxError e) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); - } - } - } - - int maxDoc = searcher.maxDoc(); - int leafCount = searcher.getTopReaderContext().leaves().size(); - - SolrRequestInfo req = SolrRequestInfo.getRequestInfo(); - boolean collectElevatedDocsWhenCollapsing = - req != null - && req.getReq().getParams().getBool(COLLECT_ELEVATED_DOCS_WHEN_COLLAPSING, true); - - if (GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type)) { - - if (collapseFieldType instanceof StrField) { - if (blockCollapse) { - return new BlockOrdScoreCollector(collapseField, nullPolicy, boostDocs); - } - return new OrdScoreCollector( - maxDoc, - leafCount, - docValuesProducer, - nullPolicy, - boostDocs, - searcher, - collectElevatedDocsWhenCollapsing); - - } else if (isNumericCollapsible(collapseFieldType)) { - if (blockCollapse) { - return new BlockIntScoreCollector(collapseField, nullPolicy, boostDocs); - } - - return new IntScoreCollector( - maxDoc, - leafCount, - nullPolicy, - size, - collapseField, - boostDocs, - searcher, - collectElevatedDocsWhenCollapsing); - - } else { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - "Collapsing field should be of either String, Int or Float type"); - } - - } else { // min, max, sort, etc.. something other then just "score" - - if (collapseFieldType instanceof StrField) { - if (blockCollapse) { - // NOTE: for now we don't worry about whether this is a sortSpec of min/max - // groupHeadSelector, we use a "sort spec' based block collector unless/until there is - // some (performance?) reason to specialize - return new BlockOrdSortSpecCollector( - collapseField, - nullPolicy, - boostDocs, - BlockOrdSortSpecCollector.getSort(groupHeadSelector, sortSpec, funcQuery, searcher), - needsScores || needsScores4Collapsing); - } - - return new OrdFieldValueCollector( - maxDoc, - leafCount, - docValuesProducer, - nullPolicy, - groupHeadSelector, - sortSpec, - needsScores4Collapsing, - needsScores, - minMaxFieldType, - boostDocs, - funcQuery, - searcher, - collectElevatedDocsWhenCollapsing); - - } else if (isNumericCollapsible(collapseFieldType)) { - - if (blockCollapse) { - // NOTE: for now we don't worry about whether this is a sortSpec of min/max - // groupHeadSelector, we use a "sort spec' based block collector unless/until there is - // some (performance?) reason to specialize - return new BlockIntSortSpecCollector( - collapseField, - nullPolicy, - boostDocs, - BlockOrdSortSpecCollector.getSort(groupHeadSelector, sortSpec, funcQuery, searcher), - needsScores || needsScores4Collapsing); - } - - return new IntFieldValueCollector( - maxDoc, - size, - leafCount, - nullPolicy, - collapseField, - groupHeadSelector, - sortSpec, - needsScores4Collapsing, - needsScores, - minMaxFieldType, - boostDocs, - funcQuery, - searcher, - collectElevatedDocsWhenCollapsing); - } else { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - "Collapsing field should be of either String, Int or Float type"); - } - } - } - } - public static final class CollapseScore { /** * Inspects the GroupHeadSelector to determine if this CollapseScore is needed. If it is, then @@ -2288,7 +2258,7 @@ private CollapseScore() { * The abstract base Strategy for collapse strategies that collapse on an ordinal using min/max * field value to select the group head. */ - private abstract static class OrdFieldValueStrategy { + protected abstract static class OrdFieldValueStrategy { protected int nullPolicy; protected IntIntDynamicMap ords; protected Scorable scorer; @@ -3356,14 +3326,14 @@ public void collapseNullGroup(int contextDoc, int globalDoc) throws IOException * *

NOTE: collect methods must be called in increasing globalDoc order */ - private static class BoostedDocsCollector { + protected static class BoostedDocsCollector { private final IntIntHashMap boostDocsMap; private final int[] sortedGlobalDocIds; private final boolean hasBoosts; private final IntArrayList boostedKeys = new IntArrayList(); private final IntArrayList boostedDocs = new IntArrayList(); - ; + private boolean boostedNullGroup = false; private final MergeBoost boostedDocsIdsIter; From 9856d84e7889ddec8580e9dbb17a29142358a042 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Fri, 6 Feb 2026 15:32:39 -0500 Subject: [PATCH 02/21] some better documentation --- .../pages/collapse-and-expand-results.adoc | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/solr/solr-ref-guide/modules/query-guide/pages/collapse-and-expand-results.adoc b/solr/solr-ref-guide/modules/query-guide/pages/collapse-and-expand-results.adoc index a2e493cacfa2..f611e60543bb 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/collapse-and-expand-results.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/collapse-and-expand-results.adoc @@ -18,25 +18,24 @@ The Collapsing query parser and the Expand component combine to form an approach to grouping documents for field collapsing in search results. -The Collapsing query parser groups documents (collapsing the result set) according to your parameters, while the Expand component provides access to documents in the collapsed group for use in results display or other processing by a client application. +The Collapsing query parser groups documents (collapsing the result set) according to your parameters, while the Expand component provides access to other documents in the collapsed group for use in results display or other processing by a client application. Collapse & Expand can together do what the older xref:result-grouping.adoc[] (`group=true`) does for _most_ use-cases but not all. Collapse and Expand are not supported when Result Grouping is enabled. Generally, you should prefer Collapse & Expand. [IMPORTANT] ==== -In order to use these features with SolrCloud, the documents must be located on the same shard. -To ensure document co-location, you can define the `router.name` parameter as `compositeId` when creating the collection. -For more information on this option, see the section xref:deployment-guide:solrcloud-shards-indexing.adoc#document-routing[Document Routing]. +Collapsing happens on each core/shard, not in a distributed sense. +Consequently, it's typically necessary to co-locate documents by collapsibility somehow. See xref:deployment-guide:solrcloud-shards-indexing.adoc#document-routing[Document Routing]. +Result Grouping doesn't have this limitation. ==== == Collapsing Query Parser -The `CollapsingQParser` is really a _post filter_ that provides more performant field collapsing than Solr's standard approach when the number of distinct groups in the result set is high. -This parser collapses the result set to a single document per group before it forwards the result set to the rest of the search components. -So all downstream components (faceting, highlighting, etc.) will work with the collapsed result set. - -The CollapsingQParserPlugin fully supports the QueryElevationComponent. +The `CollapsingQParser` produces a special type of query called a "post filter" that operates on the result set following the `q` and `fq` queries. +Given a configured group field that matches multiple documents, it will "collapse" a group to just one of those documents. +Other Solr components (faceting, highlighting, query elevation, etc.) are downstream of this, and thus will work with only the collapsed result set. +Unlike Result Grouping, Solr's response structure is completely normal, showing no indication of the collapsing behavior. === Collapsing Query Parser Options From 9e654f17fcd9a2cf2cfef83e6f45b764110a6d28 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Fri, 6 Feb 2026 16:00:10 -0500 Subject: [PATCH 03/21] extract getDocValuesProducer --- .../solr/search/CollapsingQParserPlugin.java | 85 ++++++++++--------- 1 file changed, 44 insertions(+), 41 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index 53f7d0d92acd..eeafff09f598 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -486,9 +486,6 @@ private boolean isNumericCollapsible(FieldType collapseFieldType) { protected DelegatingCollector getCollector(IntIntHashMap boostDocs, SolrIndexSearcher searcher) throws IOException { - DocValuesProducer docValuesProducer = null; - FunctionQuery funcQuery = null; - // block collapsing logic is much simpler and uses less memory, but is only viable in specific // situations final boolean blockCollapse = @@ -505,43 +502,12 @@ protected DelegatingCollector getCollector(IntIntHashMap boostDocs, SolrIndexSea HINT_BLOCK); } - FieldType collapseFieldType = searcher.getSchema().getField(collapseField).getType(); - - if (collapseFieldType instanceof StrField) { - // if we are using blockCollapse, then there is no need to bother with TOP_FC - if (HINT_TOP_FC.equals(hint) && !blockCollapse) { - @SuppressWarnings("resource") - final LeafReader uninvertingReader = getTopFieldCacheReader(searcher, collapseField); - - docValuesProducer = - new EmptyDocValuesProducer() { - @Override - public SortedDocValues getSorted(FieldInfo ignored) throws IOException { - SortedDocValues values = uninvertingReader.getSortedDocValues(collapseField); - if (values != null) { - return values; - } else { - return DocValues.emptySorted(); - } - } - }; - } else { - docValuesProducer = - new EmptyDocValuesProducer() { - @Override - public SortedDocValues getSorted(FieldInfo ignored) throws IOException { - return DocValues.getSorted(searcher.getSlowAtomicReader(), collapseField); - } - }; - } - } else { - if (HINT_TOP_FC.equals(hint)) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - "top_fc hint is only supported when collapsing on String Fields"); - } - } + SchemaField schemaField = searcher.getSchema().getField(collapseField); + DocValuesProducer docValuesProducer = + getDocValuesProducer(schemaField, hint, searcher, blockCollapse); + FieldType collapseFieldType = schemaField.getType(); + FunctionQuery funcQuery = null; FieldType minMaxFieldType = null; if (GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type)) { final String text = groupHeadSelector.selectorText; @@ -668,6 +634,43 @@ public SortedDocValues getSorted(FieldInfo ignored) throws IOException { } } } + + protected DocValuesProducer getDocValuesProducer( + SchemaField schemaField, String hint, SolrIndexSearcher searcher, boolean blockCollapse) { + if (schemaField.getType() instanceof StrField) { + // if we are using blockCollapse, then there is no need to bother with TOP_FC + if (HINT_TOP_FC.equals(hint) && !blockCollapse) { + @SuppressWarnings("resource") + final LeafReader uninvertingReader = getTopFieldCacheReader(searcher, collapseField); + + return new EmptyDocValuesProducer() { + @Override + public SortedDocValues getSorted(FieldInfo ignored) throws IOException { + SortedDocValues values = uninvertingReader.getSortedDocValues(collapseField); + if (values != null) { + return values; + } else { + return DocValues.emptySorted(); + } + } + }; + } else { + return new EmptyDocValuesProducer() { + @Override + public SortedDocValues getSorted(FieldInfo ignored) throws IOException { + return DocValues.getSorted(searcher.getSlowAtomicReader(), collapseField); + } + }; + } + } else { + if (HINT_TOP_FC.equals(hint)) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "top_fc hint is only supported when collapsing on String Fields"); + } + } + return null; + } } /** @@ -918,7 +921,7 @@ public void complete() throws IOException { int currentContext = 0; int currentDocBase = 0; - collapseValues = collapseValuesProducer.getSorted(null); + collapseValues = collapseValuesProducer.getSorted(null); // reset iterator if (collapseValues instanceof MultiDocValues.MultiSortedDocValues) { this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues) collapseValues; @@ -1380,7 +1383,7 @@ public void complete() throws IOException { int currentContext = 0; int currentDocBase = 0; - this.collapseValues = collapseValuesProducer.getSorted(null); + this.collapseValues = collapseValuesProducer.getSorted(null); // reset iterator if (collapseValues instanceof MultiDocValues.MultiSortedDocValues) { this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues) collapseValues; this.ordinalMap = multiSortedDocValues.mapping; From 76e890cf9740e935808dd0a481c9223e42c1a7cf Mon Sep 17 00:00:00 2001 From: David Smiley Date: Fri, 6 Feb 2026 16:05:45 -0500 Subject: [PATCH 04/21] GroupHeadSelectorType is now a record --- .../solr/search/CollapsingQParserPlugin.java | 53 ++++++------------- .../search/TestCollapseQParserPlugin.java | 16 +++--- 2 files changed, 23 insertions(+), 46 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index eeafff09f598..06eb4421da63 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -227,42 +227,19 @@ public enum GroupHeadSelectorType { public static final EnumSet MIN_MAX = EnumSet.of(MIN, MAX); } - /** Models all the information about how group head documents should be selected */ - public static final class GroupHeadSelector { - - /** - * The param value for this selector whose meaning depends on type. (ie: a field or valuesource - * for MIN/MAX, a sort string for SORT, "score" for SCORE). Will never be null. - */ - public final String selectorText; - - /** The type for this selector, will never be null */ - public final GroupHeadSelectorType type; + /** + * Models all the information about how group head documents should be selected + * + * @param type The type for this selector, will never be null + * @param selectorText The param value for this selector whose meaning depends on type. (ie: a + * field or valuesource for MIN/MAX, a sort string for SORT, "score" for SCORE). Will never be + * null. + */ + public record GroupHeadSelector(GroupHeadSelectorType type, String selectorText) { - private GroupHeadSelector(String s, GroupHeadSelectorType type) { - assert null != s; + public GroupHeadSelector { + assert null != selectorText; assert null != type; - - this.selectorText = s; - this.type = type; - } - - @Override - public boolean equals(final Object other) { - if (other instanceof GroupHeadSelector that) { - return (this.type == that.type) && this.selectorText.equals(that.selectorText); - } - return false; - } - - @Override - public int hashCode() { - return 17 * (31 + selectorText.hashCode()) * (31 + type.hashCode()); - } - - @Override - public String toString() { - return "GroupHeadSelector(selectorText=" + this.selectorText + ", type=" + this.type + ")"; } /** returns a new GroupHeadSelector based on the specified local params */ @@ -280,14 +257,14 @@ public static GroupHeadSelector build(final SolrParams localParams) { } if (null != sortString) { - return new GroupHeadSelector(sortString, GroupHeadSelectorType.SORT); + return new GroupHeadSelector(GroupHeadSelectorType.SORT, sortString); } else if (null != min) { - return new GroupHeadSelector(min, GroupHeadSelectorType.MIN); + return new GroupHeadSelector(GroupHeadSelectorType.MIN, min); } else if (null != max) { - return new GroupHeadSelector(max, GroupHeadSelectorType.MAX); + return new GroupHeadSelector(GroupHeadSelectorType.MAX, max); } // default - return new GroupHeadSelector("score", GroupHeadSelectorType.SCORE); + return new GroupHeadSelector(GroupHeadSelectorType.SCORE, "score"); } } diff --git a/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java b/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java index 716a4a129bc4..d9839fec5d2d 100644 --- a/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java +++ b/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java @@ -1124,23 +1124,23 @@ public void testGroupHeadSelector() { () -> GroupHeadSelector.build(params("sort", "foo_s asc", "min", "bar_s"))); s = GroupHeadSelector.build(params("min", "foo_s")); - assertEquals(GroupHeadSelectorType.MIN, s.type); - assertEquals("foo_s", s.selectorText); + assertEquals(GroupHeadSelectorType.MIN, s.type()); + assertEquals("foo_s", s.selectorText()); s = GroupHeadSelector.build(params("max", "foo_s")); - assertEquals(GroupHeadSelectorType.MAX, s.type); - assertEquals("foo_s", s.selectorText); + assertEquals(GroupHeadSelectorType.MAX, s.type()); + assertEquals("foo_s", s.selectorText()); assertNotEquals(s, GroupHeadSelector.build(params("min", "foo_s", "other", "stuff"))); s = GroupHeadSelector.build(params()); - assertEquals(GroupHeadSelectorType.SCORE, s.type); - assertNotNull(s.selectorText); + assertEquals(GroupHeadSelectorType.SCORE, s.type()); + assertNotNull(s.selectorText()); assertEquals(GroupHeadSelector.build(params()), s); assertNotEquals(s, GroupHeadSelector.build(params("min", "BAR_s"))); s = GroupHeadSelector.build(params("sort", "foo_s asc")); - assertEquals(GroupHeadSelectorType.SORT, s.type); - assertEquals("foo_s asc", s.selectorText); + assertEquals(GroupHeadSelectorType.SORT, s.type()); + assertEquals("foo_s asc", s.selectorText()); assertEquals(GroupHeadSelector.build(params("sort", "foo_s asc")), s); assertNotEquals(s, GroupHeadSelector.build(params("sort", "BAR_s asc"))); assertNotEquals(s, GroupHeadSelector.build(params("min", "BAR_s"))); From 2412a40cd8fde6303587b405a541681ea288422b Mon Sep 17 00:00:00 2001 From: David Smiley Date: Fri, 6 Feb 2026 16:13:34 -0500 Subject: [PATCH 05/21] GroupHeadSelectorType.CUSTOM And allow custom GroupHeadSelector building --- .../org/apache/solr/search/CollapsingQParserPlugin.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index 06eb4421da63..c30e762544c9 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -223,7 +223,8 @@ public enum GroupHeadSelectorType { MIN, MAX, SORT, - SCORE; + SCORE, + CUSTOM; public static final EnumSet MIN_MAX = EnumSet.of(MIN, MAX); } @@ -367,7 +368,7 @@ public CollapsingPostFilter( SolrException.ErrorCode.BAD_REQUEST, "Collapsing not supported on multivalued fields"); } - this.groupHeadSelector = GroupHeadSelector.build(localParams); + this.groupHeadSelector = buildGroupHeadSelector(localParams); if (groupHeadSelector.type.equals(GroupHeadSelectorType.SORT) && CollapseScore.wantsCScore(groupHeadSelector.selectorText)) { @@ -422,6 +423,10 @@ public CollapsingPostFilter( this.nullPolicy = NullPolicy.fromString(localParams.get("nullPolicy")); } + protected GroupHeadSelector buildGroupHeadSelector(SolrParams localParams) { + return GroupHeadSelector.build(localParams); + } + @Override @SuppressWarnings({"unchecked"}) public DelegatingCollector getFilterCollector(IndexSearcher indexSearcher) { From e066b195e35db83e25bc8b05eb266f3b1f83ceb7 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Wed, 11 Feb 2026 11:22:52 -0500 Subject: [PATCH 06/21] refactor: GroupHeadSelector.needsScores and remove many needsScores4Collapsing fields/passing --- .../solr/search/CollapsingQParserPlugin.java | 68 ++++++++----------- 1 file changed, 27 insertions(+), 41 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index c30e762544c9..dca69236100b 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -267,6 +267,20 @@ public static GroupHeadSelector build(final SolrParams localParams) { // default return new GroupHeadSelector(GroupHeadSelectorType.SCORE, "score"); } + + /** + * Determines if scores are needed for collapsing based on this selector's type and + * configuration. + * + * @param sortSpec the SortSpec if this selector is SORT type, otherwise may be null + * @return true if scores are needed for the collapsing operation + */ + public boolean needsScores(SortSpec sortSpec) { + return GroupHeadSelectorType.SCORE.equals(this.type) + || (GroupHeadSelectorType.SORT.equals(this.type) && sortSpec.includesScore()) + || (GroupHeadSelectorType.MIN_MAX.contains(this.type) + && CollapseScore.wantsCScore(this.selectorText)); + } } public static class CollapsingPostFilter extends ExtendedQueryBase implements PostFilter { @@ -276,7 +290,6 @@ public static class CollapsingPostFilter extends ExtendedQueryBase implements Po protected final SortSpec sortSpec; // may be null, parsed at most once from groupHeadSelector public final String hint; protected final boolean needsScores; - protected final boolean needsScores4Collapsing; protected final NullPolicy nullPolicy; private final int size; private Set boosted; // ordered by "priority" @@ -398,12 +411,7 @@ public CollapsingPostFilter( final ResponseBuilder rb = info.getResponseBuilder(); final SortSpec topSort = null == rb ? null : rb.getSortSpec(); - this.needsScores4Collapsing = - GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type) - || (GroupHeadSelectorType.SORT.equals(groupHeadSelector.type) - && this.sortSpec.includesScore()) - || (GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type) - && CollapseScore.wantsCScore(groupHeadSelector.selectorText)); + boolean needsScores4Collapsing = groupHeadSelector.needsScores(this.sortSpec); this.needsScores = needsScores4Collapsing || (info.getRsp().getReturnFields().wantsScore() @@ -562,7 +570,7 @@ protected DelegatingCollector getCollector(IntIntHashMap boostDocs, SolrIndexSea nullPolicy.getCode(), boostDocs, BlockOrdSortSpecCollector.getSort(groupHeadSelector, sortSpec, funcQuery, searcher), - needsScores || needsScores4Collapsing); + needsScores); } return new OrdFieldValueCollector( @@ -572,7 +580,6 @@ protected DelegatingCollector getCollector(IntIntHashMap boostDocs, SolrIndexSea nullPolicy.getCode(), groupHeadSelector, sortSpec, - needsScores4Collapsing, needsScores, minMaxFieldType, boostDocs, @@ -591,7 +598,7 @@ protected DelegatingCollector getCollector(IntIntHashMap boostDocs, SolrIndexSea nullPolicy.getCode(), boostDocs, BlockOrdSortSpecCollector.getSort(groupHeadSelector, sortSpec, funcQuery, searcher), - needsScores || needsScores4Collapsing); + needsScores); } return new IntFieldValueCollector( @@ -602,7 +609,6 @@ protected DelegatingCollector getCollector(IntIntHashMap boostDocs, SolrIndexSea collapseField, groupHeadSelector, sortSpec, - needsScores4Collapsing, needsScores, minMaxFieldType, boostDocs, @@ -1184,7 +1190,6 @@ protected static class OrdFieldValueCollector extends DelegatingCollector { protected int nullPolicy; private OrdFieldValueStrategy collapseStrategy; - protected boolean needsScores4Collapsing; protected boolean needsScores; private boolean collectElevatedDocsWhenCollapsing; @@ -1198,7 +1203,6 @@ public OrdFieldValueCollector( int nullPolicy, GroupHeadSelector groupHeadSelector, SortSpec sortSpec, - boolean needsScores4Collapsing, boolean needsScores, FieldType fieldType, IntIntHashMap boostDocsMap, @@ -1226,7 +1230,6 @@ public OrdFieldValueCollector( this.boostedDocsCollector = BoostedDocsCollector.build(boostDocsMap); this.nullPolicy = nullPolicy; - this.needsScores4Collapsing = needsScores4Collapsing; this.needsScores = needsScores; this.collapseStrategy = createCollapseStrategy( @@ -1249,7 +1252,6 @@ protected OrdFieldValueStrategy createCollapseStrategy( nullPolicy, valueCount, groupHeadSelector, - this.needsScores4Collapsing, this.needsScores, boostedDocsCollector, sortSpec, @@ -1261,7 +1263,6 @@ protected OrdFieldValueStrategy createCollapseStrategy( nullPolicy, valueCount, groupHeadSelector, - this.needsScores4Collapsing, this.needsScores, boostedDocsCollector, funcQuery, @@ -1456,7 +1457,6 @@ static class IntFieldValueCollector extends DelegatingCollector { private int nullPolicy; private IntFieldValueStrategy collapseStrategy; - private boolean needsScores4Collapsing; private boolean needsScores; private String collapseField; @@ -1471,7 +1471,6 @@ public IntFieldValueCollector( String collapseField, GroupHeadSelector groupHeadSelector, SortSpec sortSpec, - boolean needsScores4Collapsing, boolean needsScores, FieldType fieldType, IntIntHashMap boostDocsMap, @@ -1491,7 +1490,6 @@ public IntFieldValueCollector( } this.collapseField = collapseField; this.nullPolicy = nullPolicy; - this.needsScores4Collapsing = needsScores4Collapsing; this.needsScores = needsScores; this.boostedDocsCollector = BoostedDocsCollector.build(boostDocsMap); @@ -1504,7 +1502,6 @@ public IntFieldValueCollector( collapseField, nullPolicy, groupHeadSelector, - this.needsScores4Collapsing, this.needsScores, boostedDocsCollector, sortSpec, @@ -1517,7 +1514,6 @@ public IntFieldValueCollector( collapseField, nullPolicy, groupHeadSelector, - this.needsScores4Collapsing, this.needsScores, boostedDocsCollector, funcQuery, @@ -2556,6 +2552,7 @@ public void collapse(int ord, int contextDoc, int globalDoc) throws IOException */ private static class OrdValueSourceStrategy extends OrdFieldValueStrategy { + private boolean needsScores4Collapsing; private FloatCompare comp; private float nullVal; private ValueSource valueSource; @@ -2563,14 +2560,12 @@ private static class OrdValueSourceStrategy extends OrdFieldValueStrategy { private IntFloatDynamicMap ordVals; private Map rcontext; private final CollapseScore collapseScore = new CollapseScore(); - private boolean needsScores4Collapsing; public OrdValueSourceStrategy( int maxDoc, int nullPolicy, int valueCount, GroupHeadSelector groupHeadSelector, - boolean needsScores4Collapsing, boolean needsScores, BoostedDocsCollector boostedDocsCollector, FunctionQuery funcQuery, @@ -2578,7 +2573,7 @@ public OrdValueSourceStrategy( SortedDocValues values) throws IOException { super(maxDoc, valueCount, nullPolicy, needsScores, boostedDocsCollector, values); - this.needsScores4Collapsing = needsScores4Collapsing; + this.needsScores4Collapsing = groupHeadSelector.needsScores(null); this.valueSource = funcQuery.getValueSource(); this.rcontext = ValueSource.newContext(searcher); @@ -2653,18 +2648,16 @@ public void collapse(int ord, int contextDoc, int globalDoc) throws IOException */ private static class OrdSortSpecStrategy extends OrdFieldValueStrategy { + private final boolean needsScores4Collapsing; private final SortFieldsCompare compareState; - private final Sort sort; private float score; - private boolean needsScores4Collapsing; public OrdSortSpecStrategy( int maxDoc, int nullPolicy, int valueCount, GroupHeadSelector groupHeadSelector, - boolean needsScores4Collapsing, boolean needsScores, BoostedDocsCollector boostedDocsCollector, SortSpec sortSpec, @@ -2672,11 +2665,11 @@ public OrdSortSpecStrategy( SortedDocValues values) throws IOException { super(maxDoc, valueCount, nullPolicy, needsScores, boostedDocsCollector, values); - this.needsScores4Collapsing = needsScores4Collapsing; + this.needsScores4Collapsing = groupHeadSelector.needsScores(sortSpec); assert GroupHeadSelectorType.SORT.equals(groupHeadSelector.type); - this.sort = rewriteSort(sortSpec, searcher); + Sort sort = rewriteSort(sortSpec, searcher); this.compareState = new SortFieldsCompare(sort.getSort(), valueCount); } @@ -3057,6 +3050,7 @@ public void collapseNullGroup(int contextDoc, int globalDoc) throws IOException */ private static class IntValueSourceStrategy extends IntFieldValueStrategy { + private boolean needsScores4Collapsing; private FloatCompare comp; private IntFloatDynamicMap testValues; private float nullCompVal; @@ -3066,7 +3060,6 @@ private static class IntValueSourceStrategy extends IntFieldValueStrategy { private Map rcontext; private final CollapseScore collapseScore = new CollapseScore(); private int index = -1; - private boolean needsScores4Collapsing; public IntValueSourceStrategy( int maxDoc, @@ -3074,7 +3067,6 @@ public IntValueSourceStrategy( String collapseField, int nullPolicy, GroupHeadSelector groupHeadSelector, - boolean needsScores4Collapsing, boolean needsScores, BoostedDocsCollector boostedDocsCollector, FunctionQuery funcQuery, @@ -3083,7 +3075,7 @@ public IntValueSourceStrategy( super(maxDoc, size, collapseField, nullPolicy, needsScores, boostedDocsCollector); - this.needsScores4Collapsing = needsScores4Collapsing; + this.needsScores4Collapsing = groupHeadSelector.needsScores(null); this.testValues = new IntFloatDynamicMap(size, 0.0f); this.valueSource = funcQuery.getValueSource(); @@ -3185,12 +3177,10 @@ public void collapseNullGroup(int contextDoc, int globalDoc) throws IOException */ private static class IntSortSpecStrategy extends IntFieldValueStrategy { + private final boolean needsScores4Collapsing; private final SortFieldsCompare compareState; - private final SortSpec sortSpec; - private final Sort sort; private int index = -1; - private boolean needsScores4Collapsing; public IntSortSpecStrategy( int maxDoc, @@ -3198,7 +3188,6 @@ public IntSortSpecStrategy( String collapseField, int nullPolicy, GroupHeadSelector groupHeadSelector, - boolean needsScores4Collapsing, boolean needsScores, BoostedDocsCollector boostedDocsCollector, SortSpec sortSpec, @@ -3206,12 +3195,11 @@ public IntSortSpecStrategy( throws IOException { super(maxDoc, size, collapseField, nullPolicy, needsScores, boostedDocsCollector); - this.needsScores4Collapsing = needsScores4Collapsing; + this.needsScores4Collapsing = groupHeadSelector.needsScores(sortSpec); assert GroupHeadSelectorType.SORT.equals(groupHeadSelector.type); - this.sortSpec = sortSpec; - this.sort = rewriteSort(sortSpec, searcher); + Sort sort = rewriteSort(sortSpec, searcher); this.compareState = new SortFieldsCompare(sort.getSort(), size); } @@ -3312,7 +3300,6 @@ public void collapseNullGroup(int contextDoc, int globalDoc) throws IOException *

NOTE: collect methods must be called in increasing globalDoc order */ protected static class BoostedDocsCollector { - private final IntIntHashMap boostDocsMap; private final int[] sortedGlobalDocIds; private final boolean hasBoosts; @@ -3348,7 +3335,6 @@ public void purgeGroupsThatHaveBoostedDocs( } private BoostedDocsCollector(final IntIntHashMap boostDocsMap) { - this.boostDocsMap = boostDocsMap; this.hasBoosts = !boostDocsMap.isEmpty(); sortedGlobalDocIds = new int[boostDocsMap.size()]; Iterator it = boostDocsMap.iterator(); From 03083984929fd080e13131361f97dc21faf29f08 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Thu, 12 Feb 2026 01:02:26 -0500 Subject: [PATCH 07/21] getCollapsedDisi --- .../solr/search/CollapsingQParserPlugin.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index dca69236100b..04c20f530366 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -1383,8 +1383,7 @@ public void complete() throws IOException { leafDelegate = delegate.getLeafCollector(contexts[currentContext]); ScoreAndDoc dummy = new ScoreAndDoc(); leafDelegate.setScorer(dummy); - DocIdSetIterator it = - new BitSetIterator(collapseStrategy.getCollapsedSet(), 0); // cost is not useful here + DocIdSetIterator it = collapseStrategy.getCollapsedDisi(); int globalDoc = -1; int nullScoreIndex = 0; IntFloatDynamicMap scores = collapseStrategy.getScores(); @@ -1614,8 +1613,7 @@ public void complete() throws IOException { leafDelegate = delegate.getLeafCollector(contexts[currentContext]); ScoreAndDoc dummy = new ScoreAndDoc(); leafDelegate.setScorer(dummy); - DocIdSetIterator it = - new BitSetIterator(collapseStrategy.getCollapsedSet(), 0); // cost is not useful here + DocIdSetIterator it = collapseStrategy.getCollapsedDisi(); int globalDoc = -1; int nullScoreIndex = 0; IntIntHashMap cmap = collapseStrategy.getCollapseMap(); @@ -2278,7 +2276,11 @@ public OrdFieldValueStrategy( } } - public FixedBitSet getCollapsedSet() { + public DocIdSetIterator getCollapsedDisi() { + return new BitSetIterator(getCollapsedSet(), 0); // cost is not useful here + } + + protected FixedBitSet getCollapsedSet() { // Handle the boosted docs. boostedDocsCollector.purgeGroupsThatHaveBoostedDocs( collapsedSet, @@ -2802,7 +2804,11 @@ public IntFieldValueStrategy( } } - public FixedBitSet getCollapsedSet() { + public DocIdSetIterator getCollapsedDisi() { + return new BitSetIterator(getCollapsedSet(), 0); // cost is not useful here + } + + protected FixedBitSet getCollapsedSet() { // Handle the boosted docs. boostedDocsCollector.purgeGroupsThatHaveBoostedDocs( From f448d8c4d28bd46b4d9775a4b3136e8c8fbbf832 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Thu, 12 Feb 2026 09:30:54 -0500 Subject: [PATCH 08/21] remove unused "values" param --- .../solr/search/CollapsingQParserPlugin.java | 48 ++++++++----------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index 04c20f530366..5172a0c63c0e 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -1255,8 +1255,8 @@ protected OrdFieldValueStrategy createCollapseStrategy( this.needsScores, boostedDocsCollector, sortSpec, - searcher, - collapseValues); + searcher + ); } else if (funcQuery != null) { return new OrdValueSourceStrategy( maxDoc, @@ -1266,8 +1266,8 @@ protected OrdFieldValueStrategy createCollapseStrategy( this.needsScores, boostedDocsCollector, funcQuery, - searcher, - collapseValues); + searcher + ); } else { NumberType numType = fieldType.getNumberType(); if (null == numType) { @@ -1282,24 +1282,24 @@ protected OrdFieldValueStrategy createCollapseStrategy( valueCount, groupHeadSelector, this.needsScores, - boostedDocsCollector, - collapseValues); + boostedDocsCollector + ); case FLOAT -> new OrdFloatStrategy( maxDoc, nullPolicy, valueCount, groupHeadSelector, this.needsScores, - boostedDocsCollector, - collapseValues); + boostedDocsCollector + ); case LONG -> new OrdLongStrategy( maxDoc, nullPolicy, valueCount, groupHeadSelector, this.needsScores, - boostedDocsCollector, - collapseValues); + boostedDocsCollector + ); default -> throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "min/max must be either Int/Long/Float field types"); @@ -2259,8 +2259,7 @@ public OrdFieldValueStrategy( int valueCount, int nullPolicy, boolean needsScores, - BoostedDocsCollector boostedDocsCollector, - SortedDocValues values) { + BoostedDocsCollector boostedDocsCollector) { this.ords = new IntIntDynamicMap(valueCount, -1); this.nullPolicy = nullPolicy; this.needsScores = needsScores; @@ -2334,10 +2333,9 @@ public OrdIntStrategy( int valueCount, GroupHeadSelector groupHeadSelector, boolean needsScores, - BoostedDocsCollector boostedDocsCollector, - SortedDocValues values) + BoostedDocsCollector boostedDocsCollector) throws IOException { - super(maxDoc, valueCount, nullPolicy, needsScores, boostedDocsCollector, values); + super(maxDoc, valueCount, nullPolicy, needsScores, boostedDocsCollector); this.field = groupHeadSelector.selectorText; assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type); @@ -2410,10 +2408,9 @@ public OrdFloatStrategy( int valueCount, GroupHeadSelector groupHeadSelector, boolean needsScores, - BoostedDocsCollector boostedDocsCollector, - SortedDocValues values) + BoostedDocsCollector boostedDocsCollector) throws IOException { - super(maxDoc, valueCount, nullPolicy, needsScores, boostedDocsCollector, values); + super(maxDoc, valueCount, nullPolicy, needsScores, boostedDocsCollector); this.field = groupHeadSelector.selectorText; assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type); @@ -2490,10 +2487,9 @@ public OrdLongStrategy( int valueCount, GroupHeadSelector groupHeadSelector, boolean needsScores, - BoostedDocsCollector boostedDocsCollector, - SortedDocValues values) + BoostedDocsCollector boostedDocsCollector) throws IOException { - super(maxDoc, valueCount, nullPolicy, needsScores, boostedDocsCollector, values); + super(maxDoc, valueCount, nullPolicy, needsScores, boostedDocsCollector); this.field = groupHeadSelector.selectorText; assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type); @@ -2571,10 +2567,9 @@ public OrdValueSourceStrategy( boolean needsScores, BoostedDocsCollector boostedDocsCollector, FunctionQuery funcQuery, - IndexSearcher searcher, - SortedDocValues values) + IndexSearcher searcher) throws IOException { - super(maxDoc, valueCount, nullPolicy, needsScores, boostedDocsCollector, values); + super(maxDoc, valueCount, nullPolicy, needsScores, boostedDocsCollector); this.needsScores4Collapsing = groupHeadSelector.needsScores(null); this.valueSource = funcQuery.getValueSource(); this.rcontext = ValueSource.newContext(searcher); @@ -2663,10 +2658,9 @@ public OrdSortSpecStrategy( boolean needsScores, BoostedDocsCollector boostedDocsCollector, SortSpec sortSpec, - IndexSearcher searcher, - SortedDocValues values) + IndexSearcher searcher) throws IOException { - super(maxDoc, valueCount, nullPolicy, needsScores, boostedDocsCollector, values); + super(maxDoc, valueCount, nullPolicy, needsScores, boostedDocsCollector); this.needsScores4Collapsing = groupHeadSelector.needsScores(sortSpec); assert GroupHeadSelectorType.SORT.equals(groupHeadSelector.type); From e3da939d462355a103e5c81d980e3094e123ca41 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Sat, 14 Feb 2026 11:29:50 -0500 Subject: [PATCH 09/21] NullPolicy: no getCode needed OrdFieldValueCollector, IntFieldValueCollector: Replaced a strategy pattern with inheritance. Simpler. Easier to customize. Use Builder instead of passing tons of params around. Organize fields on the collectors. --- .../solr/search/CollapsingQParserPlugin.java | 3075 ++++++++--------- 1 file changed, 1382 insertions(+), 1693 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index 5172a0c63c0e..b49c5358286c 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -156,41 +156,31 @@ public class CollapsingQParserPlugin extends QParserPlugin { @Deprecated public static final String HINT_MULTI_DOCVALUES = "multi_docvalues"; public enum NullPolicy { - IGNORE("ignore", 0), - COLLAPSE("collapse", 1), - EXPAND("expand", 2); + IGNORE("ignore"), + COLLAPSE("collapse"), + EXPAND("expand"); private final String name; - private final int code; - NullPolicy(String name, int code) { + NullPolicy(String name) { this.name = name; - this.code = code; } public String getName() { return name; } - public int getCode() { - return code; - } - public static NullPolicy fromString(String nullPolicy) { if (StrUtils.isNullOrEmpty(nullPolicy)) { return DEFAULT_POLICY; } - switch (nullPolicy) { - case "ignore": - return IGNORE; - case "collapse": - return COLLAPSE; - case "expand": - return EXPAND; - default: - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "Invalid nullPolicy: " + nullPolicy); - } + return switch (nullPolicy) { + case "ignore" -> IGNORE; + case "collapse" -> COLLAPSE; + case "expand" -> EXPAND; + default -> throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "Invalid nullPolicy: " + nullPolicy); + }; } static final NullPolicy DEFAULT_POLICY = IGNORE; @@ -485,7 +475,7 @@ protected DelegatingCollector getCollector(IntIntHashMap boostDocs, SolrIndexSea // (for the same reason cscore() isn't supported in 'sort' local param) && (!CollapseScore.wantsCScore(groupHeadSelector.selectorText)) // - && NullPolicy.COLLAPSE.getCode() != nullPolicy.getCode()); + && NullPolicy.COLLAPSE != nullPolicy); if (HINT_BLOCK.equals(hint) && !blockCollapse) { log.debug( "Query specifies hint={} but other local params prevent the use block based collapse", @@ -514,9 +504,6 @@ protected DelegatingCollector getCollector(IntIntHashMap boostDocs, SolrIndexSea } } - int maxDoc = searcher.maxDoc(); - int leafCount = searcher.getTopReaderContext().leaves().size(); - SolrRequestInfo req = SolrRequestInfo.getRequestInfo(); boolean collectElevatedDocsWhenCollapsing = req != null @@ -526,26 +513,22 @@ protected DelegatingCollector getCollector(IntIntHashMap boostDocs, SolrIndexSea if (collapseFieldType instanceof StrField) { if (blockCollapse) { - return new BlockOrdScoreCollector(collapseField, nullPolicy.getCode(), boostDocs); + return new BlockOrdScoreCollector(collapseField, nullPolicy, boostDocs); } return new OrdScoreCollector( - maxDoc, - leafCount, docValuesProducer, - nullPolicy.getCode(), + nullPolicy, boostDocs, searcher, collectElevatedDocsWhenCollapsing); } else if (isNumericCollapsible(collapseFieldType)) { if (blockCollapse) { - return new BlockIntScoreCollector(collapseField, nullPolicy.getCode(), boostDocs); + return new BlockIntScoreCollector(collapseField, nullPolicy, boostDocs); } return new IntScoreCollector( - maxDoc, - leafCount, - nullPolicy.getCode(), + nullPolicy, size, collapseField, boostDocs, @@ -567,25 +550,22 @@ protected DelegatingCollector getCollector(IntIntHashMap boostDocs, SolrIndexSea // some (performance?) reason to specialize return new BlockOrdSortSpecCollector( collapseField, - nullPolicy.getCode(), + nullPolicy, boostDocs, BlockOrdSortSpecCollector.getSort(groupHeadSelector, sortSpec, funcQuery, searcher), needsScores); } - return new OrdFieldValueCollector( - maxDoc, - leafCount, - docValuesProducer, - nullPolicy.getCode(), - groupHeadSelector, - sortSpec, - needsScores, - minMaxFieldType, - boostDocs, - funcQuery, - searcher, - collectElevatedDocsWhenCollapsing); + var builder = new OrdFieldCollectorBuilder(); + builder.groupHeadSelector = groupHeadSelector; + builder.collapseValuesProducer = docValuesProducer; + builder.nullPolicy = nullPolicy; + builder.needsScores = needsScores; + builder.boostDocsMap = boostDocs; + builder.searcher = searcher; + builder.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing; + + return builder.build(sortSpec, minMaxFieldType, funcQuery); } else if (isNumericCollapsible(collapseFieldType)) { @@ -595,26 +575,23 @@ protected DelegatingCollector getCollector(IntIntHashMap boostDocs, SolrIndexSea // some (performance?) reason to specialize return new BlockIntSortSpecCollector( collapseField, - nullPolicy.getCode(), + nullPolicy, boostDocs, BlockOrdSortSpecCollector.getSort(groupHeadSelector, sortSpec, funcQuery, searcher), needsScores); } - return new IntFieldValueCollector( - maxDoc, - size, - leafCount, - nullPolicy.getCode(), - collapseField, - groupHeadSelector, - sortSpec, - needsScores, - minMaxFieldType, - boostDocs, - funcQuery, - searcher, - collectElevatedDocsWhenCollapsing); + var builder = new IntFieldCollectorBuilder(); + builder.groupHeadSelector = groupHeadSelector; + builder.nullPolicy = nullPolicy; + builder.collapseField = collapseField; + builder.size = size; + builder.needsScores = needsScores; + builder.boostDocsMap = boostDocs; + builder.searcher = searcher; + builder.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing; + + return builder.build(sortSpec, minMaxFieldType, funcQuery); } else { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, @@ -766,58 +743,56 @@ public float score() { */ static class OrdScoreCollector extends DelegatingCollector { - private LeafReaderContext[] contexts; + // Configuration + private final LeafReaderContext[] contexts; + private final int maxDoc; + private final NullPolicy nullPolicy; + private final boolean collectElevatedDocsWhenCollapsing; + + // Source data private final DocValuesProducer collapseValuesProducer; - private FixedBitSet collapsedSet; private SortedDocValues collapseValues; private OrdinalMap ordinalMap; + private MultiDocValues.MultiSortedDocValues multiSortedDocValues; private SortedDocValues segmentValues; private LongValues segmentOrdinalMap; - private MultiDocValues.MultiSortedDocValues multiSortedDocValues; - private IntIntDynamicMap ords; - private IntFloatDynamicMap scores; - private int maxDoc; - private int nullPolicy; - private float nullScore = -Float.MAX_VALUE; - private int nullDoc = -1; - private boolean collectElevatedDocsWhenCollapsing; - private FloatArrayList nullScores; + // Results/accumulator + private final IntIntDynamicMap ords; + private final IntFloatDynamicMap scores; + private final FixedBitSet collapsedSet; private final BoostedDocsCollector boostedDocsCollector; + private FloatArrayList nullScores; + private float nullScore = -Float.MAX_VALUE; + private int nullDoc = -1; public OrdScoreCollector( - int maxDoc, - int segments, DocValuesProducer collapseValuesProducer, - int nullPolicy, + NullPolicy nullPolicy, IntIntHashMap boostDocsMap, IndexSearcher searcher, boolean collectElevatedDocsWhenCollapsing) throws IOException { - this.maxDoc = maxDoc; - this.contexts = new LeafReaderContext[segments]; + this.contexts = searcher.getTopReaderContext().leaves().toArray(LeafReaderContext[]::new); + this.maxDoc = searcher.getIndexReader().maxDoc(); + this.nullPolicy = nullPolicy; this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing; - List con = searcher.getTopReaderContext().leaves(); - for (int i = 0; i < con.size(); i++) { - contexts[i] = con.get(i); - } - - this.collapsedSet = new FixedBitSet(maxDoc); this.collapseValuesProducer = collapseValuesProducer; - this.collapseValues = collapseValuesProducer.getSorted(null); + this.collapseValues = collapseValuesProducer.getSorted(null); int valueCount = collapseValues.getValueCount(); - if (collapseValues instanceof MultiDocValues.MultiSortedDocValues) { - this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues) collapseValues; - this.ordinalMap = multiSortedDocValues.mapping; + if (collapseValues instanceof MultiDocValues.MultiSortedDocValues multi) { + this.multiSortedDocValues = multi; + this.ordinalMap = multi.mapping; } + this.ords = new IntIntDynamicMap(valueCount, -1); this.scores = new IntFloatDynamicMap(valueCount, -Float.MAX_VALUE); - this.nullPolicy = nullPolicy; - if (nullPolicy == NullPolicy.EXPAND.getCode()) { + this.collapsedSet = new FixedBitSet(maxDoc); + this.boostedDocsCollector = BoostedDocsCollector.build(boostDocsMap); + if (nullPolicy == NullPolicy.EXPAND) { nullScores = new FloatArrayList(); } - this.boostedDocsCollector = BoostedDocsCollector.build(boostDocsMap); } @Override @@ -872,13 +847,13 @@ public void collect(int contextDoc) throws IOException { ords.put(ord, globalDoc); scores.put(ord, score); } - } else if (nullPolicy == NullPolicy.COLLAPSE.getCode()) { + } else if (nullPolicy == NullPolicy.COLLAPSE) { float score = scorer.score(); if (score > nullScore) { nullScore = score; nullDoc = globalDoc; } - } else if (nullPolicy == NullPolicy.EXPAND.getCode()) { + } else if (nullPolicy == NullPolicy.EXPAND) { collapsedSet.set(globalDoc); nullScores.add(scorer.score()); } @@ -965,9 +940,9 @@ public void complete() throws IOException { dummy.score = scores.get(ord); } else if (mergeBoost.boost(docId)) { // Ignore so it doesn't mess up the null scoring. - } else if (this.nullPolicy == NullPolicy.COLLAPSE.getCode()) { + } else if (this.nullPolicy == NullPolicy.COLLAPSE) { dummy.score = nullScore; - } else if (this.nullPolicy == NullPolicy.EXPAND.getCode()) { + } else if (this.nullPolicy == NullPolicy.EXPAND) { dummy.score = nullScores.get(++index); } @@ -988,46 +963,43 @@ public void complete() throws IOException { */ static class IntScoreCollector extends DelegatingCollector { - private LeafReaderContext[] contexts; - private FixedBitSet collapsedSet; + // Configuration + private final LeafReaderContext[] contexts; + private final int maxDoc; + private final NullPolicy nullPolicy; + private final String field; + private final boolean collectElevatedDocsWhenCollapsing; + + // Source data private NumericDocValues collapseValues; - private IntLongHashMap cmap; - private int maxDoc; - private int nullPolicy; - private float nullScore = -Float.MAX_VALUE; - private int nullDoc = -1; - private FloatArrayList nullScores; - private String field; - private boolean collectElevatedDocsWhenCollapsing; + // Results/accumulator + private final IntLongHashMap cmap; + private final FixedBitSet collapsedSet; private final BoostedDocsCollector boostedDocsCollector; + private FloatArrayList nullScores; + private float nullScore = -Float.MAX_VALUE; + private int nullDoc = -1; public IntScoreCollector( - int maxDoc, - int segments, - int nullPolicy, + NullPolicy nullPolicy, int size, String field, IntIntHashMap boostDocsMap, - IndexSearcher searcher, + SolrIndexSearcher searcher, boolean collectElevatedDocsWhenCollapsing) { - this.maxDoc = maxDoc; - this.contexts = new LeafReaderContext[segments]; + this.contexts = searcher.getTopReaderContext().leaves().toArray(LeafReaderContext[]::new); + this.maxDoc = searcher.maxDoc(); + this.nullPolicy = nullPolicy; + this.field = field; this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing; - List con = searcher.getTopReaderContext().leaves(); - for (int i = 0; i < con.size(); i++) { - contexts[i] = con.get(i); - } + this.cmap = new IntLongHashMap(size); this.collapsedSet = new FixedBitSet(maxDoc); - this.nullPolicy = nullPolicy; - if (nullPolicy == NullPolicy.EXPAND.getCode()) { + this.boostedDocsCollector = BoostedDocsCollector.build(boostDocsMap); + if (nullPolicy == NullPolicy.EXPAND) { nullScores = new FloatArrayList(); } - this.cmap = new IntLongHashMap(size); - this.field = field; - - this.boostedDocsCollector = BoostedDocsCollector.build(boostDocsMap); } @Override @@ -1078,13 +1050,13 @@ public void collect(int contextDoc) throws IOException { if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return; } - if (nullPolicy == NullPolicy.COLLAPSE.getCode()) { + if (nullPolicy == NullPolicy.COLLAPSE) { float score = scorer.score(); if (score > this.nullScore) { this.nullScore = score; this.nullDoc = globalDoc; } - } else if (nullPolicy == NullPolicy.EXPAND.getCode()) { + } else if (nullPolicy == NullPolicy.EXPAND) { collapsedSet.set(globalDoc); nullScores.add(scorer.score()); } @@ -1154,9 +1126,9 @@ public void complete() throws IOException { if (mergeBoost.boost(globalDoc)) { // It's an elevated doc so no score is needed (and should not have been populated) dummy.score = 0F; - } else if (nullPolicy == NullPolicy.COLLAPSE.getCode()) { + } else if (nullPolicy == NullPolicy.COLLAPSE) { dummy.score = nullScore; - } else if (nullPolicy == NullPolicy.EXPAND.getCode()) { + } else if (nullPolicy == NullPolicy.EXPAND) { dummy.score = nullScores.get(nullScoreIndex++); } } @@ -1171,139 +1143,106 @@ public void complete() throws IOException { } } + /** Builder for OrdFieldValueCollector subclasses. */ + protected static class OrdFieldCollectorBuilder { + public GroupHeadSelector groupHeadSelector; + public DocValuesProducer collapseValuesProducer; + public NullPolicy nullPolicy; + public int valueCount; + public boolean needsScores; + public IntIntHashMap boostDocsMap; + public SolrIndexSearcher searcher; + public boolean collectElevatedDocsWhenCollapsing; + + /** Builds the appropriate OrdFieldValueCollector subclass based on the selection strategy. */ + public OrdFieldValueCollector build( + SortSpec sortSpec, FieldType fieldType, FunctionQuery funcQuery) throws IOException { + + try { + SortedDocValues tempValues = collapseValuesProducer.getSorted(null); + valueCount = tempValues.getValueCount(); + } catch (IOException e) { + throw new RuntimeException(e); + } + + if (null != sortSpec) { + return new OrdSortSpecCollector(this, sortSpec); + } else if (funcQuery != null) { + return new OrdValueSourceCollector(this, funcQuery); + } else { + NumberType numType = fieldType.getNumberType(); + if (null == numType) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "min/max must be either Int/Long/Float based field types"); + } + return switch (numType) { + case INTEGER -> new OrdIntCollector(this); + case FLOAT -> new OrdFloatCollector(this); + case LONG -> new OrdLongCollector(this); + default -> throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "min/max must be either Int/Long/Float field types"); + }; + } + } + } + /** * Collapse on Ordinal value field. * * @lucene.internal */ - protected static class OrdFieldValueCollector extends DelegatingCollector { - private LeafReaderContext[] contexts; + protected abstract static class OrdFieldValueCollector extends DelegatingCollector { + // Configuration + protected final LeafReaderContext[] contexts; + protected final int maxDoc; + protected final NullPolicy nullPolicy; + protected final boolean needsScores; + protected final boolean collectElevatedDocsWhenCollapsing; - private DocValuesProducer collapseValuesProducer; + // Source data + protected final DocValuesProducer collapseValuesProducer; protected SortedDocValues collapseValues; protected OrdinalMap ordinalMap; + protected MultiDocValues.MultiSortedDocValues multiSortedDocValues; protected SortedDocValues segmentValues; protected LongValues segmentOrdinalMap; - protected MultiDocValues.MultiSortedDocValues multiSortedDocValues; - - protected int maxDoc; - protected int nullPolicy; - - private OrdFieldValueStrategy collapseStrategy; - protected boolean needsScores; - - private boolean collectElevatedDocsWhenCollapsing; + protected Scorable scorer; + // Results/accumulator + protected final IntIntDynamicMap ords; + protected final FixedBitSet collapsedSet; protected final BoostedDocsCollector boostedDocsCollector; + protected IntFloatDynamicMap scores; + protected FloatArrayList nullScores; + protected float nullScore; + protected int nullDoc = -1; - public OrdFieldValueCollector( - int maxDoc, - int segments, - DocValuesProducer collapseValuesProducer, - int nullPolicy, - GroupHeadSelector groupHeadSelector, - SortSpec sortSpec, - boolean needsScores, - FieldType fieldType, - IntIntHashMap boostDocsMap, - FunctionQuery funcQuery, - IndexSearcher searcher, - boolean collectElevatedDocsWhenCollapsing) - throws IOException { - - assert !GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type); + protected OrdFieldValueCollector(OrdFieldCollectorBuilder ctx) throws IOException { + this.contexts = ctx.searcher.getTopReaderContext().leaves().toArray(LeafReaderContext[]::new); + this.maxDoc = ctx.searcher.maxDoc(); + this.nullPolicy = ctx.nullPolicy; + this.needsScores = ctx.needsScores; + this.collectElevatedDocsWhenCollapsing = ctx.collectElevatedDocsWhenCollapsing; + this.collapseValuesProducer = ctx.collapseValuesProducer; - this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing; - this.maxDoc = maxDoc; - this.contexts = new LeafReaderContext[segments]; - List con = searcher.getTopReaderContext().leaves(); - for (int i = 0; i < con.size(); i++) { - contexts[i] = con.get(i); - } - this.collapseValuesProducer = collapseValuesProducer; - this.collapseValues = collapseValuesProducer.getSorted(null); + this.collapseValues = ctx.collapseValuesProducer.getSorted(null); if (collapseValues instanceof MultiDocValues.MultiSortedDocValues) { this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues) collapseValues; this.ordinalMap = multiSortedDocValues.mapping; } - this.boostedDocsCollector = BoostedDocsCollector.build(boostDocsMap); - - this.nullPolicy = nullPolicy; - this.needsScores = needsScores; - this.collapseStrategy = - createCollapseStrategy( - maxDoc, groupHeadSelector, sortSpec, fieldType, funcQuery, searcher); - } - - protected OrdFieldValueStrategy createCollapseStrategy( - int maxDoc, - GroupHeadSelector groupHeadSelector, - SortSpec sortSpec, - FieldType fieldType, - FunctionQuery funcQuery, - IndexSearcher searcher) - throws IOException { - int valueCount = collapseValues.getValueCount(); - - if (null != sortSpec) { - return new OrdSortSpecStrategy( - maxDoc, - nullPolicy, - valueCount, - groupHeadSelector, - this.needsScores, - boostedDocsCollector, - sortSpec, - searcher - ); - } else if (funcQuery != null) { - return new OrdValueSourceStrategy( - maxDoc, - nullPolicy, - valueCount, - groupHeadSelector, - this.needsScores, - boostedDocsCollector, - funcQuery, - searcher - ); - } else { - NumberType numType = fieldType.getNumberType(); - if (null == numType) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - "min/max must be either Int/Long/Float based field types"); + this.ords = new IntIntDynamicMap(ctx.valueCount, -1); + this.collapsedSet = new FixedBitSet(ctx.searcher.maxDoc()); + this.boostedDocsCollector = BoostedDocsCollector.build(ctx.boostDocsMap); + if (this.needsScores) { + this.scores = new IntFloatDynamicMap(ctx.valueCount, 0.0f); + if (ctx.nullPolicy == NullPolicy.EXPAND) { + this.nullScores = new FloatArrayList(); } - return switch (numType) { - case INTEGER -> new OrdIntStrategy( - maxDoc, - nullPolicy, - valueCount, - groupHeadSelector, - this.needsScores, - boostedDocsCollector - ); - case FLOAT -> new OrdFloatStrategy( - maxDoc, - nullPolicy, - valueCount, - groupHeadSelector, - this.needsScores, - boostedDocsCollector - ); - case LONG -> new OrdLongStrategy( - maxDoc, - nullPolicy, - valueCount, - groupHeadSelector, - this.needsScores, - boostedDocsCollector - ); - default -> throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - "min/max must be either Int/Long/Float field types"); - }; + } else { + this.scores = null; } } @@ -1314,14 +1253,15 @@ public ScoreMode scoreMode() { @Override public void setScorer(Scorable scorer) throws IOException { - this.collapseStrategy.setScorer(scorer); + // not calling super here; see complete() instead + this.scorer = scorer; } @Override public void doSetNextReader(LeafReaderContext context) throws IOException { + // not calling super here; see complete() instead this.contexts[context.ord] = context; this.docBase = context.docBase; - this.collapseStrategy.setNextReader(context); if (ordinalMap != null) { this.segmentValues = this.multiSortedDocValues.values[context.ord]; this.segmentOrdinalMap = ordinalMap.getGlobalOrds(context.ord); @@ -1354,7 +1294,33 @@ public void collect(int contextDoc) throws IOException { } } - collapseStrategy.collapse(ord, contextDoc, globalDoc); + collapse(ord, contextDoc, globalDoc); + } + + /** + * Strategy-specific collapse logic. Subclasses implement this to define how to select the group + * head document. + */ + protected abstract void collapse(int ord, int contextDoc, int globalDoc) throws IOException; + + protected DocIdSetIterator getCollapsedDisi() { + // Handle the boosted docs. + boostedDocsCollector.purgeGroupsThatHaveBoostedDocs( + collapsedSet, + (ord) -> { + ords.remove(ord); + }, + () -> { + nullDoc = -1; + }); + + // Build the sorted DocSet of group heads. + if (nullDoc > -1) { + this.collapsedSet.set(nullDoc); + } + ords.forEachValue(doc -> collapsedSet.set(doc)); + + return new BitSetIterator(collapsedSet, 0); // cost is not useful here } @Override @@ -1383,12 +1349,9 @@ public void complete() throws IOException { leafDelegate = delegate.getLeafCollector(contexts[currentContext]); ScoreAndDoc dummy = new ScoreAndDoc(); leafDelegate.setScorer(dummy); - DocIdSetIterator it = collapseStrategy.getCollapsedDisi(); + DocIdSetIterator it = getCollapsedDisi(); int globalDoc = -1; int nullScoreIndex = 0; - IntFloatDynamicMap scores = collapseStrategy.getScores(); - FloatArrayList nullScores = collapseStrategy.getNullScores(); - float nullScore = collapseStrategy.getNullScore(); final MergeBoost mergeBoost = boostedDocsCollector.getMergeBoost(); while ((globalDoc = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { @@ -1423,14 +1386,14 @@ public void complete() throws IOException { } if (ord > -1) { - dummy.score = scores.get(ord); + dummy.score = this.scores.get(ord); } else if (mergeBoost.boost(globalDoc)) { // It's an elevated doc so no score is needed (and should not have been populated) dummy.score = 0F; - } else if (nullPolicy == NullPolicy.COLLAPSE.getCode()) { - dummy.score = nullScore; - } else if (nullPolicy == NullPolicy.EXPAND.getCode()) { - dummy.score = nullScores.get(nullScoreIndex++); + } else if (nullPolicy == NullPolicy.COLLAPSE) { + dummy.score = this.nullScore; + } else if (nullPolicy == NullPolicy.EXPAND) { + dummy.score = this.nullScores.get(nullScoreIndex++); } } @@ -1444,847 +1407,533 @@ public void complete() throws IOException { } } - /** - * Collapses on an integer field. - * - * @lucene.internal - */ - static class IntFieldValueCollector extends DelegatingCollector { - private LeafReaderContext[] contexts; - private NumericDocValues collapseValues; - private int maxDoc; - private int nullPolicy; - - private IntFieldValueStrategy collapseStrategy; - private boolean needsScores; - private String collapseField; - - private final BoostedDocsCollector boostedDocsCollector; - private boolean collectElevatedDocsWhenCollapsing; - - public IntFieldValueCollector( - int maxDoc, - int size, - int segments, - int nullPolicy, - String collapseField, - GroupHeadSelector groupHeadSelector, - SortSpec sortSpec, - boolean needsScores, - FieldType fieldType, - IntIntHashMap boostDocsMap, - FunctionQuery funcQuery, - IndexSearcher searcher, - boolean collectElevatedDocsWhenCollapsing) - throws IOException { - this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing; - - assert !GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type); + /** Collector for collapsing on ordinal using min/max of an int field to select group head. */ + protected static class OrdIntCollector extends OrdFieldValueCollector { + // Configuration + private final String field; + private final IntCompare comp; - this.maxDoc = maxDoc; - this.contexts = new LeafReaderContext[segments]; - List con = searcher.getTopReaderContext().leaves(); - for (int i = 0; i < con.size(); i++) { - contexts[i] = con.get(i); - } - this.collapseField = collapseField; - this.nullPolicy = nullPolicy; - this.needsScores = needsScores; + // Source fields (per-segment state) + private NumericDocValues minMaxValues; - this.boostedDocsCollector = BoostedDocsCollector.build(boostDocsMap); + // Results/accumulator + private final IntIntDynamicMap ordVals; + private int nullVal; - if (null != sortSpec) { - this.collapseStrategy = - new IntSortSpecStrategy( - maxDoc, - size, - collapseField, - nullPolicy, - groupHeadSelector, - this.needsScores, - boostedDocsCollector, - sortSpec, - searcher); - } else if (funcQuery != null) { - this.collapseStrategy = - new IntValueSourceStrategy( - maxDoc, - size, - collapseField, - nullPolicy, - groupHeadSelector, - this.needsScores, - boostedDocsCollector, - funcQuery, - searcher); + public OrdIntCollector(OrdFieldCollectorBuilder ctx) throws IOException { + super(ctx); + this.field = ctx.groupHeadSelector.selectorText; + assert GroupHeadSelectorType.MIN_MAX.contains(ctx.groupHeadSelector.type); + if (GroupHeadSelectorType.MAX.equals(ctx.groupHeadSelector.type)) { + comp = new MaxIntComp(); + this.ordVals = new IntIntDynamicMap(ctx.valueCount, Integer.MIN_VALUE); } else { - NumberType numType = fieldType.getNumberType(); - assert null != numType; // shouldn't make it here for non-numeric types - switch (numType) { - case INTEGER: - { - this.collapseStrategy = - new IntIntStrategy( - maxDoc, - size, - collapseField, - nullPolicy, - groupHeadSelector, - this.needsScores, - boostedDocsCollector); - break; - } - case FLOAT: - { - this.collapseStrategy = - new IntFloatStrategy( - maxDoc, - size, - collapseField, - nullPolicy, - groupHeadSelector, - this.needsScores, - boostedDocsCollector); - break; - } - default: - { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - "min/max must be Int or Float field types when collapsing on numeric fields"); - } - } + comp = new MinIntComp(); + this.ordVals = new IntIntDynamicMap(ctx.valueCount, Integer.MAX_VALUE); + this.nullVal = Integer.MAX_VALUE; } } - @Override - public ScoreMode scoreMode() { - return needsScores ? ScoreMode.COMPLETE : super.scoreMode(); - } - - @Override - public void setScorer(Scorable scorer) throws IOException { - this.collapseStrategy.setScorer(scorer); - } - @Override public void doSetNextReader(LeafReaderContext context) throws IOException { - this.contexts[context.ord] = context; - this.docBase = context.docBase; - this.collapseStrategy.setNextReader(context); - this.collapseValues = DocValues.getNumeric(context.reader(), this.collapseField); + super.doSetNextReader(context); + this.minMaxValues = DocValues.getNumeric(context.reader(), this.field); } @Override - public void collect(int contextDoc) throws IOException { - final int globalDoc = contextDoc + this.docBase; - if (collapseValues.advanceExact(contextDoc)) { - final int collapseKey = (int) collapseValues.longValue(); - // Check to see if we have documents boosted by the QueryElevationComponent (skip normal - // strategy based collection) - if (boostedDocsCollector.collectIfBoosted(collapseKey, globalDoc)) return; - collapseStrategy.collapse(collapseKey, contextDoc, globalDoc); - - } else { // Null Group... + protected void collapse(int ord, int contextDoc, int globalDoc) throws IOException { + int currentVal; + if (minMaxValues.advanceExact(contextDoc)) { + currentVal = (int) minMaxValues.longValue(); + } else { + currentVal = 0; + } - if (collectElevatedDocsWhenCollapsing) { - // Check to see if we have documents boosted by the QueryElevationComponent (skip normal - // strategy based collection) - if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return; + if (ord > -1) { + if (comp.test(currentVal, ordVals.get(ord))) { + ords.put(ord, globalDoc); + ordVals.put(ord, currentVal); + if (needsScores) { + scores.put(ord, scorer.score()); + } + } + } else if (this.nullPolicy == NullPolicy.COLLAPSE) { + if (comp.test(currentVal, nullVal)) { + nullVal = currentVal; + nullDoc = globalDoc; + if (needsScores) { + nullScore = scorer.score(); + } } - if (NullPolicy.IGNORE.getCode() != nullPolicy) { - collapseStrategy.collapseNullGroup(contextDoc, globalDoc); + } else if (this.nullPolicy == NullPolicy.EXPAND) { + this.collapsedSet.set(globalDoc); + if (needsScores) { + nullScores.add(scorer.score()); } } } + } - @Override - public void complete() throws IOException { - if (contexts.length == 0) { - return; - } + /** Collector for collapsing on ordinal using min/max of a float field to select group head. */ + protected static class OrdFloatCollector extends OrdFieldValueCollector { + // Configuration + private final String field; + private final FloatCompare comp; - int currentContext = 0; - int currentDocBase = 0; - this.collapseValues = - DocValues.getNumeric(contexts[currentContext].reader(), this.collapseField); - int nextDocBase = - currentContext + 1 < contexts.length ? contexts[currentContext + 1].docBase : maxDoc; - leafDelegate = delegate.getLeafCollector(contexts[currentContext]); - ScoreAndDoc dummy = new ScoreAndDoc(); - leafDelegate.setScorer(dummy); - DocIdSetIterator it = collapseStrategy.getCollapsedDisi(); - int globalDoc = -1; - int nullScoreIndex = 0; - IntIntHashMap cmap = collapseStrategy.getCollapseMap(); - IntFloatDynamicMap scores = collapseStrategy.getScores(); - FloatArrayList nullScores = collapseStrategy.getNullScores(); - float nullScore = collapseStrategy.getNullScore(); - final MergeBoost mergeBoost = boostedDocsCollector.getMergeBoost(); - - while ((globalDoc = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { - - while (globalDoc >= nextDocBase) { - currentContext++; - currentDocBase = contexts[currentContext].docBase; - nextDocBase = - currentContext + 1 < contexts.length ? contexts[currentContext + 1].docBase : maxDoc; - leafDelegate = delegate.getLeafCollector(contexts[currentContext]); - leafDelegate.setScorer(dummy); - this.collapseValues = - DocValues.getNumeric(contexts[currentContext].reader(), this.collapseField); - } - - final int contextDoc = globalDoc - currentDocBase; - - if (this.needsScores) { - if (collapseValues.advanceExact(contextDoc)) { - final int collapseValue = (int) collapseValues.longValue(); - - final int pointer = cmap.get(collapseValue); - dummy.score = scores.get(pointer); - - } else { // Null Group... - - if (mergeBoost.boost(globalDoc)) { - // It's an elevated doc so no score is needed (and should not have been populated) - dummy.score = 0F; - } else if (nullPolicy == NullPolicy.COLLAPSE.getCode()) { - dummy.score = nullScore; - } else if (nullPolicy == NullPolicy.EXPAND.getCode()) { - dummy.score = nullScores.get(nullScoreIndex++); - } - } - } - - dummy.docId = contextDoc; - leafDelegate.collect(contextDoc); - } - - if (delegate instanceof DelegatingCollector) { - ((DelegatingCollector) delegate).complete(); - } - } - } - - /** - * Base class for collectors that will do collapsing using "block indexed" documents - * - * @lucene.internal - */ - private abstract static class AbstractBlockCollector extends DelegatingCollector { - - protected final BlockGroupState currentGroupState = new BlockGroupState(); - protected final String collapseField; - protected final boolean needsScores; - protected final boolean expandNulls; - private final MergeBoost boostDocs; - - protected AbstractBlockCollector( - final String collapseField, - final int nullPolicy, - final IntIntHashMap boostDocsMap, - final boolean needsScores) { - - this.collapseField = collapseField; - this.needsScores = needsScores; - - assert nullPolicy != NullPolicy.COLLAPSE.getCode(); - assert nullPolicy == NullPolicy.IGNORE.getCode() || nullPolicy == NullPolicy.EXPAND.getCode(); - this.expandNulls = (NullPolicy.EXPAND.getCode() == nullPolicy); - this.boostDocs = BoostedDocsCollector.build(boostDocsMap).getMergeBoost(); - - currentGroupState.resetForNewGroup(); - } - - @Override - public ScoreMode scoreMode() { - return needsScores ? ScoreMode.COMPLETE : super.scoreMode(); - } - - /** If we have a candidate match, delegate the collection of that match. */ - protected void maybeDelegateCollect() throws IOException { - if (currentGroupState.isCurrentDocCollectable()) { - delegateCollect(); - } - } - - /** Immediately delegate the collection of the current doc */ - protected void delegateCollect() throws IOException { - // ensure we have the 'correct' scorer - // (our supper class may have set the "real" scorer on our leafDelegate - // and it may have an incorrect docID) - leafDelegate.setScorer(currentGroupState); - leafDelegate.collect(currentGroupState.docId); - } - - /** - * NOTE: collects the best doc for the last group in the previous segment subclasses must call - * super BEFORE they make any changes to their own state that might influence - * collection - */ - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - maybeDelegateCollect(); - // Now setup for the next segment. - currentGroupState.resetForNewGroup(); - this.docBase = context.docBase; - super.doSetNextReader(context); - } - - /** - * Acts as an id iterator over the boosted docs - * - * @param contextDoc the context specific docId to check for, iterator is advanced to this id - * @return true if the contextDoc is boosted, false otherwise. - */ - protected boolean isBoostedAdvanceExact(final int contextDoc) { - return boostDocs.boost(contextDoc + docBase); - } - - @Override - public void complete() throws IOException { - // Deal with last group (if any)... - maybeDelegateCollect(); - - super.complete(); - } - - /** - * Encapsulates basic state information about the current group, and the "best matching" - * document in that group (so far) - */ - protected static final class BlockGroupState extends ScoreAndDoc { - /** - * Specific values have no intrinsic meaning, but can only be considered if the - * current docID is non-negative - */ - private int currentGroup = 0; - - private boolean groupHasBoostedDocs; - - public void setCurrentGroup(final int groupId) { - this.currentGroup = groupId; - } - - public int getCurrentGroup() { - assert -1 < this.docId; - return this.currentGroup; - } - - public void setBestDocForCurrentGroup(final int contextDoc, final boolean isBoosted) { - this.docId = contextDoc; - this.groupHasBoostedDocs |= isBoosted; - } - - public void resetForNewGroup() { - this.docId = -1; - this.score = Float.MIN_VALUE; - this.groupHasBoostedDocs = false; - } - - public boolean hasBoostedDocs() { - assert -1 < this.docId; - return groupHasBoostedDocs; - } - - /** - * Returns true if we have a valid ("best match") docId for the current group and there are no - * boosted docs for this group (If the current doc was boosted, it should have already been - * collected) - */ - public boolean isCurrentDocCollectable() { - return (-1 < this.docId && !groupHasBoostedDocs); - } - } - } - - /** - * Collapses groups on a block using a field that has values unique to that block (example: - * _root_) choosing the group head based on score - * - * @lucene.internal - */ - abstract static class AbstractBlockScoreCollector extends AbstractBlockCollector { - - public AbstractBlockScoreCollector( - final String collapseField, final int nullPolicy, final IntIntHashMap boostDocsMap) { - super(collapseField, nullPolicy, boostDocsMap, true); - } - - private void setCurrentGroupBestMatch( - final int contextDocId, final float score, final boolean isBoosted) { - currentGroupState.setBestDocForCurrentGroup(contextDocId, isBoosted); - currentGroupState.score = score; - } - - /** - * This method should be called by subclasses for each doc + group encountered - * - * @param contextDoc a valid doc id relative to the current reader context - * @param docGroup some uique identifier for the group - the base class makes no assumptions - * about it's meaning - * @see #collectDocWithNullGroup - */ - protected void collectDocWithGroup(int contextDoc, int docGroup) throws IOException { - assert 0 <= contextDoc; - - final boolean isBoosted = isBoostedAdvanceExact(contextDoc); - - if (-1 < currentGroupState.docId && docGroup == currentGroupState.getCurrentGroup()) { - // we have an existing group, and contextDoc is in that group. - - if (isBoosted) { - // this doc is the best and should be immediately collected regardless of score - setCurrentGroupBestMatch(contextDoc, scorer.score(), isBoosted); - delegateCollect(); - - } else if (currentGroupState.hasBoostedDocs()) { - // No-Op: nothing about this doc matters since we've already collected boosted docs in - // this group - - // No-Op - } else { - // check if this doc the new 'best' doc in this group... - final float score = scorer.score(); - if (score > currentGroupState.score) { - setCurrentGroupBestMatch(contextDoc, scorer.score(), isBoosted); - } - } - - } else { - // We have a document that starts a new group (or may be the first doc+group we've collected - // this segment) - - // first collect the prior group if needed... - maybeDelegateCollect(); - - // then setup the new group and current best match - currentGroupState.resetForNewGroup(); - currentGroupState.setCurrentGroup(docGroup); - setCurrentGroupBestMatch(contextDoc, scorer.score(), isBoosted); - - if (isBoosted) { // collect immediately - delegateCollect(); - } - } - } - - /** - * This method should be called by subclasses for each doc encountered that is not in a group - * (ie: null group) - * - * @param contextDoc a valid doc id relative to the current reader context - * @see #collectDocWithGroup - */ - protected void collectDocWithNullGroup(int contextDoc) throws IOException { - assert 0 <= contextDoc; - - // NOTE: with 'null group' docs, it doesn't matter if they are boosted since we don't suppor - // collapsing nulls - - // this doc is definitely not part of any prior group, so collect if needed... - maybeDelegateCollect(); - - if (expandNulls) { - // set & immediately collect our current doc... - setCurrentGroupBestMatch(contextDoc, scorer.score(), false); - delegateCollect(); - - } else { - // we're ignoring nulls, so: No-Op. - } - - // either way re-set for the next doc / group - currentGroupState.resetForNewGroup(); - } - } - - /** - * A block based score collector that uses a field's "ord" as the group ids - * - * @lucene.internal - */ - static class BlockOrdScoreCollector extends AbstractBlockScoreCollector { - private SortedDocValues segmentValues; - - public BlockOrdScoreCollector( - final String collapseField, final int nullPolicy, final IntIntHashMap boostDocsMap) - throws IOException { - super(collapseField, nullPolicy, boostDocsMap); - } - - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - super.doSetNextReader(context); - this.segmentValues = DocValues.getSorted(context.reader(), collapseField); - } - - @Override - public void collect(int contextDoc) throws IOException { - if (segmentValues.advanceExact(contextDoc)) { - int ord = segmentValues.ordValue(); - collectDocWithGroup(contextDoc, ord); - } else { - collectDocWithNullGroup(contextDoc); - } - } - } - - /** - * A block based score collector that uses a field's numeric value as the group ids - * - * @lucene.internal - */ - static class BlockIntScoreCollector extends AbstractBlockScoreCollector { - private NumericDocValues segmentValues; - - public BlockIntScoreCollector( - final String collapseField, final int nullPolicy, final IntIntHashMap boostDocsMap) - throws IOException { - super(collapseField, nullPolicy, boostDocsMap); - } - - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - super.doSetNextReader(context); - this.segmentValues = DocValues.getNumeric(context.reader(), collapseField); - } - - @Override - public void collect(int contextDoc) throws IOException { - if (segmentValues.advanceExact(contextDoc)) { - int group = (int) segmentValues.longValue(); - collectDocWithGroup(contextDoc, group); - } else { - collectDocWithNullGroup(contextDoc); - } - } - } - - /** - * Collapses groups on a block using a field that has values unique to that block (example: - * _root_) choosing the group head based on a {@link SortSpec} (which can be synthetically - * created for min/max group head selectors using {@link #getSort}) - * - *

Note that since this collector does a single pass, and unlike other collectors doesn't need - * to maintain a large data structure of scores (for all matching docs) when they might be needed - * for the response, it has no need to distinguish between the concepts of - * needsScores4Collapsing vs needsScores - * - * @lucene.internal - */ - abstract static class AbstractBlockSortSpecCollector extends AbstractBlockCollector { - - /** - * Helper method for extracting a {@link Sort} out of a {@link SortSpec} or creating - * one synthetically for "min/max" {@link GroupHeadSelector} against a {@link FunctionQuery} - * or simple field name. - * - * @return appropriate (already re-written) Sort to use with a AbstractBlockSortSpecCollector - */ - public static Sort getSort( - final GroupHeadSelector groupHeadSelector, - final SortSpec sortSpec, - final FunctionQuery funcQuery, - final SolrIndexSearcher searcher) - throws IOException { - if (null != sortSpec) { - assert GroupHeadSelectorType.SORT.equals(groupHeadSelector.type); - - // a "feature" of SortSpec is that getSort() is null if we're just using 'score desc' - if (null == sortSpec.getSort()) { - return Sort.RELEVANCE.rewrite(searcher); - } - return sortSpec.getSort().rewrite(searcher); - } // else: min/max on field or value source... - - assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type); - assert !CollapseScore.wantsCScore(groupHeadSelector.selectorText); - - final boolean reverse = GroupHeadSelectorType.MAX.equals(groupHeadSelector.type); - final SortField sf = - (null != funcQuery) - ? funcQuery.getValueSource().getSortField(reverse) - : searcher.getSchema().getField(groupHeadSelector.selectorText).getSortField(reverse); - - return (new Sort(sf)).rewrite(searcher); - } - - private final BlockBasedSortFieldsCompare sortsCompare; - - public AbstractBlockSortSpecCollector( - final String collapseField, - final int nullPolicy, - final IntIntHashMap boostDocsMap, - final Sort sort, - final boolean needsScores) { - super(collapseField, nullPolicy, boostDocsMap, needsScores); - this.sortsCompare = new BlockBasedSortFieldsCompare(sort.getSort()); - } + // Source fields (per-segment state) + private NumericDocValues minMaxValues; - @Override - public void setScorer(Scorable scorer) throws IOException { - sortsCompare.setScorer(scorer); - super.setScorer(scorer); - } + // Results/accumulator + private final IntFloatDynamicMap ordVals; + private float nullVal; - private void setCurrentGroupBestMatch(final int contextDocId, final boolean isBoosted) - throws IOException { - currentGroupState.setBestDocForCurrentGroup(contextDocId, isBoosted); - if (needsScores) { - currentGroupState.score = scorer.score(); + public OrdFloatCollector(OrdFieldCollectorBuilder ctx) throws IOException { + super(ctx); + this.field = ctx.groupHeadSelector.selectorText; + assert GroupHeadSelectorType.MIN_MAX.contains(ctx.groupHeadSelector.type); + if (GroupHeadSelectorType.MAX.equals(ctx.groupHeadSelector.type)) { + comp = new MaxFloatComp(); + this.ordVals = new IntFloatDynamicMap(ctx.valueCount, -Float.MAX_VALUE); + this.nullVal = -Float.MAX_VALUE; + } else { + comp = new MinFloatComp(); + this.ordVals = new IntFloatDynamicMap(ctx.valueCount, Float.MAX_VALUE); + this.nullVal = Float.MAX_VALUE; } } @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { + public void doSetNextReader(LeafReaderContext context) throws IOException { super.doSetNextReader(context); - this.sortsCompare.setNextReader(context); + this.minMaxValues = DocValues.getNumeric(context.reader(), this.field); } - /** - * This method should be called by subclasses for each doc + group encountered - * - * @param contextDoc a valid doc id relative to the current reader context - * @param docGroup some uique identifier for the group - the base class makes no assumptions - * about it's meaning - * @see #collectDocWithNullGroup - */ - protected void collectDocWithGroup(int contextDoc, int docGroup) throws IOException { - assert 0 <= contextDoc; - - final boolean isBoosted = isBoostedAdvanceExact(contextDoc); - - if (-1 < currentGroupState.docId && docGroup == currentGroupState.getCurrentGroup()) { - // we have an existing group, and contextDoc is in that group. - - if (isBoosted) { - // this doc is the best and should be immediately collected regardless of sort values - setCurrentGroupBestMatch(contextDoc, isBoosted); - delegateCollect(); + @Override + protected void collapse(int ord, int contextDoc, int globalDoc) throws IOException { + int currentMinMax; + if (minMaxValues.advanceExact(contextDoc)) { + currentMinMax = (int) minMaxValues.longValue(); + } else { + currentMinMax = 0; + } - } else if (currentGroupState.hasBoostedDocs()) { - // No-Op: nothing about this doc matters since we've already collected boosted docs in - // this group + float currentVal = Float.intBitsToFloat(currentMinMax); - // No-Op - } else { - // check if it's the new 'best' doc in this group... - if (sortsCompare.testAndSetGroupValues(contextDoc)) { - setCurrentGroupBestMatch(contextDoc, isBoosted); + if (ord > -1) { + if (comp.test(currentVal, ordVals.get(ord))) { + ords.put(ord, globalDoc); + ordVals.put(ord, currentVal); + if (needsScores) { + scores.put(ord, scorer.score()); } } - - } else { - // We have a document that starts a new group (or may be the first doc+group we've collected - // this segmen) - - // first collect the prior group if needed... - maybeDelegateCollect(); - - // then setup the new group and current best match - currentGroupState.resetForNewGroup(); - currentGroupState.setCurrentGroup(docGroup); - sortsCompare.setGroupValues(contextDoc); - setCurrentGroupBestMatch(contextDoc, isBoosted); - - if (isBoosted) { // collect immediately - delegateCollect(); + } else if (this.nullPolicy == NullPolicy.COLLAPSE) { + if (comp.test(currentVal, nullVal)) { + nullVal = currentVal; + nullDoc = globalDoc; + if (needsScores) { + nullScore = scorer.score(); + } + } + } else if (this.nullPolicy == NullPolicy.EXPAND) { + this.collapsedSet.set(globalDoc); + if (needsScores) { + nullScores.add(scorer.score()); } } } + } - /** - * This method should be called by subclasses for each doc encountered that is not in a group - * (ie: null group) - * - * @param contextDoc a valid doc id relative to the current reader context - * @see #collectDocWithGroup - */ - protected void collectDocWithNullGroup(int contextDoc) throws IOException { - assert 0 <= contextDoc; + /** Collector for collapsing on ordinal using min/max of a long field to select group head. */ + protected static class OrdLongCollector extends OrdFieldValueCollector { + // Configuration + private final String field; + private final LongCompare comp; - // NOTE: with 'null group' docs, it doesn't matter if they are boosted since we don't suppor - // collapsing nulls + // Source fields (per-segment state) + private NumericDocValues minMaxVals; - // this doc is definitely not part of any prior group, so collect if needed... - maybeDelegateCollect(); + // Results/accumulator + private final IntLongDynamicMap ordVals; + private long nullVal; - if (expandNulls) { - // set & immediately collect our current doc... - setCurrentGroupBestMatch(contextDoc, false); - // NOTE: sort values don't matter - delegateCollect(); + public OrdLongCollector(OrdFieldCollectorBuilder ctx) throws IOException { + super(ctx); + this.field = ctx.groupHeadSelector.selectorText; + assert GroupHeadSelectorType.MIN_MAX.contains(ctx.groupHeadSelector.type); + if (GroupHeadSelectorType.MAX.equals(ctx.groupHeadSelector.type)) { + comp = new MaxLongComp(); + this.ordVals = new IntLongDynamicMap(ctx.valueCount, Long.MIN_VALUE); } else { - // we're ignoring nulls, so: No-Op. + this.nullVal = Long.MAX_VALUE; + comp = new MinLongComp(); + this.ordVals = new IntLongDynamicMap(ctx.valueCount, Long.MAX_VALUE); } + } - // either way re-set for the next doc / group - currentGroupState.resetForNewGroup(); + @Override + public void doSetNextReader(LeafReaderContext context) throws IOException { + super.doSetNextReader(context); + this.minMaxVals = DocValues.getNumeric(context.reader(), this.field); + } + + @Override + protected void collapse(int ord, int contextDoc, int globalDoc) throws IOException { + long currentVal; + if (minMaxVals.advanceExact(contextDoc)) { + currentVal = minMaxVals.longValue(); + } else { + currentVal = 0; + } + + if (ord > -1) { + if (comp.test(currentVal, ordVals.get(ord))) { + ords.put(ord, globalDoc); + ordVals.put(ord, currentVal); + if (needsScores) { + scores.put(ord, scorer.score()); + } + } + } else if (this.nullPolicy == NullPolicy.COLLAPSE) { + if (comp.test(currentVal, nullVal)) { + nullVal = currentVal; + nullDoc = globalDoc; + if (needsScores) { + nullScore = scorer.score(); + } + } + } else if (this.nullPolicy == NullPolicy.EXPAND) { + this.collapsedSet.set(globalDoc); + if (needsScores) { + nullScores.add(scorer.score()); + } + } } } /** - * A block based score collector that uses a field's "ord" as the group ids - * - * @lucene.internal + * Collector for collapsing on ordinal using min/max of a value source function to select group + * head. */ - static class BlockOrdSortSpecCollector extends AbstractBlockSortSpecCollector { - private SortedDocValues segmentValues; + protected static class OrdValueSourceCollector extends OrdFieldValueCollector { + // Configuration + private final boolean needsScores4Collapsing; + private final FloatCompare comp; + private final ValueSource valueSource; + private final Map rcontext; + private final CollapseScore collapseScore = new CollapseScore(); - public BlockOrdSortSpecCollector( - final String collapseField, - final int nullPolicy, - final IntIntHashMap boostDocsMap, - final Sort sort, - final boolean needsScores) + // Source fields (per-segment state) + private FunctionValues functionValues; + + // Results/accumulator + private final IntFloatDynamicMap ordVals; + private float nullVal; + + public OrdValueSourceCollector(OrdFieldCollectorBuilder ctx, FunctionQuery funcQuery) throws IOException { - super(collapseField, nullPolicy, boostDocsMap, sort, needsScores); + super(ctx); + this.needsScores4Collapsing = ctx.groupHeadSelector.needsScores(null); + assert GroupHeadSelectorType.MIN_MAX.contains(ctx.groupHeadSelector.type); + if (GroupHeadSelectorType.MAX.equals(ctx.groupHeadSelector.type)) { + comp = new MaxFloatComp(); + this.ordVals = new IntFloatDynamicMap(ctx.valueCount, -Float.MAX_VALUE); + } else { + this.nullVal = Float.MAX_VALUE; + comp = new MinFloatComp(); + } + this.valueSource = funcQuery.getValueSource(); + this.rcontext = ValueSource.newContext(ctx.searcher); + collapseScore.setupIfNeeded(ctx.groupHeadSelector, rcontext); } @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { + public void doSetNextReader(LeafReaderContext context) throws IOException { super.doSetNextReader(context); - this.segmentValues = DocValues.getSorted(context.reader(), collapseField); + functionValues = this.valueSource.getValues(rcontext, context); } @Override - public void collect(int contextDoc) throws IOException { - if (segmentValues.advanceExact(contextDoc)) { - int ord = segmentValues.ordValue(); - collectDocWithGroup(contextDoc, ord); - } else { - collectDocWithNullGroup(contextDoc); + protected void collapse(int ord, int contextDoc, int globalDoc) throws IOException { + float score = 0; + + if (needsScores4Collapsing) { + score = scorer.score(); + this.collapseScore.score = score; + } + + float currentVal = functionValues.floatVal(contextDoc); + + if (ord > -1) { + if (comp.test(currentVal, ordVals.get(ord))) { + ords.put(ord, globalDoc); + ordVals.put(ord, currentVal); + if (needsScores) { + if (!needsScores4Collapsing) { + score = scorer.score(); + } + scores.put(ord, score); + } + } + } else if (this.nullPolicy == NullPolicy.COLLAPSE) { + if (comp.test(currentVal, nullVal)) { + nullVal = currentVal; + nullDoc = globalDoc; + if (needsScores) { + if (!needsScores4Collapsing) { + score = scorer.score(); + } + this.nullScore = score; + } + } + } else if (this.nullPolicy == NullPolicy.EXPAND) { + this.collapsedSet.set(globalDoc); + if (needsScores) { + if (!needsScores4Collapsing) { + score = scorer.score(); + } + nullScores.add(score); + } } } } /** - * A block based score collector that uses a field's numeric value as the group ids - * - * @lucene.internal + * Collector for collapsing on ordinal using the first document according to a complex sort as the + * group head. */ - static class BlockIntSortSpecCollector extends AbstractBlockSortSpecCollector { - private NumericDocValues segmentValues; + protected static class OrdSortSpecCollector extends OrdFieldValueCollector { + // Configuration + private final boolean needsScores4Collapsing; + private final SortFieldsCompare compareState; - public BlockIntSortSpecCollector( - final String collapseField, - final int nullPolicy, - final IntIntHashMap boostDocsMap, - final Sort sort, - final boolean needsScores) + // Results/accumulator + private float score; + + public OrdSortSpecCollector(OrdFieldCollectorBuilder ctx, SortSpec sortSpec) throws IOException { - super(collapseField, nullPolicy, boostDocsMap, sort, needsScores); + super(ctx); + + this.needsScores4Collapsing = ctx.groupHeadSelector.needsScores(sortSpec); + + assert GroupHeadSelectorType.SORT.equals(ctx.groupHeadSelector.type); + + Sort sort = rewriteSort(sortSpec, ctx.searcher); + this.compareState = new SortFieldsCompare(sort.getSort(), ctx.valueCount); } @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { + public void doSetNextReader(LeafReaderContext context) throws IOException { super.doSetNextReader(context); - this.segmentValues = DocValues.getNumeric(context.reader(), collapseField); + compareState.setNextReader(context); } @Override - public void collect(int contextDoc) throws IOException { - if (segmentValues.advanceExact(contextDoc)) { - int group = (int) segmentValues.longValue(); - collectDocWithGroup(contextDoc, group); - } else { - collectDocWithNullGroup(contextDoc); - } + public void setScorer(Scorable scorer) throws IOException { + super.setScorer(scorer); + this.compareState.setScorer(scorer); } - } - public static final class CollapseScore { - /** - * Inspects the GroupHeadSelector to determine if this CollapseScore is needed. If it is, then - * "this" will be added to the readerContext using the "CSCORE" key, and true will be returned. - * If not returns false. - */ - public boolean setupIfNeeded( - final GroupHeadSelector groupHeadSelector, - final Map readerContext) { - // HACK, but not really any better options until/unless we can recursively - // ask value sources if they depend on score - if (wantsCScore(groupHeadSelector.selectorText)) { - readerContext.put("CSCORE", this); - return true; + @Override + protected void collapse(int ord, int contextDoc, int globalDoc) throws IOException { + if (needsScores4Collapsing) { + this.score = scorer.score(); } - return false; - } - /** - * Huge HACK, but not really any better options until/unless we can recursively ask value - * sources if they depend on score - */ - public static boolean wantsCScore(final String text) { - return (text.contains("cscore()")); + if (ord > -1) { // real collapseKey + if (-1 == ords.get(ord)) { + // we've never seen this ord (aka: collapseKey) before, treat it as group head for now + compareState.setGroupValues(ord, contextDoc); + ords.put(ord, globalDoc); + if (needsScores) { + if (!needsScores4Collapsing) { + this.score = scorer.score(); + } + scores.put(ord, score); + } + } else { + // test this ord to see if it's a new group leader + if (compareState.testAndSetGroupValues(ord, contextDoc)) { + ords.put(ord, globalDoc); + if (needsScores) { + if (!needsScores4Collapsing) { + this.score = scorer.score(); + } + scores.put(ord, score); + } + } + } + } else if (this.nullPolicy == NullPolicy.COLLAPSE) { + if (-1 == nullDoc) { + // we've never seen a doc with null collapse key yet, treat it as the null group head for + // now + compareState.setNullGroupValues(contextDoc); + nullDoc = globalDoc; + if (needsScores) { + if (!needsScores4Collapsing) { + this.score = scorer.score(); + } + this.nullScore = score; + } + } else { + // test this doc to see if it's the new null leader + if (compareState.testAndSetNullGroupValues(contextDoc)) { + nullDoc = globalDoc; + if (needsScores) { + if (!needsScores4Collapsing) { + this.score = scorer.score(); + } + this.nullScore = score; + } + } + } + } else if (this.nullPolicy == NullPolicy.EXPAND) { + this.collapsedSet.set(globalDoc); + if (needsScores) { + if (!needsScores4Collapsing) { + this.score = scorer.score(); + } + nullScores.add(score); + } + } } + } - private CollapseScore() { - // No-Op + /** Builder for IntFieldValueCollector subclasses. */ + protected static class IntFieldCollectorBuilder { + public GroupHeadSelector groupHeadSelector; + public NullPolicy nullPolicy; + public String collapseField; + public int size; + public boolean needsScores; + public IntIntHashMap boostDocsMap; + public IndexSearcher searcher; + public boolean collectElevatedDocsWhenCollapsing; + + public IntFieldValueCollector build( + SortSpec sortSpec, FieldType fieldType, FunctionQuery funcQuery) throws IOException { + if (null != sortSpec) { + return new IntSortSpecCollector(this, sortSpec); + } else if (funcQuery != null) { + return new IntValueSourceCollector(this, funcQuery); + } else { + NumberType numType = fieldType.getNumberType(); + assert null != numType; // shouldn't make it here for non-numeric types + return switch (numType) { + case INTEGER -> new IntIntCollector(this); + case FLOAT -> new IntFloatCollector(this); + default -> throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "min/max must be Int or Float field types when collapsing on numeric fields"); + }; + } } - - public float score; } - /* - * Collapse Strategies - */ - /** - * The abstract base Strategy for collapse strategies that collapse on an ordinal using min/max - * field value to select the group head. + * Collapses on an integer field. + * + * @lucene.internal */ - protected abstract static class OrdFieldValueStrategy { - protected int nullPolicy; - protected IntIntDynamicMap ords; + protected abstract static class IntFieldValueCollector extends DelegatingCollector { + // Configuration + protected final LeafReaderContext[] contexts; + protected final int maxDoc; + protected final NullPolicy nullPolicy; + protected final boolean needsScores; + protected final String collapseField; + protected final boolean collectElevatedDocsWhenCollapsing; + + // Source data + protected NumericDocValues collapseValues; protected Scorable scorer; + + // Results/accumulator + protected final IntIntHashMap cmap; + protected final IntIntDynamicMap docs; + protected final FixedBitSet collapsedSet; + protected final BoostedDocsCollector boostedDocsCollector; + protected IntFloatDynamicMap scores; protected FloatArrayList nullScores; protected float nullScore; - protected IntFloatDynamicMap scores; - protected FixedBitSet collapsedSet; protected int nullDoc = -1; - protected boolean needsScores; - - private final BoostedDocsCollector boostedDocsCollector; - - public abstract void collapse(int ord, int contextDoc, int globalDoc) throws IOException; - public abstract void setNextReader(LeafReaderContext context) throws IOException; - - public OrdFieldValueStrategy( - int maxDoc, - int valueCount, - int nullPolicy, - boolean needsScores, - BoostedDocsCollector boostedDocsCollector) { - this.ords = new IntIntDynamicMap(valueCount, -1); - this.nullPolicy = nullPolicy; - this.needsScores = needsScores; - this.collapsedSet = new FixedBitSet(maxDoc); + protected IntFieldValueCollector(IntFieldCollectorBuilder ctx) throws IOException { + assert !GroupHeadSelectorType.SCORE.equals(ctx.groupHeadSelector.type); - this.boostedDocsCollector = boostedDocsCollector; + this.contexts = ctx.searcher.getTopReaderContext().leaves().toArray(LeafReaderContext[]::new); + this.maxDoc = ctx.searcher.getIndexReader().maxDoc(); + this.nullPolicy = ctx.nullPolicy; + this.needsScores = ctx.needsScores; + this.collapseField = ctx.collapseField; + this.collectElevatedDocsWhenCollapsing = ctx.collectElevatedDocsWhenCollapsing; + this.cmap = new IntIntHashMap(ctx.size); + this.docs = new IntIntDynamicMap(ctx.size, 0); + this.collapsedSet = new FixedBitSet(this.maxDoc); + this.boostedDocsCollector = BoostedDocsCollector.build(ctx.boostDocsMap); if (this.needsScores) { - this.scores = new IntFloatDynamicMap(valueCount, 0.0f); - if (nullPolicy == NullPolicy.EXPAND.getCode()) { + this.scores = new IntFloatDynamicMap(ctx.size, 0.0f); + if (ctx.nullPolicy == NullPolicy.EXPAND) { nullScores = new FloatArrayList(); } + } else { + this.scores = null; } } - public DocIdSetIterator getCollapsedDisi() { - return new BitSetIterator(getCollapsedSet(), 0); // cost is not useful here + protected abstract void collapseNullGroup(int contextDoc, int globalDoc) throws IOException; + + protected abstract void collapse(int collapseKey, int contextDoc, int globalDoc) + throws IOException; + + @Override + public ScoreMode scoreMode() { + return needsScores ? ScoreMode.COMPLETE : super.scoreMode(); + } + + @Override + public void setScorer(Scorable scorer) throws IOException { + this.scorer = scorer; + } + + @Override + public void doSetNextReader(LeafReaderContext context) throws IOException { + this.contexts[context.ord] = context; + this.docBase = context.docBase; + this.collapseValues = DocValues.getNumeric(context.reader(), this.collapseField); + } + + @Override + public void collect(int contextDoc) throws IOException { + final int globalDoc = contextDoc + this.docBase; + if (collapseValues.advanceExact(contextDoc)) { + final int collapseKey = (int) collapseValues.longValue(); + // Check to see if we have documents boosted by the QueryElevationComponent (skip normal + // strategy based collection) + if (boostedDocsCollector.collectIfBoosted(collapseKey, globalDoc)) return; + collapse(collapseKey, contextDoc, globalDoc); + + } else { // Null Group... + + if (collectElevatedDocsWhenCollapsing) { + // Check to see if we have documents boosted by the QueryElevationComponent (skip normal + // strategy based collection) + if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return; + } + if (NullPolicy.IGNORE != nullPolicy) { + collapseNullGroup(contextDoc, globalDoc); + } + } } - protected FixedBitSet getCollapsedSet() { + protected DocIdSetIterator getCollapsedDisi() { // Handle the boosted docs. boostedDocsCollector.purgeGroupsThatHaveBoostedDocs( collapsedSet, - (ord) -> { - ords.remove(ord); + (key) -> { + cmap.remove(key); }, () -> { nullDoc = -1; @@ -2294,172 +1943,162 @@ protected FixedBitSet getCollapsedSet() { if (nullDoc > -1) { this.collapsedSet.set(nullDoc); } - ords.forEachValue(doc -> collapsedSet.set(doc)); + for (IntIntCursor cursor : cmap) { + int pointer = cursor.value; + collapsedSet.set(docs.get(pointer)); + } - return collapsedSet; + return new BitSetIterator(collapsedSet, 0); // cost is not useful here } - public void setScorer(Scorable scorer) throws IOException { - this.scorer = scorer; - } + @Override + public void complete() throws IOException { + if (contexts.length == 0) { + return; + } - public FloatArrayList getNullScores() { - return nullScores; - } + int currentContext = 0; + int currentDocBase = 0; + this.collapseValues = + DocValues.getNumeric(contexts[currentContext].reader(), this.collapseField); + int nextDocBase = + currentContext + 1 < contexts.length ? contexts[currentContext + 1].docBase : maxDoc; + leafDelegate = delegate.getLeafCollector(contexts[currentContext]); + ScoreAndDoc dummy = new ScoreAndDoc(); + leafDelegate.setScorer(dummy); + DocIdSetIterator it = getCollapsedDisi(); + int globalDoc = -1; + int nullScoreIndex = 0; + final MergeBoost mergeBoost = boostedDocsCollector.getMergeBoost(); - public float getNullScore() { - return this.nullScore; - } + while ((globalDoc = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + + while (globalDoc >= nextDocBase) { + currentContext++; + currentDocBase = contexts[currentContext].docBase; + nextDocBase = + currentContext + 1 < contexts.length ? contexts[currentContext + 1].docBase : maxDoc; + leafDelegate = delegate.getLeafCollector(contexts[currentContext]); + leafDelegate.setScorer(dummy); + this.collapseValues = + DocValues.getNumeric(contexts[currentContext].reader(), this.collapseField); + } + + final int contextDoc = globalDoc - currentDocBase; + + if (this.needsScores) { + if (collapseValues.advanceExact(contextDoc)) { + final int collapseValue = (int) collapseValues.longValue(); + + final int pointer = cmap.get(collapseValue); + dummy.score = scores.get(pointer); + + } else { // Null Group... + + if (mergeBoost.boost(globalDoc)) { + // It's an elevated doc so no score is needed (and should not have been populated) + dummy.score = 0F; + } else if (nullPolicy == NullPolicy.COLLAPSE) { + dummy.score = nullScore; + } else if (nullPolicy == NullPolicy.EXPAND) { + dummy.score = nullScores.get(nullScoreIndex++); + } + } + } + + dummy.docId = contextDoc; + leafDelegate.collect(contextDoc); + } - public IntFloatDynamicMap getScores() { - return scores; + if (delegate instanceof DelegatingCollector) { + ((DelegatingCollector) delegate).complete(); + } } } - /* - * Strategy for collapsing on ordinal using min/max of an int field to select the group head. + /** + * Collector for collapsing on int field using min/max of an integer field to select group head. */ - private static class OrdIntStrategy extends OrdFieldValueStrategy { - + protected static class IntIntCollector extends IntFieldValueCollector { + // Configuration private final String field; - private NumericDocValues minMaxValues; - private IntCompare comp; - private int nullVal; - private IntIntDynamicMap ordVals; - - public OrdIntStrategy( - int maxDoc, - int nullPolicy, - int valueCount, - GroupHeadSelector groupHeadSelector, - boolean needsScores, - BoostedDocsCollector boostedDocsCollector) - throws IOException { - super(maxDoc, valueCount, nullPolicy, needsScores, boostedDocsCollector); - this.field = groupHeadSelector.selectorText; + private final IntCompare comp; - assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type); + // Source fields (per-segment state) + private NumericDocValues minMaxVals; + + // Results/accumulator + private final IntIntDynamicMap testValues; + private int nullCompVal; + private int index = -1; - if (GroupHeadSelectorType.MAX.equals(groupHeadSelector.type)) { + public IntIntCollector(IntFieldCollectorBuilder ctx) throws IOException { + super(ctx); + this.field = ctx.groupHeadSelector.selectorText; + assert GroupHeadSelectorType.MIN_MAX.contains(ctx.groupHeadSelector.type); + if (GroupHeadSelectorType.MAX.equals(ctx.groupHeadSelector.type)) { comp = new MaxIntComp(); - this.ordVals = new IntIntDynamicMap(valueCount, Integer.MIN_VALUE); + this.nullCompVal = Integer.MIN_VALUE; } else { comp = new MinIntComp(); - this.ordVals = new IntIntDynamicMap(valueCount, Integer.MAX_VALUE); - this.nullVal = Integer.MAX_VALUE; + this.nullCompVal = Integer.MAX_VALUE; } - } - @Override - public void setNextReader(LeafReaderContext context) throws IOException { - this.minMaxValues = DocValues.getNumeric(context.reader(), this.field); + this.testValues = new IntIntDynamicMap(ctx.size, 0); } @Override - public void collapse(int ord, int contextDoc, int globalDoc) throws IOException { + public void doSetNextReader(LeafReaderContext context) throws IOException { + super.doSetNextReader(context); + this.minMaxVals = DocValues.getNumeric(context.reader(), this.field); + } - int currentVal; - if (minMaxValues.advanceExact(contextDoc)) { - currentVal = (int) minMaxValues.longValue(); - } else { - currentVal = 0; + private int advanceAndGetCurrentVal(int contextDoc) throws IOException { + if (minMaxVals.advanceExact(contextDoc)) { + return (int) minMaxVals.longValue(); } + return 0; + } - if (ord > -1) { - if (comp.test(currentVal, ordVals.get(ord))) { - ords.put(ord, globalDoc); - ordVals.put(ord, currentVal); - if (needsScores) { - scores.put(ord, scorer.score()); - } - } - } else if (this.nullPolicy == NullPolicy.COLLAPSE.getCode()) { - if (comp.test(currentVal, nullVal)) { - nullVal = currentVal; - nullDoc = globalDoc; + @Override + protected void collapse(int collapseKey, int contextDoc, int globalDoc) throws IOException { + final int currentVal = advanceAndGetCurrentVal(contextDoc); + + final int idx; + if ((idx = cmap.indexOf(collapseKey)) >= 0) { + int pointer = cmap.indexGet(idx); + if (comp.test(currentVal, testValues.get(pointer))) { + testValues.put(pointer, currentVal); + docs.put(pointer, globalDoc); if (needsScores) { - nullScore = scorer.score(); + scores.put(pointer, scorer.score()); } } - } else if (this.nullPolicy == NullPolicy.EXPAND.getCode()) { - this.collapsedSet.set(globalDoc); + } else { + ++index; + cmap.put(collapseKey, index); + testValues.put(index, currentVal); + docs.put(index, globalDoc); if (needsScores) { - nullScores.add(scorer.score()); + scores.put(index, scorer.score()); } } } - } - - /** - * Strategy for collapsing on ordinal and using the min/max value of a float field to select the - * group head - */ - private static class OrdFloatStrategy extends OrdFieldValueStrategy { - - private final String field; - private NumericDocValues minMaxValues; - private FloatCompare comp; - private float nullVal; - private IntFloatDynamicMap ordVals; - - public OrdFloatStrategy( - int maxDoc, - int nullPolicy, - int valueCount, - GroupHeadSelector groupHeadSelector, - boolean needsScores, - BoostedDocsCollector boostedDocsCollector) - throws IOException { - super(maxDoc, valueCount, nullPolicy, needsScores, boostedDocsCollector); - this.field = groupHeadSelector.selectorText; - - assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type); - - if (GroupHeadSelectorType.MAX.equals(groupHeadSelector.type)) { - comp = new MaxFloatComp(); - this.ordVals = new IntFloatDynamicMap(valueCount, -Float.MAX_VALUE); - this.nullVal = -Float.MAX_VALUE; - } else { - comp = new MinFloatComp(); - this.ordVals = new IntFloatDynamicMap(valueCount, Float.MAX_VALUE); - this.nullVal = Float.MAX_VALUE; - } - } - - @Override - public void setNextReader(LeafReaderContext context) throws IOException { - this.minMaxValues = DocValues.getNumeric(context.reader(), this.field); - } @Override - public void collapse(int ord, int contextDoc, int globalDoc) throws IOException { - - int currentMinMax; - if (minMaxValues.advanceExact(contextDoc)) { - currentMinMax = (int) minMaxValues.longValue(); - } else { - currentMinMax = 0; - } - - float currentVal = Float.intBitsToFloat(currentMinMax); + protected void collapseNullGroup(int contextDoc, int globalDoc) throws IOException { + assert NullPolicy.IGNORE != this.nullPolicy; - if (ord > -1) { - if (comp.test(currentVal, ordVals.get(ord))) { - ords.put(ord, globalDoc); - ordVals.put(ord, currentVal); - if (needsScores) { - scores.put(ord, scorer.score()); - } - } - } else if (this.nullPolicy == NullPolicy.COLLAPSE.getCode()) { - if (comp.test(currentVal, nullVal)) { - nullVal = currentVal; + final int currentVal = advanceAndGetCurrentVal(contextDoc); + if (this.nullPolicy == NullPolicy.COLLAPSE) { + if (comp.test(currentVal, nullCompVal)) { + nullCompVal = currentVal; nullDoc = globalDoc; if (needsScores) { nullScore = scorer.score(); } } - } else if (this.nullPolicy == NullPolicy.EXPAND.getCode()) { + } else if (this.nullPolicy == NullPolicy.EXPAND) { this.collapsedSet.set(globalDoc); if (needsScores) { nullScores.add(scorer.score()); @@ -2468,74 +2107,86 @@ public void collapse(int ord, int contextDoc, int globalDoc) throws IOException } } - /* - * Strategy for collapsing on ordinal and using the min/max value of a long - * field to select the group head - */ - - private static class OrdLongStrategy extends OrdFieldValueStrategy { - + /** Collector for collapsing on int field using min/max of a float field to select group head. */ + protected static class IntFloatCollector extends IntFieldValueCollector { + // Configuration private final String field; + private final FloatCompare comp; + + // Source fields (per-segment state) private NumericDocValues minMaxVals; - private LongCompare comp; - private long nullVal; - private IntLongDynamicMap ordVals; - - public OrdLongStrategy( - int maxDoc, - int nullPolicy, - int valueCount, - GroupHeadSelector groupHeadSelector, - boolean needsScores, - BoostedDocsCollector boostedDocsCollector) - throws IOException { - super(maxDoc, valueCount, nullPolicy, needsScores, boostedDocsCollector); - this.field = groupHeadSelector.selectorText; - assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type); + // Results/accumulator + private final IntFloatDynamicMap testValues; + private float nullCompVal; + private int index = -1; - if (GroupHeadSelectorType.MAX.equals(groupHeadSelector.type)) { - comp = new MaxLongComp(); - this.ordVals = new IntLongDynamicMap(valueCount, Long.MIN_VALUE); + public IntFloatCollector(IntFieldCollectorBuilder ctx) throws IOException { + super(ctx); + this.field = ctx.groupHeadSelector.selectorText; + assert GroupHeadSelectorType.MIN_MAX.contains(ctx.groupHeadSelector.type); + if (GroupHeadSelectorType.MAX.equals(ctx.groupHeadSelector.type)) { + comp = new MaxFloatComp(); + this.nullCompVal = -Float.MAX_VALUE; } else { - this.nullVal = Long.MAX_VALUE; - comp = new MinLongComp(); - this.ordVals = new IntLongDynamicMap(valueCount, Long.MAX_VALUE); + comp = new MinFloatComp(); + this.nullCompVal = Float.MAX_VALUE; } + + this.testValues = new IntFloatDynamicMap(ctx.size, 0.0f); } @Override - public void setNextReader(LeafReaderContext context) throws IOException { + public void doSetNextReader(LeafReaderContext context) throws IOException { + super.doSetNextReader(context); this.minMaxVals = DocValues.getNumeric(context.reader(), this.field); } - @Override - public void collapse(int ord, int contextDoc, int globalDoc) throws IOException { - - long currentVal; + private float advanceAndGetCurrentVal(int contextDoc) throws IOException { if (minMaxVals.advanceExact(contextDoc)) { - currentVal = minMaxVals.longValue(); - } else { - currentVal = 0; + return Float.intBitsToFloat((int) minMaxVals.longValue()); } + return 0.0f; + } - if (ord > -1) { - if (comp.test(currentVal, ordVals.get(ord))) { - ords.put(ord, globalDoc); - ordVals.put(ord, currentVal); + @Override + protected void collapse(int collapseKey, int contextDoc, int globalDoc) throws IOException { + final float currentVal = advanceAndGetCurrentVal(contextDoc); + + final int idx; + if ((idx = cmap.indexOf(collapseKey)) >= 0) { + int pointer = cmap.indexGet(idx); + if (comp.test(currentVal, testValues.get(pointer))) { + testValues.put(pointer, currentVal); + docs.put(pointer, globalDoc); if (needsScores) { - scores.put(ord, scorer.score()); + scores.put(pointer, scorer.score()); } } - } else if (this.nullPolicy == NullPolicy.COLLAPSE.getCode()) { - if (comp.test(currentVal, nullVal)) { - nullVal = currentVal; + } else { + ++index; + cmap.put(collapseKey, index); + testValues.put(index, currentVal); + docs.put(index, globalDoc); + if (needsScores) { + scores.put(index, scorer.score()); + } + } + } + + @Override + protected void collapseNullGroup(int contextDoc, int globalDoc) throws IOException { + assert NullPolicy.IGNORE != this.nullPolicy; + final float currentVal = advanceAndGetCurrentVal(contextDoc); + if (this.nullPolicy == NullPolicy.COLLAPSE) { + if (comp.test(currentVal, nullCompVal)) { + nullCompVal = currentVal; nullDoc = globalDoc; if (needsScores) { nullScore = scorer.score(); } } - } else if (this.nullPolicy == NullPolicy.EXPAND.getCode()) { + } else if (this.nullPolicy == NullPolicy.EXPAND) { this.collapsedSet.set(globalDoc); if (needsScores) { nullScores.add(scorer.score()); @@ -2544,81 +2195,98 @@ public void collapse(int ord, int contextDoc, int globalDoc) throws IOException } } - /* - * Strategy for collapsing on ordinal and using the min/max value of a value source function - * to select the group head - */ - private static class OrdValueSourceStrategy extends OrdFieldValueStrategy { + /** Collector for collapsing on int field using function query to select group head. */ + protected static class IntValueSourceCollector extends IntFieldValueCollector { + // Configuration + private final boolean needsScores4Collapsing; + private final FloatCompare comp; + private final ValueSource valueSource; + private final Map rcontext; - private boolean needsScores4Collapsing; - private FloatCompare comp; - private float nullVal; - private ValueSource valueSource; + // Source fields (per-segment state) private FunctionValues functionValues; - private IntFloatDynamicMap ordVals; - private Map rcontext; + + // Results/accumulator private final CollapseScore collapseScore = new CollapseScore(); + private final IntFloatDynamicMap testValues; + private float nullCompVal; + private int index = -1; - public OrdValueSourceStrategy( - int maxDoc, - int nullPolicy, - int valueCount, - GroupHeadSelector groupHeadSelector, - boolean needsScores, - BoostedDocsCollector boostedDocsCollector, - FunctionQuery funcQuery, - IndexSearcher searcher) + public IntValueSourceCollector(IntFieldCollectorBuilder ctx, FunctionQuery funcQuery) throws IOException { - super(maxDoc, valueCount, nullPolicy, needsScores, boostedDocsCollector); - this.needsScores4Collapsing = groupHeadSelector.needsScores(null); - this.valueSource = funcQuery.getValueSource(); - this.rcontext = ValueSource.newContext(searcher); - - assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type); - - if (GroupHeadSelectorType.MAX.equals(groupHeadSelector.type)) { + super(ctx); + this.needsScores4Collapsing = ctx.groupHeadSelector.needsScores(null); + assert GroupHeadSelectorType.MIN_MAX.contains(ctx.groupHeadSelector.type); + if (GroupHeadSelectorType.MAX.equals(ctx.groupHeadSelector.type)) { comp = new MaxFloatComp(); - this.ordVals = new IntFloatDynamicMap(valueCount, -Float.MAX_VALUE); + nullCompVal = -Float.MAX_VALUE; } else { - this.nullVal = Float.MAX_VALUE; comp = new MinFloatComp(); - this.ordVals = new IntFloatDynamicMap(valueCount, Float.MAX_VALUE); + nullCompVal = Float.MAX_VALUE; } + this.valueSource = funcQuery.getValueSource(); + this.rcontext = ValueSource.newContext(ctx.searcher); + collapseScore.setupIfNeeded(ctx.groupHeadSelector, rcontext); - collapseScore.setupIfNeeded(groupHeadSelector, rcontext); + this.testValues = new IntFloatDynamicMap(ctx.size, 0.0f); } @Override - public void setNextReader(LeafReaderContext context) throws IOException { + public void doSetNextReader(LeafReaderContext context) throws IOException { + super.doSetNextReader(context); functionValues = this.valueSource.getValues(rcontext, context); } - @Override - public void collapse(int ord, int contextDoc, int globalDoc) throws IOException { - - float score = 0; - + private float computeScoreIfNeeded4Collapse() throws IOException { if (needsScores4Collapsing) { - score = scorer.score(); - this.collapseScore.score = score; + this.collapseScore.score = scorer.score(); + return this.collapseScore.score; } + return 0F; + } - float currentVal = functionValues.floatVal(contextDoc); + @Override + protected void collapse(int collapseKey, int contextDoc, int globalDoc) throws IOException { + float score = computeScoreIfNeeded4Collapse(); + final float currentVal = functionValues.floatVal(contextDoc); - if (ord > -1) { - if (comp.test(currentVal, ordVals.get(ord))) { - ords.put(ord, globalDoc); - ordVals.put(ord, currentVal); + final int idx; + if ((idx = cmap.indexOf(collapseKey)) >= 0) { + int pointer = cmap.indexGet(idx); + if (comp.test(currentVal, testValues.get(pointer))) { + testValues.put(pointer, currentVal); + docs.put(pointer, globalDoc); if (needsScores) { if (!needsScores4Collapsing) { score = scorer.score(); } - scores.put(ord, score); + scores.put(pointer, score); } } - } else if (this.nullPolicy == NullPolicy.COLLAPSE.getCode()) { - if (comp.test(currentVal, nullVal)) { - nullVal = currentVal; + } else { + ++index; + cmap.put(collapseKey, index); + docs.put(index, globalDoc); + testValues.put(index, currentVal); + if (needsScores) { + if (!needsScores4Collapsing) { + score = scorer.score(); + } + scores.put(index, score); + } + } + } + + @Override + protected void collapseNullGroup(int contextDoc, int globalDoc) throws IOException { + assert NullPolicy.IGNORE != this.nullPolicy; + + float score = computeScoreIfNeeded4Collapse(); + final float currentVal = functionValues.floatVal(contextDoc); + + if (this.nullPolicy == NullPolicy.COLLAPSE) { + if (comp.test(currentVal, nullCompVal)) { + nullCompVal = currentVal; nullDoc = globalDoc; if (needsScores) { if (!needsScores4Collapsing) { @@ -2627,7 +2295,7 @@ public void collapse(int ord, int contextDoc, int globalDoc) throws IOException nullScore = score; } } - } else if (this.nullPolicy == NullPolicy.EXPAND.getCode()) { + } else if (this.nullPolicy == NullPolicy.EXPAND) { this.collapsedSet.set(globalDoc); if (needsScores) { if (!needsScores4Collapsing) { @@ -2639,82 +2307,85 @@ public void collapse(int ord, int contextDoc, int globalDoc) throws IOException } } - /* - * Strategy for collapsing on ordinal and using the first document according to a complex sort - * as the group head - */ - private static class OrdSortSpecStrategy extends OrdFieldValueStrategy { - + /** Collector for collapsing on int field using a sort spec to select group head. */ + protected static class IntSortSpecCollector extends IntFieldValueCollector { + // Configuration private final boolean needsScores4Collapsing; private final SortFieldsCompare compareState; + // Results/accumulator private float score; + private int index = -1; - public OrdSortSpecStrategy( - int maxDoc, - int nullPolicy, - int valueCount, - GroupHeadSelector groupHeadSelector, - boolean needsScores, - BoostedDocsCollector boostedDocsCollector, - SortSpec sortSpec, - IndexSearcher searcher) + public IntSortSpecCollector(IntFieldCollectorBuilder ctx, SortSpec sortSpec) throws IOException { - super(maxDoc, valueCount, nullPolicy, needsScores, boostedDocsCollector); - this.needsScores4Collapsing = groupHeadSelector.needsScores(sortSpec); + super(ctx); - assert GroupHeadSelectorType.SORT.equals(groupHeadSelector.type); + this.needsScores4Collapsing = ctx.groupHeadSelector.needsScores(sortSpec); - Sort sort = rewriteSort(sortSpec, searcher); + assert GroupHeadSelectorType.SORT.equals(ctx.groupHeadSelector.type); - this.compareState = new SortFieldsCompare(sort.getSort(), valueCount); + Sort sort = rewriteSort(sortSpec, ctx.searcher); + this.compareState = new SortFieldsCompare(sort.getSort(), ctx.size); } @Override - public void setNextReader(LeafReaderContext context) throws IOException { + public void doSetNextReader(LeafReaderContext context) throws IOException { + super.doSetNextReader(context); compareState.setNextReader(context); } @Override - public void setScorer(Scorable s) throws IOException { - super.setScorer(s); - this.compareState.setScorer(s); + public void setScorer(Scorable scorer) throws IOException { + super.setScorer(scorer); + this.compareState.setScorer(scorer); } - @Override - public void collapse(int ord, int contextDoc, int globalDoc) throws IOException { - + private float computeScoreIfNeeded4Collapse() throws IOException { if (needsScores4Collapsing) { - this.score = scorer.score(); + return scorer.score(); } + return 0f; + } - if (ord > -1) { // real collapseKey - if (-1 == ords.get(ord)) { - // we've never seen this ord (aka: collapseKey) before, treat it as group head for now - compareState.setGroupValues(ord, contextDoc); - ords.put(ord, globalDoc); + @Override + protected void collapse(int collapseKey, int contextDoc, int globalDoc) throws IOException { + score = computeScoreIfNeeded4Collapse(); + + final int idx; + if ((idx = cmap.indexOf(collapseKey)) >= 0) { + int pointer = cmap.indexGet(idx); + if (compareState.testAndSetGroupValues(pointer, contextDoc)) { + docs.put(pointer, globalDoc); if (needsScores) { if (!needsScores4Collapsing) { this.score = scorer.score(); } - scores.put(ord, score); + scores.put(pointer, score); } - } else { - // test this ord to see if it's a new group leader - if (compareState.testAndSetGroupValues(ord, contextDoc)) { // TODO X - ords.put(ord, globalDoc); - if (needsScores) { - if (!needsScores4Collapsing) { - this.score = scorer.score(); - } - scores.put(ord, score); - } + } + } else { + ++index; + cmap.put(collapseKey, index); + docs.put(index, globalDoc); + compareState.setGroupValues(index, contextDoc); + if (needsScores) { + if (!needsScores4Collapsing) { + this.score = scorer.score(); } + scores.put(index, score); } - } else if (this.nullPolicy == NullPolicy.COLLAPSE.getCode()) { + } + } + + @Override + protected void collapseNullGroup(int contextDoc, int globalDoc) throws IOException { + assert NullPolicy.IGNORE != this.nullPolicy; + + score = computeScoreIfNeeded4Collapse(); + + if (this.nullPolicy == NullPolicy.COLLAPSE) { if (-1 == nullDoc) { - // we've never seen a doc with null collapse key yet, treat it as the null group head for - // now compareState.setNullGroupValues(contextDoc); nullDoc = globalDoc; if (needsScores) { @@ -2724,7 +2395,6 @@ public void collapse(int ord, int contextDoc, int globalDoc) throws IOException nullScore = score; } } else { - // test this doc to see if it's the new null leader if (compareState.testAndSetNullGroupValues(contextDoc)) { nullDoc = globalDoc; if (needsScores) { @@ -2735,7 +2405,7 @@ public void collapse(int ord, int contextDoc, int globalDoc) throws IOException } } } - } else if (this.nullPolicy == NullPolicy.EXPAND.getCode()) { + } else if (this.nullPolicy == NullPolicy.EXPAND) { this.collapsedSet.set(globalDoc); if (needsScores) { if (!needsScores4Collapsing) { @@ -2747,549 +2417,568 @@ public void collapse(int ord, int contextDoc, int globalDoc) throws IOException } } - /* - * Base strategy for collapsing on a 32 bit numeric field and selecting a group head - * based on min/max value of a 32 bit numeric field. + /** + * Base class for collectors that will do collapsing using "block indexed" documents + * + * @lucene.internal */ + private abstract static class AbstractBlockCollector extends DelegatingCollector { - private abstract static class IntFieldValueStrategy { - protected int nullPolicy; - protected IntIntHashMap cmap; - protected Scorable scorer; - protected FloatArrayList nullScores; - protected float nullScore; - protected IntFloatDynamicMap scores; - protected FixedBitSet collapsedSet; - protected int nullDoc = -1; - protected boolean needsScores; - protected String collapseField; - protected IntIntDynamicMap docs; - - private final BoostedDocsCollector boostedDocsCollector; - - public abstract void collapseNullGroup(int contextDoc, int globalDoc) throws IOException; + // Configuration + protected final String collapseField; + protected final boolean needsScores; + protected final boolean expandNulls; - public abstract void collapse(int collapseKey, int contextDoc, int globalDoc) - throws IOException; + // Results/accumulator + protected final BlockGroupState currentGroupState = new BlockGroupState(); + private final MergeBoost boostDocs; - public abstract void setNextReader(LeafReaderContext context) throws IOException; + protected AbstractBlockCollector( + final String collapseField, + final NullPolicy nullPolicy, + final IntIntHashMap boostDocsMap, + final boolean needsScores) { - public IntFieldValueStrategy( - int maxDoc, - int size, - String collapseField, - int nullPolicy, - boolean needsScores, - BoostedDocsCollector boostedDocsCollector) { this.collapseField = collapseField; - this.nullPolicy = nullPolicy; this.needsScores = needsScores; - this.collapsedSet = new FixedBitSet(maxDoc); - this.cmap = new IntIntHashMap(size); - this.docs = new IntIntDynamicMap(size, 0); - this.boostedDocsCollector = boostedDocsCollector; + assert nullPolicy == NullPolicy.IGNORE || nullPolicy == NullPolicy.EXPAND; + this.expandNulls = (NullPolicy.EXPAND == nullPolicy); + this.boostDocs = BoostedDocsCollector.build(boostDocsMap).getMergeBoost(); - if (needsScores) { - this.scores = new IntFloatDynamicMap(size, 0.0f); - if (nullPolicy == NullPolicy.EXPAND.getCode()) { - nullScores = new FloatArrayList(); - } + currentGroupState.resetForNewGroup(); + } + + @Override + public ScoreMode scoreMode() { + return needsScores ? ScoreMode.COMPLETE : super.scoreMode(); + } + + /** If we have a candidate match, delegate the collection of that match. */ + protected void maybeDelegateCollect() throws IOException { + if (currentGroupState.isCurrentDocCollectable()) { + delegateCollect(); } } - public DocIdSetIterator getCollapsedDisi() { - return new BitSetIterator(getCollapsedSet(), 0); // cost is not useful here + /** Immediately delegate the collection of the current doc */ + protected void delegateCollect() throws IOException { + // ensure we have the 'correct' scorer + // (our supper class may have set the "real" scorer on our leafDelegate + // and it may have an incorrect docID) + leafDelegate.setScorer(currentGroupState); + leafDelegate.collect(currentGroupState.docId); + } + + /** + * NOTE: collects the best doc for the last group in the previous segment subclasses must call + * super BEFORE they make any changes to their own state that might influence + * collection + */ + @Override + protected void doSetNextReader(LeafReaderContext context) throws IOException { + maybeDelegateCollect(); + // Now setup for the next segment. + currentGroupState.resetForNewGroup(); + this.docBase = context.docBase; + super.doSetNextReader(context); + } + + /** + * Acts as an id iterator over the boosted docs + * + * @param contextDoc the context specific docId to check for, iterator is advanced to this id + * @return true if the contextDoc is boosted, false otherwise. + */ + protected boolean isBoostedAdvanceExact(final int contextDoc) { + return boostDocs.boost(contextDoc + docBase); } - protected FixedBitSet getCollapsedSet() { + @Override + public void complete() throws IOException { + // Deal with last group (if any)... + maybeDelegateCollect(); - // Handle the boosted docs. - boostedDocsCollector.purgeGroupsThatHaveBoostedDocs( - collapsedSet, - (key) -> { - cmap.remove(key); - }, - () -> { - nullDoc = -1; - }); + super.complete(); + } - // Build the sorted DocSet of group heads. - if (nullDoc > -1) { - this.collapsedSet.set(nullDoc); + /** + * Encapsulates basic state information about the current group, and the "best matching" + * document in that group (so far) + */ + protected static final class BlockGroupState extends ScoreAndDoc { + /** + * Specific values have no intrinsic meaning, but can only be considered if the + * current docID is non-negative + */ + private int currentGroup = 0; + + private boolean groupHasBoostedDocs; + + public void setCurrentGroup(final int groupId) { + this.currentGroup = groupId; } - Iterator it1 = cmap.iterator(); - while (it1.hasNext()) { - IntIntCursor cursor = it1.next(); - int pointer = cursor.value; - collapsedSet.set(docs.get(pointer)); + + public int getCurrentGroup() { + assert -1 < this.docId; + return this.currentGroup; + } + + public void setBestDocForCurrentGroup(final int contextDoc, final boolean isBoosted) { + this.docId = contextDoc; + this.groupHasBoostedDocs |= isBoosted; + } + + public void resetForNewGroup() { + this.docId = -1; + this.score = Float.MIN_VALUE; + this.groupHasBoostedDocs = false; } - return collapsedSet; + public boolean hasBoostedDocs() { + assert -1 < this.docId; + return groupHasBoostedDocs; + } + + /** + * Returns true if we have a valid ("best match") docId for the current group and there are no + * boosted docs for this group (If the current doc was boosted, it should have already been + * collected) + */ + public boolean isCurrentDocCollectable() { + return (-1 < this.docId && !groupHasBoostedDocs); + } } + } - public void setScorer(Scorable scorer) throws IOException { - this.scorer = scorer; + /** + * Collapses groups on a block using a field that has values unique to that block (example: + * _root_) choosing the group head based on score + * + * @lucene.internal + */ + abstract static class AbstractBlockScoreCollector extends AbstractBlockCollector { + + public AbstractBlockScoreCollector( + final String collapseField, final NullPolicy nullPolicy, final IntIntHashMap boostDocsMap) { + super(collapseField, nullPolicy, boostDocsMap, true); } - public FloatArrayList getNullScores() { - return nullScores; + private void setCurrentGroupBestMatch( + final int contextDocId, final float score, final boolean isBoosted) { + currentGroupState.setBestDocForCurrentGroup(contextDocId, isBoosted); + currentGroupState.score = score; } - public IntIntHashMap getCollapseMap() { - return cmap; + /** + * This method should be called by subclasses for each doc + group encountered + * + * @param contextDoc a valid doc id relative to the current reader context + * @param docGroup some uique identifier for the group - the base class makes no assumptions + * about it's meaning + * @see #collectDocWithNullGroup + */ + protected void collectDocWithGroup(int contextDoc, int docGroup) throws IOException { + assert 0 <= contextDoc; + + final boolean isBoosted = isBoostedAdvanceExact(contextDoc); + + if (-1 < currentGroupState.docId && docGroup == currentGroupState.getCurrentGroup()) { + // we have an existing group, and contextDoc is in that group. + + if (isBoosted) { + // this doc is the best and should be immediately collected regardless of score + setCurrentGroupBestMatch(contextDoc, scorer.score(), isBoosted); + delegateCollect(); + + } else if (currentGroupState.hasBoostedDocs()) { + // No-Op: nothing about this doc matters since we've already collected boosted docs in + // this group + + // No-Op + } else { + // check if this doc the new 'best' doc in this group... + final float score = scorer.score(); + if (score > currentGroupState.score) { + setCurrentGroupBestMatch(contextDoc, scorer.score(), isBoosted); + } + } + + } else { + // We have a document that starts a new group (or may be the first doc+group we've collected + // this segment) + + // first collect the prior group if needed... + maybeDelegateCollect(); + + // then setup the new group and current best match + currentGroupState.resetForNewGroup(); + currentGroupState.setCurrentGroup(docGroup); + setCurrentGroupBestMatch(contextDoc, scorer.score(), isBoosted); + + if (isBoosted) { // collect immediately + delegateCollect(); + } + } } - public float getNullScore() { - return this.nullScore; - } + /** + * This method should be called by subclasses for each doc encountered that is not in a group + * (ie: null group) + * + * @param contextDoc a valid doc id relative to the current reader context + * @see #collectDocWithGroup + */ + protected void collectDocWithNullGroup(int contextDoc) throws IOException { + assert 0 <= contextDoc; + + // NOTE: with 'null group' docs, it doesn't matter if they are boosted since we don't suppor + // collapsing nulls + + // this doc is definitely not part of any prior group, so collect if needed... + maybeDelegateCollect(); + + if (expandNulls) { + // set & immediately collect our current doc... + setCurrentGroupBestMatch(contextDoc, scorer.score(), false); + delegateCollect(); - public IntFloatDynamicMap getScores() { - return scores; - } + } else { + // we're ignoring nulls, so: No-Op. + } - public IntIntDynamicMap getDocs() { - return docs; + // either way re-set for the next doc / group + currentGroupState.resetForNewGroup(); } } - /* - * Strategy for collapsing on a 32 bit numeric field and selecting the group head based - * on the min/max value of a 32 bit field numeric field. + /** + * A block based score collector that uses a field's "ord" as the group ids + * + * @lucene.internal */ - private static class IntIntStrategy extends IntFieldValueStrategy { - - private final String field; - private NumericDocValues minMaxVals; - private IntIntDynamicMap testValues; - private IntCompare comp; - private int nullCompVal; - - private int index = -1; + static class BlockOrdScoreCollector extends AbstractBlockScoreCollector { + private SortedDocValues segmentValues; - public IntIntStrategy( - int maxDoc, - int size, - String collapseField, - int nullPolicy, - GroupHeadSelector groupHeadSelector, - boolean needsScores, - BoostedDocsCollector boostedDocsCollector) + public BlockOrdScoreCollector( + final String collapseField, final NullPolicy nullPolicy, final IntIntHashMap boostDocsMap) throws IOException { - - super(maxDoc, size, collapseField, nullPolicy, needsScores, boostedDocsCollector); - this.field = groupHeadSelector.selectorText; - this.testValues = new IntIntDynamicMap(size, 0); - - assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type); - - if (GroupHeadSelectorType.MAX.equals(groupHeadSelector.type)) { - comp = new MaxIntComp(); - this.nullCompVal = Integer.MIN_VALUE; - } else { - comp = new MinIntComp(); - this.nullCompVal = Integer.MAX_VALUE; - } + super(collapseField, nullPolicy, boostDocsMap); } @Override - public void setNextReader(LeafReaderContext context) throws IOException { - this.minMaxVals = DocValues.getNumeric(context.reader(), this.field); - } - - private int advanceAndGetCurrentVal(int contextDoc) throws IOException { - if (minMaxVals.advanceExact(contextDoc)) { - return (int) minMaxVals.longValue(); - } // else... - return 0; + protected void doSetNextReader(LeafReaderContext context) throws IOException { + super.doSetNextReader(context); + this.segmentValues = DocValues.getSorted(context.reader(), collapseField); } @Override - public void collapse(int collapseKey, int contextDoc, int globalDoc) throws IOException { - final int currentVal = advanceAndGetCurrentVal(contextDoc); - - final int idx; - if ((idx = cmap.indexOf(collapseKey)) >= 0) { - int pointer = cmap.indexGet(idx); - if (comp.test(currentVal, testValues.get(pointer))) { - testValues.put(pointer, currentVal); - docs.put(pointer, globalDoc); - if (needsScores) { - scores.put(pointer, scorer.score()); - } - } + public void collect(int contextDoc) throws IOException { + if (segmentValues.advanceExact(contextDoc)) { + int ord = segmentValues.ordValue(); + collectDocWithGroup(contextDoc, ord); } else { - ++index; - cmap.put(collapseKey, index); - testValues.put(index, currentVal); - docs.put(index, globalDoc); - if (needsScores) { - scores.put(index, scorer.score()); - } - } - } - - @Override - public void collapseNullGroup(int contextDoc, int globalDoc) throws IOException { - assert NullPolicy.IGNORE.getCode() != this.nullPolicy; - - final int currentVal = advanceAndGetCurrentVal(contextDoc); - if (this.nullPolicy == NullPolicy.COLLAPSE.getCode()) { - if (comp.test(currentVal, nullCompVal)) { - nullCompVal = currentVal; - nullDoc = globalDoc; - if (needsScores) { - nullScore = scorer.score(); - } - } - } else if (this.nullPolicy == NullPolicy.EXPAND.getCode()) { - this.collapsedSet.set(globalDoc); - if (needsScores) { - nullScores.add(scorer.score()); - } + collectDocWithNullGroup(contextDoc); } } } - private static class IntFloatStrategy extends IntFieldValueStrategy { - - private final String field; - private NumericDocValues minMaxVals; - private IntFloatDynamicMap testValues; - private FloatCompare comp; - private float nullCompVal; - - private int index = -1; + /** + * A block based score collector that uses a field's numeric value as the group ids + * + * @lucene.internal + */ + static class BlockIntScoreCollector extends AbstractBlockScoreCollector { + private NumericDocValues segmentValues; - public IntFloatStrategy( - int maxDoc, - int size, - String collapseField, - int nullPolicy, - GroupHeadSelector groupHeadSelector, - boolean needsScores, - BoostedDocsCollector boostedDocsCollector) + public BlockIntScoreCollector( + final String collapseField, final NullPolicy nullPolicy, final IntIntHashMap boostDocsMap) throws IOException { - - super(maxDoc, size, collapseField, nullPolicy, needsScores, boostedDocsCollector); - this.field = groupHeadSelector.selectorText; - this.testValues = new IntFloatDynamicMap(size, 0.0f); - - assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type); - - if (GroupHeadSelectorType.MAX.equals(groupHeadSelector.type)) { - comp = new MaxFloatComp(); - this.nullCompVal = -Float.MAX_VALUE; - } else { - comp = new MinFloatComp(); - this.nullCompVal = Float.MAX_VALUE; - } + super(collapseField, nullPolicy, boostDocsMap); } @Override - public void setNextReader(LeafReaderContext context) throws IOException { - this.minMaxVals = DocValues.getNumeric(context.reader(), this.field); - } - - private float advanceAndGetCurrentVal(int contextDoc) throws IOException { - if (minMaxVals.advanceExact(contextDoc)) { - return Float.intBitsToFloat((int) minMaxVals.longValue()); - } // else... - return Float.intBitsToFloat(0); + protected void doSetNextReader(LeafReaderContext context) throws IOException { + super.doSetNextReader(context); + this.segmentValues = DocValues.getNumeric(context.reader(), collapseField); } @Override - public void collapse(int collapseKey, int contextDoc, int globalDoc) throws IOException { - final float currentVal = advanceAndGetCurrentVal(contextDoc); - - final int idx; - if ((idx = cmap.indexOf(collapseKey)) >= 0) { - int pointer = cmap.indexGet(idx); - if (comp.test(currentVal, testValues.get(pointer))) { - testValues.put(pointer, currentVal); - docs.put(pointer, globalDoc); - if (needsScores) { - scores.put(pointer, scorer.score()); - } - } + public void collect(int contextDoc) throws IOException { + if (segmentValues.advanceExact(contextDoc)) { + int group = (int) segmentValues.longValue(); + collectDocWithGroup(contextDoc, group); } else { - ++index; - cmap.put(collapseKey, index); - testValues.put(index, currentVal); - docs.put(index, globalDoc); - if (needsScores) { - scores.put(index, scorer.score()); - } - } - } - - @Override - public void collapseNullGroup(int contextDoc, int globalDoc) throws IOException { - assert NullPolicy.IGNORE.getCode() != this.nullPolicy; - final float currentVal = advanceAndGetCurrentVal(contextDoc); - if (this.nullPolicy == NullPolicy.COLLAPSE.getCode()) { - if (comp.test(currentVal, nullCompVal)) { - nullCompVal = currentVal; - nullDoc = globalDoc; - if (needsScores) { - nullScore = scorer.score(); - } - } - } else if (this.nullPolicy == NullPolicy.EXPAND.getCode()) { - this.collapsedSet.set(globalDoc); - if (needsScores) { - nullScores.add(scorer.score()); - } + collectDocWithNullGroup(contextDoc); } } } - /* - * Strategy for collapsing on a 32 bit numeric field and selecting the group head based - * on the min/max value of a Value Source Function. + /** + * Collapses groups on a block using a field that has values unique to that block (example: + * _root_) choosing the group head based on a {@link SortSpec} (which can be synthetically + * created for min/max group head selectors using {@link #getSort}) + * + *

Note that since this collector does a single pass, and unlike other collectors doesn't need + * to maintain a large data structure of scores (for all matching docs) when they might be needed + * for the response, it has no need to distinguish between the concepts of + * needsScores4Collapsing vs needsScores + * + * @lucene.internal */ - private static class IntValueSourceStrategy extends IntFieldValueStrategy { + abstract static class AbstractBlockSortSpecCollector extends AbstractBlockCollector { - private boolean needsScores4Collapsing; - private FloatCompare comp; - private IntFloatDynamicMap testValues; - private float nullCompVal; + /** + * Helper method for extracting a {@link Sort} out of a {@link SortSpec} or creating + * one synthetically for "min/max" {@link GroupHeadSelector} against a {@link FunctionQuery} + * or simple field name. + * + * @return appropriate (already re-written) Sort to use with a AbstractBlockSortSpecCollector + */ + public static Sort getSort( + final GroupHeadSelector groupHeadSelector, + final SortSpec sortSpec, + final FunctionQuery funcQuery, + final SolrIndexSearcher searcher) + throws IOException { + if (null != sortSpec) { + assert GroupHeadSelectorType.SORT.equals(groupHeadSelector.type); - private ValueSource valueSource; - private FunctionValues functionValues; - private Map rcontext; - private final CollapseScore collapseScore = new CollapseScore(); - private int index = -1; + // a "feature" of SortSpec is that getSort() is null if we're just using 'score desc' + if (null == sortSpec.getSort()) { + return Sort.RELEVANCE.rewrite(searcher); + } + return sortSpec.getSort().rewrite(searcher); + } // else: min/max on field or value source... - public IntValueSourceStrategy( - int maxDoc, - int size, - String collapseField, - int nullPolicy, - GroupHeadSelector groupHeadSelector, - boolean needsScores, - BoostedDocsCollector boostedDocsCollector, - FunctionQuery funcQuery, - IndexSearcher searcher) - throws IOException { + assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type); + assert !CollapseScore.wantsCScore(groupHeadSelector.selectorText); - super(maxDoc, size, collapseField, nullPolicy, needsScores, boostedDocsCollector); + final boolean reverse = GroupHeadSelectorType.MAX.equals(groupHeadSelector.type); + final SortField sf = + (null != funcQuery) + ? funcQuery.getValueSource().getSortField(reverse) + : searcher.getSchema().getField(groupHeadSelector.selectorText).getSortField(reverse); - this.needsScores4Collapsing = groupHeadSelector.needsScores(null); - this.testValues = new IntFloatDynamicMap(size, 0.0f); + return (new Sort(sf)).rewrite(searcher); + } - this.valueSource = funcQuery.getValueSource(); - this.rcontext = ValueSource.newContext(searcher); + private final BlockBasedSortFieldsCompare sortsCompare; - assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type); + public AbstractBlockSortSpecCollector( + final String collapseField, + final NullPolicy nullPolicy, + final IntIntHashMap boostDocsMap, + final Sort sort, + final boolean needsScores) { + super(collapseField, nullPolicy, boostDocsMap, needsScores); + this.sortsCompare = new BlockBasedSortFieldsCompare(sort.getSort()); + } - if (GroupHeadSelectorType.MAX.equals(groupHeadSelector.type)) { - this.nullCompVal = -Float.MAX_VALUE; - comp = new MaxFloatComp(); - } else { - this.nullCompVal = Float.MAX_VALUE; - comp = new MinFloatComp(); + @Override + public void setScorer(Scorable scorer) throws IOException { + sortsCompare.setScorer(scorer); + super.setScorer(scorer); + } + + private void setCurrentGroupBestMatch(final int contextDocId, final boolean isBoosted) + throws IOException { + currentGroupState.setBestDocForCurrentGroup(contextDocId, isBoosted); + if (needsScores) { + currentGroupState.score = scorer.score(); } + } - collapseScore.setupIfNeeded(groupHeadSelector, rcontext); + @Override + protected void doSetNextReader(LeafReaderContext context) throws IOException { + super.doSetNextReader(context); + this.sortsCompare.setNextReader(context); } - @Override - @SuppressWarnings({"unchecked"}) - public void setNextReader(LeafReaderContext context) throws IOException { - functionValues = this.valueSource.getValues(rcontext, context); - } + /** + * This method should be called by subclasses for each doc + group encountered + * + * @param contextDoc a valid doc id relative to the current reader context + * @param docGroup some uique identifier for the group - the base class makes no assumptions + * about it's meaning + * @see #collectDocWithNullGroup + */ + protected void collectDocWithGroup(int contextDoc, int docGroup) throws IOException { + assert 0 <= contextDoc; + + final boolean isBoosted = isBoostedAdvanceExact(contextDoc); - private float computeScoreIfNeeded4Collapse() throws IOException { - if (needsScores4Collapsing) { - this.collapseScore.score = scorer.score(); - return this.collapseScore.score; - } // else... - return 0F; - } + if (-1 < currentGroupState.docId && docGroup == currentGroupState.getCurrentGroup()) { + // we have an existing group, and contextDoc is in that group. - @Override - public void collapse(int collapseKey, int contextDoc, int globalDoc) throws IOException { + if (isBoosted) { + // this doc is the best and should be immediately collected regardless of sort values + setCurrentGroupBestMatch(contextDoc, isBoosted); + delegateCollect(); - float score = computeScoreIfNeeded4Collapse(); - final float currentVal = functionValues.floatVal(contextDoc); + } else if (currentGroupState.hasBoostedDocs()) { + // No-Op: nothing about this doc matters since we've already collected boosted docs in + // this group - final int idx; - if ((idx = cmap.indexOf(collapseKey)) >= 0) { - int pointer = cmap.indexGet(idx); - if (comp.test(currentVal, testValues.get(pointer))) { - testValues.put(pointer, currentVal); - docs.put(pointer, globalDoc); - if (needsScores) { - if (!needsScores4Collapsing) { - score = scorer.score(); - } - scores.put(pointer, score); + // No-Op + } else { + // check if it's the new 'best' doc in this group... + if (sortsCompare.testAndSetGroupValues(contextDoc)) { + setCurrentGroupBestMatch(contextDoc, isBoosted); } } + } else { - ++index; - cmap.put(collapseKey, index); - docs.put(index, globalDoc); - testValues.put(index, currentVal); - if (needsScores) { - if (!needsScores4Collapsing) { - score = scorer.score(); - } - scores.put(index, score); - } - } - } + // We have a document that starts a new group (or may be the first doc+group we've collected + // this segmen) - @Override - public void collapseNullGroup(int contextDoc, int globalDoc) throws IOException { - assert NullPolicy.IGNORE.getCode() != this.nullPolicy; + // first collect the prior group if needed... + maybeDelegateCollect(); - float score = computeScoreIfNeeded4Collapse(); - final float currentVal = functionValues.floatVal(contextDoc); + // then setup the new group and current best match + currentGroupState.resetForNewGroup(); + currentGroupState.setCurrentGroup(docGroup); + sortsCompare.setGroupValues(contextDoc); + setCurrentGroupBestMatch(contextDoc, isBoosted); - if (this.nullPolicy == NullPolicy.COLLAPSE.getCode()) { - if (comp.test(currentVal, nullCompVal)) { - nullCompVal = currentVal; - nullDoc = globalDoc; - if (needsScores) { - if (!needsScores4Collapsing) { - score = scorer.score(); - } - nullScore = score; - } - } - } else if (this.nullPolicy == NullPolicy.EXPAND.getCode()) { - this.collapsedSet.set(globalDoc); - if (needsScores) { - if (!needsScores4Collapsing) { - score = scorer.score(); - } - nullScores.add(score); + if (isBoosted) { // collect immediately + delegateCollect(); } } } - } - /* - * Strategy for collapsing on a 32 bit numeric field and using the first document according - * to a complex sort as the group head - */ - private static class IntSortSpecStrategy extends IntFieldValueStrategy { + /** + * This method should be called by subclasses for each doc encountered that is not in a group + * (ie: null group) + * + * @param contextDoc a valid doc id relative to the current reader context + * @see #collectDocWithGroup + */ + protected void collectDocWithNullGroup(int contextDoc) throws IOException { + assert 0 <= contextDoc; - private final boolean needsScores4Collapsing; - private final SortFieldsCompare compareState; + // NOTE: with 'null group' docs, it doesn't matter if they are boosted since we don't suppor + // collapsing nulls - private int index = -1; + // this doc is definitely not part of any prior group, so collect if needed... + maybeDelegateCollect(); - public IntSortSpecStrategy( - int maxDoc, - int size, - String collapseField, - int nullPolicy, - GroupHeadSelector groupHeadSelector, - boolean needsScores, - BoostedDocsCollector boostedDocsCollector, - SortSpec sortSpec, - IndexSearcher searcher) - throws IOException { + if (expandNulls) { + // set & immediately collect our current doc... + setCurrentGroupBestMatch(contextDoc, false); + // NOTE: sort values don't matter + delegateCollect(); + + } else { + // we're ignoring nulls, so: No-Op. + } - super(maxDoc, size, collapseField, nullPolicy, needsScores, boostedDocsCollector); - this.needsScores4Collapsing = groupHeadSelector.needsScores(sortSpec); + // either way re-set for the next doc / group + currentGroupState.resetForNewGroup(); + } + } - assert GroupHeadSelectorType.SORT.equals(groupHeadSelector.type); + /** + * A block based score collector that uses a field's "ord" as the group ids + * + * @lucene.internal + */ + static class BlockOrdSortSpecCollector extends AbstractBlockSortSpecCollector { + private SortedDocValues segmentValues; - Sort sort = rewriteSort(sortSpec, searcher); - this.compareState = new SortFieldsCompare(sort.getSort(), size); + public BlockOrdSortSpecCollector( + final String collapseField, + final NullPolicy nullPolicy, + final IntIntHashMap boostDocsMap, + final Sort sort, + final boolean needsScores) + throws IOException { + super(collapseField, nullPolicy, boostDocsMap, sort, needsScores); } @Override - public void setNextReader(LeafReaderContext context) throws IOException { - compareState.setNextReader(context); + protected void doSetNextReader(LeafReaderContext context) throws IOException { + super.doSetNextReader(context); + this.segmentValues = DocValues.getSorted(context.reader(), collapseField); } @Override - public void setScorer(Scorable s) throws IOException { - super.setScorer(s); - this.compareState.setScorer(s); + public void collect(int contextDoc) throws IOException { + if (segmentValues.advanceExact(contextDoc)) { + int ord = segmentValues.ordValue(); + collectDocWithGroup(contextDoc, ord); + } else { + collectDocWithNullGroup(contextDoc); + } } + } - private float computeScoreIfNeeded4Collapse() throws IOException { - return needsScores4Collapsing ? scorer.score() : 0F; + /** + * A block based score collector that uses a field's numeric value as the group ids + * + * @lucene.internal + */ + static class BlockIntSortSpecCollector extends AbstractBlockSortSpecCollector { + private NumericDocValues segmentValues; + + public BlockIntSortSpecCollector( + final String collapseField, + final NullPolicy nullPolicy, + final IntIntHashMap boostDocsMap, + final Sort sort, + final boolean needsScores) + throws IOException { + super(collapseField, nullPolicy, boostDocsMap, sort, needsScores); } @Override - public void collapse(int collapseKey, int contextDoc, int globalDoc) throws IOException { - float score = computeScoreIfNeeded4Collapse(); + protected void doSetNextReader(LeafReaderContext context) throws IOException { + super.doSetNextReader(context); + this.segmentValues = DocValues.getNumeric(context.reader(), collapseField); + } - final int idx; - if ((idx = cmap.indexOf(collapseKey)) >= 0) { - // we've seen this collapseKey before, test to see if it's a new group leader - int pointer = cmap.indexGet(idx); - if (compareState.testAndSetGroupValues(pointer, contextDoc)) { - docs.put(pointer, globalDoc); - if (needsScores) { - if (!needsScores4Collapsing) { - score = scorer.score(); - } - scores.put(pointer, score); - } - } + @Override + public void collect(int contextDoc) throws IOException { + if (segmentValues.advanceExact(contextDoc)) { + int group = (int) segmentValues.longValue(); + collectDocWithGroup(contextDoc, group); } else { - // we've never seen this collapseKey before, treat it as group head for now - ++index; - cmap.put(collapseKey, index); - docs.put(index, globalDoc); - compareState.setGroupValues(index, contextDoc); - if (needsScores) { - if (!needsScores4Collapsing) { - score = scorer.score(); - } - scores.put(index, score); - } + collectDocWithNullGroup(contextDoc); } } + } - @Override - public void collapseNullGroup(int contextDoc, int globalDoc) throws IOException { - assert NullPolicy.IGNORE.getCode() != this.nullPolicy; + public static final class CollapseScore { + /** + * Inspects the GroupHeadSelector to determine if this CollapseScore is needed. If it is, then + * "this" will be added to the readerContext using the "CSCORE" key, and true will be returned. + * If not returns false. + */ + public boolean setupIfNeeded( + final GroupHeadSelector groupHeadSelector, + final Map readerContext) { + // HACK, but not really any better options until/unless we can recursively + // ask value sources if they depend on score + if (wantsCScore(groupHeadSelector.selectorText)) { + readerContext.put("CSCORE", this); + return true; + } + return false; + } - float score = computeScoreIfNeeded4Collapse(); + /** + * Huge HACK, but not really any better options until/unless we can recursively ask value + * sources if they depend on score + */ + public static boolean wantsCScore(final String text) { + return (text.contains("cscore()")); + } - if (this.nullPolicy == NullPolicy.COLLAPSE.getCode()) { - if (-1 == nullDoc) { - // we've never seen a doc with null collapse key yet, treat it as the null group head for - // now - compareState.setNullGroupValues(contextDoc); - nullDoc = globalDoc; - if (needsScores) { - if (!needsScores4Collapsing) { - score = scorer.score(); - } - nullScore = score; - } - } else { - // test this doc to see if it's the new null leader - if (compareState.testAndSetNullGroupValues(contextDoc)) { - nullDoc = globalDoc; - if (needsScores) { - if (!needsScores4Collapsing) { - score = scorer.score(); - } - nullScore = score; - } - } - } - } else if (this.nullPolicy == NullPolicy.EXPAND.getCode()) { - this.collapsedSet.set(globalDoc); - if (needsScores) { - if (!needsScores4Collapsing) { - score = scorer.score(); - } - nullScores.add(score); - } - } + private CollapseScore() { + // No-Op } + + public float score; } /** From 7a4829fe433ca92f0ab90d64efec59554ba2c0dc Mon Sep 17 00:00:00 2001 From: David Smiley Date: Sat, 14 Feb 2026 13:16:26 -0500 Subject: [PATCH 10/21] woops; fix --- .../java/org/apache/solr/search/CollapsingQParserPlugin.java | 1 + .../apache/solr/search/TestRandomCollapseQParserPlugin.java | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index b49c5358286c..f78018ea508e 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -1642,6 +1642,7 @@ public OrdValueSourceCollector(OrdFieldCollectorBuilder ctx, FunctionQuery funcQ } else { this.nullVal = Float.MAX_VALUE; comp = new MinFloatComp(); + this.ordVals = new IntFloatDynamicMap(ctx.valueCount, Float.MAX_VALUE); } this.valueSource = funcQuery.getValueSource(); this.rcontext = ValueSource.newContext(ctx.searcher); diff --git a/solr/core/src/test/org/apache/solr/search/TestRandomCollapseQParserPlugin.java b/solr/core/src/test/org/apache/solr/search/TestRandomCollapseQParserPlugin.java index 09f3b7008e9d..beff4f515aeb 100644 --- a/solr/core/src/test/org/apache/solr/search/TestRandomCollapseQParserPlugin.java +++ b/solr/core/src/test/org/apache/solr/search/TestRandomCollapseQParserPlugin.java @@ -268,10 +268,9 @@ public void testParsedFilterQueryResponse() throws Exception { "CollapsingPostFilter(CollapsingPostFilter(field=id, " + "nullPolicy=" + nullPolicy - + ", GroupHeadSelector(selectorText=" + + ", GroupHeadSelector[type=SORT, selectorText=" + groupHeadSort.substring(1, groupHeadSort.length() - 1) - + ", type=SORT" - + "), hint=" + + "], hint=" + collapseHint + ", size=" + collapseSize From b100160a72db46ef7b65d1e1864207147cbd2b2c Mon Sep 17 00:00:00 2001 From: David Smiley Date: Sat, 14 Feb 2026 18:36:28 -0500 Subject: [PATCH 11/21] Introduce AbstractCollapseCollector --- .../solr/search/CollapsingQParserPlugin.java | 213 ++++++++---------- 1 file changed, 93 insertions(+), 120 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index f78018ea508e..33b3380ebd6e 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -737,17 +737,68 @@ public float score() { } /** - * Collapses on Ordinal Values using Score to select the group head. + * Abstract base class for all collapse collectors. Provides common configuration and result state + * shared across ordinal and integer-based collapse strategies. * * @lucene.internal */ - static class OrdScoreCollector extends DelegatingCollector { - + protected abstract static class AbstractCollapseCollector extends DelegatingCollector { // Configuration - private final LeafReaderContext[] contexts; - private final int maxDoc; - private final NullPolicy nullPolicy; - private final boolean collectElevatedDocsWhenCollapsing; + protected final LeafReaderContext[] contexts; + protected final int maxDoc; + protected final NullPolicy nullPolicy; + protected final boolean needsScores; + protected final boolean collectElevatedDocsWhenCollapsing; + + // Results/accumulator + protected final FixedBitSet collapsedSet; + protected final BoostedDocsCollector boostedDocsCollector; + protected FloatArrayList nullScores; + protected float nullScore; + protected int nullDoc = -1; + + protected AbstractCollapseCollector( + IndexSearcher searcher, + NullPolicy nullPolicy, + boolean needsScores, + boolean collectElevatedDocsWhenCollapsing, + IntIntHashMap boostDocsMap) { + + this.contexts = searcher.getTopReaderContext().leaves().toArray(LeafReaderContext[]::new); + this.maxDoc = searcher.getIndexReader().maxDoc(); + this.nullPolicy = nullPolicy; + this.needsScores = needsScores; + this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing; + + this.collapsedSet = new FixedBitSet(maxDoc); + this.boostedDocsCollector = BoostedDocsCollector.build(boostDocsMap); + if (needsScores && nullPolicy == NullPolicy.EXPAND) { + this.nullScores = new FloatArrayList(); + } + } + + @Override + public ScoreMode scoreMode() { + return needsScores ? ScoreMode.COMPLETE : super.scoreMode(); + } + + @Override + protected void doSetNextReader(LeafReaderContext context) throws IOException { + // Do NOT set leafDelegate (calling super() would do this). + // That's handled in complete() for these collectors. + assert this.contexts[context.ord] == context; + assert leafDelegate == null; + this.context = context; + this.docBase = context.docBase; + } + } + + /** + * Collapses on Ordinal Values using Score to select the group head. + * + * @lucene.internal + */ + static class OrdScoreCollector extends AbstractCollapseCollector { // Source data private final DocValuesProducer collapseValuesProducer; @@ -760,11 +811,6 @@ static class OrdScoreCollector extends DelegatingCollector { // Results/accumulator private final IntIntDynamicMap ords; private final IntFloatDynamicMap scores; - private final FixedBitSet collapsedSet; - private final BoostedDocsCollector boostedDocsCollector; - private FloatArrayList nullScores; - private float nullScore = -Float.MAX_VALUE; - private int nullDoc = -1; public OrdScoreCollector( DocValuesProducer collapseValuesProducer, @@ -773,10 +819,8 @@ public OrdScoreCollector( IndexSearcher searcher, boolean collectElevatedDocsWhenCollapsing) throws IOException { - this.contexts = searcher.getTopReaderContext().leaves().toArray(LeafReaderContext[]::new); - this.maxDoc = searcher.getIndexReader().maxDoc(); - this.nullPolicy = nullPolicy; - this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing; + super(searcher, nullPolicy, true, collectElevatedDocsWhenCollapsing, boostDocsMap); + this.collapseValuesProducer = collapseValuesProducer; this.collapseValues = collapseValuesProducer.getSorted(null); @@ -788,22 +832,12 @@ public OrdScoreCollector( this.ords = new IntIntDynamicMap(valueCount, -1); this.scores = new IntFloatDynamicMap(valueCount, -Float.MAX_VALUE); - this.collapsedSet = new FixedBitSet(maxDoc); - this.boostedDocsCollector = BoostedDocsCollector.build(boostDocsMap); - if (nullPolicy == NullPolicy.EXPAND) { - nullScores = new FloatArrayList(); - } - } - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE; + this.nullScore = -Float.MAX_VALUE; } @Override protected void doSetNextReader(LeafReaderContext context) throws IOException { - this.contexts[context.ord] = context; - this.docBase = context.docBase; + super.doSetNextReader(context); if (ordinalMap != null) { this.segmentValues = this.multiSortedDocValues.values[context.ord]; this.segmentOrdinalMap = ordinalMap.getGlobalOrds(context.ord); @@ -961,25 +995,16 @@ public void complete() throws IOException { * * @lucene.internal */ - static class IntScoreCollector extends DelegatingCollector { + static class IntScoreCollector extends AbstractCollapseCollector { // Configuration - private final LeafReaderContext[] contexts; - private final int maxDoc; - private final NullPolicy nullPolicy; private final String field; - private final boolean collectElevatedDocsWhenCollapsing; // Source data private NumericDocValues collapseValues; // Results/accumulator private final IntLongHashMap cmap; - private final FixedBitSet collapsedSet; - private final BoostedDocsCollector boostedDocsCollector; - private FloatArrayList nullScores; - private float nullScore = -Float.MAX_VALUE; - private int nullDoc = -1; public IntScoreCollector( NullPolicy nullPolicy, @@ -988,29 +1013,17 @@ public IntScoreCollector( IntIntHashMap boostDocsMap, SolrIndexSearcher searcher, boolean collectElevatedDocsWhenCollapsing) { - this.contexts = searcher.getTopReaderContext().leaves().toArray(LeafReaderContext[]::new); - this.maxDoc = searcher.maxDoc(); - this.nullPolicy = nullPolicy; + super(searcher, nullPolicy, true, collectElevatedDocsWhenCollapsing, boostDocsMap); + this.field = field; - this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing; this.cmap = new IntLongHashMap(size); - this.collapsedSet = new FixedBitSet(maxDoc); - this.boostedDocsCollector = BoostedDocsCollector.build(boostDocsMap); - if (nullPolicy == NullPolicy.EXPAND) { - nullScores = new FloatArrayList(); - } - } - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE; + this.nullScore = -Float.MAX_VALUE; } @Override protected void doSetNextReader(LeafReaderContext context) throws IOException { - this.contexts[context.ord] = context; - this.docBase = context.docBase; + super.doSetNextReader(context); this.collapseValues = DocValues.getNumeric(context.reader(), this.field); } @@ -1193,14 +1206,7 @@ public OrdFieldValueCollector build( * * @lucene.internal */ - protected abstract static class OrdFieldValueCollector extends DelegatingCollector { - // Configuration - protected final LeafReaderContext[] contexts; - protected final int maxDoc; - protected final NullPolicy nullPolicy; - protected final boolean needsScores; - protected final boolean collectElevatedDocsWhenCollapsing; - + protected abstract static class OrdFieldValueCollector extends AbstractCollapseCollector { // Source data protected final DocValuesProducer collapseValuesProducer; protected SortedDocValues collapseValues; @@ -1212,19 +1218,16 @@ protected abstract static class OrdFieldValueCollector extends DelegatingCollect // Results/accumulator protected final IntIntDynamicMap ords; - protected final FixedBitSet collapsedSet; - protected final BoostedDocsCollector boostedDocsCollector; protected IntFloatDynamicMap scores; - protected FloatArrayList nullScores; - protected float nullScore; - protected int nullDoc = -1; protected OrdFieldValueCollector(OrdFieldCollectorBuilder ctx) throws IOException { - this.contexts = ctx.searcher.getTopReaderContext().leaves().toArray(LeafReaderContext[]::new); - this.maxDoc = ctx.searcher.maxDoc(); - this.nullPolicy = ctx.nullPolicy; - this.needsScores = ctx.needsScores; - this.collectElevatedDocsWhenCollapsing = ctx.collectElevatedDocsWhenCollapsing; + super( + ctx.searcher, + ctx.nullPolicy, + ctx.needsScores, + ctx.collectElevatedDocsWhenCollapsing, + ctx.boostDocsMap); + this.collapseValuesProducer = ctx.collapseValuesProducer; this.collapseValues = ctx.collapseValuesProducer.getSorted(null); @@ -1234,23 +1237,13 @@ protected OrdFieldValueCollector(OrdFieldCollectorBuilder ctx) throws IOExceptio } this.ords = new IntIntDynamicMap(ctx.valueCount, -1); - this.collapsedSet = new FixedBitSet(ctx.searcher.maxDoc()); - this.boostedDocsCollector = BoostedDocsCollector.build(ctx.boostDocsMap); if (this.needsScores) { this.scores = new IntFloatDynamicMap(ctx.valueCount, 0.0f); - if (ctx.nullPolicy == NullPolicy.EXPAND) { - this.nullScores = new FloatArrayList(); - } } else { this.scores = null; } } - @Override - public ScoreMode scoreMode() { - return needsScores ? ScoreMode.COMPLETE : super.scoreMode(); - } - @Override public void setScorer(Scorable scorer) throws IOException { // not calling super here; see complete() instead @@ -1259,9 +1252,7 @@ public void setScorer(Scorable scorer) throws IOException { @Override public void doSetNextReader(LeafReaderContext context) throws IOException { - // not calling super here; see complete() instead - this.contexts[context.ord] = context; - this.docBase = context.docBase; + super.doSetNextReader(context); if (ordinalMap != null) { this.segmentValues = this.multiSortedDocValues.values[context.ord]; this.segmentOrdinalMap = ordinalMap.getGlobalOrds(context.ord); @@ -1413,7 +1404,7 @@ protected static class OrdIntCollector extends OrdFieldValueCollector { private final String field; private final IntCompare comp; - // Source fields (per-segment state) + // Source data private NumericDocValues minMaxValues; // Results/accumulator @@ -1480,7 +1471,7 @@ protected static class OrdFloatCollector extends OrdFieldValueCollector { private final String field; private final FloatCompare comp; - // Source fields (per-segment state) + // Source data private NumericDocValues minMaxValues; // Results/accumulator @@ -1550,7 +1541,7 @@ protected static class OrdLongCollector extends OrdFieldValueCollector { private final String field; private final LongCompare comp; - // Source fields (per-segment state) + // Source data private NumericDocValues minMaxVals; // Results/accumulator @@ -1624,7 +1615,7 @@ protected static class OrdValueSourceCollector extends OrdFieldValueCollector { private final Map rcontext; private final CollapseScore collapseScore = new CollapseScore(); - // Source fields (per-segment state) + // Source data private FunctionValues functionValues; // Results/accumulator @@ -1837,14 +1828,9 @@ public IntFieldValueCollector build( * * @lucene.internal */ - protected abstract static class IntFieldValueCollector extends DelegatingCollector { + protected abstract static class IntFieldValueCollector extends AbstractCollapseCollector { // Configuration - protected final LeafReaderContext[] contexts; - protected final int maxDoc; - protected final NullPolicy nullPolicy; - protected final boolean needsScores; protected final String collapseField; - protected final boolean collectElevatedDocsWhenCollapsing; // Source data protected NumericDocValues collapseValues; @@ -1853,32 +1839,24 @@ protected abstract static class IntFieldValueCollector extends DelegatingCollect // Results/accumulator protected final IntIntHashMap cmap; protected final IntIntDynamicMap docs; - protected final FixedBitSet collapsedSet; - protected final BoostedDocsCollector boostedDocsCollector; protected IntFloatDynamicMap scores; - protected FloatArrayList nullScores; - protected float nullScore; - protected int nullDoc = -1; protected IntFieldValueCollector(IntFieldCollectorBuilder ctx) throws IOException { + super( + ctx.searcher, + ctx.nullPolicy, + ctx.needsScores, + ctx.collectElevatedDocsWhenCollapsing, + ctx.boostDocsMap); + assert !GroupHeadSelectorType.SCORE.equals(ctx.groupHeadSelector.type); - this.contexts = ctx.searcher.getTopReaderContext().leaves().toArray(LeafReaderContext[]::new); - this.maxDoc = ctx.searcher.getIndexReader().maxDoc(); - this.nullPolicy = ctx.nullPolicy; - this.needsScores = ctx.needsScores; this.collapseField = ctx.collapseField; - this.collectElevatedDocsWhenCollapsing = ctx.collectElevatedDocsWhenCollapsing; this.cmap = new IntIntHashMap(ctx.size); this.docs = new IntIntDynamicMap(ctx.size, 0); - this.collapsedSet = new FixedBitSet(this.maxDoc); - this.boostedDocsCollector = BoostedDocsCollector.build(ctx.boostDocsMap); if (this.needsScores) { this.scores = new IntFloatDynamicMap(ctx.size, 0.0f); - if (ctx.nullPolicy == NullPolicy.EXPAND) { - nullScores = new FloatArrayList(); - } } else { this.scores = null; } @@ -1889,11 +1867,6 @@ protected IntFieldValueCollector(IntFieldCollectorBuilder ctx) throws IOExceptio protected abstract void collapse(int collapseKey, int contextDoc, int globalDoc) throws IOException; - @Override - public ScoreMode scoreMode() { - return needsScores ? ScoreMode.COMPLETE : super.scoreMode(); - } - @Override public void setScorer(Scorable scorer) throws IOException { this.scorer = scorer; @@ -1901,8 +1874,7 @@ public void setScorer(Scorable scorer) throws IOException { @Override public void doSetNextReader(LeafReaderContext context) throws IOException { - this.contexts[context.ord] = context; - this.docBase = context.docBase; + super.doSetNextReader(context); this.collapseValues = DocValues.getNumeric(context.reader(), this.collapseField); } @@ -2025,7 +1997,7 @@ protected static class IntIntCollector extends IntFieldValueCollector { private final String field; private final IntCompare comp; - // Source fields (per-segment state) + // Source data private NumericDocValues minMaxVals; // Results/accumulator @@ -2114,7 +2086,7 @@ protected static class IntFloatCollector extends IntFieldValueCollector { private final String field; private final FloatCompare comp; - // Source fields (per-segment state) + // Source data private NumericDocValues minMaxVals; // Results/accumulator @@ -2204,7 +2176,7 @@ protected static class IntValueSourceCollector extends IntFieldValueCollector { private final ValueSource valueSource; private final Map rcontext; - // Source fields (per-segment state) + // Source data private FunctionValues functionValues; // Results/accumulator @@ -2227,6 +2199,7 @@ public IntValueSourceCollector(IntFieldCollectorBuilder ctx, FunctionQuery funcQ } this.valueSource = funcQuery.getValueSource(); this.rcontext = ValueSource.newContext(ctx.searcher); + collapseScore.setupIfNeeded(ctx.groupHeadSelector, rcontext); this.testValues = new IntFloatDynamicMap(ctx.size, 0.0f); From 11b1f7a5b5527076e1ca9bbe3ac63486f52f7bc2 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Sat, 14 Feb 2026 18:56:44 -0500 Subject: [PATCH 12/21] setScorer harmonization --- .../solr/search/CollapsingQParserPlugin.java | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index 33b3380ebd6e..bcb6b1fe375d 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -1214,7 +1214,6 @@ protected abstract static class OrdFieldValueCollector extends AbstractCollapseC protected MultiDocValues.MultiSortedDocValues multiSortedDocValues; protected SortedDocValues segmentValues; protected LongValues segmentOrdinalMap; - protected Scorable scorer; // Results/accumulator protected final IntIntDynamicMap ords; @@ -1244,12 +1243,6 @@ protected OrdFieldValueCollector(OrdFieldCollectorBuilder ctx) throws IOExceptio } } - @Override - public void setScorer(Scorable scorer) throws IOException { - // not calling super here; see complete() instead - this.scorer = scorer; - } - @Override public void doSetNextReader(LeafReaderContext context) throws IOException { super.doSetNextReader(context); @@ -1834,7 +1827,6 @@ protected abstract static class IntFieldValueCollector extends AbstractCollapseC // Source data protected NumericDocValues collapseValues; - protected Scorable scorer; // Results/accumulator protected final IntIntHashMap cmap; @@ -1867,11 +1859,6 @@ protected IntFieldValueCollector(IntFieldCollectorBuilder ctx) throws IOExceptio protected abstract void collapse(int collapseKey, int contextDoc, int globalDoc) throws IOException; - @Override - public void setScorer(Scorable scorer) throws IOException { - this.scorer = scorer; - } - @Override public void doSetNextReader(LeafReaderContext context) throws IOException { super.doSetNextReader(context); @@ -2751,8 +2738,8 @@ public AbstractBlockSortSpecCollector( @Override public void setScorer(Scorable scorer) throws IOException { - sortsCompare.setScorer(scorer); super.setScorer(scorer); + sortsCompare.setScorer(scorer); } private void setCurrentGroupBestMatch(final int contextDocId, final boolean isBoosted) From 1823e603f98058a0eb595e6112d5bed8976b8509 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Sat, 14 Feb 2026 22:07:22 -0500 Subject: [PATCH 13/21] factor out a common complete() --- .../solr/search/CollapsingQParserPlugin.java | 437 ++++++------------ 1 file changed, 152 insertions(+), 285 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index bcb6b1fe375d..d70ac3391f50 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -791,6 +791,76 @@ protected void doSetNextReader(LeafReaderContext context) throws IOException { this.context = context; this.docBase = context.docBase; } + + /** + * Build the collapsed doc ID set. Implementations should purge boosted docs, populate + * collapsedSet with group head doc IDs, and reset any doc values iterators needed for the + * complete() replay pass. + */ + protected abstract DocIdSetIterator getCollapsedDisi() throws IOException; + + /** + * Update per-segment state for the given segment during the complete() replay pass. Called for + * every segment visited, including the first. + */ + protected abstract void advanceCompleteSegment(int contextIndex) throws IOException; + + /** + * Look up the stored score for a collapsed doc during the complete() replay pass. Returns the + * score if the doc is in a non-null group, or {@link Float#NaN} for null-group docs. + */ + protected abstract float getGroupHeadScore(int globalDoc, int contextDoc) throws IOException; + + @Override + public final void complete() throws IOException { + if (contexts.length == 0) { + return; + } + + DocIdSetIterator collapsedDocs = getCollapsedDisi(); + + int currentContext = -1; + int currentDocBase = 0; + int nextDocBase = 0; + ScoreAndDoc dummy = new ScoreAndDoc(); + final MergeBoost mergeBoost = boostedDocsCollector.getMergeBoost(); + int nullScoreIndex = 0; + int globalDoc; + + while ((globalDoc = collapsedDocs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + while (globalDoc >= nextDocBase) { + currentContext++; + currentDocBase = contexts[currentContext].docBase; + nextDocBase = + currentContext + 1 < contexts.length ? contexts[currentContext + 1].docBase : maxDoc; + leafDelegate = delegate.getLeafCollector(contexts[currentContext]); + leafDelegate.setScorer(dummy); + advanceCompleteSegment(currentContext); + } + + int contextDoc = globalDoc - currentDocBase; + + if (needsScores) { + float score = getGroupHeadScore(globalDoc, contextDoc); + if (!Float.isNaN(score)) { + dummy.score = score; + } else if (mergeBoost.boost(globalDoc)) { + dummy.score = 0F; + } else if (nullPolicy == NullPolicy.COLLAPSE) { + dummy.score = nullScore; + } else if (nullPolicy == NullPolicy.EXPAND) { + dummy.score = nullScores.get(nullScoreIndex++); + } + } + + dummy.docId = contextDoc; + leafDelegate.collect(contextDoc); + } + + if (delegate instanceof DelegatingCollector) { + ((DelegatingCollector) delegate).complete(); + } + } } /** @@ -894,99 +964,51 @@ public void collect(int contextDoc) throws IOException { } @Override - public void complete() throws IOException { - if (contexts.length == 0) { - return; - } - - // Handle the boosted docs. + protected DocIdSetIterator getCollapsedDisi() throws IOException { boostedDocsCollector.purgeGroupsThatHaveBoostedDocs( - collapsedSet, - (ord) -> { - ords.remove(ord); - }, - () -> { - nullDoc = -1; - }); + collapsedSet, ords::remove, () -> nullDoc = -1); - // Build the sorted DocSet of group heads. if (nullDoc > -1) { collapsedSet.set(nullDoc); } ords.forEachValue(doc -> collapsedSet.set(doc)); - int currentContext = 0; - int currentDocBase = 0; - - collapseValues = collapseValuesProducer.getSorted(null); // reset iterator - + // Reset iterators for the complete pass + collapseValues = collapseValuesProducer.getSorted(null); if (collapseValues instanceof MultiDocValues.MultiSortedDocValues) { this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues) collapseValues; this.ordinalMap = multiSortedDocValues.mapping; } + return new BitSetIterator(collapsedSet, 0); + } + + @Override + protected void advanceCompleteSegment(int contextIndex) { if (ordinalMap != null) { - this.segmentValues = this.multiSortedDocValues.values[currentContext]; - this.segmentOrdinalMap = this.ordinalMap.getGlobalOrds(currentContext); + this.segmentValues = this.multiSortedDocValues.values[contextIndex]; + this.segmentOrdinalMap = this.ordinalMap.getGlobalOrds(contextIndex); } else { this.segmentValues = collapseValues; } + } - int nextDocBase = - currentContext + 1 < contexts.length ? contexts[currentContext + 1].docBase : maxDoc; - leafDelegate = delegate.getLeafCollector(contexts[currentContext]); - ScoreAndDoc dummy = new ScoreAndDoc(); - leafDelegate.setScorer(dummy); - DocIdSetIterator it = new BitSetIterator(collapsedSet, 0L); // cost is not useful here - final MergeBoost mergeBoost = boostedDocsCollector.getMergeBoost(); - int docId = -1; - int index = -1; - while ((docId = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { - while (docId >= nextDocBase) { - currentContext++; - currentDocBase = contexts[currentContext].docBase; - nextDocBase = - currentContext + 1 < contexts.length ? contexts[currentContext + 1].docBase : maxDoc; - leafDelegate = delegate.getLeafCollector(contexts[currentContext]); - leafDelegate.setScorer(dummy); - if (ordinalMap != null) { - this.segmentValues = this.multiSortedDocValues.values[currentContext]; - this.segmentOrdinalMap = this.ordinalMap.getGlobalOrds(currentContext); - } - } - - int contextDoc = docId - currentDocBase; - - int ord = -1; - if (this.ordinalMap != null) { - // Handle ordinalMapping case - if (segmentValues.advanceExact(contextDoc)) { - ord = (int) segmentOrdinalMap.get(segmentValues.ordValue()); - } - } else { - // Handle top Level FieldCache or Single Segment Case - if (segmentValues.advanceExact(docId)) { - ord = segmentValues.ordValue(); - } + @Override + protected float getGroupHeadScore(int globalDoc, int contextDoc) throws IOException { + int ord = -1; + if (this.ordinalMap != null) { + if (segmentValues.advanceExact(contextDoc)) { + ord = (int) segmentOrdinalMap.get(segmentValues.ordValue()); } - - if (ord > -1) { - dummy.score = scores.get(ord); - } else if (mergeBoost.boost(docId)) { - // Ignore so it doesn't mess up the null scoring. - } else if (this.nullPolicy == NullPolicy.COLLAPSE) { - dummy.score = nullScore; - } else if (this.nullPolicy == NullPolicy.EXPAND) { - dummy.score = nullScores.get(++index); + } else { + if (segmentValues.advanceExact(globalDoc)) { + ord = segmentValues.ordValue(); } - - dummy.docId = contextDoc; - leafDelegate.collect(contextDoc); } - - if (delegate instanceof DelegatingCollector) { - ((DelegatingCollector) delegate).complete(); + if (ord > -1) { + return scores.get(ord); } + return Float.NaN; } } @@ -1077,82 +1099,33 @@ public void collect(int contextDoc) throws IOException { } @Override - public void complete() throws IOException { - if (contexts.length == 0) { - return; - } - - // Handle the boosted docs. + protected DocIdSetIterator getCollapsedDisi() { boostedDocsCollector.purgeGroupsThatHaveBoostedDocs( - collapsedSet, - (key) -> { - cmap.remove(key); - }, - () -> { - nullDoc = -1; - }); + collapsedSet, cmap::remove, () -> nullDoc = -1); - // Build the sorted DocSet of group heads. if (nullDoc > -1) { collapsedSet.set(nullDoc); } - Iterator it1 = cmap.iterator(); - while (it1.hasNext()) { - IntLongCursor cursor = it1.next(); - int doc = (int) cursor.value; - collapsedSet.set(doc); + for (IntLongCursor cursor : cmap) { + collapsedSet.set((int) cursor.value); } - int currentContext = 0; - int currentDocBase = 0; - - collapseValues = DocValues.getNumeric(contexts[currentContext].reader(), this.field); - int nextDocBase = - currentContext + 1 < contexts.length ? contexts[currentContext + 1].docBase : maxDoc; - leafDelegate = delegate.getLeafCollector(contexts[currentContext]); - ScoreAndDoc dummy = new ScoreAndDoc(); - leafDelegate.setScorer(dummy); - DocIdSetIterator it = new BitSetIterator(collapsedSet, 0L); // cost is not useful here - final MergeBoost mergeBoost = boostedDocsCollector.getMergeBoost(); - int globalDoc = -1; - int nullScoreIndex = 0; - while ((globalDoc = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { - - while (globalDoc >= nextDocBase) { - currentContext++; - currentDocBase = contexts[currentContext].docBase; - nextDocBase = - currentContext + 1 < contexts.length ? contexts[currentContext + 1].docBase : maxDoc; - leafDelegate = delegate.getLeafCollector(contexts[currentContext]); - leafDelegate.setScorer(dummy); - collapseValues = DocValues.getNumeric(contexts[currentContext].reader(), this.field); - } - - final int contextDoc = globalDoc - currentDocBase; - if (collapseValues.advanceExact(contextDoc)) { - final int collapseValue = (int) collapseValues.longValue(); - final long scoreDoc = cmap.get(collapseValue); - dummy.score = Float.intBitsToFloat((int) (scoreDoc >> 32)); - - } else { // Null Group... - - if (mergeBoost.boost(globalDoc)) { - // It's an elevated doc so no score is needed (and should not have been populated) - dummy.score = 0F; - } else if (nullPolicy == NullPolicy.COLLAPSE) { - dummy.score = nullScore; - } else if (nullPolicy == NullPolicy.EXPAND) { - dummy.score = nullScores.get(nullScoreIndex++); - } - } + return new BitSetIterator(collapsedSet, 0); + } - dummy.docId = contextDoc; - leafDelegate.collect(contextDoc); - } + @Override + protected void advanceCompleteSegment(int contextIndex) throws IOException { + this.collapseValues = DocValues.getNumeric(contexts[contextIndex].reader(), this.field); + } - if (delegate instanceof DelegatingCollector) { - ((DelegatingCollector) delegate).complete(); + @Override + protected float getGroupHeadScore(int globalDoc, int contextDoc) throws IOException { + if (collapseValues.advanceExact(contextDoc)) { + int collapseValue = (int) collapseValues.longValue(); + long scoreDoc = cmap.get(collapseValue); + return Float.intBitsToFloat((int) (scoreDoc >> 32)); } + return Float.NaN; } } @@ -1287,107 +1260,54 @@ public void collect(int contextDoc) throws IOException { */ protected abstract void collapse(int ord, int contextDoc, int globalDoc) throws IOException; - protected DocIdSetIterator getCollapsedDisi() { - // Handle the boosted docs. + @Override + protected DocIdSetIterator getCollapsedDisi() throws IOException { boostedDocsCollector.purgeGroupsThatHaveBoostedDocs( - collapsedSet, - (ord) -> { - ords.remove(ord); - }, - () -> { - nullDoc = -1; - }); + collapsedSet, ords::remove, () -> nullDoc = -1); - // Build the sorted DocSet of group heads. if (nullDoc > -1) { - this.collapsedSet.set(nullDoc); + collapsedSet.set(nullDoc); } ords.forEachValue(doc -> collapsedSet.set(doc)); - return new BitSetIterator(collapsedSet, 0); // cost is not useful here - } - - @Override - public void complete() throws IOException { - if (contexts.length == 0) { - return; - } - - int currentContext = 0; - int currentDocBase = 0; - - this.collapseValues = collapseValuesProducer.getSorted(null); // reset iterator + // Reset iterators for the complete pass + collapseValues = collapseValuesProducer.getSorted(null); if (collapseValues instanceof MultiDocValues.MultiSortedDocValues) { this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues) collapseValues; this.ordinalMap = multiSortedDocValues.mapping; } + + return new BitSetIterator(collapsedSet, 0); + } + + @Override + protected void advanceCompleteSegment(int contextIndex) { if (ordinalMap != null) { - this.segmentValues = this.multiSortedDocValues.values[currentContext]; - this.segmentOrdinalMap = this.ordinalMap.getGlobalOrds(currentContext); + this.segmentValues = this.multiSortedDocValues.values[contextIndex]; + this.segmentOrdinalMap = this.ordinalMap.getGlobalOrds(contextIndex); } else { this.segmentValues = collapseValues; } + } - int nextDocBase = - currentContext + 1 < contexts.length ? contexts[currentContext + 1].docBase : maxDoc; - leafDelegate = delegate.getLeafCollector(contexts[currentContext]); - ScoreAndDoc dummy = new ScoreAndDoc(); - leafDelegate.setScorer(dummy); - DocIdSetIterator it = getCollapsedDisi(); - int globalDoc = -1; - int nullScoreIndex = 0; - final MergeBoost mergeBoost = boostedDocsCollector.getMergeBoost(); - - while ((globalDoc = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { - - while (globalDoc >= nextDocBase) { - currentContext++; - currentDocBase = contexts[currentContext].docBase; - nextDocBase = - currentContext + 1 < contexts.length ? contexts[currentContext + 1].docBase : maxDoc; - leafDelegate = delegate.getLeafCollector(contexts[currentContext]); - leafDelegate.setScorer(dummy); - if (ordinalMap != null) { - this.segmentValues = this.multiSortedDocValues.values[currentContext]; - this.segmentOrdinalMap = this.ordinalMap.getGlobalOrds(currentContext); - } + @Override + protected float getGroupHeadScore(int globalDoc, int contextDoc) throws IOException { + int ord = -1; + if (this.ordinalMap != null) { + // Handle ordinalMapping case + if (segmentValues.advanceExact(contextDoc)) { + ord = (int) segmentOrdinalMap.get(segmentValues.ordValue()); } - - int contextDoc = globalDoc - currentDocBase; - - if (this.needsScores) { - int ord = -1; - if (this.ordinalMap != null) { - // Handle ordinalMapping case - if (segmentValues.advanceExact(contextDoc)) { - ord = (int) segmentOrdinalMap.get(segmentValues.ordValue()); - } - } else { - // Handle top Level FieldCache or Single Segment Case - if (segmentValues.advanceExact(globalDoc)) { - ord = segmentValues.ordValue(); - } - } - - if (ord > -1) { - dummy.score = this.scores.get(ord); - } else if (mergeBoost.boost(globalDoc)) { - // It's an elevated doc so no score is needed (and should not have been populated) - dummy.score = 0F; - } else if (nullPolicy == NullPolicy.COLLAPSE) { - dummy.score = this.nullScore; - } else if (nullPolicy == NullPolicy.EXPAND) { - dummy.score = this.nullScores.get(nullScoreIndex++); - } + } else { + // Handle top Level FieldCache or Single Segment Case + if (segmentValues.advanceExact(globalDoc)) { + ord = segmentValues.ordValue(); } - - dummy.docId = contextDoc; - leafDelegate.collect(contextDoc); } - - if (delegate instanceof DelegatingCollector) { - ((DelegatingCollector) delegate).complete(); + if (ord > -1) { + return scores.get(ord); } + return Float.NaN; } } @@ -1888,16 +1808,11 @@ public void collect(int contextDoc) throws IOException { } } + @Override protected DocIdSetIterator getCollapsedDisi() { // Handle the boosted docs. boostedDocsCollector.purgeGroupsThatHaveBoostedDocs( - collapsedSet, - (key) -> { - cmap.remove(key); - }, - () -> { - nullDoc = -1; - }); + collapsedSet, cmap::remove, () -> nullDoc = -1); // Build the sorted DocSet of group heads. if (nullDoc > -1) { @@ -1912,67 +1827,19 @@ protected DocIdSetIterator getCollapsedDisi() { } @Override - public void complete() throws IOException { - if (contexts.length == 0) { - return; - } - - int currentContext = 0; - int currentDocBase = 0; + protected void advanceCompleteSegment(int contextIndex) throws IOException { this.collapseValues = - DocValues.getNumeric(contexts[currentContext].reader(), this.collapseField); - int nextDocBase = - currentContext + 1 < contexts.length ? contexts[currentContext + 1].docBase : maxDoc; - leafDelegate = delegate.getLeafCollector(contexts[currentContext]); - ScoreAndDoc dummy = new ScoreAndDoc(); - leafDelegate.setScorer(dummy); - DocIdSetIterator it = getCollapsedDisi(); - int globalDoc = -1; - int nullScoreIndex = 0; - final MergeBoost mergeBoost = boostedDocsCollector.getMergeBoost(); - - while ((globalDoc = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { - - while (globalDoc >= nextDocBase) { - currentContext++; - currentDocBase = contexts[currentContext].docBase; - nextDocBase = - currentContext + 1 < contexts.length ? contexts[currentContext + 1].docBase : maxDoc; - leafDelegate = delegate.getLeafCollector(contexts[currentContext]); - leafDelegate.setScorer(dummy); - this.collapseValues = - DocValues.getNumeric(contexts[currentContext].reader(), this.collapseField); - } - - final int contextDoc = globalDoc - currentDocBase; - - if (this.needsScores) { - if (collapseValues.advanceExact(contextDoc)) { - final int collapseValue = (int) collapseValues.longValue(); - - final int pointer = cmap.get(collapseValue); - dummy.score = scores.get(pointer); - - } else { // Null Group... - - if (mergeBoost.boost(globalDoc)) { - // It's an elevated doc so no score is needed (and should not have been populated) - dummy.score = 0F; - } else if (nullPolicy == NullPolicy.COLLAPSE) { - dummy.score = nullScore; - } else if (nullPolicy == NullPolicy.EXPAND) { - dummy.score = nullScores.get(nullScoreIndex++); - } - } - } - - dummy.docId = contextDoc; - leafDelegate.collect(contextDoc); - } + DocValues.getNumeric(contexts[contextIndex].reader(), this.collapseField); + } - if (delegate instanceof DelegatingCollector) { - ((DelegatingCollector) delegate).complete(); + @Override + protected float getGroupHeadScore(int globalDoc, int contextDoc) throws IOException { + if (collapseValues.advanceExact(contextDoc)) { + final int collapseValue = (int) collapseValues.longValue(); + final int pointer = cmap.get(collapseValue); + return scores.get(pointer); } + return Float.NaN; } } From 056b6002023dc9ce7ba366d7d9f68c14f9cf17fe Mon Sep 17 00:00:00 2001 From: David Smiley Date: Sat, 14 Feb 2026 22:27:10 -0500 Subject: [PATCH 14/21] contexts is now a List not array --- .../solr/search/CollapsingQParserPlugin.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index d70ac3391f50..932ce6f0abfc 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -744,7 +744,7 @@ public float score() { */ protected abstract static class AbstractCollapseCollector extends DelegatingCollector { // Configuration - protected final LeafReaderContext[] contexts; + protected final List contexts; protected final int maxDoc; protected final NullPolicy nullPolicy; protected final boolean needsScores; @@ -764,7 +764,7 @@ protected AbstractCollapseCollector( boolean collectElevatedDocsWhenCollapsing, IntIntHashMap boostDocsMap) { - this.contexts = searcher.getTopReaderContext().leaves().toArray(LeafReaderContext[]::new); + this.contexts = searcher.getTopReaderContext().leaves(); this.maxDoc = searcher.getIndexReader().maxDoc(); this.nullPolicy = nullPolicy; this.needsScores = needsScores; @@ -786,7 +786,7 @@ public ScoreMode scoreMode() { protected void doSetNextReader(LeafReaderContext context) throws IOException { // Do NOT set leafDelegate (calling super() would do this). // That's handled in complete() for these collectors. - assert this.contexts[context.ord] == context; + assert this.contexts.get(context.ord) == context; assert leafDelegate == null; this.context = context; this.docBase = context.docBase; @@ -813,7 +813,7 @@ protected void doSetNextReader(LeafReaderContext context) throws IOException { @Override public final void complete() throws IOException { - if (contexts.length == 0) { + if (contexts.isEmpty()) { return; } @@ -830,10 +830,12 @@ public final void complete() throws IOException { while ((globalDoc = collapsedDocs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { while (globalDoc >= nextDocBase) { currentContext++; - currentDocBase = contexts[currentContext].docBase; + currentDocBase = contexts.get(currentContext).docBase; nextDocBase = - currentContext + 1 < contexts.length ? contexts[currentContext + 1].docBase : maxDoc; - leafDelegate = delegate.getLeafCollector(contexts[currentContext]); + currentContext + 1 < contexts.size() + ? contexts.get(currentContext + 1).docBase + : maxDoc; + leafDelegate = delegate.getLeafCollector(contexts.get(currentContext)); leafDelegate.setScorer(dummy); advanceCompleteSegment(currentContext); } @@ -1115,7 +1117,7 @@ protected DocIdSetIterator getCollapsedDisi() { @Override protected void advanceCompleteSegment(int contextIndex) throws IOException { - this.collapseValues = DocValues.getNumeric(contexts[contextIndex].reader(), this.field); + this.collapseValues = DocValues.getNumeric(contexts.get(contextIndex).reader(), this.field); } @Override @@ -1829,7 +1831,7 @@ protected DocIdSetIterator getCollapsedDisi() { @Override protected void advanceCompleteSegment(int contextIndex) throws IOException { this.collapseValues = - DocValues.getNumeric(contexts[contextIndex].reader(), this.collapseField); + DocValues.getNumeric(contexts.get(contextIndex).reader(), this.collapseField); } @Override From 4ef267c3515c13fc5ee1f3c65c2f608b07e3b4c2 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Sun, 15 Feb 2026 15:50:30 -0500 Subject: [PATCH 15/21] =?UTF-8?q?refactors=20AbstractCollapseCollector.com?= =?UTF-8?q?plete()=20into=20a=20template=20method=20pattern:=20=20=201.=20?= =?UTF-8?q?complete()=20is=20now=20final=20on=20the=20base=20class,=20with?= =?UTF-8?q?=20abstract=20callbacks:=20getCollapsedDisi()=20(returns=20coll?= =?UTF-8?q?apsed=20doc=20IDs)=20and=20getCollapsedScores()=20(returns=20a?= =?UTF-8?q?=20Scorable=20per=20segment)=20=20=202.=20CachedScoreScorable?= =?UTF-8?q?=20=E2=80=94=20new=20abstract=20Scorable=20subclass=20that=20ca?= =?UTF-8?q?ches=20score()=20by=20doc=20ID,=20delegating=20to=20computeScor?= =?UTF-8?q?e(int=20globalDoc)=20=20=203.=20scoreNullGroup()=20=E2=80=94=20?= =?UTF-8?q?base=20class=20helper=20extracting=20null-group=20score=20logic?= =?UTF-8?q?=20from=20complete()=20=20=204.=20contexts=20changed=20from=20L?= =?UTF-8?q?eafReaderContext[]=20to=20List=20=20=205.=20?= =?UTF-8?q?Scoring=20setup=20moved=20from=20getCollapsedDisi()=20to=20getC?= =?UTF-8?q?ollapsedScores()=20(lazy=20init)=20in=20both=20Ord=20collectors?= =?UTF-8?q?=20=20=206.=20Removed=20advanceCompleteSegment()=20and=20getGro?= =?UTF-8?q?upHeadScore()=20abstract=20methods=20(replaced=20by=20getCollap?= =?UTF-8?q?sedScores)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../solr/search/CollapsingQParserPlugin.java | 324 ++++++++++-------- 1 file changed, 180 insertions(+), 144 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index 932ce6f0abfc..e370df2b3c6c 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -48,6 +48,7 @@ import org.apache.lucene.index.MultiDocValues; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.OrdinalMap; +import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.queries.function.FunctionQuery; import org.apache.lucene.queries.function.FunctionValues; @@ -736,6 +737,28 @@ public float score() { } } + private abstract static class CachedScoreScorable extends Scorable { + private final DocIdSetIterator disi; + private int cachedDocId = -1; + private float cachedScore; + + CachedScoreScorable(DocIdSetIterator disi) { + this.disi = disi; + } + + @Override + public final float score() throws IOException { + int docId = disi.docID(); + if (docId != cachedDocId) { + cachedDocId = docId; + cachedScore = computeScore(docId); + } + return cachedScore; + } + + protected abstract float computeScore(int globalDoc) throws IOException; + } + /** * Abstract base class for all collapse collectors. Provides common configuration and result state * shared across ordinal and integer-based collapse strategies. @@ -792,25 +815,7 @@ protected void doSetNextReader(LeafReaderContext context) throws IOException { this.docBase = context.docBase; } - /** - * Build the collapsed doc ID set. Implementations should purge boosted docs, populate - * collapsedSet with group head doc IDs, and reset any doc values iterators needed for the - * complete() replay pass. - */ - protected abstract DocIdSetIterator getCollapsedDisi() throws IOException; - - /** - * Update per-segment state for the given segment during the complete() replay pass. Called for - * every segment visited, including the first. - */ - protected abstract void advanceCompleteSegment(int contextIndex) throws IOException; - - /** - * Look up the stored score for a collapsed doc during the complete() replay pass. Returns the - * score if the doc is in a non-null group, or {@link Float#NaN} for null-group docs. - */ - protected abstract float getGroupHeadScore(int globalDoc, int contextDoc) throws IOException; - + /** Core/general algorithm */ @Override public final void complete() throws IOException { if (contexts.isEmpty()) { @@ -819,50 +824,69 @@ public final void complete() throws IOException { DocIdSetIterator collapsedDocs = getCollapsedDisi(); - int currentContext = -1; - int currentDocBase = 0; int nextDocBase = 0; - ScoreAndDoc dummy = new ScoreAndDoc(); - final MergeBoost mergeBoost = boostedDocsCollector.getMergeBoost(); - int nullScoreIndex = 0; int globalDoc; while ((globalDoc = collapsedDocs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { - while (globalDoc >= nextDocBase) { - currentContext++; - currentDocBase = contexts.get(currentContext).docBase; - nextDocBase = - currentContext + 1 < contexts.size() - ? contexts.get(currentContext + 1).docBase - : maxDoc; - leafDelegate = delegate.getLeafCollector(contexts.get(currentContext)); - leafDelegate.setScorer(dummy); - advanceCompleteSegment(currentContext); - } - - int contextDoc = globalDoc - currentDocBase; + if (globalDoc >= nextDocBase) { + // finish previous leaf + if (leafDelegate != null) { + leafDelegate.finish(); + } - if (needsScores) { - float score = getGroupHeadScore(globalDoc, contextDoc); - if (!Float.isNaN(score)) { - dummy.score = score; - } else if (mergeBoost.boost(globalDoc)) { - dummy.score = 0F; - } else if (nullPolicy == NullPolicy.COLLAPSE) { - dummy.score = nullScore; - } else if (nullPolicy == NullPolicy.EXPAND) { - dummy.score = nullScores.get(nullScoreIndex++); + int ctxIdx = ReaderUtil.subIndex(globalDoc, contexts); + context = contexts.get(ctxIdx); + docBase = context.docBase; + nextDocBase = ctxIdx + 1 < contexts.size() ? contexts.get(ctxIdx + 1).docBase : maxDoc; + leafDelegate = delegate.getLeafCollector(context); + if (delegate.scoreMode().needsScores()) { + leafDelegate.setScorer(getCollapsedScores(collapsedDocs, context)); } } - dummy.docId = contextDoc; - leafDelegate.collect(contextDoc); + leafDelegate.collect(globalDoc - docBase); + } + + if (leafDelegate != null) { + leafDelegate.finish(); } if (delegate instanceof DelegatingCollector) { ((DelegatingCollector) delegate).complete(); } } + + /** + * Return a DISI of global doc IDs that the collector has matched. This is the first step of + * {@link #complete()}. + */ + protected abstract DocIdSetIterator getCollapsedDisi() throws IOException; + + /** + * Return a {@link Scorable} that provides the score for the current doc in the given {@code + * disi} (global doc IDs). Called once per segment during {@link #complete()}. + */ + protected abstract Scorable getCollapsedScores(DocIdSetIterator disi, LeafReaderContext context) + throws IOException; + + private MergeBoost mergeBoost; + private int nullScoreIndex = 0; // next nullScores index to return from scoreNullGroup if EXPAND + + /** Returns the score for a null-group doc during {@link #complete()}. Only called once. */ + protected float scoreNullGroup(int globalDoc) { + if (mergeBoost == null) { // lazy init + mergeBoost = boostedDocsCollector.getMergeBoost(); + } + if (mergeBoost.boost(globalDoc)) { // has side effect + return 0F; + } else if (nullPolicy == NullPolicy.COLLAPSE) { + return nullScore; + } else if (nullPolicy == NullPolicy.EXPAND) { + return nullScores.get(nullScoreIndex++); // side effect + } + return 0F; + } + } /** @@ -966,51 +990,58 @@ public void collect(int contextDoc) throws IOException { } @Override - protected DocIdSetIterator getCollapsedDisi() throws IOException { + protected DocIdSetIterator getCollapsedDisi() { boostedDocsCollector.purgeGroupsThatHaveBoostedDocs( collapsedSet, ords::remove, () -> nullDoc = -1); if (nullDoc > -1) { collapsedSet.set(nullDoc); } - ords.forEachValue(doc -> collapsedSet.set(doc)); - // Reset iterators for the complete pass - collapseValues = collapseValuesProducer.getSorted(null); - if (collapseValues instanceof MultiDocValues.MultiSortedDocValues) { - this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues) collapseValues; - this.ordinalMap = multiSortedDocValues.mapping; - } + ords.forEachValue(collapsedSet::set); return new BitSetIterator(collapsedSet, 0); } + private boolean collapsedScoresInitialized; + @Override - protected void advanceCompleteSegment(int contextIndex) { + protected Scorable getCollapsedScores(DocIdSetIterator disi, LeafReaderContext context) + throws IOException { + if (!collapsedScoresInitialized) { + collapsedScoresInitialized = true; + collapseValues = collapseValuesProducer.getSorted(null); + if (collapseValues instanceof MultiDocValues.MultiSortedDocValues) { + this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues) collapseValues; + this.ordinalMap = multiSortedDocValues.mapping; + } + } if (ordinalMap != null) { - this.segmentValues = this.multiSortedDocValues.values[contextIndex]; - this.segmentOrdinalMap = this.ordinalMap.getGlobalOrds(contextIndex); + this.segmentValues = this.multiSortedDocValues.values[context.ord]; + this.segmentOrdinalMap = this.ordinalMap.getGlobalOrds(context.ord); } else { this.segmentValues = collapseValues; } - } - - @Override - protected float getGroupHeadScore(int globalDoc, int contextDoc) throws IOException { - int ord = -1; - if (this.ordinalMap != null) { - if (segmentValues.advanceExact(contextDoc)) { - ord = (int) segmentOrdinalMap.get(segmentValues.ordValue()); - } - } else { - if (segmentValues.advanceExact(globalDoc)) { - ord = segmentValues.ordValue(); + return new CachedScoreScorable(disi) { + @Override + protected float computeScore(int globalDoc) throws IOException { + int ord = -1; + if (ordinalMap != null) { + int contextDoc = globalDoc - context.docBase; + if (segmentValues.advanceExact(contextDoc)) { + ord = (int) segmentOrdinalMap.get(segmentValues.ordValue()); + } + } else { + if (segmentValues.advanceExact(globalDoc)) { + ord = segmentValues.ordValue(); + } + } + if (ord > -1) { + return scores.get(ord); + } + return scoreNullGroup(globalDoc); } - } - if (ord > -1) { - return scores.get(ord); - } - return Float.NaN; + }; } } @@ -1058,8 +1089,7 @@ public void collect(int contextDoc) throws IOException { final int collapseValue = (int) collapseValues.longValue(); if (collectElevatedDocsWhenCollapsing) { - // Check to see if we have documents boosted by the QueryElevationComponent (skip normal - // strategy based collection) + // Check to see if we have documents boosted by the QueryElevationComponent if (boostedDocsCollector.collectIfBoosted(collapseValue, globalDoc)) return; } @@ -1082,8 +1112,7 @@ public void collect(int contextDoc) throws IOException { } else { // Null Group... if (collectElevatedDocsWhenCollapsing) { - // Check to see if we have documents boosted by the QueryElevationComponent (skip normal - // strategy based collection) + // Check to see if we have documents boosted by the QueryElevationComponent if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return; } @@ -1108,6 +1137,7 @@ protected DocIdSetIterator getCollapsedDisi() { if (nullDoc > -1) { collapsedSet.set(nullDoc); } + for (IntLongCursor cursor : cmap) { collapsedSet.set((int) cursor.value); } @@ -1116,18 +1146,21 @@ protected DocIdSetIterator getCollapsedDisi() { } @Override - protected void advanceCompleteSegment(int contextIndex) throws IOException { - this.collapseValues = DocValues.getNumeric(contexts.get(contextIndex).reader(), this.field); - } - - @Override - protected float getGroupHeadScore(int globalDoc, int contextDoc) throws IOException { - if (collapseValues.advanceExact(contextDoc)) { - int collapseValue = (int) collapseValues.longValue(); - long scoreDoc = cmap.get(collapseValue); - return Float.intBitsToFloat((int) (scoreDoc >> 32)); - } - return Float.NaN; + protected Scorable getCollapsedScores(DocIdSetIterator disi, LeafReaderContext context) + throws IOException { + this.collapseValues = DocValues.getNumeric(context.reader(), this.field); + return new CachedScoreScorable(disi) { + @Override + protected float computeScore(int globalDoc) throws IOException { + int contextDoc = globalDoc - context.docBase; + if (collapseValues.advanceExact(contextDoc)) { + int collapseValue = (int) collapseValues.longValue(); + long scoreDoc = cmap.get(collapseValue); + return Float.intBitsToFloat((int) (scoreDoc >> 32)); + } + return scoreNullGroup(globalDoc); + } + }; } } @@ -1244,8 +1277,7 @@ public void collect(int contextDoc) throws IOException { } if (collectElevatedDocsWhenCollapsing) { - // Check to see if we have documents boosted by the QueryElevationComponent (skip normal - // strategy based collection) + // Check to see if we have documents boosted by the QueryElevationComponent if (-1 == ord) { if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return; } else { @@ -1263,53 +1295,58 @@ public void collect(int contextDoc) throws IOException { protected abstract void collapse(int ord, int contextDoc, int globalDoc) throws IOException; @Override - protected DocIdSetIterator getCollapsedDisi() throws IOException { + protected DocIdSetIterator getCollapsedDisi() { boostedDocsCollector.purgeGroupsThatHaveBoostedDocs( collapsedSet, ords::remove, () -> nullDoc = -1); if (nullDoc > -1) { collapsedSet.set(nullDoc); } - ords.forEachValue(doc -> collapsedSet.set(doc)); - // Reset iterators for the complete pass - collapseValues = collapseValuesProducer.getSorted(null); - if (collapseValues instanceof MultiDocValues.MultiSortedDocValues) { - this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues) collapseValues; - this.ordinalMap = multiSortedDocValues.mapping; - } + ords.forEachValue(collapsedSet::set); return new BitSetIterator(collapsedSet, 0); } + private boolean collapsedScoresInitialized; + @Override - protected void advanceCompleteSegment(int contextIndex) { + protected Scorable getCollapsedScores(DocIdSetIterator disi, LeafReaderContext context) + throws IOException { + if (!collapsedScoresInitialized) { + collapsedScoresInitialized = true; + collapseValues = collapseValuesProducer.getSorted(null); + if (collapseValues instanceof MultiDocValues.MultiSortedDocValues) { + this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues) collapseValues; + this.ordinalMap = multiSortedDocValues.mapping; + } + } if (ordinalMap != null) { - this.segmentValues = this.multiSortedDocValues.values[contextIndex]; - this.segmentOrdinalMap = this.ordinalMap.getGlobalOrds(contextIndex); + this.segmentValues = this.multiSortedDocValues.values[context.ord]; + this.segmentOrdinalMap = this.ordinalMap.getGlobalOrds(context.ord); } else { this.segmentValues = collapseValues; } - } - - @Override - protected float getGroupHeadScore(int globalDoc, int contextDoc) throws IOException { - int ord = -1; - if (this.ordinalMap != null) { - // Handle ordinalMapping case - if (segmentValues.advanceExact(contextDoc)) { - ord = (int) segmentOrdinalMap.get(segmentValues.ordValue()); - } - } else { - // Handle top Level FieldCache or Single Segment Case - if (segmentValues.advanceExact(globalDoc)) { - ord = segmentValues.ordValue(); + return new CachedScoreScorable(disi) { + @Override + protected float computeScore(int globalDoc) throws IOException { + int ord = -1; + if (ordinalMap != null) { + int contextDoc = globalDoc - context.docBase; + if (segmentValues.advanceExact(contextDoc)) { + ord = (int) segmentOrdinalMap.get(segmentValues.ordValue()); + } + } else { + if (segmentValues.advanceExact(globalDoc)) { + ord = segmentValues.ordValue(); + } + } + if (ord > -1) { + return scores.get(ord); + } + return scoreNullGroup(globalDoc); } - } - if (ord > -1) { - return scores.get(ord); - } - return Float.NaN; + }; } } @@ -1792,16 +1829,14 @@ public void collect(int contextDoc) throws IOException { final int globalDoc = contextDoc + this.docBase; if (collapseValues.advanceExact(contextDoc)) { final int collapseKey = (int) collapseValues.longValue(); - // Check to see if we have documents boosted by the QueryElevationComponent (skip normal - // strategy based collection) + // Check to see if we have documents boosted by the QueryElevationComponent if (boostedDocsCollector.collectIfBoosted(collapseKey, globalDoc)) return; collapse(collapseKey, contextDoc, globalDoc); } else { // Null Group... if (collectElevatedDocsWhenCollapsing) { - // Check to see if we have documents boosted by the QueryElevationComponent (skip normal - // strategy based collection) + // Check to see if we have documents boosted by the QueryElevationComponent if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return; } if (NullPolicy.IGNORE != nullPolicy) { @@ -1812,14 +1847,13 @@ public void collect(int contextDoc) throws IOException { @Override protected DocIdSetIterator getCollapsedDisi() { - // Handle the boosted docs. boostedDocsCollector.purgeGroupsThatHaveBoostedDocs( collapsedSet, cmap::remove, () -> nullDoc = -1); - // Build the sorted DocSet of group heads. if (nullDoc > -1) { this.collapsedSet.set(nullDoc); } + for (IntIntCursor cursor : cmap) { int pointer = cursor.value; collapsedSet.set(docs.get(pointer)); @@ -1829,19 +1863,21 @@ protected DocIdSetIterator getCollapsedDisi() { } @Override - protected void advanceCompleteSegment(int contextIndex) throws IOException { - this.collapseValues = - DocValues.getNumeric(contexts.get(contextIndex).reader(), this.collapseField); - } - - @Override - protected float getGroupHeadScore(int globalDoc, int contextDoc) throws IOException { - if (collapseValues.advanceExact(contextDoc)) { - final int collapseValue = (int) collapseValues.longValue(); - final int pointer = cmap.get(collapseValue); - return scores.get(pointer); - } - return Float.NaN; + protected Scorable getCollapsedScores(DocIdSetIterator disi, LeafReaderContext context) + throws IOException { + this.collapseValues = DocValues.getNumeric(context.reader(), this.collapseField); + return new CachedScoreScorable(disi) { + @Override + protected float computeScore(int globalDoc) throws IOException { + int contextDoc = globalDoc - context.docBase; + if (collapseValues.advanceExact(contextDoc)) { + final int collapseValue = (int) collapseValues.longValue(); + final int pointer = cmap.get(collapseValue); + return scores.get(pointer); + } + return scoreNullGroup(globalDoc); + } + }; } } From 8ea64838e8d4a0623cfe68bb44e3cea4c8178398 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Sun, 15 Feb 2026 22:03:30 -0500 Subject: [PATCH 16/21] Defer needsScores detection until we have the delegate (then we know), thus avoiding questionable eager detection algorithm. An optimization. --- .../solr/search/CollapsingQParserPlugin.java | 168 ++++++++++-------- 1 file changed, 89 insertions(+), 79 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index e370df2b3c6c..16430c9306e9 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -280,9 +280,9 @@ public static class CollapsingPostFilter extends ExtendedQueryBase implements Po protected final GroupHeadSelector groupHeadSelector; protected final SortSpec sortSpec; // may be null, parsed at most once from groupHeadSelector public final String hint; - protected final boolean needsScores; + protected final boolean needsScores4Collapsing; protected final NullPolicy nullPolicy; - private final int size; + private final int initialSize; private Set boosted; // ordered by "priority" public String getField() { @@ -339,7 +339,7 @@ public String toString(String s) { + this.groupHeadSelector + (hint == null ? "" : ", hint=" + this.hint) + ", size=" - + this.size + + this.initialSize + ")"; } @@ -392,7 +392,8 @@ public CollapsingPostFilter( : null; this.hint = localParams.get("hint"); - this.size = localParams.getInt("size", 100000); // Only used for collapsing on int fields. + this.initialSize = + localParams.getInt("size", 100_000); // Only used for collapsing on int fields. { final SolrRequestInfo info = SolrRequestInfo.getRequestInfo(); @@ -400,21 +401,12 @@ public CollapsingPostFilter( // may be null in some esoteric corner usages final ResponseBuilder rb = info.getResponseBuilder(); - final SortSpec topSort = null == rb ? null : rb.getSortSpec(); - boolean needsScores4Collapsing = groupHeadSelector.needsScores(this.sortSpec); - this.needsScores = - needsScores4Collapsing - || (info.getRsp().getReturnFields().wantsScore() - || (null != topSort && topSort.includesScore()) - || (this.boosted != null)); - - if (this.needsScores && null != rb) { - // regardless of why we need scores ensure the IndexSearcher will compute them - // for the "real" docs. (ie: maybe we need them because we were - // asked to compute them for the collapsed docs, maybe we need them because in - // order to find the groupHead we need them computed for us. + this.needsScores4Collapsing = groupHeadSelector.needsScores(this.sortSpec); + if (this.needsScores4Collapsing && null != rb) { + // ensure the IndexSearcher will compute scores for the "real" docs + // when we need them to find the groupHead rb.setFieldFlags(rb.getFieldFlags() | SolrIndexSearcher.GET_SCORES); } } @@ -517,10 +509,10 @@ protected DelegatingCollector getCollector(IntIntHashMap boostDocs, SolrIndexSea return new BlockOrdScoreCollector(collapseField, nullPolicy, boostDocs); } return new OrdScoreCollector( + searcher, docValuesProducer, nullPolicy, boostDocs, - searcher, collectElevatedDocsWhenCollapsing); } else if (isNumericCollapsible(collapseFieldType)) { @@ -529,12 +521,12 @@ protected DelegatingCollector getCollector(IntIntHashMap boostDocs, SolrIndexSea } return new IntScoreCollector( + searcher, nullPolicy, - size, - collapseField, boostDocs, - searcher, - collectElevatedDocsWhenCollapsing); + collectElevatedDocsWhenCollapsing, + collapseField, + initialSize); } else { throw new SolrException( @@ -553,19 +545,20 @@ protected DelegatingCollector getCollector(IntIntHashMap boostDocs, SolrIndexSea collapseField, nullPolicy, boostDocs, - BlockOrdSortSpecCollector.getSort(groupHeadSelector, sortSpec, funcQuery, searcher), - needsScores); + BlockOrdSortSpecCollector.getSort( + groupHeadSelector, sortSpec, funcQuery, searcher)); } var builder = new OrdFieldCollectorBuilder(); + builder.searcher = searcher; builder.groupHeadSelector = groupHeadSelector; - builder.collapseValuesProducer = docValuesProducer; builder.nullPolicy = nullPolicy; - builder.needsScores = needsScores; + builder.needsScores4Collapsing = needsScores4Collapsing; builder.boostDocsMap = boostDocs; - builder.searcher = searcher; builder.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing; + builder.collapseValuesProducer = docValuesProducer; + return builder.build(sortSpec, minMaxFieldType, funcQuery); } else if (isNumericCollapsible(collapseFieldType)) { @@ -578,20 +571,21 @@ protected DelegatingCollector getCollector(IntIntHashMap boostDocs, SolrIndexSea collapseField, nullPolicy, boostDocs, - BlockOrdSortSpecCollector.getSort(groupHeadSelector, sortSpec, funcQuery, searcher), - needsScores); + BlockOrdSortSpecCollector.getSort( + groupHeadSelector, sortSpec, funcQuery, searcher)); } var builder = new IntFieldCollectorBuilder(); + builder.searcher = searcher; builder.groupHeadSelector = groupHeadSelector; builder.nullPolicy = nullPolicy; - builder.collapseField = collapseField; - builder.size = size; - builder.needsScores = needsScores; + builder.needsScores4Collapsing = needsScores4Collapsing; builder.boostDocsMap = boostDocs; - builder.searcher = searcher; builder.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing; + builder.collapseField = collapseField; + builder.initialSize = initialSize; + return builder.build(sortSpec, minMaxFieldType, funcQuery); } else { throw new SolrException( @@ -770,7 +764,8 @@ protected abstract static class AbstractCollapseCollector extends DelegatingColl protected final List contexts; protected final int maxDoc; protected final NullPolicy nullPolicy; - protected final boolean needsScores; + protected final boolean needsScores4Collapsing; + protected boolean needsScores; // cached from scoreMode() protected final boolean collectElevatedDocsWhenCollapsing; // Results/accumulator @@ -783,30 +778,39 @@ protected abstract static class AbstractCollapseCollector extends DelegatingColl protected AbstractCollapseCollector( IndexSearcher searcher, NullPolicy nullPolicy, - boolean needsScores, + boolean needsScores4Collapsing, boolean collectElevatedDocsWhenCollapsing, IntIntHashMap boostDocsMap) { this.contexts = searcher.getTopReaderContext().leaves(); this.maxDoc = searcher.getIndexReader().maxDoc(); this.nullPolicy = nullPolicy; - this.needsScores = needsScores; + this.needsScores4Collapsing = needsScores4Collapsing; this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing; this.collapsedSet = new FixedBitSet(maxDoc); this.boostedDocsCollector = BoostedDocsCollector.build(boostDocsMap); - if (needsScores && nullPolicy == NullPolicy.EXPAND) { - this.nullScores = new FloatArrayList(); - } } @Override public ScoreMode scoreMode() { - return needsScores ? ScoreMode.COMPLETE : super.scoreMode(); + return needsScores4Collapsing ? ScoreMode.COMPLETE : super.scoreMode(); + } + + /** Initialize score-related data structures. Called once before collecting begins. */ + protected void initializeCollection() { + this.needsScores = scoreMode().needsScores(); + if (needsScores && nullPolicy == NullPolicy.EXPAND) { + this.nullScores = new FloatArrayList(); + } } @Override protected void doSetNextReader(LeafReaderContext context) throws IOException { + if (this.context == null) { + initializeCollection(); + } + // Do NOT set leafDelegate (calling super() would do this). // That's handled in complete() for these collectors. assert this.contexts.get(context.ord) == context; @@ -886,7 +890,6 @@ protected float scoreNullGroup(int globalDoc) { } return 0F; } - } /** @@ -909,10 +912,10 @@ static class OrdScoreCollector extends AbstractCollapseCollector { private final IntFloatDynamicMap scores; public OrdScoreCollector( + IndexSearcher searcher, DocValuesProducer collapseValuesProducer, NullPolicy nullPolicy, IntIntHashMap boostDocsMap, - IndexSearcher searcher, boolean collectElevatedDocsWhenCollapsing) throws IOException { super(searcher, nullPolicy, true, collectElevatedDocsWhenCollapsing, boostDocsMap); @@ -1062,17 +1065,17 @@ static class IntScoreCollector extends AbstractCollapseCollector { private final IntLongHashMap cmap; public IntScoreCollector( + SolrIndexSearcher searcher, NullPolicy nullPolicy, - int size, - String field, IntIntHashMap boostDocsMap, - SolrIndexSearcher searcher, - boolean collectElevatedDocsWhenCollapsing) { + boolean collectElevatedDocsWhenCollapsing, + String field, + int initialSize) { super(searcher, nullPolicy, true, collectElevatedDocsWhenCollapsing, boostDocsMap); this.field = field; - this.cmap = new IntLongHashMap(size); + this.cmap = new IntLongHashMap(initialSize); this.nullScore = -Float.MAX_VALUE; } @@ -1170,7 +1173,7 @@ protected static class OrdFieldCollectorBuilder { public DocValuesProducer collapseValuesProducer; public NullPolicy nullPolicy; public int valueCount; - public boolean needsScores; + public boolean needsScores4Collapsing; public IntIntHashMap boostDocsMap; public SolrIndexSearcher searcher; public boolean collectElevatedDocsWhenCollapsing; @@ -1216,6 +1219,7 @@ public OrdFieldValueCollector build( */ protected abstract static class OrdFieldValueCollector extends AbstractCollapseCollector { // Source data + protected final int valueCount; protected final DocValuesProducer collapseValuesProducer; protected SortedDocValues collapseValues; protected OrdinalMap ordinalMap; @@ -1231,10 +1235,11 @@ protected OrdFieldValueCollector(OrdFieldCollectorBuilder ctx) throws IOExceptio super( ctx.searcher, ctx.nullPolicy, - ctx.needsScores, + ctx.needsScores4Collapsing, ctx.collectElevatedDocsWhenCollapsing, ctx.boostDocsMap); + this.valueCount = ctx.valueCount; this.collapseValuesProducer = ctx.collapseValuesProducer; this.collapseValues = ctx.collapseValuesProducer.getSorted(null); @@ -1244,10 +1249,13 @@ protected OrdFieldValueCollector(OrdFieldCollectorBuilder ctx) throws IOExceptio } this.ords = new IntIntDynamicMap(ctx.valueCount, -1); + } + + @Override + protected void initializeCollection() { + super.initializeCollection(); if (this.needsScores) { - this.scores = new IntFloatDynamicMap(ctx.valueCount, 0.0f); - } else { - this.scores = null; + this.scores = new IntFloatDynamicMap(valueCount, 0.0f); } } @@ -1749,8 +1757,8 @@ protected static class IntFieldCollectorBuilder { public GroupHeadSelector groupHeadSelector; public NullPolicy nullPolicy; public String collapseField; - public int size; - public boolean needsScores; + public int initialSize; + public boolean needsScores4Collapsing; public IntIntHashMap boostDocsMap; public IndexSearcher searcher; public boolean collectElevatedDocsWhenCollapsing; @@ -1783,6 +1791,7 @@ public IntFieldValueCollector build( protected abstract static class IntFieldValueCollector extends AbstractCollapseCollector { // Configuration protected final String collapseField; + protected final int initialSize; // data structure size hint // Source data protected NumericDocValues collapseValues; @@ -1796,20 +1805,24 @@ protected IntFieldValueCollector(IntFieldCollectorBuilder ctx) throws IOExceptio super( ctx.searcher, ctx.nullPolicy, - ctx.needsScores, + ctx.needsScores4Collapsing, ctx.collectElevatedDocsWhenCollapsing, ctx.boostDocsMap); assert !GroupHeadSelectorType.SCORE.equals(ctx.groupHeadSelector.type); this.collapseField = ctx.collapseField; + this.initialSize = ctx.initialSize; - this.cmap = new IntIntHashMap(ctx.size); - this.docs = new IntIntDynamicMap(ctx.size, 0); - if (this.needsScores) { - this.scores = new IntFloatDynamicMap(ctx.size, 0.0f); - } else { - this.scores = null; + this.cmap = new IntIntHashMap(ctx.initialSize); + this.docs = new IntIntDynamicMap(ctx.initialSize, 0); + } + + @Override + protected void initializeCollection() { + super.initializeCollection(); + if (needsScores) { + this.scores = new IntFloatDynamicMap(initialSize, 0.0f); } } @@ -1909,7 +1922,7 @@ public IntIntCollector(IntFieldCollectorBuilder ctx) throws IOException { this.nullCompVal = Integer.MAX_VALUE; } - this.testValues = new IntIntDynamicMap(ctx.size, 0); + this.testValues = new IntIntDynamicMap(ctx.initialSize, 0); } @Override @@ -1998,7 +2011,7 @@ public IntFloatCollector(IntFieldCollectorBuilder ctx) throws IOException { this.nullCompVal = Float.MAX_VALUE; } - this.testValues = new IntFloatDynamicMap(ctx.size, 0.0f); + this.testValues = new IntFloatDynamicMap(ctx.initialSize, 0.0f); } @Override @@ -2094,7 +2107,7 @@ public IntValueSourceCollector(IntFieldCollectorBuilder ctx, FunctionQuery funcQ collapseScore.setupIfNeeded(ctx.groupHeadSelector, rcontext); - this.testValues = new IntFloatDynamicMap(ctx.size, 0.0f); + this.testValues = new IntFloatDynamicMap(ctx.initialSize, 0.0f); } @Override @@ -2192,7 +2205,7 @@ public IntSortSpecCollector(IntFieldCollectorBuilder ctx, SortSpec sortSpec) assert GroupHeadSelectorType.SORT.equals(ctx.groupHeadSelector.type); Sort sort = rewriteSort(sortSpec, ctx.searcher); - this.compareState = new SortFieldsCompare(sort.getSort(), ctx.size); + this.compareState = new SortFieldsCompare(sort.getSort(), ctx.initialSize); } @Override @@ -2292,7 +2305,7 @@ private abstract static class AbstractBlockCollector extends DelegatingCollector // Configuration protected final String collapseField; - protected final boolean needsScores; + protected final boolean needsScores4Collapsing; protected final boolean expandNulls; // Results/accumulator @@ -2303,10 +2316,10 @@ protected AbstractBlockCollector( final String collapseField, final NullPolicy nullPolicy, final IntIntHashMap boostDocsMap, - final boolean needsScores) { + final boolean needsScores4Collapsing) { this.collapseField = collapseField; - this.needsScores = needsScores; + this.needsScores4Collapsing = needsScores4Collapsing; assert nullPolicy == NullPolicy.IGNORE || nullPolicy == NullPolicy.EXPAND; this.expandNulls = (NullPolicy.EXPAND == nullPolicy); @@ -2317,7 +2330,7 @@ protected AbstractBlockCollector( @Override public ScoreMode scoreMode() { - return needsScores ? ScoreMode.COMPLETE : super.scoreMode(); + return needsScores4Collapsing ? ScoreMode.COMPLETE : super.scoreMode(); } /** If we have a candidate match, delegate the collection of that match. */ @@ -2635,9 +2648,8 @@ public AbstractBlockSortSpecCollector( final String collapseField, final NullPolicy nullPolicy, final IntIntHashMap boostDocsMap, - final Sort sort, - final boolean needsScores) { - super(collapseField, nullPolicy, boostDocsMap, needsScores); + final Sort sort) { + super(collapseField, nullPolicy, boostDocsMap, sort.needsScores()); this.sortsCompare = new BlockBasedSortFieldsCompare(sort.getSort()); } @@ -2650,7 +2662,7 @@ public void setScorer(Scorable scorer) throws IOException { private void setCurrentGroupBestMatch(final int contextDocId, final boolean isBoosted) throws IOException { currentGroupState.setBestDocForCurrentGroup(contextDocId, isBoosted); - if (needsScores) { + if (scoreMode().needsScores()) { currentGroupState.score = scorer.score(); } } @@ -2756,10 +2768,9 @@ public BlockOrdSortSpecCollector( final String collapseField, final NullPolicy nullPolicy, final IntIntHashMap boostDocsMap, - final Sort sort, - final boolean needsScores) + final Sort sort) throws IOException { - super(collapseField, nullPolicy, boostDocsMap, sort, needsScores); + super(collapseField, nullPolicy, boostDocsMap, sort); } @Override @@ -2791,10 +2802,9 @@ public BlockIntSortSpecCollector( final String collapseField, final NullPolicy nullPolicy, final IntIntHashMap boostDocsMap, - final Sort sort, - final boolean needsScores) + final Sort sort) throws IOException { - super(collapseField, nullPolicy, boostDocsMap, sort, needsScores); + super(collapseField, nullPolicy, boostDocsMap, sort); } @Override From ea6c4ab4418817bc504e51d9670110682c578fc8 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Sun, 15 Feb 2026 23:12:15 -0500 Subject: [PATCH 17/21] initCollapseValues() / initSegmentValues() helper methods valueCount moved from builder to collector constructor Removed redundant needsScores4Collapsing fields Removed deprecated things --- .../solr/search/CollapsingQParserPlugin.java | 109 +++++++----------- 1 file changed, 42 insertions(+), 67 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index 16430c9306e9..a452480d6518 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -147,15 +147,6 @@ public class CollapsingQParserPlugin extends QParserPlugin { */ public static String COLLECT_ELEVATED_DOCS_WHEN_COLLAPSING = "collectElevatedDocsWhenCollapsing"; - /** - * @deprecated use {@link NullPolicy} instead. - */ - @Deprecated public static final String NULL_COLLAPSE = "collapse"; - - @Deprecated public static final String NULL_IGNORE = "ignore"; - @Deprecated public static final String NULL_EXPAND = "expand"; - @Deprecated public static final String HINT_MULTI_DOCVALUES = "multi_docvalues"; - public enum NullPolicy { IGNORE("ignore"), COLLAPSE("collapse"), @@ -769,7 +760,7 @@ protected abstract static class AbstractCollapseCollector extends DelegatingColl protected final boolean collectElevatedDocsWhenCollapsing; // Results/accumulator - protected final FixedBitSet collapsedSet; + protected FixedBitSet collapsedSet; protected final BoostedDocsCollector boostedDocsCollector; protected FloatArrayList nullScores; protected float nullScore; @@ -788,7 +779,6 @@ protected AbstractCollapseCollector( this.needsScores4Collapsing = needsScores4Collapsing; this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing; - this.collapsedSet = new FixedBitSet(maxDoc); this.boostedDocsCollector = BoostedDocsCollector.build(boostDocsMap); } @@ -797,9 +787,10 @@ public ScoreMode scoreMode() { return needsScores4Collapsing ? ScoreMode.COMPLETE : super.scoreMode(); } - /** Initialize score-related data structures. Called once before collecting begins. */ + /** Initialize data structures for collection. Called once before collecting begins. */ protected void initializeCollection() { this.needsScores = scoreMode().needsScores(); + this.collapsedSet = new FixedBitSet(maxDoc); if (needsScores && nullPolicy == NullPolicy.EXPAND) { this.nullScores = new FloatArrayList(); } @@ -822,9 +813,6 @@ protected void doSetNextReader(LeafReaderContext context) throws IOException { /** Core/general algorithm */ @Override public final void complete() throws IOException { - if (contexts.isEmpty()) { - return; - } DocIdSetIterator collapsedDocs = getCollapsedDisi(); @@ -855,9 +843,7 @@ public final void complete() throws IOException { leafDelegate.finish(); } - if (delegate instanceof DelegatingCollector) { - ((DelegatingCollector) delegate).complete(); - } + super.complete(); } /** @@ -921,22 +907,29 @@ public OrdScoreCollector( super(searcher, nullPolicy, true, collectElevatedDocsWhenCollapsing, boostDocsMap); this.collapseValuesProducer = collapseValuesProducer; + initCollapseValues(); + int valueCount = collapseValues.getValueCount(); + + this.ords = new IntIntDynamicMap(valueCount, -1); + this.scores = new IntFloatDynamicMap(valueCount, -Float.MAX_VALUE); + this.nullScore = -Float.MAX_VALUE; + } + private void initCollapseValues() throws IOException { this.collapseValues = collapseValuesProducer.getSorted(null); - int valueCount = collapseValues.getValueCount(); if (collapseValues instanceof MultiDocValues.MultiSortedDocValues multi) { this.multiSortedDocValues = multi; this.ordinalMap = multi.mapping; } - - this.ords = new IntIntDynamicMap(valueCount, -1); - this.scores = new IntFloatDynamicMap(valueCount, -Float.MAX_VALUE); - this.nullScore = -Float.MAX_VALUE; } @Override protected void doSetNextReader(LeafReaderContext context) throws IOException { super.doSetNextReader(context); + initSegmentValues(context); + } + + private void initSegmentValues(LeafReaderContext context) { if (ordinalMap != null) { this.segmentValues = this.multiSortedDocValues.values[context.ord]; this.segmentOrdinalMap = ordinalMap.getGlobalOrds(context.ord); @@ -1013,18 +1006,9 @@ protected Scorable getCollapsedScores(DocIdSetIterator disi, LeafReaderContext c throws IOException { if (!collapsedScoresInitialized) { collapsedScoresInitialized = true; - collapseValues = collapseValuesProducer.getSorted(null); - if (collapseValues instanceof MultiDocValues.MultiSortedDocValues) { - this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues) collapseValues; - this.ordinalMap = multiSortedDocValues.mapping; - } - } - if (ordinalMap != null) { - this.segmentValues = this.multiSortedDocValues.values[context.ord]; - this.segmentOrdinalMap = this.ordinalMap.getGlobalOrds(context.ord); - } else { - this.segmentValues = collapseValues; + initCollapseValues(); } + initSegmentValues(context); return new CachedScoreScorable(disi) { @Override protected float computeScore(int globalDoc) throws IOException { @@ -1142,6 +1126,7 @@ protected DocIdSetIterator getCollapsedDisi() { } for (IntLongCursor cursor : cmap) { + // the low bits of the long is the global doc ID collapsedSet.set((int) cursor.value); } @@ -1167,12 +1152,11 @@ protected float computeScore(int globalDoc) throws IOException { } } - /** Builder for OrdFieldValueCollector subclasses. */ + /** Builder for {@link OrdFieldValueCollector}. */ protected static class OrdFieldCollectorBuilder { public GroupHeadSelector groupHeadSelector; public DocValuesProducer collapseValuesProducer; public NullPolicy nullPolicy; - public int valueCount; public boolean needsScores4Collapsing; public IntIntHashMap boostDocsMap; public SolrIndexSearcher searcher; @@ -1182,13 +1166,6 @@ protected static class OrdFieldCollectorBuilder { public OrdFieldValueCollector build( SortSpec sortSpec, FieldType fieldType, FunctionQuery funcQuery) throws IOException { - try { - SortedDocValues tempValues = collapseValuesProducer.getSorted(null); - valueCount = tempValues.getValueCount(); - } catch (IOException e) { - throw new RuntimeException(e); - } - if (null != sortSpec) { return new OrdSortSpecCollector(this, sortSpec); } else if (funcQuery != null) { @@ -1239,16 +1216,20 @@ protected OrdFieldValueCollector(OrdFieldCollectorBuilder ctx) throws IOExceptio ctx.collectElevatedDocsWhenCollapsing, ctx.boostDocsMap); - this.valueCount = ctx.valueCount; this.collapseValuesProducer = ctx.collapseValuesProducer; - this.collapseValues = ctx.collapseValuesProducer.getSorted(null); + initCollapseValues(); + this.valueCount = collapseValues.getValueCount(); + + this.ords = new IntIntDynamicMap(valueCount, -1); + } + + private void initCollapseValues() throws IOException { + this.collapseValues = collapseValuesProducer.getSorted(null); if (collapseValues instanceof MultiDocValues.MultiSortedDocValues) { this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues) collapseValues; this.ordinalMap = multiSortedDocValues.mapping; } - - this.ords = new IntIntDynamicMap(ctx.valueCount, -1); } @Override @@ -1262,6 +1243,10 @@ protected void initializeCollection() { @Override public void doSetNextReader(LeafReaderContext context) throws IOException { super.doSetNextReader(context); + initSegmentValues(context); + } + + private void initSegmentValues(LeafReaderContext context) { if (ordinalMap != null) { this.segmentValues = this.multiSortedDocValues.values[context.ord]; this.segmentOrdinalMap = ordinalMap.getGlobalOrds(context.ord); @@ -1377,10 +1362,10 @@ public OrdIntCollector(OrdFieldCollectorBuilder ctx) throws IOException { assert GroupHeadSelectorType.MIN_MAX.contains(ctx.groupHeadSelector.type); if (GroupHeadSelectorType.MAX.equals(ctx.groupHeadSelector.type)) { comp = new MaxIntComp(); - this.ordVals = new IntIntDynamicMap(ctx.valueCount, Integer.MIN_VALUE); + this.ordVals = new IntIntDynamicMap(valueCount, Integer.MIN_VALUE); } else { comp = new MinIntComp(); - this.ordVals = new IntIntDynamicMap(ctx.valueCount, Integer.MAX_VALUE); + this.ordVals = new IntIntDynamicMap(valueCount, Integer.MAX_VALUE); this.nullVal = Integer.MAX_VALUE; } } @@ -1444,11 +1429,11 @@ public OrdFloatCollector(OrdFieldCollectorBuilder ctx) throws IOException { assert GroupHeadSelectorType.MIN_MAX.contains(ctx.groupHeadSelector.type); if (GroupHeadSelectorType.MAX.equals(ctx.groupHeadSelector.type)) { comp = new MaxFloatComp(); - this.ordVals = new IntFloatDynamicMap(ctx.valueCount, -Float.MAX_VALUE); + this.ordVals = new IntFloatDynamicMap(valueCount, -Float.MAX_VALUE); this.nullVal = -Float.MAX_VALUE; } else { comp = new MinFloatComp(); - this.ordVals = new IntFloatDynamicMap(ctx.valueCount, Float.MAX_VALUE); + this.ordVals = new IntFloatDynamicMap(valueCount, Float.MAX_VALUE); this.nullVal = Float.MAX_VALUE; } } @@ -1515,11 +1500,11 @@ public OrdLongCollector(OrdFieldCollectorBuilder ctx) throws IOException { assert GroupHeadSelectorType.MIN_MAX.contains(ctx.groupHeadSelector.type); if (GroupHeadSelectorType.MAX.equals(ctx.groupHeadSelector.type)) { comp = new MaxLongComp(); - this.ordVals = new IntLongDynamicMap(ctx.valueCount, Long.MIN_VALUE); + this.ordVals = new IntLongDynamicMap(valueCount, Long.MIN_VALUE); } else { this.nullVal = Long.MAX_VALUE; comp = new MinLongComp(); - this.ordVals = new IntLongDynamicMap(ctx.valueCount, Long.MAX_VALUE); + this.ordVals = new IntLongDynamicMap(valueCount, Long.MAX_VALUE); } } @@ -1569,7 +1554,6 @@ protected void collapse(int ord, int contextDoc, int globalDoc) throws IOExcepti */ protected static class OrdValueSourceCollector extends OrdFieldValueCollector { // Configuration - private final boolean needsScores4Collapsing; private final FloatCompare comp; private final ValueSource valueSource; private final Map rcontext; @@ -1585,15 +1569,14 @@ protected static class OrdValueSourceCollector extends OrdFieldValueCollector { public OrdValueSourceCollector(OrdFieldCollectorBuilder ctx, FunctionQuery funcQuery) throws IOException { super(ctx); - this.needsScores4Collapsing = ctx.groupHeadSelector.needsScores(null); assert GroupHeadSelectorType.MIN_MAX.contains(ctx.groupHeadSelector.type); if (GroupHeadSelectorType.MAX.equals(ctx.groupHeadSelector.type)) { comp = new MaxFloatComp(); - this.ordVals = new IntFloatDynamicMap(ctx.valueCount, -Float.MAX_VALUE); + this.ordVals = new IntFloatDynamicMap(valueCount, -Float.MAX_VALUE); } else { this.nullVal = Float.MAX_VALUE; comp = new MinFloatComp(); - this.ordVals = new IntFloatDynamicMap(ctx.valueCount, Float.MAX_VALUE); + this.ordVals = new IntFloatDynamicMap(valueCount, Float.MAX_VALUE); } this.valueSource = funcQuery.getValueSource(); this.rcontext = ValueSource.newContext(ctx.searcher); @@ -1657,7 +1640,6 @@ protected void collapse(int ord, int contextDoc, int globalDoc) throws IOExcepti */ protected static class OrdSortSpecCollector extends OrdFieldValueCollector { // Configuration - private final boolean needsScores4Collapsing; private final SortFieldsCompare compareState; // Results/accumulator @@ -1667,12 +1649,10 @@ public OrdSortSpecCollector(OrdFieldCollectorBuilder ctx, SortSpec sortSpec) throws IOException { super(ctx); - this.needsScores4Collapsing = ctx.groupHeadSelector.needsScores(sortSpec); - assert GroupHeadSelectorType.SORT.equals(ctx.groupHeadSelector.type); Sort sort = rewriteSort(sortSpec, ctx.searcher); - this.compareState = new SortFieldsCompare(sort.getSort(), ctx.valueCount); + this.compareState = new SortFieldsCompare(sort.getSort(), valueCount); } @Override @@ -1752,7 +1732,7 @@ protected void collapse(int ord, int contextDoc, int globalDoc) throws IOExcepti } } - /** Builder for IntFieldValueCollector subclasses. */ + /** Builder for {@link IntFieldValueCollector}. */ protected static class IntFieldCollectorBuilder { public GroupHeadSelector groupHeadSelector; public NullPolicy nullPolicy; @@ -2076,7 +2056,6 @@ protected void collapseNullGroup(int contextDoc, int globalDoc) throws IOExcepti /** Collector for collapsing on int field using function query to select group head. */ protected static class IntValueSourceCollector extends IntFieldValueCollector { // Configuration - private final boolean needsScores4Collapsing; private final FloatCompare comp; private final ValueSource valueSource; private final Map rcontext; @@ -2093,7 +2072,6 @@ protected static class IntValueSourceCollector extends IntFieldValueCollector { public IntValueSourceCollector(IntFieldCollectorBuilder ctx, FunctionQuery funcQuery) throws IOException { super(ctx); - this.needsScores4Collapsing = ctx.groupHeadSelector.needsScores(null); assert GroupHeadSelectorType.MIN_MAX.contains(ctx.groupHeadSelector.type); if (GroupHeadSelectorType.MAX.equals(ctx.groupHeadSelector.type)) { comp = new MaxFloatComp(); @@ -2189,7 +2167,6 @@ protected void collapseNullGroup(int contextDoc, int globalDoc) throws IOExcepti /** Collector for collapsing on int field using a sort spec to select group head. */ protected static class IntSortSpecCollector extends IntFieldValueCollector { // Configuration - private final boolean needsScores4Collapsing; private final SortFieldsCompare compareState; // Results/accumulator @@ -2200,8 +2177,6 @@ public IntSortSpecCollector(IntFieldCollectorBuilder ctx, SortSpec sortSpec) throws IOException { super(ctx); - this.needsScores4Collapsing = ctx.groupHeadSelector.needsScores(sortSpec); - assert GroupHeadSelectorType.SORT.equals(ctx.groupHeadSelector.type); Sort sort = rewriteSort(sortSpec, ctx.searcher); From 1b1e0cdd235d5758823090a37ad82f4e9ce8f5b6 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Sun, 15 Feb 2026 23:52:24 -0500 Subject: [PATCH 18/21] Base collector: remove collectElevatedDocsWhenCollapsing (redundant) Move boostDocs initialization. --- .../solr/search/CollapsingQParserPlugin.java | 142 ++++++------------ 1 file changed, 48 insertions(+), 94 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index a452480d6518..47f897f87310 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -273,8 +273,8 @@ public static class CollapsingPostFilter extends ExtendedQueryBase implements Po public final String hint; protected final boolean needsScores4Collapsing; protected final NullPolicy nullPolicy; - private final int initialSize; - private Set boosted; // ordered by "priority" + protected final boolean collectElevatedDocsWhenCollapsing; + protected final int initialSize; public String getField() { return this.collapseField; @@ -307,7 +307,8 @@ public boolean equals(Object other) { private boolean equalsTo(CollapsingPostFilter other) { return collapseField.equals(other.collapseField) && groupHeadSelector.equals(other.groupHeadSelector) - && nullPolicy == other.nullPolicy; + && nullPolicy == other.nullPolicy + && collectElevatedDocsWhenCollapsing == other.collectElevatedDocsWhenCollapsing; } @Override @@ -403,6 +404,9 @@ public CollapsingPostFilter( } this.nullPolicy = NullPolicy.fromString(localParams.get("nullPolicy")); + + this.collectElevatedDocsWhenCollapsing = + params.getBool(COLLECT_ELEVATED_DOCS_WHEN_COLLAPSING, true); } protected GroupHeadSelector buildGroupHeadSelector(SolrParams localParams) { @@ -410,45 +414,19 @@ protected GroupHeadSelector buildGroupHeadSelector(SolrParams localParams) { } @Override - @SuppressWarnings({"unchecked"}) public DelegatingCollector getFilterCollector(IndexSearcher indexSearcher) { try { - // Deal with boosted docs. - // We have to deal with it here rather then the constructor because - // because the QueryElevationComponent runs after the Queries are constructed. - - IntIntHashMap boostDocsMap = null; - Map context = null; - SolrRequestInfo info = SolrRequestInfo.getRequestInfo(); - if (info != null) { - context = info.getReq().getContext(); - } - - if (this.boosted == null && context != null) { - this.boosted = (Set) context.get(QueryElevationComponent.BOOSTED); - } - - SolrIndexSearcher searcher = (SolrIndexSearcher) indexSearcher; - boostDocsMap = QueryElevationComponent.getBoostDocs(searcher, this.boosted, context); - return this.getCollector(boostDocsMap, searcher); - + return this.getFilterCollector((SolrIndexSearcher) indexSearcher); } catch (IOException e) { throw new RuntimeException(e); } } - /** - * @see #isNumericCollapsible - */ - private static final EnumSet NUMERIC_COLLAPSIBLE_TYPES = - EnumSet.of(NumberType.INTEGER, NumberType.FLOAT); - - private boolean isNumericCollapsible(FieldType collapseFieldType) { - return NUMERIC_COLLAPSIBLE_TYPES.contains(collapseFieldType.getNumberType()); - } - - protected DelegatingCollector getCollector(IntIntHashMap boostDocs, SolrIndexSearcher searcher) + protected DelegatingCollector getFilterCollector(SolrIndexSearcher searcher) throws IOException { + // We have to deal with it here rather than the constructor because + // the QueryElevationComponent runs after the Queries are constructed. + IntIntHashMap boostDocs = getElevatedBoostDocsMap(searcher); // block collapsing logic is much simpler and uses less memory, but is only viable in specific // situations @@ -488,36 +466,20 @@ protected DelegatingCollector getCollector(IntIntHashMap boostDocs, SolrIndexSea } } - SolrRequestInfo req = SolrRequestInfo.getRequestInfo(); - boolean collectElevatedDocsWhenCollapsing = - req != null - && req.getReq().getParams().getBool(COLLECT_ELEVATED_DOCS_WHEN_COLLAPSING, true); - if (GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type)) { if (collapseFieldType instanceof StrField) { if (blockCollapse) { return new BlockOrdScoreCollector(collapseField, nullPolicy, boostDocs); } - return new OrdScoreCollector( - searcher, - docValuesProducer, - nullPolicy, - boostDocs, - collectElevatedDocsWhenCollapsing); + return new OrdScoreCollector(searcher, docValuesProducer, nullPolicy, boostDocs); } else if (isNumericCollapsible(collapseFieldType)) { if (blockCollapse) { return new BlockIntScoreCollector(collapseField, nullPolicy, boostDocs); } - return new IntScoreCollector( - searcher, - nullPolicy, - boostDocs, - collectElevatedDocsWhenCollapsing, - collapseField, - initialSize); + return new IntScoreCollector(searcher, nullPolicy, boostDocs, collapseField, initialSize); } else { throw new SolrException( @@ -546,7 +508,6 @@ protected DelegatingCollector getCollector(IntIntHashMap boostDocs, SolrIndexSea builder.nullPolicy = nullPolicy; builder.needsScores4Collapsing = needsScores4Collapsing; builder.boostDocsMap = boostDocs; - builder.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing; builder.collapseValuesProducer = docValuesProducer; @@ -572,7 +533,6 @@ protected DelegatingCollector getCollector(IntIntHashMap boostDocs, SolrIndexSea builder.nullPolicy = nullPolicy; builder.needsScores4Collapsing = needsScores4Collapsing; builder.boostDocsMap = boostDocs; - builder.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing; builder.collapseField = collapseField; builder.initialSize = initialSize; @@ -586,6 +546,28 @@ protected DelegatingCollector getCollector(IntIntHashMap boostDocs, SolrIndexSea } } + @SuppressWarnings({"unchecked"}) + protected IntIntHashMap getElevatedBoostDocsMap(SolrIndexSearcher indexSearcher) + throws IOException { + if (!collectElevatedDocsWhenCollapsing) { + return null; + } + SolrRequestInfo info = SolrRequestInfo.getRequestInfo(); + if (info != null) { + var context = info.getReq().getContext(); + var boosted = (Set) context.get(QueryElevationComponent.BOOSTED); + return QueryElevationComponent.getBoostDocs(indexSearcher, boosted, context); + } + return null; + } + + private boolean isNumericCollapsible(FieldType collapseFieldType) { + return switch (collapseFieldType.getNumberType()) { + case NumberType.INTEGER, NumberType.FLOAT -> true; + default -> false; + }; + } + protected DocValuesProducer getDocValuesProducer( SchemaField schemaField, String hint, SolrIndexSearcher searcher, boolean blockCollapse) { if (schemaField.getType() instanceof StrField) { @@ -757,7 +739,6 @@ protected abstract static class AbstractCollapseCollector extends DelegatingColl protected final NullPolicy nullPolicy; protected final boolean needsScores4Collapsing; protected boolean needsScores; // cached from scoreMode() - protected final boolean collectElevatedDocsWhenCollapsing; // Results/accumulator protected FixedBitSet collapsedSet; @@ -770,14 +751,12 @@ protected AbstractCollapseCollector( IndexSearcher searcher, NullPolicy nullPolicy, boolean needsScores4Collapsing, - boolean collectElevatedDocsWhenCollapsing, IntIntHashMap boostDocsMap) { this.contexts = searcher.getTopReaderContext().leaves(); this.maxDoc = searcher.getIndexReader().maxDoc(); this.nullPolicy = nullPolicy; this.needsScores4Collapsing = needsScores4Collapsing; - this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing; this.boostedDocsCollector = BoostedDocsCollector.build(boostDocsMap); } @@ -901,10 +880,9 @@ public OrdScoreCollector( IndexSearcher searcher, DocValuesProducer collapseValuesProducer, NullPolicy nullPolicy, - IntIntHashMap boostDocsMap, - boolean collectElevatedDocsWhenCollapsing) + IntIntHashMap boostDocsMap) throws IOException { - super(searcher, nullPolicy, true, collectElevatedDocsWhenCollapsing, boostDocsMap); + super(searcher, nullPolicy, true, boostDocsMap); this.collapseValuesProducer = collapseValuesProducer; initCollapseValues(); @@ -958,8 +936,7 @@ public void collect(int contextDoc) throws IOException { } } - if (collectElevatedDocsWhenCollapsing) { - // Check to see if we have documents boosted by the QueryElevationComponent + if (boostedDocsCollector.hasBoosts()) { if (0 <= ord) { if (boostedDocsCollector.collectIfBoosted(ord, globalDoc)) return; } else { @@ -1052,10 +1029,9 @@ public IntScoreCollector( SolrIndexSearcher searcher, NullPolicy nullPolicy, IntIntHashMap boostDocsMap, - boolean collectElevatedDocsWhenCollapsing, String field, int initialSize) { - super(searcher, nullPolicy, true, collectElevatedDocsWhenCollapsing, boostDocsMap); + super(searcher, nullPolicy, true, boostDocsMap); this.field = field; @@ -1075,10 +1051,7 @@ public void collect(int contextDoc) throws IOException { if (collapseValues.advanceExact(contextDoc)) { final int collapseValue = (int) collapseValues.longValue(); - if (collectElevatedDocsWhenCollapsing) { - // Check to see if we have documents boosted by the QueryElevationComponent - if (boostedDocsCollector.collectIfBoosted(collapseValue, globalDoc)) return; - } + if (boostedDocsCollector.collectIfBoosted(collapseValue, globalDoc)) return; float score = scorer.score(); final int idx; @@ -1098,10 +1071,7 @@ public void collect(int contextDoc) throws IOException { } else { // Null Group... - if (collectElevatedDocsWhenCollapsing) { - // Check to see if we have documents boosted by the QueryElevationComponent - if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return; - } + if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return; if (nullPolicy == NullPolicy.COLLAPSE) { float score = scorer.score(); @@ -1160,7 +1130,6 @@ protected static class OrdFieldCollectorBuilder { public boolean needsScores4Collapsing; public IntIntHashMap boostDocsMap; public SolrIndexSearcher searcher; - public boolean collectElevatedDocsWhenCollapsing; /** Builds the appropriate OrdFieldValueCollector subclass based on the selection strategy. */ public OrdFieldValueCollector build( @@ -1209,12 +1178,7 @@ protected abstract static class OrdFieldValueCollector extends AbstractCollapseC protected IntFloatDynamicMap scores; protected OrdFieldValueCollector(OrdFieldCollectorBuilder ctx) throws IOException { - super( - ctx.searcher, - ctx.nullPolicy, - ctx.needsScores4Collapsing, - ctx.collectElevatedDocsWhenCollapsing, - ctx.boostDocsMap); + super(ctx.searcher, ctx.nullPolicy, ctx.needsScores4Collapsing, ctx.boostDocsMap); this.collapseValuesProducer = ctx.collapseValuesProducer; @@ -1269,8 +1233,7 @@ public void collect(int contextDoc) throws IOException { } } - if (collectElevatedDocsWhenCollapsing) { - // Check to see if we have documents boosted by the QueryElevationComponent + if (boostedDocsCollector.hasBoosts()) { if (-1 == ord) { if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return; } else { @@ -1741,7 +1704,6 @@ protected static class IntFieldCollectorBuilder { public boolean needsScores4Collapsing; public IntIntHashMap boostDocsMap; public IndexSearcher searcher; - public boolean collectElevatedDocsWhenCollapsing; public IntFieldValueCollector build( SortSpec sortSpec, FieldType fieldType, FunctionQuery funcQuery) throws IOException { @@ -1782,12 +1744,7 @@ protected abstract static class IntFieldValueCollector extends AbstractCollapseC protected IntFloatDynamicMap scores; protected IntFieldValueCollector(IntFieldCollectorBuilder ctx) throws IOException { - super( - ctx.searcher, - ctx.nullPolicy, - ctx.needsScores4Collapsing, - ctx.collectElevatedDocsWhenCollapsing, - ctx.boostDocsMap); + super(ctx.searcher, ctx.nullPolicy, ctx.needsScores4Collapsing, ctx.boostDocsMap); assert !GroupHeadSelectorType.SCORE.equals(ctx.groupHeadSelector.type); @@ -1827,11 +1784,8 @@ public void collect(int contextDoc) throws IOException { collapse(collapseKey, contextDoc, globalDoc); } else { // Null Group... - - if (collectElevatedDocsWhenCollapsing) { - // Check to see if we have documents boosted by the QueryElevationComponent - if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return; - } + // Check to see if we have documents boosted by the QueryElevationComponent + if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return; if (NullPolicy.IGNORE != nullPolicy) { collapseNullGroup(contextDoc, globalDoc); } @@ -2952,7 +2906,7 @@ public void apply(int globalDoc) { } } - static class MergeBoost { + protected static class MergeBoost { private int[] boostDocs; private int index = 0; From c98e5bf8bbdef41dfd0112826b478821732201c8 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Mon, 16 Feb 2026 02:02:34 -0500 Subject: [PATCH 19/21] Restore collapsedSet eager initialization --- .../org/apache/solr/search/CollapsingQParserPlugin.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index 47f897f87310..1cde70d0b6c0 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -741,7 +741,7 @@ protected abstract static class AbstractCollapseCollector extends DelegatingColl protected boolean needsScores; // cached from scoreMode() // Results/accumulator - protected FixedBitSet collapsedSet; + protected final FixedBitSet collapsedSet; // todo use DocIdSetBuilder instead protected final BoostedDocsCollector boostedDocsCollector; protected FloatArrayList nullScores; protected float nullScore; @@ -758,6 +758,7 @@ protected AbstractCollapseCollector( this.nullPolicy = nullPolicy; this.needsScores4Collapsing = needsScores4Collapsing; + this.collapsedSet = new FixedBitSet(maxDoc); this.boostedDocsCollector = BoostedDocsCollector.build(boostDocsMap); } @@ -769,7 +770,6 @@ public ScoreMode scoreMode() { /** Initialize data structures for collection. Called once before collecting begins. */ protected void initializeCollection() { this.needsScores = scoreMode().needsScores(); - this.collapsedSet = new FixedBitSet(maxDoc); if (needsScores && nullPolicy == NullPolicy.EXPAND) { this.nullScores = new FloatArrayList(); } @@ -1798,7 +1798,7 @@ protected DocIdSetIterator getCollapsedDisi() { collapsedSet, cmap::remove, () -> nullDoc = -1); if (nullDoc > -1) { - this.collapsedSet.set(nullDoc); + collapsedSet.set(nullDoc); } for (IntIntCursor cursor : cmap) { From ed3f6757d1c10a8b79dc7813fe6f3612d7f58c49 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Mon, 16 Feb 2026 17:02:01 -0500 Subject: [PATCH 20/21] Use DocIdSetBuilder instead of FixedBitSet (save memory). Refactor BoostedDocsCollector a little. DynamicMap: track size --- .../org/apache/solr/search/BitDocSet.java | 3 +- .../solr/search/CollapsingQParserPlugin.java | 175 +++++++++--------- .../java/org/apache/solr/util/DynamicMap.java | 3 + .../apache/solr/util/IntFloatDynamicMap.java | 19 +- .../apache/solr/util/IntIntDynamicMap.java | 19 +- .../apache/solr/util/IntLongDynamicMap.java | 19 +- 6 files changed, 140 insertions(+), 98 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/BitDocSet.java b/solr/core/src/java/org/apache/solr/search/BitDocSet.java index c45cbd5d8932..cd6fd8ee3e68 100644 --- a/solr/core/src/java/org/apache/solr/search/BitDocSet.java +++ b/solr/core/src/java/org/apache/solr/search/BitDocSet.java @@ -20,6 +20,7 @@ import java.util.Collections; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Query; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.BitSetIterator; import org.apache.lucene.util.FixedBitSet; @@ -320,7 +321,7 @@ public long cost() { } @Override - public DocSetQuery makeQuery() { + public Query makeQuery() { return new DocSetQuery(this); } diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index 1cde70d0b6c0..87f276c537fd 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -65,9 +65,8 @@ import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.BitSetIterator; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.DocIdSetBuilder; import org.apache.lucene.util.LongValues; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.GroupParams; @@ -741,7 +740,7 @@ protected abstract static class AbstractCollapseCollector extends DelegatingColl protected boolean needsScores; // cached from scoreMode() // Results/accumulator - protected final FixedBitSet collapsedSet; // todo use DocIdSetBuilder instead + protected final DocIdSetBuilder collapsedSet; protected final BoostedDocsCollector boostedDocsCollector; protected FloatArrayList nullScores; protected float nullScore; @@ -758,7 +757,7 @@ protected AbstractCollapseCollector( this.nullPolicy = nullPolicy; this.needsScores4Collapsing = needsScores4Collapsing; - this.collapsedSet = new FixedBitSet(maxDoc); + this.collapsedSet = new DocIdSetBuilder(Math.max(1, maxDoc)); this.boostedDocsCollector = BoostedDocsCollector.build(boostDocsMap); } @@ -793,7 +792,7 @@ protected void doSetNextReader(LeafReaderContext context) throws IOException { @Override public final void complete() throws IOException { - DocIdSetIterator collapsedDocs = getCollapsedDisi(); + DocIdSetIterator collapsedDocs = getCollapsedDocs(); int nextDocBase = 0; int globalDoc; @@ -811,7 +810,8 @@ public final void complete() throws IOException { nextDocBase = ctxIdx + 1 < contexts.size() ? contexts.get(ctxIdx + 1).docBase : maxDoc; leafDelegate = delegate.getLeafCollector(context); if (delegate.scoreMode().needsScores()) { - leafDelegate.setScorer(getCollapsedScores(collapsedDocs, context)); + scorer = getCollapsedScores(collapsedDocs, context); + leafDelegate.setScorer(scorer); } } @@ -826,10 +826,41 @@ public final void complete() throws IOException { } /** - * Return a DISI of global doc IDs that the collector has matched. This is the first step of - * {@link #complete()}. + * Produce the final list of global docs that we'll pass on through to delegated/chained + * collectors. This is the first step of {@link #complete()}. */ - protected abstract DocIdSetIterator getCollapsedDisi() throws IOException; + protected DocIdSetIterator getCollapsedDocs() throws IOException { + // all subclasses have this common logic to do first regarding boosted/elevated & nullDoc: + + boostedDocsCollector.addBoostedDocsTo(collapsedSet); + + if (boostedDocsCollector.isBoostedNullGroup()) { + // If we're using IGNORE then no (matching) null docs were collected (by caller) + // If we're using EXPAND then all (matching) null docs were already collected (by us) + // ...and that's *good* because each is treated like its own group, our boosts don't + // matter + // We only have to worry about removing null docs when using COLLAPSE, in which case any + // boosted null doc means we clear the group head of the null group. + nullDoc = -1; + } + + if (nullDoc > -1) { + collapsedSet.grow(1).add(nullDoc); + } + + finishCollapsedSet(); + + DocIdSetIterator iterator = collapsedSet.build().iterator(); + return iterator != null ? iterator : DocIdSetIterator.empty(); + } + + /** + * Finishes adding docs to {@link #collapsedSet} so that it's ready. First step is usually to + * call {@link BoostedDocsCollector#visitBoostedGroupKeys(IntProcedure)} to ensure we don't add + * docs for these group keys as they have already been added by the caller, which are query + * elevated / boosted docs. Called by {@link #getCollapsedDocs()} + */ + protected abstract void finishCollapsedSet(); /** * Return a {@link Scorable} that provides the score for the current doc in the given {@code @@ -957,23 +988,17 @@ public void collect(int contextDoc) throws IOException { nullDoc = globalDoc; } } else if (nullPolicy == NullPolicy.EXPAND) { - collapsedSet.set(globalDoc); + collapsedSet.grow(1).add(globalDoc); nullScores.add(scorer.score()); } } @Override - protected DocIdSetIterator getCollapsedDisi() { - boostedDocsCollector.purgeGroupsThatHaveBoostedDocs( - collapsedSet, ords::remove, () -> nullDoc = -1); + protected void finishCollapsedSet() { + boostedDocsCollector.visitBoostedGroupKeys(ords::remove); - if (nullDoc > -1) { - collapsedSet.set(nullDoc); - } - - ords.forEachValue(collapsedSet::set); - - return new BitSetIterator(collapsedSet, 0); + DocIdSetBuilder.BulkAdder adder = collapsedSet.grow(ords.size()); + ords.forEachValue(adder::add); } private boolean collapsedScoresInitialized; @@ -1080,27 +1105,21 @@ public void collect(int contextDoc) throws IOException { this.nullDoc = globalDoc; } } else if (nullPolicy == NullPolicy.EXPAND) { - collapsedSet.set(globalDoc); + collapsedSet.grow(1).add(globalDoc); nullScores.add(scorer.score()); } } } @Override - protected DocIdSetIterator getCollapsedDisi() { - boostedDocsCollector.purgeGroupsThatHaveBoostedDocs( - collapsedSet, cmap::remove, () -> nullDoc = -1); - - if (nullDoc > -1) { - collapsedSet.set(nullDoc); - } + protected void finishCollapsedSet() { + boostedDocsCollector.visitBoostedGroupKeys(cmap::remove); + DocIdSetBuilder.BulkAdder adder = collapsedSet.grow(cmap.size()); for (IntLongCursor cursor : cmap) { // the low bits of the long is the global doc ID - collapsedSet.set((int) cursor.value); + adder.add((int) cursor.value); } - - return new BitSetIterator(collapsedSet, 0); } @Override @@ -1251,17 +1270,11 @@ public void collect(int contextDoc) throws IOException { protected abstract void collapse(int ord, int contextDoc, int globalDoc) throws IOException; @Override - protected DocIdSetIterator getCollapsedDisi() { - boostedDocsCollector.purgeGroupsThatHaveBoostedDocs( - collapsedSet, ords::remove, () -> nullDoc = -1); - - if (nullDoc > -1) { - collapsedSet.set(nullDoc); - } - - ords.forEachValue(collapsedSet::set); + protected void finishCollapsedSet() { + boostedDocsCollector.visitBoostedGroupKeys(ords::remove); - return new BitSetIterator(collapsedSet, 0); + DocIdSetBuilder.BulkAdder adder = collapsedSet.grow(ords.size()); + ords.forEachValue(adder::add); } private boolean collapsedScoresInitialized; @@ -1365,7 +1378,7 @@ protected void collapse(int ord, int contextDoc, int globalDoc) throws IOExcepti } } } else if (this.nullPolicy == NullPolicy.EXPAND) { - this.collapsedSet.set(globalDoc); + this.collapsedSet.grow(1).add(globalDoc); if (needsScores) { nullScores.add(scorer.score()); } @@ -1435,7 +1448,7 @@ protected void collapse(int ord, int contextDoc, int globalDoc) throws IOExcepti } } } else if (this.nullPolicy == NullPolicy.EXPAND) { - this.collapsedSet.set(globalDoc); + this.collapsedSet.grow(1).add(globalDoc); if (needsScores) { nullScores.add(scorer.score()); } @@ -1503,7 +1516,7 @@ protected void collapse(int ord, int contextDoc, int globalDoc) throws IOExcepti } } } else if (this.nullPolicy == NullPolicy.EXPAND) { - this.collapsedSet.set(globalDoc); + this.collapsedSet.grow(1).add(globalDoc); if (needsScores) { nullScores.add(scorer.score()); } @@ -1586,7 +1599,7 @@ protected void collapse(int ord, int contextDoc, int globalDoc) throws IOExcepti } } } else if (this.nullPolicy == NullPolicy.EXPAND) { - this.collapsedSet.set(globalDoc); + this.collapsedSet.grow(1).add(globalDoc); if (needsScores) { if (!needsScores4Collapsing) { score = scorer.score(); @@ -1684,7 +1697,7 @@ protected void collapse(int ord, int contextDoc, int globalDoc) throws IOExcepti } } } else if (this.nullPolicy == NullPolicy.EXPAND) { - this.collapsedSet.set(globalDoc); + this.collapsedSet.grow(1).add(globalDoc); if (needsScores) { if (!needsScores4Collapsing) { this.score = scorer.score(); @@ -1793,20 +1806,14 @@ public void collect(int contextDoc) throws IOException { } @Override - protected DocIdSetIterator getCollapsedDisi() { - boostedDocsCollector.purgeGroupsThatHaveBoostedDocs( - collapsedSet, cmap::remove, () -> nullDoc = -1); - - if (nullDoc > -1) { - collapsedSet.set(nullDoc); - } + protected void finishCollapsedSet() { + boostedDocsCollector.visitBoostedGroupKeys(cmap::remove); + DocIdSetBuilder.BulkAdder adder = collapsedSet.grow(cmap.size()); for (IntIntCursor cursor : cmap) { int pointer = cursor.value; - collapsedSet.set(docs.get(pointer)); + adder.add(docs.get(pointer)); } - - return new BitSetIterator(collapsedSet, 0); // cost is not useful here } @Override @@ -1911,7 +1918,7 @@ protected void collapseNullGroup(int contextDoc, int globalDoc) throws IOExcepti } } } else if (this.nullPolicy == NullPolicy.EXPAND) { - this.collapsedSet.set(globalDoc); + this.collapsedSet.grow(1).add(globalDoc); if (needsScores) { nullScores.add(scorer.score()); } @@ -1999,7 +2006,7 @@ protected void collapseNullGroup(int contextDoc, int globalDoc) throws IOExcepti } } } else if (this.nullPolicy == NullPolicy.EXPAND) { - this.collapsedSet.set(globalDoc); + this.collapsedSet.grow(1).add(globalDoc); if (needsScores) { nullScores.add(scorer.score()); } @@ -2107,7 +2114,7 @@ protected void collapseNullGroup(int contextDoc, int globalDoc) throws IOExcepti } } } else if (this.nullPolicy == NullPolicy.EXPAND) { - this.collapsedSet.set(globalDoc); + this.collapsedSet.grow(1).add(globalDoc); if (needsScores) { if (!needsScores4Collapsing) { score = scorer.score(); @@ -2214,7 +2221,7 @@ protected void collapseNullGroup(int contextDoc, int globalDoc) throws IOExcepti } } } else if (this.nullPolicy == NullPolicy.EXPAND) { - this.collapsedSet.set(globalDoc); + this.collapsedSet.grow(1).add(globalDoc); if (needsScores) { if (!needsScores4Collapsing) { this.score = scorer.score(); @@ -2819,12 +2826,6 @@ public boolean collectIfBoosted(int groupKey, int globalDoc) { public boolean collectInNullGroupIfBoosted(int globalDoc) { return false; } - - @Override - public void purgeGroupsThatHaveBoostedDocs( - final FixedBitSet collapsedSet, - final IntProcedure removeGroupKey, - final Runnable resetNullGroupHead) {} }; } @@ -2876,33 +2877,25 @@ public boolean collectInNullGroupIfBoosted(int globalDoc) { return false; } + // REMAINING METHODS ARE USED IN COMPLETE() TO CONSUME + + /** Add all collected boosted docs to the collapsed set. */ + public void addBoostedDocsTo(DocIdSetBuilder collapsedSet) { + DocIdSetBuilder.BulkAdder adder = collapsedSet.grow(boostedDocs.size()); + boostedDocs.forEach((IntProcedure) adder::add); + } + + /** Visit group keys that have boosted docs */ + public void visitBoostedGroupKeys(IntProcedure procedure) { + boostedKeys.forEach(procedure); + } + /** - * Kludgy API neccessary to deal with diff collectors/strategies using diff data structs for - * tracking collapse keys... + * Whether a boosted doc was collected in the null group. If true, the null group head should be + * reset so the boosted doc takes precedence. */ - public void purgeGroupsThatHaveBoostedDocs( - final FixedBitSet collapsedSet, - final IntProcedure removeGroupKey, - final Runnable resetNullGroupHead) { - // Add the (collected) boosted docs to the collapsedSet - boostedDocs.forEach( - new IntProcedure() { - @Override - public void apply(int globalDoc) { - collapsedSet.set(globalDoc); - } - }); - // Remove any group heads that are in the same groups as (collected) boosted documents. - boostedKeys.forEach(removeGroupKey); - if (boostedNullGroup) { - // If we're using IGNORE then no (matching) null docs were collected (by caller) - // If we're using EXPAND then all (matching) null docs were already collected (by us) - // ...and that's *good* because each is treated like it's own group, our boosts don't - // matter - // We only have to worry about removing null docs when using COLLAPSE, in which case any - // boosted null doc means we clear the group head of the null group.. - resetNullGroupHead.run(); - } + public boolean isBoostedNullGroup() { + return boostedNullGroup; } } diff --git a/solr/core/src/java/org/apache/solr/util/DynamicMap.java b/solr/core/src/java/org/apache/solr/util/DynamicMap.java index eac7c3f6d207..a3eb135ce6a9 100644 --- a/solr/core/src/java/org/apache/solr/util/DynamicMap.java +++ b/solr/core/src/java/org/apache/solr/util/DynamicMap.java @@ -51,4 +51,7 @@ default int mapExpectedElements(int expectedKeyMax) { // +2 let's us not to worry about which comparison operator to choose return threshold(expectedKeyMax) + 2; } + + /** The number of entries in this map. */ + int size(); } diff --git a/solr/core/src/java/org/apache/solr/util/IntFloatDynamicMap.java b/solr/core/src/java/org/apache/solr/util/IntFloatDynamicMap.java index 07b2d98dde82..3f9318b677fb 100644 --- a/solr/core/src/java/org/apache/solr/util/IntFloatDynamicMap.java +++ b/solr/core/src/java/org/apache/solr/util/IntFloatDynamicMap.java @@ -29,6 +29,7 @@ public class IntFloatDynamicMap implements DynamicMap { private float[] keyValues; private float emptyValue; private int threshold; + private int keyValuesSize; /** * Create map with expected max value of key. Although the map will automatically do resizing to @@ -52,6 +53,7 @@ private void upgradeToArray() { Arrays.fill(keyValues, emptyValue); } if (hashMap != null) { + keyValuesSize = hashMap.size(); hashMap.forEach((IntFloatProcedure) (key, value) -> keyValues[key] = value); hashMap = null; } @@ -73,7 +75,12 @@ public void put(int key, float value) { if (key >= keyValues.length) { growBuffer(key + 1); } - keyValues[key] = value; + if (keyValues[key] != value) { + if (keyValues[key] == emptyValue) { + keyValuesSize++; + } + keyValues[key] = value; + } } else { this.hashMap.put(key, value); this.maxSize = Math.max(key + 1, maxSize); @@ -108,9 +115,17 @@ public void forEachValue(FloatConsumer consumer) { public void remove(int key) { if (keyValues != null) { - if (key < keyValues.length) keyValues[key] = emptyValue; + if (key < keyValues.length && keyValues[key] != emptyValue) { + keyValues[key] = emptyValue; + keyValuesSize--; + } } else { hashMap.remove(key); } } + + @Override + public int size() { + return keyValues != null ? keyValuesSize : hashMap.size(); + } } diff --git a/solr/core/src/java/org/apache/solr/util/IntIntDynamicMap.java b/solr/core/src/java/org/apache/solr/util/IntIntDynamicMap.java index aa498f39b182..09d384869e1a 100644 --- a/solr/core/src/java/org/apache/solr/util/IntIntDynamicMap.java +++ b/solr/core/src/java/org/apache/solr/util/IntIntDynamicMap.java @@ -30,6 +30,7 @@ public class IntIntDynamicMap implements DynamicMap { private int[] keyValues; private int emptyValue; private int threshold; + private int keyValuesSize; /** * Create map with expected max value of key. Although the map will automatically do resizing to @@ -53,6 +54,7 @@ private void upgradeToArray() { Arrays.fill(keyValues, emptyValue); } if (hashMap != null) { + keyValuesSize = hashMap.size(); hashMap.forEach((IntIntProcedure) (key, value) -> keyValues[key] = value); hashMap = null; } @@ -74,7 +76,12 @@ public void put(int key, int value) { if (key >= keyValues.length) { growBuffer(key + 1); } - keyValues[key] = value; + if (keyValues[key] != value) { + if (keyValues[key] == emptyValue) { + keyValuesSize++; + } + keyValues[key] = value; + } } else { this.maxSize = Math.max(key + 1, maxSize); this.hashMap.put(key, value); @@ -109,9 +116,17 @@ public void forEachValue(IntConsumer consumer) { public void remove(int key) { if (keyValues != null) { - if (key < keyValues.length) keyValues[key] = emptyValue; + if (key < keyValues.length && keyValues[key] != emptyValue) { + keyValues[key] = emptyValue; + keyValuesSize--; + } } else { hashMap.remove(key); } } + + @Override + public int size() { + return keyValues != null ? keyValuesSize : hashMap.size(); + } } diff --git a/solr/core/src/java/org/apache/solr/util/IntLongDynamicMap.java b/solr/core/src/java/org/apache/solr/util/IntLongDynamicMap.java index f9fcbabe3335..141e091f61c1 100644 --- a/solr/core/src/java/org/apache/solr/util/IntLongDynamicMap.java +++ b/solr/core/src/java/org/apache/solr/util/IntLongDynamicMap.java @@ -30,6 +30,7 @@ public class IntLongDynamicMap implements DynamicMap { private long[] keyValues; private long emptyValue; private int threshold; + private int keyValuesSize; /** * Create map with expected max value of key. Although the map will automatically do resizing to @@ -53,6 +54,7 @@ private void upgradeToArray() { Arrays.fill(keyValues, emptyValue); } if (hashMap != null) { + keyValuesSize = hashMap.size(); hashMap.forEach((IntLongProcedure) (key, value) -> keyValues[key] = value); hashMap = null; } @@ -74,7 +76,12 @@ public void put(int key, long value) { if (key >= keyValues.length) { growBuffer(key + 1); } - keyValues[key] = value; + if (keyValues[key] != value) { + if (keyValues[key] == emptyValue) { + keyValuesSize++; + } + keyValues[key] = value; + } } else { this.maxSize = Math.max(key + 1, maxSize); this.hashMap.put(key, value); @@ -109,9 +116,17 @@ public void forEachValue(LongConsumer consumer) { public void remove(int key) { if (keyValues != null) { - if (key < keyValues.length) keyValues[key] = emptyValue; + if (key < keyValues.length && keyValues[key] != emptyValue) { + keyValues[key] = emptyValue; + keyValuesSize--; + } } else { hashMap.remove(key); } } + + @Override + public int size() { + return keyValues != null ? keyValuesSize : hashMap.size(); + } } From 80e31fa9d0c29a5e80efd57525538dfd5c0ad64c Mon Sep 17 00:00:00 2001 From: David Smiley Date: Tue, 17 Feb 2026 10:51:41 -0500 Subject: [PATCH 21/21] builders should be records. initializeCollection clarity --- .../solr/search/CollapsingQParserPlugin.java | 136 +++++++++--------- 1 file changed, 65 insertions(+), 71 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index 87f276c537fd..ee69e1d646a1 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -205,6 +205,7 @@ public enum GroupHeadSelectorType { MAX, SORT, SCORE, + /** For use outside the Solr codebase */ CUSTOM; public static final EnumSet MIN_MAX = EnumSet.of(MIN, MAX); } @@ -226,6 +227,7 @@ public record GroupHeadSelector(GroupHeadSelectorType type, String selectorText) /** returns a new GroupHeadSelector based on the specified local params */ public static GroupHeadSelector build(final SolrParams localParams) { + // note: subclasses using CUSTOM should do their own build logic final String sortString = StrUtils.isBlank(localParams.get(SORT)) ? null : localParams.get(SORT); final String max = StrUtils.isBlank(localParams.get("max")) ? null : localParams.get("max"); @@ -501,16 +503,14 @@ protected DelegatingCollector getFilterCollector(SolrIndexSearcher searcher) groupHeadSelector, sortSpec, funcQuery, searcher)); } - var builder = new OrdFieldCollectorBuilder(); - builder.searcher = searcher; - builder.groupHeadSelector = groupHeadSelector; - builder.nullPolicy = nullPolicy; - builder.needsScores4Collapsing = needsScores4Collapsing; - builder.boostDocsMap = boostDocs; - - builder.collapseValuesProducer = docValuesProducer; - - return builder.build(sortSpec, minMaxFieldType, funcQuery); + return new OrdFieldCollectorBuilder( + groupHeadSelector, + docValuesProducer, + nullPolicy, + needsScores4Collapsing, + boostDocs, + searcher) + .build(sortSpec, minMaxFieldType, funcQuery); } else if (isNumericCollapsible(collapseFieldType)) { @@ -526,17 +526,15 @@ protected DelegatingCollector getFilterCollector(SolrIndexSearcher searcher) groupHeadSelector, sortSpec, funcQuery, searcher)); } - var builder = new IntFieldCollectorBuilder(); - builder.searcher = searcher; - builder.groupHeadSelector = groupHeadSelector; - builder.nullPolicy = nullPolicy; - builder.needsScores4Collapsing = needsScores4Collapsing; - builder.boostDocsMap = boostDocs; - - builder.collapseField = collapseField; - builder.initialSize = initialSize; - - return builder.build(sortSpec, minMaxFieldType, funcQuery); + return new IntFieldCollectorBuilder( + groupHeadSelector, + nullPolicy, + collapseField, + initialSize, + needsScores4Collapsing, + boostDocs, + searcher) + .build(sortSpec, minMaxFieldType, funcQuery); } else { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, @@ -766,9 +764,12 @@ public ScoreMode scoreMode() { return needsScores4Collapsing ? ScoreMode.COMPLETE : super.scoreMode(); } - /** Initialize data structures for collection. Called once before collecting begins. */ - protected void initializeCollection() { - this.needsScores = scoreMode().needsScores(); + /** + * Initialize data structures for collection. Called once before collecting begins. This is + * useful for initialization dependent on {@link #needsScores}, which isn't known in the + * constructor. + */ + protected void initializeCollection(boolean needsScores) { if (needsScores && nullPolicy == NullPolicy.EXPAND) { this.nullScores = new FloatArrayList(); } @@ -776,8 +777,8 @@ protected void initializeCollection() { @Override protected void doSetNextReader(LeafReaderContext context) throws IOException { - if (this.context == null) { - initializeCollection(); + if (this.context == null) { // first time detection + initializeCollection(this.needsScores = scoreMode().needsScores()); } // Do NOT set leafDelegate (calling super() would do this). @@ -1142,16 +1143,16 @@ protected float computeScore(int globalDoc) throws IOException { } /** Builder for {@link OrdFieldValueCollector}. */ - protected static class OrdFieldCollectorBuilder { - public GroupHeadSelector groupHeadSelector; - public DocValuesProducer collapseValuesProducer; - public NullPolicy nullPolicy; - public boolean needsScores4Collapsing; - public IntIntHashMap boostDocsMap; - public SolrIndexSearcher searcher; + public record OrdFieldCollectorBuilder( + GroupHeadSelector groupHeadSelector, + DocValuesProducer collapseValuesProducer, + NullPolicy nullPolicy, + boolean needsScores4Collapsing, + IntIntHashMap boostDocsMap, + SolrIndexSearcher searcher) { /** Builds the appropriate OrdFieldValueCollector subclass based on the selection strategy. */ - public OrdFieldValueCollector build( + public DelegatingCollector build( SortSpec sortSpec, FieldType fieldType, FunctionQuery funcQuery) throws IOException { if (null != sortSpec) { @@ -1216,15 +1217,15 @@ private void initCollapseValues() throws IOException { } @Override - protected void initializeCollection() { - super.initializeCollection(); - if (this.needsScores) { + protected void initializeCollection(boolean needsScores) { + super.initializeCollection(needsScores); + if (needsScores) { this.scores = new IntFloatDynamicMap(valueCount, 0.0f); } } @Override - public void doSetNextReader(LeafReaderContext context) throws IOException { + protected void doSetNextReader(LeafReaderContext context) throws IOException { super.doSetNextReader(context); initSegmentValues(context); } @@ -1284,18 +1285,11 @@ protected Scorable getCollapsedScores(DocIdSetIterator disi, LeafReaderContext c throws IOException { if (!collapsedScoresInitialized) { collapsedScoresInitialized = true; - collapseValues = collapseValuesProducer.getSorted(null); - if (collapseValues instanceof MultiDocValues.MultiSortedDocValues) { - this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues) collapseValues; - this.ordinalMap = multiSortedDocValues.mapping; - } - } - if (ordinalMap != null) { - this.segmentValues = this.multiSortedDocValues.values[context.ord]; - this.segmentOrdinalMap = this.ordinalMap.getGlobalOrds(context.ord); - } else { - this.segmentValues = collapseValues; + initCollapseValues(); } + + initSegmentValues(context); + return new CachedScoreScorable(disi) { @Override protected float computeScore(int globalDoc) throws IOException { @@ -1347,7 +1341,7 @@ public OrdIntCollector(OrdFieldCollectorBuilder ctx) throws IOException { } @Override - public void doSetNextReader(LeafReaderContext context) throws IOException { + protected void doSetNextReader(LeafReaderContext context) throws IOException { super.doSetNextReader(context); this.minMaxValues = DocValues.getNumeric(context.reader(), this.field); } @@ -1415,7 +1409,7 @@ public OrdFloatCollector(OrdFieldCollectorBuilder ctx) throws IOException { } @Override - public void doSetNextReader(LeafReaderContext context) throws IOException { + protected void doSetNextReader(LeafReaderContext context) throws IOException { super.doSetNextReader(context); this.minMaxValues = DocValues.getNumeric(context.reader(), this.field); } @@ -1485,7 +1479,7 @@ public OrdLongCollector(OrdFieldCollectorBuilder ctx) throws IOException { } @Override - public void doSetNextReader(LeafReaderContext context) throws IOException { + protected void doSetNextReader(LeafReaderContext context) throws IOException { super.doSetNextReader(context); this.minMaxVals = DocValues.getNumeric(context.reader(), this.field); } @@ -1560,7 +1554,7 @@ public OrdValueSourceCollector(OrdFieldCollectorBuilder ctx, FunctionQuery funcQ } @Override - public void doSetNextReader(LeafReaderContext context) throws IOException { + protected void doSetNextReader(LeafReaderContext context) throws IOException { super.doSetNextReader(context); functionValues = this.valueSource.getValues(rcontext, context); } @@ -1632,7 +1626,7 @@ public OrdSortSpecCollector(OrdFieldCollectorBuilder ctx, SortSpec sortSpec) } @Override - public void doSetNextReader(LeafReaderContext context) throws IOException { + protected void doSetNextReader(LeafReaderContext context) throws IOException { super.doSetNextReader(context); compareState.setNextReader(context); } @@ -1709,16 +1703,16 @@ protected void collapse(int ord, int contextDoc, int globalDoc) throws IOExcepti } /** Builder for {@link IntFieldValueCollector}. */ - protected static class IntFieldCollectorBuilder { - public GroupHeadSelector groupHeadSelector; - public NullPolicy nullPolicy; - public String collapseField; - public int initialSize; - public boolean needsScores4Collapsing; - public IntIntHashMap boostDocsMap; - public IndexSearcher searcher; - - public IntFieldValueCollector build( + public record IntFieldCollectorBuilder( + GroupHeadSelector groupHeadSelector, + NullPolicy nullPolicy, + String collapseField, + int initialSize, + boolean needsScores4Collapsing, + IntIntHashMap boostDocsMap, + IndexSearcher searcher) { + + public DelegatingCollector build( SortSpec sortSpec, FieldType fieldType, FunctionQuery funcQuery) throws IOException { if (null != sortSpec) { return new IntSortSpecCollector(this, sortSpec); @@ -1769,8 +1763,8 @@ protected IntFieldValueCollector(IntFieldCollectorBuilder ctx) throws IOExceptio } @Override - protected void initializeCollection() { - super.initializeCollection(); + protected void initializeCollection(boolean needsScores) { + super.initializeCollection(needsScores); if (needsScores) { this.scores = new IntFloatDynamicMap(initialSize, 0.0f); } @@ -1782,7 +1776,7 @@ protected abstract void collapse(int collapseKey, int contextDoc, int globalDoc) throws IOException; @Override - public void doSetNextReader(LeafReaderContext context) throws IOException { + protected void doSetNextReader(LeafReaderContext context) throws IOException { super.doSetNextReader(context); this.collapseValues = DocValues.getNumeric(context.reader(), this.collapseField); } @@ -1867,7 +1861,7 @@ public IntIntCollector(IntFieldCollectorBuilder ctx) throws IOException { } @Override - public void doSetNextReader(LeafReaderContext context) throws IOException { + protected void doSetNextReader(LeafReaderContext context) throws IOException { super.doSetNextReader(context); this.minMaxVals = DocValues.getNumeric(context.reader(), this.field); } @@ -1956,7 +1950,7 @@ public IntFloatCollector(IntFieldCollectorBuilder ctx) throws IOException { } @Override - public void doSetNextReader(LeafReaderContext context) throws IOException { + protected void doSetNextReader(LeafReaderContext context) throws IOException { super.doSetNextReader(context); this.minMaxVals = DocValues.getNumeric(context.reader(), this.field); } @@ -2050,7 +2044,7 @@ public IntValueSourceCollector(IntFieldCollectorBuilder ctx, FunctionQuery funcQ } @Override - public void doSetNextReader(LeafReaderContext context) throws IOException { + protected void doSetNextReader(LeafReaderContext context) throws IOException { super.doSetNextReader(context); functionValues = this.valueSource.getValues(rcontext, context); } @@ -2145,7 +2139,7 @@ public IntSortSpecCollector(IntFieldCollectorBuilder ctx, SortSpec sortSpec) } @Override - public void doSetNextReader(LeafReaderContext context) throws IOException { + protected void doSetNextReader(LeafReaderContext context) throws IOException { super.doSetNextReader(context); compareState.setNextReader(context); }