Use stock DVSkipper competitive iterator instead of SecondarySortIterator (#138087)

We seem to get the pruning performance back on the stock lucene iterator
with apache/lucene#15397, which is also present in
our forked version of the code.

Author: romseygeek

server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java

@@ -8,15 +8,13 @@
  */
 package org.elasticsearch.index.fielddata.fieldcomparator;
 
-import org.apache.lucene.index.DocValuesSkipper;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.FieldComparator;
 import org.apache.lucene.search.LeafFieldComparator;
 import org.apache.lucene.search.LongValues;
 import org.apache.lucene.search.Pruning;
-import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.BitSet;
 import org.elasticsearch.common.time.DateUtils;
@@ -31,7 +29,6 @@
 import org.elasticsearch.index.fielddata.SortedNumericLongValues;
 import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData;
 import org.elasticsearch.lucene.comparators.XLongComparator;
-import org.elasticsearch.lucene.comparators.XNumericComparator;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.sort.BucketedSort;
@@ -114,40 +111,6 @@ public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws I
                     protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
                         return wrap(getLongValues(context, lMissingValue), maxDoc);
                     }
-
-                    @Override
-                    protected XNumericComparator<Long>.CompetitiveDISIBuilder buildCompetitiveDISIBuilder(LeafReaderContext context)
-                        throws IOException {
-                        Sort indexSort = context.reader().getMetaData().sort();
-                        if (indexSort == null) {
-                            return super.buildCompetitiveDISIBuilder(context);
-                        }
-                        SortField[] sortFields = indexSort.getSort();
-                        if (sortFields.length != 2) {
-                            return super.buildCompetitiveDISIBuilder(context);
-                        }
-                        if (sortFields[1].getField().equals(field) == false) {
-                            return super.buildCompetitiveDISIBuilder(context);
-                        }
-                        DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field);
-                        DocValuesSkipper primaryFieldSkipper = context.reader().getDocValuesSkipper(sortFields[0].getField());
-                        if (primaryFieldSkipper == null || skipper.docCount() != maxDoc || primaryFieldSkipper.docCount() != maxDoc) {
-                            return super.buildCompetitiveDISIBuilder(context);
-                        }
-                        return new CompetitiveDISIBuilder(this) {
-                            @Override
-                            protected int docCount() {
-                                return skipper.docCount();
-                            }
-
-                            @Override
-                            protected void doUpdateCompetitiveIterator() {
-                                competitiveIterator.update(
-                                    new SecondarySortIterator(docValues, skipper, primaryFieldSkipper, minValueAsLong, maxValueAsLong)
-                                );
-                            }
-                        };
-                    }
                 };
             }
         };

server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/SecondarySortIterator.java

@@ -41,6 +41,7 @@
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.test.index.IndexVersionUtils;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.junit.Assert;
 
 import java.io.IOException;
 import java.math.BigDecimal;
@@ -934,7 +935,7 @@ protected List<SortShortcutSupport> getSortShortcutSupport() {
                 b -> b.field("type", "date").field("ignore_malformed", false),
                 b -> b.startObject("host.name").field("type", "keyword").endObject(),
                 b -> b.field("@timestamp", "2025-10-30T00:00:00").field("host.name", "foo"),
-                checkClass
+                Assert::assertNotNull
             ),
             new SortShortcutSupport(b -> b.field("type", "date"), b -> b.field("field", "2025-10-30T00:00:00"), true),
             new SortShortcutSupport(b -> b.field("type", "date_nanos"), b -> b.field("field", "2025-10-30T00:00:00"), true),