diff --git a/solr/core/src/java/org/apache/solr/schema/FieldProperties.java b/solr/core/src/java/org/apache/solr/schema/FieldProperties.java index 91f3caa38e63..ce502a56bcbf 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldProperties.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldProperties.java @@ -52,6 +52,7 @@ public abstract class FieldProperties { protected static final int USE_DOCVALUES_AS_STORED = 0b100000000000000000; protected static final int LARGE_FIELD = 0b1000000000000000000; protected static final int UNINVERTIBLE = 0b10000000000000000000; + protected static final int DOC_VALUES_SKIP_LIST = 0b100000000000000000000; static final String[] propertyNames = { "indexed", @@ -73,7 +74,8 @@ public abstract class FieldProperties { "termPayloads", "useDocValuesAsStored", "large", - "uninvertible" + "uninvertible", + "skipList" }; static final Map propertyMap = new HashMap<>(); diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java index 1452233beee3..24546627ece9 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java @@ -42,6 +42,8 @@ import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.document.Field; import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.DocValuesSkipIndexType; +import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; import org.apache.lucene.index.Term; @@ -1148,6 +1150,26 @@ public void checkSchemaField(final SchemaField field) { if (field.hasDocValues()) { checkSupportsDocValues(); } + if (field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) { + if (!field.hasDocValues()) { + throw new SolrException( + ErrorCode.SERVER_ERROR, + "Field " + field.getName() + " cannot use skipList=true without docValues=true"); + } + final DocValuesType docValuesType = getDocValuesTypeForSkipIndex(field); + if (docValuesType != DocValuesType.NUMERIC && docValuesType != DocValuesType.SORTED_NUMERIC) { + throw new SolrException( + ErrorCode.SERVER_ERROR, + "Field " + + field.getName() + + " of type " + + this + + " cannot use skipList=true because it is currently only supported on PointField" + + "-based numeric and date fields; docValues type " + + docValuesType + + " is unsupported"); + } + } if (field.isLarge() && field.multiValued()) { throw new SolrException( ErrorCode.SERVER_ERROR, "Field type " + this + " is 'large'; can't support multiValued"); @@ -1167,6 +1189,16 @@ protected void checkSupportsDocValues() { ErrorCode.SERVER_ERROR, "Field type " + this + " does not support doc values"); } + /** + * Returns the concrete docValues type used for the field when indexing. Field types that support + * {@code skipList=true} must override this method so schema validation can reject unsupported + * docValues shapes during core load. The default implementation means the field type does not + * currently support {@code skipList=true}. + */ + protected DocValuesType getDocValuesTypeForSkipIndex(SchemaField field) { + return DocValuesType.NONE; + } + /** * Returns whether this field type should enable docValues by default for schemaVersion >= 1.7. * This should not be enabled for fields that did not have docValues implemented by Solr 9.7, as diff --git a/solr/core/src/java/org/apache/solr/schema/PointField.java b/solr/core/src/java/org/apache/solr/schema/PointField.java index e74e73f13207..ae126579b10c 100644 --- a/solr/core/src/java/org/apache/solr/schema/PointField.java +++ b/solr/core/src/java/org/apache/solr/schema/PointField.java @@ -26,6 +26,7 @@ import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.IndexOrDocValuesQuery; @@ -292,7 +293,10 @@ public List createFields(SchemaField sf, Object value) { assert numericValue instanceof Double; bits = Double.doubleToLongBits(numericValue.doubleValue()); } - fields.add(new NumericDocValuesField(sf.getName(), bits)); + fields.add( + sf.hasDocValuesSkipList() + ? NumericDocValuesField.indexedField(sf.getName(), bits) + : new NumericDocValuesField(sf.getName(), bits)); } else { // MultiValued if (numericValue instanceof Integer || numericValue instanceof Long) { @@ -303,7 +307,10 @@ public List createFields(SchemaField sf, Object value) { assert numericValue instanceof Double; bits = NumericUtils.doubleToSortableLong(numericValue.doubleValue()); } - fields.add(new SortedNumericDocValuesField(sf.getName(), bits)); + fields.add( + sf.hasDocValuesSkipList() + ? SortedNumericDocValuesField.indexedField(sf.getName(), bits) + : new SortedNumericDocValuesField(sf.getName(), bits)); } } if (sf.stored()) { @@ -314,6 +321,11 @@ public List createFields(SchemaField sf, Object value) { protected abstract StoredField getStoredField(SchemaField sf, Object value); + @Override + protected DocValuesType getDocValuesTypeForSkipIndex(SchemaField field) { + return field.multiValued() ? DocValuesType.SORTED_NUMERIC : DocValuesType.NUMERIC; + } + @Override public SortField getSortField(SchemaField field, boolean top) { return getNumericSort(field, getNumberType(), top); diff --git a/solr/core/src/java/org/apache/solr/schema/SchemaField.java b/solr/core/src/java/org/apache/solr/schema/SchemaField.java index ef3c5559affe..5c186f27e210 100644 --- a/solr/core/src/java/org/apache/solr/schema/SchemaField.java +++ b/solr/core/src/java/org/apache/solr/schema/SchemaField.java @@ -244,6 +244,10 @@ public String getDocValuesFormat() { return (String) args.getOrDefault(DOC_VALUES_FORMAT, type.getDocValuesFormat()); } + public boolean hasDocValuesSkipList() { + return (properties & DOC_VALUES_SKIP_LIST) != 0; + } + /** * Sanity checks that the properties of this field type are plausible for a field that may be used * in sorting, throwing an appropriate exception (including the field name) if it is not. @@ -466,6 +470,7 @@ public SimpleOrderedMap getNamedPropertyValues(boolean showDefaults) { properties.add(getPropertyName(REQUIRED), isRequired()); properties.add(getPropertyName(TOKENIZED), isTokenized()); properties.add(getPropertyName(USE_DOCVALUES_AS_STORED), useDocValuesAsStored()); + properties.add(getPropertyName(DOC_VALUES_SKIP_LIST), hasDocValuesSkipList()); // The BINARY property is always false // properties.add(getPropertyName(BINARY), isBinary()); } else { @@ -534,7 +539,7 @@ public DocValuesType docValuesType() { @Override public DocValuesSkipIndexType docValuesSkipIndexType() { - return DocValuesSkipIndexType.NONE; + return hasDocValuesSkipList() ? DocValuesSkipIndexType.RANGE : DocValuesSkipIndexType.NONE; } @Override diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-unsupported-skip-list.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-unsupported-skip-list.xml new file mode 100644 index 000000000000..7c9097c65855 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-unsupported-skip-list.xml @@ -0,0 +1,23 @@ + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema_codec.xml b/solr/core/src/test-files/solr/collection1/conf/schema_codec.xml index c442cdd7bfca..2abc769eaa1c 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema_codec.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema_codec.xml @@ -25,6 +25,8 @@ + + @@ -41,6 +43,8 @@ + + @@ -49,6 +53,8 @@ + + string_f diff --git a/solr/core/src/test-files/solr/collection1/conf/schema_postingsformat.xml b/solr/core/src/test-files/solr/collection1/conf/schema_postingsformat.xml index 32dd7403d28b..00dd2887e7b7 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema_postingsformat.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema_postingsformat.xml @@ -20,6 +20,8 @@ + + @@ -30,6 +32,8 @@ + + @@ -37,5 +41,7 @@ + + diff --git a/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java b/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java index 919777dc3e79..7dd60f504fe0 100644 --- a/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java +++ b/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java @@ -24,6 +24,8 @@ import org.apache.lucene.codecs.lucene104.Lucene104Codec; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; +import org.apache.lucene.index.DocValuesSkipIndexType; +import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.tests.util.TestUtil; @@ -107,6 +109,26 @@ public void testDynamicFieldsDocValuesFormats() { assertEquals("Asserting", format.getDocValuesFormatForField("bar_direct").getName()); } + public void testDocValuesSkipListPersistsFieldInfo() throws Exception { + assertU(delQ("*:*")); + assertU(commit()); + assertU(add(doc("string_f", "id", "int_skip_f", "7", "long_skip_mv_f", "11"))); + assertU(commit()); + + h.getCore() + .withSearcher( + searcher -> { + FieldInfos fieldInfos = FieldInfos.getMergedFieldInfos(searcher.getIndexReader()); + assertEquals( + DocValuesSkipIndexType.RANGE, + fieldInfos.fieldInfo("int_skip_f").docValuesSkipIndexType()); + assertEquals( + DocValuesSkipIndexType.RANGE, + fieldInfos.fieldInfo("long_skip_mv_f").docValuesSkipIndexType()); + return null; + }); + } + private void reloadCoreAndRecreateIndex() { h.getCoreContainer().reload(h.coreName); assertU(delQ("*:*")); diff --git a/solr/core/src/test/org/apache/solr/rest/schema/TestFieldResource.java b/solr/core/src/test/org/apache/solr/rest/schema/TestFieldResource.java index 5eb97247a399..1e629f2f8f6c 100644 --- a/solr/core/src/test/org/apache/solr/rest/schema/TestFieldResource.java +++ b/solr/core/src/test/org/apache/solr/rest/schema/TestFieldResource.java @@ -25,7 +25,7 @@ public void testGetField() { assertQ( "/schema/fields/test_postv?indent=on&wt=xml&showDefaults=true", "count(/response/lst[@name='field']) = 1", - "count(/response/lst[@name='field']/*) = 19", + "count(/response/lst[@name='field']/*) = 20", "/response/lst[@name='field']/str[@name='name'] = 'test_postv'", "/response/lst[@name='field']/str[@name='type'] = 'text'", "/response/lst[@name='field']/bool[@name='indexed'] = 'true'", @@ -44,7 +44,8 @@ public void testGetField() { "/response/lst[@name='field']/bool[@name='large'] = 'false'", "/response/lst[@name='field']/bool[@name='required'] = 'false'", "/response/lst[@name='field']/bool[@name='tokenized'] = 'true'", - "/response/lst[@name='field']/bool[@name='useDocValuesAsStored'] = 'true'"); + "/response/lst[@name='field']/bool[@name='useDocValuesAsStored'] = 'true'", + "/response/lst[@name='field']/bool[@name='skipList'] = 'false'"); } @Test diff --git a/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java b/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java index 3f3e372399ae..13fa19810f0f 100644 --- a/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java +++ b/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java @@ -121,6 +121,12 @@ public void testDocValuesUnsupported() throws Exception { doTest("bad-schema-unsupported-docValues.xml", "does not support doc values"); } + public void testDocValuesSkipListUnsupported() throws Exception { + doTest( + "bad-schema-unsupported-skip-list.xml", + "currently only supported on PointField-based numeric and date fields"); + } + public void testRootTypeMissmatchWithUniqueKey() throws Exception { doTest( "bad-schema-uniquekey-diff-type-root.xml", diff --git a/solr/core/src/test/org/apache/solr/schema/TestSchemaField.java b/solr/core/src/test/org/apache/solr/schema/TestSchemaField.java index e404c48a238a..f4553e44bc0d 100644 --- a/solr/core/src/test/org/apache/solr/schema/TestSchemaField.java +++ b/solr/core/src/test/org/apache/solr/schema/TestSchemaField.java @@ -75,6 +75,8 @@ public void testFields() { assertFieldFormats("str_none_asserting_f", null, "Asserting"); assertFieldFormats("str_standard_asserting_f", "Lucene84", "Asserting"); + assertFieldHasSkipList("int_skip_f", true); + assertFieldHasSkipList("long_skip_mv_f", true); } public void testDynamicFields() { @@ -84,6 +86,23 @@ public void testDynamicFields() { assertFieldFormats("any_asserting", null, "Asserting"); assertFieldFormats("any_simple", "Direct", "Lucene80"); + assertFieldHasSkipList("any_skip_i", true); + assertFieldHasSkipList("any_skip_l", true); + } + + private void assertFieldHasSkipList(String fieldName, boolean expectedSkipList) { + SchemaField field = h.getCore().getLatestSchema().getField(fieldName); + assertNotNull("Field " + fieldName + " not found - schema got changed?", field); + final String skipListPropertyName = + FieldProperties.getPropertyName(FieldProperties.DOC_VALUES_SKIP_LIST); + assertEquals( + "Field " + field.getName() + " wrong " + skipListPropertyName + " value", + expectedSkipList, + field.hasDocValuesSkipList()); + assertEquals( + "Field " + field.getName() + " wrong schema property value for " + skipListPropertyName, + expectedSkipList, + field.getNamedPropertyValues(true).get(skipListPropertyName)); } private void assertFieldFormats(