Skip to content

Commit d2facd5

Browse files
mark-vieiramark-vieira
authored
[9.1] Add percolator field fallback compatibility (#137466) (#138315)
1 parent c1a46d5 commit d2facd5

File tree

3 files changed

+91
-84
lines changed

3 files changed

+91
-84
lines changed

‎modules/percolator/src/main/java/module-info.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,5 @@
1616
requires org.apache.lucene.memory;
1717
requires org.apache.lucene.queries;
1818
requires org.apache.lucene.sandbox;
19+
requires org.elasticsearch.logging;
1920
}

‎modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java‎

Lines changed: 74 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import org.elasticsearch.TransportVersion;
3838
import org.elasticsearch.TransportVersions;
3939
import org.elasticsearch.action.get.GetRequest;
40+
import org.elasticsearch.common.CheckedSupplier;
4041
import org.elasticsearch.common.bytes.BytesReference;
4142
import org.elasticsearch.common.io.stream.InputStreamStreamInput;
4243
import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
@@ -63,6 +64,10 @@
6364
import org.elasticsearch.index.query.SearchExecutionContext;
6465
import org.elasticsearch.indices.breaker.CircuitBreakerService;
6566
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
67+
import org.elasticsearch.logging.LogManager;
68+
import org.elasticsearch.logging.Logger;
69+
import org.elasticsearch.search.lookup.Source;
70+
import org.elasticsearch.search.lookup.SourceProvider;
6671
import org.elasticsearch.xcontent.ConstructingObjectParser;
6772
import org.elasticsearch.xcontent.NamedXContentRegistry;
6873
import org.elasticsearch.xcontent.ParseField;
@@ -86,6 +91,8 @@
8691
import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
8792

8893
public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBuilder> {
94+
private static final Logger LOGGER = LogManager.getLogger(PercolateQueryBuilder.class);
95+
8996
public static final String NAME = "percolate";
9097

9198
static final ParseField DOCUMENT_FIELD = new ParseField("document");
@@ -557,41 +564,81 @@ static PercolateQuery.QueryStore createStore(MappedFieldType queryBuilderFieldTy
557564
return docId -> {
558565
if (binaryDocValues.advanceExact(docId)) {
559566
BytesRef qbSource = binaryDocValues.binaryValue();
560-
try (
561-
InputStream in = new ByteArrayInputStream(qbSource.bytes, qbSource.offset, qbSource.length);
562-
StreamInput input = new NamedWriteableAwareStreamInput(new InputStreamStreamInput(in, qbSource.length), registry)
563-
) {
564-
// Query builder's content is stored via BinaryFieldMapper, which has a custom encoding
565-
// to encode multiple binary values into a single binary doc values field.
566-
// This is the reason we need to first read the number of values and
567-
// then the length of the field value in bytes.
568-
int numValues = input.readVInt();
569-
assert numValues == 1;
570-
int valueLength = input.readVInt();
571-
assert valueLength > 0;
572-
573-
TransportVersion transportVersion;
574-
if (indexVersion.before(IndexVersions.V_8_8_0)) {
575-
transportVersion = TransportVersion.fromId(indexVersion.id());
576-
} else {
577-
transportVersion = TransportVersion.readVersion(input);
567+
QueryBuilder queryBuilder = readQueryBuilder(qbSource, registry, indexVersion, () -> {
568+
// query builder is written in an incompatible format, fall-back to reading it from source
569+
if (context.isSourceEnabled() == false) {
570+
throw new ElasticsearchException(
571+
"Unable to read percolator query. Original transport version is incompatible and source is "
572+
+ "unavailable on index [{}].",
573+
context.index().getName()
574+
);
578575
}
579-
// set the transportversion here - only read vints so far, so can change the version freely at this point
580-
input.setTransportVersion(transportVersion);
581-
582-
QueryBuilder queryBuilder = input.readNamedWriteable(QueryBuilder.class);
583-
assert in.read() == -1;
584-
queryBuilder = Rewriteable.rewrite(queryBuilder, context);
585-
return queryBuilder.toQuery(context);
586-
}
587-
576+
LOGGER.warn(
577+
"Reading percolator query from source. For best performance, reindexing of index [{}] is required.",
578+
context.index().getName()
579+
);
580+
SourceProvider sourceProvider = context.createSourceProvider();
581+
Source source = sourceProvider.getSource(ctx, docId);
582+
SourceToParse sourceToParse = new SourceToParse(
583+
String.valueOf(docId),
584+
source.internalSourceRef(),
585+
source.sourceContentType()
586+
);
587+
588+
return context.parseDocument(sourceToParse).rootDoc().getBinaryValue(queryBuilderFieldType.name());
589+
});
590+
591+
queryBuilder = Rewriteable.rewrite(queryBuilder, context);
592+
return queryBuilder.toQuery(context);
588593
} else {
589594
return null;
590595
}
591596
};
592597
};
593598
}
594599

600+
private static QueryBuilder readQueryBuilder(
601+
BytesRef bytesRef,
602+
NamedWriteableRegistry registry,
603+
IndexVersion indexVersion,
604+
CheckedSupplier<BytesRef, IOException> fallbackSource
605+
) throws IOException {
606+
try (
607+
InputStream in = new ByteArrayInputStream(bytesRef.bytes, bytesRef.offset, bytesRef.length);
608+
StreamInput input = new NamedWriteableAwareStreamInput(new InputStreamStreamInput(in, bytesRef.length), registry)
609+
) {
610+
// Query builder's content is stored via BinaryFieldMapper, which has a custom encoding
611+
// to encode multiple binary values into a single binary doc values field.
612+
// This is the reason we need to first read the number of values and
613+
// then the length of the field value in bytes.
614+
int numValues = input.readVInt();
615+
assert numValues == 1;
616+
int valueLength = input.readVInt();
617+
assert valueLength > 0;
618+
619+
TransportVersion transportVersion;
620+
if (indexVersion.before(IndexVersions.V_8_8_0)) {
621+
transportVersion = TransportVersion.fromId(indexVersion.id());
622+
} else {
623+
transportVersion = TransportVersion.readVersion(input);
624+
}
625+
626+
QueryBuilder queryBuilder;
627+
628+
if (TransportVersion.isCompatible(transportVersion) || fallbackSource == null) {
629+
// set the transportversion here - only read vints so far, so can change the version freely at this point
630+
input.setTransportVersion(transportVersion);
631+
queryBuilder = input.readNamedWriteable(QueryBuilder.class);
632+
assert in.read() == -1;
633+
} else {
634+
// incompatible transport version, try the fallback
635+
queryBuilder = readQueryBuilder(fallbackSource.get(), registry, indexVersion, null);
636+
}
637+
638+
return queryBuilder;
639+
}
640+
}
641+
595642
static SearchExecutionContext wrap(SearchExecutionContext delegate) {
596643
return new SearchExecutionContext(delegate) {
597644

‎qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java‎

Lines changed: 16 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -11,18 +11,10 @@
1111

1212
import com.carrotsearch.randomizedtesting.annotations.Name;
1313

14-
import org.elasticsearch.TransportVersion;
15-
import org.elasticsearch.Version;
1614
import org.elasticsearch.client.Request;
1715
import org.elasticsearch.client.Response;
1816
import org.elasticsearch.common.Strings;
19-
import org.elasticsearch.common.io.stream.InputStreamStreamInput;
20-
import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
21-
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
22-
import org.elasticsearch.common.io.stream.StreamInput;
23-
import org.elasticsearch.common.settings.Settings;
2417
import org.elasticsearch.common.unit.Fuzziness;
25-
import org.elasticsearch.core.UpdateForV10;
2618
import org.elasticsearch.index.query.BoolQueryBuilder;
2719
import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
2820
import org.elasticsearch.index.query.DisMaxQueryBuilder;
@@ -37,33 +29,23 @@
3729
import org.elasticsearch.index.query.SpanTermQueryBuilder;
3830
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
3931
import org.elasticsearch.index.query.functionscore.RandomScoreFunctionBuilder;
40-
import org.elasticsearch.search.SearchModule;
4132
import org.elasticsearch.test.cluster.ElasticsearchCluster;
4233
import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider;
4334
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
4435
import org.elasticsearch.xcontent.XContentBuilder;
4536
import org.junit.ClassRule;
4637

47-
import java.io.ByteArrayInputStream;
48-
import java.io.InputStream;
4938
import java.util.ArrayList;
50-
import java.util.Base64;
51-
import java.util.Collections;
5239
import java.util.List;
53-
import java.util.Map;
5440

55-
import static org.elasticsearch.cluster.ClusterState.VERSION_INTRODUCING_TRANSPORT_VERSIONS;
5641
import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
5742

5843
/**
5944
* An integration test that tests whether percolator queries stored in older supported ES version can still be read by the
6045
* current ES version. Percolator queries are stored in the binary format in a dedicated doc values field (see
61-
* PercolatorFieldMapper#createQueryBuilderField(...) method). Using the query builders writable contract. This test
62-
* does best effort verifying that we don't break bwc for query builders between the first previous major version and
63-
* the latest current major release.
64-
*
65-
* The queries to test are specified in json format, which turns out to work because we tend break here rarely. If the
66-
* json format of a query being tested here then feel free to change this.
46+
* PercolatorFieldMapper#createQueryBuilderField(...) method). We don't attempt to assert anything on results here, simply executing
47+
* a percolator query will force deserialization of the old query builder. This also verifies that our fallback compatibility
48+
* functionality is working correctly, otherwise the search request will throw an exception.
6749
*/
6850
public class QueryBuilderBWCIT extends ParameterizedFullClusterRestartTestCase {
6951
private static final List<Object[]> CANDIDATES = new ArrayList<>();
@@ -227,43 +209,20 @@ public void testQueryBuilderBWC() throws Exception {
227209
assertEquals(201, rsp.getStatusLine().getStatusCode());
228210
}
229211
} else {
230-
NamedWriteableRegistry registry = new NamedWriteableRegistry(
231-
new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedWriteables()
232-
);
233-
234-
for (int i = 0; i < CANDIDATES.size(); i++) {
235-
QueryBuilder expectedQueryBuilder = (QueryBuilder) CANDIDATES.get(i)[1];
236-
Request request = new Request("GET", "/" + index + "/_search");
237-
request.setJsonEntity(Strings.format("""
238-
{"query": {"ids": {"values": ["%s"]}}, "docvalue_fields": [{"field":"query.query_builder_field"}]}
239-
""", i));
240-
Response rsp = client().performRequest(request);
241-
assertEquals(200, rsp.getStatusLine().getStatusCode());
242-
var hitRsp = (Map<?, ?>) ((List<?>) ((Map<?, ?>) responseAsMap(rsp).get("hits")).get("hits")).get(0);
243-
String queryBuilderStr = (String) ((List<?>) ((Map<?, ?>) hitRsp.get("fields")).get("query.query_builder_field")).get(0);
244-
byte[] qbSource = Base64.getDecoder().decode(queryBuilderStr);
245-
try (
246-
InputStream in = new ByteArrayInputStream(qbSource, 0, qbSource.length);
247-
StreamInput input = new NamedWriteableAwareStreamInput(new InputStreamStreamInput(in), registry)
248-
) {
249-
@UpdateForV10(owner = UpdateForV10.Owner.SEARCH_FOUNDATIONS) // won't need to read <8.8 data anymore
250-
boolean originalClusterHasTransportVersion = parseLegacyVersion(getOldClusterVersion()).map(
251-
v -> v.onOrAfter(VERSION_INTRODUCING_TRANSPORT_VERSIONS)
252-
).orElse(true);
253-
TransportVersion transportVersion;
254-
if (originalClusterHasTransportVersion == false) {
255-
transportVersion = TransportVersion.fromId(
256-
parseLegacyVersion(getOldClusterVersion()).map(Version::id).orElse(TransportVersion.minimumCompatible().id())
257-
);
258-
} else {
259-
transportVersion = TransportVersion.readVersion(input);
212+
Request request = new Request("GET", "/" + index + "/_search");
213+
request.setJsonEntity("""
214+
{
215+
"query": {
216+
"percolate": {
217+
"field": "query",
218+
"document": {
219+
"foo": "bar"
220+
}
260221
}
261-
input.setTransportVersion(transportVersion);
262-
QueryBuilder queryBuilder = input.readNamedWriteable(QueryBuilder.class);
263-
assert in.read() == -1;
264-
assertEquals(expectedQueryBuilder, queryBuilder);
265-
}
266-
}
222+
}
223+
}""");
224+
Response rsp = client().performRequest(request);
225+
assertEquals(200, rsp.getStatusLine().getStatusCode());
267226
}
268227
}
269228
}

0 commit comments

Comments
 (0)