Skip to content

Commit 5a542b0

Browse files
parkertimminselasticsearchmachine
parkertimmins
and
elasticsearchmachine
authored
[9.2] Fix index.mapping.pattern_text.disable_templating not registered issue (#138265) (#138324)
* Fix index.mapping.pattern_text.disable_templating not registered issue (#138265) * Fix index.mapping.pattern_text.disable_templating has not been registered issue * Update docs/changelog/138265.yaml * gate versions that do not have pattern_text (cherry picked from commit 279c349) * Copy from MatchOnlyTextRollingUpgradeIT * gate bwc rolling upgrade test on version * [CI] Auto commit changes from spotless --------- Co-authored-by: elasticsearchmachine <infra-root+elasticsearchmachine@elastic.co>
1 parent b4086c9 commit 5a542b0

File tree

3 files changed

+307
-1
lines changed

3 files changed

+307
-1
lines changed

‎docs/changelog/138265.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 138265
2+
summary: Fix `index.mapping.pattern_text.disable_templating` not registered issue
3+
area: Mapping
4+
type: bug
5+
issues: []

‎x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/PatternTextRollingUpgradeIT.java‎

Lines changed: 301 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,301 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.upgrades;
11+
12+
import com.carrotsearch.randomizedtesting.annotations.Name;
13+
14+
import org.elasticsearch.client.Request;
15+
import org.elasticsearch.client.Response;
16+
import org.elasticsearch.client.ResponseException;
17+
import org.elasticsearch.common.network.NetworkAddress;
18+
import org.elasticsearch.common.time.DateFormatter;
19+
import org.elasticsearch.common.time.FormatNames;
20+
import org.elasticsearch.common.xcontent.XContentHelper;
21+
import org.elasticsearch.test.rest.ObjectPath;
22+
import org.elasticsearch.xcontent.XContentType;
23+
24+
import java.io.IOException;
25+
import java.io.InputStream;
26+
import java.time.Instant;
27+
import java.util.List;
28+
import java.util.Locale;
29+
import java.util.Map;
30+
31+
import static org.elasticsearch.upgrades.StandardToLogsDbIndexModeRollingUpgradeIT.enableLogsdbByDefault;
32+
import static org.elasticsearch.upgrades.StandardToLogsDbIndexModeRollingUpgradeIT.getWriteBackingIndex;
33+
import static org.elasticsearch.upgrades.TextRollingUpgradeIT.randomAlphasDelimitedBySpace;
34+
import static org.hamcrest.Matchers.containsString;
35+
import static org.hamcrest.Matchers.equalTo;
36+
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
37+
import static org.hamcrest.Matchers.notNullValue;
38+
39+
public class PatternTextRollingUpgradeIT extends AbstractRollingUpgradeWithSecurityTestCase {
40+
41+
private static final String DATA_STREAM = "logs-bwc-test";
42+
43+
private static final int IGNORE_ABOVE_MAX = 256;
44+
private static final int NUM_REQUESTS = 4;
45+
private static final int NUM_DOCS_PER_REQUEST = 1024;
46+
47+
static String BULK_ITEM_TEMPLATE =
48+
"""
49+
{ "create": {} }
50+
{"@timestamp": "$now", "host.name": "$host", "method": "$method", "ip": "$ip", "message": "$message", "length": $length, "factor": $factor}
51+
""";
52+
53+
private static final String TEMPLATE = """
54+
{
55+
"mappings": {
56+
"properties": {
57+
"@timestamp" : {
58+
"type": "date"
59+
},
60+
"method": {
61+
"type": "keyword"
62+
},
63+
"message": {
64+
"type": "pattern_text",
65+
"fields": {
66+
"keyword": {
67+
"ignore_above": $IGNORE_ABOVE,
68+
"type": "keyword"
69+
}
70+
}
71+
},
72+
"ip": {
73+
"type": "ip"
74+
},
75+
"length": {
76+
"type": "long"
77+
},
78+
"factor": {
79+
"type": "double"
80+
}
81+
}
82+
}
83+
}""";
84+
85+
// when sorted, this message will appear at the top and hence can be used to validate query results
86+
private static String smallestMessage;
87+
88+
public PatternTextRollingUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
89+
super(upgradedNodes);
90+
}
91+
92+
public void testIndexing() throws Exception {
93+
assumeTrue("pattern_text only available from 9.2.0 onward", oldClusterHasFeature("gte_v9.2.0"));
94+
95+
if (isOldCluster()) {
96+
// given - enable logsdb and create a template
97+
startTrial();
98+
enableLogsdbByDefault();
99+
String templateId = getClass().getSimpleName().toLowerCase(Locale.ROOT);
100+
createTemplate(DATA_STREAM, templateId, prepareTemplate());
101+
102+
// when - index some documents
103+
bulkIndex(NUM_REQUESTS, NUM_DOCS_PER_REQUEST);
104+
105+
// then - verify that logsdb and synthetic source are both enabled
106+
String firstBackingIndex = getWriteBackingIndex(client(), DATA_STREAM, 0);
107+
var settings = (Map<?, ?>) getIndexSettingsWithDefaults(firstBackingIndex).get(firstBackingIndex);
108+
assertThat(((Map<?, ?>) settings.get("settings")).get("index.mode"), equalTo("logsdb"));
109+
assertThat(((Map<?, ?>) settings.get("defaults")).get("index.mapping.source.mode"), equalTo("SYNTHETIC"));
110+
111+
// then continued - verify that the created data stream uses the created template
112+
LogsdbIndexingRollingUpgradeIT.assertDataStream(DATA_STREAM, templateId);
113+
114+
// when/then - run some queries and verify results
115+
ensureGreen(DATA_STREAM);
116+
search(DATA_STREAM);
117+
query(DATA_STREAM);
118+
119+
} else if (isMixedCluster()) {
120+
// when
121+
bulkIndex(NUM_REQUESTS, NUM_DOCS_PER_REQUEST);
122+
123+
// when/then
124+
ensureGreen(DATA_STREAM);
125+
search(DATA_STREAM);
126+
query(DATA_STREAM);
127+
128+
} else if (isUpgradedCluster()) {
129+
// when/then
130+
ensureGreen(DATA_STREAM);
131+
bulkIndex(NUM_REQUESTS, NUM_DOCS_PER_REQUEST);
132+
search(DATA_STREAM);
133+
query(DATA_STREAM);
134+
135+
// when/then continued - force merge all shard segments into one
136+
var forceMergeRequest = new Request("POST", "/" + DATA_STREAM + "/_forcemerge");
137+
forceMergeRequest.addParameter("max_num_segments", "1");
138+
assertOK(client().performRequest(forceMergeRequest));
139+
140+
// then continued
141+
ensureGreen(DATA_STREAM);
142+
search(DATA_STREAM);
143+
query(DATA_STREAM);
144+
}
145+
}
146+
147+
private String prepareTemplate() {
148+
boolean shouldSetIgnoreAbove = randomBoolean();
149+
if (shouldSetIgnoreAbove) {
150+
return TEMPLATE.replace("$IGNORE_ABOVE", String.valueOf(randomInt(IGNORE_ABOVE_MAX)));
151+
}
152+
153+
// removes the entire line that defines ignore_above
154+
return TEMPLATE.replaceAll("(?m)^\\s*\"ignore_above\":\\s*\\$IGNORE_ABOVE\\s*,?\\s*\\n?", "");
155+
}
156+
157+
static void createTemplate(String dataStreamName, String id, String template) throws IOException {
158+
final String INDEX_TEMPLATE = """
159+
{
160+
"priority": 500,
161+
"index_patterns": ["$DATASTREAM"],
162+
"template": $TEMPLATE,
163+
"data_stream": {
164+
}
165+
}""";
166+
var putIndexTemplateRequest = new Request("POST", "/_index_template/" + id);
167+
putIndexTemplateRequest.setJsonEntity(INDEX_TEMPLATE.replace("$TEMPLATE", template).replace("$DATASTREAM", dataStreamName));
168+
assertOK(client().performRequest(putIndexTemplateRequest));
169+
}
170+
171+
private void bulkIndex(int numRequest, int numDocs) throws Exception {
172+
String firstIndex = null;
173+
Instant startTime = Instant.now().minusSeconds(60 * 60);
174+
175+
for (int i = 0; i < numRequest; i++) {
176+
var bulkRequest = new Request("POST", "/" + DATA_STREAM + "/_bulk");
177+
bulkRequest.setJsonEntity(bulkIndexRequestBody(numDocs, startTime));
178+
bulkRequest.addParameter("refresh", "true");
179+
180+
var response = client().performRequest(bulkRequest);
181+
var responseBody = entityAsMap(response);
182+
183+
assertOK(response);
184+
assertThat("errors in response:\n " + responseBody, responseBody.get("errors"), equalTo(false));
185+
if (firstIndex == null) {
186+
firstIndex = (String) ((Map<?, ?>) ((Map<?, ?>) ((List<?>) responseBody.get("items")).get(0)).get("create")).get("_index");
187+
}
188+
}
189+
}
190+
191+
private String bulkIndexRequestBody(int numDocs, Instant startTime) {
192+
StringBuilder requestBody = new StringBuilder();
193+
194+
for (int j = 0; j < numDocs; j++) {
195+
String hostName = "host" + j % 50; // Not realistic, but makes asserting search / query response easier.
196+
String methodName = "method" + j % 5;
197+
String ip = NetworkAddress.format(randomIp(true));
198+
String message = randomAlphasDelimitedBySpace(10, 1, 15);
199+
recordSmallestMessage(message);
200+
long length = randomLong();
201+
double factor = randomDouble();
202+
203+
requestBody.append(
204+
BULK_ITEM_TEMPLATE.replace("$now", formatInstant(startTime))
205+
.replace("$host", hostName)
206+
.replace("$method", methodName)
207+
.replace("$ip", ip)
208+
.replace("$message", message)
209+
.replace("$length", Long.toString(length))
210+
.replace("$factor", Double.toString(factor))
211+
);
212+
requestBody.append('\n');
213+
214+
startTime = startTime.plusMillis(1);
215+
}
216+
217+
return requestBody.toString();
218+
}
219+
220+
private void recordSmallestMessage(final String message) {
221+
if (smallestMessage == null || message.compareTo(smallestMessage) < 0) {
222+
smallestMessage = message;
223+
}
224+
}
225+
226+
void search(String dataStreamName) throws Exception {
227+
var searchRequest = new Request("POST", "/" + dataStreamName + "/_search");
228+
searchRequest.addParameter("pretty", "true");
229+
searchRequest.setJsonEntity("""
230+
{
231+
"size": 500
232+
}
233+
""");
234+
var response = client().performRequest(searchRequest);
235+
assertOK(response);
236+
var responseBody = entityAsMap(response);
237+
logger.info("{}", responseBody);
238+
239+
Integer totalCount = ObjectPath.evaluate(responseBody, "hits.total.value");
240+
assertThat(totalCount, greaterThanOrEqualTo(NUM_REQUESTS * NUM_DOCS_PER_REQUEST));
241+
}
242+
243+
private void query(String dataStreamName) throws Exception {
244+
var queryRequest = new Request("POST", "/_query");
245+
queryRequest.addParameter("pretty", "true");
246+
queryRequest.setJsonEntity("""
247+
{
248+
"query": "FROM $ds | STATS max(length), max(factor) BY message | SORT message | LIMIT 5"
249+
}
250+
""".replace("$ds", dataStreamName));
251+
var response = client().performRequest(queryRequest);
252+
assertOK(response);
253+
var responseBody = entityAsMap(response);
254+
logger.info("{}", responseBody);
255+
256+
String column1 = ObjectPath.evaluate(responseBody, "columns.0.name");
257+
assertThat(column1, equalTo("max(length)"));
258+
String column2 = ObjectPath.evaluate(responseBody, "columns.1.name");
259+
assertThat(column2, equalTo("max(factor)"));
260+
String column3 = ObjectPath.evaluate(responseBody, "columns.2.name");
261+
assertThat(column3, equalTo("message"));
262+
263+
Long maxRx = ObjectPath.evaluate(responseBody, "values.0.0");
264+
assertThat(maxRx, notNullValue());
265+
Double maxTx = ObjectPath.evaluate(responseBody, "values.0.1");
266+
assertThat(maxTx, notNullValue());
267+
String key = ObjectPath.evaluate(responseBody, "values.0.2");
268+
assertThat(key, equalTo(smallestMessage));
269+
}
270+
271+
protected static void startTrial() throws IOException {
272+
Request startTrial = new Request("POST", "/_license/start_trial");
273+
startTrial.addParameter("acknowledge", "true");
274+
try {
275+
assertOK(client().performRequest(startTrial));
276+
} catch (ResponseException e) {
277+
var responseBody = entityAsMap(e.getResponse());
278+
String error = ObjectPath.evaluate(responseBody, "error_message");
279+
assertThat(error, containsString("Trial was already activated."));
280+
}
281+
}
282+
283+
static Map<String, Object> getIndexSettingsWithDefaults(String index) throws IOException {
284+
Request request = new Request("GET", "/" + index + "/_settings");
285+
request.addParameter("flat_settings", "true");
286+
request.addParameter("include_defaults", "true");
287+
Response response = client().performRequest(request);
288+
try (InputStream is = response.getEntity().getContent()) {
289+
return XContentHelper.convertToMap(
290+
XContentType.fromMediaType(response.getEntity().getContentType().getValue()).xContent(),
291+
is,
292+
true
293+
);
294+
}
295+
}
296+
297+
static String formatInstant(Instant instant) {
298+
return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(instant);
299+
}
300+
301+
}

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ private static Parameter<NamedAnalyzer> analyzerParam(String name, Function<Fiel
169169
* associated index setting, which is set from the current license status.
170170
*/
171171
private static Parameter<Boolean> disableTemplatingParameter(IndexSettings indexSettings) {
172-
boolean forceDisable = indexSettings.getValue(DISABLE_TEMPLATING_SETTING);
172+
boolean forceDisable = DISABLE_TEMPLATING_SETTING.get(indexSettings.getSettings());
173173
return Parameter.boolParam(
174174
"disable_templating",
175175
false,

0 commit comments

Comments
 (0)