diff --git a/solr/core/src/test/org/apache/solr/handler/admin/LukeHandlerCloudTest.java b/solr/core/src/test/org/apache/solr/handler/admin/LukeHandlerCloudTest.java index f49ac3ccca0..c9ddbabe2bb 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/LukeHandlerCloudTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/LukeHandlerCloudTest.java @@ -41,9 +41,29 @@ /** Cloud-specific Luke tests that require SolrCloud features like managed schema and Schema API. */ public class LukeHandlerCloudTest extends SolrCloudTestCase { + private static final String DISTRIB_COLLECTION = "lukeDistribTests"; + private static final int NUM_DOCS = 20; + @BeforeClass public static void setupCluster() throws Exception { - configureCluster(2).addConfig("managed", configset("cloud-managed")).configure(); + System.setProperty("managed.schema.mutable", "true"); + configureCluster(2) + .addConfig("managed", configset("cloud-managed")) + .addConfig("dynamic", configset("cloud-dynamic")) + .configure(); + + CollectionAdminRequest.createCollection(DISTRIB_COLLECTION, "dynamic", 2, 1) + .processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT); + cluster.waitForActiveCollection(DISTRIB_COLLECTION, 2, 2); + + for (int i = 0; i < NUM_DOCS; i++) { + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", String.valueOf(i)); + doc.addField("name", "name_" + i); + doc.addField("subject", "subject value " + (i % 5)); + cluster.getSolrClient().add(DISTRIB_COLLECTION, doc); + } + cluster.getSolrClient().commit(DISTRIB_COLLECTION); } private void requestLuke(String collection, SolrParams extra) throws Exception { @@ -52,6 +72,28 @@ private void requestLuke(String collection, SolrParams extra) throws Exception { req.process(cluster.getSolrClient(), collection); } + @Test + public void testDistributedShardError() { + SolrParams lukeParams = params("id", "0", "show", "schema"); + + Exception ex = expectThrows(Exception.class, () -> requestLuke(DISTRIB_COLLECTION, lukeParams)); + String fullMessage = SolrException.getRootCause(ex).getMessage(); + assertTrue( + "exception should mention doc style mismatch: " + fullMessage, + fullMessage.contains("missing doc param for doc style")); + } + + @Test + public void testDistributedDocIdRejected() { + SolrParams lukeParams = params("docId", "0"); + + Exception ex = expectThrows(Exception.class, () -> requestLuke(DISTRIB_COLLECTION, lukeParams)); + String fullMessage = SolrException.getRootCause(ex).getMessage(); + assertTrue( + "exception should mention docId not supported: " + fullMessage, + fullMessage.contains("docId parameter is not supported in distributed mode")); + } + /** * Verifies that distributed Luke detects inconsistent index flags across shards. Uses Schema API * to change a field's {@code stored} property between indexing on different shards, producing @@ -60,7 +102,6 @@ private void requestLuke(String collection, SolrParams extra) throws Exception { @Test public void testInconsistentIndexFlagsAcrossShards() throws Exception { String collection = "lukeInconsistentFlags"; - System.setProperty("managed.schema.mutable", "true"); CollectionAdminRequest.createCollection(collection, "managed", 2, 1) .processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/LukeRequestHandlerDistribTest.java b/solr/core/src/test/org/apache/solr/handler/admin/LukeRequestHandlerDistribTest.java index 15296027756..b727114572f 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/LukeRequestHandlerDistribTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/LukeRequestHandlerDistribTest.java @@ -18,8 +18,8 @@ import java.util.Map; import org.apache.solr.BaseDistributedSearchTestCase; +import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.request.LukeRequest; -import org.apache.solr.client.solrj.response.InputStreamResponseParser; import org.apache.solr.client.solrj.response.LukeResponse; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrException; @@ -30,7 +30,6 @@ import org.apache.solr.request.SolrQueryRequestBase; import org.apache.solr.update.AddUpdateCommand; import org.apache.solr.update.CommitUpdateCommand; -import org.apache.solr.util.BaseTestHarness; import org.junit.Test; public class LukeRequestHandlerDistribTest extends BaseDistributedSearchTestCase { @@ -72,15 +71,8 @@ private LukeResponse requestLuke(ModifiableSolrParams extra) throws Exception { private void assertLukeXPath(ModifiableSolrParams extra, String... xpaths) throws Exception { ModifiableSolrParams params = new ModifiableSolrParams(); params.set("shards", shards); - params.set("shards.info", "true"); params.add(extra); - LukeRequest req = new LukeRequest(params); - req.setNumTerms(0); - req.setResponseParser(new InputStreamResponseParser("xml")); - NamedList raw = controlClient.request(req); - String xml = InputStreamResponseParser.consumeResponseToString(raw); - String failedXpath = BaseTestHarness.validateXPath(xml, xpaths); - assertNull("XPath validation failed: " + failedXpath + "\nResponse:\n" + xml, failedXpath); + LukeTestUtil.assertLukeXPath(controlClient, null, params, xpaths); } private void indexTestData() throws Exception { @@ -91,10 +83,10 @@ private void indexTestData() throws Exception { } @Test - @ShardsFixed(num = 2) - public void testDistributedAggregate() throws Exception { + public void testDistributedAggregateAndFields() throws Exception { indexTestData(); + // --- Aggregate index stats --- LukeResponse rsp = requestLuke(); assertEquals("aggregated numDocs should equal total docs", NUM_DOCS, rsp.getNumDocs()); @@ -114,15 +106,8 @@ public void testDistributedAggregate() throws Exception { } assertEquals( "sum of per-shard numDocs should equal aggregated numDocs", rsp.getNumDocs(), sumShardDocs); - } - - @Test - @ShardsFixed(num = 2) - public void testDistributedFieldsAggregate() throws Exception { - indexTestData(); - - LukeResponse rsp = requestLuke(); + // --- Field-level aggregation --- Map fields = rsp.getFieldInfo(); assertNotNull("fields should be present", fields); @@ -149,34 +134,26 @@ public void testDistributedFieldsAggregate() throws Exception { "//lst[@name='fields']/lst[@name='name']/long[@name='docs'][.='20']", "//lst[@name='fields']/lst[@name='id']/str[@name='type'][.='string']", "//lst[@name='fields']/lst[@name='id']/long[@name='docs'][.='20']"); - } - - @Test - @ShardsFixed(num = 2) - public void testDetailedFieldStatsPerShard() throws Exception { - indexTestData(); + // --- Detailed per-shard stats (topTerms, histogram, distinct) --- ModifiableSolrParams params = new ModifiableSolrParams(); params.set("fl", "name"); params.set("numTerms", "5"); - LukeResponse rsp = requestLuke(params); + rsp = requestLuke(params); // Top-level fields should NOT have topTerms, distinct, histogram - LukeResponse.FieldInfo nameField = rsp.getFieldInfo().get("name"); + nameField = rsp.getFieldInfo().get("name"); assertNotNull("'name' field should be present", nameField); assertNull("topTerms should NOT be in top-level fields", nameField.getTopTerms()); assertEquals("distinct should NOT be in top-level fields", 0, nameField.getDistinct()); // Per-shard entries should have detailed stats - Map shardResponses = rsp.getShardResponses(); + shardResponses = rsp.getShardResponses(); assertNotNull("shards section should be present", shardResponses); - ModifiableSolrParams detailedParams = new ModifiableSolrParams(); - detailedParams.set("fl", "name"); - detailedParams.set("numTerms", "5"); assertLukeXPath( - detailedParams, + params, "/response/lst[@name='fields']/lst[@name='name']/str[@name='type'][.='nametext']", "/response/lst[@name='fields']/lst[@name='name']/long[@name='docs'][.='20']", "not(/response/lst[@name='fields']/lst[@name='name']/lst[@name='topTerms'])", @@ -185,39 +162,70 @@ public void testDetailedFieldStatsPerShard() throws Exception { "//lst[@name='shards']/lst/lst[@name='fields']/lst[@name='name']/lst[@name='topTerms']", "//lst[@name='shards']/lst/lst[@name='fields']/lst[@name='name']/lst[@name='histogram']/int[@name='1']", "//lst[@name='shards']/lst/lst[@name='fields']/lst[@name='name']/int[@name='distinct']"); - } - - @Test - @ShardsFixed(num = 2) - public void testLocalModeDefault() throws Exception { - indexTestData(); // Query a single client without the shards param — local mode LukeRequest req = new LukeRequest(); req.setNumTerms(0); - LukeResponse rsp = req.process(controlClient); + rsp = req.process(controlClient); assertNotNull("index info should be present", rsp.getIndexInfo()); assertNull("shards should NOT be present in local mode", rsp.getShardResponses()); - } - - @Test - @ShardsFixed(num = 2) - public void testExplicitDistribFalse() throws Exception { - indexTestData(); // Query a single client with distrib=false — no shards param - LukeRequest req = new LukeRequest(params("distrib", "false")); + req = new LukeRequest(params("distrib", "false")); req.setNumTerms(0); - LukeResponse rsp = req.process(controlClient); + rsp = req.process(controlClient); assertNotNull("index info should be present", rsp.getIndexInfo()); assertNull("shards should NOT be present with distrib=false", rsp.getShardResponses()); + + // --- Schema view --- + params = new ModifiableSolrParams(); + params.set("show", "schema"); + + assertLukeXPath( + params, + "//lst[@name='schema']/lst[@name='fields']/lst[@name='id']/str[@name='type'][.='string']", + "//lst[@name='schema']/lst[@name='fields']/lst[@name='name']/str[@name='type'][.='nametext']", + "//lst[@name='schema']/lst[@name='dynamicFields']/lst[@name='*_s']", + "//lst[@name='schema']/str[@name='uniqueKeyField'][.='id']", + "//lst[@name='schema']/lst[@name='types']/lst[@name='string']", + "//lst[@name='schema']/lst[@name='types']/lst[@name='nametext']", + "//lst[@name='schema']/lst[@name='similarity']", + "not(/response/lst[@name='fields'])", + "count(//lst[@name='shards']/lst)=2"); + + // --- Doc lookup not found --- + params = new ModifiableSolrParams(); + params.set("id", "999888777"); + + rsp = requestLuke(params); + + NamedList raw = rsp.getResponse(); + assertNull("doc section should NOT be present for missing ID", raw.get("doc")); + + assertLukeXPath(params, "not(//lst[@name='doc'])"); + + // --- Doc lookup found --- + params = new ModifiableSolrParams(); + params.set("id", "0"); + + assertLukeXPath( + params, + "//lst[@name='doc']/int[@name='docId']", + "//lst[@name='doc']/lst[@name='lucene']/lst[@name='id']/str[@name='type'][.='string']", + "//lst[@name='doc']/lst[@name='lucene']/lst[@name='id']/str[@name='value'][.='0']", + "//lst[@name='doc']/lst[@name='lucene']/lst[@name='name']/str[@name='type'][.='nametext']", + "//lst[@name='doc']/lst[@name='lucene']/lst[@name='name']/str[@name='value'][.='name_0']", + "//lst[@name='doc']/arr[@name='solr']/str[.='0']", + "//lst[@name='doc']/arr[@name='solr']/str[.='name_0']", + "//lst[@name='index']", + "//lst[@name='info']"); } @Test - @ShardsFixed(num = 12) - public void testSparseShards() throws Exception { + @ShardsFixed(num = 4) + public void testSparseShardsAndDeferredIndexFlags() throws Exception { // Index a single doc on shard 0 index_specific( 0, "id", "100", "name", "sparse test", "subject", "subject value", "cat_s", "category"); @@ -232,7 +240,10 @@ public void testSparseShards() throws Exception { Map shardResponses = rsp.getShardResponses(); assertNotNull("shards section should be present", shardResponses); - assertEquals("should have 12 shard entries", 12, shardResponses.size()); + assertEquals( + "should have " + getShardCount() + " shard entries", + getShardCount(), + shardResponses.size()); long sumShardDocs = 0; for (Map.Entry entry : shardResponses.entrySet()) { @@ -267,7 +278,7 @@ public void testSparseShards() throws Exception { new ModifiableSolrParams(), "//lst[@name='index']/long[@name='numDocs'][.='1']", "//lst[@name='index']/long[@name='deletedDocs'][.='0']", - "count(//lst[@name='shards']/lst)=12", + "count(//lst[@name='shards']/lst)=" + getShardCount(), "//lst[@name='fields']/lst[@name='name']/str[@name='type'][.='nametext']", "//lst[@name='fields']/lst[@name='name']/str[@name='schema']", "//lst[@name='fields']/lst[@name='name']/str[@name='index']", @@ -275,36 +286,11 @@ public void testSparseShards() throws Exception { "//lst[@name='fields']/lst[@name='cat_s']/str[@name='type'][.='string']", "//lst[@name='fields']/lst[@name='cat_s']/str[@name='dynamicBase'][.='*_s']", "//lst[@name='fields']/lst[@name='cat_s']/long[@name='docs'][.='1']"); - } - - @Test - @ShardsFixed(num = 2) - public void testDistribShowSchema() throws Exception { - indexTestData(); - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("show", "schema"); - - assertLukeXPath( - params, - "//lst[@name='schema']/lst[@name='fields']/lst[@name='id']/str[@name='type'][.='string']", - "//lst[@name='schema']/lst[@name='fields']/lst[@name='name']/str[@name='type'][.='nametext']", - "//lst[@name='schema']/lst[@name='dynamicFields']/lst[@name='*_s']", - "//lst[@name='schema']/str[@name='uniqueKeyField'][.='id']", - "//lst[@name='schema']/lst[@name='types']/lst[@name='string']", - "//lst[@name='schema']/lst[@name='types']/lst[@name='nametext']", - "//lst[@name='schema']/lst[@name='similarity']", - "not(/response/lst[@name='fields'])", - "count(//lst[@name='shards']/lst)=2"); - } - - @Test - @ShardsFixed(num = 16) - public void testDeferredIndexFlags() throws Exception { // Index docs with the target field across shards, plus anchor docs without it. // Use numeric IDs (the default test schema copies id to integer fields). // Target docs get even IDs starting at 1000, anchor docs get odd IDs. - for (int i = 0; i < 16 * 4; i++) { + for (int i = 0; i < getShardCount() * 4; i++) { index("id", String.valueOf(1000 + i * 2), "flag_target_s", "value_" + i); index("id", String.valueOf(1001 + i * 2), "name", "anchor"); } @@ -313,9 +299,9 @@ public void testDeferredIndexFlags() throws Exception { // Delete all target docs except the first one, using per-shard deletes. // Then optimize to force segment merge — expunges soft-deleted docs so // Terms.getDocCount() (which backs docs) reflects only live docs. - for (int i = 0; i < clients.size(); i++) { - clients.get(i).deleteByQuery("flag_target_s:* AND -id:1000"); - clients.get(i).optimize(); + for (SolrClient client : clients) { + client.deleteByQuery("flag_target_s:* AND -id:1000"); + client.optimize(); } controlClient.deleteByQuery("flag_target_s:* AND -id:1000"); controlClient.optimize(); @@ -323,17 +309,15 @@ public void testDeferredIndexFlags() throws Exception { ModifiableSolrParams params = new ModifiableSolrParams(); params.set("fl", "flag_target_s"); - LukeResponse rsp = requestLuke(params); + rsp = requestLuke(params); - Map fields = rsp.getFieldInfo(); + fields = rsp.getFieldInfo(); assertNotNull("fields should be present", fields); LukeResponse.FieldInfo targetField = fields.get("flag_target_s"); assertNotNull("'flag_target_s' field should be present", targetField); - ModifiableSolrParams xpathParams = new ModifiableSolrParams(); - xpathParams.set("fl", "flag_target_s"); assertLukeXPath( - xpathParams, + params, "//lst[@name='fields']/lst[@name='flag_target_s']/str[@name='type'][.='string']", "//lst[@name='fields']/lst[@name='flag_target_s']/str[@name='dynamicBase'][.='*_s']", "//lst[@name='fields']/lst[@name='flag_target_s']/str[@name='index']", @@ -341,75 +325,6 @@ public void testDeferredIndexFlags() throws Exception { } @Test - @ShardsFixed(num = 2) - public void testDistributedShardError() throws Exception { - indexTestData(); - - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("id", "0"); - params.set("show", "schema"); - - Exception ex = expectThrows(Exception.class, () -> requestLuke(params)); - String fullMessage = SolrException.getRootCause(ex).getMessage(); - assertTrue( - "exception should mention doc style mismatch: " + fullMessage, - fullMessage.contains("missing doc param for doc style")); - } - - @Test - @ShardsFixed(num = 2) - public void testDistributedDocIdRejected() throws Exception { - indexTestData(); - - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("docId", "0"); - - Exception ex = expectThrows(Exception.class, () -> requestLuke(params)); - String fullMessage = SolrException.getRootCause(ex).getMessage(); - assertTrue( - "exception should mention docId not supported: " + fullMessage, - fullMessage.contains("docId parameter is not supported in distributed mode")); - } - - @Test - @ShardsFixed(num = 2) - public void testDistributedDocLookupFound() throws Exception { - indexTestData(); - - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("id", "0"); - - assertLukeXPath( - params, - "//lst[@name='doc']/int[@name='docId']", - "//lst[@name='doc']/lst[@name='lucene']/lst[@name='id']/str[@name='type'][.='string']", - "//lst[@name='doc']/lst[@name='lucene']/lst[@name='id']/str[@name='value'][.='0']", - "//lst[@name='doc']/lst[@name='lucene']/lst[@name='name']/str[@name='type'][.='nametext']", - "//lst[@name='doc']/lst[@name='lucene']/lst[@name='name']/str[@name='value'][.='name_0']", - "//lst[@name='doc']/arr[@name='solr']/str[.='0']", - "//lst[@name='doc']/arr[@name='solr']/str[.='name_0']", - "//lst[@name='index']", - "//lst[@name='info']"); - } - - @Test - @ShardsFixed(num = 2) - public void testDistributedDocLookupNotFound() throws Exception { - indexTestData(); - - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("id", "999888777"); - - LukeResponse rsp = requestLuke(params); - - NamedList raw = rsp.getResponse(); - assertNull("doc section should NOT be present for missing ID", raw.get("doc")); - - assertLukeXPath(params, "not(//lst[@name='doc'])"); - } - - @Test - @ShardsFixed(num = 2) public void testDistributedDocLookupDuplicateId() throws Exception { String dupId = "99999"; @@ -446,7 +361,6 @@ public void testDistributedDocLookupDuplicateId() throws Exception { } @Test - @ShardsFixed(num = 2) public void testShardsParamRoutesToSpecificShard() throws Exception { // Index a doc with a dynamic field only to shard 0 index_specific(0, "id", "700", "name", "shard0_only", "only_on_shard0_s", "present"); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/LukeTestUtil.java b/solr/core/src/test/org/apache/solr/handler/admin/LukeTestUtil.java new file mode 100644 index 00000000000..308bed73714 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/LukeTestUtil.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin; + +import static org.junit.Assert.assertNull; + +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.request.LukeRequest; +import org.apache.solr.client.solrj.response.InputStreamResponseParser; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.util.BaseTestHarness; + +final class LukeTestUtil { + + private LukeTestUtil() {} + + static void assertLukeXPath( + SolrClient client, String collection, SolrParams extra, String... xpaths) throws Exception { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("shards.info", "true"); + params.add(extra); + LukeRequest req = new LukeRequest(params); + req.setNumTerms(0); + req.setResponseParser(new InputStreamResponseParser("xml")); + NamedList raw = + collection != null ? client.request(req, collection) : client.request(req); + String xml = InputStreamResponseParser.consumeResponseToString(raw); + String failedXpath = BaseTestHarness.validateXPath(xml, xpaths); + assertNull("XPath validation failed: " + failedXpath + "\nResponse:\n" + xml, failedXpath); + } +} diff --git a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java index e72ffaa213c..5249457ecd8 100644 --- a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java @@ -93,6 +93,13 @@ * without annotations in that class hierarchy. Ideally this function should be retired in favour of * better annotations. * + *

WARNING each test annotated with @Shards* will spin up its own set of Jetty servers which can + * be a substantial performance hit. Therefore, one should be mindful about the total number of + * independent tests using such annotations. One approach is to pool assertions in a single test to + * minimize jetty server construction overhead. If the test doesn't rely on the comparison features + * of this class, i.e. {@link #query} it may be wise to make it a {@link + * org.apache.solr.cloud.SolrCloudTestCase} instead. + * * @since solr 1.5 */ public abstract class BaseDistributedSearchTestCase extends SolrTestCaseJ4 {