diff --git a/changelog/unreleased/SOLR-16390-cluster-command-status.yml b/changelog/unreleased/SOLR-16390-cluster-command-status.yml new file mode 100644 index 00000000000..775b6364937 --- /dev/null +++ b/changelog/unreleased/SOLR-16390-cluster-command-status.yml @@ -0,0 +1,8 @@ +title: "v2 cluster command-status APIs are now more REST-ful at GET/DELETE /api/cluster/commands/{id} and DELETE /api/cluster/commands (the previous /api/cluster/command-status paths are gone). SolrJ provides ClusterApi.GetClusterCommandStatus, ClusterApi.DeleteClusterCommandStatus, and ClusterApi.DeleteAllClusterCommandStatuses." +type: changed +authors: + - name: Prithvi S + nick: iprithv +links: + - name: SOLR-16390 + url: https://issues.apache.org/jira/browse/SOLR-16390 diff --git a/dev-docs/v2-api-conventions.adoc b/dev-docs/v2-api-conventions.adoc index d40d7dd4a48..08a9d01fdba 100644 --- a/dev-docs/v2-api-conventions.adoc +++ b/dev-docs/v2-api-conventions.adoc @@ -23,6 +23,8 @@ Following these guidelines has given us the following (non-exhaustive) list of v * `/api/backups/specificBackupName/versions/specificVersion` * `/api/cluster/nodes/specificNodeName/roles` * `/api/cluster/nodes/specificNodeName/roles/specificRoleName` +* `/api/cluster/commands` +* `/api/cluster/commands/specificCommandId` * `/api/cluster/properties` * `/api/cluster/properties/specificPropertyName` * `/api/collections` diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterCommandsApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterCommandsApi.java new file mode 100644 index 00000000000..bfd3c19eda8 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterCommandsApi.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.endpoint; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import org.apache.solr.client.api.model.DeleteClusterCommandStatusResponse; +import org.apache.solr.client.api.model.GetClusterCommandStatusResponse; + +/** + * V2 API definitions for cluster-level asynchronous Collection API command status. + * + *

These APIs are analogous to the v1 {@code /admin/collections?action=REQUESTSTATUS} and {@code + * /admin/collections?action=DELETESTATUS} commands. They are not to be confused with the node-local + * {@link GetNodeCommandStatusApi} under {@code /api/node/commands}. + */ +@Path("/cluster/commands") +public interface ClusterCommandsApi { + + @GET + @Path("/{requestId}") + @Operation( + summary = "Request the status of an already submitted asynchronous Collection API call.", + tags = {"cluster"}) + GetClusterCommandStatusResponse getClusterCommandStatus( + @Parameter( + description = "The user defined request-id for the asynchronous request.", + required = true) + @PathParam("requestId") + String requestId) + throws Exception; + + @DELETE + @Path("/{requestId}") + @Operation( + summary = + "Delete the stored status of a completed or failed asynchronous Collection API call.", + tags = {"cluster"}) + DeleteClusterCommandStatusResponse deleteClusterCommandStatus( + @Parameter( + description = "The user defined request-id whose stored response should be cleared.", + required = true) + @PathParam("requestId") + String requestId) + throws Exception; + + @DELETE + @Operation( + summary = + "Delete the stored status of all completed and failed asynchronous Collection API calls.", + tags = {"cluster"}) + DeleteClusterCommandStatusResponse deleteAllClusterCommandStatuses() throws Exception; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/GetNodeCommandStatusApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/GetNodeCommandStatusApi.java index a2b5bcc5986..cc807074d59 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/GetNodeCommandStatusApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/GetNodeCommandStatusApi.java @@ -28,7 +28,7 @@ * *

This API is analogous to the v1 /admin/cores?action=REQUESTSTATUS command. It is not to be * confused with the more robust asynchronous command support offered under the v2 - * `/cluster/command-status` path (or the corresponding v1 path + * `/cluster/commands` path (or the corresponding v1 path * `/solr/admin/collections?action=REQUESTSTATUS`). Async support at the core level differs in that * command IDs are local to individual Solr nodes and are not persisted across restarts. * diff --git a/solr/api/src/java/org/apache/solr/client/api/model/DeleteClusterCommandStatusResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/DeleteClusterCommandStatusResponse.java new file mode 100644 index 00000000000..4571ec19f26 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/DeleteClusterCommandStatusResponse.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; + +/** + * Response body for {@code DELETE /api/cluster/commands} and {@code DELETE + * /api/cluster/commands/{requestId}}. + */ +public class DeleteClusterCommandStatusResponse extends SolrJerseyResponse { + + @JsonProperty("status") + @Schema(description = "A message describing the result of the delete.") + public String status; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/GetClusterCommandStatusResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/GetClusterCommandStatusResponse.java new file mode 100644 index 00000000000..6b96427cacd --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/GetClusterCommandStatusResponse.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonValue; +import io.swagger.v3.oas.annotations.media.Schema; +import java.util.HashMap; +import java.util.Map; +import org.apache.solr.client.api.util.ReflectWritable; + +/** + * Response body for {@code GET /api/cluster/commands/{requestId}}. + * + *

Completed and failed commands also flatten the original command response (for example {@code + * success} / {@code failure} sub-responses) into this object as additional properties. + */ +public class GetClusterCommandStatusResponse extends SolrJerseyResponse { + + @JsonProperty("status") + @Schema(description = "The current state of the asynchronous request and a descriptive message.") + public CommandStatus status; + + private Map unknownFields = new HashMap<>(); + + @JsonAnyGetter + public Map unknownProperties() { + return unknownFields; + } + + @JsonAnySetter + public void setUnknownProperty(String field, Object value) { + unknownFields.put(field, value); + } + + /** Nested {@code status} object returned by REQUESTSTATUS. */ + public static class CommandStatus implements ReflectWritable { + @JsonProperty("state") + @Schema(description = "Request state: submitted, running, completed, failed, or notfound.") + public State state; + + @JsonProperty("msg") + @Schema(description = "A message describing where the request was found, if at all.") + public String msg; + + /** + * The state of an asynchronous request. Mirrors {@code + * org.apache.solr.client.solrj.response.RequestStatusState}'s constants and wire keys; kept as + * a separate type here since this module (solr:api) cannot depend on solrj. + */ + public enum State { + SUBMITTED("submitted"), + RUNNING("running"), + COMPLETED("completed"), + FAILED("failed"), + NOT_FOUND("notfound"); + + private final String key; + + State(String key) { + this.key = key; + } + + @JsonValue + public String getKey() { + return key; + } + + @JsonCreator + public static State fromKey(String key) { + for (State state : values()) { + if (state.key.equalsIgnoreCase(key)) { + return state; + } + } + throw new IllegalArgumentException("Unknown request status state: " + key); + } + } + } +} diff --git a/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java b/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java index 36ee024f7f2..4e672348292 100644 --- a/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java @@ -17,16 +17,11 @@ package org.apache.solr.handler; -import static org.apache.solr.client.solrj.SolrRequest.METHOD.DELETE; import static org.apache.solr.client.solrj.SolrRequest.METHOD.GET; import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST; -import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.REQUESTID; -import static org.apache.solr.common.params.CollectionParams.ACTION; import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDROLE; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETESTATUS; import static org.apache.solr.common.params.CollectionParams.CollectionAction.OVERSEERSTATUS; import static org.apache.solr.common.params.CollectionParams.CollectionAction.REMOVEROLE; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.REQUESTSTATUS; import static org.apache.solr.core.RateLimiterConfig.RL_CONFIG_KEY; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_READ_PERM; @@ -199,18 +194,6 @@ public void getOverseerStatus(SolrQueryRequest req, SolrQueryResponse rsp) throw collectionsHandler.handleRequestBody(wrapParams(req, "action", OVERSEERSTATUS.lowerName), rsp); } - @EndPoint(method = DELETE, path = "/cluster/command-status/{id}", permission = COLL_EDIT_PERM) - public void deleteCommandStatus(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { - final Map v1Params = - Map.of(ACTION, DELETESTATUS.lowerName, REQUESTID, req.getPathTemplateValues().get("id")); - collectionsHandler.handleRequestBody(wrapParams(req, v1Params), rsp); - } - - @EndPoint(method = DELETE, path = "/cluster/command-status", permission = COLL_EDIT_PERM) - public void flushCommandStatus(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { - CollectionsHandler.CollectionOperation.DELETESTATUS_OP.execute(req, rsp, collectionsHandler); - } - public static SolrQueryRequest wrapParams(SolrQueryRequest req, Object... def) { Map m = Utils.makeMap(def); return wrapParams(req, m); @@ -232,13 +215,6 @@ public static SolrQueryRequest wrapParams(SolrQueryRequest req, Map v1Params = - Map.of(ACTION, REQUESTSTATUS.lowerName, REQUESTID, req.getPathTemplateValues().get("id")); - collectionsHandler.handleRequestBody(wrapParams(req, v1Params), rsp); - } - @EndPoint(method = GET, path = "/cluster/nodes", permission = COLL_READ_PERM) public void getNodes(SolrQueryRequest req, SolrQueryResponse rsp) { rsp.add("nodes", getCoreContainer().getZkController().getClusterState().getLiveNodes()); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 270afc24906..77ec443e38f 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -16,16 +16,10 @@ */ package org.apache.solr.handler.admin; -import static org.apache.solr.client.solrj.response.RequestStatusState.COMPLETED; -import static org.apache.solr.client.solrj.response.RequestStatusState.FAILED; -import static org.apache.solr.client.solrj.response.RequestStatusState.NOT_FOUND; -import static org.apache.solr.client.solrj.response.RequestStatusState.RUNNING; -import static org.apache.solr.client.solrj.response.RequestStatusState.SUBMITTED; import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.CREATE_NODE_SET; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.CREATE_NODE_SET_SHUFFLE; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.NUM_SLICES; -import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.REQUESTID; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.SHARD_UNIQUE; import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; @@ -133,10 +127,8 @@ import org.apache.solr.client.api.model.UpdateCollectionPropertyRequestBody; import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.response.RequestStatusState; import org.apache.solr.cloud.OverseerSolrResponse; import org.apache.solr.cloud.OverseerSolrResponseSerializer; -import org.apache.solr.cloud.OverseerTaskQueue; import org.apache.solr.cloud.OverseerTaskQueue.QueueEvent; import org.apache.solr.cloud.ZkController; import org.apache.solr.cloud.ZkController.NotInClusterStateException; @@ -162,8 +154,6 @@ import org.apache.solr.common.params.RequiredSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.Pair; -import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CloudConfig; @@ -176,6 +166,7 @@ import org.apache.solr.handler.admin.api.AliasProperty; import org.apache.solr.handler.admin.api.BalanceReplicas; import org.apache.solr.handler.admin.api.BalanceShardUnique; +import org.apache.solr.handler.admin.api.ClusterCommands; import org.apache.solr.handler.admin.api.ClusterProperty; import org.apache.solr.handler.admin.api.CollectionProperty; import org.apache.solr.handler.admin.api.CollectionStatus; @@ -437,13 +428,6 @@ public SolrResponse submitCollectionApiCommand( return submitCollectionApiCommand(coreContainer.getZkController(), adminCmdContext, m, timeout); } - private boolean overseerCollectionQueueContains(String asyncId) - throws KeeperException, InterruptedException { - OverseerTaskQueue collectionQueue = - coreContainer.getZkController().getOverseerCollectionQueue(); - return collectionQueue.containsTaskWithRequestId(ASYNC, asyncId); - } - /** * Copy prefixed params into a map. There must only be one value for these parameters. * @@ -490,14 +474,6 @@ public Category getCategory() { return Category.ADMIN; } - private static void addStatusToResponse( - NamedList results, RequestStatusState state, String msg) { - SimpleOrderedMap status = new SimpleOrderedMap<>(); - status.add("state", state.getKey()); - status.add("msg", msg); - results.add("status", status); - } - @SuppressWarnings("ImmutableEnumChecker") public enum CollectionOperation implements CollectionOp { CREATE_OP( @@ -789,141 +765,14 @@ collection, propName, new UpdateCollectionPropertyRequestBody(val)) REQUESTSTATUS_OP( REQUESTSTATUS, (req, rsp, h) -> { - req.getParams().required().check(REQUESTID); - - final CoreContainer coreContainer = h.coreContainer; - final String requestId = req.getParams().get(REQUESTID); - final ZkController zkController = coreContainer.getZkController(); - - final NamedList status = new NamedList<>(); - if (zkController.getDistributedCommandRunner().isEmpty()) { - if (zkController.getOverseerRunningMap().contains(requestId)) { - addStatusToResponse(status, RUNNING, "found [" + requestId + "] in running tasks"); - } else if (zkController.getOverseerCompletedMap().contains(requestId)) { - final byte[] mapEntry = zkController.getOverseerCompletedMap().get(requestId); - rsp.getValues() - .addAll(OverseerSolrResponseSerializer.deserialize(mapEntry).getResponse()); - addStatusToResponse( - status, COMPLETED, "found [" + requestId + "] in completed tasks"); - } else if (zkController.getOverseerFailureMap().contains(requestId)) { - final byte[] mapEntry = zkController.getOverseerFailureMap().get(requestId); - rsp.getValues() - .addAll(OverseerSolrResponseSerializer.deserialize(mapEntry).getResponse()); - addStatusToResponse(status, FAILED, "found [" + requestId + "] in failed tasks"); - } else if (h.overseerCollectionQueueContains(requestId)) { - addStatusToResponse( - status, SUBMITTED, "found [" + requestId + "] in submitted tasks"); - } else { - addStatusToResponse( - status, NOT_FOUND, "Did not find [" + requestId + "] in any tasks queue"); - } - } else { - Pair sr = - zkController - .getDistributedCommandRunner() - .get() - .getAsyncTaskRequestStatus(requestId); - final String message; - switch (sr.first()) { - case COMPLETED: - message = "found [" + requestId + "] in completed tasks"; - rsp.getValues().addAll(sr.second().getResponse()); - break; - case FAILED: - message = "found [" + requestId + "] in failed tasks"; - rsp.getValues().addAll(sr.second().getResponse()); - break; - case RUNNING: - message = "found [" + requestId + "] in running tasks"; - break; - case SUBMITTED: - message = "found [" + requestId + "] in submitted tasks"; - break; - default: - message = "Did not find [" + requestId + "] in any tasks queue"; - } - addStatusToResponse(status, sr.first(), message); - } - - rsp.getValues().addAll(status); + ClusterCommands.invokeGetFromV1Params(h.coreContainer, req, rsp); return null; }), DELETESTATUS_OP( DELETESTATUS, - new CollectionOp() { - @Override - public Map execute( - SolrQueryRequest req, SolrQueryResponse rsp, CollectionsHandler h) throws Exception { - final CoreContainer coreContainer = h.coreContainer; - final String requestId = req.getParams().get(REQUESTID); - final ZkController zkController = coreContainer.getZkController(); - boolean flush = req.getParams().getBool(CollectionAdminParams.FLUSH, false); - - if (requestId == null && !flush) { - throw new SolrException( - ErrorCode.BAD_REQUEST, "Either requestid or flush parameter must be specified."); - } - - if (requestId != null && flush) { - throw new SolrException( - ErrorCode.BAD_REQUEST, - "Both requestid and flush parameters can not be specified together."); - } - - if (zkController.getDistributedCommandRunner().isEmpty()) { - if (flush) { - Collection completed = zkController.getOverseerCompletedMap().keys(); - Collection failed = zkController.getOverseerFailureMap().keys(); - for (String asyncId : completed) { - zkController.getOverseerCompletedMap().remove(asyncId); - zkController.clearAsyncId(asyncId); - } - for (String asyncId : failed) { - zkController.getOverseerFailureMap().remove(asyncId); - zkController.clearAsyncId(asyncId); - } - rsp.getValues() - .add("status", "successfully cleared stored collection api responses"); - } else { - // Request to cleanup - if (zkController.getOverseerCompletedMap().remove(requestId)) { - zkController.clearAsyncId(requestId); - rsp.getValues() - .add( - "status", "successfully removed stored response for [" + requestId + "]"); - } else if (zkController.getOverseerFailureMap().remove(requestId)) { - zkController.clearAsyncId(requestId); - rsp.getValues() - .add( - "status", "successfully removed stored response for [" + requestId + "]"); - } else { - rsp.getValues() - .add("status", "[" + requestId + "] not found in stored responses"); - // Don't call zkController.clearAsyncId for this, since it could be a - // running/pending task - } - } - } else { - if (flush) { - zkController.getDistributedCommandRunner().get().deleteAllAsyncIds(); - rsp.getValues() - .add("status", "successfully cleared stored collection api responses"); - } else { - if (zkController - .getDistributedCommandRunner() - .get() - .deleteSingleAsyncId(requestId)) { - rsp.getValues() - .add( - "status", "successfully removed stored response for [" + requestId + "]"); - } else { - rsp.getValues() - .add("status", "[" + requestId + "] not found in stored responses"); - } - } - } - return null; - } + (req, rsp, h) -> { + ClusterCommands.invokeDeleteFromV1Params(h.coreContainer, req, rsp); + return null; }), ADDREPLICA_OP( ADDREPLICA, @@ -1379,7 +1228,8 @@ public Collection> getJerseyResources() { ListCollectionSnapshots.class, CreateCollectionSnapshot.class, DeleteCollectionSnapshot.class, - ClusterProperty.class); + ClusterProperty.class, + ClusterCommands.class); } @Override diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/ClusterCommands.java b/solr/core/src/java/org/apache/solr/handler/admin/api/ClusterCommands.java new file mode 100644 index 00000000000..792642dbe6c --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/ClusterCommands.java @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin.api; + +import static org.apache.solr.client.solrj.response.RequestStatusState.COMPLETED; +import static org.apache.solr.client.solrj.response.RequestStatusState.FAILED; +import static org.apache.solr.client.solrj.response.RequestStatusState.NOT_FOUND; +import static org.apache.solr.client.solrj.response.RequestStatusState.RUNNING; +import static org.apache.solr.client.solrj.response.RequestStatusState.SUBMITTED; +import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.REQUESTID; +import static org.apache.solr.common.params.CommonAdminParams.ASYNC; +import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; +import static org.apache.solr.security.PermissionNameProvider.Name.COLL_READ_PERM; + +import jakarta.inject.Inject; +import java.util.Collection; +import java.util.Map; +import org.apache.solr.client.api.endpoint.ClusterCommandsApi; +import org.apache.solr.client.api.model.DeleteClusterCommandStatusResponse; +import org.apache.solr.client.api.model.GetClusterCommandStatusResponse; +import org.apache.solr.client.api.model.GetClusterCommandStatusResponse.CommandStatus; +import org.apache.solr.client.solrj.response.RequestStatusState; +import org.apache.solr.cloud.OverseerSolrResponse; +import org.apache.solr.cloud.OverseerSolrResponseSerializer; +import org.apache.solr.cloud.ZkController; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.params.CollectionAdminParams; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.Pair; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.handler.api.V2ApiUtils; +import org.apache.solr.jersey.PermissionName; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; + +/** + * V2 APIs for checking and deleting cluster-level asynchronous Collection API command status. + * + *

{@code GET /api/cluster/commands/{requestId}} is analogous to v1 {@code + * /admin/collections?action=REQUESTSTATUS}. {@code DELETE /api/cluster/commands/{requestId}} and + * {@code DELETE /api/cluster/commands} are analogous to v1 {@code + * /admin/collections?action=DELETESTATUS}. + * + *

The v1 CollectionsHandler operations delegate to this class. + */ +public class ClusterCommands extends AdminAPIBase implements ClusterCommandsApi { + + @Inject + public ClusterCommands( + CoreContainer coreContainer, + SolrQueryRequest solrQueryRequest, + SolrQueryResponse solrQueryResponse) { + super(coreContainer, solrQueryRequest, solrQueryResponse); + } + + @Override + @PermissionName(COLL_READ_PERM) + public GetClusterCommandStatusResponse getClusterCommandStatus(String requestId) + throws Exception { + ensureRequiredParameterProvided(CoreAdminParams.REQUESTID, requestId); + fetchAndValidateZooKeeperAwareCoreContainer(); + + final GetClusterCommandStatusResponse response = + instantiateJerseyResponse(GetClusterCommandStatusResponse.class); + final ZkController zkController = coreContainer.getZkController(); + + if (zkController.getDistributedCommandRunner().isEmpty()) { + if (zkController.getOverseerRunningMap().contains(requestId)) { + setStatus(response, RUNNING, requestId); + } else if (zkController.getOverseerCompletedMap().contains(requestId)) { + copyStoredCommandResponse( + response, + OverseerSolrResponseSerializer.deserialize( + zkController.getOverseerCompletedMap().get(requestId)) + .getResponse()); + setStatus(response, COMPLETED, requestId); + } else if (zkController.getOverseerFailureMap().contains(requestId)) { + copyStoredCommandResponse( + response, + OverseerSolrResponseSerializer.deserialize( + zkController.getOverseerFailureMap().get(requestId)) + .getResponse()); + setStatus(response, FAILED, requestId); + } else if (overseerCollectionQueueContains(requestId)) { + setStatus(response, SUBMITTED, requestId); + } else { + setStatus(response, NOT_FOUND, requestId); + } + } else { + Pair sr = + zkController.getDistributedCommandRunner().get().getAsyncTaskRequestStatus(requestId); + switch (sr.first()) { + case COMPLETED: + case FAILED: + copyStoredCommandResponse(response, sr.second().getResponse()); + break; + default: + break; + } + setStatus(response, sr.first(), requestId); + } + + return response; + } + + @Override + @PermissionName(COLL_EDIT_PERM) + public DeleteClusterCommandStatusResponse deleteClusterCommandStatus(String requestId) + throws Exception { + ensureRequiredParameterProvided(CoreAdminParams.REQUESTID, requestId); + fetchAndValidateZooKeeperAwareCoreContainer(); + + final DeleteClusterCommandStatusResponse response = + instantiateJerseyResponse(DeleteClusterCommandStatusResponse.class); + final ZkController zkController = coreContainer.getZkController(); + + if (zkController.getDistributedCommandRunner().isEmpty()) { + if (zkController.getOverseerCompletedMap().remove(requestId)) { + zkController.clearAsyncId(requestId); + response.status = "successfully removed stored response for [" + requestId + "]"; + } else if (zkController.getOverseerFailureMap().remove(requestId)) { + zkController.clearAsyncId(requestId); + response.status = "successfully removed stored response for [" + requestId + "]"; + } else { + // Don't call zkController.clearAsyncId for this, since it could be a running/pending task + response.status = "[" + requestId + "] not found in stored responses"; + } + } else if (zkController.getDistributedCommandRunner().get().deleteSingleAsyncId(requestId)) { + response.status = "successfully removed stored response for [" + requestId + "]"; + } else { + response.status = "[" + requestId + "] not found in stored responses"; + } + + return response; + } + + @Override + @PermissionName(COLL_EDIT_PERM) + public DeleteClusterCommandStatusResponse deleteAllClusterCommandStatuses() throws Exception { + fetchAndValidateZooKeeperAwareCoreContainer(); + + final DeleteClusterCommandStatusResponse response = + instantiateJerseyResponse(DeleteClusterCommandStatusResponse.class); + final ZkController zkController = coreContainer.getZkController(); + + if (zkController.getDistributedCommandRunner().isEmpty()) { + Collection completed = zkController.getOverseerCompletedMap().keys(); + Collection failed = zkController.getOverseerFailureMap().keys(); + for (String asyncId : completed) { + zkController.getOverseerCompletedMap().remove(asyncId); + zkController.clearAsyncId(asyncId); + } + for (String asyncId : failed) { + zkController.getOverseerFailureMap().remove(asyncId); + zkController.clearAsyncId(asyncId); + } + } else { + zkController.getDistributedCommandRunner().get().deleteAllAsyncIds(); + } + response.status = "successfully cleared stored collection api responses"; + return response; + } + + /** + * v1 {@code REQUESTSTATUS} entrypoint. Squashes the JAX-RS response into {@code rsp}, converting + * the nested status object to a {@link SimpleOrderedMap} so SolrJ's v1 {@code RequestStatus} + * parser continues to work. + */ + public static void invokeGetFromV1Params( + CoreContainer coreContainer, SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { + req.getParams().required().check(REQUESTID); + final ClusterCommands api = new ClusterCommands(coreContainer, req, rsp); + final GetClusterCommandStatusResponse jerseyResponse = + api.getClusterCommandStatus(req.getParams().get(REQUESTID)); + V2ApiUtils.squashIntoSolrResponseWithoutHeader(rsp, jerseyResponse); + convertStatusToNamedList(rsp); + } + + /** + * v1 {@code DELETESTATUS} entrypoint. {@code requestid} and {@code flush} remain mutually + * exclusive query parameters on v1; the v2 API uses distinct paths instead. + */ + public static void invokeDeleteFromV1Params( + CoreContainer coreContainer, SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { + final String requestId = req.getParams().get(REQUESTID); + final boolean flush = req.getParams().getBool(CollectionAdminParams.FLUSH, false); + + if (requestId == null && !flush) { + throw new SolrException( + ErrorCode.BAD_REQUEST, "Either requestid or flush parameter must be specified."); + } + if (requestId != null && flush) { + throw new SolrException( + ErrorCode.BAD_REQUEST, + "Both requestid and flush parameters can not be specified together."); + } + + final ClusterCommands api = new ClusterCommands(coreContainer, req, rsp); + final DeleteClusterCommandStatusResponse jerseyResponse = + flush ? api.deleteAllClusterCommandStatuses() : api.deleteClusterCommandStatus(requestId); + V2ApiUtils.squashIntoSolrResponseWithoutHeader(rsp, jerseyResponse); + } + + private boolean overseerCollectionQueueContains(String asyncId) throws Exception { + return coreContainer + .getZkController() + .getOverseerCollectionQueue() + .containsTaskWithRequestId(ASYNC, asyncId); + } + + private static void setStatus( + GetClusterCommandStatusResponse response, RequestStatusState state, String requestId) { + final CommandStatus status = new CommandStatus(); + status.state = CommandStatus.State.valueOf(state.name()); + status.msg = statusMessage(state, requestId); + response.status = status; + } + + private static String statusMessage(RequestStatusState state, String requestId) { + return switch (state) { + case RUNNING -> "found [" + requestId + "] in running tasks"; + case COMPLETED -> "found [" + requestId + "] in completed tasks"; + case FAILED -> "found [" + requestId + "] in failed tasks"; + case SUBMITTED -> "found [" + requestId + "] in submitted tasks"; + default -> "Did not find [" + requestId + "] in any tasks queue"; + }; + } + + private static void copyStoredCommandResponse( + GetClusterCommandStatusResponse response, NamedList stored) { + if (stored == null) { + return; + } + for (Map.Entry entry : stored) { + final String key = entry.getKey(); + if ("responseHeader".equals(key) || "error".equals(key) || "status".equals(key)) { + continue; + } + response.setUnknownProperty(key, entry.getValue()); + } + } + + /** + * v1 SolrJ reads {@code status} as a {@link NamedList}. Squash leaves the JAX-RS {@link + * CommandStatus} POJO in place; replace it so existing clients keep working. + */ + private static void convertStatusToNamedList(SolrQueryResponse rsp) { + final NamedList values = rsp.getValues(); + final int idx = values.indexOf("status", 0); + if (idx < 0) { + return; + } + final Object statusVal = values.getVal(idx); + if (statusVal instanceof CommandStatus commandStatus) { + final SimpleOrderedMap status = new SimpleOrderedMap<>(); + status.add("state", commandStatus.state.getKey()); + status.add("msg", commandStatus.msg); + values.setVal(idx, status); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/V2ClusterAPIMappingTest.java b/solr/core/src/test/org/apache/solr/handler/V2ClusterAPIMappingTest.java index b41640f1534..4fc458bc589 100644 --- a/solr/core/src/test/org/apache/solr/handler/V2ClusterAPIMappingTest.java +++ b/solr/core/src/test/org/apache/solr/handler/V2ClusterAPIMappingTest.java @@ -17,7 +17,6 @@ package org.apache.solr.handler; -import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.REQUESTID; import static org.apache.solr.common.params.CommonParams.ACTION; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; @@ -69,15 +68,6 @@ public void setupApiBag() { apiBag.registerObject(clusterAPI.commands); } - @Test - public void testAsyncCommandStatusAllParams() throws Exception { - final SolrParams v1Params = - captureConvertedV1Params("/cluster/command-status/someId", "GET", null); - - assertEquals(CollectionParams.CollectionAction.REQUESTSTATUS.lowerName, v1Params.get(ACTION)); - assertEquals("someId", v1Params.get(REQUESTID)); - } - @Test public void testClusterOverseerAllParams() throws Exception { final SolrParams v1Params = captureConvertedV1Params("/cluster/overseer", "GET", null); @@ -92,15 +82,6 @@ public void testClusterStatusAllParams() throws Exception { assertEquals(CollectionAction.CLUSTERSTATUS.lowerName, v1Params.get(ACTION)); } - @Test - public void testDeleteCommandStatusAllParams() throws Exception { - final SolrParams v1Params = - captureConvertedV1Params("/cluster/command-status/someId", "DELETE", null); - - assertEquals(CollectionParams.CollectionAction.DELETESTATUS.lowerName, v1Params.get(ACTION)); - assertEquals("someId", v1Params.get(REQUESTID)); - } - @Test public void testAddRoleAllParams() throws Exception { final SolrParams v1Params = diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/ClusterCommandsTest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/ClusterCommandsTest.java new file mode 100644 index 00000000000..dc6857cfa28 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/ClusterCommandsTest.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin.api; + +import java.util.concurrent.TimeUnit; +import org.apache.solr.client.api.model.DeleteClusterCommandStatusResponse; +import org.apache.solr.client.api.model.GetClusterCommandStatusResponse; +import org.apache.solr.client.api.model.GetClusterCommandStatusResponse.CommandStatus.State; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.request.ClusterApi; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * HTTP tests for {@code GET/DELETE /api/cluster/commands} via the generated SolrJ client classes. + */ +public class ClusterCommandsTest extends SolrCloudTestCase { + + public static final int MAX_WAIT_TIMEOUT = 30; + + @BeforeClass + public static void setupCluster() throws Exception { + configureCluster(1) + .addConfig( + "conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) + .configure(); + } + + @Test + public void testGetCommandStatusNotFound() throws Exception { + GetClusterCommandStatusResponse rsp = + new ClusterApi.GetClusterCommandStatus("does-not-exist").process(cluster.getSolrClient()); + + assertNotNull(rsp); + assertNull(rsp.error); + assertEquals(State.NOT_FOUND, rsp.status.state); + assertEquals("Did not find [does-not-exist] in any tasks queue", rsp.status.msg); + } + + @Test + public void testGetAndDeleteSingleCommandStatus() throws Exception { + final SolrClient client = cluster.getSolrClient(); + final String collection = "cluster-commands-single"; + final String asyncId = + CollectionAdminRequest.createCollection(collection, "conf1", 1, 1).processAsync(client); + + GetClusterCommandStatusResponse getRsp = waitForCompleted(asyncId, client); + assertEquals(State.COMPLETED, getRsp.status.state); + assertEquals("found [" + asyncId + "] in completed tasks", getRsp.status.msg); + assertTrue( + "completed create should include sub-responses from the original command", + getRsp.unknownProperties().containsKey("success") + || getRsp.unknownProperties().containsKey("failure")); + + DeleteClusterCommandStatusResponse deleteRsp = + new ClusterApi.DeleteClusterCommandStatus(asyncId).process(client); + assertEquals("successfully removed stored response for [" + asyncId + "]", deleteRsp.status); + + GetClusterCommandStatusResponse afterDelete = + new ClusterApi.GetClusterCommandStatus(asyncId).process(client); + assertEquals(State.NOT_FOUND, afterDelete.status.state); + } + + @Test + public void testDeleteUnknownCommandStatus() throws Exception { + DeleteClusterCommandStatusResponse rsp = + new ClusterApi.DeleteClusterCommandStatus("foo").process(cluster.getSolrClient()); + assertEquals("[foo] not found in stored responses", rsp.status); + } + + @Test + public void testDeleteAllCommandStatuses() throws Exception { + final SolrClient client = cluster.getSolrClient(); + final String id1 = + CollectionAdminRequest.createCollection("cluster-commands-flush-1", "conf1", 1, 1) + .processAsync(client); + final String id2 = + CollectionAdminRequest.createCollection("cluster-commands-flush-2", "conf1", 1, 1) + .processAsync(client); + + waitForCompleted(id1, client); + waitForCompleted(id2, client); + + DeleteClusterCommandStatusResponse flushRsp = + new ClusterApi.DeleteAllClusterCommandStatuses().process(client); + assertEquals("successfully cleared stored collection api responses", flushRsp.status); + + assertEquals( + State.NOT_FOUND, new ClusterApi.GetClusterCommandStatus(id1).process(client).status.state); + assertEquals( + State.NOT_FOUND, new ClusterApi.GetClusterCommandStatus(id2).process(client).status.state); + } + + private static GetClusterCommandStatusResponse waitForCompleted(String id, SolrClient client) + throws Exception { + GetClusterCommandStatusResponse rsp = null; + long endTime = System.nanoTime() + TimeUnit.SECONDS.toNanos(MAX_WAIT_TIMEOUT); + while (System.nanoTime() < endTime) { + rsp = new ClusterApi.GetClusterCommandStatus(id).process(client); + State state = rsp.status.state; + assumeTrue("Error creating collection - skipping test", state != State.FAILED); + if (state == State.COMPLETED) { + return rsp; + } + TimeUnit.SECONDS.sleep(1); + } + assumeTrue( + "Timed out waiting for async request " + id, + rsp != null && State.COMPLETED.equals(rsp.status.state)); + return rsp; + } +} diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/collections-api.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/collections-api.adoc index 0ed88262b49..71a20e63acf 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/collections-api.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/collections-api.adoc @@ -117,7 +117,7 @@ V2 API:: ==== [source,bash] ---- -curl -X GET http://localhost:8983/api/cluster/command-status/1000 +curl -X GET http://localhost:8983/api/cluster/commands/1000 ---- ==== ====== @@ -205,13 +205,13 @@ V2 API:: Delete a single request response: [source,bash] ---- -curl -X DELETE http://localhost:8983/api/cluster/command-status/1000 +curl -X DELETE http://localhost:8983/api/cluster/commands/1000 ---- Flush out all stored completed and failed async request responses: [source,bash] ---- -curl -X DELETE http://localhost:8983/api/cluster/command-status?flush=true +curl -X DELETE http://localhost:8983/api/cluster/commands ---- ==== ====== @@ -234,7 +234,8 @@ The request ID of the asynchronous call whose stored response should be cleared. |Optional |Default: none |=== + -Set to `true` to clear all stored completed and failed async request responses. +v1 only. Set to `true` to clear all stored completed and failed async request responses. +The v2 API uses `DELETE /api/cluster/commands` for the same operation and does not take this parameter. === Examples using DELETESTATUS