Repository navigation
Init message queue integration for async pipeline execution #264
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -156,12 +156,15 @@ paths: | |
| $ref: "#/components/schemas/PipelineExecuteRequest" | ||
| required: true | ||
| responses: | ||
| "203": | ||
| description: Non-Authoritative Information | ||
| "202": | ||
| description: Accepted | ||
| content: | ||
| application/json: | ||
| schema: | ||
| $ref: "#/components/schemas/PipelineBuild" | ||
| type: object | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. shouldn't it point to PipelineBuild? |
||
| additionalProperties: | ||
| type: string | ||
| format: uuid | ||
| security: | ||
| - bearerAuth: [] | ||
| /api/v1/stage/build/logs/{stageBuildId}: | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -67,6 +67,19 @@ services: | |
| - "8016:8016" | ||
| networks: | ||
| - app | ||
| rabbitmq: | ||
| image: rabbitmq:3-management | ||
| container_name: rabbitmq | ||
| ports: | ||
| - "5672:5672" | ||
| - "15672:15672" | ||
| networks: | ||
| - app | ||
| healthcheck: | ||
| test: ["CMD", "rabbitmq-diagnostics", "ping"] | ||
| interval: 10s | ||
| timeout: 5s | ||
| retries: 5 | ||
| pipeline-app: | ||
| image: pipeline-app | ||
| container_name: pipeline-app | ||
|
|
@@ -79,6 +92,7 @@ services: | |
| AUTH_PORT: 8080 | ||
| OTEL_SERVICE_NAME: pipeline-app | ||
| FEATURE_PROVIDER_URI: http://flagd:8016 | ||
| RABBITMQ_URL: amqp://guest:guest@rabbitmq:5672 | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Let's call the env a generic name.. something like MQ_URL. This will help us swap the queue vendor in future if required. |
||
| ports: | ||
| - "8080:8080" | ||
| volumes: | ||
|
|
@@ -92,4 +106,6 @@ services: | |
| auth-service: | ||
| condition: service_healthy | ||
| flagd: | ||
| condition: service_started | ||
| condition: service_started | ||
| rabbitmq: | ||
| condition: service_healthy | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| package com.autobuild.pipeline.configuration; | ||
|
|
||
| import org.springframework.amqp.core.Queue; | ||
| import org.springframework.amqp.rabbit.connection.ConnectionFactory; | ||
| import org.springframework.amqp.rabbit.core.RabbitTemplate; | ||
| import org.springframework.amqp.support.converter.Jackson2JsonMessageConverter; | ||
| import org.springframework.context.annotation.Bean; | ||
| import org.springframework.context.annotation.Configuration; | ||
|
|
||
| /** | ||
| * Message queue configuration. | ||
| */ | ||
| @Configuration | ||
| public class MessagingConfig { | ||
|
|
||
| /** Name of the queue used to dispatch pipeline build execution requests. */ | ||
| public static final String PIPELINE_BUILD_REQUEST_QUEUE = "pipeline-build-request-queue"; | ||
|
|
||
| @Bean | ||
| public Queue pipelineBuildQueue() { | ||
| return new Queue(PIPELINE_BUILD_REQUEST_QUEUE, true); | ||
| } | ||
|
|
||
| @Bean | ||
| public Jackson2JsonMessageConverter messageConverter() { | ||
| return new Jackson2JsonMessageConverter(); | ||
| } | ||
|
|
||
| @Bean | ||
| public RabbitTemplate rabbitTemplate(ConnectionFactory connectionFactory) { | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. can we have it as a conditional bean? like if rabbitmq is being used then only create this bean. Otherwise an unnecessary bean will remain in spring pool. |
||
| RabbitTemplate template = new RabbitTemplate(connectionFactory); | ||
| template.setMessageConverter(messageConverter()); | ||
| return template; | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,16 +1,16 @@ | ||
| package com.autobuild.pipeline.executor.controller; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.Map; | ||
| import java.util.UUID; | ||
|
|
||
| import org.springframework.http.MediaType; | ||
| import org.springframework.beans.factory.annotation.Autowired; | ||
| import org.springframework.http.MediaType; | ||
| import org.springframework.http.ResponseEntity; | ||
| import org.springframework.web.bind.annotation.PostMapping; | ||
| import org.springframework.web.bind.annotation.RequestBody; | ||
| import org.springframework.web.bind.annotation.RequestMapping; | ||
| import org.springframework.web.bind.annotation.RestController; | ||
|
|
||
| import com.autobuild.pipeline.executor.dto.PipelineBuildDTO; | ||
| import com.autobuild.pipeline.executor.dto.PipelineExecuteRequest; | ||
| import com.autobuild.pipeline.executor.service.PipelineExecutorService; | ||
|
|
||
|
|
@@ -24,8 +24,6 @@ | |
|
|
||
| /** | ||
| * Controller for executing a pipeline. | ||
| * | ||
| * @author Suvabrata Chowdhury | ||
| */ | ||
| @Tag(name = "Pipeline") | ||
| @SecurityScheme( | ||
|
|
@@ -43,12 +41,10 @@ public class PipelineExecutorController { | |
| @Autowired | ||
| private PipelineExecutorService service; | ||
|
|
||
| @Operation(summary = "Execute a pipeline", responses = @ApiResponse(responseCode = "203")) | ||
| @Operation(summary = "Execute a pipeline", responses = @ApiResponse(responseCode = "202")) | ||
| @PostMapping(value = "/pipeline", produces = MediaType.APPLICATION_JSON_VALUE) | ||
| public ResponseEntity<PipelineBuildDTO> executePipeline(@RequestBody @Valid PipelineExecuteRequest request) | ||
| throws IOException { | ||
| PipelineBuildDTO build = service.executePipeline(request); | ||
|
|
||
| return ResponseEntity.accepted().body(build); | ||
| public ResponseEntity<Map<String, UUID>> executePipeline(@RequestBody @Valid PipelineExecuteRequest request) { | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I get the Map<String,UUID> response.. Please have it as some DTO so that both OpenAPI spec and UI objects have a predictable schema. Please test with UI as well.. I feel this might break the execute flow as we were previously returning the full build object but now we are returning just the UUID. |
||
| UUID buildId = service.executePipeline(request); | ||
| return ResponseEntity.accepted().body(Map.of("buildId", buildId)); | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| package com.autobuild.pipeline.executor.dto.messaging; | ||
|
|
||
| import java.util.List; | ||
| import java.util.UUID; | ||
|
|
||
| import lombok.AllArgsConstructor; | ||
| import lombok.Getter; | ||
| import lombok.NoArgsConstructor; | ||
| import lombok.Setter; | ||
|
|
||
| /** | ||
| * Message payload published to the queue when a pipeline execution is requested. | ||
| * Contains the pipeline's definition so the executor can process it independently. | ||
| */ | ||
| @NoArgsConstructor | ||
| @AllArgsConstructor | ||
| @Getter | ||
| @Setter | ||
| public class PipelineBuildRequestMessage { | ||
|
|
||
| private UUID buildId; | ||
| private UUID pipelineId; | ||
| private String pipelineName; | ||
| private List<StageDefinition> stages; | ||
| } |
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -0,0 +1,21 @@ | ||||||
| package com.autobuild.pipeline.executor.dto.messaging; | ||||||
|
|
||||||
| import lombok.AllArgsConstructor; | ||||||
| import lombok.Getter; | ||||||
| import lombok.NoArgsConstructor; | ||||||
| import lombok.Setter; | ||||||
|
|
||||||
| /** | ||||||
| * Represents a single stage within a pipeline build request message. | ||||||
| */ | ||||||
| @NoArgsConstructor | ||||||
| @AllArgsConstructor | ||||||
| @Getter | ||||||
| @Setter | ||||||
| public class StageDefinition { | ||||||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. as this is also a message let's use the same convention and call it StageBuildRequestMessage,
Suggested change
|
||||||
|
|
||||||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Let's add stageId also here for reference purpose. |
||||||
| private String name; | ||||||
| private String scriptType; | ||||||
| private String command; | ||||||
| private int stageOrder; | ||||||
| } | ||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| package com.autobuild.pipeline.executor.messaging; | ||
|
|
||
| import com.autobuild.pipeline.executor.dto.messaging.PipelineBuildRequestMessage; | ||
|
|
||
| /** | ||
| * Contract for publishing pipeline build request messages to a message queue. | ||
| */ | ||
| public interface PipelineBuildMessagePublisher { | ||
|
|
||
| /** | ||
| * Publishes a pipeline build request message. | ||
| * | ||
| * @param message the build request containing the pipeline definition and generated build ID | ||
| */ | ||
| void publish(PipelineBuildRequestMessage message); | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| package com.autobuild.pipeline.executor.messaging; | ||
|
|
||
| import org.springframework.amqp.rabbit.core.RabbitTemplate; | ||
| import org.springframework.beans.factory.annotation.Autowired; | ||
| import org.springframework.stereotype.Service; | ||
|
|
||
| import com.autobuild.pipeline.configuration.MessagingConfig; | ||
| import com.autobuild.pipeline.executor.dto.messaging.PipelineBuildRequestMessage; | ||
|
|
||
| import lombok.extern.slf4j.Slf4j; | ||
|
|
||
| /** | ||
| * RabbitMQ implementation of {@link PipelineBuildMessagePublisher}. | ||
| */ | ||
| @Slf4j | ||
| @Service | ||
| public class RabbitMqPipelineBuildMessagePublisher implements PipelineBuildMessagePublisher { | ||
|
|
||
| @Autowired | ||
| private RabbitTemplate rabbitTemplate; | ||
|
|
||
| @Override | ||
| public void publish(PipelineBuildRequestMessage message) { | ||
| rabbitTemplate.convertAndSend(MessagingConfig.PIPELINE_BUILD_REQUEST_QUEUE, message); | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. does it throw some exception? please check if any runtime exception it throws that needs to be handled properly.
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It might throw AmqpException so adding a check will help |
||
| log.info("Published build request {} to queue '{}'", | ||
| message.getBuildId(), MessagingConfig.PIPELINE_BUILD_REQUEST_QUEUE); | ||
| } | ||
| } | ||
|
SuvabrataChowdhury marked this conversation as resolved.
Resolved
|
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -9,8 +9,6 @@ | |
|
|
||
| /** | ||
| * Repository layer for PipelineBuild. | ||
| * | ||
| * @author Suvabrata Chowdhury | ||
| */ | ||
|
|
||
| @Repository | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
interesting.. didn't know it was set to 203 in the openapi spec 😅