Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,7 @@ pipeline_build_logs/*
.vscode/*

# Bazel symlinks — point into the local cache, machine-specific
/bazel-*
/bazel-*

# Protobuf generated files
**/definiton/gen/
3 changes: 2 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
#
# Build stage
#
FROM maven:3.8.3-openjdk-17 AS build
FROM maven:3.9.6-eclipse-temurin-17 AS build
WORKDIR /home/app
COPY pipeline/pom.xml .
COPY pipeline/src ./src
COPY apis/protobuf /home/apis/protobuf
RUN mvn clean package -DskipTests -Pmonitoring

#
Expand Down
68 changes: 26 additions & 42 deletions apis/openapi-spec/pipeline-service.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -249,35 +249,32 @@ components:
properties:
id:
type: string
format: uuid
name:
type: string
stages:
type: array
items:
$ref: "#/components/schemas/Stage"
required:
- name
- stages
Stage:
PipelineBuild:
type: object
properties:
id:
currentState:
type: string
format: uuid
name:
enum:
- WAITING
- RUNNING
- SUCCESS
- FAILED
id:
type: string
scriptType:
pipelineId:
type: string
command:
pipelineName:
type: string
stageOrder:
type: integer
format: int32
required:
- command
- name
- scriptType
stageBuilds:
type: array
items:
$ref: "#/components/schemas/StageBuild"
PipelineExecuteRequest:
type: object
properties:
Expand All @@ -286,53 +283,40 @@ components:
format: uuid
required:
- pipelineId
PipelineBuild:
Stage:
type: object
properties:
command:
type: string
id:
type: string
format: uuid
pipelineId:
name:
type: string
format: uuid
pipelineName:
scriptType:
type: string
stageOrder:
type: integer
format: int32
StageBuild:
type: object
properties:
currentState:
type: string
enum:
- WAITING
- RUNNING
- SUCCESS
- FAILED
startedAt:
type: string
format: date-time
stageBuilds:
type: array
items:
$ref: "#/components/schemas/StageBuild"
StageBuild:
type: object
properties:
- STOPPED
id:
type: string
format: uuid
stageId:
type: string
format: uuid
stageName:
type: string
stageOrder:
type: integer
format: int32
currentState:
type: string
enum:
- WAITING
- RUNNING
- SUCCESS
- FAILED
- STOPPED
securitySchemes:
bearerAuth:
type: http
Expand Down
5 changes: 5 additions & 0 deletions apis/protobuf/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
filegroup(
name = "protobuf_files",
srcs = glob(["**/*.proto"]),
visibility = ["//visibility:public"],
)
1 change: 1 addition & 0 deletions apis/protobuf/buf.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
version: v2
23 changes: 23 additions & 0 deletions apis/protobuf/definition/v1/pipeline_definition.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
syntax = "proto3";

Comment thread
SuvabrataChowdhury marked this conversation as resolved.
Resolved
package definition.v1;

option java_package = "com.autobuild.pipeline.definiton.gen.dto";
option java_multiple_files = true;

// option go_package = "";
// go_multiple_files doesn't exist - Go always generates one file per proto file.

message Stage {
string id = 1;
string name = 2;
string script_type = 3;
string command = 4;
int32 stage_order = 5;
}

message Pipeline {
string id = 1;
string name = 2;
repeated Stage stages = 3;
}
49 changes: 49 additions & 0 deletions apis/protobuf/execution/v1/pipeline_execution.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
syntax = "proto3";
Comment thread
SuvabrataChowdhury marked this conversation as resolved.
Resolved

package execution.v1;

import "google/protobuf/timestamp.proto";

option java_package = "com.autobuild.pipeline.executor.gen.dto";
option java_multiple_files = true;

// option go_package = "";
// go_multiple_files doesn't exist - Go always generates one file per proto file.

enum PipelineExecutionState {
PIPELINE_WAITING = 0; // Pipeline is waiting to be executed.
PIPELINE_RUNNING = 1; // Pipeline execution has started.
PIPELINE_SUCCESS = 2; // Pipeline execution has finished and it succeeded.
PIPELINE_FAILED = 3; // Pipeline execution has finished and it failed.
}

enum StageExecutionState {
STAGE_WAITING = 0; // Stage is waiting to be executed.
STAGE_RUNNING = 1; // Stage execution has started.
STAGE_SUCCESS = 2; // Stage execution has finished and it succeeded.
STAGE_FAILED = 3; // Stage execution has finished and it failed.
STAGE_STOPPED = 4; // Stage execution stopped.
}

message StageBuild {
string id = 1;
string stage_id = 2;
string stage_name = 3;
StageExecutionState current_state = 4;
int32 stage_order = 5;
}

message PipelineBuild {
string id = 1;
string pipeline_id = 2;
string pipeline_name = 3;
PipelineExecutionState current_state = 4;
repeated StageBuild stage_builds = 5;
google.protobuf.Timestamp started_at = 6;
}

// A single line of output from a running stage, streamed back as it is produced.
message StageLog {
string stage_build_id = 1;
string line = 2;
}
13 changes: 12 additions & 1 deletion autobuild-docs/setup/backend/local-backend-execution.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,15 @@
1. Locate [backend launch.json](https://github.com/SuvabrataChowdhury/AutoBuild/blob/launch-json-template/pipeline/.vscode/launch.json.template) and rename it to just `launch.json`.
2. From VS Code `Run and Debug` (keybind SHIFT+CMD+D for Mac) select `AutoBuild: Local` and run the backend service in debug mode.
![VS Code Execution](./images/vs-code-execution.png)
Ensure the application is running fine by hitting the endpoints through bruno.
Ensure the application is running fine by hitting the endpoints through bruno.


3. [Optional] To generate Protobuf classes locally:
From the `pipeline/` directory, run: `mvn compile -s maven-settings.xml`
Without `-s maven-settings.xml`, Maven falls back to repository mentioned at `~/.m2/repository/settings.xml` and might fail if it does not have the necessary dependencies. Mentioning it like that ensures the dependencies are pulled from open maven repository.
(The files are generated at the `generate-sources` and writes the Java files into target/generated-sources/. Then compile picks them up alongside your hand-written code and compiles everything together.)
4. Recommended VS Code extension for .proto files:
**Buf** (`bufbuild.vscode-buf`) — syntax highlighting and linting
https://marketplace.visualstudio.com/items?itemName=bufbuild.vscode-buf
5. To understand Protocol Buffers (the format used for DTO generation):
Protocol Buffers Crash Course — https://youtu.be/46O73On0gyI
4 changes: 4 additions & 0 deletions autobuild-docs/setup/backend/running-backend-containers.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,3 +51,7 @@ docker compose -p autobuild down
docker compose down -v
```
> Note: this will delete all persisted volumes including database data, Grafana dashboards, Loki logs, and Prometheus metrics.

## Protobuf

Proto classes are auto-generated during `docker build` — Maven runs the protobuf compiler as part of the build, so no separate step is needed.
6 changes: 3 additions & 3 deletions pipeline/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,17 @@ load("@rules_shell//shell:sh_test.bzl", "sh_test")
# Inputs Bazel tracks for cache invalidation — if none of these change,
# both targets below are skipped entirely (cache hit).
_SRCS = glob(
["src/**", "pom.xml", "checkstyle.xml"],
["src/**", "pom.xml", "checkstyle.xml", "maven-settings.xml"],
exclude = ["target/**"],
)
) + ["//apis/protobuf:protobuf_files"]

# Runs the full Maven test suite.
# Run with: bazel test //pipeline:test
sh_test(
name = "test",
srcs = ["run_tests.sh"],
data = _SRCS,
tags = [],
tags = ["local"],
)

# Generates the OpenAPI spec by starting the app and hitting /v3/api-docs.yaml.
Expand Down
4 changes: 4 additions & 0 deletions pipeline/maven-settings.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
<settings xmlns="http://maven.apache.org/SETTINGS/1.2.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.2.0 https://maven.apache.org/xsd/settings-1.2.0.xsd">
</settings>
40 changes: 40 additions & 0 deletions pipeline/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
</scm>
<properties>
<java.version>17</java.version>
<protobuf.version>4.29.3</protobuf.version>
</properties>
<dependencies>
<dependency>
Expand Down Expand Up @@ -156,6 +157,18 @@
<artifactId>spring-boot-starter-graphql</artifactId>
</dependency>

<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>${protobuf.version}</version>
</dependency>

<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java-util</artifactId>
<version>${protobuf.version}</version>
</dependency>

</dependencies>

<build>
Expand Down Expand Up @@ -202,6 +215,7 @@
<encoding>UTF-8</encoding>
<consoleOutput>true</consoleOutput>
<failsOnError>true</failsOnError>
<excludeGeneratedSources>true</excludeGeneratedSources>
</configuration>
</plugin>
<plugin>
Expand All @@ -224,6 +238,9 @@
<version>0.8.15</version>

<configuration>
<excludes>
<exclude>com/autobuild/pipeline/**/gen/**</exclude>
</excludes>
<rules>
<rule>
<element>BUNDLE</element>
Expand Down Expand Up @@ -261,8 +278,31 @@
<goals>
<goal>check</goal>
</goals>
<configuration>
<skip>${skipTests}</skip>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>io.github.ascopes</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
<version>5.1.11</version>
<configuration>
<protoc kind="binary-maven">
<version>${protobuf.version}</version>
Comment thread
SuvabrataChowdhury marked this conversation as resolved.
Resolved
</protoc>
<sourceDirectories>
<sourceDirectory>${project.basedir}/../apis/protobuf</sourceDirectory>
</sourceDirectories>
</configuration>
<executions>
<execution>
<goals>
<goal>generate</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
Expand Down
2 changes: 1 addition & 1 deletion pipeline/run_generate_openapi.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@
set -euo pipefail

cd "./pipeline/"
mvn --batch-mode verify -P integration-test -DskipTests
mvn --batch-mode -s maven-settings.xml verify -P integration-test -DskipTests
2 changes: 1 addition & 1 deletion pipeline/run_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@
set -euo pipefail

cd "./pipeline/"
mvn --batch-mode clean verify
mvn --batch-mode -s maven-settings.xml clean verify
Loading
Loading