From a1b7ce870076982a03bd37ed5bc21d63c59aa94e Mon Sep 17 00:00:00 2001 From: Jelle Hoffman Date: Mon, 24 Mar 2025 14:15:18 +0100 Subject: [PATCH 01/64] FDP-3056: Added smhe containers Signed-off-by: Jelle Hoffman jelle.hoffman@hotmail.com --- .../osgp-secret-management/Dockerfile | 19 +++++++++ .../osgp-throttling-service/Dockerfile | 14 +++++++ osgp/platform/osgp-throttling-service/pom.xml | 1 + .../osgp-protocol-adapter-dlms/Dockerfile | 17 ++++++++ .../osgp-protocol-adapter-dlms/pom.xml | 1 + .../osgp-simulator-dlms-triggered/Dockerfile | 15 +++++++ .../osgp-simulator-dlms-triggered/context.xml | 39 +++++++++++++++++++ .../osgp-simulator-dlms-triggered/pom.xml | 2 + .../dlms-device-simulator-starter/Dockerfile | 7 ++++ 9 files changed, 115 insertions(+) create mode 100644 osgp/platform/osgp-secret-management/Dockerfile create mode 100644 osgp/platform/osgp-throttling-service/Dockerfile create mode 100644 osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms/Dockerfile create mode 100644 osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered/Dockerfile create mode 100644 osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered/context.xml create mode 100644 osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/simulator/dlms-device-simulator-starter/Dockerfile diff --git a/osgp/platform/osgp-secret-management/Dockerfile b/osgp/platform/osgp-secret-management/Dockerfile new file mode 100644 index 00000000000..65219d66db6 --- /dev/null +++ b/osgp/platform/osgp-secret-management/Dockerfile @@ -0,0 +1,19 @@ +FROM tomcat:10.1-jre17-temurin-noble + +RUN rm -rf /usr/local/tomcat/webapps.dist + +# Enable the use of system environment variables in Tomcat configuration files. +RUN echo "org.apache.tomcat.util.digester.PROPERTY_SOURCE=org.apache.tomcat.util.digester.EnvironmentPropertySource" >> /usr/local/tomcat/conf/catalina.properties + +RUN mkdir -p /var/log/osp && \ + rm -rf /usr/local/tomcat/webapps/* && \ + chmod 755 /usr/local/tomcat/webapps /var/log/osp && \ + mkdir -p /usr/local/tomcat/webapps/probe/ && \ + touch /usr/local/tomcat/webapps/probe/probe.txt \ + +WORKDIR /opt/app +COPY target/osgp-secret-management-*exec.jar osgp-secret-management-exec.jar + +ENV SERVER_PORT=8443 + +CMD java -Djava.security.properties=/opt/app/java.security --module-path /opt/nfast/java/classes -jar osgp-secret-management-exec.jar diff --git a/osgp/platform/osgp-throttling-service/Dockerfile b/osgp/platform/osgp-throttling-service/Dockerfile new file mode 100644 index 00000000000..a222ca4466f --- /dev/null +++ b/osgp/platform/osgp-throttling-service/Dockerfile @@ -0,0 +1,14 @@ +FROM tomcat:10.1-jre17-temurin-noble + +RUN rm -rf /usr/local/tomcat/webapps.dist + +# Enable the use of system environment variables in Tomcat configuration files. +RUN echo "org.apache.tomcat.util.digester.PROPERTY_SOURCE=org.apache.tomcat.util.digester.EnvironmentPropertySource" >> /usr/local/tomcat/conf/catalina.properties + +RUN mkdir -p /var/log/osp && \ + rm -rf /usr/local/tomcat/webapps/* && \ + chmod 755 /usr/local/tomcat/webapps /var/log/osp && \ + mkdir -p /usr/local/tomcat/webapps/probe/ && \ + touch /usr/local/tomcat/webapps/probe/probe.txt \ + +COPY target/gxf /usr/local/tomcat/webapps/ \ No newline at end of file diff --git a/osgp/platform/osgp-throttling-service/pom.xml b/osgp/platform/osgp-throttling-service/pom.xml index 82c2fa555ef..73a53713806 100644 --- a/osgp/platform/osgp-throttling-service/pom.xml +++ b/osgp/platform/osgp-throttling-service/pom.xml @@ -39,6 +39,7 @@ SPDX-License-Identifier: Apache-2.0 maven-war-plugin ${maven.war.plugin.version} + ${project.build.directory}/gxf/${project.artifactId} true diff --git a/osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms/Dockerfile b/osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms/Dockerfile new file mode 100644 index 00000000000..72fa446bada --- /dev/null +++ b/osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms/Dockerfile @@ -0,0 +1,17 @@ +FROM tomcat:10.1-jre17-temurin-noble + +RUN rm -rf /usr/local/tomcat/webapps.dist + +# Enable the use of system environment variables in Tomcat configuration files. +RUN echo "org.apache.tomcat.util.digester.PROPERTY_SOURCE=org.apache.tomcat.util.digester.EnvironmentPropertySource" >> /usr/local/tomcat/conf/catalina.properties + +RUN mkdir -p /var/log/osp && \ + rm -rf /usr/local/tomcat/webapps/* && \ + chmod 755 /usr/local/tomcat/webapps /var/log/osp && \ + mkdir -p /usr/local/tomcat/webapps/probe/ && \ + touch /usr/local/tomcat/webapps/probe/probe.txt \ + + +RUN apt-get update && apt-get install iputils-ping -y && rm -rf /var/lib/apt/lists/* + +COPY target/gxf /usr/local/tomcat/webapps/ \ No newline at end of file diff --git a/osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms/pom.xml b/osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms/pom.xml index 3fae73e82a0..20e412f6868 100644 --- a/osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms/pom.xml +++ b/osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms/pom.xml @@ -39,6 +39,7 @@ SPDX-License-Identifier: Apache-2.0 maven-war-plugin ${maven.war.plugin.version} + ${project.build.directory}/gxf/${project.artifactId} true false diff --git a/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered/Dockerfile b/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered/Dockerfile new file mode 100644 index 00000000000..1fd27e0ea6b --- /dev/null +++ b/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered/Dockerfile @@ -0,0 +1,15 @@ +FROM tomcat:10.1-jre17-temurin-noble + +RUN rm -rf /usr/local/tomcat/webapps.dist + +# Enable the use of system environment variables in Tomcat configuration files. +RUN echo "org.apache.tomcat.util.digester.PROPERTY_SOURCE=org.apache.tomcat.util.digester.EnvironmentPropertySource" >> /usr/local/tomcat/conf/catalina.properties + +RUN mkdir -p /var/log/osp && \ + rm -rf /usr/local/tomcat/webapps/* && \ + chmod 755 /usr/local/tomcat/webapps /var/log/osp && \ + mkdir -p /usr/local/tomcat/webapps/probe/ && \ + touch /usr/local/tomcat/webapps/probe/probe.txt \ + +COPY context.xml /usr/local/tomcat/conf +COPY target/gxf /usr/local/tomcat/webapps/ \ No newline at end of file diff --git a/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered/context.xml b/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered/context.xml new file mode 100644 index 00000000000..d6b425d0384 --- /dev/null +++ b/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered/context.xml @@ -0,0 +1,39 @@ + + + + + + + WEB-INF/web.xml + + + + + + + + + + + + + + \ No newline at end of file diff --git a/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered/pom.xml b/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered/pom.xml index 466c6a5a2b8..62ae42ce7fe 100644 --- a/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered/pom.xml +++ b/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered/pom.xml @@ -57,6 +57,8 @@ SPDX-License-Identifier: Apache-2.0 maven-war-plugin ${maven.war.plugin.version} + ${project.build.directory}/gxf/${project.artifactId} + true false gif diff --git a/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/simulator/dlms-device-simulator-starter/Dockerfile b/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/simulator/dlms-device-simulator-starter/Dockerfile new file mode 100644 index 00000000000..3bbd96fcd20 --- /dev/null +++ b/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/simulator/dlms-device-simulator-starter/Dockerfile @@ -0,0 +1,7 @@ +FROM amazoncorretto:17-alpine-jdk + +RUN apk add --update coreutils tzdata && rm -rf /var/cache/apk/* + +COPY target/dlms-device-simulator-starter*.jar /app/dlms-device-simulator-starter.jar + +ENTRYPOINT ["java", "-jar", "/app/dlms-device-simulator-starter.jar", "/config/simulators.json", "start"] From 20251913d8f088f9199ba77046c7c2e0fcba1952 Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Thu, 17 Apr 2025 11:43:55 +0200 Subject: [PATCH 02/64] Added smhe containers to build.yml --- .github/workflows/build.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 19642711082..86db4519b80 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -91,6 +91,16 @@ jobs: context: ./osgp/platform/osgp-adapter-ws-smartmetering/ - image: ghcr.io/osgp/osgp-adapter-ws-tariffswitching context: ./osgp/platform/osgp-adapter-ws-tariffswitching/ + - image: ghcr.io/osgp/osgp-secret-management/ + context: ./osgp/platform/osgp-secret-management + - image: ghcr.io/osgp/platform/osgp-throttling-service/ + context: ./osgp/platform/osgp-throttling-service/ + - image: ghcr.io/osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms/ + context: ./osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms + - image: ghcr.io/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered/ + context: ./osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered + - image: ghcr.io/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/simulator/dlms-device-simulator-starter + context: ./osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/simulator/dlms-device-simulator-starter - image: ghcr.io/osgp/osgp-protocol-adapter-iec60870 context: ./osgp/protocol-adapter-iec60870/osgp-protocol-adapter-iec60870 - image: ghcr.io/osgp/osgp-protocol-simulator-iec60870 From 1b4f6e95fa10c89ab97599d7afa78b1aed60ac99 Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Thu, 17 Apr 2025 12:20:02 +0200 Subject: [PATCH 03/64] Added newlines. --- osgp/platform/osgp-secret-management/Dockerfile | 1 + osgp/platform/osgp-throttling-service/Dockerfile | 2 +- .../osgp-protocol-adapter-dlms/Dockerfile | 3 +-- .../osgp-simulator-dlms-triggered/Dockerfile | 3 ++- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/osgp/platform/osgp-secret-management/Dockerfile b/osgp/platform/osgp-secret-management/Dockerfile index 65219d66db6..76bca0c2562 100644 --- a/osgp/platform/osgp-secret-management/Dockerfile +++ b/osgp/platform/osgp-secret-management/Dockerfile @@ -12,6 +12,7 @@ RUN mkdir -p /var/log/osp && \ touch /usr/local/tomcat/webapps/probe/probe.txt \ WORKDIR /opt/app + COPY target/osgp-secret-management-*exec.jar osgp-secret-management-exec.jar ENV SERVER_PORT=8443 diff --git a/osgp/platform/osgp-throttling-service/Dockerfile b/osgp/platform/osgp-throttling-service/Dockerfile index a222ca4466f..7026ed2e358 100644 --- a/osgp/platform/osgp-throttling-service/Dockerfile +++ b/osgp/platform/osgp-throttling-service/Dockerfile @@ -11,4 +11,4 @@ RUN mkdir -p /var/log/osp && \ mkdir -p /usr/local/tomcat/webapps/probe/ && \ touch /usr/local/tomcat/webapps/probe/probe.txt \ -COPY target/gxf /usr/local/tomcat/webapps/ \ No newline at end of file +COPY target/gxf /usr/local/tomcat/webapps/ diff --git a/osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms/Dockerfile b/osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms/Dockerfile index 72fa446bada..bf0ab73ff36 100644 --- a/osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms/Dockerfile +++ b/osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms/Dockerfile @@ -11,7 +11,6 @@ RUN mkdir -p /var/log/osp && \ mkdir -p /usr/local/tomcat/webapps/probe/ && \ touch /usr/local/tomcat/webapps/probe/probe.txt \ - RUN apt-get update && apt-get install iputils-ping -y && rm -rf /var/lib/apt/lists/* -COPY target/gxf /usr/local/tomcat/webapps/ \ No newline at end of file +COPY target/gxf /usr/local/tomcat/webapps/ diff --git a/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered/Dockerfile b/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered/Dockerfile index 1fd27e0ea6b..1d0b64efc7b 100644 --- a/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered/Dockerfile +++ b/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered/Dockerfile @@ -12,4 +12,5 @@ RUN mkdir -p /var/log/osp && \ touch /usr/local/tomcat/webapps/probe/probe.txt \ COPY context.xml /usr/local/tomcat/conf -COPY target/gxf /usr/local/tomcat/webapps/ \ No newline at end of file + +COPY target/gxf /usr/local/tomcat/webapps/ From 465d75e8ffd3362f72156bf109579b7b9e8780d9 Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Thu, 17 Apr 2025 12:52:08 +0200 Subject: [PATCH 04/64] Removed slash from throttling service. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 86db4519b80..1fda1bc37cc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -94,7 +94,7 @@ jobs: - image: ghcr.io/osgp/osgp-secret-management/ context: ./osgp/platform/osgp-secret-management - image: ghcr.io/osgp/platform/osgp-throttling-service/ - context: ./osgp/platform/osgp-throttling-service/ + context: ./osgp/platform/osgp-throttling-service - image: ghcr.io/osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms/ context: ./osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms - image: ghcr.io/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered/ From c19d4ec465440c7e139279189b230adff037a750 Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Tue, 22 Apr 2025 15:06:53 +0200 Subject: [PATCH 05/64] Added checkout to branch for testing. --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1fda1bc37cc..704f525547c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -178,6 +178,7 @@ jobs: uses: actions/checkout@v4 with: repository: 'OSGP/gxf-gitops' + ref: 'feature/FDP-3056-smhe-cucumber-containers' - name: Setup k3d cluster run: bash setup.sh - name: Create gxf platform From d41386d7f2fc2bafc23599568517bdf82e2c0df3 Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Tue, 22 Apr 2025 15:17:00 +0200 Subject: [PATCH 06/64] Added Readme with test instructions. --- .github/workflows/README.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .github/workflows/README.md diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 00000000000..b581ab0f7aa --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,4 @@ +# GXF github workflows +## Run build.yml for testing +To run build.yml for testing add the text cucumber_testing anywhere in your PR title, this will immediately trigger build.yml. +Afterwards you can remove cucumber_testing again. \ No newline at end of file From 6fe12e5bede8fb28fcc03a48271082bda5968690 Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Tue, 22 Apr 2025 15:23:44 +0200 Subject: [PATCH 07/64] Corrected README for triggering build.yml. --- .github/workflows/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index b581ab0f7aa..d3d0887981d 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -1,4 +1,4 @@ # GXF github workflows ## Run build.yml for testing -To run build.yml for testing add the text cucumber_testing anywhere in your PR title, this will immediately trigger build.yml. -Afterwards you can remove cucumber_testing again. \ No newline at end of file +To run build.yml for testing you need to add a label to your PR containing the text cucumber_testing. You can add a label by opening your PR in the github interface and clicking labels in the right side bar according to https://docs.github.com/en/issues/using-labels-and-milestones-to-track-work/managing-labels. +Afterwards you can remove the label again. \ No newline at end of file From 4a6e971c9a7d4e138af5957a197984bc46e6e485 Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Tue, 22 Apr 2025 15:37:39 +0200 Subject: [PATCH 08/64] Small change in README.md. --- .github/workflows/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index d3d0887981d..d805f82a924 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -1,4 +1,7 @@ # GXF github workflows -## Run build.yml for testing +## build.yml +Responsible for building docker containers as artifacts and running cucumber tests. + +### Run build.yml for testing To run build.yml for testing you need to add a label to your PR containing the text cucumber_testing. You can add a label by opening your PR in the github interface and clicking labels in the right side bar according to https://docs.github.com/en/issues/using-labels-and-milestones-to-track-work/managing-labels. Afterwards you can remove the label again. \ No newline at end of file From 8dfe4534740bee8a493baa2900347344aa73aed7 Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Tue, 22 Apr 2025 15:54:25 +0200 Subject: [PATCH 09/64] Disabled pr tag. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 704f525547c..7b35b37373b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -150,7 +150,7 @@ jobs: type=semver,pattern={{version}},value=${{ steps.maven.outputs.version }} type=ref,event=branch type=ref,event=tag - type=ref,event=pr + type=ref,event=pr,enable=false type=raw,value=latest,enable={{is_default_branch}} - name: Build and push Docker image id: push From fd1f84f6a615bc825cc75b1563e3ac23e871a6e3 Mon Sep 17 00:00:00 2001 From: paulus Date: Wed, 23 Apr 2025 13:14:56 +0200 Subject: [PATCH 10/64] Re-enabled pr event. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7b35b37373b..704f525547c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -150,7 +150,7 @@ jobs: type=semver,pattern={{version}},value=${{ steps.maven.outputs.version }} type=ref,event=branch type=ref,event=tag - type=ref,event=pr,enable=false + type=ref,event=pr type=raw,value=latest,enable={{is_default_branch}} - name: Build and push Docker image id: push From b44c4330ed51dc635dd8ccc9e4d7a75bbe28b3b8 Mon Sep 17 00:00:00 2001 From: paulus Date: Wed, 23 Apr 2025 14:14:54 +0200 Subject: [PATCH 11/64] Removed slashes for images. --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 704f525547c..28fd29b8ebd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -91,13 +91,13 @@ jobs: context: ./osgp/platform/osgp-adapter-ws-smartmetering/ - image: ghcr.io/osgp/osgp-adapter-ws-tariffswitching context: ./osgp/platform/osgp-adapter-ws-tariffswitching/ - - image: ghcr.io/osgp/osgp-secret-management/ + - image: ghcr.io/osgp/osgp-secret-management context: ./osgp/platform/osgp-secret-management - - image: ghcr.io/osgp/platform/osgp-throttling-service/ + - image: ghcr.io/osgp/platform/osgp-throttling-service context: ./osgp/platform/osgp-throttling-service - - image: ghcr.io/osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms/ + - image: ghcr.io/osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms context: ./osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms - - image: ghcr.io/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered/ + - image: ghcr.io/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered context: ./osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered - image: ghcr.io/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/simulator/dlms-device-simulator-starter context: ./osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/simulator/dlms-device-simulator-starter From 26e7df2cb7bc21d129917d9500a9f17547b9cf5d Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Thu, 24 Apr 2025 14:54:54 +0200 Subject: [PATCH 12/64] Removed final backslash to prevent docker warning --- osgp/platform/osgp-secret-management/Dockerfile | 2 +- osgp/platform/osgp-throttling-service/Dockerfile | 2 +- .../protocol-adapter-dlms/osgp-protocol-adapter-dlms/Dockerfile | 2 +- .../osgp-simulator-dlms-triggered/Dockerfile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/osgp/platform/osgp-secret-management/Dockerfile b/osgp/platform/osgp-secret-management/Dockerfile index 76bca0c2562..90c9f41234c 100644 --- a/osgp/platform/osgp-secret-management/Dockerfile +++ b/osgp/platform/osgp-secret-management/Dockerfile @@ -9,7 +9,7 @@ RUN mkdir -p /var/log/osp && \ rm -rf /usr/local/tomcat/webapps/* && \ chmod 755 /usr/local/tomcat/webapps /var/log/osp && \ mkdir -p /usr/local/tomcat/webapps/probe/ && \ - touch /usr/local/tomcat/webapps/probe/probe.txt \ + touch /usr/local/tomcat/webapps/probe/probe.txt WORKDIR /opt/app diff --git a/osgp/platform/osgp-throttling-service/Dockerfile b/osgp/platform/osgp-throttling-service/Dockerfile index 7026ed2e358..43126cd629a 100644 --- a/osgp/platform/osgp-throttling-service/Dockerfile +++ b/osgp/platform/osgp-throttling-service/Dockerfile @@ -9,6 +9,6 @@ RUN mkdir -p /var/log/osp && \ rm -rf /usr/local/tomcat/webapps/* && \ chmod 755 /usr/local/tomcat/webapps /var/log/osp && \ mkdir -p /usr/local/tomcat/webapps/probe/ && \ - touch /usr/local/tomcat/webapps/probe/probe.txt \ + touch /usr/local/tomcat/webapps/probe/probe.txt COPY target/gxf /usr/local/tomcat/webapps/ diff --git a/osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms/Dockerfile b/osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms/Dockerfile index bf0ab73ff36..df76f68320f 100644 --- a/osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms/Dockerfile +++ b/osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms/Dockerfile @@ -9,7 +9,7 @@ RUN mkdir -p /var/log/osp && \ rm -rf /usr/local/tomcat/webapps/* && \ chmod 755 /usr/local/tomcat/webapps /var/log/osp && \ mkdir -p /usr/local/tomcat/webapps/probe/ && \ - touch /usr/local/tomcat/webapps/probe/probe.txt \ + touch /usr/local/tomcat/webapps/probe/probe.txt RUN apt-get update && apt-get install iputils-ping -y && rm -rf /var/lib/apt/lists/* diff --git a/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered/Dockerfile b/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered/Dockerfile index 1d0b64efc7b..97adbab802b 100644 --- a/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered/Dockerfile +++ b/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered/Dockerfile @@ -9,7 +9,7 @@ RUN mkdir -p /var/log/osp && \ rm -rf /usr/local/tomcat/webapps/* && \ chmod 755 /usr/local/tomcat/webapps /var/log/osp && \ mkdir -p /usr/local/tomcat/webapps/probe/ && \ - touch /usr/local/tomcat/webapps/probe/probe.txt \ + touch /usr/local/tomcat/webapps/probe/probe.txt COPY context.xml /usr/local/tomcat/conf From 7364ff6409fffe81640fe37962a32e8969ee3645 Mon Sep 17 00:00:00 2001 From: paulus Date: Wed, 30 Apr 2025 11:24:25 +0200 Subject: [PATCH 13/64] Added secret management to find command. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 28fd29b8ebd..0bed72684ad 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -44,7 +44,7 @@ jobs: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} run: ./mvnw -T1C -B verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=gxf -Dsonar.projectKey=OSGP_open-smart-grid-platform ${{ steps.maven_options.outputs.maven_options }} - name: Staging war and jar files - run: mkdir staging && find . -path ./staging -prune -o \( -name gxf -o -name cucumber-*-test-jar-with-dependencies.jar -o -name *simulator*.war \) -exec cp -prv --parents '{}' staging/ \; + run: mkdir staging && find . -path ./staging -prune -o \( -name gxf -o -name cucumber-*-test-jar-with-dependencies.jar -o -name *simulator*.war -o -name *osgp-secret* \) -exec cp -prv --parents '{}' staging/ \; if: contains(github.ref, 'release') || contains(github.ref, 'development') || contains(github.event.pull_request.labels.*.name, 'build_containers') || contains(github.event.pull_request.labels.*.name, 'cucumber_testing') - uses: actions/upload-artifact@v4 if: contains(github.ref, 'release') || contains(github.ref, 'development') || contains(github.event.pull_request.labels.*.name, 'build_containers') || contains(github.event.pull_request.labels.*.name, 'cucumber_testing') From 763bf0c0493341f4fcd993378be8617fd8c53622 Mon Sep 17 00:00:00 2001 From: paulus Date: Wed, 30 Apr 2025 11:56:17 +0200 Subject: [PATCH 14/64] Added device simulator start to find command. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0bed72684ad..921932f077b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -44,7 +44,7 @@ jobs: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} run: ./mvnw -T1C -B verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=gxf -Dsonar.projectKey=OSGP_open-smart-grid-platform ${{ steps.maven_options.outputs.maven_options }} - name: Staging war and jar files - run: mkdir staging && find . -path ./staging -prune -o \( -name gxf -o -name cucumber-*-test-jar-with-dependencies.jar -o -name *simulator*.war -o -name *osgp-secret* \) -exec cp -prv --parents '{}' staging/ \; + run: mkdir staging && find . -path ./staging -prune -o \( -name gxf -o -name cucumber-*-test-jar-with-dependencies.jar -o -name *simulator*.war -o -name *osgp-secret* -o -name *dlms-device-simulator-starter*.jar \) -exec cp -prv --parents '{}' staging/ \; if: contains(github.ref, 'release') || contains(github.ref, 'development') || contains(github.event.pull_request.labels.*.name, 'build_containers') || contains(github.event.pull_request.labels.*.name, 'cucumber_testing') - uses: actions/upload-artifact@v4 if: contains(github.ref, 'release') || contains(github.ref, 'development') || contains(github.event.pull_request.labels.*.name, 'build_containers') || contains(github.event.pull_request.labels.*.name, 'cucumber_testing') From 5849ffa03e757735a6db17c4a88e8dad0d741c65 Mon Sep 17 00:00:00 2001 From: paulus Date: Wed, 30 Apr 2025 17:05:37 +0200 Subject: [PATCH 15/64] Added ci/smartmetering-values.yaml --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0852c592a62..c4183fa719e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -175,7 +175,7 @@ jobs: checks: write strategy: matrix: - values_file: [ci/common-values.yaml, ci/publiclighting-values.yaml] + values_file: [ci/common-values.yaml, ci/publiclighting-values.yaml, ci/smartmetering-values.yaml] steps: - name: Checkout gxf-gitops uses: actions/checkout@v4 From 355386f2d7441cf7a0dd6894c5536b9df9392b99 Mon Sep 17 00:00:00 2001 From: paulus Date: Wed, 30 Apr 2025 22:03:54 +0200 Subject: [PATCH 16/64] Tryout failed kubectl. --- .github/workflows/build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c4183fa719e..8eb64685dba 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -189,7 +189,10 @@ jobs: - name: Create and run gxf cucumber tests run: ./charts/gxf-cucumber-tests/template-apply.sh --imageTag $(echo -e "${{ needs.docker.outputs.tags }}" | head -n 1 | cut -d':' -f2) --valuesFile ${{ matrix.values_file }} - name: Wait for jobs to finish - run: kubectl wait --for=condition=complete --all jobs --timeout=30m + run: > + if ! kubectl wait --for=condition=complete --all jobs --timeout=1s; then + kubectl get pods + fi - name: Run maven cucumber reporting run: mvn -f charts/gxf-cucumber-tests/pom.xml verify - name: Annotate cucumber report to github action From 439eb86e4c54bee54b9ef04a26f3927d10c99b60 Mon Sep 17 00:00:00 2001 From: paulus Date: Wed, 30 Apr 2025 22:20:45 +0200 Subject: [PATCH 17/64] Added exit 1 on failure. --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8eb64685dba..3440a2b5b73 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -192,6 +192,8 @@ jobs: run: > if ! kubectl wait --for=condition=complete --all jobs --timeout=1s; then kubectl get pods + kubectl get ns + exit 1 fi - name: Run maven cucumber reporting run: mvn -f charts/gxf-cucumber-tests/pom.xml verify From 19a45205ae352c56590421860180c5270822d608 Mon Sep 17 00:00:00 2001 From: paulus Date: Wed, 30 Apr 2025 22:31:05 +0200 Subject: [PATCH 18/64] Added debug logs on timeout. --- .github/workflows/build.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3440a2b5b73..aae0bb81233 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -190,9 +190,13 @@ jobs: run: ./charts/gxf-cucumber-tests/template-apply.sh --imageTag $(echo -e "${{ needs.docker.outputs.tags }}" | head -n 1 | cut -d':' -f2) --valuesFile ${{ matrix.values_file }} - name: Wait for jobs to finish run: > - if ! kubectl wait --for=condition=complete --all jobs --timeout=1s; then - kubectl get pods - kubectl get ns + if ! kubectl wait --for=condition=complete --all jobs --timeout=30m; then + for ns in $(kubectl get ns --no-headers); do + for pod in $(kubectl -n $ns get pod -o name); do + logs=$(kubectl -n $ns logs $pod --all-containers=true) + done; + done + echo "::error::Kubectl timed out with logs:\n $logs" exit 1 fi - name: Run maven cucumber reporting From 246596304271ea5fffc2bc5072983df5ca6229d5 Mon Sep 17 00:00:00 2001 From: paulus Date: Wed, 30 Apr 2025 22:42:31 +0200 Subject: [PATCH 19/64] Namespace is not needed. --- .github/workflows/build.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aae0bb81233..68c766c52c0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -191,11 +191,9 @@ jobs: - name: Wait for jobs to finish run: > if ! kubectl wait --for=condition=complete --all jobs --timeout=30m; then - for ns in $(kubectl get ns --no-headers); do - for pod in $(kubectl -n $ns get pod -o name); do - logs=$(kubectl -n $ns logs $pod --all-containers=true) - done; - done + for pod in $(kubectl -n $ns get pod -o name); do + logs=$(kubectl -n $ns logs $pod --all-containers=true) + done; echo "::error::Kubectl timed out with logs:\n $logs" exit 1 fi From ee53f1c9f8f9ca7ba8c4f587ce49b2c62bd663a4 Mon Sep 17 00:00:00 2001 From: paulus Date: Wed, 30 Apr 2025 22:45:51 +0200 Subject: [PATCH 20/64] Added semicolons. --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 68c766c52c0..ee9f769a975 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -192,10 +192,10 @@ jobs: run: > if ! kubectl wait --for=condition=complete --all jobs --timeout=30m; then for pod in $(kubectl -n $ns get pod -o name); do - logs=$(kubectl -n $ns logs $pod --all-containers=true) + logs=$(kubectl -n $ns logs $pod --all-containers=true); done; - echo "::error::Kubectl timed out with logs:\n $logs" - exit 1 + echo "::error::Kubectl timed out with logs:\n $logs"; + exit 1; fi - name: Run maven cucumber reporting run: mvn -f charts/gxf-cucumber-tests/pom.xml verify From 0f72e809493f5b76fd4ce4ca19995d2017428621 Mon Sep 17 00:00:00 2001 From: paulus Date: Wed, 30 Apr 2025 23:32:24 +0200 Subject: [PATCH 21/64] Removed ns variables. --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ee9f769a975..8f9d3d8f92b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -191,8 +191,8 @@ jobs: - name: Wait for jobs to finish run: > if ! kubectl wait --for=condition=complete --all jobs --timeout=30m; then - for pod in $(kubectl -n $ns get pod -o name); do - logs=$(kubectl -n $ns logs $pod --all-containers=true); + for pod in $(kubectl get pod -o name); do + logs=$(kubectl logs $pod --all-containers=true); done; echo "::error::Kubectl timed out with logs:\n $logs"; exit 1; From c19b18ffce3ddb10e90c2824a680213288077004 Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Thu, 1 May 2025 10:13:52 +0200 Subject: [PATCH 22/64] Show only running containers logs on error. --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8f9d3d8f92b..ed41ba0f3ed 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -190,8 +190,8 @@ jobs: run: ./charts/gxf-cucumber-tests/template-apply.sh --imageTag $(echo -e "${{ needs.docker.outputs.tags }}" | head -n 1 | cut -d':' -f2) --valuesFile ${{ matrix.values_file }} - name: Wait for jobs to finish run: > - if ! kubectl wait --for=condition=complete --all jobs --timeout=30m; then - for pod in $(kubectl get pod -o name); do + if ! kubectl wait --for=condition=complete --all jobs --timeout=5m; then + for pod in $(kubectl get pod --field-selector=status.phase==Running -o name); do logs=$(kubectl logs $pod --all-containers=true); done; echo "::error::Kubectl timed out with logs:\n $logs"; From 48f32ec25ea310b736dc55424ef67c0c4eeafd58 Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Thu, 1 May 2025 10:39:17 +0200 Subject: [PATCH 23/64] Back to 30min --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ed41ba0f3ed..c7202d689c2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -190,7 +190,7 @@ jobs: run: ./charts/gxf-cucumber-tests/template-apply.sh --imageTag $(echo -e "${{ needs.docker.outputs.tags }}" | head -n 1 | cut -d':' -f2) --valuesFile ${{ matrix.values_file }} - name: Wait for jobs to finish run: > - if ! kubectl wait --for=condition=complete --all jobs --timeout=5m; then + if ! kubectl wait --for=condition=complete --all jobs --timeout=30m; then for pod in $(kubectl get pod --field-selector=status.phase==Running -o name); do logs=$(kubectl logs $pod --all-containers=true); done; From 63ec08f2979238ae609044c0116d0a49065b997b Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Thu, 1 May 2025 11:31:12 +0200 Subject: [PATCH 24/64] Added list events. --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c7202d689c2..70e74b1c7fe 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -195,6 +195,7 @@ jobs: logs=$(kubectl logs $pod --all-containers=true); done; echo "::error::Kubectl timed out with logs:\n $logs"; + kubectl get events --all-namespaces --sort-by='.metadata.creationTimestamp' exit 1; fi - name: Run maven cucumber reporting From 9ae68566687e4a56cb95bf5a475d67b2c3ff10c0 Mon Sep 17 00:00:00 2001 From: paulus Date: Mon, 5 May 2025 18:19:57 +0200 Subject: [PATCH 25/64] Add authentication to ghcr. --- .github/workflows/build.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8f9d3d8f92b..85ef81240d8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -182,6 +182,12 @@ jobs: with: repository: 'OSGP/gxf-gitops' ref: 'feature/FDP-3056-smhe-cucumber-containers' + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Setup k3d cluster run: bash setup.sh - name: Create gxf platform From d6e35a000dd9aa998f9ec1c405ea33046b4aee66 Mon Sep 17 00:00:00 2001 From: paulus Date: Mon, 5 May 2025 22:17:37 +0200 Subject: [PATCH 26/64] Removed incorrect paths for images. --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1fd0d339f18..80344861fc8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -93,13 +93,13 @@ jobs: context: ./osgp/platform/osgp-adapter-ws-tariffswitching/ - image: ghcr.io/osgp/osgp-secret-management context: ./osgp/platform/osgp-secret-management - - image: ghcr.io/osgp/platform/osgp-throttling-service + - image: ghcr.io/osgp/osgp-throttling-service context: ./osgp/platform/osgp-throttling-service - - image: ghcr.io/osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms + - image: ghcr.io/osgp/osgp-protocol-adapter-dlms context: ./osgp/protocol-adapter-dlms/osgp-protocol-adapter-dlms - - image: ghcr.io/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered + - image: ghcr.io/osgp/osgp-simulator-dlms-triggered context: ./osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/osgp-simulator-dlms-triggered - - image: ghcr.io/osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/simulator/dlms-device-simulator-starter + - image: ghcr.io/osgp/dlms-device-simulator-starter context: ./osgp/protocol-adapter-dlms/osgp-protocol-simulator-dlms/simulator/dlms-device-simulator-starter - image: ghcr.io/osgp/osgp-protocol-adapter-iec60870 context: ./osgp/protocol-adapter-iec60870/osgp-protocol-adapter-iec60870 From 7430542d7a29530fcde3cb8ad0bb4e41f7785720 Mon Sep 17 00:00:00 2001 From: paulus Date: Mon, 5 May 2025 23:09:41 +0200 Subject: [PATCH 27/64] Better logging. --- .github/workflows/build.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 80344861fc8..5c3f3591fa3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -197,11 +197,12 @@ jobs: - name: Wait for jobs to finish run: > if ! kubectl wait --for=condition=complete --all jobs --timeout=30m; then + echo "::error::Kubectl timed out..."; + kubectl get events --all-namespaces --sort-by='.metadata.creationTimestamp' + kubectl get pods for pod in $(kubectl get pod --field-selector=status.phase==Running -o name); do - logs=$(kubectl logs $pod --all-containers=true); + kubectl logs $pod --all-containers=true done; - echo "::error::Kubectl timed out with logs:\n $logs"; - kubectl get events --all-namespaces --sort-by='.metadata.creationTimestamp' exit 1; fi - name: Run maven cucumber reporting From 70b170a909adfcc47cb1022ee499c057adcb8ffa Mon Sep 17 00:00:00 2001 From: paulus Date: Tue, 6 May 2025 00:06:16 +0200 Subject: [PATCH 28/64] Better logging 2. --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5c3f3591fa3..a8614744ad1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -199,10 +199,11 @@ jobs: if ! kubectl wait --for=condition=complete --all jobs --timeout=30m; then echo "::error::Kubectl timed out..."; kubectl get events --all-namespaces --sort-by='.metadata.creationTimestamp' - kubectl get pods for pod in $(kubectl get pod --field-selector=status.phase==Running -o name); do + echo "Logs for $pod:" kubectl logs $pod --all-containers=true done; + kubectl get pods exit 1; fi - name: Run maven cucumber reporting From 50ce8be76d8365741593a661e2ca78505720c324 Mon Sep 17 00:00:00 2001 From: paulus Date: Tue, 6 May 2025 01:02:22 +0200 Subject: [PATCH 29/64] Better logging 3. --- .github/workflows/README.md | 5 ++++- .github/workflows/build.yml | 7 +++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index d805f82a924..6adb53bf6b6 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -4,4 +4,7 @@ Responsible for building docker containers as artifacts and running cucumber tes ### Run build.yml for testing To run build.yml for testing you need to add a label to your PR containing the text cucumber_testing. You can add a label by opening your PR in the github interface and clicking labels in the right side bar according to https://docs.github.com/en/issues/using-labels-and-milestones-to-track-work/managing-labels. -Afterwards you can remove the label again. \ No newline at end of file +Afterwards you can remove the label again. + +### Error logs on cucumber test timeout +If cucumber tests time out error logs are generated and uploaded as artifacts. \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a8614744ad1..0aecc42f420 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -206,6 +206,13 @@ jobs: kubectl get pods exit 1; fi + - name: Upload error logs + uses: actions/upload-artifact@v4 + with: + name: kube_error_logs + path: *.log + if_no_files_found: ignore + overwrite: true - name: Run maven cucumber reporting run: mvn -f charts/gxf-cucumber-tests/pom.xml verify - name: Annotate cucumber report to github action From accd17881805c9e59f21bbf61aa179a8dccfb145 Mon Sep 17 00:00:00 2001 From: paulus Date: Tue, 6 May 2025 01:04:11 +0200 Subject: [PATCH 30/64] Better logging 4. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0aecc42f420..e590cea3c50 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -211,7 +211,7 @@ jobs: with: name: kube_error_logs path: *.log - if_no_files_found: ignore + if-no-files-found: ignore overwrite: true - name: Run maven cucumber reporting run: mvn -f charts/gxf-cucumber-tests/pom.xml verify From 7db3595be36ac5ac7f108d54ae5f77d630af503d Mon Sep 17 00:00:00 2001 From: paulus Date: Tue, 6 May 2025 01:05:29 +0200 Subject: [PATCH 31/64] Better logging 5. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e590cea3c50..797e53aa36b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -210,7 +210,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: kube_error_logs - path: *.log + path: ./*.log if-no-files-found: ignore overwrite: true - name: Run maven cucumber reporting From 485ad5b76f0c9376cd6423e0407f9b289a67ce99 Mon Sep 17 00:00:00 2001 From: paulus Date: Tue, 6 May 2025 02:02:45 +0200 Subject: [PATCH 32/64] Removed upload artifact. --- .github/workflows/build.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 797e53aa36b..a8614744ad1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -206,13 +206,6 @@ jobs: kubectl get pods exit 1; fi - - name: Upload error logs - uses: actions/upload-artifact@v4 - with: - name: kube_error_logs - path: ./*.log - if-no-files-found: ignore - overwrite: true - name: Run maven cucumber reporting run: mvn -f charts/gxf-cucumber-tests/pom.xml verify - name: Annotate cucumber report to github action From bb722a7b6dbfcc04ae75340614c4fd62bcf96d04 Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Tue, 6 May 2025 11:11:43 +0200 Subject: [PATCH 33/64] Changed localhost to postgres. --- .../platform/smartmetering/PlatformSmartmeteringDefaults.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/cucumber-tests-platform-smartmetering/src/test/java/org/opensmartgridplatform/cucumber/platform/smartmetering/PlatformSmartmeteringDefaults.java b/integration-tests/cucumber-tests-platform-smartmetering/src/test/java/org/opensmartgridplatform/cucumber/platform/smartmetering/PlatformSmartmeteringDefaults.java index 5054c8df078..751a16afd33 100644 --- a/integration-tests/cucumber-tests-platform-smartmetering/src/test/java/org/opensmartgridplatform/cucumber/platform/smartmetering/PlatformSmartmeteringDefaults.java +++ b/integration-tests/cucumber-tests-platform-smartmetering/src/test/java/org/opensmartgridplatform/cucumber/platform/smartmetering/PlatformSmartmeteringDefaults.java @@ -108,7 +108,7 @@ public class PlatformSmartmeteringDefaults static { InetAddress localhost; try { - localhost = InetAddress.getByName(PlatformDefaults.LOCALHOST); + localhost = InetAddress.getByName("postgres"); } catch (final UnknownHostException e) { localhost = null; } From 5c231c13776f48baf705190e1912ac2ae1b7f8d6 Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Tue, 6 May 2025 11:34:51 +0200 Subject: [PATCH 34/64] Added constant for postgres host string. --- .../platform/smartmetering/PlatformSmartmeteringDefaults.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration-tests/cucumber-tests-platform-smartmetering/src/test/java/org/opensmartgridplatform/cucumber/platform/smartmetering/PlatformSmartmeteringDefaults.java b/integration-tests/cucumber-tests-platform-smartmetering/src/test/java/org/opensmartgridplatform/cucumber/platform/smartmetering/PlatformSmartmeteringDefaults.java index 751a16afd33..d203baa7523 100644 --- a/integration-tests/cucumber-tests-platform-smartmetering/src/test/java/org/opensmartgridplatform/cucumber/platform/smartmetering/PlatformSmartmeteringDefaults.java +++ b/integration-tests/cucumber-tests-platform-smartmetering/src/test/java/org/opensmartgridplatform/cucumber/platform/smartmetering/PlatformSmartmeteringDefaults.java @@ -42,6 +42,7 @@ public class PlatformSmartmeteringDefaults public static final boolean HLS3ACTIVE = false; public static final boolean HLS4ACTIVE = false; public static final boolean HLS5ACTIVE = true; + public static final String HOST = "postgres"; public static final String ICC_ID = "iccid"; public static final boolean LLS1_ACTIVE = false; public static final boolean USE_HDLC = false; @@ -108,7 +109,7 @@ public class PlatformSmartmeteringDefaults static { InetAddress localhost; try { - localhost = InetAddress.getByName("postgres"); + localhost = InetAddress.getByName(PlatformSmartmeteringDefaults.HOST); } catch (final UnknownHostException e) { localhost = null; } From c6fce72217cde4029c3dde7b702383eecb49793a Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Tue, 6 May 2025 11:47:31 +0200 Subject: [PATCH 35/64] Fixed formatting for cucumber integration test. --- .../platform/smartmetering/PlatformSmartmeteringDefaults.java | 1 - 1 file changed, 1 deletion(-) diff --git a/integration-tests/cucumber-tests-platform-smartmetering/src/test/java/org/opensmartgridplatform/cucumber/platform/smartmetering/PlatformSmartmeteringDefaults.java b/integration-tests/cucumber-tests-platform-smartmetering/src/test/java/org/opensmartgridplatform/cucumber/platform/smartmetering/PlatformSmartmeteringDefaults.java index d203baa7523..115de9766cc 100644 --- a/integration-tests/cucumber-tests-platform-smartmetering/src/test/java/org/opensmartgridplatform/cucumber/platform/smartmetering/PlatformSmartmeteringDefaults.java +++ b/integration-tests/cucumber-tests-platform-smartmetering/src/test/java/org/opensmartgridplatform/cucumber/platform/smartmetering/PlatformSmartmeteringDefaults.java @@ -10,7 +10,6 @@ import java.util.Date; import java.util.HashMap; import java.util.Map; -import org.opensmartgridplatform.cucumber.platform.PlatformDefaults; import org.opensmartgridplatform.domain.core.entities.DeviceModel; import org.opensmartgridplatform.domain.core.entities.ProtocolInfo; From 0371480238d19e10fa925e1d5fce9993dc5f8daa Mon Sep 17 00:00:00 2001 From: paulus Date: Tue, 6 May 2025 12:43:38 +0200 Subject: [PATCH 36/64] Revert "Fixed formatting for cucumber integration test." This reverts commit c6fce72217cde4029c3dde7b702383eecb49793a. --- .../platform/smartmetering/PlatformSmartmeteringDefaults.java | 1 + 1 file changed, 1 insertion(+) diff --git a/integration-tests/cucumber-tests-platform-smartmetering/src/test/java/org/opensmartgridplatform/cucumber/platform/smartmetering/PlatformSmartmeteringDefaults.java b/integration-tests/cucumber-tests-platform-smartmetering/src/test/java/org/opensmartgridplatform/cucumber/platform/smartmetering/PlatformSmartmeteringDefaults.java index 115de9766cc..d203baa7523 100644 --- a/integration-tests/cucumber-tests-platform-smartmetering/src/test/java/org/opensmartgridplatform/cucumber/platform/smartmetering/PlatformSmartmeteringDefaults.java +++ b/integration-tests/cucumber-tests-platform-smartmetering/src/test/java/org/opensmartgridplatform/cucumber/platform/smartmetering/PlatformSmartmeteringDefaults.java @@ -10,6 +10,7 @@ import java.util.Date; import java.util.HashMap; import java.util.Map; +import org.opensmartgridplatform.cucumber.platform.PlatformDefaults; import org.opensmartgridplatform.domain.core.entities.DeviceModel; import org.opensmartgridplatform.domain.core.entities.ProtocolInfo; From c882a21ebd640191b60c59fa742b40c808005118 Mon Sep 17 00:00:00 2001 From: paulus Date: Tue, 6 May 2025 12:50:44 +0200 Subject: [PATCH 37/64] Add db options for postgres kubernetes. --- .../cucumber-tests-platform-smartmetering/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/integration-tests/cucumber-tests-platform-smartmetering/Dockerfile b/integration-tests/cucumber-tests-platform-smartmetering/Dockerfile index 46f4c646399..d9a1c2cf37b 100644 --- a/integration-tests/cucumber-tests-platform-smartmetering/Dockerfile +++ b/integration-tests/cucumber-tests-platform-smartmetering/Dockerfile @@ -4,6 +4,7 @@ COPY target/cucumber-*-test-jar-with-dependencies.jar / COPY src/test/resources/logback.xml /logback.xml RUN wget https://repo1.maven.org/maven2/org/jacoco/org.jacoco.agent/0.8.12/org.jacoco.agent-0.8.12-runtime.jar CMD java -javaagent:org.jacoco.agent-0.8.12-runtime.jar=destfile=/target/code-coverage/jacoco-it.exec \ + -Ddb.username=osp_admin -Ddb.password=1234 -Ddb.hostname=postgres \ -Dcucumber.execution.strict=true\ -Dcucumber.filter.tags='not @Skip' \ -DskipITs=false\ From e7dbb0592cd5164ffa1f31afddb3b51ef9f718bb Mon Sep 17 00:00:00 2001 From: paulus Date: Tue, 6 May 2025 13:06:16 +0200 Subject: [PATCH 38/64] Revert "Added constant for postgres host string." This reverts commit 5c231c13776f48baf705190e1912ac2ae1b7f8d6. --- .../platform/smartmetering/PlatformSmartmeteringDefaults.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/integration-tests/cucumber-tests-platform-smartmetering/src/test/java/org/opensmartgridplatform/cucumber/platform/smartmetering/PlatformSmartmeteringDefaults.java b/integration-tests/cucumber-tests-platform-smartmetering/src/test/java/org/opensmartgridplatform/cucumber/platform/smartmetering/PlatformSmartmeteringDefaults.java index d203baa7523..751a16afd33 100644 --- a/integration-tests/cucumber-tests-platform-smartmetering/src/test/java/org/opensmartgridplatform/cucumber/platform/smartmetering/PlatformSmartmeteringDefaults.java +++ b/integration-tests/cucumber-tests-platform-smartmetering/src/test/java/org/opensmartgridplatform/cucumber/platform/smartmetering/PlatformSmartmeteringDefaults.java @@ -42,7 +42,6 @@ public class PlatformSmartmeteringDefaults public static final boolean HLS3ACTIVE = false; public static final boolean HLS4ACTIVE = false; public static final boolean HLS5ACTIVE = true; - public static final String HOST = "postgres"; public static final String ICC_ID = "iccid"; public static final boolean LLS1_ACTIVE = false; public static final boolean USE_HDLC = false; @@ -109,7 +108,7 @@ public class PlatformSmartmeteringDefaults static { InetAddress localhost; try { - localhost = InetAddress.getByName(PlatformSmartmeteringDefaults.HOST); + localhost = InetAddress.getByName("postgres"); } catch (final UnknownHostException e) { localhost = null; } From 7e1327b45e7587a2bae98ef6519183b8df419590 Mon Sep 17 00:00:00 2001 From: paulus Date: Tue, 6 May 2025 13:06:50 +0200 Subject: [PATCH 39/64] Revert "Changed localhost to postgres." This reverts commit bb722a7b6dbfcc04ae75340614c4fd62bcf96d04. --- .../platform/smartmetering/PlatformSmartmeteringDefaults.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/cucumber-tests-platform-smartmetering/src/test/java/org/opensmartgridplatform/cucumber/platform/smartmetering/PlatformSmartmeteringDefaults.java b/integration-tests/cucumber-tests-platform-smartmetering/src/test/java/org/opensmartgridplatform/cucumber/platform/smartmetering/PlatformSmartmeteringDefaults.java index 751a16afd33..5054c8df078 100644 --- a/integration-tests/cucumber-tests-platform-smartmetering/src/test/java/org/opensmartgridplatform/cucumber/platform/smartmetering/PlatformSmartmeteringDefaults.java +++ b/integration-tests/cucumber-tests-platform-smartmetering/src/test/java/org/opensmartgridplatform/cucumber/platform/smartmetering/PlatformSmartmeteringDefaults.java @@ -108,7 +108,7 @@ public class PlatformSmartmeteringDefaults static { InetAddress localhost; try { - localhost = InetAddress.getByName("postgres"); + localhost = InetAddress.getByName(PlatformDefaults.LOCALHOST); } catch (final UnknownHostException e) { localhost = null; } From a1abe1ea51e0e74195d912b35f51f0c647c06c96 Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Tue, 6 May 2025 15:20:31 +0200 Subject: [PATCH 40/64] Revert "Add db options for postgres kubernetes." This reverts commit c882a21ebd640191b60c59fa742b40c808005118. --- .../cucumber-tests-platform-smartmetering/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/integration-tests/cucumber-tests-platform-smartmetering/Dockerfile b/integration-tests/cucumber-tests-platform-smartmetering/Dockerfile index d9a1c2cf37b..46f4c646399 100644 --- a/integration-tests/cucumber-tests-platform-smartmetering/Dockerfile +++ b/integration-tests/cucumber-tests-platform-smartmetering/Dockerfile @@ -4,7 +4,6 @@ COPY target/cucumber-*-test-jar-with-dependencies.jar / COPY src/test/resources/logback.xml /logback.xml RUN wget https://repo1.maven.org/maven2/org/jacoco/org.jacoco.agent/0.8.12/org.jacoco.agent-0.8.12-runtime.jar CMD java -javaagent:org.jacoco.agent-0.8.12-runtime.jar=destfile=/target/code-coverage/jacoco-it.exec \ - -Ddb.username=osp_admin -Ddb.password=1234 -Ddb.hostname=postgres \ -Dcucumber.execution.strict=true\ -Dcucumber.filter.tags='not @Skip' \ -DskipITs=false\ From 13ad4ab09ce2532f6a26aed3df13e60324f9e324 Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Tue, 6 May 2025 21:27:03 +0200 Subject: [PATCH 41/64] Update osgp-throttling-service.properties Changed host to postgres Signed-off-by: Paulus Nunnink --- .../src/main/resources/osgp-throttling-service.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osgp/platform/osgp-throttling-service/src/main/resources/osgp-throttling-service.properties b/osgp/platform/osgp-throttling-service/src/main/resources/osgp-throttling-service.properties index 7db3feb61d4..ae1da1c88f0 100644 --- a/osgp/platform/osgp-throttling-service/src/main/resources/osgp-throttling-service.properties +++ b/osgp/platform/osgp-throttling-service/src/main/resources/osgp-throttling-service.properties @@ -1,6 +1,6 @@ # Datasource Configuration spring.datasource.driver-class-name=org.postgresql.Driver -spring.datasource.url=jdbc:postgresql://127.0.0.1:5432/osgp_throttling +spring.datasource.url=jdbc:postgresql://postgres:5432/osgp_throttling spring.datasource.username=osp_admin spring.datasource.password=1234 # JPA / Hibernate Configuration From c6a48f69a79446269f6a4ffa52bdcfba18bf1c37 Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Wed, 7 May 2025 11:07:16 +0200 Subject: [PATCH 42/64] Added redis host. --- .../src/main/resources/osgp-throttling-service.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osgp/platform/osgp-throttling-service/src/main/resources/osgp-throttling-service.properties b/osgp/platform/osgp-throttling-service/src/main/resources/osgp-throttling-service.properties index 7db3feb61d4..68a3a058fad 100644 --- a/osgp/platform/osgp-throttling-service/src/main/resources/osgp-throttling-service.properties +++ b/osgp/platform/osgp-throttling-service/src/main/resources/osgp-throttling-service.properties @@ -40,7 +40,7 @@ wait.for.high.prio.pause.in.ms=200 # The task to reset in memory counters with db state is executed by cron expression. scheduling.task.reinitialize.state.cron.expression=30 0/30 * * * ? # Redis -redis.host=localhost +redis.host=redis redis.port=6379 redis.ssl=false redis.ssl.keystore.location= From ff7e0616113b5418d1c8f75840cc01c5df812ef6 Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Thu, 8 May 2025 12:22:14 +0200 Subject: [PATCH 43/64] Upload kubernetes log artifacts on error. --- .github/workflows/build.yml | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a8614744ad1..62dfc61e653 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -195,16 +195,31 @@ jobs: - name: Create and run gxf cucumber tests run: ./charts/gxf-cucumber-tests/template-apply.sh --imageTag $(echo -e "${{ needs.docker.outputs.tags }}" | head -n 1 | cut -d':' -f2) --valuesFile ${{ matrix.values_file }} - name: Wait for jobs to finish + id: cucumber_job run: > if ! kubectl wait --for=condition=complete --all jobs --timeout=30m; then echo "::error::Kubectl timed out..."; - kubectl get events --all-namespaces --sort-by='.metadata.creationTimestamp' + kubectl get events --all-namespaces --sort-by='.metadata.creationTimestamp' 2> kube_events.log for pod in $(kubectl get pod --field-selector=status.phase==Running -o name); do - echo "Logs for $pod:" - kubectl logs $pod --all-containers=true + echo "::debug:: Created logs for pod $pod" + kubectl logs $pod --all-containers=true 2> ${pod}.log done; - kubectl get pods - exit 1; + kubectl get pods 2> kube_status.log + echo "status=ERROR" >> $GITHUB_OUTPUT + fi + echo "status=SUCCES" >> $GITHUB_OUTPUT + - name: Upload error logs + uses: actions/upload-artifact@v4 + with: + name: kube_error_logs + path: ./*.log + if-no-files-found: ignore + overwrite: true + - name: Fail on error + run: > + if ${{ steps.cucumber_job.outputs.status }} == "ERROR"; + then + exit 1; fi - name: Run maven cucumber reporting run: mvn -f charts/gxf-cucumber-tests/pom.xml verify From e994e50758e8cdf9c2b8b39fc8c1e4375512fce2 Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Thu, 8 May 2025 12:55:46 +0200 Subject: [PATCH 44/64] Don't run on single line. --- .github/workflows/build.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 62dfc61e653..95a99039d8d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -196,18 +196,18 @@ jobs: run: ./charts/gxf-cucumber-tests/template-apply.sh --imageTag $(echo -e "${{ needs.docker.outputs.tags }}" | head -n 1 | cut -d':' -f2) --valuesFile ${{ matrix.values_file }} - name: Wait for jobs to finish id: cucumber_job - run: > + run: | if ! kubectl wait --for=condition=complete --all jobs --timeout=30m; then echo "::error::Kubectl timed out..."; kubectl get events --all-namespaces --sort-by='.metadata.creationTimestamp' 2> kube_events.log for pod in $(kubectl get pod --field-selector=status.phase==Running -o name); do - echo "::debug:: Created logs for pod $pod" - kubectl logs $pod --all-containers=true 2> ${pod}.log + echo "::debug:: Created logs for pod $pod"; + kubectl logs $pod --all-containers=true 2> ${pod}.log ; done; - kubectl get pods 2> kube_status.log - echo "status=ERROR" >> $GITHUB_OUTPUT - fi - echo "status=SUCCES" >> $GITHUB_OUTPUT + kubectl get pods 2> kube_status.log; + echo "status=ERROR" >> $GITHUB_OUTPUT; + fi; + echo "status=SUCCES" >> $GITHUB_OUTPUT; - name: Upload error logs uses: actions/upload-artifact@v4 with: @@ -216,8 +216,8 @@ jobs: if-no-files-found: ignore overwrite: true - name: Fail on error - run: > - if ${{ steps.cucumber_job.outputs.status }} == "ERROR"; + run: | + if [ ${{ steps.cucumber_job.outputs.status }} = "ERROR" ]; then exit 1; fi From 4aac3438f1cef930f77b560f0b1a2a7dc3626516 Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Thu, 8 May 2025 15:23:03 +0200 Subject: [PATCH 45/64] Added continue on error for cucumber_job. --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 95a99039d8d..791859cd980 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -196,6 +196,7 @@ jobs: run: ./charts/gxf-cucumber-tests/template-apply.sh --imageTag $(echo -e "${{ needs.docker.outputs.tags }}" | head -n 1 | cut -d':' -f2) --valuesFile ${{ matrix.values_file }} - name: Wait for jobs to finish id: cucumber_job + continue-on-error: true run: | if ! kubectl wait --for=condition=complete --all jobs --timeout=30m; then echo "::error::Kubectl timed out..."; From 0b40a443c5aca6b5aecf6c14cf11ce9ee7d013c7 Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Mon, 12 May 2025 13:29:47 +0200 Subject: [PATCH 46/64] Try to fix cucumber job output. --- .github/workflows/build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 791859cd980..b12c91418d1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -198,7 +198,7 @@ jobs: id: cucumber_job continue-on-error: true run: | - if ! kubectl wait --for=condition=complete --all jobs --timeout=30m; then + if ! kubectl wait --for=condition=complete --all jobs --timeout=10s; then echo "::error::Kubectl timed out..."; kubectl get events --all-namespaces --sort-by='.metadata.creationTimestamp' 2> kube_events.log for pod in $(kubectl get pod --field-selector=status.phase==Running -o name); do @@ -206,7 +206,8 @@ jobs: kubectl logs $pod --all-containers=true 2> ${pod}.log ; done; kubectl get pods 2> kube_status.log; - echo "status=ERROR" >> $GITHUB_OUTPUT; + echo "status='ERROR'" >> $GITHUB_OUTPUT; + exit 1 fi; echo "status=SUCCES" >> $GITHUB_OUTPUT; - name: Upload error logs From 4bb96cfa46fa3ea81b422d89be160aabebac77f8 Mon Sep 17 00:00:00 2001 From: paulus Date: Mon, 12 May 2025 13:33:24 +0200 Subject: [PATCH 47/64] Fixed incorrect syntax --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b12c91418d1..4765cf3e430 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -207,7 +207,7 @@ jobs: done; kubectl get pods 2> kube_status.log; echo "status='ERROR'" >> $GITHUB_OUTPUT; - exit 1 + exit 1 fi; echo "status=SUCCES" >> $GITHUB_OUTPUT; - name: Upload error logs From e7b3fa574cd9342e722c069311ae28d79356b87c Mon Sep 17 00:00:00 2001 From: paulus Date: Mon, 12 May 2025 14:21:18 +0200 Subject: [PATCH 48/64] Moved status error assignment to top. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4765cf3e430..4b38dc400ca 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -200,13 +200,13 @@ jobs: run: | if ! kubectl wait --for=condition=complete --all jobs --timeout=10s; then echo "::error::Kubectl timed out..."; + echo "status=ERROR" >> $GITHUB_OUTPUT; kubectl get events --all-namespaces --sort-by='.metadata.creationTimestamp' 2> kube_events.log for pod in $(kubectl get pod --field-selector=status.phase==Running -o name); do echo "::debug:: Created logs for pod $pod"; kubectl logs $pod --all-containers=true 2> ${pod}.log ; done; kubectl get pods 2> kube_status.log; - echo "status='ERROR'" >> $GITHUB_OUTPUT; exit 1 fi; echo "status=SUCCES" >> $GITHUB_OUTPUT; From 05fb2ccdf1f11a1c21cce08ad331e1b91027efff Mon Sep 17 00:00:00 2001 From: paulus Date: Mon, 12 May 2025 14:41:29 +0200 Subject: [PATCH 49/64] Re-added timeout 30m --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4b38dc400ca..b8c41d2eba3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -198,7 +198,7 @@ jobs: id: cucumber_job continue-on-error: true run: | - if ! kubectl wait --for=condition=complete --all jobs --timeout=10s; then + if ! kubectl wait --for=condition=complete --all jobs --timeout=30m; then echo "::error::Kubectl timed out..."; echo "status=ERROR" >> $GITHUB_OUTPUT; kubectl get events --all-namespaces --sort-by='.metadata.creationTimestamp' 2> kube_events.log From 7a04223ebe60968ce3e620fc231404f6607084e1 Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Tue, 13 May 2025 10:42:54 +0200 Subject: [PATCH 50/64] Write both stderr and stdout to log files. --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b8c41d2eba3..6e8d3f0d9a6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -201,12 +201,12 @@ jobs: if ! kubectl wait --for=condition=complete --all jobs --timeout=30m; then echo "::error::Kubectl timed out..."; echo "status=ERROR" >> $GITHUB_OUTPUT; - kubectl get events --all-namespaces --sort-by='.metadata.creationTimestamp' 2> kube_events.log + kubectl get events --all-namespaces --sort-by='.metadata.creationTimestamp' &> kube_events.log for pod in $(kubectl get pod --field-selector=status.phase==Running -o name); do echo "::debug:: Created logs for pod $pod"; - kubectl logs $pod --all-containers=true 2> ${pod}.log ; + kubectl logs $pod --all-containers=true &> ${pod}.log ; done; - kubectl get pods 2> kube_status.log; + kubectl get pods &> kube_status.log; exit 1 fi; echo "status=SUCCES" >> $GITHUB_OUTPUT; From 4b32d737b15f9bc3d128cef95dd4550b1bb0fd6c Mon Sep 17 00:00:00 2001 From: paulus Date: Wed, 14 May 2025 08:47:29 +0200 Subject: [PATCH 51/64] Removed comma's, reduced timeout. --- .github/workflows/build.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6e8d3f0d9a6..dac0885733b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -198,18 +198,18 @@ jobs: id: cucumber_job continue-on-error: true run: | - if ! kubectl wait --for=condition=complete --all jobs --timeout=30m; then - echo "::error::Kubectl timed out..."; - echo "status=ERROR" >> $GITHUB_OUTPUT; + if ! kubectl wait --for=condition=complete --all jobs --timeout=1m; then + echo "::error::Kubectl timed out..." + echo "status=ERROR" >> $GITHUB_OUTPUT kubectl get events --all-namespaces --sort-by='.metadata.creationTimestamp' &> kube_events.log for pod in $(kubectl get pod --field-selector=status.phase==Running -o name); do - echo "::debug:: Created logs for pod $pod"; - kubectl logs $pod --all-containers=true &> ${pod}.log ; + echo "::debug:: Created logs for pod $pod" + kubectl logs $pod --all-containers=true &> ${pod}.log done; - kubectl get pods &> kube_status.log; + kubectl get pods &> kube_status.log exit 1 fi; - echo "status=SUCCES" >> $GITHUB_OUTPUT; + echo "status=SUCCES" >> $GITHUB_OUTPUT - name: Upload error logs uses: actions/upload-artifact@v4 with: From 4b797dffaea6c18fdb60283882f6f75238087e24 Mon Sep 17 00:00:00 2001 From: paulus Date: Wed, 14 May 2025 09:23:42 +0200 Subject: [PATCH 52/64] Removed pod logs for testing. --- .github/workflows/build.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dac0885733b..7664596989b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -202,10 +202,7 @@ jobs: echo "::error::Kubectl timed out..." echo "status=ERROR" >> $GITHUB_OUTPUT kubectl get events --all-namespaces --sort-by='.metadata.creationTimestamp' &> kube_events.log - for pod in $(kubectl get pod --field-selector=status.phase==Running -o name); do - echo "::debug:: Created logs for pod $pod" - kubectl logs $pod --all-containers=true &> ${pod}.log - done; + kubectl get pods &> kube_status.log exit 1 fi; From 32e949d95c40138630810bae212969192025b93e Mon Sep 17 00:00:00 2001 From: paulus Date: Wed, 14 May 2025 09:51:15 +0200 Subject: [PATCH 53/64] Added lognames for values_files --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7664596989b..9705612a9b9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -201,6 +201,8 @@ jobs: if ! kubectl wait --for=condition=complete --all jobs --timeout=1m; then echo "::error::Kubectl timed out..." echo "status=ERROR" >> $GITHUB_OUTPUT + valuesfile=${{ matrix.values_file }} + echo "logname=$(basename $valuesfile .yaml)" >> $GITHUB_OUTPUT kubectl get events --all-namespaces --sort-by='.metadata.creationTimestamp' &> kube_events.log kubectl get pods &> kube_status.log @@ -210,7 +212,7 @@ jobs: - name: Upload error logs uses: actions/upload-artifact@v4 with: - name: kube_error_logs + name: kube_${{ steps.cucumber_job.outputs.logname }}_error_logs path: ./*.log if-no-files-found: ignore overwrite: true From e0df0977cb9c016b9a818d606a129c78edf31714 Mon Sep 17 00:00:00 2001 From: paulus Date: Wed, 14 May 2025 10:29:18 +0200 Subject: [PATCH 54/64] Removed -values from logname --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9705612a9b9..41f2af21a21 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -201,8 +201,8 @@ jobs: if ! kubectl wait --for=condition=complete --all jobs --timeout=1m; then echo "::error::Kubectl timed out..." echo "status=ERROR" >> $GITHUB_OUTPUT - valuesfile=${{ matrix.values_file }} - echo "logname=$(basename $valuesfile .yaml)" >> $GITHUB_OUTPUT + valuesfile=$(basename ${{ matrix.values_file }} .yaml) + echo "logname=${valuesfile%%-*}" >> $GITHUB_OUTPUT kubectl get events --all-namespaces --sort-by='.metadata.creationTimestamp' &> kube_events.log kubectl get pods &> kube_status.log From 031e08f36806a8c5587a567e597a5670cf17901a Mon Sep 17 00:00:00 2001 From: paulus Date: Wed, 14 May 2025 10:50:06 +0200 Subject: [PATCH 55/64] Stop failing fast so we can collect logs. --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 41f2af21a21..d9363c359f2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -174,6 +174,7 @@ jobs: permissions: checks: write strategy: + fail-fast: false matrix: values_file: [ci/common-values.yaml, ci/publiclighting-values.yaml, ci/smartmetering-values.yaml] steps: From 5863daa0d7e24d953459c9f86f2215fe27db50b4 Mon Sep 17 00:00:00 2001 From: paulus Date: Wed, 14 May 2025 11:15:01 +0200 Subject: [PATCH 56/64] Readded logs for pods. --- .github/workflows/build.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d9363c359f2..39da35c214f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -205,7 +205,11 @@ jobs: valuesfile=$(basename ${{ matrix.values_file }} .yaml) echo "logname=${valuesfile%%-*}" >> $GITHUB_OUTPUT kubectl get events --all-namespaces --sort-by='.metadata.creationTimestamp' &> kube_events.log - + for pod in $(kubectl get pod --field-selector=status.phase==Running -o name); do + echo "::debug:: Created logs for pod $pod" + kubectl logs $pod --all-containers=true &> ${pod}.log + done + ls kubectl get pods &> kube_status.log exit 1 fi; From 12ccfaa6d181abce21591d6836878c374f4dbcab Mon Sep 17 00:00:00 2001 From: paulus Date: Wed, 14 May 2025 11:32:33 +0200 Subject: [PATCH 57/64] Added ls. --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 39da35c214f..ee844d26c8a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -207,6 +207,7 @@ jobs: kubectl get events --all-namespaces --sort-by='.metadata.creationTimestamp' &> kube_events.log for pod in $(kubectl get pod --field-selector=status.phase==Running -o name); do echo "::debug:: Created logs for pod $pod" + ls kubectl logs $pod --all-containers=true &> ${pod}.log done ls From 242d225c36d4aaea96c001fff76a1633547ae0f5 Mon Sep 17 00:00:00 2001 From: paulus Date: Wed, 14 May 2025 11:35:33 +0200 Subject: [PATCH 58/64] Removed slash from pod filename. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ee844d26c8a..735809b12d5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -208,7 +208,7 @@ jobs: for pod in $(kubectl get pod --field-selector=status.phase==Running -o name); do echo "::debug:: Created logs for pod $pod" ls - kubectl logs $pod --all-containers=true &> ${pod}.log + kubectl logs $pod --all-containers=true &> $(basename $pod).log done ls kubectl get pods &> kube_status.log From dcebb5c2abbaa1da05cb4a1a50f9a24b8fbd58f0 Mon Sep 17 00:00:00 2001 From: paulus Date: Wed, 14 May 2025 12:10:04 +0200 Subject: [PATCH 59/64] timeout back to 30m --- .github/workflows/build.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 735809b12d5..c9f5e2a010b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -199,7 +199,7 @@ jobs: id: cucumber_job continue-on-error: true run: | - if ! kubectl wait --for=condition=complete --all jobs --timeout=1m; then + if ! kubectl wait --for=condition=complete --all jobs --timeout=30m; then echo "::error::Kubectl timed out..." echo "status=ERROR" >> $GITHUB_OUTPUT valuesfile=$(basename ${{ matrix.values_file }} .yaml) @@ -207,10 +207,8 @@ jobs: kubectl get events --all-namespaces --sort-by='.metadata.creationTimestamp' &> kube_events.log for pod in $(kubectl get pod --field-selector=status.phase==Running -o name); do echo "::debug:: Created logs for pod $pod" - ls kubectl logs $pod --all-containers=true &> $(basename $pod).log done - ls kubectl get pods &> kube_status.log exit 1 fi; From baaacc0f3e109d54c47ee7c6421783a5c17c96d8 Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Wed, 21 May 2025 09:33:59 +0200 Subject: [PATCH 60/64] Removed local changes to properties and updated readme. --- .github/workflows/README.md | 5 ++++- .../src/main/resources/osgp-throttling-service.properties | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 6adb53bf6b6..5b2912bd838 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -7,4 +7,7 @@ To run build.yml for testing you need to add a label to your PR containing the t Afterwards you can remove the label again. ### Error logs on cucumber test timeout -If cucumber tests time out error logs are generated and uploaded as artifacts. \ No newline at end of file +If cucumber tests time out error logs are generated and uploaded as artifacts. + +## Editing properties for devops +If you need to edit the configuration for OPS purposes, this can be done in the gxf-gitops repository under platform/config.yaml -- global properties. \ No newline at end of file diff --git a/osgp/platform/osgp-throttling-service/src/main/resources/osgp-throttling-service.properties b/osgp/platform/osgp-throttling-service/src/main/resources/osgp-throttling-service.properties index 37014c4a275..b3e20a8e9f1 100644 --- a/osgp/platform/osgp-throttling-service/src/main/resources/osgp-throttling-service.properties +++ b/osgp/platform/osgp-throttling-service/src/main/resources/osgp-throttling-service.properties @@ -1,6 +1,6 @@ # Datasource Configuration spring.datasource.driver-class-name=org.postgresql.Driver -spring.datasource.url=jdbc:postgresql://postgres:5432/osgp_throttling +spring.datasource.url=jdbc:postgresql://localhost:5432/osgp_throttling spring.datasource.username=osp_admin spring.datasource.password=1234 # JPA / Hibernate Configuration @@ -40,7 +40,7 @@ wait.for.high.prio.pause.in.ms=200 # The task to reset in memory counters with db state is executed by cron expression. scheduling.task.reinitialize.state.cron.expression=30 0/30 * * * ? # Redis -redis.host=redis +redis.host=localhost redis.port=6379 redis.ssl=false redis.ssl.keystore.location= From 0ff7b660fe8ed9b5267322916b0e1813f60996fd Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Wed, 21 May 2025 11:00:35 +0200 Subject: [PATCH 61/64] Added valuesfile for gxf platform. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c9f5e2a010b..7429d39f5df 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -192,7 +192,7 @@ jobs: - name: Setup k3d cluster run: bash setup.sh - name: Create gxf platform - run: ./charts/gxf-platform/template-apply.sh --imageTag $(echo -e "${{ needs.docker.outputs.tags }}" | head -n 1 | cut -d':' -f2) + run: ./charts/gxf-platform/template-apply.sh --imageTag $(echo -e "${{ needs.docker.outputs.tags }}" | head -n 1 | cut -d':' -f2) --valuesFile ${{ matrix.values_file }} - name: Create and run gxf cucumber tests run: ./charts/gxf-cucumber-tests/template-apply.sh --imageTag $(echo -e "${{ needs.docker.outputs.tags }}" | head -n 1 | cut -d':' -f2) --valuesFile ${{ matrix.values_file }} - name: Wait for jobs to finish From 6841e879a54c6aef9805f4f5385dc5666a8b9867 Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Mon, 16 Jun 2025 18:14:48 +0200 Subject: [PATCH 62/64] Removed unused tomcat. --- osgp/platform/osgp-secret-management/Dockerfile | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/osgp/platform/osgp-secret-management/Dockerfile b/osgp/platform/osgp-secret-management/Dockerfile index 90c9f41234c..6e9b08f8f0a 100644 --- a/osgp/platform/osgp-secret-management/Dockerfile +++ b/osgp/platform/osgp-secret-management/Dockerfile @@ -1,15 +1,4 @@ -FROM tomcat:10.1-jre17-temurin-noble - -RUN rm -rf /usr/local/tomcat/webapps.dist - -# Enable the use of system environment variables in Tomcat configuration files. -RUN echo "org.apache.tomcat.util.digester.PROPERTY_SOURCE=org.apache.tomcat.util.digester.EnvironmentPropertySource" >> /usr/local/tomcat/conf/catalina.properties - -RUN mkdir -p /var/log/osp && \ - rm -rf /usr/local/tomcat/webapps/* && \ - chmod 755 /usr/local/tomcat/webapps /var/log/osp && \ - mkdir -p /usr/local/tomcat/webapps/probe/ && \ - touch /usr/local/tomcat/webapps/probe/probe.txt +FROM eclipse-temurin:17 WORKDIR /opt/app From 6b86d078eda7bd45dbdf0bc3090cff928ad3c2a5 Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Mon, 16 Jun 2025 18:30:29 +0200 Subject: [PATCH 63/64] Added java.security properties file. --- osgp/platform/osgp-secret-management/Dockerfile | 3 ++- .../osgp-secret-management/java.security | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 osgp/platform/osgp-secret-management/java.security diff --git a/osgp/platform/osgp-secret-management/Dockerfile b/osgp/platform/osgp-secret-management/Dockerfile index 6e9b08f8f0a..619fd00d33a 100644 --- a/osgp/platform/osgp-secret-management/Dockerfile +++ b/osgp/platform/osgp-secret-management/Dockerfile @@ -3,7 +3,8 @@ FROM eclipse-temurin:17 WORKDIR /opt/app COPY target/osgp-secret-management-*exec.jar osgp-secret-management-exec.jar +COPY java.security . ENV SERVER_PORT=8443 -CMD java -Djava.security.properties=/opt/app/java.security --module-path /opt/nfast/java/classes -jar osgp-secret-management-exec.jar +CMD java -Djava.security.properties=/opt/app/java.security -jar osgp-secret-management-exec.jar diff --git a/osgp/platform/osgp-secret-management/java.security b/osgp/platform/osgp-secret-management/java.security new file mode 100644 index 00000000000..37ce1c693a9 --- /dev/null +++ b/osgp/platform/osgp-secret-management/java.security @@ -0,0 +1,16 @@ +# +# List of providers and their preference orders (see above): +# +security.provider.1=SUN +security.provider.2=SunRsaSign +security.provider.3=SunEC +security.provider.4=nCipherKM +security.provider.5=SunJSSE +security.provider.6=SunJCE +security.provider.7=SunJGSS +security.provider.8=SunSASL +security.provider.9=XMLDSig +security.provider.10=SunPCSC +security.provider.11=JdkLDAP +security.provider.12=JdkSASL +security.provider.13=SunPKCS11 \ No newline at end of file From a148c0e3292ac45b7073b2fa62daa0ad7cf1af9a Mon Sep 17 00:00:00 2001 From: Paulus Nunnink Date: Mon, 16 Jun 2025 20:19:49 +0200 Subject: [PATCH 64/64] Added properties file to starting command. --- osgp/platform/osgp-secret-management/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osgp/platform/osgp-secret-management/Dockerfile b/osgp/platform/osgp-secret-management/Dockerfile index 619fd00d33a..bcf7f583226 100644 --- a/osgp/platform/osgp-secret-management/Dockerfile +++ b/osgp/platform/osgp-secret-management/Dockerfile @@ -7,4 +7,4 @@ COPY java.security . ENV SERVER_PORT=8443 -CMD java -Djava.security.properties=/opt/app/java.security -jar osgp-secret-management-exec.jar +CMD java -Djava.security.properties=/opt/app/java.security -jar osgp-secret-management-exec.jar --SECRET_MANAGEMENT_PROPERTIES=/etc/osp/osgp-secret-management.properties