diff --git a/.github/workflows/publish_dev.yml b/.github/workflows/publish_dev.yml index 86cb76b6..a717d818 100644 --- a/.github/workflows/publish_dev.yml +++ b/.github/workflows/publish_dev.yml @@ -1,8 +1,8 @@ name: Publish Docker Images (dev) -# Triggers on direct pushes to the 'dev' branch, which includes PR merges. on: - push: + pull_request_target: + types: [ closed ] branches: [ "dev" ] env: @@ -11,41 +11,48 @@ env: jobs: bump-version: name: Bump VERSION file + if: github.event.pull_request.merged == true runs-on: ubuntu-latest permissions: contents: write # needed to push the updated VERSION file back outputs: new_version: ${{ steps.bump.outputs.version }} + version_changed: ${{ steps.bump.outputs.changed }} steps: - name: Checkout repository uses: actions/checkout@v4 with: token: ${{ secrets.PIPELINE_PAT }} + ref: ${{ github.event.pull_request.base.ref }} fetch-depth: 0 - name: Detect source branch and bump version id: bump run: | - # ── Detect source branch from merge commit subject ────────────────── - # Standard merge commit message: "Merge branch 'feature/foo' into dev" - MERGE_SUBJECT="$(git log --merges -1 --pretty=%s HEAD)" - echo "Merge subject: ${MERGE_SUBJECT}" - - SOURCE=$(echo "$MERGE_SUBJECT" | sed -n "s/Merge branch '\([^']*\)'.*/\1/p") - SOURCE="${SOURCE#origin/}" # strip remote prefix if present + SOURCE="${{ github.event.pull_request.head.ref }}" if [[ -z "$SOURCE" ]]; then SOURCE="unknown" - echo "::warning::Could not detect source branch; defaulting to 'unknown' (no bump)." + echo "::warning::Could not detect source branch from pull request metadata; defaulting to 'unknown' (no bump)." fi echo "Detected source branch: ${SOURCE}" # ── Read and parse current VERSION ────────────────────────────────── RAW="$(cat VERSION | tr -d '[:space:]')" - CLEAN="${RAW#v}" # strip leading v - CLEAN="${CLEAN%-dev}" # strip any existing -dev suffix - IFS='.' read -r MAJOR MINOR PATCH <<< "$CLEAN" + CLEAN="${RAW#v}" # strip leading v + while [[ "$CLEAN" == *-dev ]]; do + CLEAN="${CLEAN%-dev}" + done + + if [[ ! "$CLEAN" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then + echo "::error::VERSION must be in the form v..[-dev], got '${RAW}'" + exit 1 + fi + + MAJOR="${BASH_REMATCH[1]}" + MINOR="${BASH_REMATCH[2]}" + PATCH="${BASH_REMATCH[3]}" # ── Apply bump based on branch prefix ──────────────────────────────── PREFIX="${SOURCE%%/*}" @@ -69,18 +76,29 @@ jobs: echo "${NEW_VERSION}" > VERSION echo "version=${NEW_VERSION}" >> "$GITHUB_OUTPUT" + if [[ "$RAW" != "$NEW_VERSION" ]]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + else + echo "changed=false" >> "$GITHUB_OUTPUT" + fi echo "New version: ${NEW_VERSION}" - name: Commit and push updated VERSION + if: steps.bump.outputs.changed == 'true' run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" git add VERSION + if git diff --cached --quiet; then + echo "No VERSION changes to commit." + exit 0 + fi git commit -m "chore: bump version to ${{ steps.bump.outputs.version }} [skip ci]" git push build-and-push: name: Build & Push ${{ matrix.container }} + if: github.event.pull_request.merged == true needs: bump-version runs-on: ubuntu-latest strategy: @@ -103,7 +121,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 with: - ref: dev # pick up the commit that includes the updated VERSION + ref: ${{ github.event.pull_request.base.ref }} - name: Setup Docker buildx uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/publish_main.yml b/.github/workflows/publish_main.yml index b6be3712..6040a85d 100644 --- a/.github/workflows/publish_main.yml +++ b/.github/workflows/publish_main.yml @@ -1,8 +1,8 @@ name: Publish Docker Images (main) -# Triggers on direct pushes to 'main', which includes PR/merge-commit merges. on: - push: + pull_request_target: + types: [ closed ] branches: [ "main" ] env: @@ -11,41 +11,48 @@ env: jobs: bump-version: name: Bump VERSION file + if: github.event.pull_request.merged == true runs-on: ubuntu-latest permissions: contents: write # needed to push back VERSION + Git tag outputs: new_version: ${{ steps.bump.outputs.version }} + version_changed: ${{ steps.bump.outputs.changed }} steps: - name: Checkout repository uses: actions/checkout@v4 with: token: ${{ secrets.PIPELINE_PAT }} + ref: ${{ github.event.pull_request.base.ref }} fetch-depth: 0 - name: Detect source branch and bump version id: bump run: | - # ── Detect source branch from merge commit subject ────────────────── - # Standard merge commit message: "Merge branch 'feature/foo' into main" - MERGE_SUBJECT="$(git log --merges -1 --pretty=%s HEAD)" - echo "Merge subject: ${MERGE_SUBJECT}" - - SOURCE=$(echo "$MERGE_SUBJECT" | sed -n "s/Merge branch '\([^']*\)'.*/\1/p") - SOURCE="${SOURCE#origin/}" # strip remote prefix if present + SOURCE="${{ github.event.pull_request.head.ref }}" if [[ -z "$SOURCE" ]]; then SOURCE="unknown" - echo "::warning::Could not detect source branch; defaulting to 'unknown' (no bump)." + echo "::warning::Could not detect source branch from pull request metadata; defaulting to 'unknown' (no bump)." fi echo "Detected source branch: ${SOURCE}" # ── Read and parse current VERSION ────────────────────────────────── RAW="$(cat VERSION | tr -d '[:space:]')" - CLEAN="${RAW#v}" # strip leading v - CLEAN="${CLEAN%-dev}" # strip any existing -dev suffix - IFS='.' read -r MAJOR MINOR PATCH <<< "$CLEAN" + CLEAN="${RAW#v}" # strip leading v + while [[ "$CLEAN" == *-dev ]]; do + CLEAN="${CLEAN%-dev}" + done + + if [[ ! "$CLEAN" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then + echo "::error::VERSION must be in the form v..[-dev], got '${RAW}'" + exit 1 + fi + + MAJOR="${BASH_REMATCH[1]}" + MINOR="${BASH_REMATCH[2]}" + PATCH="${BASH_REMATCH[3]}" # ── Decide bump strategy ───────────────────────────────────────────── # When the source branch is 'dev', the version was already incremented @@ -74,23 +81,38 @@ jobs: echo "${NEW_VERSION}" > VERSION echo "version=${NEW_VERSION}" >> "$GITHUB_OUTPUT" + if [[ "$RAW" != "$NEW_VERSION" ]]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + else + echo "changed=false" >> "$GITHUB_OUTPUT" + fi echo "New version: ${NEW_VERSION}" - name: Commit and push updated VERSION + if: steps.bump.outputs.changed == 'true' run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" git add VERSION + if git diff --cached --quiet; then + echo "No VERSION changes to commit." + exit 0 + fi git commit -m "chore: release ${{ steps.bump.outputs.version }} [skip ci]" git push - name: Create and push Git release tag run: | + if git rev-parse -q --verify "refs/tags/${{ steps.bump.outputs.version }}" >/dev/null 2>&1; then + echo "Tag '${{ steps.bump.outputs.version }}' already exists; skipping." + exit 0 + fi git tag "${{ steps.bump.outputs.version }}" git push origin "${{ steps.bump.outputs.version }}" build-and-push: name: Build & Push ${{ matrix.container }} + if: github.event.pull_request.merged == true needs: bump-version runs-on: ubuntu-latest strategy: @@ -113,7 +135,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 with: - ref: main # pick up the commit that includes the updated VERSION + ref: ${{ github.event.pull_request.base.ref }} - name: Derive partial version tags id: version diff --git a/README.md b/README.md index c246509c..cdb9c8a8 100644 --- a/README.md +++ b/README.md @@ -43,16 +43,13 @@ MacOS WorkFlows - - Windows WorkFlows - ## About the Project -![Pipeline overview](./assets/heidgaf_architecture.svg) +![Pipeline overview](./assets/hamstring_architecture.svg) ## Getting Started @@ -60,16 +57,15 @@ ```sh HOST_IP=127.0.0.1 docker compose -f docker/docker-compose.yml --profile prod up ``` -

- Terminal example -

- #### Use the dev profile for testing out changes in docker containers: ```sh HOST_IP=127.0.0.1 docker compose -f docker/docker-compose.yml --profile dev up ``` +

+ Terminal example +

+ -

(back to top)

@@ -85,7 +81,7 @@ possibly infrastructure. The section `pipeline.log_collection.collector.logline_format` has to be adjusted to reflect your specific input log line format. Using our adjustable and flexible log line configuration, you can rename, reorder and fully configure each -field of a valid log line. Freely define timestamps, RegEx patterns, lists, and IP addresses. For example, your +field of a valid log line. You can freely define timestamps, RegEx patterns, lists, and IP addresses. For example, your configuration might look as follows: ```yml @@ -187,24 +183,11 @@ Have a look at the following pictures showing examples of how these dashboards m -

(back to top)

- - ## Models and Training -To train and test our and possibly your own models, we currently rely on the following datasets: - -- [DGTA Benchmark](https://data.mendeley.com/datasets/2wzf9bz7xr/1) -- [DNS Tunneling Queries for Binary Classification](https://data.mendeley.com/datasets/mzn9hvdcxg/1) -- [UMUDGA - University of Murcia Domain Generation Algorithm Dataset](https://data.mendeley.com/datasets/y8ph45msv8/1) -- [DGArchive](https://dgarchive.caad.fkie.fraunhofer.de/) -- [DNS Exfiltration](https://data.mendeley.com/datasets/c4n7fckkz3/3) - -We compute all features separately and only rely on the `domain` and `class` for binary classification. - ### Inserting Data for Testing -For testing purposes, you can ingest PCAPs or tap on network interfaces using the zeek-based sensor in its `1.0.0` release. For more information on it, please refer to [the documentation](https://github.com/Hamstring-NDR/hamstring-zeek). +For testing purposes, you can ingest PCAPs or tap on network interfaces using the zeek-based sensor that is integrated into the docker-compose file. For more information on the sensor, please refer to [the documentation](https://github.com/Hamstring-NDR/hamstring-zeek). ### Training Your Own Models @@ -260,33 +243,6 @@ The results will be saved per default to `./results`, if not configured otherwis ``` This will create a `rules.txt` file containing the innards of the model, explaining the rules it created. -

(back to top)

- - -### Data - -> [!IMPORTANT] -> We support custom schemes. - -Depending on your data and usecase, you can customize the data scheme to fit your needs. -The below configuration is part of the [main configuration file](./config.yaml) which is detailed in our [documentation](https://HAMSTRING.readthedocs.io/en/latest/usage.html#id2) - -```yml -loglines: - fields: - - [ "timestamp", RegEx, '^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$' ] - - [ "status_code", ListItem, [ "NOERROR", "NXDOMAIN" ], [ "NXDOMAIN" ] ] - - [ "src_ip", IpAddress ] - - [ "dns_server_ip", IpAddress ] - - [ "domain_name", RegEx, '^(?=.{1,253}$)((?!-)[A-Za-z0-9-]{1,63}(?(back to top)

## Contributing @@ -305,16 +261,12 @@ Don't forget to give the project a star! Thanks again! -

(back to top)

- ## License Distributed under the EUPL License. See `LICENSE.txt` for more information. -

(back to top)

- diff --git a/assets/heidgaf_architecture.svg b/assets/hamstring_architecture.svg similarity index 100% rename from assets/heidgaf_architecture.svg rename to assets/hamstring_architecture.svg diff --git a/assets/hamstring_terminal.gif b/assets/hamstring_terminal.gif new file mode 100644 index 00000000..932d3a18 Binary files /dev/null and b/assets/hamstring_terminal.gif differ diff --git a/assets/terminal_example.gif b/assets/terminal_example.gif deleted file mode 100644 index bcaf0e1d..00000000 Binary files a/assets/terminal_example.gif and /dev/null differ diff --git a/docker/docker-compose/prod/docker-compose.monitoring.yml b/docker/docker-compose/prod/docker-compose.monitoring.yml index 060b6ec2..c721b26b 100644 --- a/docker/docker-compose/prod/docker-compose.monitoring.yml +++ b/docker/docker-compose/prod/docker-compose.monitoring.yml @@ -1,6 +1,6 @@ services: monitoring_agent: - image: ghcr.io/hamstring-ndr/hamstring-monitoring:v1.0.0 + image: ghcr.io/hamstring-ndr/hamstring-monitoring:v2.0.0 restart: "unless-stopped" volumes: - ../../../config.yaml:/app/config.yaml diff --git a/docs/media/hamstring_architecture.png b/docs/media/hamstring_architecture.png new file mode 100644 index 00000000..996ae064 Binary files /dev/null and b/docs/media/hamstring_architecture.png differ diff --git a/docs/media/hamstring_architecture.svg b/docs/media/hamstring_architecture.svg new file mode 100644 index 00000000..4c26e527 --- /dev/null +++ b/docs/media/hamstring_architecture.svg @@ -0,0 +1,4 @@ + + + +
Log Server
Log Collector
Batch Sender
Prefilter
Inspector
Detector
ZooKeeper
Zeek Sensor
Kafka Broker
Log Generation
Log Aggregation
Collection
Filtering
Inspection
Detection
Legend
Consume
Produce
Alerter
Alerting
Kafka Broker
Kafka Broker
Log Metadata & Analysis Aggregation
Log File
PCAPs & 
TAP interfaces
 diff --git a/docs/media/heidgaf_architecture.svg b/docs/media/heidgaf_architecture.svg deleted file mode 100644 index bef8170d..00000000 --- a/docs/media/heidgaf_architecture.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Log Server
Log Collector
Batch Sender
Prefilter
Inspector
Detector
ZooKeeper
Kafka Broker
Zeek Sensor
Kafka Broker
Kafka Broker
Log Generation
Log Aggregation
Collection
Filtering
Inspection
Detection
Legend
Consume
Produce
 diff --git a/docs/media/heidgaf_overview_detailed.drawio.png b/docs/media/heidgaf_overview_detailed.drawio.png deleted file mode 100644 index 95dfd511..00000000 Binary files a/docs/media/heidgaf_overview_detailed.drawio.png and /dev/null differ diff --git a/docs/media/monitoring_pipeline.png b/docs/media/monitoring_pipeline.png deleted file mode 100644 index a5f95511..00000000 Binary files a/docs/media/monitoring_pipeline.png and /dev/null differ diff --git a/docs/media/pipeline_overview.png b/docs/media/pipeline_overview.png deleted file mode 100644 index debd8988..00000000 Binary files a/docs/media/pipeline_overview.png and /dev/null differ