Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/dev_dbt.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,6 @@ jobs:
with:
image-name: dbt-demo
# TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that
# the demo is reproducable for the release and it will be automatically replaced for the release branch.
# the demo is reproducible for the release and it will be automatically replaced for the release branch.
image-version: 0.0.1
containerfile-path: demos/airflow-scheduled-job/dbt/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,6 @@ jobs:
with:
image-name: jupyter-pyspark-with-alibi-detect
# TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that
# the demo is reproducable for the release and it will be automatically replaced for the release branch.
# the demo is reproducible for the release and it will be automatically replaced for the release branch.
image-version: python-3.12
containerfile-path: demos/signal-processing/Dockerfile-jupyter
2 changes: 1 addition & 1 deletion .github/workflows/dev_jupyterhub-keycloak-spark.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,6 @@ jobs:
with:
image-name: spark
# TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that
# the demo is reproducable for the release and it will be automatically replaced for the release branch.
# the demo is reproducible for the release and it will be automatically replaced for the release branch.
image-version: 3.5.2-python311
containerfile-path: demos/jupyterhub-keycloak/Dockerfile
2 changes: 1 addition & 1 deletion .github/workflows/dev_nifi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,6 @@ jobs:
with:
image-name: nifi
# TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that
# the demo is reproducable for the release and it will be automatically replaced for the release branch.
# the demo is reproducible for the release and it will be automatically replaced for the release branch.
image-version: 2.12.0-postgresql
containerfile-path: demos/signal-processing/Dockerfile-nifi
20 changes: 15 additions & 5 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,18 @@ default_language_version:
exclude: '(stacks/_templates/minio-.*/rendered-chart\.yaml|\.svg)$'

repos:
- repo: https://github.com/crate-ci/typos
rev: 00f422f3b19c57bc6338715ebfe3316d38768461 # v1.50.3
hooks:
- id: typos
# Drop the upstream default `--write-changes` so the hook reports and
# fails instead of rewriting files.
# Keep `--force-exclude` so the
# excludes in typos.toml still apply to the paths prek passes in.
args: ["--force-exclude"]

- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v6.0.0 # 4.6.0
rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # 6.0.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
Expand All @@ -16,23 +26,23 @@ repos:
exclude: '^stacks/argo-cd-git-ops/secrets/sealed-secrets-key\.yaml$'

- repo: https://github.com/adrienverge/yamllint
rev: v1.37.1 # 1.35.1
rev: cba56bcde1fdd01c1deb3f945e69764c291a6530 # 1.38.0
hooks:
- id: yamllint

- repo: https://github.com/igorshubovych/markdownlint-cli
rev: v0.46.0 # 0.41.0
rev: a4d5d37e66ebcd6b3705204a1d6dbb56dea66338 # 0.49.0
hooks:
- id: markdownlint

- repo: https://github.com/koalaman/shellcheck-precommit
rev: v0.11.0 # 0.10.0
rev: 99470f5e12208ff0fb17ab81c3c494f7620a1d8d # 0.11.0
hooks:
- id: shellcheck
args: ["--severity=info"]

- repo: https://github.com/rhysd/actionlint
rev: e7d448ef7507c20fc4c88a95d0c448b848cd6127 # 1.7.8
rev: 914e7df21a07ef503a81201c76d2b11c789d3fca # 1.7.12
hooks:
- id: actionlint

Expand Down
2 changes: 1 addition & 1 deletion .scripts/update_refs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ if [[ "$CURRENT_BRANCH" == release-* ]]; then
STACKABLE_RELEASE="${CURRENT_BRANCH#release-}"
MESSAGE="Update stackableRelease to $STACKABLE_RELEASE"
echo "$MESSAGE"
# NOTE (@NickLarsenNZ): find is not required for such a trivial case, but it is done for consitency
# NOTE (@NickLarsenNZ): find is not required for such a trivial case, but it is done for consistency
find stacks/stacks-v2.yaml \
-exec grep --color=always -l stackableRelease {} \; \
-exec sed -i -E "s#(stackableRelease:\s+)(\S+)#\1${STACKABLE_RELEASE}#" {} \; \
Expand Down
2 changes: 1 addition & 1 deletion docs/modules/demos/pages/airflow-scheduled-job.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ Enable and run `core_deferrable_sleep_demo`:
image::airflow-scheduled-job/airflow_17.png[]

Click on the DAG, switching to the task view.
This DAG uses a deferrable operator which, in conjunction with the triggerer process, "offloads" the DAG from its worker for a specfic period of time, before being picked up and again and executed.
This DAG uses a deferrable operator which, in conjunction with the triggerer process, "offloads" the DAG from its worker for a specific period of time, before being picked up and again and executed.
From the Task Instances view (under Dags), you will see the task cycle through the following states:

image::airflow-scheduled-job/deferrable_01_running.png[]
Expand Down
4 changes: 2 additions & 2 deletions docs/modules/demos/pages/argo-cd-git-ops.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ When managing all resources and configuration via Git, deploying sensitive data
There are multiple solutions to this such as Hashicorp or Bitwarden, which depend heavily on the infrastructure already available.

For the sake of this demo, {sealed-secrets}[Bitnami's Sealed Secrets] are utilized.
Sensitive data is encrypted as a `SealedSecret` before commiting to the Git repository, synced via ArgoCD and decrypted by the Sealed Secrets controller into a standard Kubernetes `Secret`.
Sensitive data is encrypted as a `SealedSecret` before committing to the Git repository, synced via ArgoCD and decrypted by the Sealed Secrets controller into a standard Kubernetes `Secret`.

This way, everything will be stored and managed in Git.

Expand Down Expand Up @@ -121,7 +121,7 @@ Detailed information about the Airflow cluster, the cluster status and deployed
Additionally, if the Git repository and the cluster state itself differ, these differences can be previewed in a code diff preview.

Per default in this demo, the ArgoCD `Sync Policy` is set to `auto-sync`.
This means that changes to the Git repository are immediatly synced into the cluster.
This means that changes to the Git repository are immediately synced into the cluster.
This is nice in the demo case, but should be disabled for production use cases.

image::argo-cd-git-ops/argocd-airflow-application-details.png[]
Expand Down
2 changes: 1 addition & 1 deletion docs/modules/demos/pages/signal-processing.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ There are two located in the "Stackable Data Platform" folder.

=== Measurements

The _Gas measurements_ dashboard shows the original data. The first graph plots two measurments (`r1`, `r2`), together with the model scores (`r1_score`, `r2_score`, `r1_score_lttb`).#
The _Gas measurements_ dashboard shows the original data. The first graph plots two measurements (`r1`, `r2`), together with the model scores (`r1_score`, `r2_score`, `r1_score_lttb`).#
These are superimposed on each other for ease of comparison.

image::signal-processing/measurements.png[]
Expand Down
18 changes: 9 additions & 9 deletions stacks/monitoring/grafana-dashboards.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1253,7 +1253,7 @@ data:
"uid": "prometheus"
},
"editorMode": "code",
"expr": "sum by (namespae, job) (trino_execution_querymanager_runningqueries{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})",
"expr": "sum by (namespace, job) (trino_execution_querymanager_runningqueries{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})",
"legendFormat": "running",
"range": true,
"refId": "A"
Expand All @@ -1264,9 +1264,9 @@ data:
"uid": "prometheus"
},
"editorMode": "code",
"expr": "sum by (namespae, job) (trino_execution_querymanager_queuedqueries{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})",
"expr": "sum by (namespace, job) (trino_execution_querymanager_queuedqueries{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})",
"hide": false,
"legendFormat": "queueud",
"legendFormat": "queued",
"range": true,
"refId": "B"
}
Expand Down Expand Up @@ -1409,7 +1409,7 @@ data:
"uid": "prometheus"
},
"editorMode": "code",
"expr": "sum by (namespae, job) (trino_execution_querymanager_completedqueries_totalcount{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})",
"expr": "sum by (namespace, job) (trino_execution_querymanager_completedqueries_totalcount{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})",
"legendFormat": "completed",
"range": true,
"refId": "A"
Expand All @@ -1420,7 +1420,7 @@ data:
"uid": "prometheus"
},
"editorMode": "code",
"expr": "sum by (namespae, job) (trino_execution_querymanager_failedqueries_totalcount{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})",
"expr": "sum by (namespace, job) (trino_execution_querymanager_failedqueries_totalcount{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})",
"hide": false,
"legendFormat": "failed",
"range": true,
Expand All @@ -1432,7 +1432,7 @@ data:
"uid": "prometheus"
},
"editorMode": "code",
"expr": "sum by (namespae, job) (trino_execution_querymanager_abandonedqueries_totalcount{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})",
"expr": "sum by (namespace, job) (trino_execution_querymanager_abandonedqueries_totalcount{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})",
"hide": false,
"legendFormat": "abandoned",
"range": true,
Expand All @@ -1444,7 +1444,7 @@ data:
"uid": "prometheus"
},
"editorMode": "code",
"expr": "sum by (namespae, job) (trino_execution_querymanager_canceledqueries_totalcount{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})",
"expr": "sum by (namespace, job) (trino_execution_querymanager_canceledqueries_totalcount{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})",
"hide": false,
"legendFormat": "canceled",
"range": true,
Expand Down Expand Up @@ -9095,7 +9095,7 @@ data:
{
"matcher": {
"id": "byName",
"options": "mssing"
"options": "missing"
},
"properties": [
{
Expand Down Expand Up @@ -9183,7 +9183,7 @@ data:
"editorMode": "code",
"expr": "max(fs_namesystem_missing_blocks{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})",
"hide": false,
"legendFormat": "mssing",
"legendFormat": "missing",
"range": true,
"refId": "B"
},
Expand Down
2 changes: 1 addition & 1 deletion stacks/signal-processing/jupyterhub.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ options:
cmd: null
image:
# TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that
# the demo is reproducable for the release and it will be automatically replaced for the release branch.
# the demo is reproducible for the release and it will be automatically replaced for the release branch.
name: oci.stackable.tech/demos/jupyter-pyspark-with-alibi-detect
tag: python-3.12
cloudMetadata:
Expand Down
2 changes: 1 addition & 1 deletion stacks/signal-processing/nifi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ spec:
image:
productVersion: 2.12.0
# TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that
# the demo is reproducable for the release and it will be automatically replaced for the release branch.
# the demo is reproducible for the release and it will be automatically replaced for the release branch.
custom: oci.stackable.tech/demos/nifi:2.12.0-postgresql
# pullPolicy: IfNotPresent
clusterConfig:
Expand Down
58 changes: 58 additions & 0 deletions typos.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
# Configuration for typos (https://github.com/crate-ci/typos), run via the prek
# hook in .pre-commit-config.yaml.
#
# Before adding an entry here, consider an in-place marker instead. Use one when
# the word is correct at this one site and would still be a typo elsewhere:
#
# # spellchecker:disable-line at the end of the line it applies to
# # spellchecker:ignore-next-line on its own line, above the offending line
# # spellchecker:off / :on around a block
#
# Every entry below gets a one-line comment saying what the word is.

[files]
# Bare `typos` skips hidden dirs by default, but prek passes explicit paths and
# so does check them. Turn it off so both agree.
ignore-hidden = false

extend-exclude = [
# `.git` itself, which ignore-hidden = false would otherwise pull in.
".git/",
# Generated by generate-embeddings-local.py from a crawl of
# docs.stackable.tech. Typos in it belong to the documentation repository.
"demos/opensearch-rag/embeddings/stackable-docs-embeddings.json",
# Base64-encoded demo key material, not prose.
"stacks/argo-cd-git-ops/secrets/sealed-secrets-key.yaml",
]

[default]
# typos has no native suppression directive
# (https://github.com/crate-ci/typos/issues/316), so these regexes provide one.
# They cover `#`, `//`, `<!-- -->`, `;`, `/* */` and Jinja `{# #}` comments,
# which spans every file type in this repo.
#
# Both failure modes are safe:
# * unterminated `:off` suppresses nothing rather than swallowing the rest of the file.
# * `disable-line` only matches when the marker ends the line.
extend-ignore-re = [
"(?Rm)^.*(#|//|<!--|;|/\\*)\\s*spellchecker:disable-line\\s*(-->|#\\}|\\*/)?\\s*$",
"(#|//|<!--|;|/\\*)\\s*spellchecker:ignore-next-line\\s*(-->|#\\}|\\*/)?\\s*\\n.*",
"(?s)(#|//|<!--|;|/\\*|\")\\s*spellchecker:off\\s*(-->|#\\}|\\*/|\")?.*?(#|//|<!--|;|/\\*|\")\\s*spellchecker:on\\s*(-->|#\\}|\\*/|\")?",
# Random Kafka topic IDs in `kafka-topics.sh --describe` output. They sit
# inside AsciiDoc listing blocks, where a marker would render verbatim.
"TopicId:\\s*\\S+",
]

[type.jupyter]
# Random notebook cell IDs. JSON has no comments, so no marker can go here.
extend-ignore-re = [
"\"id\": \"[0-9a-z]+\"",
]

[default.extend-words]
# Azure Kubernetes Service. Appears in docs/modules/demos/pages/argo-cd-git-ops.adoc.
# A single lowercase entry covers every casing, so no separate `AKS` entry is needed.
aks = "aks"
# Apache Druid's ingestion row-stats field name, referenced verbatim in the
# Druid demo docs.
unparseable = "unparseable"
Loading