diff --git a/CHANGELOG.md b/CHANGELOG.md index 2845c927..6a7ab30d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Added + +- BREAKING: Enable auto-purge feature for data directory cleanup ([#1068]). + ### Changed - Internal operator refactoring: introduce a build() step in the reconciler that @@ -17,6 +21,7 @@ All notable changes to this project will be documented in this file. [#1053]: https://github.com/stackabletech/zookeeper-operator/pull/1053 [#1060]: https://github.com/stackabletech/zookeeper-operator/pull/1060 [#1063]: https://github.com/stackabletech/zookeeper-operator/pull/1063 +[#1068]: https://github.com/stackabletech/zookeeper-operator/pull/1068 ## [26.7.0] - 2026-07-21 diff --git a/deploy/helm/chart_testing.yaml b/deploy/helm/chart_testing.yaml index 82b39c26..253af46d 100644 --- a/deploy/helm/chart_testing.yaml +++ b/deploy/helm/chart_testing.yaml @@ -1,3 +1,4 @@ +--- remote: origin target-branch: main chart-dirs: diff --git a/docs/modules/zookeeper/pages/data_directory.adoc b/docs/modules/zookeeper/pages/data_directory.adoc new file mode 100644 index 00000000..d95899c5 --- /dev/null +++ b/docs/modules/zookeeper/pages/data_directory.adoc @@ -0,0 +1,39 @@ += Data directory +:description: How ZooKeeper manages its data directory and which zoo.cfg properties control it. +:zk-admin-advanced: https://zookeeper.apache.org/doc/r3.9.5/zookeeperAdmin.html#sc_advancedConfiguration + +This page is meant as a general explanation of how ZooKeeper manages the files in its data directory over time. +To see a simple list of the properties the operator sets in relation to that and their defaults, see xref:reference/zoo_cfg_properties.adoc#data-dir-props[`zoo.cfg` properties reference]. + +== Data directory files + +The data directory is located at the path specified by `dataDir` in `zoo.cfg`. +The operator sets this to `/stackable/data` by default. +This directory contains two types of files, which together persist the ZNode data: + +* Transaction log files: An append-only write-ahead log. Every change to the ZNodes is appended here first. +* Snapshot files: A full dump of the current state of all ZNodes. Snapshots let ZooKeeper recover quickly after a restart. + +ZooKeeper can write the transaction logs to a separate directory, configured with `dataLogDir`. +The operator does not set this property, so both file types share `dataDir` and therefore the same volume. + +ZooKeeper pre-allocates its transaction log file in fixed-size blocks of a certain size (`preAllocSize`) so it does not have to grow the file with every write, which keeps writes fast. + +== Snapshot creation and auto-purge + +A snapshot is written and the transaction log rolled to a fresh file when either of two independent limits is reached: + +`snapCount`:: Create snapshot after roughly this many transactions. The operator does not set this property, consult the {zk-admin-advanced}[ZooKeeper Administrator's Guide{external-link-icon}^] for its default. +`snapSizeLimitInKb`:: Create snapshot after the transaction log reaches roughly this size. + +Snapshots and their logs accumulate indefinitely unless the auto-purge feature is enabled. +The operator enables this feature by default. + +There are two settings to control auto-purge: + +`autopurge.purgeInterval`:: Time between cleanup runs, in hours. +`autopurge.snapRetainCount`:: How many of the most recent snapshots to keep, together with the transaction logs belonging to them. The rest are deleted during the cleanup. + +Together these two settings control how large the data directory can grow. +`autopurge.snapRetainCount` sets how much history a cleanup keeps, while `autopurge.purgeInterval` sets how long it takes until the next cleanup, during which new snapshots and logs keep accumulating. +The volume needs to be sized accordingly, see xref:usage_guide/resource_configuration.adoc[]. diff --git a/docs/modules/zookeeper/pages/reference/index.adoc b/docs/modules/zookeeper/pages/reference/index.adoc index eeb572f1..9cf74cba 100644 --- a/docs/modules/zookeeper/pages/reference/index.adoc +++ b/docs/modules/zookeeper/pages/reference/index.adoc @@ -5,3 +5,4 @@ Consult the reference documentation section to find exhaustive information on: * Descriptions and default values of all properties in the CRDs used by this operator in the xref:reference/crds.adoc[]. * The properties in the xref:reference/discovery.adoc[]. * The xref:reference/commandline-parameters.adoc[] and xref:reference/environment-variables.adoc[] accepted by the operator. +* The default settings for `zoo.cfg` properties in xref:reference/zoo_cfg_properties.adoc[]. diff --git a/docs/modules/zookeeper/pages/reference/zoo_cfg_properties.adoc b/docs/modules/zookeeper/pages/reference/zoo_cfg_properties.adoc new file mode 100644 index 00000000..eb39fb81 --- /dev/null +++ b/docs/modules/zookeeper/pages/reference/zoo_cfg_properties.adoc @@ -0,0 +1,40 @@ += `zoo.cfg` properties +:description: Reference for the zoo.cfg properties set by the Stackable Operator for Apache ZooKeeper. + +This is a reference page for `zoo.cfg` properties set by the operator. + +[[data-dir-props]] +== Data directory properties + +The following `zoo.cfg` properties control how ZooKeeper manages the files in its data directory. +You can override them using xref:usage_guide/overrides.adoc[configOverrides]. + +[cols="2,1,2,3",options="header"] +|=== +| Property | Unit | Default set by the operator | Description + +| `dataDir` +| path +| `/stackable/data` +| The location where ZooKeeper stores the snapshots of its in-memory database and the transaction log of updates to that database. + +| `autopurge.purgeInterval` +| hours +| `6` +| Time between cleanup runs. `0` disables auto-purge. + +| `autopurge.snapRetainCount` +| count +| `3` +| Number of recent snapshots (and their transaction logs) to keep. Minimum `3`. + +| `snapSizeLimitInKb` +| KiB +| `102400` (= 100 MiB) +| Take a snapshot after the log reaches roughly this size. A non-positive value disables this trigger. + +| `preAllocSize` +| KiB +| `16384` (= 16 MiB) +| Block size in which the transaction log file is pre-allocated. +|=== diff --git a/docs/modules/zookeeper/pages/usage_guide/resource_configuration.adoc b/docs/modules/zookeeper/pages/usage_guide/resource_configuration.adoc index ee6018b7..f40bea6c 100644 --- a/docs/modules/zookeeper/pages/usage_guide/resource_configuration.adoc +++ b/docs/modules/zookeeper/pages/usage_guide/resource_configuration.adoc @@ -22,6 +22,9 @@ In the above example, all ZooKeeper nodes in the default group will store data ( You can also configure which StorageClass to use, consult the xref:concepts:resources.adoc#storageclass[resources docs] to learn more. +ZooKeeper stores both its snapshots and transaction logs on this volume. +See the xref:data_directory.adoc[data directory docs] for how that data grows and is reclaimed over time. + == Resource requests include::home:concepts:stackable_resource_requests.adoc[] diff --git a/docs/modules/zookeeper/partials/nav.adoc b/docs/modules/zookeeper/partials/nav.adoc index bd21ee88..7554c825 100644 --- a/docs/modules/zookeeper/partials/nav.adoc +++ b/docs/modules/zookeeper/partials/nav.adoc @@ -3,6 +3,7 @@ ** xref:zookeeper:getting_started/first_steps.adoc[] * Concepts ** xref:zookeeper:znodes.adoc[] +** xref:zookeeper:data_directory.adoc[] * xref:zookeeper:usage_guide/index.adoc[] ** xref:zookeeper:usage_guide/listenerclass.adoc[] ** xref:zookeeper:usage_guide/encryption.adoc[] @@ -25,3 +26,4 @@ ** xref:zookeeper:reference/discovery.adoc[] ** xref:zookeeper:reference/commandline-parameters.adoc[] ** xref:zookeeper:reference/environment-variables.adoc[] +** xref:zookeeper:reference/zoo_cfg_properties.adoc[] diff --git a/rust/operator-binary/src/zk_controller/build/properties/zoo_cfg.rs b/rust/operator-binary/src/zk_controller/build/properties/zoo_cfg.rs index b7230c1b..e5dd1483 100644 --- a/rust/operator-binary/src/zk_controller/build/properties/zoo_cfg.rs +++ b/rust/operator-binary/src/zk_controller/build/properties/zoo_cfg.rs @@ -22,6 +22,15 @@ const DEFAULT_INIT_LIMIT: &str = "5"; const DEFAULT_SYNC_LIMIT: &str = "2"; const DEFAULT_TICK_TIME: &str = "3000"; +const AUTOPURGE_SNAP_RETAIN_COUNT_KEY: &str = "autopurge.snapRetainCount"; +const DEFAULT_AUTOPURGE_SNAP_RETAIN_COUNT: &str = "3"; +const AUTOPURGE_PURGE_INTERVAL_KEY: &str = "autopurge.purgeInterval"; +const DEFAULT_AUTOPURGE_PURGE_INTERVAL: &str = "6"; // in hours +const SNAP_SIZE_LIMIT_IN_KB_KEY: &str = "snapSizeLimitInKb"; +const DEFAULT_SNAP_SIZE_LIMIT_IN_KB: &str = "102400"; // 100MiB +const PRE_ALLOC_SIZE_KEY: &str = "preAllocSize"; +const DEFAULT_PRE_ALLOC_SIZE: &str = "16384"; // 16MiB + /// Builds the `server.` quorum entries for `zoo.cfg` from the expected pods. /// /// The pods are predicted from the validated role-group configs (`replicas` + `myidOffset`) @@ -114,6 +123,22 @@ fn build_base( METRICS_PROVIDER_HTTP_PORT_KEY.to_string(), METRICS_PROVIDER_HTTP_PORT.to_string(), ); + zoo_cfg.insert( + AUTOPURGE_SNAP_RETAIN_COUNT_KEY.to_string(), + DEFAULT_AUTOPURGE_SNAP_RETAIN_COUNT.to_string(), + ); + zoo_cfg.insert( + AUTOPURGE_PURGE_INTERVAL_KEY.to_string(), + DEFAULT_AUTOPURGE_PURGE_INTERVAL.to_string(), + ); + zoo_cfg.insert( + SNAP_SIZE_LIMIT_IN_KB_KEY.to_string(), + DEFAULT_SNAP_SIZE_LIMIT_IN_KB.to_string(), + ); + zoo_cfg.insert( + PRE_ALLOC_SIZE_KEY.to_string(), + DEFAULT_PRE_ALLOC_SIZE.to_string(), + ); // 2. TLS / quorum settings. zoo_cfg.extend(security.config_settings()); diff --git a/tests/templates/kuttl/smoke/10-assert.yaml b/tests/templates/kuttl/smoke/10-assert.yaml index 1b6cb482..3db663b2 100644 --- a/tests/templates/kuttl/smoke/10-assert.yaml +++ b/tests/templates/kuttl/smoke/10-assert.yaml @@ -6,3 +6,4 @@ metadata: timeout: 300 commands: - script: kubectl -n $NAMESPACE wait --for=condition=available=true zookeeperclusters.zookeeper.stackable.tech/test-zk --timeout 301s + - script: kubectl -n $NAMESPACE wait --for=create configmap/test-znode --timeout 301s diff --git a/tests/templates/kuttl/smoke/14-assert.yaml.j2 b/tests/templates/kuttl/smoke/14-assert.yaml.j2 index 804d6a41..49ddadf7 100644 --- a/tests/templates/kuttl/smoke/14-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/14-assert.yaml.j2 @@ -322,6 +322,8 @@ commands: zoo.cfg: | admin.serverPort=8080 authProvider.x509=org.apache.zookeeper.server.auth.X509AuthenticationProvider + autopurge.purgeInterval=6 + autopurge.snapRetainCount=3 {% if use_client_tls %} client.portUnification=true {% endif %} @@ -330,6 +332,7 @@ commands: initLimit=5 metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider metricsProvider.httpPort=7000 + preAllocSize=16384 prop.common=group prop.group=group prop.role=role @@ -337,6 +340,7 @@ commands: server.11=test-zk-server-primary-1.test-zk-server-primary-headless.__NAMESPACE__.svc.cluster.local\:2888\:3888;{{ zk_client_port }} server.20=test-zk-server-secondary-0.test-zk-server-secondary-headless.__NAMESPACE__.svc.cluster.local\:2888\:3888;{{ zk_client_port }} serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory + snapSizeLimitInKb=102400 {% if use_client_auth %} ssl.clientAuth=need {% endif %} @@ -601,6 +605,8 @@ commands: zoo.cfg: | admin.serverPort=8080 authProvider.x509=org.apache.zookeeper.server.auth.X509AuthenticationProvider + autopurge.purgeInterval=6 + autopurge.snapRetainCount=3 {% if use_client_tls %} client.portUnification=true {% endif %} @@ -609,12 +615,14 @@ commands: initLimit=5 metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider metricsProvider.httpPort=7000 + preAllocSize=16384 prop.common=role prop.role=role server.10=test-zk-server-primary-0.test-zk-server-primary-headless.__NAMESPACE__.svc.cluster.local\:2888\:3888;{{ zk_client_port }} server.11=test-zk-server-primary-1.test-zk-server-primary-headless.__NAMESPACE__.svc.cluster.local\:2888\:3888;{{ zk_client_port }} server.20=test-zk-server-secondary-0.test-zk-server-secondary-headless.__NAMESPACE__.svc.cluster.local\:2888\:3888;{{ zk_client_port }} serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory + snapSizeLimitInKb=102400 {% if use_client_auth %} ssl.clientAuth=need {% endif %}