fix(store): size the RocksDB memory budget from available memory

.github/scripts/ci-policy.py

@@ -101,6 +101,9 @@
     TRAVIS + "test-check-jacoco-report.sh": {"struct"},
     TRAVIS + "test-codecov-upload-config.sh": {"struct"},
     TRAVIS + "test-riscv64-java-runtime.sh": {"server", "riscv64"},
+    TRAVIS + "test-store-docker-entrypoint.sh": {"store"},
+    TRAVIS + "test-start-hugegraph-store.sh": {"store"},
+    TRAVIS + "test-start-server.sh": {"server"},
 }
 
 

.github/scripts/test_ci_policy.py

@@ -356,14 +356,17 @@ def test_contract_scripts_reach_their_actual_inner_jobs_and_steps(self):
                  ("test-check-java-runtime-logs.sh", {"struct"}),
                  ("test-check-jacoco-report.sh", {"struct"}),
                  ("test-codecov-upload-config.sh", {"struct"}),
-                 ("test-riscv64-java-runtime.sh", {"server", "riscv64"})]
+                 ("test-riscv64-java-runtime.sh", {"server", "riscv64"}),
+                 ("test-store-docker-entrypoint.sh", {"store"}),
+                 ("test-start-hugegraph-store.sh", {"store"}),
+                 ("test-start-server.sh", {"server"})]
         for name, suites in cases:
             script = policy.TRAVIS + name
             with self.subTest(script=script):
                 plan = self.plan_for([script])
                 self.assertEqual(suites, set(plan["selected"]))
                 callers = self.coordinator_jobs(plan)
-                if "struct" in suites:
+                if "struct" in suites or "store" in suites:
                     self.assertEqual({"pd_store"}, callers)
                     root = (workflows / "server-ci.yml").read_text()
                     caller = re.search(r"^  pd_store:\n(.*?)(?=^  [\w-]+:|\Z)", root, re.M | re.S).group(1)
@@ -373,13 +376,20 @@ def test_contract_scripts_reach_their_actual_inner_jobs_and_steps(self):
                         binding = binding.replace("needs.preflight.outputs." + key, repr(str(plan[key]).lower()))
                         inputs[key] = eval(binding, {"__builtins__": {}})
                     workflow = (workflows / "pd-store-ci.yml").read_text()
-                    struct = workflow.split("\n  struct:\n", 1)[1].split("\n  pd:\n", 1)[0]
-                    condition = re.search(r"^    if: (.+)$", struct, re.M).group(1)
+                    job = "struct" if "struct" in suites else "store"
+                    body = re.search(r"^  " + job + r":\n(.*?)(?=^  [\w-]+:|\Z)", workflow, re.M | re.S).group(1)
+                    condition = re.search(r"^    if: (.+)$", body, re.M).group(1)
                     condition = condition.replace("inputs.run-struct", str(inputs["struct"]))
                     condition = condition.replace("inputs.run-store", str(inputs["store"]))
                     self.assertTrue(eval(condition.replace("||", "or"), {"__builtins__": {}}))
-                    step = next(step for step in struct.split("\n      - ") if Path(script).name in step)
-                    self.assertNotRegex(step, r"(?m)^        if:")
+                    step = next(step for step in body.split("\n      - ") if Path(script).name in step)
+                    gate = re.search(r"(?m)^        if: (.+)$", step)
+                    if gate:
+                        # Only the job's own tool preflight may gate a contract step, and that
+                        # preflight step must itself always run
+                        self.assertEqual("steps.store-preflight.outputs.can_run == 'true'", gate.group(1))
+                        preflight = next(s for s in body.split("\n      - ") if "id: store-preflight" in s)
+                        self.assertNotRegex(preflight, r"(?m)^        if:")
                 else:
                     self.assertIn("server-rocksdb", callers)
                     workflow = (workflows / "server-ci.yml").read_text()

.github/workflows/pd-store-ci.yml

@@ -412,6 +412,10 @@ jobs:
         run: |
           mvn editorconfig:check -pl hugegraph-store/hg-store-test -am -ntp
 
+      - name: Run Store docker entrypoint rocksdb memory tests
+        run: |
+          $TRAVIS_DIR/test-store-docker-entrypoint.sh
+
       - name: Check startup test prerequisites (Store)
         id: store-preflight
         run: |

helm/hugegraph/values-cluster.yaml

@@ -57,9 +57,11 @@ store:
     -XX:MaxMetaspaceSize=256m
     -XX:MaxDirectMemorySize=512m
     -XX:+UseContainerSupport
-  # Container limits do not configure JVM heap or RocksDB native cache budgets.
-  # Size this preset for the workload and configure those application budgets
-  # separately; javaOpts accepts explicit JVM/system-property overrides.
+  # Container limits do not configure the JVM heap; javaOpts does. Store images
+  # with auto-sized RocksDB memory (apache/hugegraph#3254) give RocksDB half of
+  # the container limit left after heap, direct memory and a 1.5 GiB reserve;
+  # with this preset that is 2.5 GiB. Set store.extraEnv
+  # HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE (bytes) to use a different value.
   resources:
     requests:
       cpu: "1"

hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh

@@ -172,14 +172,23 @@ JAVA
     printf '%s\n' '#!/bin/sh' 'echo launcher-test' > "$fixture/jdk/bin/uname"
     chmod +x "$fixture/jdk/bin/"* "$fixture/bin/start-hugegraph-store.sh"
     for option in default custom-java-options user-options default-serial default-zgc \
-                  g1 G1 zgc ZGC invalid missing-module-options; do
+                  g1 G1 zgc ZGC direct-default direct-heap-lower direct-heap-raised \
+                  direct-heap-tab direct-heap-alias direct-explicit-cap \
+                  invalid missing-module-options; do
         capture="$fixture/$option.args"
         local -a args=(-d false)
         jvm_options="-Xms64m -Xmx64m"
         case "$option" in
             default) jvm_options="" ;;
             custom-java-options) jvm_options="$jvm_options -Dlauncher.test.custom=true" ;;
             user-options) jvm_options=""; args+=(-j "-Xmx96m -Dlauncher.test=true") ;;
+            # The direct-memory cap follows the heap the JVM actually gets (calc_xmx: 1024)
+            direct-default) jvm_options="" ;;
+            direct-heap-lower) jvm_options=""; args+=(-j "-Xmx512m") ;;
+            direct-heap-raised) jvm_options=""; args+=(-j "-Dlauncher.test=true -Xmx4g") ;;
+            direct-heap-tab) jvm_options=""; args+=(-j $'-Dlauncher.test=true\t-Xmx4g') ;;
+            direct-heap-alias) jvm_options=""; args+=(-j "-XX:MaxHeapSize=4g") ;;
+            direct-explicit-cap) jvm_options=""; args+=(-j "-XX:MaxDirectMemorySize=256m") ;;
             missing-module-options) rm "$fixture/bin/jvm-module.options" ;;
             default-serial) jvm_options="$jvm_options -XX:+UseSerialGC" ;;
             default-zgc) jvm_options="$jvm_options -XX:+UseZGC" ;;
@@ -217,6 +226,21 @@ JAVA
         elif [[ "$option" == user-options ]] &&
              ! grep -Fxq -- '-Dlauncher.test=true' "$capture"; then
             fail "launcher dropped custom -j options"
+        elif [[ "$option" == direct-* ]]; then
+            local cap want
+            # The JVM uses the last -XX:MaxDirectMemorySize on the command line
+            cap=$(grep -E '^-XX:MaxDirectMemorySize=' "$capture" | tail -1)
+            case "$option" in
+                direct-default) want='-XX:MaxDirectMemorySize=1024m' ;;
+                direct-explicit-cap) want='-XX:MaxDirectMemorySize=256m' ;;
+                # A heap-only -j leaves the cap to the JVM, which defaults it to that heap
+                *) want='' ;;
+            esac
+            if [[ "$cap" == "$want" ]]; then
+                pass "launcher '$option' sets direct memory to '${want:-the JVM default (-j heap)}'"
+            else
+                fail "launcher '$option' set direct memory to '${cap:-none}', expected '${want:-none}'"
+            fi
         elif [[ "$option" == zgc || "$option" == ZGC ]]; then
             if grep -Fxq -- '-XX:+UseZGC' "$capture" && \
                ! grep -Fxq -- '-XX:InitiatingHeapOccupancyPercent=50' "$capture"; then

hugegraph-server/hugegraph-dist/src/assembly/travis/test-store-docker-entrypoint.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Checks that the Store Docker entrypoint passes HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE
+# into SPRING_APPLICATION_JSON as rocksdb.total_memory_size, leaves the rocksdb key
+# out when the variable is unset (so conf/application-pd.yml or auto-sizing applies),
+# and refuses values that are not a plain number of bytes.
+
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../.." && pwd)"
+ENTRYPOINT="${REPO_ROOT}/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh"
+TMP_DIR=$(mktemp -d "${TMPDIR:-/tmp}/store-entrypoint-test.XXXXXX")
+trap 'rm -rf "${TMP_DIR}"' EXIT
+
+PASS=0
+FAIL=0
+
+[[ -f "${ENTRYPOINT}" ]] || { echo "entrypoint not found at ${ENTRYPOINT}" >&2; exit 1; }
+command -v python3 >/dev/null || { echo "python3 required" >&2; exit 1; }
+
+mkdir -p "${TMP_DIR}/bin"
+cp "${ENTRYPOINT}" "${TMP_DIR}/docker-entrypoint.sh"
+# Stand in for the launcher: record the generated config instead of starting Store
+cat > "${TMP_DIR}/bin/start-hugegraph-store.sh" <<'STUB'
+#!/usr/bin/env bash
+printf '%s' "${SPRING_APPLICATION_JSON}" > ./spring.json
+STUB
+chmod +x "${TMP_DIR}/bin/start-hugegraph-store.sh" "${TMP_DIR}/docker-entrypoint.sh"
+
+run_entrypoint() {
+    rm -f "${TMP_DIR}/spring.json"
+    # Drop any value from the caller's environment so "unset" really is unset
+    (cd "${TMP_DIR}" && env -u HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE \
+        HG_STORE_PD_ADDRESS=pd0:8686 \
+        HG_STORE_GRPC_HOST=store0 \
+        HG_STORE_RAFT_ADDRESS=store0:8510 \
+        "$@" \
+        ./docker-entrypoint.sh 2>&1)
+}
+
+expect_memory() {
+    # expect_memory <name> <expected value, or "absent"> [NAME=VALUE ...]
+    local name="$1" expected="$2"
+    shift 2
+    local out
+    if ! out=$(run_entrypoint "$@"); then
+        echo "  FAIL ${name}: entrypoint exited non-zero"
+        printf '%s\n' "${out}" | tail -3
+        FAIL=$((FAIL + 1))
+        return
+    fi
+    if ! EXPECTED="${expected}" python3 - "${TMP_DIR}/spring.json" <<'PY'
+import json, os, sys
+with open(sys.argv[1], encoding="utf-8") as fh:
+    doc = json.load(fh)
+want = os.environ["EXPECTED"]
+got = doc.get("rocksdb", {}).get("total_memory_size", "absent")
+if got != want:
+    print("  rocksdb.total_memory_size is %r, expected %r" % (got, want))
+    sys.exit(1)
+if doc["app"]["data-path"] != "/hugegraph-store/storage":
+    print("  app.data-path was lost")
+    sys.exit(1)
+PY
+    then
+        echo "  FAIL ${name}: invalid JSON or wrong rocksdb.total_memory_size"
+        FAIL=$((FAIL + 1))
+        return
+    fi
+    echo "  PASS ${name}"
+    PASS=$((PASS + 1))
+}
+
+echo "Store docker-entrypoint rocksdb memory override"
+expect_memory "unset leaves the key out" absent
+expect_memory "empty leaves the key out" absent HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE=
+expect_memory "bytes are passed through" 2147483648 \
+              HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE=2147483648
+expect_memory "0 is passed through for auto-sizing" 0 HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE=0
+
+for bad in '4GB' '-1' '1.5e9' '1 2' '1"}'; do
+    if run_entrypoint HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE="${bad}" >/dev/null; then
+        echo "  FAIL '${bad}' was accepted"
+        FAIL=$((FAIL + 1))
+    else
+        echo "  PASS '${bad}' is refused"
+        PASS=$((PASS + 1))
+    fi
+done
+
+echo "${PASS} passed, ${FAIL} failed"
+[[ "${FAIL}" -eq 0 ]]

hugegraph-store/README.md

@@ -249,7 +249,9 @@ bin/start-hugegraph-store.sh [-g GC_TYPE] [-j "JVM_OPTIONS"] [-d DAEMON]
   set in `JAVA_OPTIONS` apply. Explicit `-g g1` selects G1 even when JVM ergonomics would choose Serial GC;
   unknown values are rejected. When using explicit `-g`, do not select another collector in `JAVA_OPTIONS`
   or `-j`; conflicting selections are rejected by the JVM.
-- `-j`: Custom JVM options (e.g., `-j "-Xmx16g -Xms8g"`)
+- `-j`: Custom JVM options (e.g., `-j "-Xmx16g -Xms8g -XX:MaxDirectMemorySize=2g"`). A `-j` heap replaces the
+  launcher's direct-memory cap, so pass one with it; otherwise the RocksDB budget counts a second heap
+  of direct memory (see the deployment guide's Store Configuration).
 - `-d`: Daemon mode (`true` = daemon, `false` = foreground; default: `true`). Set to `false` when running under Docker or a process supervisor so the container exits if Java dies.
 
 The launcher always loads `bin/jvm-module.options`, including with custom `JAVA_OPTIONS` or `-j`.
@@ -389,6 +391,10 @@ docker run -d \
   hugegraph/store:latest
 ```
 
+Add `-e HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE=<bytes>` to pin the off-heap RocksDB budget.
+Without it the Store sizes the budget from the container memory limit; see
+[Store Configuration](docs/deployment-guide.md#store-configuration) in the deployment guide.
+
 **Exposed Ports**:
 - `8520`: REST API (management, metrics)
 - `8500`: gRPC (client connections)

hugegraph-store/docs/best-practices.md

@@ -85,7 +85,7 @@ app:
 
 **JVM Settings**:
 ```bash
-JAVA_OPTS="-Xms32g -Xmx32g -XX:+UseG1GC -XX:MaxGCPauseMillis=100 -XX:G1ReservePercent=20"
+JAVA_OPTS="-Xms32g -Xmx32g -XX:MaxDirectMemorySize=2g -XX:+UseG1GC -XX:MaxGCPauseMillis=100 -XX:G1ReservePercent=20"
 ```
 
 ### PD Node Recommendations
@@ -252,6 +252,7 @@ store.rpc_timeout=30000            # 30 seconds
 ```bash
 JAVA_OPTS="
   -Xms32g -Xmx32g                   # Fixed heap size
+  -XX:MaxDirectMemorySize=2g        # Direct-memory cap; RocksDB auto-sizing counts it
   -XX:+UseG1GC                      # G1 garbage collector
   -XX:MaxGCPauseMillis=200          # Target pause time
   -XX:G1ReservePercent=20           # Reserve for to-space
@@ -267,6 +268,7 @@ JAVA_OPTS="
 ```bash
 JAVA_OPTS="
   -Xms32g -Xmx32g
+  -XX:MaxDirectMemorySize=2g        # Direct-memory cap; RocksDB auto-sizing counts it
   -XX:+UseZGC                       # ZGC (sub-10ms pauses)
   -XX:ZCollectionInterval=120       # GC interval (seconds)
   -XX:+UnlockDiagnosticVMOptions