fix(dist): diagnose jvm thread creation failures during startup

docker/README.md

@@ -374,6 +374,30 @@ The launcher never deletes dumps or `heapdump_*` directories. Each start that ge
 
 To turn heap dumps off for every JVM, set `JAVA_TOOL_OPTIONS=-XX:-HeapDumpOnOutOfMemoryError`, which also keeps the image's default `JAVA_OPTS`. Set it in the container's environment, not in the host shell: `docker run -e JAVA_TOOL_OPTIONS=...`, an `environment:` entry on the Server service in Compose (the shipped Compose files do not pass it through), or `env` on the container in Kubernetes. Putting the flag in `JAVA_OPTS` turns dumps off for the Server JVM only, because the JVMs it starts do not see its command line, and setting `JAVA_OPTS` replaces the image default (`-XX:+UseContainerSupport -XX:MaxRAMPercentage=50 ...`), so repeat those flags. Either way, each start still creates its empty `heapdump_*` directory, unless creating it failed or the logs path contains a double quote or `%`.
 
+### JVM thread creation failures during startup
+
+If startup reports `pthread_create failed (EPERM)`, the host may be denying the
+JVM's thread-creation syscall through its seccomp profile or another security
+policy. HotSpot prints this warning to its stdout (the container log with
+`STDOUT_MODE=true`), while its crash file may contain only the generic
+`Cannot create worker GC thread` message. After an initialization or Server
+readiness failure, the entrypoint checks new crash files and newly appended
+Server log output and prints possible causes without changing the exit code.
+With `STDOUT_MODE=false`, it scans newly written `logs/hugegraph-server-stdout.log`
+bytes before crash files, so a generic crash header cannot hide the EPERM warning.
+Check the preceding JVM output for the specific error. Crash files left by
+previous starts do not trigger the hint.
+
+To confirm a seccomp restriction, retry the same container with
+`--security-opt seccomp=unconfined`. Use a compatible Docker/runtime seccomp profile
+for normal operation. EPERM indicates a security denial, not a process-limit
+failure. If the JVM instead reports EAGAIN, inspect process/thread limits and
+use `--ulimit nproc=65535:65535` if appropriate. A generic
+`Cannot create worker GC thread` message alone does not distinguish seccomp
+denial from insufficient memory or process/thread limits. Check the container's
+memory budget and the preceding JVM output. The entrypoint changes neither
+the host security policy nor the container limits.
+
 ## Developers
 
 ### Images and Compose files

hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh

@@ -51,6 +51,8 @@ fi
 EOF
 cat > "${TEST_HOME}/bin/util.sh" <<'EOF'
 # Configuration tests isolate readiness; process lifecycle is tested separately.
+record_jvm_startup_diagnostics() { :; }
+diagnose_jvm_startup_failure() { :; }
 read_property() { sed -n "s/^$2=//p" "$1"; }
 wait_for_startup() { return 0; }
 EOF

hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh

@@ -26,6 +26,17 @@ mkdir -p "${DOCKER_FOLDER}"
 
 log() { echo "[hugegraph-server-entrypoint] $*"; }
 
+source ./bin/util.sh
+record_jvm_startup_diagnostics
+run_startup_step() {
+    local status=0
+    "$@" || status=$?
+    if [[ "$status" -ne 0 ]]; then
+        diagnose_jvm_startup_failure
+    fi
+    return "$status"
+}
+
 # Keep typed/list mappings on their existing encoder; auth strings use native-reader escapes below.
 # TODO: relax Helm's password-space guard once its minimum Server image includes this auth encoder.
 encode_prop_value() {
@@ -296,7 +307,7 @@ if [[ ! -f "${INIT_MARKER_PATH}" ]]; then
 
     if [[ -z "${PASSWORD:-}" ]]; then
         log "init hugegraph with non-auth mode"
-        ./bin/init-store.sh
+        run_startup_step ./bin/init-store.sh
     else
         log "init hugegraph with auth mode"
         # init-store reads the password from stdin, and a disabled one returns
@@ -307,7 +318,7 @@ if [[ ! -f "${INIT_MARKER_PATH}" ]]; then
                     "the entrypoint applies it through 'auth.admin_pa' for" \
                     "the PD startup path" ;;
         esac
-        printf '%s\n' "${PASSWORD}" | ./bin/init-store.sh
+        printf '%s\n' "${PASSWORD}" | run_startup_step ./bin/init-store.sh
     fi
 else
     log "HugeGraph initialization already done. Revalidating the config..."
@@ -317,12 +328,11 @@ else
     # says nothing about whether the admin the current config relies on is
     # reachable. An enabled one returns at the marker, before it touches the
     # backend or reads stdin, so neither wait-storage nor PASSWORD is needed.
-    ./bin/init-store.sh
+    run_startup_step ./bin/init-store.sh
 fi
 
 # Own the foreground launcher so wait can recover the JVM's actual status.
 # A daemon JVM is not our child; watching its PID with tail loses that status.
-source ./bin/util.sh
 LAUNCHER_PID=""
 READINESS_PID=""
 PARTITION_PID=""
@@ -370,6 +380,7 @@ wait "$READINESS_PID" || READY_STATUS=$?
 READINESS_PID=""
 if [[ "$READY_STATUS" -ne 0 ]]; then
     log "ERROR: HugeGraphServer did not become ready; see the container logs"
+    diagnose_jvm_startup_failure
     if kill -0 "$LAUNCHER_PID" 2>/dev/null; then
         # Keep the startup deadline even if the JVM ignores TERM. Container
         # termination cleans up any remaining processes; don't wait forever.

hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh

@@ -36,6 +36,13 @@ repo = pathlib.Path(sys.argv[1])
 static = repo / 'hugegraph-server/hugegraph-dist/src/assembly/static/bin'
 entrypoint = repo / 'hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh'
 
+denied = subprocess.run(['bash', '-c', 'source "$1"; print_jvm_thread_hint',
+                         'bash', str(static / 'util.sh')],
+                        input='pthread_create failed (EPERM)\n', text=True, capture_output=True)
+assert denied.returncode == 0, denied.stderr
+assert '--security-opt seccomp=unconfined' in denied.stderr, denied.stderr
+assert 'nproc' not in denied.stderr and 'thread limits' not in denied.stderr, denied.stderr
+
 
 def await_file(path, process):
     deadline = time.monotonic() + 5
@@ -49,6 +56,12 @@ with tempfile.TemporaryDirectory(prefix='hugegraph-entrypoint-process-') as dire
     for mode, expected in [('exit0', 0), ('exit42', 42), ('kill137', 137),
                            ('TERM', 143), ('INT', 130), ('groupTERM', 143),
                            ('early42', 42), ('early0', 1), ('timeout', 1),
+                           ('early42-eperm-stdout', 42),
+                           ('early42-file-eperm-stdout', 42), ('early42-file-eperm-log', 42),
+                           ('early42-file-output', 42), ('early42-file-output-no-crash', 42),
+                           ('early42-stale-stdout', 42), ('early42-truncated-stdout', 42),
+                           ('early42-resource', 42), ('early42-stale-file', 42),
+                           ('early42-stale-log', 42), ('early37-init', 37),
                            ('timeout-ignore', 1), ('partition42', 42),
                            ('partitionTERM', 143), ('partitionINT', 130),
                            ('partitiongroupTERM', 143), ('readinessTERM', 143),
@@ -62,6 +75,14 @@ with tempfile.TemporaryDirectory(prefix='hugegraph-entrypoint-process-') as dire
         for name in ['start-hugegraph.sh', 'util.sh']:
             shutil.copy(static / name, work / 'bin' / name)
         (work / 'docker/init_complete').touch()
+        if mode == 'early42-stale-file':
+            stale = work / 'logs/hs_err_pid1.log'
+            stale.write_text('pthread_create failed (EPERM)\n')
+            os.utime(stale, (1, 1))
+        if mode == 'early42-stale-log':
+            (work / 'logs/hugegraph-server.log').write_text('pthread_create failed (EPERM)\n')
+        if mode in ['early42-stale-stdout', 'early42-truncated-stdout']:
+            (work / 'logs/hugegraph-server-stdout.log').write_text('pthread_create failed (EPERM)\n' * 10)
         backend = 'hstore' if mode.startswith('partition') else 'rocksdb'
         (work / 'conf/graphs/hugegraph.properties').write_text('backend=' + backend + '\n')
         (work / 'conf/rest-server.properties').write_text(
@@ -112,6 +133,11 @@ with tempfile.TemporaryDirectory(prefix='hugegraph-entrypoint-process-') as dire
 
             threading.Thread(target=hold_http, daemon=True).start()
         (work / 'bin/init-store.sh').write_text('#!/bin/bash\nexit 0\n')
+        if mode == 'early37-init':
+            (work / 'bin/init-store.sh').write_text(
+                '#!/bin/bash\nprintf \'Failed to start thread "GC Thread#0" - '
+                'pthread_create failed (EPERM)\\n\'\n'
+                'printf "Cannot create worker GC thread\\n" > logs/hs_err_pid37.log\nexit 37\n')
         (work / 'bin/wait-partition.sh').write_text(
             '#!/bin/bash\ntouch ./partition-ran\nexit 7\n')
         if mode.startswith('partition') and mode != 'partition42':
@@ -167,7 +193,27 @@ def stop(signum, frame):
 signal.signal(signal.SIGTERM, signal.SIG_IGN if mode == 'timeout-ignore' else stop)
 (work / 'child-running').touch()
 if mode.startswith('early'):
-    sys.exit(int(mode[5:]))
+    if mode in ['early42-file-output', 'early42-file-output-no-crash', 'early42-truncated-stdout']:
+        (work / 'logs/hugegraph-server-stdout.log').write_text('pthread_create failed (EPERM)\\n')
+        if mode == 'early42-file-output':
+            (work / 'logs/hs_err_pid42.log').write_text('Cannot create worker GC thread\\n')
+    elif mode in ['early42-file-eperm-stdout', 'early42-file-eperm-log']:
+        if mode == 'early42-file-eperm-stdout':
+            target = work / 'logs/hugegraph-server-stdout.log'
+            message = 'Exception in thread "main" java.nio.file.FileSystemException: /data/LOCK: Operation not permitted'
+        else:
+            target = work / 'logs/hugegraph-server.log'
+            message = '[pool-3-thread-1] [ERROR] RocksDBStore - Failed to open RocksDB: lock: Operation not permitted'
+        with target.open('a') as output:
+            output.write(message + '\\n')
+    elif mode in ['early42-eperm-stdout', 'early42-resource']:
+        if mode == 'early42-eperm-stdout':
+            print('Failed to start thread "GC Thread#0" - pthread_create failed (EPERM)', flush=True)
+        (work / 'logs/hs_err_pid42.log').write_text('Cannot create worker GC thread\\n')
+    elif mode == 'early42-stale-log':
+        with (work / 'logs/hugegraph-server.log').open('a') as output:
+            output.write('unrelated startup failure\\n')
+    sys.exit(int(mode[5:].split('-')[0]))
 while not (work / 'child-exit').exists():
     time.sleep(0.02)
 sys.exit(int((work / 'child-exit').read_text()))
@@ -180,6 +226,9 @@ sys.exit(int((work / 'child-exit').read_text()))
         env.update(PATH=str(work / 'tools') + ':' + os.environ['PATH'],
                    TEST_CHILD_MODE=mode, HG_SERVER_STARTUP_TIMEOUT_S='3',
                    TEST_REAL_CURL=shutil.which('curl'), NO_PROXY='127.0.0.1', no_proxy='127.0.0.1')
+        if mode in ['early42-file-output', 'early42-file-output-no-crash',
+                    'early42-stale-stdout', 'early42-truncated-stdout']:
+            env['STDOUT_MODE'] = 'false'
         if mode.startswith('readiness'):
             env['HG_SERVER_STARTUP_TIMEOUT_S'] = '60'
         if mode.startswith('partitionDiagnostic'):
@@ -221,14 +270,32 @@ sys.exit(int((work / 'child-exit').read_text()))
                         (work / 'child-exit').write_text(str(status))
                 actual = process.wait(timeout=5)
                 assert actual == expected, (mode, expected, actual, (work / 'output').read_text())
+                diagnostic = (work / 'output').read_text()
+                if mode in ['early42-file-eperm-stdout', 'early42-file-eperm-log']:
+                    assert 'JVM thread creation was denied' not in diagnostic, diagnostic
+                    assert '--security-opt seccomp=unconfined' not in diagnostic, diagnostic
+                if mode in ['early42-file-output', 'early42-file-output-no-crash', 'early42-truncated-stdout']:
+                    assert 'JVM thread creation was denied (EPERM)' in diagnostic, diagnostic
+                    assert 'Possible causes' not in diagnostic, diagnostic
+                    assert '--ulimit nproc' not in diagnostic, diagnostic
+                elif mode in ['early42-eperm-stdout', 'early42-resource', 'early37-init']:
+                    assert '--security-opt seccomp=unconfined' in diagnostic, diagnostic
+                    assert '--ulimit nproc=65535:65535' in diagnostic, diagnostic
+                    assert 'Possible causes include seccomp denial, insufficient memory' in diagnostic, diagnostic
+                    assert 'logs/hugegraph-server-stdout.log with STDOUT_MODE=false' in diagnostic, diagnostic
+                    assert 'JVM thread creation was denied' not in diagnostic, diagnostic
+                    if mode != 'early42-resource':
+                        assert 'Failed to start thread "GC Thread#0"' in diagnostic, diagnostic
+                else:
+                    assert '--security-opt seccomp=unconfined' not in diagnostic, diagnostic
                 if mode.startswith('timeout'):
                     assert time.monotonic() - started < 5, 'startup cleanup exceeded the deadline'
                 if mode in ['TERM', 'INT', 'groupTERM', 'partitionTERM', 'partitionINT',
                             'partitiongroupTERM', 'readinessTERM', 'readinessINT',
                             'readinessgroupTERM', 'partitionDiagnosticTERM',
                             'partitionDiagnosticINT', 'partitionDiagnosticgroupTERM']:
                     assert (work / 'term-observed').exists(), 'TERM did not reach the JVM substitute'
-                if not mode.startswith('timeout'):
+                if not mode.startswith('timeout') and mode != 'early37-init':
                     owned = ([work / 'bin/pid'] + list(work.glob('partition-*.pid')) +
                              list(work.glob('readiness-*.pid')))
                     for pid_file in owned:

hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh

@@ -16,6 +16,59 @@
 # limitations under the License.
 #
 
+function record_jvm_startup_diagnostics() {
+    JVM_STARTUP_TIME=$(date -u +'%Y-%m-%d %H:%M:%S.%N UTC')
+    JVM_STARTUP_LOG_OFFSET=$(stat -c %s ./logs/hugegraph-server.log 2>/dev/null || echo 0)
+    JVM_STARTUP_STDOUT_OFFSET=$(stat -c %s ./logs/hugegraph-server-stdout.log 2>/dev/null || echo 0)
+}
+
+function print_jvm_thread_hint() {
+    local kind
+    kind=$(awk '
+        /pthread_create.*\(EPERM\)|pthread_create.*Operation not permitted/ { denied = 1 }
+        /Cannot create worker GC thread/ { resources = 1 }
+        END { if (denied) print "denied"; else if (resources) print "resources" }
+    ')
+    case "$kind" in
+        denied)
+            echo 'ERROR: JVM thread creation was denied (EPERM). Check the Docker host seccomp profile and security policy.' >&2
+            echo 'To diagnose seccomp restrictions, retry with --security-opt seccomp=unconfined; use a compatible host/runtime profile for normal operation.' >&2
+            ;;
+        resources)
+            echo 'ERROR: JVM could not create a GC worker thread. Possible causes include seccomp denial, insufficient memory, or process/thread limits.' >&2
+            echo 'Look for pthread_create failed (EPERM) in the JVM output: the container log, or logs/hugegraph-server-stdout.log with STDOUT_MODE=false.' >&2
+            echo 'To diagnose seccomp restrictions, retry with --security-opt seccomp=unconfined; use a compatible host/runtime profile for normal operation.' >&2
+            echo 'If the process limit is too low, set --ulimit nproc=65535:65535.' >&2
+            ;;
+        *) return 1 ;;
+    esac
+}
+
+function diagnose_jvm_startup_failure() {
+    local crash size offset
+    if [[ -r ./logs/hugegraph-server-stdout.log ]]; then
+        size=$(stat -c %s ./logs/hugegraph-server-stdout.log 2>/dev/null || echo 0)
+        offset=$JVM_STARTUP_STDOUT_OFFSET
+        [[ "$size" -ge "$offset" ]] || offset=0
+        if tail -c +$((offset + 1)) ./logs/hugegraph-server-stdout.log | print_jvm_thread_hint; then
+            return 0
+        fi
+    fi
+    while IFS= read -r -d '' crash; do
+        if [[ -r "$crash" ]] && print_jvm_thread_hint < "$crash"; then
+            return 0
+        fi
+    done < <(find -H . ./logs -maxdepth 1 -type f -name 'hs_err_pid*.log' \
+                 -newermt "$JVM_STARTUP_TIME" -print0 2>/dev/null)
+    if [[ -r ./logs/hugegraph-server.log ]]; then
+        size=$(stat -c %s ./logs/hugegraph-server.log 2>/dev/null || echo 0)
+        offset=$JVM_STARTUP_LOG_OFFSET
+        [[ "$size" -ge "$offset" ]] || offset=0
+        tail -c +$((offset + 1)) ./logs/hugegraph-server.log | print_jvm_thread_hint || true
+    fi
+    return 0
+}
+
 function command_available() {
     local cmd=$1
     if command -v "$cmd" >/dev/null 2>&1; then