From fe5adb239eaabc86f5f048fa4ecc1e1df9891d90 Mon Sep 17 00:00:00 2001 From: pallavithegod Date: Fri, 16 Oct 2026 15:47:14 +0530 Subject: [PATCH 1/4] fix(dist): diagnose jvm thread creation failures during startup --- docker/README.md | 16 +++++++ .../docker/docker-entrypoint-test.sh | 2 + .../docker/docker-entrypoint.sh | 19 ++++++-- .../test/test-docker-entrypoint-process.sh | 34 +++++++++++++- .../src/assembly/static/bin/util.sh | 44 +++++++++++++++++++ 5 files changed, 109 insertions(+), 6 deletions(-) diff --git a/docker/README.md b/docker/README.md index 233cd4ef71..d61e2dd7c1 100644 --- a/docker/README.md +++ b/docker/README.md @@ -358,6 +358,22 @@ The launcher never deletes dumps or `heapdump_*` directories. Each start that ge To turn heap dumps off for every JVM, set `JAVA_TOOL_OPTIONS=-XX:-HeapDumpOnOutOfMemoryError`, which also keeps the image's default `JAVA_OPTS`. Set it in the container's environment, not in the host shell: `docker run -e JAVA_TOOL_OPTIONS=...`, an `environment:` entry on the Server service in Compose (the shipped Compose files do not pass it through), or `env` on the container in Kubernetes. Putting the flag in `JAVA_OPTS` turns dumps off for the Server JVM only, because the JVMs it starts do not see its command line, and setting `JAVA_OPTS` replaces the image default (`-XX:+UseContainerSupport -XX:MaxRAMPercentage=50 ...`), so repeat those flags. Either way, each start still creates its empty `heapdump_*` directory, unless creating it failed or the logs path contains a double quote or `%`. +### JVM thread creation failures during startup + +If startup reports `pthread_create failed (EPERM)`, the host may be denying the +JVM's thread-creation syscall through its seccomp profile. The entrypoint checks +new JVM crash files and newly appended Server log output after an initialization +or Server readiness failure, and prints a diagnostic without changing the exit code. +Crash files left by previous starts do not trigger the hint. + +To confirm a seccomp restriction, retry the same container with +`--security-opt seccomp=unconfined`. Use a compatible Docker/runtime seccomp profile +for normal operation. Inspect host process/thread limits too; if they are too low, +set `--ulimit nproc=65535:65535`. A generic `Cannot create worker GC thread` message +can also indicate insufficient memory, so check the container's memory budget +before attributing that error to seccomp. The entrypoint changes neither the +host security policy nor the container limits. + ## Developers ### Images and Compose files diff --git a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh index 68dc76afdf..fa4f9bcfbd 100755 --- a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh +++ b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh @@ -51,6 +51,8 @@ fi EOF cat > "${TEST_HOME}/bin/util.sh" <<'EOF' # Configuration tests isolate readiness; process lifecycle is tested separately. +record_jvm_startup_diagnostics() { :; } +diagnose_jvm_startup_failure() { :; } read_property() { sed -n "s/^$2=//p" "$1"; } wait_for_startup() { return 0; } EOF diff --git a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh index d360d84a68..4835f45e5e 100755 --- a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh +++ b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh @@ -26,6 +26,17 @@ mkdir -p "${DOCKER_FOLDER}" log() { echo "[hugegraph-server-entrypoint] $*"; } +source ./bin/util.sh +record_jvm_startup_diagnostics +run_startup_step() { + local status=0 + "$@" || status=$? + if [[ "$status" -ne 0 ]]; then + diagnose_jvm_startup_failure + fi + return "$status" +} + # Keep typed/list mappings on their existing encoder; auth strings use native-reader escapes below. # TODO: relax Helm's password-space guard once its minimum Server image includes this auth encoder. encode_prop_value() { @@ -296,7 +307,7 @@ if [[ ! -f "${INIT_MARKER_PATH}" ]]; then if [[ -z "${PASSWORD:-}" ]]; then log "init hugegraph with non-auth mode" - ./bin/init-store.sh + run_startup_step ./bin/init-store.sh else log "init hugegraph with auth mode" # init-store reads the password from stdin, and a disabled one returns @@ -307,7 +318,7 @@ if [[ ! -f "${INIT_MARKER_PATH}" ]]; then "the entrypoint applies it through 'auth.admin_pa' for" \ "the PD startup path" ;; esac - printf '%s\n' "${PASSWORD}" | ./bin/init-store.sh + printf '%s\n' "${PASSWORD}" | run_startup_step ./bin/init-store.sh fi else log "HugeGraph initialization already done. Revalidating the config..." @@ -317,12 +328,11 @@ else # says nothing about whether the admin the current config relies on is # reachable. An enabled one returns at the marker, before it touches the # backend or reads stdin, so neither wait-storage nor PASSWORD is needed. - ./bin/init-store.sh + run_startup_step ./bin/init-store.sh fi # Own the foreground launcher so wait can recover the JVM's actual status. # A daemon JVM is not our child; watching its PID with tail loses that status. -source ./bin/util.sh LAUNCHER_PID="" READINESS_PID="" PARTITION_PID="" @@ -370,6 +380,7 @@ wait "$READINESS_PID" || READY_STATUS=$? READINESS_PID="" if [[ "$READY_STATUS" -ne 0 ]]; then log "ERROR: HugeGraphServer did not become ready; see the container logs" + diagnose_jvm_startup_failure if kill -0 "$LAUNCHER_PID" 2>/dev/null; then # Keep the startup deadline even if the JVM ignores TERM. Container # termination cleans up any remaining processes; don't wait forever. diff --git a/hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh b/hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh index 7a3a90e600..171ce724ae 100644 --- a/hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh +++ b/hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh @@ -49,6 +49,9 @@ with tempfile.TemporaryDirectory(prefix='hugegraph-entrypoint-process-') as dire for mode, expected in [('exit0', 0), ('exit42', 42), ('kill137', 137), ('TERM', 143), ('INT', 130), ('groupTERM', 143), ('early42', 42), ('early0', 1), ('timeout', 1), + ('early42-eperm-file', 42), ('early42-eperm-log', 42), + ('early42-resource', 42), ('early42-stale-file', 42), + ('early42-stale-log', 42), ('early37-init', 37), ('timeout-ignore', 1), ('partition42', 42), ('partitionTERM', 143), ('partitionINT', 130), ('partitiongroupTERM', 143), ('readinessTERM', 143), @@ -62,6 +65,12 @@ with tempfile.TemporaryDirectory(prefix='hugegraph-entrypoint-process-') as dire for name in ['start-hugegraph.sh', 'util.sh']: shutil.copy(static / name, work / 'bin' / name) (work / 'docker/init_complete').touch() + if mode == 'early42-stale-file': + stale = work / 'logs/hs_err_pid1.log' + stale.write_text('pthread_create failed (EPERM)\n') + os.utime(stale, (1, 1)) + if mode == 'early42-stale-log': + (work / 'logs/hugegraph-server.log').write_text('pthread_create failed (EPERM)\n') backend = 'hstore' if mode.startswith('partition') else 'rocksdb' (work / 'conf/graphs/hugegraph.properties').write_text('backend=' + backend + '\n') (work / 'conf/rest-server.properties').write_text( @@ -112,6 +121,9 @@ with tempfile.TemporaryDirectory(prefix='hugegraph-entrypoint-process-') as dire threading.Thread(target=hold_http, daemon=True).start() (work / 'bin/init-store.sh').write_text('#!/bin/bash\nexit 0\n') + if mode == 'early37-init': + (work / 'bin/init-store.sh').write_text( + '#!/bin/bash\nprintf "pthread_create failed (EPERM)\\n" > logs/hs_err_pid37.log\nexit 37\n') (work / 'bin/wait-partition.sh').write_text( '#!/bin/bash\ntouch ./partition-ran\nexit 7\n') if mode.startswith('partition') and mode != 'partition42': @@ -167,7 +179,16 @@ def stop(signum, frame): signal.signal(signal.SIGTERM, signal.SIG_IGN if mode == 'timeout-ignore' else stop) (work / 'child-running').touch() if mode.startswith('early'): - sys.exit(int(mode[5:])) + if mode == 'early42-eperm-file': + (work / 'logs/hs_err_pid42.log').write_text('pthread_create failed (EPERM)\\n') + elif mode == 'early42-eperm-log': + (work / 'logs/hugegraph-server.log').write_text('pthread_create failed: Operation not permitted\\n') + elif mode == 'early42-resource': + (work / 'logs/hs_err_pid42.log').write_text('Cannot create worker GC thread\\n') + elif mode == 'early42-stale-log': + with (work / 'logs/hugegraph-server.log').open('a') as output: + output.write('unrelated startup failure\\n') + sys.exit(int(mode[5:].split('-')[0])) while not (work / 'child-exit').exists(): time.sleep(0.02) sys.exit(int((work / 'child-exit').read_text())) @@ -221,6 +242,15 @@ sys.exit(int((work / 'child-exit').read_text())) (work / 'child-exit').write_text(str(status)) actual = process.wait(timeout=5) assert actual == expected, (mode, expected, actual, (work / 'output').read_text()) + diagnostic = (work / 'output').read_text() + if mode in ['early42-eperm-file', 'early42-eperm-log', 'early37-init']: + assert '--security-opt seccomp=unconfined' in diagnostic, diagnostic + assert '--ulimit nproc=65535:65535' in diagnostic, diagnostic + elif mode == 'early42-resource': + assert 'Check available memory' in diagnostic, diagnostic + assert '--security-opt seccomp=unconfined' not in diagnostic, diagnostic + else: + assert '--security-opt seccomp=unconfined' not in diagnostic, diagnostic if mode.startswith('timeout'): assert time.monotonic() - started < 5, 'startup cleanup exceeded the deadline' if mode in ['TERM', 'INT', 'groupTERM', 'partitionTERM', 'partitionINT', @@ -228,7 +258,7 @@ sys.exit(int((work / 'child-exit').read_text())) 'readinessgroupTERM', 'partitionDiagnosticTERM', 'partitionDiagnosticINT', 'partitionDiagnosticgroupTERM']: assert (work / 'term-observed').exists(), 'TERM did not reach the JVM substitute' - if not mode.startswith('timeout'): + if not mode.startswith('timeout') and mode != 'early37-init': owned = ([work / 'bin/pid'] + list(work.glob('partition-*.pid')) + list(work.glob('readiness-*.pid'))) for pid_file in owned: diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh index 65720021eb..9fa9f20466 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh @@ -16,6 +16,50 @@ # limitations under the License. # +function record_jvm_startup_diagnostics() { + JVM_STARTUP_TIME=$(date -u +'%Y-%m-%d %H:%M:%S.%N UTC') + JVM_STARTUP_LOG_OFFSET=$(stat -c %s ./logs/hugegraph-server.log 2>/dev/null || echo 0) +} + +function print_jvm_thread_hint() { + local kind + kind=$(awk ' + /pthread_create.*\(EPERM\)|pthread_create.*Operation not permitted/ { denied = 1 } + /[Tt]hread.*Operation not permitted/ { denied = 1 } + /Cannot create worker GC thread/ { resources = 1 } + END { if (denied) print "denied"; else if (resources) print "resources" } + ') + case "$kind" in + denied) + echo 'ERROR: JVM thread creation was denied (EPERM). Check the Docker host seccomp profile and thread limits.' >&2 + echo 'To diagnose seccomp restrictions, retry with --security-opt seccomp=unconfined; use a compatible host/runtime profile for normal operation.' >&2 + echo 'If the process limit is too low, set --ulimit nproc=65535:65535.' >&2 + ;; + resources) + echo 'ERROR: JVM could not create a GC worker thread. Check available memory and process/thread limits.' >&2 + echo 'If the process limit is too low, set --ulimit nproc=65535:65535.' >&2 + ;; + *) return 1 ;; + esac +} + +function diagnose_jvm_startup_failure() { + local crash size offset + while IFS= read -r -d '' crash; do + if [[ -r "$crash" ]] && print_jvm_thread_hint < "$crash"; then + return 0 + fi + done < <(find -H . ./logs -maxdepth 1 -type f -name 'hs_err_pid*.log' \ + -newermt "$JVM_STARTUP_TIME" -print0 2>/dev/null) + if [[ -r ./logs/hugegraph-server.log ]]; then + size=$(stat -c %s ./logs/hugegraph-server.log 2>/dev/null || echo 0) + offset=$JVM_STARTUP_LOG_OFFSET + [[ "$size" -ge "$offset" ]] || offset=0 + tail -c +$((offset + 1)) ./logs/hugegraph-server.log | print_jvm_thread_hint || true + fi + return 0 +} + function command_available() { local cmd=$1 if command -v "$cmd" >/dev/null 2>&1; then From a24c32d1b3794b302859ecde97158b75de570f3f Mon Sep 17 00:00:00 2001 From: pallavithegod Date: Fri, 16 Oct 2026 18:48:01 +0530 Subject: [PATCH 2/4] fix(dist): clarify gc thread failure hints Cover possible seccomp denial in generic crash-file hints. Keep process-limit advice separate from EPERM denials. Exercise stdout warnings with realistic crash-file fixtures. --- docker/README.md | 24 ++++++++------ .../test/test-docker-entrypoint-process.sh | 31 ++++++++++++------- .../src/assembly/static/bin/util.sh | 7 +++-- 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/docker/README.md b/docker/README.md index d61e2dd7c1..db793d32f6 100644 --- a/docker/README.md +++ b/docker/README.md @@ -361,18 +361,24 @@ To turn heap dumps off for every JVM, set `JAVA_TOOL_OPTIONS=-XX:-HeapDumpOnOutO ### JVM thread creation failures during startup If startup reports `pthread_create failed (EPERM)`, the host may be denying the -JVM's thread-creation syscall through its seccomp profile. The entrypoint checks -new JVM crash files and newly appended Server log output after an initialization -or Server readiness failure, and prints a diagnostic without changing the exit code. -Crash files left by previous starts do not trigger the hint. +JVM's thread-creation syscall through its seccomp profile or another security +policy. HotSpot prints this warning to its stdout (the container log with +`STDOUT_MODE=true`), while its crash file may contain only the generic +`Cannot create worker GC thread` message. After an initialization or Server +readiness failure, the entrypoint checks new crash files and newly appended +Server log output and prints possible causes without changing the exit code. +Check the preceding JVM output for the specific error. Crash files left by +previous starts do not trigger the hint. To confirm a seccomp restriction, retry the same container with `--security-opt seccomp=unconfined`. Use a compatible Docker/runtime seccomp profile -for normal operation. Inspect host process/thread limits too; if they are too low, -set `--ulimit nproc=65535:65535`. A generic `Cannot create worker GC thread` message -can also indicate insufficient memory, so check the container's memory budget -before attributing that error to seccomp. The entrypoint changes neither the -host security policy nor the container limits. +for normal operation. EPERM indicates a security denial, not a process-limit +failure. If the JVM instead reports EAGAIN, inspect process/thread limits and +use `--ulimit nproc=65535:65535` if appropriate. A generic +`Cannot create worker GC thread` message alone does not distinguish seccomp +denial from insufficient memory or process/thread limits. Check the container's +memory budget and the preceding JVM output. The entrypoint changes neither +the host security policy nor the container limits. ## Developers diff --git a/hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh b/hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh index 171ce724ae..0d9f3d6fe3 100644 --- a/hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh +++ b/hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh @@ -36,6 +36,13 @@ repo = pathlib.Path(sys.argv[1]) static = repo / 'hugegraph-server/hugegraph-dist/src/assembly/static/bin' entrypoint = repo / 'hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh' +denied = subprocess.run(['bash', '-c', 'source "$1"; print_jvm_thread_hint', + 'bash', str(static / 'util.sh')], + input='pthread_create failed (EPERM)\n', text=True, capture_output=True) +assert denied.returncode == 0, denied.stderr +assert '--security-opt seccomp=unconfined' in denied.stderr, denied.stderr +assert 'nproc' not in denied.stderr and 'thread limits' not in denied.stderr, denied.stderr + def await_file(path, process): deadline = time.monotonic() + 5 @@ -49,7 +56,7 @@ with tempfile.TemporaryDirectory(prefix='hugegraph-entrypoint-process-') as dire for mode, expected in [('exit0', 0), ('exit42', 42), ('kill137', 137), ('TERM', 143), ('INT', 130), ('groupTERM', 143), ('early42', 42), ('early0', 1), ('timeout', 1), - ('early42-eperm-file', 42), ('early42-eperm-log', 42), + ('early42-eperm-stdout', 42), ('early42-resource', 42), ('early42-stale-file', 42), ('early42-stale-log', 42), ('early37-init', 37), ('timeout-ignore', 1), ('partition42', 42), @@ -123,7 +130,9 @@ with tempfile.TemporaryDirectory(prefix='hugegraph-entrypoint-process-') as dire (work / 'bin/init-store.sh').write_text('#!/bin/bash\nexit 0\n') if mode == 'early37-init': (work / 'bin/init-store.sh').write_text( - '#!/bin/bash\nprintf "pthread_create failed (EPERM)\\n" > logs/hs_err_pid37.log\nexit 37\n') + '#!/bin/bash\nprintf \'Failed to start thread "GC Thread#0" - ' + 'pthread_create failed (EPERM)\\n\'\n' + 'printf "Cannot create worker GC thread\\n" > logs/hs_err_pid37.log\nexit 37\n') (work / 'bin/wait-partition.sh').write_text( '#!/bin/bash\ntouch ./partition-ran\nexit 7\n') if mode.startswith('partition') and mode != 'partition42': @@ -179,11 +188,9 @@ def stop(signum, frame): signal.signal(signal.SIGTERM, signal.SIG_IGN if mode == 'timeout-ignore' else stop) (work / 'child-running').touch() if mode.startswith('early'): - if mode == 'early42-eperm-file': - (work / 'logs/hs_err_pid42.log').write_text('pthread_create failed (EPERM)\\n') - elif mode == 'early42-eperm-log': - (work / 'logs/hugegraph-server.log').write_text('pthread_create failed: Operation not permitted\\n') - elif mode == 'early42-resource': + if mode in ['early42-eperm-stdout', 'early42-resource']: + if mode == 'early42-eperm-stdout': + print('Failed to start thread "GC Thread#0" - pthread_create failed (EPERM)', flush=True) (work / 'logs/hs_err_pid42.log').write_text('Cannot create worker GC thread\\n') elif mode == 'early42-stale-log': with (work / 'logs/hugegraph-server.log').open('a') as output: @@ -243,12 +250,14 @@ sys.exit(int((work / 'child-exit').read_text())) actual = process.wait(timeout=5) assert actual == expected, (mode, expected, actual, (work / 'output').read_text()) diagnostic = (work / 'output').read_text() - if mode in ['early42-eperm-file', 'early42-eperm-log', 'early37-init']: + if mode in ['early42-eperm-stdout', 'early42-resource', 'early37-init']: assert '--security-opt seccomp=unconfined' in diagnostic, diagnostic assert '--ulimit nproc=65535:65535' in diagnostic, diagnostic - elif mode == 'early42-resource': - assert 'Check available memory' in diagnostic, diagnostic - assert '--security-opt seccomp=unconfined' not in diagnostic, diagnostic + assert 'Possible causes include seccomp denial, insufficient memory' in diagnostic, diagnostic + assert 'Look for pthread_create failed (EPERM) above' in diagnostic, diagnostic + assert 'JVM thread creation was denied' not in diagnostic, diagnostic + if mode != 'early42-resource': + assert 'Failed to start thread "GC Thread#0"' in diagnostic, diagnostic else: assert '--security-opt seccomp=unconfined' not in diagnostic, diagnostic if mode.startswith('timeout'): diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh index 9fa9f20466..5bacb08705 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh @@ -31,12 +31,13 @@ function print_jvm_thread_hint() { ') case "$kind" in denied) - echo 'ERROR: JVM thread creation was denied (EPERM). Check the Docker host seccomp profile and thread limits.' >&2 + echo 'ERROR: JVM thread creation was denied (EPERM). Check the Docker host seccomp profile and security policy.' >&2 echo 'To diagnose seccomp restrictions, retry with --security-opt seccomp=unconfined; use a compatible host/runtime profile for normal operation.' >&2 - echo 'If the process limit is too low, set --ulimit nproc=65535:65535.' >&2 ;; resources) - echo 'ERROR: JVM could not create a GC worker thread. Check available memory and process/thread limits.' >&2 + echo 'ERROR: JVM could not create a GC worker thread. Possible causes include seccomp denial, insufficient memory, or process/thread limits.' >&2 + echo 'Look for pthread_create failed (EPERM) above in the container log; if present, check the host seccomp profile and security policy.' >&2 + echo 'To diagnose seccomp restrictions, retry with --security-opt seccomp=unconfined; use a compatible host/runtime profile for normal operation.' >&2 echo 'If the process limit is too low, set --ulimit nproc=65535:65535.' >&2 ;; *) return 1 ;; From ae3e701cc0a599b57379ccd5010ea60a5fbe5e74 Mon Sep 17 00:00:00 2001 From: pallavithegod Date: Sun, 18 Oct 2026 16:07:06 +0530 Subject: [PATCH 3/4] fix(dist): inspect redirected jvm startup output Scan new stdout-file bytes before generic crash headers. Ignore stale output and handle truncation between starts. Cover file-output startup failures with and without crash files. --- docker/README.md | 2 ++ .../test/test-docker-entrypoint-process.sh | 21 ++++++++++++++++--- .../src/assembly/static/bin/util.sh | 11 +++++++++- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/docker/README.md b/docker/README.md index db793d32f6..c9cac00142 100644 --- a/docker/README.md +++ b/docker/README.md @@ -367,6 +367,8 @@ policy. HotSpot prints this warning to its stdout (the container log with `Cannot create worker GC thread` message. After an initialization or Server readiness failure, the entrypoint checks new crash files and newly appended Server log output and prints possible causes without changing the exit code. +With `STDOUT_MODE=false`, it scans newly written `logs/hugegraph-server-stdout.log` +bytes before crash files, so a generic crash header cannot hide the EPERM warning. Check the preceding JVM output for the specific error. Crash files left by previous starts do not trigger the hint. diff --git a/hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh b/hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh index 0d9f3d6fe3..05eb40abc0 100644 --- a/hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh +++ b/hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh @@ -57,6 +57,8 @@ with tempfile.TemporaryDirectory(prefix='hugegraph-entrypoint-process-') as dire ('TERM', 143), ('INT', 130), ('groupTERM', 143), ('early42', 42), ('early0', 1), ('timeout', 1), ('early42-eperm-stdout', 42), + ('early42-file-output', 42), ('early42-file-output-no-crash', 42), + ('early42-stale-stdout', 42), ('early42-truncated-stdout', 42), ('early42-resource', 42), ('early42-stale-file', 42), ('early42-stale-log', 42), ('early37-init', 37), ('timeout-ignore', 1), ('partition42', 42), @@ -78,6 +80,8 @@ with tempfile.TemporaryDirectory(prefix='hugegraph-entrypoint-process-') as dire os.utime(stale, (1, 1)) if mode == 'early42-stale-log': (work / 'logs/hugegraph-server.log').write_text('pthread_create failed (EPERM)\n') + if mode in ['early42-stale-stdout', 'early42-truncated-stdout']: + (work / 'logs/hugegraph-server-stdout.log').write_text('pthread_create failed (EPERM)\n' * 10) backend = 'hstore' if mode.startswith('partition') else 'rocksdb' (work / 'conf/graphs/hugegraph.properties').write_text('backend=' + backend + '\n') (work / 'conf/rest-server.properties').write_text( @@ -188,7 +192,11 @@ def stop(signum, frame): signal.signal(signal.SIGTERM, signal.SIG_IGN if mode == 'timeout-ignore' else stop) (work / 'child-running').touch() if mode.startswith('early'): - if mode in ['early42-eperm-stdout', 'early42-resource']: + if mode in ['early42-file-output', 'early42-file-output-no-crash', 'early42-truncated-stdout']: + (work / 'logs/hugegraph-server-stdout.log').write_text('pthread_create failed (EPERM)\\n') + if mode == 'early42-file-output': + (work / 'logs/hs_err_pid42.log').write_text('Cannot create worker GC thread\\n') + elif mode in ['early42-eperm-stdout', 'early42-resource']: if mode == 'early42-eperm-stdout': print('Failed to start thread "GC Thread#0" - pthread_create failed (EPERM)', flush=True) (work / 'logs/hs_err_pid42.log').write_text('Cannot create worker GC thread\\n') @@ -208,6 +216,9 @@ sys.exit(int((work / 'child-exit').read_text())) env.update(PATH=str(work / 'tools') + ':' + os.environ['PATH'], TEST_CHILD_MODE=mode, HG_SERVER_STARTUP_TIMEOUT_S='3', TEST_REAL_CURL=shutil.which('curl'), NO_PROXY='127.0.0.1', no_proxy='127.0.0.1') + if mode in ['early42-file-output', 'early42-file-output-no-crash', + 'early42-stale-stdout', 'early42-truncated-stdout']: + env['STDOUT_MODE'] = 'false' if mode.startswith('readiness'): env['HG_SERVER_STARTUP_TIMEOUT_S'] = '60' if mode.startswith('partitionDiagnostic'): @@ -250,11 +261,15 @@ sys.exit(int((work / 'child-exit').read_text())) actual = process.wait(timeout=5) assert actual == expected, (mode, expected, actual, (work / 'output').read_text()) diagnostic = (work / 'output').read_text() - if mode in ['early42-eperm-stdout', 'early42-resource', 'early37-init']: + if mode in ['early42-file-output', 'early42-file-output-no-crash', 'early42-truncated-stdout']: + assert 'JVM thread creation was denied (EPERM)' in diagnostic, diagnostic + assert 'Possible causes' not in diagnostic, diagnostic + assert '--ulimit nproc' not in diagnostic, diagnostic + elif mode in ['early42-eperm-stdout', 'early42-resource', 'early37-init']: assert '--security-opt seccomp=unconfined' in diagnostic, diagnostic assert '--ulimit nproc=65535:65535' in diagnostic, diagnostic assert 'Possible causes include seccomp denial, insufficient memory' in diagnostic, diagnostic - assert 'Look for pthread_create failed (EPERM) above' in diagnostic, diagnostic + assert 'logs/hugegraph-server-stdout.log with STDOUT_MODE=false' in diagnostic, diagnostic assert 'JVM thread creation was denied' not in diagnostic, diagnostic if mode != 'early42-resource': assert 'Failed to start thread "GC Thread#0"' in diagnostic, diagnostic diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh index 5bacb08705..536f500660 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh @@ -19,6 +19,7 @@ function record_jvm_startup_diagnostics() { JVM_STARTUP_TIME=$(date -u +'%Y-%m-%d %H:%M:%S.%N UTC') JVM_STARTUP_LOG_OFFSET=$(stat -c %s ./logs/hugegraph-server.log 2>/dev/null || echo 0) + JVM_STARTUP_STDOUT_OFFSET=$(stat -c %s ./logs/hugegraph-server-stdout.log 2>/dev/null || echo 0) } function print_jvm_thread_hint() { @@ -36,7 +37,7 @@ function print_jvm_thread_hint() { ;; resources) echo 'ERROR: JVM could not create a GC worker thread. Possible causes include seccomp denial, insufficient memory, or process/thread limits.' >&2 - echo 'Look for pthread_create failed (EPERM) above in the container log; if present, check the host seccomp profile and security policy.' >&2 + echo 'Look for pthread_create failed (EPERM) in the JVM output: the container log, or logs/hugegraph-server-stdout.log with STDOUT_MODE=false.' >&2 echo 'To diagnose seccomp restrictions, retry with --security-opt seccomp=unconfined; use a compatible host/runtime profile for normal operation.' >&2 echo 'If the process limit is too low, set --ulimit nproc=65535:65535.' >&2 ;; @@ -46,6 +47,14 @@ function print_jvm_thread_hint() { function diagnose_jvm_startup_failure() { local crash size offset + if [[ -r ./logs/hugegraph-server-stdout.log ]]; then + size=$(stat -c %s ./logs/hugegraph-server-stdout.log 2>/dev/null || echo 0) + offset=$JVM_STARTUP_STDOUT_OFFSET + [[ "$size" -ge "$offset" ]] || offset=0 + if tail -c +$((offset + 1)) ./logs/hugegraph-server-stdout.log | print_jvm_thread_hint; then + return 0 + fi + fi while IFS= read -r -d '' crash; do if [[ -r "$crash" ]] && print_jvm_thread_hint < "$crash"; then return 0 From 67d1a37fd0f904056518c8570afdf141df26adfd Mon Sep 17 00:00:00 2001 From: pallavithegod Date: Mon, 19 Oct 2026 22:53:13 +0530 Subject: [PATCH 4/4] fix(dist): narrow thread denial diagnostics Recognize pthread_create failures without matching unrelated thread names. Cover filesystem EPERM in fresh stdout and server log bytes. Preserve existing startup hints and lifecycle fixture coverage. --- .../docker/test/test-docker-entrypoint-process.sh | 13 +++++++++++++ .../hugegraph-dist/src/assembly/static/bin/util.sh | 1 - 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh b/hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh index 05eb40abc0..d83736b715 100644 --- a/hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh +++ b/hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh @@ -57,6 +57,7 @@ with tempfile.TemporaryDirectory(prefix='hugegraph-entrypoint-process-') as dire ('TERM', 143), ('INT', 130), ('groupTERM', 143), ('early42', 42), ('early0', 1), ('timeout', 1), ('early42-eperm-stdout', 42), + ('early42-file-eperm-stdout', 42), ('early42-file-eperm-log', 42), ('early42-file-output', 42), ('early42-file-output-no-crash', 42), ('early42-stale-stdout', 42), ('early42-truncated-stdout', 42), ('early42-resource', 42), ('early42-stale-file', 42), @@ -196,6 +197,15 @@ if mode.startswith('early'): (work / 'logs/hugegraph-server-stdout.log').write_text('pthread_create failed (EPERM)\\n') if mode == 'early42-file-output': (work / 'logs/hs_err_pid42.log').write_text('Cannot create worker GC thread\\n') + elif mode in ['early42-file-eperm-stdout', 'early42-file-eperm-log']: + if mode == 'early42-file-eperm-stdout': + target = work / 'logs/hugegraph-server-stdout.log' + message = 'Exception in thread "main" java.nio.file.FileSystemException: /data/LOCK: Operation not permitted' + else: + target = work / 'logs/hugegraph-server.log' + message = '[pool-3-thread-1] [ERROR] RocksDBStore - Failed to open RocksDB: lock: Operation not permitted' + with target.open('a') as output: + output.write(message + '\\n') elif mode in ['early42-eperm-stdout', 'early42-resource']: if mode == 'early42-eperm-stdout': print('Failed to start thread "GC Thread#0" - pthread_create failed (EPERM)', flush=True) @@ -261,6 +271,9 @@ sys.exit(int((work / 'child-exit').read_text())) actual = process.wait(timeout=5) assert actual == expected, (mode, expected, actual, (work / 'output').read_text()) diagnostic = (work / 'output').read_text() + if mode in ['early42-file-eperm-stdout', 'early42-file-eperm-log']: + assert 'JVM thread creation was denied' not in diagnostic, diagnostic + assert '--security-opt seccomp=unconfined' not in diagnostic, diagnostic if mode in ['early42-file-output', 'early42-file-output-no-crash', 'early42-truncated-stdout']: assert 'JVM thread creation was denied (EPERM)' in diagnostic, diagnostic assert 'Possible causes' not in diagnostic, diagnostic diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh index 536f500660..af83844a5f 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh @@ -26,7 +26,6 @@ function print_jvm_thread_hint() { local kind kind=$(awk ' /pthread_create.*\(EPERM\)|pthread_create.*Operation not permitted/ { denied = 1 } - /[Tt]hread.*Operation not permitted/ { denied = 1 } /Cannot create worker GC thread/ { resources = 1 } END { if (denied) print "denied"; else if (resources) print "resources" } ')