Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/trigger_files/beam_PostCommit_Go_VR_Spark.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to run",
"modification": 1,
"https://github.com/apache/beam/pull/36527": "skip a processing time timer test in spark",
"modification": 2,
"https://github.com/apache/beam/pull/36527": "skip a processing time timer test in spark"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to run",
"modification": 1
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,6 @@
"https://github.com/apache/beam/pull/34830": "testing",
"https://github.com/apache/beam/issues/35429": "testing",
"trigger-2026-04-04": "portable_runner expand_sdf opt-in",
"https://github.com/apache/beam/pull/38892": "UnboundedSource portable VR test"
"https://github.com/apache/beam/pull/38892": "UnboundedSource portable VR test",
"modification": 1
}
4 changes: 4 additions & 0 deletions sdks/go/test/run_validatesrunner_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,10 @@ if [[ "$RUNNER" == "flink" || "$RUNNER" == "spark" || "$RUNNER" == "portable" ||
--artifact-port 0 &
elif [[ "$RUNNER" == "spark" ]]; then
"$JAVA_CMD" \
--add-opens=java.base/sun.nio.ch=ALL-UNNAMED \
--add-opens=java.base/java.nio=ALL-UNNAMED \
--add-opens=java.base/java.util=ALL-UNNAMED \
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED \
-jar $SPARK_JOB_SERVER_JAR \
--spark-master-url local \
--job-port $JOB_PORT \
Expand Down
4 changes: 2 additions & 2 deletions sdks/python/apache_beam/runners/portability/job_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -175,8 +175,8 @@ def subprocess_cmd_and_endpoint(self):
self._artifacts_dir if self._artifacts_dir else self.local_temp_dir(
prefix='artifacts'))
job_port, = subprocess_server.pick_port(self._job_port)
subprocess_cmd = [self._java_launcher, '-jar'] + self._jvm_properties + [
jar_path
subprocess_cmd = [self._java_launcher] + list(self._jvm_properties) + [
'-jar', jar_path
Comment on lines +178 to +179

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

If self._jvm_properties is None, calling list(self._jvm_properties) will raise a TypeError. Using self._jvm_properties or [] provides a safe fallback to prevent potential runtime errors.

Suggested change
subprocess_cmd = [self._java_launcher] + list(self._jvm_properties) + [
'-jar', jar_path
subprocess_cmd = [self._java_launcher] + list(self._jvm_properties or []) + [
'-jar', jar_path

] + list(
self.java_arguments(
job_port, self._artifact_port, self._expansion_port, artifacts_dir))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,8 @@ def test_subprocess_cmd_and_endpoint(self):
subprocess_cmd,
[
'/path/to/java',
'-jar',
'-Dsome.property=value',
'-jar',
'/path/to/jar',
'--artifacts-dir',
'/path/to/artifacts/',
Expand Down
11 changes: 11 additions & 0 deletions sdks/python/apache_beam/runners/portability/spark_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,13 @@
# https://spark.apache.org/docs/latest/submitting-applications.html#master-urls
LOCAL_MASTER_PATTERN = r'^local(\[.+\])?$'

SPARK_JAR_JOB_SERVER_JVM_ARGS = [
'--add-opens=java.base/sun.nio.ch=ALL-UNNAMED',
'--add-opens=java.base/java.nio=ALL-UNNAMED',
'--add-opens=java.base/java.util=ALL-UNNAMED',
'--add-opens=java.base/java.lang.invoke=ALL-UNNAMED',
]

# Since Java job servers are heavyweight external processes, cache them.
# This applies only to SparkJarJobServer, not SparkUberJarJobServer.
JOB_SERVER_CACHE = {}
Expand Down Expand Up @@ -84,6 +91,10 @@ def __init__(self, options):
self._jar = options.spark_job_server_jar
self._master_url = options.spark_master_url
self._spark_version = options.spark_version
self._jvm_properties = list(self._jvm_properties)
for arg in SPARK_JAR_JOB_SERVER_JVM_ARGS:
if arg not in self._jvm_properties:
self._jvm_properties.append(arg)
Comment on lines +94 to +97

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

If self._jvm_properties is None, calling list(self._jvm_properties) will raise a TypeError. Using self._jvm_properties or [] ensures the code is robust against None values.

Suggested change
self._jvm_properties = list(self._jvm_properties)
for arg in SPARK_JAR_JOB_SERVER_JVM_ARGS:
if arg not in self._jvm_properties:
self._jvm_properties.append(arg)
self._jvm_properties = list(self._jvm_properties or [])
for arg in SPARK_JAR_JOB_SERVER_JVM_ARGS:
if arg not in self._jvm_properties:
self._jvm_properties.append(arg)


def path_to_jar(self):
if self._jar:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
from apache_beam.runners.portability import job_server
from apache_beam.runners.portability import portable_runner
from apache_beam.runners.portability import portable_runner_test
from apache_beam.runners.portability.spark_runner import SPARK_JAR_JOB_SERVER_JVM_ARGS
from apache_beam.utils import subprocess_server

# Run as
Expand Down Expand Up @@ -100,6 +101,7 @@ def _subprocess_command(cls, job_port, expansion_port):
try:
return [
subprocess_server.JavaHelper.get_java(),
*SPARK_JAR_JOB_SERVER_JVM_ARGS,
'-Dbeam.spark.test.reuseSparkContext=true',
'-jar',
cls.spark_job_server_jar,
Expand Down
Loading