From 2b4d123f73767ce6e5442e732d6ba560f012ad35 Mon Sep 17 00:00:00 2001 From: tanishq-chugh Date: Mon, 17 Aug 2026 12:46:45 +0530 Subject: [PATCH 1/5] [WIP] Optimize K8s Operator LLAP scaling --- .../crds/hiveclusters.hive.apache.org-v1.yml | 52 +++++++++++--- .../hive-operator/templates/hivecluster.yaml | 1 + .../kubernetes/helm/hive-operator/values.yaml | 3 +- .../autoscaling/ComponentAutoscaler.java | 9 ++- .../autoscaling/HiveClusterAutoscaler.java | 15 ++++ .../autoscaling/LlapScalingStrategy.java | 69 ++++++++++++------- .../dependent/HiveDependentResource.java | 7 ++ .../operator/model/spec/AutoscalingSpec.java | 7 +- .../operator/model/spec/HiveServer2Spec.java | 2 +- .../operator/model/spec/LlapSpec.java | 4 +- .../operator/model/spec/MetastoreSpec.java | 2 +- .../operator/model/spec/TezAmSpec.java | 2 +- .../kubernetes/operator/util/ConfigUtils.java | 2 + .../operator/util/HiveConfigBuilder.java | 5 ++ 14 files changed, 137 insertions(+), 43 deletions(-) diff --git a/packaging/src/kubernetes/helm/hive-operator/crds/hiveclusters.hive.apache.org-v1.yml b/packaging/src/kubernetes/helm/hive-operator/crds/hiveclusters.hive.apache.org-v1.yml index 9947f44b75b2..9bdb3863c526 100644 --- a/packaging/src/kubernetes/helm/hive-operator/crds/hiveclusters.hive.apache.org-v1.yml +++ b/packaging/src/kubernetes/helm/hive-operator/crds/hiveclusters.hive.apache.org-v1.yml @@ -129,6 +129,12 @@ spec: replicas before scale-down occurs. Also acts as the cooldown between consecutive scale-downs. type: integer + scaleDownThreshold: + default: 20 + description: "Threshold that triggers scale-down (LLAP only):\ + \ average daemon utilization percentage (0-100) below which\ + \ scale-down triggers." + type: integer scaleUpStabilizationSeconds: default: 60 description: Stabilization window in seconds for scale-up @@ -138,8 +144,8 @@ spec: scaleUpThreshold: default: 80 description: "Threshold that triggers scale-up (component-specific:\ - \ sessions per pod for HS2, request rate for HMS, busy slots\ - \ per daemon for LLAP). Not used by TezAM (demand-based:\ + \ sessions per pod for HS2, request rate for HMS, load percentage\ + \ pending in TezAM for LLAP). Not used by TezAM (demand-based:\ \ 1 TezAM per session)." type: integer type: object @@ -312,6 +318,12 @@ spec: fewer replicas before scale-down occurs. Also acts as the cooldown between consecutive scale-downs. type: integer + scaleDownThreshold: + default: 20 + description: "Threshold that triggers scale-down (LLAP only):\ + \ average daemon utilization percentage (0-100) below\ + \ which scale-down triggers." + type: integer scaleUpStabilizationSeconds: default: 60 description: Stabilization window in seconds for scale-up @@ -321,9 +333,9 @@ spec: scaleUpThreshold: default: 80 description: "Threshold that triggers scale-up (component-specific:\ - \ sessions per pod for HS2, request rate for HMS, busy\ - \ slots per daemon for LLAP). Not used by TezAM (demand-based:\ - \ 1 TezAM per session)." + \ sessions per pod for HS2, request rate for HMS, load\ + \ percentage pending in TezAM for LLAP). Not used by TezAM\ + \ (demand-based: 1 TezAM per session)." type: integer type: object configOverrides: @@ -467,6 +479,12 @@ spec: fewer replicas before scale-down occurs. Also acts as the cooldown between consecutive scale-downs. type: integer + scaleDownThreshold: + default: 20 + description: "Threshold that triggers scale-down (LLAP\ + \ only): average daemon utilization percentage (0-100)\ + \ below which scale-down triggers." + type: integer scaleUpStabilizationSeconds: default: 60 description: Stabilization window in seconds for scale-up @@ -477,8 +495,8 @@ spec: default: 80 description: "Threshold that triggers scale-up (component-specific:\ \ sessions per pod for HS2, request rate for HMS,\ - \ busy slots per daemon for LLAP). Not used by TezAM\ - \ (demand-based: 1 TezAM per session)." + \ load percentage pending in TezAM for LLAP). Not\ + \ used by TezAM (demand-based: 1 TezAM per session)." type: integer type: object replicas: @@ -545,6 +563,12 @@ spec: replicas before scale-down occurs. Also acts as the cooldown between consecutive scale-downs. type: integer + scaleDownThreshold: + default: 20 + description: "Threshold that triggers scale-down (LLAP only):\ + \ average daemon utilization percentage (0-100) below which\ + \ scale-down triggers." + type: integer scaleUpStabilizationSeconds: default: 60 description: Stabilization window in seconds for scale-up @@ -554,8 +578,8 @@ spec: scaleUpThreshold: default: 80 description: "Threshold that triggers scale-up (component-specific:\ - \ sessions per pod for HS2, request rate for HMS, busy slots\ - \ per daemon for LLAP). Not used by TezAM (demand-based:\ + \ sessions per pod for HS2, request rate for HMS, load percentage\ + \ pending in TezAM for LLAP). Not used by TezAM (demand-based:\ \ 1 TezAM per session)." type: integer type: object @@ -753,6 +777,12 @@ spec: replicas before scale-down occurs. Also acts as the cooldown between consecutive scale-downs. type: integer + scaleDownThreshold: + default: 20 + description: "Threshold that triggers scale-down (LLAP only):\ + \ average daemon utilization percentage (0-100) below which\ + \ scale-down triggers." + type: integer scaleUpStabilizationSeconds: default: 60 description: Stabilization window in seconds for scale-up @@ -762,8 +792,8 @@ spec: scaleUpThreshold: default: 80 description: "Threshold that triggers scale-up (component-specific:\ - \ sessions per pod for HS2, request rate for HMS, busy slots\ - \ per daemon for LLAP). Not used by TezAM (demand-based:\ + \ sessions per pod for HS2, request rate for HMS, load percentage\ + \ pending in TezAM for LLAP). Not used by TezAM (demand-based:\ \ 1 TezAM per session)." type: integer type: object diff --git a/packaging/src/kubernetes/helm/hive-operator/templates/hivecluster.yaml b/packaging/src/kubernetes/helm/hive-operator/templates/hivecluster.yaml index 278ae40e8df0..a2b465ba0b37 100644 --- a/packaging/src/kubernetes/helm/hive-operator/templates/hivecluster.yaml +++ b/packaging/src/kubernetes/helm/hive-operator/templates/hivecluster.yaml @@ -157,6 +157,7 @@ spec: enabled: true minReplicas: {{ .autoscaling.minReplicas }} scaleUpThreshold: {{ .autoscaling.scaleUpThreshold }} + scaleDownThreshold: {{ .autoscaling.scaleDownThreshold }} scaleUpStabilizationSeconds: {{ .autoscaling.scaleUpStabilizationSeconds }} scaleDownStabilizationSeconds: {{ .autoscaling.scaleDownStabilizationSeconds }} gracePeriodSeconds: {{ .autoscaling.gracePeriodSeconds }} diff --git a/packaging/src/kubernetes/helm/hive-operator/values.yaml b/packaging/src/kubernetes/helm/hive-operator/values.yaml index c16f7240ea20..0c1881a4dac2 100644 --- a/packaging/src/kubernetes/helm/hive-operator/values.yaml +++ b/packaging/src/kubernetes/helm/hive-operator/values.yaml @@ -200,7 +200,8 @@ cluster: autoscaling: enabled: false minReplicas: 0 - scaleUpThreshold: 10 + scaleUpThreshold: 70 + scaleDownThreshold: 20 scaleUpStabilizationSeconds: 60 scaleDownStabilizationSeconds: 900 gracePeriodSeconds: 600 diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/ComponentAutoscaler.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/ComponentAutoscaler.java index fcf4da2aefba..a55973bfe343 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/ComponentAutoscaler.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/ComponentAutoscaler.java @@ -94,8 +94,13 @@ public EvaluationResult evaluate(List metrics, AutoscalingSpec spec, int target; if (clamped > currentReplicas) { - // Scale up: use stabilized max (highest recommendation in window — don't under-scale) - target = scaleUpWindow.stabilizedMax(); + if (component.startsWith(ConfigUtils.COMPONENT_LLAP + "-")) { + // HS2 sessions activation gate scales up the LLAP pods to atleast 1 + // in presence of sessions. Avoid stabilizedMin in this start-up case. + target = currentReplicas == 0 ? clamped : scaleUpWindow.stabilizedMin(); + } else { + target = scaleUpWindow.stabilizedMax(); + } } else if (clamped < currentReplicas) { // Scale down: use stabilized max (highest/most conservative recommendation in window — // prevents premature scale-down, matches HPA selectPolicy: Max behavior). diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/HiveClusterAutoscaler.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/HiveClusterAutoscaler.java index 3c69a957ef93..f6016043fc63 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/HiveClusterAutoscaler.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/HiveClusterAutoscaler.java @@ -321,6 +321,21 @@ public List getHs2MetricsFromCache(HiveCluster cluster) { return metricsCache.getOrEmpty(key, maxStale); } + /** + * Returns cached TezAM metrics for the given LLAP cluster (used by LlapScalingStrategy). + */ + public List getTezAmMetricsFromCache(HiveCluster cluster, String llapName) { + String namespace = cluster.getMetadata().getNamespace(); + String clusterName = cluster.getMetadata().getName(); + int maxStale = cluster.getSpec().llapClusters().stream() + .filter(l -> llapName.equals(l.name())) + .findFirst() + .map(l -> l.tezAm().autoscaling().metricsScrapeIntervalSeconds() * 3) + .orElse(30); + return metricsCache.getOrEmpty( + cacheKey(namespace, clusterName, ConfigUtils.tezAmComponentKey(llapName)), maxStale); + } + private void evaluateComponent(HiveCluster cluster, KubernetesClient client, String namespace, String clusterName, String component, AutoscalingSpec autoscaling, int maxReplicas, diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/LlapScalingStrategy.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/LlapScalingStrategy.java index 368d38474d6c..5f7388ec1186 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/LlapScalingStrategy.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/LlapScalingStrategy.java @@ -28,11 +28,11 @@ /** * Scaling strategy for LLAP daemons. - * Formula: avg(QueuedRequests + Configured - Available) across all pods. - * This represents average "busy slots + queued" per daemon. - * desired = ceil(avg_busy / scaleUpThreshold) + * For Scale-Up: Pending Load across all TezAM pods should be above the threshold. + * For Scale-Down: Running Load across all LLAP pods should be below the threshold. + * desired = ceil(totalClusterLoad / capacityPerDaemon) *

- * Activation gate: only scale if HS2 has open sessions (prevents zombie scaling). + * Activation gate: only scale if HS2 has open sessions & TezAMs are running DAGs (prevents zombie scaling). */ public class LlapScalingStrategy implements ScalingStrategy { @@ -41,7 +41,9 @@ public class LlapScalingStrategy implements ScalingStrategy { static final String METRIC_QUEUED = "hadoop_llapdaemon_executornumqueuedrequests"; static final String METRIC_CONFIGURED = "hadoop_llapdaemon_executornumexecutorsconfigured"; static final String METRIC_AVAILABLE = "hadoop_llapdaemon_executornumexecutorsavailable"; + static final String METRIC_MAX_FREE_SLOTS_CONFIGURED = "hadoop_llapdaemon_executormaxfreeslotsconfigured"; static final String METRIC_LLAP_TARGET_PREFIX = "hs2_llap_target_sessions_"; + static final String METRIC_TEZ_PENDING_TASKS = "tez_am_pending_tasks"; private final HiveClusterAutoscaler orchestrator; private final HiveCluster cluster; @@ -83,36 +85,57 @@ public int computeDesiredReplicas(List podMetrics, return minReplica; } - // Compute average busy slots across all LLAP pods - double totalBusy = 0; - int podCount = 0; + List tezAmMetrics = orchestrator.getTezAmMetricsFromCache(cluster, llapName); + double totalPending = 0; + for (PodMetrics pm : tezAmMetrics) { + totalPending += pm.metrics().getOrDefault(METRIC_TEZ_PENDING_TASKS, 0.0); + } + + double totalLLAPCapacity = 0; + double totalLLAPLoad = 0; for (PodMetrics pm : podMetrics) { double queued = pm.metrics().getOrDefault(METRIC_QUEUED, 0.0); double configured = pm.metrics().getOrDefault(METRIC_CONFIGURED, 0.0); double available = pm.metrics().getOrDefault(METRIC_AVAILABLE, 0.0); - double busy = queued + configured - available; - totalBusy += busy; - podCount++; + totalLLAPCapacity += pm.metrics().getOrDefault(METRIC_MAX_FREE_SLOTS_CONFIGURED, 0.0); + totalLLAPLoad += queued + configured - available; } - double avgBusy = totalBusy / podCount; - lastMetric = (int) Math.round(avgBusy); - - if (avgBusy <= 0) { - // HS2 has sessions (passed activation gate above) but executors are idle between queries. - // Keep at least 1 daemon to avoid flapping: scaling to 0 here would cause immediate - // scale-back-up on the next evaluation when the empty-pod path triggers. + // HS2 has sessions (passed activation gate above) but either + // 1. there is no tezAM running, so LLAP running any work is zombie if any. + // 2. there are tezAMs running, but no tasks running or pending. + if(tezAmMetrics.isEmpty() || (totalPending + totalLLAPLoad) == 0) { return Math.max(1, autoscaling.minReplicas()); } - if (LOG.isDebugEnabled()) { - LOG.debug("[llap] avgBusy={}, threshold={}", String.format("%.2f", avgBusy), - autoscaling.scaleUpThreshold()); + double capacityPerDaemon = Math.max(1.0, totalLLAPCapacity / podMetrics.size()); + double freeLLAPCapacity = totalLLAPCapacity - totalLLAPLoad; + double avgLLAPLoadPercent = totalLLAPCapacity > 0 ? (totalLLAPLoad / totalLLAPCapacity) * 100.00 : 0.0; + + double totalClusterLoad = totalPending + totalLLAPLoad; + double pendingLoadPercent = totalClusterLoad > 0 ? (totalPending / totalClusterLoad) * 100.0 : 0.0; + + int scaleUpThreshold = autoscaling.scaleUpThreshold(); + int scaleDownThreshold = autoscaling.scaleDownThreshold(); + + lastMetric = (int) totalClusterLoad; + + // Scale-up: pending load share of total load exceeds threshold + // Scale-down: no pending work AND daemon load below threshold + if ((pendingLoadPercent >= scaleUpThreshold && totalPending > freeLLAPCapacity) || + (totalPending == 0 && avgLLAPLoadPercent <= scaleDownThreshold)) { + int desired = (int) Math.ceil(totalClusterLoad / capacityPerDaemon); + if (LOG.isDebugEnabled()) { + LOG.debug("[llap-{}] totalClusterLoad={}, capacityPerDaemon={}, pendingLoadPercent={}, avgLLAPLoadPercent={}", + llapName, totalClusterLoad, capacityPerDaemon, String.format("%.2f", pendingLoadPercent), + String.format("%.2f", avgLLAPLoadPercent)); + } + return desired; } - int threshold = Math.max(1, autoscaling.scaleUpThreshold()); - int desired = (int) Math.ceil(avgBusy / threshold); - return Math.max(desired, autoscaling.minReplicas()); + // Work is in flight (pending share below threshold) or daemons are moderately loaded. + // Return current pod count so the stabilization window keeps the replica count stable. + return podMetrics.size(); } @Override diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveDependentResource.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveDependentResource.java index ebe89d061431..4cb348bfd476 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveDependentResource.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveDependentResource.java @@ -854,14 +854,21 @@ private static String buildJmxExporterConfig(String component) { sb.append("- pattern: '").append(llapBean).append("ExecutorNumExecutors'\n"); sb.append(" name: hadoop_llapdaemon_executornumexecutors\n"); sb.append(" type: GAUGE\n"); + sb.append("- pattern: '").append(llapBean).append("ExecutorMaxFreeSlotsConfigured'\n"); + sb.append(" name: hadoop_llapdaemon_executormaxfreeslotsconfigured\n"); + sb.append(" type: GAUGE\n"); break; case ConfigUtils.COMPONENT_TEZAM: // LlapMetricsSystem registers beans under LlapTaskScheduler service // SchedulerDagStatus tracks if the AM is running a dag or is idle so exported as GAUGE. + // SchedulerPendingTaskCount tracks currently waiting tasks in the AM to be scheduled. String schedulerBean = "Hadoop<>"; sb.append("- pattern: '").append(schedulerBean).append("SchedulerDagStatus'\n"); sb.append(" name: tez_am_dag_running\n"); sb.append(" type: GAUGE\n"); + sb.append("- pattern: '").append(schedulerBean).append("SchedulerPendingTaskCount'\n"); + sb.append(" name: tez_am_pending_tasks\n"); + sb.append(" type: GAUGE\n"); break; default: sb.append("- pattern: '.*'\n"); diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/AutoscalingSpec.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/AutoscalingSpec.java index 1cdb06bdad46..a1bc6ea3a105 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/AutoscalingSpec.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/AutoscalingSpec.java @@ -33,10 +33,14 @@ public record AutoscalingSpec( @Default("0") Integer minReplicas, @JsonPropertyDescription("Threshold that triggers scale-up (component-specific: " - + "sessions per pod for HS2, request rate for HMS, busy slots per daemon for LLAP). " + + "sessions per pod for HS2, request rate for HMS, load percentage pending in TezAM for LLAP). " + "Not used by TezAM (demand-based: 1 TezAM per session).") @Default("80") Integer scaleUpThreshold, + @JsonPropertyDescription("Threshold that triggers scale-down (LLAP only): " + + "average daemon utilization percentage (0-100) below which scale-down triggers.") + @Default("20") + Integer scaleDownThreshold, @JsonPropertyDescription("Stabilization window in seconds for scale-up decisions. " + "Picks the highest recommendation within this window to prevent flapping.") @Default("60") @@ -73,6 +77,7 @@ public record AutoscalingSpec( enabled = enabled != null ? enabled : false; minReplicas = minReplicas != null ? minReplicas : 0; scaleUpThreshold = scaleUpThreshold != null ? scaleUpThreshold : 80; + scaleDownThreshold = scaleDownThreshold != null ? scaleDownThreshold : 20; scaleUpStabilizationSeconds = scaleUpStabilizationSeconds != null ? scaleUpStabilizationSeconds : 60; scaleDownStabilizationSeconds = scaleDownStabilizationSeconds != null ? scaleDownStabilizationSeconds : 600; gracePeriodSeconds = gracePeriodSeconds != null ? gracePeriodSeconds : 3600; diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/HiveServer2Spec.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/HiveServer2Spec.java index 62855537bcdb..b4962d7e6f34 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/HiveServer2Spec.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/HiveServer2Spec.java @@ -63,6 +63,6 @@ public record HiveServer2Spec( extraVolumeMounts = extraVolumeMounts != null ? extraVolumeMounts : List.of(); externalJars = externalJars != null ? externalJars : List.of(); autoscaling = autoscaling != null ? autoscaling : new AutoscalingSpec( - false, 1, 80, 60, 600, 300, 10, 90, 30, null); + false, 1, 80, 0, 60, 600, 300, 10, 90, 30, null); } } diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/LlapSpec.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/LlapSpec.java index 61279829c908..eb42d8c8c0ee 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/LlapSpec.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/LlapSpec.java @@ -81,7 +81,7 @@ public record LlapTezAmSpec( public LlapTezAmSpec { replicas = replicas != null ? replicas : 1; autoscaling = autoscaling != null ? autoscaling : new AutoscalingSpec( - false, 0, 0, 60, 600, 120, 10, 0, 0, null); + false, 0, 0, 0, 60, 600, 120, 10, 0, 0, null); } } @@ -102,7 +102,7 @@ public record LlapTezAmSpec( extraVolumes = extraVolumes != null ? extraVolumes : List.of(); extraVolumeMounts = extraVolumeMounts != null ? extraVolumeMounts : List.of(); autoscaling = autoscaling != null ? autoscaling : new AutoscalingSpec( - false, 0, 1, 60, 900, 600, 10, 0, 0, null); + false, 0, 1, 20, 60, 900, 600, 10, 0, 0, null); tezAm = tezAm != null ? tezAm : new LlapTezAmSpec(null, null); } diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/MetastoreSpec.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/MetastoreSpec.java index 956bac65bcb9..6548d999918a 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/MetastoreSpec.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/MetastoreSpec.java @@ -70,7 +70,7 @@ public record MetastoreSpec( extraVolumes = extraVolumes != null ? extraVolumes : List.of(); extraVolumeMounts = extraVolumeMounts != null ? extraVolumeMounts : List.of(); autoscaling = autoscaling != null ? autoscaling : new AutoscalingSpec( - false, 1, 75, 60, 300, 60, 10, 90, 30, null); + false, 1, 75, 0, 60, 300, 60, 10, 90, 30, null); } public boolean isEnabled() { diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/TezAmSpec.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/TezAmSpec.java index 98d1c976dd9d..8165b9b829ae 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/TezAmSpec.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/TezAmSpec.java @@ -64,7 +64,7 @@ public record TezAmSpec( extraVolumes = extraVolumes != null ? extraVolumes : List.of(); extraVolumeMounts = extraVolumeMounts != null ? extraVolumeMounts : List.of(); autoscaling = autoscaling != null ? autoscaling : new AutoscalingSpec( - false, 0, 0, 60, 600, 120, 10, 0, 0, null); + false, 0, 0, 0, 60, 600, 120, 10, 0, 0, null); } public boolean isEnabled() { diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/ConfigUtils.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/ConfigUtils.java index 6253681455b2..76dac7a640b9 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/ConfigUtils.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/ConfigUtils.java @@ -141,6 +141,8 @@ public static String tezAmComponentKey(String llapName) { public static final String HIVE_LLAP_DAEMON_UMBILICAL_PORT_KEY = "hive.llap.daemon.umbilical.port"; public static final String HIVE_LLAP_DAEMON_UMBILICAL_PORT_DEFAULT = "0"; + public static final String HIVE_LLAP_TASK_SCHEDULER_LOCALITY_DELAY_KEY = "hive.llap.task.scheduler.locality.delay"; + public static final String METASTORE_SERVER_TRANSPORT_MODE_KEY = "metastore.server.thrift.transport.mode"; public static final String METASTORE_SERVER_TRANSPORT_MODE_DEFAULT = "http"; diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/HiveConfigBuilder.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/HiveConfigBuilder.java index d71011f92899..49d69f5c52db 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/HiveConfigBuilder.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/HiveConfigBuilder.java @@ -204,6 +204,11 @@ public static Map getTezSite(HiveClusterSpec spec, LlapSpec llap if (spec.tezAm().configOverrides() != null) { tezProps.putAll(spec.tezAm().configOverrides()); } + if (llap != null && llap.isEnabled() && llap.autoscaling().isEnabled()) { + if (ConfigUtils.getTimeMs(tezProps, ConfigUtils.HIVE_LLAP_TASK_SCHEDULER_LOCALITY_DELAY_KEY, 0) == -1) { + tezProps.put(ConfigUtils.HIVE_LLAP_TASK_SCHEDULER_LOCALITY_DELAY_KEY, "0ms"); + } + } return tezProps; } From 807ffa167b2ffaeb73b133309d734fd5beadb8e9 Mon Sep 17 00:00:00 2001 From: tanishq-chugh Date: Wed, 26 Aug 2026 23:27:24 +0530 Subject: [PATCH 2/5] Fix TezAM starting at replica count when llap minReplicas is more than 0 --- .../operator/reconciler/HiveClusterReconciler.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java index ebc5c4f1ae8c..a7da524a3dbd 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java @@ -704,10 +704,13 @@ private int resolveTezAmReplicaCount(HiveCluster resource, if (llapManaged != null && llapManaged == 0) { return 0; } - if (llapSpec.autoscaling().isEnabled() && llapManaged == null) { - // First reconcile, LLAP starts at minReplicas (likely 0) — TezAM matches - return llapSpec.autoscaling().minReplicas() > 0 - ? tezAmSpec.replicas() : 0; + if (llapSpec.autoscaling().isEnabled() && llapManaged == null + && llapSpec.autoscaling().minReplicas() == 0) { + // First reconcile before autoscaler runs: LLAP starts at 0, so TezAM stays down too. + return 0; + } + if (tezAmSpec.autoscaling().isEnabled()) { + return tezAmSpec.autoscaling().minReplicas(); } return tezAmSpec.replicas(); } From 77c776509e6285f929e899a4831820c7e6b387b0 Mon Sep 17 00:00:00 2001 From: tanishq-chugh Date: Thu, 27 Aug 2026 13:05:57 +0530 Subject: [PATCH 3/5] Remove the redundant scale-up check --- .../kubernetes/operator/autoscaling/LlapScalingStrategy.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/LlapScalingStrategy.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/LlapScalingStrategy.java index 5f7388ec1186..8c4e0adf1e17 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/LlapScalingStrategy.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/LlapScalingStrategy.java @@ -109,7 +109,6 @@ public int computeDesiredReplicas(List podMetrics, } double capacityPerDaemon = Math.max(1.0, totalLLAPCapacity / podMetrics.size()); - double freeLLAPCapacity = totalLLAPCapacity - totalLLAPLoad; double avgLLAPLoadPercent = totalLLAPCapacity > 0 ? (totalLLAPLoad / totalLLAPCapacity) * 100.00 : 0.0; double totalClusterLoad = totalPending + totalLLAPLoad; @@ -122,8 +121,7 @@ public int computeDesiredReplicas(List podMetrics, // Scale-up: pending load share of total load exceeds threshold // Scale-down: no pending work AND daemon load below threshold - if ((pendingLoadPercent >= scaleUpThreshold && totalPending > freeLLAPCapacity) || - (totalPending == 0 && avgLLAPLoadPercent <= scaleDownThreshold)) { + if (pendingLoadPercent >= scaleUpThreshold || (totalPending == 0 && avgLLAPLoadPercent <= scaleDownThreshold)) { int desired = (int) Math.ceil(totalClusterLoad / capacityPerDaemon); if (LOG.isDebugEnabled()) { LOG.debug("[llap-{}] totalClusterLoad={}, capacityPerDaemon={}, pendingLoadPercent={}, avgLLAPLoadPercent={}", From 9056cc84c06bd9958c165b29a74822378f7a9132 Mon Sep 17 00:00:00 2001 From: tanishq-chugh Date: Thu, 27 Aug 2026 12:32:46 +0530 Subject: [PATCH 4/5] Address Co-pilot comments --- .../operator/autoscaling/LlapScalingStrategy.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/LlapScalingStrategy.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/LlapScalingStrategy.java index 8c4e0adf1e17..d700a644a2fc 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/LlapScalingStrategy.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/LlapScalingStrategy.java @@ -32,7 +32,8 @@ * For Scale-Down: Running Load across all LLAP pods should be below the threshold. * desired = ceil(totalClusterLoad / capacityPerDaemon) *

- * Activation gate: only scale if HS2 has open sessions & TezAMs are running DAGs (prevents zombie scaling). + * Activation gate: only scale if HS2 has open sessions, TezAM metrics are present & + * cluster has load (prevents zombie scaling). */ public class LlapScalingStrategy implements ScalingStrategy { @@ -97,7 +98,11 @@ public int computeDesiredReplicas(List podMetrics, double queued = pm.metrics().getOrDefault(METRIC_QUEUED, 0.0); double configured = pm.metrics().getOrDefault(METRIC_CONFIGURED, 0.0); double available = pm.metrics().getOrDefault(METRIC_AVAILABLE, 0.0); - totalLLAPCapacity += pm.metrics().getOrDefault(METRIC_MAX_FREE_SLOTS_CONFIGURED, 0.0); + double capacity = pm.metrics().getOrDefault(METRIC_MAX_FREE_SLOTS_CONFIGURED, 0.0); + if (capacity <= 0) { + capacity = configured; + } + totalLLAPCapacity += capacity; totalLLAPLoad += queued + configured - available; } From 4b5074814fac5655961f29225eeb35e37f90c3bf Mon Sep 17 00:00:00 2001 From: tanishq-chugh Date: Thu, 27 Aug 2026 13:25:21 +0530 Subject: [PATCH 5/5] Address SonarQube --- .../operator/autoscaling/LlapScalingStrategy.java | 2 +- .../hive/kubernetes/operator/util/HiveConfigBuilder.java | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/LlapScalingStrategy.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/LlapScalingStrategy.java index d700a644a2fc..4c7e006fc798 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/LlapScalingStrategy.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/LlapScalingStrategy.java @@ -117,7 +117,7 @@ public int computeDesiredReplicas(List podMetrics, double avgLLAPLoadPercent = totalLLAPCapacity > 0 ? (totalLLAPLoad / totalLLAPCapacity) * 100.00 : 0.0; double totalClusterLoad = totalPending + totalLLAPLoad; - double pendingLoadPercent = totalClusterLoad > 0 ? (totalPending / totalClusterLoad) * 100.0 : 0.0; + double pendingLoadPercent = (totalPending / totalClusterLoad) * 100.0; int scaleUpThreshold = autoscaling.scaleUpThreshold(); int scaleDownThreshold = autoscaling.scaleDownThreshold(); diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/HiveConfigBuilder.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/HiveConfigBuilder.java index 49d69f5c52db..9c78fa153425 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/HiveConfigBuilder.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/HiveConfigBuilder.java @@ -204,10 +204,11 @@ public static Map getTezSite(HiveClusterSpec spec, LlapSpec llap if (spec.tezAm().configOverrides() != null) { tezProps.putAll(spec.tezAm().configOverrides()); } - if (llap != null && llap.isEnabled() && llap.autoscaling().isEnabled()) { - if (ConfigUtils.getTimeMs(tezProps, ConfigUtils.HIVE_LLAP_TASK_SCHEDULER_LOCALITY_DELAY_KEY, 0) == -1) { - tezProps.put(ConfigUtils.HIVE_LLAP_TASK_SCHEDULER_LOCALITY_DELAY_KEY, "0ms"); - } + + // Disable Infinite locality Delay when LLAP Auto-scaling is enabled, as they are mutually exclusive. + if (llap != null && llap.isEnabled() && llap.autoscaling().isEnabled() && + ConfigUtils.getTimeMs(tezProps, ConfigUtils.HIVE_LLAP_TASK_SCHEDULER_LOCALITY_DELAY_KEY, 0) == -1) { + tezProps.put(ConfigUtils.HIVE_LLAP_TASK_SCHEDULER_LOCALITY_DELAY_KEY, "0ms"); } return tezProps; }