From 5f8aee53fbe1acc48dc220aa5246d4c731fa4879 Mon Sep 17 00:00:00 2001 From: shuke <37901441+shuke987@users.noreply.github.com> Date: Mon, 29 Jun 2026 11:41:44 +0800 Subject: [PATCH 1/4] [fix](regression) wait for adaptive scan profile --- ..._pipeline_task_serial_read_on_limit.groovy | 40 +++++++++++++------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/regression-test/suites/query_profile/adaptive_pipeline_task_serial_read_on_limit.groovy b/regression-test/suites/query_profile/adaptive_pipeline_task_serial_read_on_limit.groovy index 4ccd45af04a51c..f55d7df2811a52 100644 --- a/regression-test/suites/query_profile/adaptive_pipeline_task_serial_read_on_limit.groovy +++ b/regression-test/suites/query_profile/adaptive_pipeline_task_serial_read_on_limit.groovy @@ -21,28 +21,42 @@ import groovy.json.StringEscapeUtils import org.apache.doris.regression.action.ProfileAction def verifyProfileContent = { suiteContext, stmt, serialReadOnLimit -> - // Sleep 1000ms to wait for the profile collection - Thread.sleep(1000) - // Get profile list by using getProfileList def profileAction = new ProfileAction(suiteContext) - List profileData = profileAction.getProfileList() - // Find the profile id for the query that we just emitted String profileId = "" - for (def profileItem : profileData) { - if (profileItem["Sql Statement"].toString().contains(stmt)) { - profileId = profileItem["Profile ID"].toString() - logger.info("Profile ID of ${stmt} is ${profileId}") - break + String profileContent = "" + long deadline = System.currentTimeMillis() + 10000 + while (System.currentTimeMillis() <= deadline) { + List profileData = profileAction.getProfileList() + for (def profileItem : profileData) { + if (profileItem["Sql Statement"].toString().contains(stmt)) { + profileId = profileItem["Profile ID"].toString() + break + } + } + + if (profileId != "" && profileId != null) { + profileContent = profileAction.getProfile(profileId) + if (profileContent.contains("MaxScanConcurrency") + || profileContent.contains("Profile Completion State: COMPLETE")) { + logger.info("Profile ID of ${stmt} is ${profileId}") + logger.info("Profile content of ${stmt} is\n${profileContent}") + break + } + logger.info("Profile of ${stmt} is not ready, profileId=${profileId}") + } else { + logger.info("Profile ID of ${stmt} is not found yet") } + Thread.sleep(500) } if (profileId == "" || profileId == null) { logger.error("Profile ID of ${stmt} is not found") return false } - // Get profile content by using getProfile - String profileContent = profileAction.getProfile(profileId) - logger.info("Profile content of ${stmt} is\n${profileContent}") + if (!profileContent.contains("MaxScanConcurrency")) { + logger.error("Profile of ${stmt} does not contain MaxScanConcurrency, profileId=${profileId}, content:\n${profileContent}") + return false + } // Check if the profile contains the expected content if (serialReadOnLimit) { return profileContent.contains("- MaxScanConcurrency: 1") == true From 17710b29590cf8a424673dabb30d5efb46d75d2e Mon Sep 17 00:00:00 2001 From: shuke <37901441+shuke987@users.noreply.github.com> Date: Mon, 29 Jun 2026 11:50:14 +0800 Subject: [PATCH 2/4] [fix](regression) add profile readiness helper --- .../regression/action/ProfileAction.groovy | 138 +++++++++--------- ..._pipeline_task_serial_read_on_limit.groovy | 35 +---- .../query_profile/scanner_profile.groovy | 17 +-- 3 files changed, 79 insertions(+), 111 deletions(-) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ProfileAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ProfileAction.groovy index d4985d0fe4f0b0..196f15e53b678c 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ProfileAction.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ProfileAction.groovy @@ -26,6 +26,10 @@ import org.apache.doris.regression.util.JdbcUtils @Slf4j class ProfileAction implements SuiteAction { + private static final long DEFAULT_PROFILE_WAIT_TIMEOUT_MS = 10000 + private static final long DEFAULT_PROFILE_WAIT_INTERVAL_MS = 500 + private static final String PROFILE_COMPLETE = "Profile Completion State: COMPLETE" + private String tag private Runnable runCallback private Closure check @@ -76,6 +80,26 @@ class ProfileAction implements SuiteAction { return profileData } + private String normalizeProfileText(String profileText) { + // Convert HTML entities and actual NBSPs to regular spaces, + // retain line breaks, and then compress multiple consecutive spaces into a single space. + profileText = profileText.replace(" ", " ") + profileText = profileText.replace("
", "\n") + profileText = profileText.replace('\u00A0' as char, ' ' as char) + return profileText.replaceAll(" {2,}", " ") + } + + boolean isProfileReady(String profileText, List requiredContents = []) { + if (profileText == null || profileText.isEmpty()) { + return false + } + if (requiredContents != null && !requiredContents.isEmpty() + && requiredContents.every { profileText.contains(it) }) { + return true + } + return profileText.contains(PROFILE_COMPLETE) + } + String getProfile(String profileId) { def profileCli = new HttpCliAction(context) def addr = context.getFeHttpAddress() @@ -97,20 +121,58 @@ class ProfileAction implements SuiteAction { def jsonSlurper2 = new JsonSlurper() def profileText = jsonSlurper2.parseText(profileResp).data - // Convert HTML entities and actual NBSPs to regular spaces, - // retain line breaks, and then compress multiple consecutive spaces into a single space - profileText = profileText.replace(" ", " ") - profileText = profileText.replace("
", "\n") - // Replace the actual NBSP characters with regular spaces - profileText = profileText.replace('\u00A0' as char, ' ' as char) - // Compress two or more consecutive regular spaces into a single space (without affecting line breaks) - profileText = profileText.replaceAll(" {2,}", " ") - result.text = profileText + result.text = normalizeProfileText(profileText) } profileCli.run() return result.text } + String getProfile(String profileId, List requiredContents, long timeoutMs = DEFAULT_PROFILE_WAIT_TIMEOUT_MS, + long intervalMs = DEFAULT_PROFILE_WAIT_INTERVAL_MS) { + String profileText = "" + long deadline = System.currentTimeMillis() + timeoutMs + while (System.currentTimeMillis() <= deadline) { + profileText = getProfile(profileId) + if (isProfileReady(profileText, requiredContents)) { + return profileText + } + log.info("Profile {} is not ready, required contents: {}", profileId, requiredContents) + Thread.sleep(intervalMs) + } + return profileText + } + + String getProfileBySql(String sqlPattern, List requiredContents = [], + long timeoutMs = DEFAULT_PROFILE_WAIT_TIMEOUT_MS, long intervalMs = DEFAULT_PROFILE_WAIT_INTERVAL_MS) { + String profileId = "" + String profileText = "" + long deadline = System.currentTimeMillis() + timeoutMs + while (System.currentTimeMillis() <= deadline) { + for (final def profileItem in getProfileList()) { + if (profileItem["Sql Statement"].toString().contains(sqlPattern)) { + profileId = profileItem["Profile ID"].toString() + long remainingMs = Math.max(1, deadline - System.currentTimeMillis()) + profileText = getProfile(profileId, requiredContents, remainingMs, intervalMs) + if (isProfileReady(profileText, requiredContents)) { + return profileText + } + break + } + } + if (profileId == "") { + log.info("Profile with sql pattern {} is not found yet", sqlPattern) + } else { + log.info("Profile {} with sql pattern {} is not ready", profileId, sqlPattern) + } + Thread.sleep(intervalMs) + } + + if (profileId == "") { + throw new IllegalStateException("Missing profile with sql pattern: " + sqlPattern) + } + return profileText + } + @Override void run() { if (runCallback.is(null)) { @@ -130,63 +192,7 @@ class ProfileAction implements SuiteAction { exception = t } - def httpCli = new HttpCliAction(context) - def addr = context.getFeHttpAddress() - httpCli.endpoint("${addr.hostString}:${addr.port}") - httpCli.uri("/rest/v1/query_profile") - httpCli.op("get") - httpCli.printResponse(false) - - if (context.config.isCloudMode()) { - httpCli.basicAuthorization(context.config.feCloudHttpUser, context.config.feCloudHttpPassword) - } else { - httpCli.basicAuthorization(context.config.feHttpUser, context.config.feHttpPassword) - } - httpCli.check { code, body -> - if (code != 200) { - throw new IllegalStateException("Get profile list failed, code: ${code}, body:\n${body}") - } - - def jsonSlurper = new JsonSlurper() - List profileData = jsonSlurper.parseText(body).data.rows - def canFindProfile = false; - for (final def profileItem in profileData) { - if (profileItem["Sql Statement"].toString().contains(tag)) { - canFindProfile = true - def profileId = profileItem["Profile ID"].toString() - - def profileCli = new HttpCliAction(context) - profileCli.endpoint("${addr.hostString}:${addr.port}") - profileCli.uri("/rest/v1/query_profile/${profileId}") - profileCli.op("get") - profileCli.printResponse(false) - - if (context.config.isCloudMode()) { - profileCli.basicAuthorization(context.config.feCloudHttpUser, context.config.feCloudHttpPassword) - } else { - profileCli.basicAuthorization(context.config.feHttpUser, context.config.feHttpPassword) - } - profileCli.check { profileCode, profileResp -> - if (profileCode != 200) { - throw new IllegalStateException("Get profile failed, url: ${"/rest/v1/query_profile/${profileId}"}, code: ${profileCode}, body:\n${profileResp}") - } - - def jsonSlurper2 = new JsonSlurper() - def profileText = jsonSlurper2.parseText(profileResp).data - profileText = profileText.replace(" ", " ") - profileText = profileText.replace("
", "\n") - this.check(profileText, exception) - } - profileCli.run() - - break - } - } - if (!canFindProfile) { - throw new IllegalStateException("Missing profile with tag: " + tag) - } - } - httpCli.run() + this.check(getProfileBySql(tag), exception) } finally { JdbcUtils.executeToList(conn, "set enable_profile=false") } diff --git a/regression-test/suites/query_profile/adaptive_pipeline_task_serial_read_on_limit.groovy b/regression-test/suites/query_profile/adaptive_pipeline_task_serial_read_on_limit.groovy index f55d7df2811a52..35bd9e8e242077 100644 --- a/regression-test/suites/query_profile/adaptive_pipeline_task_serial_read_on_limit.groovy +++ b/regression-test/suites/query_profile/adaptive_pipeline_task_serial_read_on_limit.groovy @@ -22,39 +22,10 @@ import org.apache.doris.regression.action.ProfileAction def verifyProfileContent = { suiteContext, stmt, serialReadOnLimit -> def profileAction = new ProfileAction(suiteContext) - String profileId = "" - String profileContent = "" - long deadline = System.currentTimeMillis() + 10000 - while (System.currentTimeMillis() <= deadline) { - List profileData = profileAction.getProfileList() - for (def profileItem : profileData) { - if (profileItem["Sql Statement"].toString().contains(stmt)) { - profileId = profileItem["Profile ID"].toString() - break - } - } - - if (profileId != "" && profileId != null) { - profileContent = profileAction.getProfile(profileId) - if (profileContent.contains("MaxScanConcurrency") - || profileContent.contains("Profile Completion State: COMPLETE")) { - logger.info("Profile ID of ${stmt} is ${profileId}") - logger.info("Profile content of ${stmt} is\n${profileContent}") - break - } - logger.info("Profile of ${stmt} is not ready, profileId=${profileId}") - } else { - logger.info("Profile ID of ${stmt} is not found yet") - } - Thread.sleep(500) - } - - if (profileId == "" || profileId == null) { - logger.error("Profile ID of ${stmt} is not found") - return false - } + String profileContent = profileAction.getProfileBySql(stmt, ["MaxScanConcurrency"]) + logger.info("Profile content of ${stmt} is\n${profileContent}") if (!profileContent.contains("MaxScanConcurrency")) { - logger.error("Profile of ${stmt} does not contain MaxScanConcurrency, profileId=${profileId}, content:\n${profileContent}") + logger.error("Profile of ${stmt} does not contain MaxScanConcurrency, content:\n${profileContent}") return false } // Check if the profile contains the expected content diff --git a/regression-test/suites/query_profile/scanner_profile.groovy b/regression-test/suites/query_profile/scanner_profile.groovy index 9b26ce7ec3adc1..520b4c963f7cf2 100644 --- a/regression-test/suites/query_profile/scanner_profile.groovy +++ b/regression-test/suites/query_profile/scanner_profile.groovy @@ -62,20 +62,11 @@ suite('scanner_profile') { select "${token}", * from scanner_profile limit 10; """ def profileAction = new ProfileAction(context) - def getProfileByToken = { pattern -> - List profileData = profileAction.getProfileList() - def profileContent = "" - for (final def profileItem in profileData) { - if (profileItem["Sql Statement"].toString().contains(pattern)) { - profileContent = profileAction.getProfile(profileItem["Profile ID"].toString()) - break - } - } - return profileContent + def getProfileByToken = { pattern, requiredContents -> + return profileAction.getProfileBySql(pattern, requiredContents) } - List profileData = profileAction.getProfileList() - def profileWithLimit1 = getProfileByToken(token) + def profileWithLimit1 = getProfileByToken(token, ["TaskCpuTime", "MaxScanConcurrency"]) logger.info("${token} Profile Data: ${profileWithLimit1}") assertTrue(profileWithLimit1.toString().contains("- TaskCpuTime:")) assertTrue(profileWithLimit1.toString().contains("- MaxScanConcurrency: 1")) @@ -85,7 +76,7 @@ suite('scanner_profile') { select "${token}", * from scanner_profile where id < 10; """ - String profileWithFilter = getProfileByToken(token) + String profileWithFilter = getProfileByToken(token, ["TaskCpuTime", "ScannerCpuTime"]) logger.info("${token} Profile Data: ${profileWithFilter}") assertTrue(profileWithFilter.toString().contains("- TaskCpuTime:")) assertTrue(profileWithFilter.toString().contains("- ScannerCpuTime:")) From 02071ce98492144b683797c2637987ba10e09268 Mon Sep 17 00:00:00 2001 From: shuke <37901441+shuke987@users.noreply.github.com> Date: Wed, 1 Jul 2026 15:44:08 +0800 Subject: [PATCH 3/4] [fix](regression) keep profile DSL compatible --- .../regression/action/ProfileAction.groovy | 58 ++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ProfileAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ProfileAction.groovy index 196f15e53b678c..a8e16099c6fd90 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ProfileAction.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ProfileAction.groovy @@ -192,7 +192,63 @@ class ProfileAction implements SuiteAction { exception = t } - this.check(getProfileBySql(tag), exception) + def httpCli = new HttpCliAction(context) + def addr = context.getFeHttpAddress() + httpCli.endpoint("${addr.hostString}:${addr.port}") + httpCli.uri("/rest/v1/query_profile") + httpCli.op("get") + httpCli.printResponse(false) + + if (context.config.isCloudMode()) { + httpCli.basicAuthorization(context.config.feCloudHttpUser, context.config.feCloudHttpPassword) + } else { + httpCli.basicAuthorization(context.config.feHttpUser, context.config.feHttpPassword) + } + httpCli.check { code, body -> + if (code != 200) { + throw new IllegalStateException("Get profile list failed, code: ${code}, body:\n${body}") + } + + def jsonSlurper = new JsonSlurper() + List profileData = jsonSlurper.parseText(body).data.rows + def canFindProfile = false; + for (final def profileItem in profileData) { + if (profileItem["Sql Statement"].toString().contains(tag)) { + canFindProfile = true + def profileId = profileItem["Profile ID"].toString() + + def profileCli = new HttpCliAction(context) + profileCli.endpoint("${addr.hostString}:${addr.port}") + profileCli.uri("/rest/v1/query_profile/${profileId}") + profileCli.op("get") + profileCli.printResponse(false) + + if (context.config.isCloudMode()) { + profileCli.basicAuthorization(context.config.feCloudHttpUser, context.config.feCloudHttpPassword) + } else { + profileCli.basicAuthorization(context.config.feHttpUser, context.config.feHttpPassword) + } + profileCli.check { profileCode, profileResp -> + if (profileCode != 200) { + throw new IllegalStateException("Get profile failed, url: ${"/rest/v1/query_profile/${profileId}"}, code: ${profileCode}, body:\n${profileResp}") + } + + def jsonSlurper2 = new JsonSlurper() + def profileText = jsonSlurper2.parseText(profileResp).data + profileText = profileText.replace(" ", " ") + profileText = profileText.replace("
", "\n") + this.check(profileText, exception) + } + profileCli.run() + + break + } + } + if (!canFindProfile) { + throw new IllegalStateException("Missing profile with tag: " + tag) + } + } + httpCli.run() } finally { JdbcUtils.executeToList(conn, "set enable_profile=false") } From 56871bdb8d09bff3ef160426f501ac67a03feaee Mon Sep 17 00:00:00 2001 From: shuke <37901441+shuke987@users.noreply.github.com> Date: Wed, 1 Jul 2026 19:22:40 +0800 Subject: [PATCH 4/4] [fix](regression) extend profile readiness wait --- .../org/apache/doris/regression/action/ProfileAction.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ProfileAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ProfileAction.groovy index a8e16099c6fd90..4bf544934b7eab 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ProfileAction.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ProfileAction.groovy @@ -26,7 +26,7 @@ import org.apache.doris.regression.util.JdbcUtils @Slf4j class ProfileAction implements SuiteAction { - private static final long DEFAULT_PROFILE_WAIT_TIMEOUT_MS = 10000 + private static final long DEFAULT_PROFILE_WAIT_TIMEOUT_MS = 30000 private static final long DEFAULT_PROFILE_WAIT_INTERVAL_MS = 500 private static final String PROFILE_COMPLETE = "Profile Completion State: COMPLETE"