diff --git a/.jenkins/settings.xml b/.jenkins/settings.xml index b1144552d2cfc..dbec1ec424316 100644 --- a/.jenkins/settings.xml +++ b/.jenkins/settings.xml @@ -14,11 +14,6 @@ ${env.GITHUB_LOGIN} ${env.GITHUB_TOKEN} - - jira - ${env.JIRA_USER} - ${env.JIRA_PASS} - jetbrains ${env.JETBRAINS_USER} diff --git a/Jenkinsfile b/Jenkinsfile index edec2e2e469ea..3450a579f5d2b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -29,10 +29,6 @@ final def jetbrainsCredentials = usernamePassword( credentialsId: 'jetbrains-credentials', usernameVariable: 'JETBRAINS_USER', passwordVariable: 'JETBRAINS_PASS') -final def jiraCredentials = usernamePassword( - credentialsId: 'jira-credentials', - usernameVariable: 'JIRA_USER', - passwordVariable: 'JIRA_PASS') final def gitCredentials = usernamePassword( credentialsId: 'github-credentials', usernameVariable: 'GITHUB_LOGIN', @@ -424,7 +420,6 @@ pipeline { dockerCredentials, ossrhCredentials, jetbrainsCredentials, - jiraCredentials, gpgCredentials]) { script { sh """\ @@ -719,7 +714,7 @@ pipeline { } steps { script { - withCredentials([gitCredentials, dockerCredentials, ossrhCredentials, jetbrainsCredentials, jiraCredentials, gpgCredentials, nexusCredentials]) { + withCredentials([gitCredentials, dockerCredentials, ossrhCredentials, jetbrainsCredentials, gpgCredentials, nexusCredentials]) { configFileProvider([configFile(fileId: 'maven-settings-nexus-zl', variable: 'MAVEN_SETTINGS')]) { sh """ bash .jenkins/scripts/release_legacy.sh $branch_name $finalVersion $extraBuildParams diff --git a/ci/Jenkinsfile-release b/ci/Jenkinsfile-release index 32e17721deb40..d289174fc2ad1 100644 --- a/ci/Jenkinsfile-release +++ b/ci/Jenkinsfile-release @@ -27,10 +27,6 @@ final def jetbrainsCredentials = usernamePassword( credentialsId: 'jetbrains-credentials', usernameVariable: 'JETBRAINS_USER', passwordVariable: 'JETBRAINS_PASS') -final def jiraCredentials = usernamePassword( - credentialsId: 'jira-credentials', - usernameVariable: 'JIRA_USER', - passwordVariable: 'JIRA_PASS') final def gitCredentials = usernamePassword( credentialsId: 'github-credentials', usernameVariable: 'GITHUB_LOGIN', @@ -296,7 +292,6 @@ pipeline { dockerCredentials, ossrhCredentials, jetbrainsCredentials, - jiraCredentials, gpgCredentials]) { script { sh """\ @@ -451,7 +446,6 @@ pipeline { withCredentials([nexusCredentials, ossrhCredentials, jetbrainsCredentials, - jiraCredentials, gpgCredentials]) { script { @@ -469,7 +463,6 @@ pipeline { steps { withCredentials([nexusCredentials, ossrhCredentials, - jiraCredentials, gpgCredentials]) { script { diff --git a/documentation/pom.xml b/documentation/pom.xml index 73b04ac898bc1..a303264786191 100644 --- a/documentation/pom.xml +++ b/documentation/pom.xml @@ -447,8 +447,6 @@ true ${project.basedir}/src/main/antora/modules/ROOT/pages - ${talend.jira.username} - ${talend.jira.password} ${project.version} offline=${settings.offline} ${talend.github.username} diff --git a/documentation/src/build/DecryptCredentials.groovy b/documentation/src/build/DecryptCredentials.groovy index 211fa052dfcc3..916dd4edc01e5 100644 --- a/documentation/src/build/DecryptCredentials.groovy +++ b/documentation/src/build/DecryptCredentials.groovy @@ -20,7 +20,7 @@ def decryptAndSet(name) { def serverId = project.properties.getProperty("talend.${name}.serverId", name) def serverIt = session.settings.servers.findAll { it.id == serverId }.iterator() if (!serverIt.hasNext()) { - log.warn("no server '${serverId}' in your settings.xml, will skip changelog generation") + log.warn("no server '${serverId}' in your settings.xml, will skip contributors generation") project.properties.setProperty("talend.${name}.username", 'skip') project.properties.setProperty("talend.${name}.password", 'skip') return @@ -35,6 +35,5 @@ def decryptAndSet(name) { } -decryptAndSet('jira') decryptAndSet('github') diff --git a/documentation/src/main/antora/modules/ROOT/pages/release-process.adoc b/documentation/src/main/antora/modules/ROOT/pages/release-process.adoc index 9557dbb17e7a6..218cc89494316 100644 --- a/documentation/src/main/antora/modules/ROOT/pages/release-process.adoc +++ b/documentation/src/main/antora/modules/ROOT/pages/release-process.adoc @@ -34,11 +34,6 @@ Add the required credentials to your Maven `settings.xml` file for the release. ${env.TLND_GITHUB_USER} ${env.TLND_GITHUB_PASSWORD} - - jira - ${env.TLND_JIRA_USER} - ${env.TLND_JIRA_PASS} - jetbrains ${env.JETBRAINS_USER} @@ -83,15 +78,8 @@ NOTE: All the environment variables should be either set in your environment or 1. The OSSH variables are your Sonatype OSS repository credentials with the permissions to deploy on org.talend. If you don't have them already, you can ask for them at https://issues.sonatype.org/projects/OSSRH using your Talend address. 2. The GitHub variables are your GitHub account credentials. they are mainly used to update the documentation. -3. The JIRA variables are your Talend account credentials with read permissions on https://jira.talendforge.org/projects/TCOMP/. -4. The Blackduck configuration is used for security scans. -5. The GPG variables reference the key mentioned in <>. - -== Cleaning up JIRA - -The release connects to JIRA to retrieve and create the release notes. It uses all `TCOMP` issues which have the `changelog` label. - -Before any release, go through all the issues of the version to release and add or remove the `changelog` label depending on the issues that need to appear in the release notes. +3. The Blackduck configuration is used for security scans. +4. The GPG variables reference the key mentioned in <>. == Releasing diff --git a/documentation/src/main/java/org/talend/runtime/documentation/Generator.java b/documentation/src/main/java/org/talend/runtime/documentation/Generator.java index 78e93ab542e17..b74d68140f876 100644 --- a/documentation/src/main/java/org/talend/runtime/documentation/Generator.java +++ b/documentation/src/main/java/org/talend/runtime/documentation/Generator.java @@ -15,7 +15,6 @@ */ package org.talend.runtime.documentation; -import static java.lang.Math.min; import static java.util.Collections.emptyMap; import static java.util.Comparator.comparing; import static java.util.Locale.ENGLISH; @@ -23,24 +22,16 @@ import static java.util.Optional.of; import static java.util.Optional.ofNullable; import static java.util.concurrent.TimeUnit.SECONDS; -import static java.util.stream.Collectors.collectingAndThen; -import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.joining; -import static java.util.stream.Collectors.mapping; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; -import static java.util.stream.Collectors.toSet; -import static javax.ws.rs.core.MediaType.APPLICATION_JSON_TYPE; import static lombok.AccessLevel.PRIVATE; import static org.apache.ziplock.JarLocation.jarLocation; -import java.io.BufferedReader; import java.io.File; -import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.PrintStream; -import java.io.StringReader; import java.lang.annotation.Annotation; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Modifier; @@ -53,14 +44,10 @@ import java.time.LocalTime; import java.time.ZonedDateTime; import java.util.ArrayList; -import java.util.Base64; import java.util.Collection; import java.util.Comparator; -import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Set; import java.util.TreeMap; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; @@ -71,10 +58,8 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; -import java.util.function.Supplier; import java.util.regex.Pattern; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -90,14 +75,8 @@ import javax.json.bind.JsonbBuilder; import javax.json.bind.JsonbConfig; import javax.json.bind.config.PropertyOrderStrategy; -import javax.ws.rs.client.Client; -import javax.ws.rs.client.ClientBuilder; -import javax.ws.rs.client.WebTarget; -import javax.ws.rs.core.GenericType; -import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.text.WordUtils; -import org.apache.johnzon.jaxrs.jsonb.jaxrs.JsonbJaxrsProvider; import org.apache.xbean.finder.AnnotationFinder; import org.apache.xbean.finder.archive.FileArchive; import org.apache.xbean.finder.archive.JarArchive; @@ -151,14 +130,14 @@ public class Generator { private static final Pattern SNAPSHOT = Pattern.compile("(-SNAPSHOT|M\\d+-SNAPSHOT)"); public static void main(final String[] args) { - if (Boolean.parseBoolean(args[7]) || Boolean.getBoolean(System.getenv("TRAVIS"))) { + if (Boolean.parseBoolean(args[5]) || Boolean.getBoolean(System.getenv("TRAVIS"))) { log.info("Skipping doc generation as requested"); return; } final File generatedDir = new File(args[0], "_partials"); generatedDir.mkdirs(); - final String version = SNAPSHOT.matcher(args[3]).replaceAll(""); + final String version = SNAPSHOT.matcher(args[1]).replaceAll(""); try (final Tasks tasks = new Tasks()) { tasks.register(Asciidoctor.Factory::create).thenApply(adoc -> { @@ -180,11 +159,11 @@ public static void main(final String[] args) { tasks.register(() -> generatedScanningExclusions(generatedDir)); tasks.register(() -> generatedRemoteEngineCustomizerHelp(generatedDir)); - final boolean offline = "offline=true".equals(args[4]); + final boolean offline = "offline=true".equals(args[2]); if (offline) { - log.info("System is offline, skipping jira changelog and github contributor generation"); + log.info("System is offline, skipping github contributor generation"); } else { - tasks.register(() -> generatedContributors(generatedDir, args[5], args[6])); + tasks.register(() -> generatedContributors(generatedDir, args[3], args[4])); } } } @@ -444,224 +423,6 @@ private static void generatedContributors(final File generatedDir, final String } } - // to avoid to be very slow we just grab current version and extract other versions - // from the previous file - private static void generatedJira(final File generatedDir, final String username, final String password, - final String version) { - if (username == null || username.trim().isEmpty() || "skip".equals(username)) { - log.error("No JIRA credentials, will skip changelog generation"); - return; - } - - final String project = "TCOMP"; - final String jiraBase = "https://jira.talendforge.org"; - - final File file = new File(generatedDir, "generated_changelog.adoc"); - final Client client = ClientBuilder.newClient().register(new JsonbJaxrsProvider<>()); - final String auth = "Basic " - + Base64.getEncoder().encodeToString((username + ':' + password).getBytes(StandardCharsets.UTF_8)); - - try { - final WebTarget restApi = - client.target(jiraBase + "/rest/api/2").property("http.connection.timeout", 60000L); - final List versions = restApi - .path("project/{project}/versions") - .resolveTemplate("project", project) - .request(APPLICATION_JSON_TYPE) - .header("Authorization", auth) - .get(new GenericType>() { - }); - - final List jiraLoggedVersions = versions - .stream() - .filter(v -> (v.isReleased() || jiraVersionMatches(version, v.getName()))) - .toList(); - if (jiraLoggedVersions.isEmpty()) { - try (final PrintStream stream = new PrintStream(new WriteIfDifferentStream(file))) { - stream.println("No version found."); - } - return; - } - - final Map changelogPerVersion = new HashMap<>(); - try (final BufferedReader reader = - new BufferedReader(new StringReader(String.join("\n", Files.readAllLines(file.toPath()))))) { - final StringBuilder builder = new StringBuilder(); - String line; - String versionRead = null; - while ((line = reader.readLine()) != null) { - if (builder.length() == 0 && line.trim().isEmpty()) { - continue; - } - if (line.startsWith("== Version ")) { - if (builder.length() != 0) { - changelogPerVersion.put(versionRead, builder.toString()); - builder.setLength(0); - } - versionRead = line.substring("== Version ".length()).replace(" (dev)", ""); - } - builder.append(line).append('\n'); - } - if (builder.length() != 0) { - changelogPerVersion.put(versionRead, builder.toString()); - builder.setLength(0); - } - } catch (final IOException e) { - throw new IllegalStateException(e); - } - - final int maxVersionPerQuery = 10; - final BiFunction searchFrom = (jql, startAt) -> restApi - .path("search") - .queryParam("jql", jql) - .queryParam("startAt", startAt) - .request(APPLICATION_JSON_TYPE) - .header("Authorization", auth) - .get(JiraIssues.class); - final BiFunction> paginate = - new BiFunction>() { - - @Override - public Stream apply(final String jql, final JiraIssues issues) { - final long nextStartAt = issues.getStartAt() + issues.getMaxResults(); - final Stream fetched = Stream.of(issues); - return issues.getTotal() > nextStartAt - ? Stream.concat(fetched, apply(jql, searchFrom.apply(jql, nextStartAt))).parallel() - : fetched; - } - }; - final Set includeStatus = - Stream.of("closed", "resolved", "development done", "qa done", "done").collect(toSet()); - - final List queriedVersion = jiraLoggedVersions - .stream() - .filter(it -> !changelogPerVersion.containsKey(it.getName()) || version.equals(it.getName())) - .toList(); - final Map>> issues = IntStream - .range(0, (queriedVersion.size() + maxVersionPerQuery - 1) / maxVersionPerQuery) - .mapToObj(pageIdx -> queriedVersion - .subList(pageIdx * maxVersionPerQuery, - min(maxVersionPerQuery * (pageIdx + 1), queriedVersion.size()))) - .map(pageVersions -> "project=" + project + " AND labels=\"changelog\"" - + pageVersions - .stream() - .map(v -> "fixVersion=" + v.getName()) - .collect(joining(" OR ", " AND (", ")"))) - .flatMap(jql -> Stream - .of(searchFrom.apply(jql, 0L)) - .flatMap(it -> paginate.apply(jql, it)) - .flatMap(i -> ofNullable(i.getIssues()).map(Collection::stream).orElseGet(Stream::empty)) - .filter(issue -> includeStatus - .contains(issue.getFields().getStatus().getName().toLowerCase(ENGLISH))) - .flatMap(i -> i.getFields().getFixVersions().stream().map(v -> Pair.of(v, i)))) - .collect(groupingBy(pair -> pair.getKey().getName(), () -> new TreeMap<>(versionComparator()), - groupingBy(pair -> pair.getValue().getFields().getIssuetype().getName(), TreeMap::new, - collectingAndThen(mapping(Pair::getValue, toList()), l -> { - l.sort(comparing(JiraIssue::getKey)); - return l; - })))); - issues - .forEach((name, issuesMap) -> changelogPerVersion - .put(name, "\n\n== Version " + name + issuesMap - .entrySet() - .stream() - .collect((Supplier) StringBuilder::new, - (builder, issuesByType) -> builder - .append("\n\n=== ") - .append(issuesByType.getKey()) - .append("\n\n") - .append(issuesByType - .getValue() - .stream() - .collect((Supplier) StringBuilder::new, - // note: for now we don't use the - // description since - // it is not that useful - (a, i) -> a - .append("- link:") - .append(jiraBase) - .append("/browse/") - .append(i.getKey()) - .append("[") - .append(i.getKey()) - .append("^]") - .append(": ") - .append(i.getFields().getSummary().trim()) - .append(" ") - .append(i.getFields() - .getComponents() - .stream() - .map(c -> c.getName() - .trim()) - .filter(c -> !"dependency-update" - .equals(c)) - .map(n -> String.format( - "link:search.html?query=%s[%s^,role='dockey']", - n, n)) - .collect(joining(" "))) - .append("\n"), - StringBuilder::append)) - .append('\n'), - StringBuilder::append))); - - final String changelog = changelogPerVersion.entrySet() - .stream() - .sorted((v1, v2) -> { - if (v1.equals(v2)) { - return 0; - } - final int[] parts1 = Stream.of(v1.getKey().replace(" (dev)", "").replace("M", ".").split("\\.")) - .mapToInt(Integer::parseInt) - .toArray(); - final int[] parts2 = Stream.of(v2.getKey().replace(" (dev)", "").replace("M", ".").split("\\.")) - .mapToInt(Integer::parseInt) - .toArray(); - for (int i = 0; i < parts1.length; i++) { - if (parts2.length <= i) { - return 1; - } - final int comp = parts2[i] - parts1[i]; - if (comp != 0) { - return comp; - } - } - return 0; - }) - .map(Map.Entry::getValue) - .collect(StringBuilder::new, StringBuilder::append, StringBuilder::append) - .toString(); - - try (final PrintStream stream = new PrintStream(new WriteIfDifferentStream(file))) { - stream.println(changelog); - } - } finally { - client.close(); - } - } - - private static Comparator versionComparator() { - return (o1, o2) -> { - final String[] parts1 = o1.split("\\."); - final String[] parts2 = o2.split("\\."); - for (int i = 0; i < Math.max(parts1.length, parts2.length); i++) { - try { - final int major = (parts2.length > i ? Integer.parseInt(parts2[i]) : 0) - - (parts1.length > i ? Integer.parseInt(parts1[i]) : 0); - if (major != 0) { - return major; - } - } catch (final NumberFormatException nfe) { - // no-op - } - } - return o2.compareTo(o1); - }; - } - - private static boolean jiraVersionMatches(final String ref, final String name) { - return ref.equals(name) || ref.equals(name + ".0"); - } - private static void generatedServerConfiguration(final File generatedDir) { final File file = new File(generatedDir, "generated_server-configuration.adoc"); try (final PrintStream stream = new PrintStream(new WriteIfDifferentStream(file))) { @@ -1303,74 +1064,6 @@ private static final class Constraint { private final String description; } - @Data - public static class JiraVersion { - - private String id; - - private String name; - - private boolean released; - - private boolean archived; - - private long projectId; - } - - @Data - public static class JiraIssues { - - private long startAt; - - private long maxResults; - - private long total; - - private Collection issues; - } - - @Data - public static class IssueType { - - private String name; - } - - @Data - public static class JiraIssue { - - private String id; - - private String key; - - private Fields fields; - } - - @Data - public static class JiraComponent { - - private String self; - - private String id; - - private String name; - } - - @Data - public static class Fields { - - private String summary; - - private String description; - - private IssueType issuetype; - - private Status status; - - private Collection fixVersions; - - private Collection components; - } - @Data @AllArgsConstructor @NoArgsConstructor @@ -1387,12 +1080,6 @@ public static class DocumentationItem { private String link; } - @Data - public static class Status { - - private String name; - } - @Data @AllArgsConstructor @NoArgsConstructor