diff --git a/gradle.properties b/gradle.properties index 48687e86bb..9716bfead6 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ publishedPluginVersion=1.67.2 -pluginVersion=1.68.0-pre8 +pluginVersion=1.68.0-pre9 pluginName=readonlyrest org.gradle.jvmargs=-Xmx6144m diff --git a/ror-tools-core/src/main/scala/tech/beshu/ror/tools/core/patches/internal/filePatchers/JarManifestModifier.scala b/ror-tools-core/src/main/scala/tech/beshu/ror/tools/core/patches/internal/filePatchers/JarManifestModifier.scala index 678123e557..8f1c6a2273 100644 --- a/ror-tools-core/src/main/scala/tech/beshu/ror/tools/core/patches/internal/filePatchers/JarManifestModifier.scala +++ b/ror-tools-core/src/main/scala/tech/beshu/ror/tools/core/patches/internal/filePatchers/JarManifestModifier.scala @@ -21,7 +21,7 @@ import tech.beshu.ror.tools.core.utils.EsDirectory import tech.beshu.ror.tools.core.utils.FileUtils.{getFilePermissionsAndOwner, setFilePermissionsAndOwner} import java.util.UUID -import java.util.jar.{JarFile, JarOutputStream} +import java.util.jar.{JarEntry, JarFile, JarOutputStream} import scala.jdk.CollectionConverters.IteratorHasAsScala import scala.util.Using @@ -67,7 +67,9 @@ object JarManifestModifier { originalJarFile.entries().asIterator().asScala.foreach { entry => val name = entry.getName if (!name.equalsIgnoreCase("META-INF/MANIFEST.MF")) { - jarOutput.putNextEntry(entry) + val newEntry = new JarEntry(name) + newEntry.setTime(entry.getTime) + jarOutput.putNextEntry(newEntry) Using(originalJarFile.getInputStream(entry))(_.transferTo(jarOutput)).fold( ex => throw IllegalStateException(s"Could not copy content of ${entry.getName} because of [${ex.getMessage}]", ex), (_: Long) => () diff --git a/tests-utils/src/main/scala/tech/beshu/ror/utils/containers/windows/WindowsEsDirectoryManager.scala b/tests-utils/src/main/scala/tech/beshu/ror/utils/containers/windows/WindowsEsDirectoryManager.scala index 6d0b10bfab..437755d1d0 100644 --- a/tests-utils/src/main/scala/tech/beshu/ror/utils/containers/windows/WindowsEsDirectoryManager.scala +++ b/tests-utils/src/main/scala/tech/beshu/ror/utils/containers/windows/WindowsEsDirectoryManager.scala @@ -17,14 +17,11 @@ package tech.beshu.ror.utils.containers.windows import com.typesafe.scalalogging.LazyLogging -import os.* -import tech.beshu.ror.utils.containers.images.Elasticsearch import tech.beshu.ror.utils.containers.images.Elasticsearch.Config import java.io.{BufferedInputStream, FileOutputStream} import java.nio.file.{Files, StandardCopyOption} import java.util.zip.ZipInputStream -import scala.jdk.CollectionConverters.* import scala.language.postfixOps import scala.util.Using @@ -36,9 +33,6 @@ object WindowsEsDirectoryManager extends LazyLogging { def basePath: os.Path = os.pwd / "windows-es" - def tempPath: os.Path = - os.pwd / "temp" - def downloadsPath: os.Path = basePath / "downloads" @@ -69,6 +63,11 @@ object WindowsEsDirectoryManager extends LazyLogging { } } + def cleanDownloadsDirectory(): Unit = { + logger.info(s"Removing all files from Windows ES downloads directory") + os.remove.all(downloadsPath) + } + private def doDownloadEsZipFileWithProgress(esVersion: String): Unit = { val url = downloadUrl(esVersion) val dest = zipFilePath(esVersion) diff --git a/tests-utils/src/main/scala/tech/beshu/ror/utils/containers/windows/WindowsEsSetup.scala b/tests-utils/src/main/scala/tech/beshu/ror/utils/containers/windows/WindowsEsSetup.scala index 801ad8ff64..2705ac8c85 100644 --- a/tests-utils/src/main/scala/tech/beshu/ror/utils/containers/windows/WindowsEsSetup.scala +++ b/tests-utils/src/main/scala/tech/beshu/ror/utils/containers/windows/WindowsEsSetup.scala @@ -25,6 +25,7 @@ import tech.beshu.ror.utils.containers.images.Elasticsearch.Plugin.PluginInstall import tech.beshu.ror.utils.containers.windows.WindowsEsDirectoryManager.* import tech.beshu.ror.utils.containers.windows.WindowsEsPortProvider.* import tech.beshu.ror.utils.containers.windows.WindowsEsRunner.{WindowsEsProcess, startEs} +import tech.beshu.ror.utils.misc.ScalaUtils.retry import java.util.function.Consumer import scala.language.postfixOps @@ -38,8 +39,11 @@ object WindowsEsSetup extends LazyLogging { } def prepareEs(elasticsearch: Elasticsearch): Unit = { - downloadEsZipFileWithProgress(elasticsearch.esVersion) - unzipEs(elasticsearch.esVersion, elasticsearch.config) + // The ES zip file sometimes cannot be unzipped when running CI job. In that case we delete the downloaded file and try again. + retry(times = 3, cleanBeforeRetrying = cleanDownloadsDirectory()) { + downloadEsZipFileWithProgress(elasticsearch.esVersion) + unzipEs(elasticsearch.esVersion, elasticsearch.config) + } replaceConfigFile(elasticsearch) installPlugins(elasticsearch) } diff --git a/tests-utils/src/main/scala/tech/beshu/ror/utils/gradle/RorPluginGradleProject.scala b/tests-utils/src/main/scala/tech/beshu/ror/utils/gradle/RorPluginGradleProject.scala index af03e98137..0fa787581f 100644 --- a/tests-utils/src/main/scala/tech/beshu/ror/utils/gradle/RorPluginGradleProject.scala +++ b/tests-utils/src/main/scala/tech/beshu/ror/utils/gradle/RorPluginGradleProject.scala @@ -17,6 +17,7 @@ package tech.beshu.ror.utils.gradle import better.files.* +import com.typesafe.scalalogging.LazyLogging import org.gradle.tooling.GradleConnector import java.io.File as JFile @@ -48,7 +49,7 @@ object RorPluginGradleProject { .toList } -class RorPluginGradleProject(val moduleName: String) { +class RorPluginGradleProject(val moduleName: String) extends LazyLogging { private val project = esProject(moduleName) private val esProjectProperties = GradleProperties @@ -60,8 +61,10 @@ class RorPluginGradleProject(val moduleName: String) { .getOrElse(throw new IllegalStateException("cannot load root project gradle.properties file")) def assemble: Option[JFile] = { + logger.info(s"Assembling ROR in module $moduleName") runTask(moduleName + ":packageRorPlugin") val plugin = new JFile(project, "build/distributions/" + pluginName) + logger.info(s"Finished assembling ROR in module $moduleName") if (!plugin.exists) None else Some(plugin) } diff --git a/tests-utils/src/main/scala/tech/beshu/ror/utils/misc/EsStartupChecker.scala b/tests-utils/src/main/scala/tech/beshu/ror/utils/misc/EsStartupChecker.scala index 9bb5137fd7..5fe884a0ef 100644 --- a/tests-utils/src/main/scala/tech/beshu/ror/utils/misc/EsStartupChecker.scala +++ b/tests-utils/src/main/scala/tech/beshu/ror/utils/misc/EsStartupChecker.scala @@ -25,6 +25,7 @@ import org.apache.http.client.methods.HttpGet import tech.beshu.ror.utils.httpclient.HttpResponseHelper.deserializeJsonBody import tech.beshu.ror.utils.httpclient.RestClient import tech.beshu.ror.utils.misc.EsStartupChecker.{ClusterNotReady, Mode} +import tech.beshu.ror.utils.misc.ScalaUtils.retryBackoff import scala.concurrent.duration.* import scala.language.postfixOps @@ -35,24 +36,12 @@ class EsStartupChecker private(name: String, extends LazyLogging { def waitForStart(): Boolean = { - retryBackoff(clusterIsReady(client), maxRetries = 150, interval = 2 seconds) + retryBackoff(clusterIsReady(client), maxRetries = 150, firstDelay = 2 seconds, backOffScaler = 1) .map((_: Unit) => true) .onErrorRecover(_ => false) .runSyncUnsafe(5 minutes) } - private def retryBackoff[A](source: Task[A], - maxRetries: Int, - interval: FiniteDuration): Task[A] = { - source.onErrorHandleWith { - case ex: Exception => - if (maxRetries > 0) - retryBackoff(source, maxRetries - 1, interval).delayExecution(interval) - else - Task.raiseError(ex) - } - } - private def clusterIsReady(client: RestClient): Task[Unit] = { Resource .make( diff --git a/tests-utils/src/main/scala/tech/beshu/ror/utils/misc/ScalaUtils.scala b/tests-utils/src/main/scala/tech/beshu/ror/utils/misc/ScalaUtils.scala index dbc5278f6c..f680af7729 100644 --- a/tests-utils/src/main/scala/tech/beshu/ror/utils/misc/ScalaUtils.scala +++ b/tests-utils/src/main/scala/tech/beshu/ror/utils/misc/ScalaUtils.scala @@ -19,14 +19,17 @@ package tech.beshu.ror.utils.misc import java.time.Duration import cats.Functor import cats.implicits.* +import com.typesafe.scalalogging.LazyLogging import monix.eval.Task import java.time.format.DateTimeFormatter +import scala.annotation.tailrec import scala.concurrent.duration.* import scala.language.{implicitConversions, postfixOps} -import scala.util.{Success, Try} +import scala.util.Try +import scala.util.control.NonFatal -object ScalaUtils { +object ScalaUtils extends LazyLogging { implicit class StringOps(val value: String) extends AnyVal { def stripMarginAndReplaceWindowsLineBreak: String = { @@ -73,14 +76,28 @@ object ScalaUtils { implicit def finiteDurationToJavaDuration(interval: FiniteDuration): Duration = Duration.ofMillis(interval.toMillis) - def retry(times: Int)(action: Unit): Unit = { - LazyList - .fill(times)(()) - .foldLeft(Success(()): Try[Unit]) { - case (Success(_), _) => Try(action) - case (failure, _) => failure + def retry(times: Int, cleanBeforeRetrying: => Unit = ())(action: => Unit): Unit = { + @tailrec + def loop(attempt: Int): Unit = { + try { + action + } catch { + case NonFatal(e) => + val nextAttempt = attempt + 1 + if (nextAttempt > times) { + logger.error(s"Attempt $attempt failed: ${e.getMessage}. Retries exhausted, failing with exception.") + throw e + } else { + logger.error(s"Attempt $attempt failed: ${e.getMessage}", e) + logger.warn(s"Starting cleaning after failed attempt") + cleanBeforeRetrying + logger.warn(s"Retrying...") + loop(nextAttempt) + } } - .get + } + + loop(1) } def retryBackoff[A](source: Task[A],