A Spark bootstrap project written in Scala with gradle as build tool.

Author: mahen-n

.gitignore

@@ -0,0 +1,49 @@
+# IntelliJ
+.idea
+*.iml
+*.ipr
+*.iws
+out
+# sbt-idea plugin
+.idea_modules/
+
+# scala fmt
+.scalafmt.conf
+
+# Ignore Gradle project-specific cache directory
+.gradle
+
+# local file to ignore
+local
+plugin
+.gitlab-ci.yml
+# ignore build directory
+build
+
+#hive
+derby.log
+metastore_db
+
+### Java template
+*.class
+
+# Package Files #
+*.jar
+*.war
+*.ear
+
+#
+.git-jwn
+Notes
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
+### Emacs template
+# -*- mode: gitignore; -*-
+\#*\#
+/.emacs.desktop
+/.emacs.desktop.lock
+*.elc
+auto-save-list
+tramp
+.\#*

README.md

@@ -1,2 +1,44 @@
-# spark-scala-gradle-bootstrap
-# spark-scala-gradle-bootstrap
+# _spark-scala-gradle-bootstrap_
+
+A Spark bootstrap project written in Scala with gradle as build tool.
+
+## Prerequisites
+
+- [Java](https://java.com/en/download/)
+- [Gradle](https://gradle.org/)
+- [Scala](https://www.scala-lang.org/)
+
+### Build
+
+`./gradlew clean build`
+
+### Coverage
+
+https://github.com/scoverage/gradle-scoverage
+
+## Functional Test Examples
+
+https://github.com/scoverage/gradle-scoverage/blob/master/build.gradle
+
+## Libraries Included
+
+- Spark - 3.4.1
+
+## Useful Links
+
+- [Spark Docs - Root Page](http://spark.apache.org/docs/latest/)
+- [Spark Programming Guide](http://spark.apache.org/docs/latest/programming-guide.html)
+- [Spark Latest API docs](http://spark.apache.org/docs/latest/api/)
+- [Scala API Docs](http://www.scala-lang.org/api/2.12.1/scala/)
+- https://barrelsofdata.com/spark-boilerplate-using-scala
+
+## Issues or Suggestions
+
+# Learn Spark
+
+https://www.databricks.com/wp-content/uploads/2021/06/Ebook_8-Steps-V2.pdf
+
+# References
+
+https://github.com/spark-examples/spark-scala-examples
+> > > > > > > 002f78d (A Spark bootstrap project written in Scala with gradle as build tool.)

build.gradle

@@ -0,0 +1,115 @@
+//buildscript {
+//    repositories {
+//        mavenCentral()
+//    }
+//}
+plugins {
+    id 'scala'
+    id 'application'
+    id "com.github.spotbugs-base"
+    id "com.diffplug.spotless"
+    id 'net.nemerosa.versioning'
+    id 'com.jfrog.artifactory'
+    id 'org.scoverage'
+    id 'template.spark.livyPlugin'
+}
+
+//apply plugin: LivyPlugin
+apply plugin: 'java'
+apply plugin: 'scala'
+apply plugin: 'idea'
+apply plugin: 'maven-publish'
+apply from: "$rootDir/gradle/artifactory.gradle"
+apply from: "$rootDir/gradle/checkstyle.gradle"
+apply from: "$rootDir/gradle/spotless.gradle"
+apply from: "$rootDir/gradle/scoverage.gradle"
+
+group group
+version version
+
+sourceCompatibility = JavaVersion.VERSION_1_8
+targetCompatibility = JavaVersion.VERSION_1_8
+
+configurations {
+    provided
+}
+
+sourceSets {
+    main {
+        compileClasspath += configurations.provided
+    }
+}
+
+application {
+    mainClassName = 'dev.template.spark.Main'
+}
+
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    implementation "org.scalameta:scalafmt-core_${scalaVersion}:${scalafmt}"
+    implementation "org.apache.spark:spark-sql_${scalaVersion}:${sparkVersion}"
+    implementation "org.apache.spark:spark-graphx_${scalaVersion}:${sparkVersion}"
+    implementation "org.apache.spark:spark-launcher_${scalaVersion}:${sparkVersion}"
+    implementation "org.apache.spark:spark-catalyst_${scalaVersion}:${sparkVersion}"
+    implementation "org.apache.spark:spark-streaming_${scalaVersion}:${sparkVersion}"
+    implementation "org.apache.spark:spark-core_${scalaVersion}:${sparkVersion}"
+    implementation "commons-io:commons-io:${commonsIO}"
+
+    implementation "org.apache.hadoop:hadoop-aws:${hadoopAWS}"
+    implementation "org.apache.spark:spark-hive_${scalaVersion}:${sparkVersion}"
+    implementation "io.delta:delta-core_${scalaVersion}:${deltaVersion}"
+    implementation "com.google.guava:guava:31.1-jre"
+    compileOnly "org.scala-lang:scala-library:$scalaVersion"
+    compileOnly "org.scala-lang:scala-compiler:${scalaVersion}"
+
+    testImplementation "org.scalatestplus:junit-4-13_${scalaVersion}:3.2.2.0"
+    testImplementation "junit:junit:${junitVersion}"
+    testRuntimeOnly "org.scala-lang.modules:scala-xml_${scalaVersion}:1.2.0"
+    testImplementation 'org.mockito:mockito-core:5.3.1'
+
+    testImplementation "org.junit.jupiter:junit-jupiter-api:${jupiterApi}"
+    testImplementation "org.scalatest:scalatest_${scalaVersion}:${scalaTests}"
+}
+
+
+jar {
+    classifier 'all'
+    manifest {
+        attributes 'Implementation-Title': title,
+                'Implementation-Version': archiveVersion,
+                'Main-Class': mainClassFile
+    }
+    exclude 'META-INF/*.RSA', 'META-INF/*.SF', 'META-INF/*.DSA'
+    from files(sourceSets.main.output.classesDirs)
+    zip64 true
+}
+
+
+tasks.register('scalaTest', JavaExec) {
+    dependsOn['testClasses']
+    mainClass.set("org.scalatest.tools.Runner")
+    args = ['-R', 'build/classes/scala/test', '-o']
+    classpath = sourceSets.test.runtimeClasspath
+}
+
+test.dependsOn scalaTest
+
+idea {
+    module {
+        // IntelliJ does not know about the standard idiom of provided as used in managing
+        // uber/shaded jar dependencies. Make it so!
+        scopes.PROVIDED.plus += [configurations.provided]
+    }
+}
+
+def runLivy = tasks.register("runLivy", template.spark.Publish) {
+    def avscTree = project.fileTree(allAvscs)
+
+    targetFiles.from(avscTree)
+    schemaFile.value(layout.projectDirectory.file("validator/avro-schema.json"))
+}
+