diff --git a/contrib/templates/scala-job/.gitignore b/contrib/templates/scala-job/.gitignore new file mode 100644 index 0000000..de811f1 --- /dev/null +++ b/contrib/templates/scala-job/.gitignore @@ -0,0 +1,2 @@ + +.databricks diff --git a/contrib/templates/scala-job/.idea/.gitignore b/contrib/templates/scala-job/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/contrib/templates/scala-job/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/contrib/templates/scala-job/.idea/codeStyles/Project.xml b/contrib/templates/scala-job/.idea/codeStyles/Project.xml new file mode 100644 index 0000000..919ce1f --- /dev/null +++ b/contrib/templates/scala-job/.idea/codeStyles/Project.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/contrib/templates/scala-job/.idea/codeStyles/codeStyleConfig.xml b/contrib/templates/scala-job/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 0000000..a55e7a1 --- /dev/null +++ b/contrib/templates/scala-job/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/contrib/templates/scala-job/.idea/misc.xml b/contrib/templates/scala-job/.idea/misc.xml new file mode 100644 index 0000000..07115cd --- /dev/null +++ b/contrib/templates/scala-job/.idea/misc.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/contrib/templates/scala-job/.idea/modules.xml b/contrib/templates/scala-job/.idea/modules.xml new file mode 100644 index 0000000..26c34d5 --- /dev/null +++ b/contrib/templates/scala-job/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/contrib/templates/scala-job/.idea/mydabs-template.iml b/contrib/templates/scala-job/.idea/mydabs-template.iml new file mode 100644 index 0000000..d6ebd48 --- /dev/null +++ b/contrib/templates/scala-job/.idea/mydabs-template.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/contrib/templates/scala-job/.idea/vcs.xml b/contrib/templates/scala-job/.idea/vcs.xml new file mode 100644 index 0000000..bf09c00 --- /dev/null +++ b/contrib/templates/scala-job/.idea/vcs.xml @@ -0,0 +1,33 @@ + + + + + + \ No newline at end of file diff --git a/contrib/templates/scala-job/databricks_template_schema.json b/contrib/templates/scala-job/databricks_template_schema.json new file mode 100644 index 0000000..55b6de2 --- /dev/null +++ b/contrib/templates/scala-job/databricks_template_schema.json @@ -0,0 +1,72 @@ +{ + "properties": { + "project_name": { + "type": "string", + "default": "project_name", + "description": "Name of project (press enter to use default name)", + "order": 1 + }, + "jar_dest_path": { + "type": "string", + "description": "Destination path in Databricks where the JAR will be stored. TODO: instructions on creating /Volumes", + "default": "/Volumes/main/nija/dabs-scala/${workspace.current_user.short_name}", + "order": 3 + }, + "cluster_key": { + "type": "string", + "description": "Enter the cluster key for an existing cluster or leave empty to create a new cluster", + "default": "", + "order": 5 + }, + "cluster_type": { + "type": "string", + "enum": ["Standard", "Dedicated"], + "default": "Standard", + "description": "Select cluster type: Dedicated or Standard", + "order": 8 + }, + "jar_allowlisted": { + "skip_prompt_if": { + "properties": { + "cluster_type": { + "const": "Dedicated" + } + } + }, + "type": "boolean", + "description": "If Standard, is the JAR allowlisted by the admin for your workspace? (If not, inform admin: https://docs.databricks.com/en/data-governance/unity-catalog/manage-privileges/allowlist.html, otherwise type true)", + "default": true, + "order": 9 + }, + "main_class_name": { + "type": "string", + "description": "Main class name including package (entry point of the JAR execution)", + "default": "at.nija.Main", + "order": 10 + }, + "dbr_version": { + "type": "string", + "description": "Databricks Runtime (DBR) version (e.g., 15.4)", + "default": "15.4", + "order": 11 + }, + "scala_version": { + "type": "string", + "description": "Scala version (e.g., 2.12). Run scala -version to find it. Note: Only support 2.12 and 2.13", + "default": "2.12", + "order": 12 + }, + "scala_maintenance_version": { + "type": "string", + "description": "Scala maintenance version (e.g., 12). Run scala -version to find it.", + "default": "18", + "order": 13 + }, + "organization": { + "type": "string", + "description": "Organization name", + "default": "at.nija", + "order": 14 + } + } +} \ No newline at end of file diff --git a/contrib/templates/scala-job/template/build.sbt.tmpl b/contrib/templates/scala-job/template/build.sbt.tmpl new file mode 100644 index 0000000..28887ba --- /dev/null +++ b/contrib/templates/scala-job/template/build.sbt.tmpl @@ -0,0 +1,17 @@ +scalaVersion := "{{.scala_version}}.{{.scala_maintenance_version}}" + +name := "{{.project_name}}" +organization := "{{.organization}}" +version := "0.1" + +libraryDependencies += "com.databricks" % "databricks-connect" % "{{.dbr_version}}.+" + +assembly / assemblyOption ~= { _.withIncludeScala(false) } +assembly / assemblyExcludedJars := { + val cp = (assembly / fullClasspath).value + cp filter { _.data.getName.matches("scala-.*") } // remove Scala libraries +} + +assemblyMergeStrategy := { + case _ => MergeStrategy.preferProject +} \ No newline at end of file diff --git a/contrib/templates/scala-job/template/databricks.yml.tmpl b/contrib/templates/scala-job/template/databricks.yml.tmpl new file mode 100644 index 0000000..7b4657a --- /dev/null +++ b/contrib/templates/scala-job/template/databricks.yml.tmpl @@ -0,0 +1,54 @@ +# This is a Databricks asset bundle definition for {{.project_name}}. +# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. +bundle: + name: {{.project_name}} + +workspace: + host: {{workspace_host}} + artifact_path: {{.jar_dest_path}} + +{{- $version := "0.1" }} + +artifacts: + default: + type: jar + build: sbt package && sbt assembly + path: . + files: + - source: ./target/scala-{{.scala_version}}/{{.project_name}}-assembly-{{$version}}.jar + +resources: + jobs: + {{.project_name}}: + name: {{.project_name}} + tasks: + - task_key: main_task + {{- if .cluster_key }} + existing_cluster_id: {{.cluster_key}} + {{- else }} + job_cluster_key: {{.project_name}}_job_cluster + {{- end }} + spark_jar_task: + main_class_name: {{.main_class_name}} + libraries: + - jar: ./target/scala-{{.scala_version}}/{{.project_name}}-assembly-{{$version}}.jar + {{- if not .cluster_key }} + job_clusters: + - job_cluster_key: {{.project_name}}_job_cluster + new_cluster: + spark_version: {{.dbr_version}}.x-scala{{.scala_version}} + node_type_id: i3.xlarge # Default instance type (can be changed) + autoscale: + max_workers: 2 + min_workers: 2 + {{- if eq .cluster_type "Standard" }} + data_security_mode: USER_ISOLATION + {{- end }} + {{- end }} + +targets: + dev: + mode: development + default: true + workspace: + host: {{workspace_host}} \ No newline at end of file diff --git a/contrib/templates/scala-job/template/project/plugins.sbt.tmpl b/contrib/templates/scala-job/template/project/plugins.sbt.tmpl new file mode 100644 index 0000000..48464b8 --- /dev/null +++ b/contrib/templates/scala-job/template/project/plugins.sbt.tmpl @@ -0,0 +1 @@ +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.0.0") \ No newline at end of file