Adds a uber chart (#47)

* Add krb5-server chart

* krb5-server runs

* Support multiple test cases

* Add cases

* Support skip cleaning up

* Kerberos test sets up config and keytabs

* Fix bugs

* Fix typo

* Enable kerberos for journal nodes

* Fix a bug

* Fix a bug

* Specify journalnode princiapls in the namenode config

* Test client use kerberos

* Use nn0 pod for hdfs admin commands

* Run test cases back to back

* Describe pod when tests fail

* Debug travis

* debug travis

* debug travis

* debug travis

* debug travis

* debug travis

* debug travis

* debug travis

* debug travis

* debug travis

* debug travis

* debug travis

* debug travis

* Fix typo

* debug travis

* debug travis

* Do mount --make-share for travis

* Works end-to-end even in Travis

* Document the krb5-server chart

* Minor clean up

* Clean up

* Use -k option

* Add a chart for common configmap

* Remove vim file

* Switch charts to use the new configmap

* Fix a minor bug

* Fix a typo

* Fix a bug

* Disable service level RPC authorization

* Add group mapping for the hdfs user

* Remove redundant license notices

* CI clean up the hdfs-config chart

* Remove redundant zookeeper param in CI

* Test multiple datanode data dirs

* Test custom Hadoop config values in CI

* Update README.md's

* Cover non-HA in CI

* Add a uber-chart

* zookeeper and journalnode launch

* Clean up structure

* clean up

* journalnode works

* clean up

* Include namenode in uber-chart

* Fix bugs

* debugging namenode startup

* Work around zkfc bug

* Namenodes get ready after retrying a few times

* Include datanodes in uber-chart

* Restart datanode pod using livenessProbe when failing to register

* Fix a bug

* Working on CI now

* Basic CI passes

* Kerberos CI passes

* fix a typo

* fix a bug

* Add a helm repo

* Add subcharts CI case

* CI fo non-HA namenode

* Use zookeeper 1.0.0

* Fix zookeeper options

* Fix a bug in test code

* Update travis yaml

* Simplifies tags

* Include kerberos in uber-chart

* Fix a bug in test

* Use a tag for kerberos

* Check helm diff

* Clean up

* Add new instructions in README.md

* Fix a bug in test code

* Delete old instructions.

* Fix a bug in tests

* Update test instructions

* Add more instruction for PVs

* Support storage class and pv selector

* Add more gold files

* Clean up README.md

* Fix README.md

* Fix a template bug

* Fix a template bug

* Fix a bug from merge

* Updated gold files

* Fix non-HA instruction

* Mount volumes on the parent dir of metadata dir

* Use subpath for metadata

* Mount name subdir in journalnodes just in case

* Update gold files

* Cover how to use an existing zookeeper in README.md

* Moved all default options to uber-chart

* Update gold

* Put some default options for datanode

* Update test/README.md

* Addressed review comments

* Fix tests

Author: kimoonkim

.travis.yml

@@ -2,6 +2,7 @@ sudo: required
 
 env:
   - CASES=_basic.sh
+  - CASES=_basic-subcharts.sh
   - CASES=_kerberos.sh
   - CASES=_single-namenode.sh
 

charts/README.md

@@ -0,0 +1,4 @@
+apiVersion: v1
+name: hdfs-client-k8s
+version: 0.1.0
+description: A client for HDFS on Kubernetes.

charts/hdfs-client-k8s/Chart.yaml

@@ -2,18 +2,22 @@ apiVersion: apps/v1
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
-  name: hdfs-client
+  name: {{ template "hdfs-k8s.client.fullname" . }}
   labels:
-    app: hdfs-client
+    app: {{ template "hdfs-k8s.client.name" . }}
+    chart: {{ template "hdfs-k8s.subchart" . }}
+    release: {{ .Release.Name }}
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app: hdfs-client
+      app: {{ template "hdfs-k8s.client.name" . }}
+      release: {{ .Release.Name }}
   template:
     metadata:
       labels:
-        app: hdfs-client
+        app: {{ template "hdfs-k8s.client.name" . }}
+        release: {{ .Release.Name }}
     spec:
       containers:
         - name: hdfs-client
@@ -30,19 +34,19 @@ spec:
             - name: hdfs-config
               mountPath: /etc/hadoop-custom-conf
               readOnly: true
-            {{- if .Values.kerberosEnabled }}
+            {{- if .Values.global.kerberosEnabled }}
             - name: kerberos-config
               mountPath: /etc/krb5.conf
-              subPath: {{ .Values.kerberosConfigFileName }}
+              subPath: {{ .Values.global.kerberosConfigFileName }}
               readOnly: true
             {{- end }}
       restartPolicy: Always
       volumes:
         - name: hdfs-config
           configMap:
-            name: {{ .Values.hdfsConfigMap }}
-        {{- if .Values.kerberosEnabled }}
+            name: {{ template "hdfs-k8s.config.fullname" . }}
+        {{- if .Values.global.kerberosEnabled }}
         - name: kerberos-config
           configMap:
-            name: {{ .Values.kerberosConfigMap }}
+            name: {{ template "krb5-configmap" . }}
         {{- end }}

charts/hdfs-client/templates/client-deployment.yaml renamed to charts/hdfs-client-k8s/templates/client-deployment.yaml

@@ -1,18 +1,17 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ template "hdfs-config-k8s.fullname" . }}
+  name: {{ template "hdfs-k8s.config.fullname" . }}
   labels:
-    app: {{ template "hdfs-config-k8s.name" . }}
-    chart: {{ template "hdfs-config-k8s.chart" . }}
+    app: {{ template "hdfs-k8s.client.name" . }}
+    chart: {{ template "hdfs-k8s.subchart" . }}
     release: {{ .Release.Name }}
-    heritage: {{ .Release.Service }}
 data:
   core-site.xml: |
     <?xml version="1.0"?>
     <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
     <configuration>
-    {{- if .Values.kerberosEnabled }}
+    {{- if .Values.global.kerberosEnabled }}
       <property>
         <name>hadoop.security.authentication</name>
         <value>kerberos</value>
@@ -48,27 +47,27 @@ data:
         <value>{{ $value }}</value>
       </property>
     {{- end }}
-    {{- if .Values.namenodeHAEnabled }}
+    {{- if .Values.global.namenodeHAEnabled }}
       <property>
         <name>fs.defaultFS</name>
         <value>hdfs://hdfs-k8s</value>
       </property>
       <property>
         <name>ha.zookeeper.quorum</name>
-        <value>{{ .Values.zookeeperQuorum }}</value>
+        <value>{{ template "zookeeper-quorum" . }}</value>
       </property>
     {{- else }}
       <property>
         <name>fs.defaultFS</name>
-        <value>hdfs://hdfs-namenode-0.hdfs-namenode.default.svc.cluster.local:8020</value>
+        <value>hdfs://{{ template "namenode-svc-0" . }}:8020</value>
       </property>
     {{- end }}
     </configuration>
   hdfs-site.xml: |
     <?xml version="1.0"?>
     <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
     <configuration>
-    {{- if .Values.kerberosEnabled }}
+    {{- if .Values.global.kerberosEnabled }}
       <property>
         <name>dfs.block.access.token.enable</name>
         <value>true</value>
@@ -120,7 +119,7 @@ data:
         <name>dfs.datanode.keytab.file</name>
         <value>/etc/security/hdfs.keytab</value>
       </property>
-      {{- if .Values.jsvcEnabled }}
+      {{- if .Values.global.jsvcEnabled }}
       <property>
         <name>dfs.datanode.address</name>
         <value>0.0.0.0:1004</value>
@@ -137,7 +136,7 @@ data:
         <value>{{ $value }}</value>
       </property>
     {{- end }}
-    {{- if .Values.namenodeHAEnabled }}
+    {{- if .Values.global.namenodeHAEnabled }}
       <property>
         <name>dfs.nameservices</name>
         <value>hdfs-k8s</value>
@@ -148,23 +147,23 @@ data:
       </property>
       <property>
         <name>dfs.namenode.rpc-address.hdfs-k8s.nn0</name>
-        <value>hdfs-namenode-0.hdfs-namenode.default.svc.cluster.local:8020</value>
+        <value>{{ template "namenode-svc-0" . }}:8020</value>
       </property>
       <property>
         <name>dfs.namenode.rpc-address.hdfs-k8s.nn1</name>
-        <value>hdfs-namenode-1.hdfs-namenode.default.svc.cluster.local:8020</value>
+        <value>{{ template "namenode-svc-1" . }}:8020</value>
       </property>
       <property>
         <name>dfs.namenode.http-address.hdfs-k8s.nn0</name>
-        <value>hdfs-namenode-0.hdfs-namenode.default.svc.cluster.local:50070</value>
+        <value>{{ template "namenode-svc-0" . }}:50070</value>
       </property>
       <property>
         <name>dfs.namenode.http-address.hdfs-k8s.nn1</name>
-        <value>hdfs-namenode-1.hdfs-namenode.default.svc.cluster.local:50070</value>
+        <value>{{ template "namenode-svc-1" . }}:50070</value>
       </property>
       <property>
         <name>dfs.namenode.shared.edits.dir</name>
-        <value>qjournal://{{ .Values.journalQuorum }}/hdfs-k8s</value>
+        <value>qjournal://{{ template "journalnode-quorum" . }}/hdfs-k8s</value>
       </property>
       <property>
         <name>dfs.ha.automatic-failover.enabled</name>
@@ -193,6 +192,6 @@ data:
       </property>
       <property>
         <name>dfs.datanode.data.dir</name>
-        <value>{{ template "datanode-data-dirs" . }}</value>
+        <value>{{ join "," .Values.global.dataNodeHostPath }}</value>
       </property>
     </configuration>

charts/hdfs-client/Chart.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 name: hdfs-datanode-k8s
-version: 0.2.0
-description: Hadoop Distributed File System (HDFS) hosted by Kubernetes.
+version: 0.1.0
+description: Datanodes for HDFS on Kubernetes.