Merge pull request #74 from feiskyer/disable-haproxy-opts

Add options to disable hyper pod-internal services

Author: feiskyer

cmd/kube-proxy/app/options/options.go

@@ -50,6 +50,7 @@ type ProxyServerConfig struct {
 	NodeRef                        *api.ObjectReference // Reference to this node.
 	MasqueradeAll                  bool
 	CleanupAndExit                 bool
+	DisableHyperInternalService    bool
 	KubeAPIQPS                     float32
 	KubeAPIBurst                   int
 	UDPIdleTimeout                 time.Duration
@@ -77,6 +78,7 @@ func NewProxyConfig() *ProxyServerConfig {
 // AddFlags adds flags for a specific ProxyServer to the specified FlagSet
 func (s *ProxyServerConfig) AddFlags(fs *pflag.FlagSet) {
 	fs.IPVar(&s.BindAddress, "bind-address", s.BindAddress, "The IP address for the proxy server to serve on (set to 0.0.0.0 for all interfaces)")
+	fs.BoolVar(&s.DisableHyperInternalService, "disable-hyper-internal-service", s.DisableHyperInternalService, "Disable the internal haproxy service in Hyper pods")
 	fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
 	fs.IntVar(&s.HealthzPort, "healthz-port", s.HealthzPort, "The port to bind the health check server. Use 0 to disable.")
 	fs.IPVar(&s.HealthzBindAddress, "healthz-bind-address", s.HealthzBindAddress, "The IP address for the health check server to serve on, defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces)")

cmd/kube-proxy/app/server.go

@@ -200,7 +200,7 @@ func NewProxyServerDefault(config *options.ProxyServerConfig) (*ProxyServer, err
 		userspace.CleanupLeftovers(iptInterface)
 	case proxyModeHaproxy:
 		glog.V(2).Info("Using pod-buildin-haproxy proxy.")
-		proxierBuildin, err := haproxy.NewProxier(config.ConfigSyncPeriod, client)
+		proxierBuildin, err := haproxy.NewProxier(config.ConfigSyncPeriod, client, config.DisableHyperInternalService)
 		if err != nil {
 			glog.Fatalf("Unable to create proxier: %v", err)
 		}

cmd/kubelet/app/options/options.go

@@ -55,6 +55,7 @@ type KubeletServer struct {
 	ConfigureCBR0                  bool
 	ContainerRuntime               string
 	CPUCFSQuota                    bool
+	DisableHyperInternalService    bool
 	DockerDaemonContainer          string
 	DockerEndpoint                 string
 	DockerExecHandlerName          string
@@ -202,6 +203,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.DurationVar(&s.HTTPCheckFrequency, "http-check-frequency", s.HTTPCheckFrequency, "Duration between checking http for new data")
 	fs.StringVar(&s.ManifestURL, "manifest-url", s.ManifestURL, "URL for accessing the container manifest")
 	fs.StringVar(&s.ManifestURLHeader, "manifest-url-header", s.ManifestURLHeader, "HTTP header to use when accessing the manifest URL, with the key separated from the value with a ':', as in 'key:value'")
+	fs.BoolVar(&s.DisableHyperInternalService, "disable-hyper-internal-service", s.DisableHyperInternalService, "Disable the internal haproxy service in Hyper pods")
 	fs.BoolVar(&s.EnableServer, "enable-server", s.EnableServer, "Enable the Kubelet's server")
 	fs.IPVar(&s.Address, "address", s.Address, "The IP address for the Kubelet to serve on (set to 0.0.0.0 for all interfaces)")
 	fs.UintVar(&s.Port, "port", s.Port, "The port for the Kubelet to serve on.")

cmd/kubelet/app/server.go

@@ -186,57 +186,58 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) {
 	}
 
 	return &KubeletConfig{
-		Address:                   s.Address,
-		AllowPrivileged:           s.AllowPrivileged,
-		Auth:                      nil, // default does not enforce auth[nz]
-		CAdvisorInterface:         nil, // launches background processes, not set here
-		CgroupRoot:                s.CgroupRoot,
-		CinderConfig:              s.CinderConfig,
-		Cloud:                     nil, // cloud provider might start background processes
-		ClusterDNS:                s.ClusterDNS,
-		ClusterDomain:             s.ClusterDomain,
-		ConfigFile:                s.Config,
-		ConfigureCBR0:             s.ConfigureCBR0,
-		ContainerManager:          nil,
-		ContainerRuntime:          s.ContainerRuntime,
-		CPUCFSQuota:               s.CPUCFSQuota,
-		DiskSpacePolicy:           diskSpacePolicy,
-		DockerClient:              dockertools.ConnectToDockerOrDie(s.DockerEndpoint),
-		DockerDaemonContainer:     s.DockerDaemonContainer,
-		DockerExecHandler:         dockerExecHandler,
-		EnableDebuggingHandlers:   s.EnableDebuggingHandlers,
-		EnableServer:              s.EnableServer,
-		EventBurst:                s.EventBurst,
-		EventRecordQPS:            s.EventRecordQPS,
-		FileCheckFrequency:        s.FileCheckFrequency,
-		HostnameOverride:          s.HostnameOverride,
-		HostNetworkSources:        hostNetworkSources,
-		HostPIDSources:            hostPIDSources,
-		HostIPCSources:            hostIPCSources,
-		HTTPCheckFrequency:        s.HTTPCheckFrequency,
-		ImageGCPolicy:             imageGCPolicy,
-		KubeClient:                nil,
-		ManifestURL:               s.ManifestURL,
-		ManifestURLHeader:         manifestURLHeader,
-		MasterServiceNamespace:    s.MasterServiceNamespace,
-		MaxContainerCount:         s.MaxContainerCount,
-		MaxOpenFiles:              s.MaxOpenFiles,
-		MaxPerPodContainerCount:   s.MaxPerPodContainerCount,
-		MaxPods:                   s.MaxPods,
-		MinimumGCAge:              s.MinimumGCAge,
-		Mounter:                   mounter,
-		ChownRunner:               chownRunner,
-		ChmodRunner:               chmodRunner,
-		NetworkPluginName:         networkPluginName,
-		NetworkPlugins:            networkPlugins,
-		NodeLabels:                s.NodeLabels,
-		NodeLabelsFile:            s.NodeLabelsFile,
-		NodeStatusUpdateFrequency: s.NodeStatusUpdateFrequency,
-		OOMAdjuster:               oom.NewOOMAdjuster(),
-		OSInterface:               kubecontainer.RealOS{},
-		PodCIDR:                   s.PodCIDR,
-		ReconcileCIDR:             s.ReconcileCIDR,
-		PodInfraContainerImage:    s.PodInfraContainerImage,
+		Address:                     s.Address,
+		AllowPrivileged:             s.AllowPrivileged,
+		Auth:                        nil, // default does not enforce auth[nz]
+		CAdvisorInterface:           nil, // launches background processes, not set here
+		CgroupRoot:                  s.CgroupRoot,
+		CinderConfig:                s.CinderConfig,
+		Cloud:                       nil, // cloud provider might start background processes
+		ClusterDNS:                  s.ClusterDNS,
+		ClusterDomain:               s.ClusterDomain,
+		ConfigFile:                  s.Config,
+		ConfigureCBR0:               s.ConfigureCBR0,
+		ContainerManager:            nil,
+		ContainerRuntime:            s.ContainerRuntime,
+		CPUCFSQuota:                 s.CPUCFSQuota,
+		DisableHyperInternalService: s.DisableHyperInternalService,
+		DiskSpacePolicy:             diskSpacePolicy,
+		DockerClient:                dockertools.ConnectToDockerOrDie(s.DockerEndpoint),
+		DockerDaemonContainer:       s.DockerDaemonContainer,
+		DockerExecHandler:           dockerExecHandler,
+		EnableDebuggingHandlers:     s.EnableDebuggingHandlers,
+		EnableServer:                s.EnableServer,
+		EventBurst:                  s.EventBurst,
+		EventRecordQPS:              s.EventRecordQPS,
+		FileCheckFrequency:          s.FileCheckFrequency,
+		HostnameOverride:            s.HostnameOverride,
+		HostNetworkSources:          hostNetworkSources,
+		HostPIDSources:              hostPIDSources,
+		HostIPCSources:              hostIPCSources,
+		HTTPCheckFrequency:          s.HTTPCheckFrequency,
+		ImageGCPolicy:               imageGCPolicy,
+		KubeClient:                  nil,
+		ManifestURL:                 s.ManifestURL,
+		ManifestURLHeader:           manifestURLHeader,
+		MasterServiceNamespace:      s.MasterServiceNamespace,
+		MaxContainerCount:           s.MaxContainerCount,
+		MaxOpenFiles:                s.MaxOpenFiles,
+		MaxPerPodContainerCount:     s.MaxPerPodContainerCount,
+		MaxPods:                     s.MaxPods,
+		MinimumGCAge:                s.MinimumGCAge,
+		Mounter:                     mounter,
+		ChownRunner:                 chownRunner,
+		ChmodRunner:                 chmodRunner,
+		NetworkPluginName:           networkPluginName,
+		NetworkPlugins:              networkPlugins,
+		NodeLabels:                  s.NodeLabels,
+		NodeLabelsFile:              s.NodeLabelsFile,
+		NodeStatusUpdateFrequency:   s.NodeStatusUpdateFrequency,
+		OOMAdjuster:                 oom.NewOOMAdjuster(),
+		OSInterface:                 kubecontainer.RealOS{},
+		PodCIDR:                     s.PodCIDR,
+		ReconcileCIDR:               s.ReconcileCIDR,
+		PodInfraContainerImage:      s.PodInfraContainerImage,
 		Port:                           s.Port,
 		ReadOnlyPort:                   s.ReadOnlyPort,
 		RegisterNode:                   s.RegisterNode,
@@ -676,6 +677,7 @@ type KubeletConfig struct {
 	ContainerManager               cm.ContainerManager
 	ContainerRuntime               string
 	CPUCFSQuota                    bool
+	DisableHyperInternalService    bool
 	DiskSpacePolicy                kubelet.DiskSpacePolicy
 	DockerClient                   dockertools.DockerInterface
 	DockerDaemonContainer          string
@@ -828,6 +830,7 @@ func CreateAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod
 		kc.OutOfDiskTransitionFrequency,
 		kc.ExperimentalFlannelOverlay,
 		kc.NodeIP,
+		kc.DisableHyperInternalService,
 	)
 
 	if err != nil {

docs/admin/kube-proxy.md

@@ -59,6 +59,7 @@ kube-proxy
       --config-sync-period=15m0s: How often configuration from the apiserver is refreshed.  Must be greater than 0.
       --conntrack-max=262144: Maximum number of NAT connections to track (0 to leave as-is)
       --conntrack-tcp-timeout-established=86400: Idle timeout for established TCP connections (0 to leave as-is)
+      --disable-hyper-internal-service[=false]: Disable the internal haproxy service in Hyper pods
       --google-json-key="": The Google Cloud Platform Service Account JSON Key to use for authentication.
       --healthz-bind-address=127.0.0.1: The IP address for the health check server to serve on, defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces)
       --healthz-port=10249: The port to bind the health check server. Use 0 to disable.
@@ -76,7 +77,7 @@ kube-proxy
       --udp-timeout=250ms: How long an idle UDP connection will be kept open (e.g. '250ms', '2s').  Must be greater than 0. Only applicable for proxy-mode=userspace
 ```
 
-###### Auto generated by spf13/cobra on 30-Dec-2015
+###### Auto generated by spf13/cobra on 23-Feb-2016
 
 
 <!-- BEGIN MUNGE: GENERATED_ANALYTICS -->

docs/admin/kubelet.md

@@ -81,6 +81,7 @@ kubelet
       --container-runtime="docker": The container runtime to use. Possible values: 'docker', 'rkt'. Default: 'docker'.
       --containerized[=false]: Experimental support for running kubelet in a container.  Intended for testing. [default=false]
       --cpu-cfs-quota[=false]: Enable CPU CFS quota enforcement for containers that specify CPU limits
+      --disable-hyper-internal-service[=false]: Disable the internal haproxy service in Hyper pods
       --docker-endpoint="": If non-empty, use this for the docker endpoint to communicate with
       --docker-exec-handler="native": Handler to use when executing a command in a container. Valid values are 'native' and 'nsenter'. Defaults to 'native'.
       --enable-debugging-handlers[=true]: Enables server endpoints for log collection and local running of containers and commands
@@ -146,7 +147,7 @@ kubelet
       --volume-plugin-dir="/usr/libexec/kubernetes/kubelet-plugins/volume/exec/": <Warning: Alpha feature> The full path of the directory in which to search for additional third party volume plugins
 ```
 
-###### Auto generated by spf13/cobra on 12-Jan-2016
+###### Auto generated by spf13/cobra on 23-Feb-2016
 
 
 <!-- BEGIN MUNGE: GENERATED_ANALYTICS -->

hack/verify-flags/known-flags.txt

@@ -75,6 +75,7 @@ disable-filter
 docker-email
 docker-endpoint
 docker-exec-handler
+disable-hyper-internal-service
 docker-password
 docker-server
 docker-username

pkg/kubelet/hyper/hyper.go

@@ -74,6 +74,9 @@ type runtime struct {
 	imagePuller         kubecontainer.ImagePuller
 	version             kubecontainer.Version
 
+	// Disable the internal haproxy service in Hyper pods
+	disableHyperInternalService bool
+
 	// Runner of lifecycle events.
 	runner kubecontainer.HandlerRunner
 }
@@ -95,6 +98,7 @@ func New(generator kubecontainer.RunContainerOptionsGenerator,
 	imageBackOff *util.Backoff,
 	serializeImagePulls bool,
 	httpClient kubetypes.HttpGetter,
+	disableHyperInternalService bool,
 ) (kubecontainer.Runtime, error) {
 	// check hyper has already installed
 	hyperBinAbsPath, err := exec.LookPath(hyperBinName)
@@ -104,16 +108,17 @@ func New(generator kubecontainer.RunContainerOptionsGenerator,
 	}
 
 	hyper := &runtime{
-		hyperBinAbsPath:     hyperBinAbsPath,
-		dockerKeyring:       credentialprovider.NewDockerKeyring(),
-		containerRefManager: containerRefManager,
-		generator:           generator,
-		livenessManager:     livenessManager,
-		recorder:            recorder,
-		networkPlugin:       networkPlugin,
-		volumeGetter:        volumeGetter,
-		hyperClient:         NewHyperClient(),
-		kubeClient:          kubeClient,
+		hyperBinAbsPath:             hyperBinAbsPath,
+		dockerKeyring:               credentialprovider.NewDockerKeyring(),
+		containerRefManager:         containerRefManager,
+		generator:                   generator,
+		livenessManager:             livenessManager,
+		recorder:                    recorder,
+		networkPlugin:               networkPlugin,
+		volumeGetter:                volumeGetter,
+		hyperClient:                 NewHyperClient(),
+		kubeClient:                  kubeClient,
+		disableHyperInternalService: disableHyperInternalService,
 	}
 
 	if serializeImagePulls {
@@ -446,17 +451,19 @@ func (r *runtime) buildHyperPod(pod *api.Pod, restartCount int, pullSecrets []ap
 
 	glog.V(4).Infof("Hyper volumes: %v", volumes)
 
-	services := r.buildHyperPodServices(pod)
-	if services == nil {
-		// services can't be null for kubernetes, so fake one if it is null
-		services = []HyperService{
-			{
-				ServiceIP:   "127.0.0.2",
-				ServicePort: 65534,
-			},
+	if !r.disableHyperInternalService {
+		services := r.buildHyperPodServices(pod)
+		if services == nil {
+			// services can't be null for kubernetes, so fake one if it is null
+			services = []HyperService{
+				{
+					ServiceIP:   "127.0.0.2",
+					ServicePort: 65534,
+				},
+			}
 		}
+		specMap["services"] = services
 	}
-	specMap["services"] = services
 
 	// build hyper containers spec
 	var containers []map[string]interface{}

pkg/kubelet/hyper/hyperclient.go

@@ -39,7 +39,7 @@ const (
 	HYPER_PROTO       = "unix"
 	HYPER_ADDR        = "/var/run/hyper.sock"
 	HYPER_SCHEME      = "http"
-	HYPER_MINVERSION  = "0.4.0"
+	HYPER_MINVERSION  = "0.5.0"
 	DEFAULT_IMAGE_TAG = "latest"
 
 	KEY_COMMAND        = "command"

pkg/kubelet/kubelet.go

@@ -198,6 +198,7 @@ func NewMainKubelet(
 	outOfDiskTransitionFrequency time.Duration,
 	flannelExperimentalOverlay bool,
 	nodeIP net.IP,
+	disableHyperInternalService bool,
 ) (*Kubelet, error) {
 	if rootDirectory == "" {
 		return nil, fmt.Errorf("invalid root directory %q", rootDirectory)
@@ -317,6 +318,7 @@ func NewMainKubelet(
 		nodeIP:                         nodeIP,
 		clock:                          util.RealClock{},
 		outOfDiskTransitionFrequency: outOfDiskTransitionFrequency,
+		disableHyperInternalService:  disableHyperInternalService,
 	}
 	if klet.flannelExperimentalOverlay {
 		glog.Infof("Flannel is in charge of podCIDR and overlay networking.")
@@ -403,6 +405,7 @@ func NewMainKubelet(
 			imageBackOff,
 			serializeImagePulls,
 			klet.httpClient,
+			klet.disableHyperInternalService,
 		)
 		if err != nil {
 			return nil, err
@@ -686,6 +689,9 @@ type Kubelet struct {
 	// not-out-of-disk. This prevents a pod that causes out-of-disk condition from repeatedly
 	// getting rescheduled onto the node.
 	outOfDiskTransitionFrequency time.Duration
+
+	// Disable the internal haproxy service in Hyper pods
+	disableHyperInternalService bool
 }
 
 // Validate given node IP belongs to the current host