Merge pull request #140 from vdemeester/carry-pr-116

vdemeester · vdemeester · commit 18211431fd3c · 2016-01-22T21:46:51.000+01:00
Carry pr 116  Multiple -f
diff --git a/README.md b/README.md
@@ -20,8 +20,8 @@ import (
 func main() {
 	project, err := docker.NewProject(&docker.Context{
 		Context: project.Context{
-			ComposeFile: "docker-compose.yml",
-			ProjectName: "my-compose",
+			ComposeFiles: []string{"docker-compose.yml"},
+			ProjectName:  "my-compose",
 		},
 	})
 
diff --git a/cli/command/command.go b/cli/command/command.go
@@ -1,6 +1,8 @@
 package command
 
 import (
+	"os"
+
 	"github.com/codegangsta/cli"
 	"github.com/docker/libcompose/cli/app"
 	"github.com/docker/libcompose/project"
@@ -239,10 +241,10 @@ func CommonFlags() []cli.Flag {
 		cli.BoolFlag{
 			Name: "verbose,debug",
 		},
-		cli.StringFlag{
+		cli.StringSliceFlag{
 			Name:   "file,f",
-			Usage:  "Specify an alternate compose file (default: docker-compose.yml)",
-			Value:  "docker-compose.yml",
+			Usage:  "Specify one or more alternate compose files (default: docker-compose.yml)",
+			Value:  &cli.StringSlice{},
 			EnvVar: "COMPOSE_FILE",
 		},
 		cli.StringFlag{
@@ -254,7 +256,15 @@ func CommonFlags() []cli.Flag {
 
 // Populate updates the specified project context based on command line arguments and subcommands.
 func Populate(context *project.Context, c *cli.Context) {
-	context.ComposeFile = c.GlobalString("file")
+	context.ComposeFiles = c.GlobalStringSlice("file")
+
+	if len(context.ComposeFiles) == 0 {
+		context.ComposeFiles = []string{"docker-compose.yml"}
+		if _, err := os.Stat("docker-compose.override.yml"); err == nil {
+			context.ComposeFiles = append(context.ComposeFiles, "docker-compose.override.yml")
+		}
+	}
+
 	context.ProjectName = c.GlobalString("project-name")
 
 	if c.Command.Name == "logs" {
diff --git a/docker/convert.go b/docker/convert.go
@@ -48,7 +48,7 @@ func ConvertToAPI(s *Service, name string) (*dockerclient.CreateContainerOptions
 func volumes(c *project.ServiceConfig, ctx *Context) map[string]struct{} {
 	volumes := make(map[string]struct{}, len(c.Volumes))
 	for k, v := range c.Volumes {
-		vol := ctx.ResourceLookup.ResolvePath(v, ctx.ComposeFile)
+		vol := ctx.ResourceLookup.ResolvePath(v, ctx.ComposeFiles[0])
 
 		c.Volumes[k] = vol
 		if isVolume(vol) {
diff --git a/docker/convert_test.go b/docker/convert_test.go
@@ -19,7 +19,7 @@ func TestParseCommand(t *testing.T) {
 
 func TestParseBindsAndVolumes(t *testing.T) {
 	ctx := &Context{}
-	ctx.ComposeFile = "foo/docker-compose.yml"
+	ctx.ComposeFiles = []string{"foo/docker-compose.yml"}
 	ctx.ResourceLookup = &lookup.FileConfigLookup{}
 
 	abs, err := filepath.Abs(".")
@@ -34,7 +34,7 @@ func TestParseBindsAndVolumes(t *testing.T) {
 
 func TestParseLabels(t *testing.T) {
 	ctx := &Context{}
-	ctx.ComposeFile = "foo/docker-compose.yml"
+	ctx.ComposeFiles = []string{"foo/docker-compose.yml"}
 	ctx.ResourceLookup = &lookup.FileConfigLookup{}
 	bashCmd := "bash"
 	fooLabel := "foo.label"
diff --git a/example/main.go b/example/main.go
@@ -10,8 +10,8 @@ import (
 func main() {
 	project, err := docker.NewProject(&docker.Context{
 		Context: project.Context{
-			ComposeFile: "docker-compose.yml",
-			ProjectName: "yeah-compose",
+			ComposeFiles: []string{"docker-compose.yml"},
+			ProjectName:  "yeah-compose",
 		},
 	})
 
diff --git a/integration/assets/multiple-composefiles-default/docker-compose.override.yml b/integration/assets/multiple-composefiles-default/docker-compose.override.yml
@@ -0,0 +1,3 @@
+yetanother:
+  image: busybox:latest
+  command: top
diff --git a/integration/assets/multiple-composefiles-default/docker-compose.yml b/integration/assets/multiple-composefiles-default/docker-compose.yml
@@ -0,0 +1,6 @@
+simple:
+  image: busybox:latest
+  command: top
+another:
+  image: busybox:latest
+  command: top
diff --git a/integration/assets/multiple/one.yml b/integration/assets/multiple/one.yml
@@ -0,0 +1,10 @@
+multiple:
+  image: tianon/true
+  environment:
+    - KEY1=VAL1
+simple:
+  image: busybox:latest
+  command: top
+another:
+  image: busybox:latest
+  command: top
diff --git a/integration/assets/multiple/two.yml b/integration/assets/multiple/two.yml
@@ -0,0 +1,8 @@
+multiple:
+  image: busybox
+  command: echo two
+  environment:
+    - KEY2=VAL2
+yetanother:
+  image: busybox:latest
+  command: top
diff --git a/integration/create_test.go b/integration/create_test.go
@@ -3,8 +3,10 @@ package integration
 import (
 	"fmt"
 	"os"
+	"os/exec"
 
 	. "gopkg.in/check.v1"
+	"path/filepath"
 )
 
 func (s *RunSuite) TestFields(c *C) {
@@ -170,3 +172,79 @@ func (s *RunSuite) TestFieldTypeConversions(c *C) {
 
 	os.Unsetenv("LIMIT")
 }
+
+func (s *RunSuite) TestMultipleComposeFilesOneTwo(c *C) {
+	p := "multiple"
+	cmd := exec.Command(s.command, "-f", "./assets/multiple/one.yml", "-f", "./assets/multiple/two.yml", "create")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Stdin = os.Stdin
+	err := cmd.Run()
+
+	c.Assert(err, IsNil)
+
+	containerNames := []string{"multiple", "simple", "another", "yetanother"}
+
+	for _, containerName := range containerNames {
+		name := fmt.Sprintf("%s_%s_1", p, containerName)
+		container := s.GetContainerByName(c, name)
+
+		c.Assert(container, NotNil)
+	}
+
+	name := fmt.Sprintf("%s_%s_1", p, "multiple")
+	container := s.GetContainerByName(c, name)
+
+	c.Assert(container.Config.Image, Equals, "busybox")
+	c.Assert(container.Config.Cmd, DeepEquals, []string{"echo", "two"})
+	c.Assert(container.Config.Env, DeepEquals, []string{"KEY1=VAL1", "KEY2=VAL2"})
+}
+
+func (s *RunSuite) TestMultipleComposeFilesTwoOne(c *C) {
+	p := "multiple"
+	cmd := exec.Command(s.command, "-f", "./assets/multiple/two.yml", "-f", "./assets/multiple/one.yml", "create")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Stdin = os.Stdin
+	err := cmd.Run()
+
+	c.Assert(err, IsNil)
+
+	containerNames := []string{"multiple", "simple", "another", "yetanother"}
+
+	for _, containerName := range containerNames {
+		name := fmt.Sprintf("%s_%s_1", p, containerName)
+		container := s.GetContainerByName(c, name)
+
+		c.Assert(container, NotNil)
+	}
+
+	name := fmt.Sprintf("%s_%s_1", p, "multiple")
+	container := s.GetContainerByName(c, name)
+
+	c.Assert(container.Config.Image, Equals, "tianon/true")
+	c.Assert(container.Config.Cmd, DeepEquals, []string{"echo", "two"})
+	c.Assert(container.Config.Env, DeepEquals, []string{"KEY2=VAL2", "KEY1=VAL1"})
+}
+
+func (s *RunSuite) TestDefaultMultipleComposeFiles(c *C) {
+	p := s.RandomProject()
+	cmd := exec.Command(filepath.Join("../../", s.command), "-p", p, "create")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Stdin = os.Stdin
+	cmd.Dir = "./assets/multiple-composefiles-default/"
+	err := cmd.Run()
+
+	c.Assert(err, IsNil)
+
+	containerNames := []string{"simple", "another", "yetanother"}
+
+	for _, containerName := range containerNames {
+		name := fmt.Sprintf("%s_%s_1", p, containerName)
+		container := s.GetContainerByName(c, name)
+
+		c.Assert(container, NotNil)
+	}
+
+}
diff --git a/project/context.go b/project/context.go
@@ -25,8 +25,8 @@ type Context struct {
 	NoRecreate          bool
 	NoCache             bool
 	Signal              int
-	ComposeFile         string
-	ComposeBytes        []byte
+	ComposeFiles        []string
+	ComposeBytes        [][]byte
 	ProjectName         string
 	isOpen              bool
 	ServiceFactory      ServiceFactory
@@ -37,33 +37,35 @@ type Context struct {
 	Project             *Project
 }
 
-func (c *Context) readComposeFile() error {
+func (c *Context) readComposeFiles() error {
 	if c.ComposeBytes != nil {
 		return nil
 	}
 
-	logrus.Debugf("Opening compose file: %s", c.ComposeFile)
+	logrus.Debugf("Opening compose files: %s", strings.Join(c.ComposeFiles, ","))
 
-	if c.ComposeFile == "-" {
+	// Handle STDIN (`-f -`)
+	if len(c.ComposeFiles) == 1 && c.ComposeFiles[0] == "-" {
 		composeBytes, err := ioutil.ReadAll(os.Stdin)
 		if err != nil {
 			logrus.Errorf("Failed to read compose file from stdin: %v", err)
 			return err
 		}
-		c.ComposeBytes = composeBytes
-	} else if c.ComposeFile != "" {
-		if composeBytes, err := ioutil.ReadFile(c.ComposeFile); os.IsNotExist(err) {
-			if c.IgnoreMissingConfig {
-				return nil
-			}
-			logrus.Errorf("Failed to find %s", c.ComposeFile)
+		c.ComposeBytes = [][]byte{composeBytes}
+		return nil
+	}
+
+	for _, composeFile := range c.ComposeFiles {
+		composeBytes, err := ioutil.ReadFile(composeFile)
+		if err != nil && !os.IsNotExist(err) {
+			logrus.Errorf("Failed to open the compose file: %s", composeFile)
 			return err
-		} else if err != nil {
-			logrus.Errorf("Failed to open %s", c.ComposeFile)
+		}
+		if err != nil && !c.IgnoreMissingConfig {
+			logrus.Errorf("Failed to find the compose file: %s", composeFile)
 			return err
-		} else {
-			c.ComposeBytes = composeBytes
 		}
+		c.ComposeBytes = append(c.ComposeBytes, composeBytes)
 	}
 
 	return nil
@@ -97,9 +99,14 @@ func (c *Context) lookupProjectName() (string, error) {
 		return envProject, nil
 	}
 
-	f, err := filepath.Abs(c.ComposeFile)
+	file := "."
+	if len(c.ComposeFiles) > 0 {
+		file = c.ComposeFiles[0]
+	}
+
+	f, err := filepath.Abs(file)
 	if err != nil {
-		logrus.Errorf("Failed to get absolute directory for: %s", c.ComposeFile)
+		logrus.Errorf("Failed to get absolute directory for: %s", file)
 		return "", err
 	}
 
@@ -124,7 +131,7 @@ func (c *Context) open() error {
 		return nil
 	}
 
-	if err := c.readComposeFile(); err != nil {
+	if err := c.readComposeFiles(); err != nil {
 		return err
 	}
 
diff --git a/project/merge.go b/project/merge.go
@@ -31,7 +31,7 @@ var (
 type rawService map[string]interface{}
 type rawServiceMap map[string]rawService
 
-func mergeProject(p *Project, bytes []byte) (map[string]*ServiceConfig, error) {
+func mergeProject(p *Project, file string, bytes []byte) (map[string]*ServiceConfig, error) {
 	configs := make(map[string]*ServiceConfig)
 
 	datas := make(rawServiceMap)
@@ -44,12 +44,21 @@ func mergeProject(p *Project, bytes []byte) (map[string]*ServiceConfig, error) {
 	}
 
 	for name, data := range datas {
-		data, err := parse(p.context.ResourceLookup, p.context.EnvironmentLookup, p.File, data, datas)
+		data, err := parse(p.context.ResourceLookup, p.context.EnvironmentLookup, file, data, datas)
 		if err != nil {
 			logrus.Errorf("Failed to parse service %s: %v", name, err)
 			return nil, err
 		}
 
+		if _, ok := p.Configs[name]; ok {
+			var rawExistingService rawService
+			if err := utils.Convert(p.Configs[name], &rawExistingService); err != nil {
+				return nil, err
+			}
+
+			data = mergeConfig(rawExistingService, data)
+		}
+
 		datas[name] = data
 	}
 
@@ -58,6 +67,7 @@ func mergeProject(p *Project, bytes []byte) (map[string]*ServiceConfig, error) {
 	}
 
 	adjustValues(configs)
+
 	return configs, nil
 }
 
@@ -237,6 +247,14 @@ func parse(resourceLookup ResourceLookup, environmentLookup EnvironmentLookup, i
 		}
 	}
 
+	baseService = mergeConfig(baseService, serviceData)
+
+	logrus.Debugf("Merged result %#v", baseService)
+
+	return baseService, nil
+}
+
+func mergeConfig(baseService, serviceData rawService) rawService {
 	for k, v := range serviceData {
 		// Image and build are mutually exclusive in merge
 		if k == "image" {
@@ -252,9 +270,7 @@ func parse(resourceLookup ResourceLookup, environmentLookup EnvironmentLookup, i
 		}
 	}
 
-	logrus.Debugf("Merged result %#v", baseService)
-
-	return baseService, nil
+	return baseService
 }
 
 func merge(existing, value interface{}) interface{} {
diff --git a/project/merge_test.go b/project/merge_test.go
diff --git a/project/project.go b/project/project.go
diff --git a/project/project_test.go b/project/project_test.go
diff --git a/project/types.go b/project/types.go

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+yetanother:`
	`2`	`+ image: busybox:latest`
	`3`	`+ command: top`