Fixed upload issue on esp32 (partitions artifacts copy) (#457)

* Copy multiple build artifacts matching a name pattern.

Previously the compile command copied only the following artifacts:

  sketch.ino.hex (or .bin)
  sketch.ino.elf

now also the files matching the glob sketch.ino.*.hex will be copied.
This allows for some 3rd party core (like esp32) to copy also the extra
file sketch.ino.partitions.bin.

Should fix #163

* Added test for improved support for esp32:esp32

#163

* Run of "go mod tidy"

Author: cmaglie

cli/cli_test.go

@@ -396,6 +396,39 @@ board_manager:
 	require.NotZero(t, exitCode)
 }
 
+func Test3rdPartyCoreIntegration(t *testing.T) {
+	// override SetUp dirs
+	tmp := tmpDirOrDie()
+	defer os.RemoveAll(tmp)
+	os.Setenv("ARDUINO_SKETCHBOOK_DIR", tmp)
+	currSketchbookDir = tmp
+
+	configFile := filepath.Join(currDataDir, "arduino-cli.yaml")
+	err := ioutil.WriteFile(configFile, []byte(`
+board_manager:
+  additional_urls:
+    - https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_index.json
+`), os.FileMode(0644))
+	require.NoError(t, err, "writing dummy config "+configFile)
+
+	// Update index and install esp32:esp32
+	exitCode, _ := executeWithArgs("--config-file", configFile, "core", "update-index")
+	require.Zero(t, exitCode)
+	exitCode, d := executeWithArgs("--config-file", configFile, "core", "install", "esp32:esp32")
+	require.Zero(t, exitCode)
+	require.Contains(t, string(d), "installed")
+
+	// Build a simple sketch and check if all artifacts are copied
+	tmpSketch := paths.New(currSketchbookDir).Join("Blink")
+	err = paths.New("testdata/Blink").CopyDirTo(tmpSketch)
+	require.NoError(t, err, "copying test sketch into temp dir")
+	exitCode, d = executeWithArgs("--config-file", configFile, "compile", "-b", "esp32:esp32:esp32", tmpSketch.String())
+	require.Zero(t, exitCode)
+	require.True(t, tmpSketch.Join("Blink.esp32.esp32.esp32.bin").Exist())
+	require.True(t, tmpSketch.Join("Blink.esp32.esp32.esp32.elf").Exist())
+	require.True(t, tmpSketch.Join("Blink.esp32.esp32.esp32.partitions.bin").Exist()) // https://github.com/arduino/arduino-cli/issues/163
+}
+
 func TestCoreCommandsIntegration(t *testing.T) {
 	// override SetUp dirs
 	tmp := tmpDirOrDie()

cli/testdata/Blink/Blink.ino

@@ -0,0 +1,3 @@
+
+void setup() {}
+void loop() {}

commands/compile/compile.go

@@ -174,8 +174,11 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
 	}
 
 	// FIXME: Make a function to obtain these info...
-	outputPath := builderCtx.BuildProperties.ExpandPropsInString("{build.path}/{recipe.output.tmp_file}")
-	ext := filepath.Ext(outputPath)
+	outputPath := paths.New(
+		builderCtx.BuildProperties.ExpandPropsInString("{build.path}/{recipe.output.tmp_file}")) // "/build/path/sketch.ino.bin"
+	ext := outputPath.Ext()          // ".hex" | ".bin"
+	base := outputPath.Base()        // "sketch.ino.hex"
+	base = base[:len(base)-len(ext)] // "sketch.ino"
 
 	// FIXME: Make a function to produce a better name...
 	// Make the filename without the FQBN configs part
@@ -190,7 +193,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
 		} else {
 			exportPath = sketch.FullPath.Parent()
 		}
-		exportFile = sketch.Name + "." + fqbnSuffix
+		exportFile = sketch.Name + "." + fqbnSuffix // "sketch.arduino.avr.uno"
 	} else {
 		exportPath = paths.New(req.GetExportFile()).Parent()
 		exportFile = paths.New(req.GetExportFile()).Base()
@@ -199,16 +202,25 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
 		}
 	}
 
-	// Copy .hex file to sketch directory
-	srcHex := paths.New(outputPath)
-	dstHex := exportPath.Join(exportFile + ext)
-	logrus.WithField("from", srcHex).WithField("to", dstHex).Debug("copying sketch build output")
-	if err = srcHex.CopyTo(dstHex); err != nil {
-		return nil, fmt.Errorf("copying output file: %s", err)
+	// Copy "sketch.ino.*.hex" / "sketch.ino.*.bin" artifacts to sketch directory
+	srcDir, err := outputPath.Parent().ReadDir() // read "/build/path/*"
+	if err != nil {
+		return nil, fmt.Errorf("reading build directory: %s", err)
+	}
+	srcDir.FilterPrefix(base + ".")
+	srcDir.FilterSuffix(ext)
+	for _, srcOutput := range srcDir {
+		srcFilename := srcOutput.Base()       // "sketch.ino.*.bin"
+		srcFilename = srcFilename[len(base):] // ".*.bin"
+		dstOutput := exportPath.Join(exportFile + srcFilename)
+		logrus.WithField("from", srcOutput).WithField("to", dstOutput).Debug("copying sketch build output")
+		if err = srcOutput.CopyTo(dstOutput); err != nil {
+			return nil, fmt.Errorf("copying output file: %s", err)
+		}
 	}
 
 	// Copy .elf file to sketch directory
-	srcElf := paths.New(outputPath[:len(outputPath)-3] + "elf")
+	srcElf := outputPath.Parent().Join(base + ".elf")
 	if srcElf.Exist() {
 		dstElf := exportPath.Join(exportFile + ".elf")
 		logrus.WithField("from", srcElf).WithField("to", dstElf).Debug("copying sketch build output")

go.mod

@@ -43,7 +43,6 @@ require (
 	go.bug.st/serial.v1 v0.0.0-20180827123349-5f7892a7bb45
 	golang.org/x/net v0.0.0-20190311183353-d8887717615a
 	golang.org/x/text v0.3.0
-	google.golang.org/appengine v1.4.0 // indirect
 	google.golang.org/genproto v0.0.0-20190327125643-d831d65fe17d // indirect
 	google.golang.org/grpc v1.21.1
 	gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect

go.sum

@@ -4,8 +4,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/arduino/board-discovery v0.0.0-20180823133458-1ba29327fb0c h1:agh2JT96G8egU7FEb13L4dq3fnCN7lxXhJ86t69+W7s=
 github.com/arduino/board-discovery v0.0.0-20180823133458-1ba29327fb0c/go.mod h1:HK7SpkEax/3P+0w78iRQx1sz1vCDYYw9RXwHjQTB5i8=
-github.com/arduino/go-paths-helper v0.0.0-20190214132331-c3c98d1bf2e1 h1:S0NpDSqjlkNA510vmRCP5Cq9mPgu3rWDSdeN4SI1Mwc=
-github.com/arduino/go-paths-helper v0.0.0-20190214132331-c3c98d1bf2e1/go.mod h1:OGL+FS3aTrS01YsBAJJhkGuxtEGsFRSgZYo8b3vefdc=
 github.com/arduino/go-paths-helper v1.0.1 h1:utYXLM2RfFlc9qp/MJTIYp3t6ux/xM6mWjeEb/WLK4Q=
 github.com/arduino/go-paths-helper v1.0.1/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3v5YYu35Yb+w31Ck=
 github.com/arduino/go-properties-orderedmap v0.0.0-20190828172252-05018b28ff6c h1:4z4PJqNH8WGXtm9ix2muUOAP7gxTGBOdQTuKEDyCnsA=

test/requirements.txt

@@ -14,6 +14,7 @@ pluggy==0.12.0
 py==1.8.0
 pylint==2.3.1
 pyparsing==2.4.0
+pyserial==3.4
 pytest==5.1.3
 semver==2.8.1
 simplejson==3.16.0