Identify managed platforms not tracked by a package index (#2174)

* Load indexes first

* add new props to Index and Platform to understand if the platform is globally indexed

* add the new fields in the protobuf api

* generate protobuf stuff

* add the new fields in the PlatformReleaseToRPC mapping

* return the Platform struct from the PlatformUpgrade commands

* update the PlatformUpgrade daemon

* add the new props in the boards.ListAll func

* add the new props in the boards.Search func

* fix CR

Author: alessio-perugini

arduino/cores/cores.go

@@ -44,6 +44,7 @@ type Platform struct {
 	Package           *Package                                     `json:"-"`
 	ManuallyInstalled bool                                         // true if the Platform has been installed without the CLI
 	Deprecated        bool                                         // true if the Platform has been deprecated
+	Indexed           bool                                         // true if the Platform has been indexed from additional-urls
 }
 
 // PlatformReleaseHelp represents the help URL for this Platform release
@@ -409,3 +410,13 @@ func (release *PlatformRelease) MarshalJSON() ([]byte, error) {
 		Name:      release.Platform.Name,
 	})
 }
+
+// HasMetadata returns true if the PlatformRelease installation dir contains the installed.json file
+func (release *PlatformRelease) HasMetadata() bool {
+	if release.InstallDir == nil {
+		return false
+	}
+
+	installedJSONPath := release.InstallDir.Join("installed.json")
+	return installedJSONPath.Exist()
+}

arduino/cores/packageindex/index.go

@@ -33,8 +33,9 @@ import (
 //
 //easyjson:json
 type Index struct {
-	Packages  []*indexPackage `json:"packages"`
-	IsTrusted bool
+	Packages        []*indexPackage `json:"packages"`
+	IsTrusted       bool
+	isInstalledJSON bool
 }
 
 // indexPackage represents a single entry from package_index.json file.
@@ -144,7 +145,7 @@ var tr = i18n.Tr
 // with the existing contents of the cores.Packages passed as parameter.
 func (index Index) MergeIntoPackages(outPackages cores.Packages) {
 	for _, inPackage := range index.Packages {
-		inPackage.extractPackageIn(outPackages, index.IsTrusted)
+		inPackage.extractPackageIn(outPackages, index.IsTrusted, index.isInstalledJSON)
 	}
 }
 
@@ -243,7 +244,7 @@ func IndexFromPlatformRelease(pr *cores.PlatformRelease) Index {
 	}
 }
 
-func (inPackage indexPackage) extractPackageIn(outPackages cores.Packages, trusted bool) {
+func (inPackage indexPackage) extractPackageIn(outPackages cores.Packages, trusted bool, isInstallJSON bool) {
 	outPackage := outPackages.GetOrCreatePackage(inPackage.Name)
 	outPackage.Maintainer = inPackage.Maintainer
 	outPackage.WebsiteURL = inPackage.WebsiteURL
@@ -256,15 +257,19 @@ func (inPackage indexPackage) extractPackageIn(outPackages cores.Packages, trust
 	}
 
 	for _, inPlatform := range inPackage.Platforms {
-		inPlatform.extractPlatformIn(outPackage, trusted)
+		inPlatform.extractPlatformIn(outPackage, trusted, isInstallJSON)
 	}
 }
 
-func (inPlatformRelease indexPlatformRelease) extractPlatformIn(outPackage *cores.Package, trusted bool) error {
+func (inPlatformRelease indexPlatformRelease) extractPlatformIn(outPackage *cores.Package, trusted bool, isInstallJSON bool) error {
 	outPlatform := outPackage.GetOrCreatePlatform(inPlatformRelease.Architecture)
 	// FIXME: shall we use the Name and Category of the latest release? or maybe move Name and Category in PlatformRelease?
 	outPlatform.Name = inPlatformRelease.Name
 	outPlatform.Category = inPlatformRelease.Category
+	// If the variable `isInstallJSON` is false it means that the index we're reading is coming from the additional-urls.
+	// Therefore, the `outPlatform.Indexed` will be set at `true`.
+	outPlatform.Indexed = outPlatform.Indexed || !isInstallJSON
+
 	// If the Platform is installed before deprecation installed.json file does not include "deprecated" field.
 	// The installed.json is read during loading phase of an installed Platform, if the deprecated field is not found
 	// the package_index.json field would be overwritten and the deprecation info would be lost.
@@ -398,6 +403,11 @@ func LoadIndex(jsonIndexFile *paths.Path) (*Index, error) {
 	} else {
 		logrus.WithField("index", jsonIndexFile).Infof("Missing signature file")
 	}
+
+	if jsonIndexFile.Base() == "installed.json" {
+		index.isInstalledJSON = true
+	}
+
 	return &index, nil
 }
 

arduino/cores/packagemanager/install_uninstall.go

@@ -38,35 +38,35 @@ func (pme *Explorer) DownloadAndInstallPlatformUpgrades(
 	downloadCB rpc.DownloadProgressCB,
 	taskCB rpc.TaskProgressCB,
 	skipPostInstall bool,
-) error {
+) (*cores.PlatformRelease, error) {
 	if platformRef.PlatformVersion != nil {
-		return &arduino.InvalidArgumentError{Message: tr("Upgrade doesn't accept parameters with version")}
+		return nil, &arduino.InvalidArgumentError{Message: tr("Upgrade doesn't accept parameters with version")}
 	}
 
 	// Search the latest version for all specified platforms
 	platform := pme.FindPlatform(platformRef)
 	if platform == nil {
-		return &arduino.PlatformNotFoundError{Platform: platformRef.String()}
+		return nil, &arduino.PlatformNotFoundError{Platform: platformRef.String()}
 	}
 	installed := pme.GetInstalledPlatformRelease(platform)
 	if installed == nil {
-		return &arduino.PlatformNotFoundError{Platform: platformRef.String()}
+		return nil, &arduino.PlatformNotFoundError{Platform: platformRef.String()}
 	}
 	latest := platform.GetLatestRelease()
 	if !latest.Version.GreaterThan(installed.Version) {
-		return &arduino.PlatformAlreadyAtTheLatestVersionError{}
+		return installed, &arduino.PlatformAlreadyAtTheLatestVersionError{Platform: platformRef.String()}
 	}
 	platformRef.PlatformVersion = latest.Version
 
 	platformRelease, tools, err := pme.FindPlatformReleaseDependencies(platformRef)
 	if err != nil {
-		return &arduino.PlatformNotFoundError{Platform: platformRef.String()}
+		return nil, &arduino.PlatformNotFoundError{Platform: platformRef.String()}
 	}
 	if err := pme.DownloadAndInstallPlatformAndTools(platformRelease, tools, downloadCB, taskCB, skipPostInstall); err != nil {
-		return err
+		return nil, err
 	}
 
-	return nil
+	return platformRelease, nil
 }
 
 // DownloadAndInstallPlatformAndTools runs a full installation process for the given platform and tools.

commands/board/listall.go

@@ -62,6 +62,8 @@ func ListAll(ctx context.Context, req *rpc.BoardListAllRequest) (*rpc.BoardListA
 				Website:           platform.Package.WebsiteURL,
 				Email:             platform.Package.Email,
 				ManuallyInstalled: platform.ManuallyInstalled,
+				Indexed:           platform.Indexed,
+				MissingMetadata:   !installedPlatformRelease.HasMetadata(),
 			}
 
 			toTest := []string{

commands/board/search.go

@@ -54,13 +54,15 @@ func Search(ctx context.Context, req *rpc.BoardSearchRequest) (*rpc.BoardSearchR
 				Website:           platform.Package.WebsiteURL,
 				Email:             platform.Package.Email,
 				ManuallyInstalled: platform.ManuallyInstalled,
+				Indexed:           platform.Indexed,
 			}
 
 			if latestPlatformRelease != nil {
 				rpcPlatform.Latest = latestPlatformRelease.Version.String()
 			}
 			if installedPlatformRelease != nil {
 				rpcPlatform.Installed = installedPlatformRelease.Version.String()
+				rpcPlatform.MissingMetadata = !installedPlatformRelease.HasMetadata()
 			}
 
 			// Platforms that are not installed don't have a list of boards

commands/core.go

@@ -62,6 +62,8 @@ func PlatformReleaseToRPC(platformRelease *cores.PlatformRelease) *rpc.Platform
 		ManuallyInstalled: platformRelease.Platform.ManuallyInstalled,
 		Deprecated:        platformRelease.Platform.Deprecated,
 		Type:              []string{platformRelease.Platform.Category},
+		Indexed:           platformRelease.Platform.Indexed,
+		MissingMetadata:   !platformRelease.HasMetadata(),
 	}
 
 	return result