Support for git-downloaded libraries in profiles.

commands/instances.go

@@ -388,6 +388,40 @@ func (s *arduinoCoreServerImpl) Init(req *rpc.InitRequest, stream rpc.ArduinoCor
 				continue
 			}
 
+			if libraryRef.GitURL != nil {
+				uid := libraryRef.InternalUniqueIdentifier()
+				libRoot := s.settings.ProfilesCacheDir().Join(uid)
+				libDir := libRoot.Join(libraryRef.Library)
+
+				if !libDir.IsDir() {
+					// Clone repo and install
+					tmpDir, err := librariesmanager.CloneLibraryGitRepository(ctx, libraryRef.GitURL.String())
+					if err != nil {
+						taskCallback(&rpc.TaskProgress{Name: i18n.Tr("Error downloading library %s", libraryRef)})
+						e := &cmderrors.FailedLibraryInstallError{Cause: err}
+						responseError(e.GRPCStatus())
+						continue
+					}
+
+					// Install library into profile cache
+					copyErr := tmpDir.CopyDirTo(libDir)
+					_ = tmpDir.RemoveAll()
+					if copyErr != nil {
+						taskCallback(&rpc.TaskProgress{Name: i18n.Tr("Error installing library %s", libraryRef)})
+						e := &cmderrors.FailedLibraryInstallError{Cause: fmt.Errorf("copying library to profile cache: %w", err)}
+						responseError(e.GRPCStatus())
+						continue
+					}
+				}
+
+				lmb.AddLibrariesDir(librariesmanager.LibrariesDir{
+					Path:            libDir,
+					Location:        libraries.Profile,
+					IsSingleLibrary: true,
+				})
+				continue
+			}
+
 			uid := libraryRef.InternalUniqueIdentifier()
 			libRoot := s.settings.ProfilesCacheDir().Join(uid)
 			libDir := libRoot.Join(libraryRef.Library)

commands/service_library_install.go

@@ -267,9 +267,8 @@ func (s *arduinoCoreServerImpl) GitLibraryInstall(req *rpc.GitLibraryInstallRequ
 	lmi, release := lm.NewInstaller()
 	defer release()
 
-	// TODO: pass context
-	// ctx := stream.Context()
-	if err := lmi.InstallGitLib(req.GetUrl(), req.GetOverwrite()); err != nil {
+	ctx := stream.Context()
+	if err := lmi.InstallGitLib(ctx, req.GetUrl(), req.GetOverwrite()); err != nil {
 		return &cmderrors.FailedLibraryInstallError{Cause: err}
 	}
 	taskCB(&rpc.TaskProgress{Message: i18n.Tr("Library installed"), Completed: true})

commands/service_profile_lib_add.go

@@ -18,6 +18,7 @@ package commands
 import (
 	"cmp"
 	"context"
+	"net/url"
 	"slices"
 
 	"github.com/arduino/arduino-cli/commands/cmderrors"
@@ -59,6 +60,15 @@ func (s *arduinoCoreServerImpl) ProfileLibAdd(ctx context.Context, req *rpc.Prof
 		addedLib := &sketch.ProfileLibraryReference{InstallDir: path}
 		profile.Libraries = append(profile.Libraries, addedLib)
 		addedLibs = append(addedLibs, addedLib)
+	} else if reqGitLib := req.GetLibrary().GetGitLibrary(); reqGitLib != nil {
+		// Add a git library
+		gitURL, err := url.Parse(reqGitLib.GetUrl())
+		if err != nil {
+			return nil, &cmderrors.InvalidURLError{Cause: err}
+		}
+		addedLib := &sketch.ProfileLibraryReference{GitURL: gitURL}
+		profile.Libraries = append(profile.Libraries, addedLib)
+		addedLibs = append(addedLibs, addedLib)
 	} else if reqIndexLib := req.GetLibrary().GetIndexLibrary(); reqIndexLib != nil {
 		// Obtain the library index from the manager
 		li, err := instances.GetLibrariesIndex(req.GetInstance())

internal/arduino/libraries/librariesmanager/install.go

@@ -200,54 +200,64 @@ func (lmi *Installer) InstallZipLib(ctx context.Context, archivePath *paths.Path
 }
 
 // InstallGitLib installs a library hosted on a git repository on the specified path.
-func (lmi *Installer) InstallGitLib(argURL string, overwrite bool) error {
-	libraryName, gitURL, ref, err := parseGitArgURL(argURL)
+func (lmi *Installer) InstallGitLib(ctx context.Context, argURL string, overwrite bool) error {
+	tmpInstallPath, err := CloneLibraryGitRepository(ctx, argURL)
 	if err != nil {
 		return err
 	}
+	defer tmpInstallPath.RemoveAll()
+
+	// Install extracted library in the destination directory
+	if err := lmi.importLibraryFromDirectory(tmpInstallPath, overwrite); err != nil {
+		return errors.New(i18n.Tr("moving extracted archive to destination dir: %s", err))
+	}
+
+	return nil
+}
+
+// CloneLibraryGitRepository clones a git repository containing a library
+// into a temporary directory and returns the path to the cloned library.
+func CloneLibraryGitRepository(ctx context.Context, argURL string) (*paths.Path, error) {
+	libraryName, gitURL, ref, err := parseGitArgURL(argURL)
+	if err != nil {
+		return nil, err
+	}
 
 	// Clone library in a temporary directory
 	tmp, err := paths.MkTempDir("", "")
 	if err != nil {
-		return err
+		return nil, err
 	}
-	defer tmp.RemoveAll()
 	tmpInstallPath := tmp.Join(libraryName)
 
-	if _, err := git.PlainClone(tmpInstallPath.String(), false, &git.CloneOptions{
+	if _, err := git.PlainCloneContext(ctx, tmpInstallPath.String(), false, &git.CloneOptions{
 		URL:           gitURL,
 		ReferenceName: plumbing.ReferenceName(ref),
 	}); err != nil {
 		if err.Error() != "reference not found" {
-			return err
+			return nil, err
 		}
 
 		// We did not find the requested reference, let's do a PlainClone and use
 		// "ResolveRevision" to find and checkout the requested revision
-		if repo, err := git.PlainClone(tmpInstallPath.String(), false, &git.CloneOptions{
+		if repo, err := git.PlainCloneContext(ctx, tmpInstallPath.String(), false, &git.CloneOptions{
 			URL: gitURL,
 		}); err != nil {
-			return err
+			return nil, err
 		} else if h, err := repo.ResolveRevision(plumbing.Revision(ref)); err != nil {
-			return err
+			return nil, err
 		} else if w, err := repo.Worktree(); err != nil {
-			return err
+			return nil, err
 		} else if err := w.Checkout(&git.CheckoutOptions{
 			Force: true, // workaround for: https://github.com/go-git/go-git/issues/1411
 			Hash:  plumbing.NewHash(h.String())}); err != nil {
-			return err
+			return nil, err
 		}
 	}
 
 	// We don't want the installed library to be a git repository thus we delete this folder
 	tmpInstallPath.Join(".git").RemoveAll()
-
-	// Install extracted library in the destination directory
-	if err := lmi.importLibraryFromDirectory(tmpInstallPath, overwrite); err != nil {
-		return errors.New(i18n.Tr("moving extracted archive to destination dir: %s", err))
-	}
-
-	return nil
+	return tmpInstallPath, nil
 }
 
 // parseGitArgURL tries to recover a library name from a git URL.

internal/arduino/sketch/profiles.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"path/filepath"
 	"regexp"
 	"slices"
 	"strings"
@@ -354,6 +355,7 @@ type ProfileLibraryReference struct {
 	Version      *semver.Version
 	IsDependency bool
 	InstallDir   *paths.Path
+	GitURL       *url.URL
 }
 
 // UnmarshalYAML decodes a ProfileLibraryReference from YAML source.
@@ -375,6 +377,19 @@ func (l *ProfileLibraryReference) UnmarshalYAML(unmarshal func(interface{}) erro
 			}
 			l.IsDependency = true
 			// Fallback
+		} else if gitUrl, ok := dataMap["git"]; ok {
+			if gitUrlStr, ok := gitUrl.(string); !ok {
+				return fmt.Errorf("%s: %s", i18n.Tr("invalid library reference"), dataMap)
+			} else if parsedUrl, err := url.Parse(gitUrlStr); err != nil {
+				return fmt.Errorf("%s: %w", i18n.Tr("invalid git URL"), err)
+			} else {
+				l.GitURL = parsedUrl
+				if l.Library = filepath.Base(parsedUrl.Path); l.Library == "" {
+					l.Library = "lib"
+				}
+				l.Library = strings.TrimSuffix(l.Library, ".git")
+				return nil
+			}
 		} else {
 			return fmt.Errorf("%s: %s", i18n.Tr("invalid library reference"), dataMap)
 		}
@@ -401,6 +416,9 @@ func (l *ProfileLibraryReference) AsYaml() string {
 	if l.InstallDir != nil {
 		return fmt.Sprintf("      - dir: %s\n", l.InstallDir)
 	}
+	if l.GitURL != nil {
+		return fmt.Sprintf("      - git: %s\n", l.GitURL)
+	}
 	dep := ""
 	if l.IsDependency {
 		dep = "dependency: "
@@ -412,6 +430,9 @@ func (l *ProfileLibraryReference) String() string {
 	if l.InstallDir != nil {
 		return "@dir:" + l.InstallDir.String()
 	}
+	if l.GitURL != nil {
+		return "@git:" + l.GitURL.String()
+	}
 	dep := ""
 	if l.IsDependency {
 		dep = " (dep)"
@@ -431,6 +452,14 @@ func (l *ProfileLibraryReference) Match(other *ProfileLibraryReference) bool {
 	if other.InstallDir != nil {
 		return false
 	}
+
+	if l.GitURL != nil {
+		return other.GitURL != nil && l.GitURL.String() == other.GitURL.String()
+	}
+	if other.GitURL != nil {
+		return false
+	}
+
 	if l.Library != other.Library {
 		return false
 	}
@@ -451,6 +480,15 @@ func (l *ProfileLibraryReference) ToRpc() *rpc.ProfileLibraryReference {
 			},
 		}
 	}
+	if l.GitURL != nil {
+		return &rpc.ProfileLibraryReference{
+			Library: &rpc.ProfileLibraryReference_GitLibrary_{
+				GitLibrary: &rpc.ProfileLibraryReference_GitLibrary{
+					Url: l.GitURL.String(),
+				},
+			},
+		}
+	}
 	return &rpc.ProfileLibraryReference{
 		Library: &rpc.ProfileLibraryReference_IndexLibrary_{
 			IndexLibrary: &rpc.ProfileLibraryReference_IndexLibrary{
@@ -471,6 +509,13 @@ func FromRpcProfileLibraryReference(l *rpc.ProfileLibraryReference) (*ProfileLib
 		}
 		return &ProfileLibraryReference{InstallDir: path}, nil
 	}
+	if gitLib := l.GetGitLibrary(); gitLib != nil {
+		gitURL, err := url.Parse(gitLib.GetUrl())
+		if err != nil {
+			return nil, &cmderrors.InvalidURLError{Cause: err}
+		}
+		return &ProfileLibraryReference{GitURL: gitURL}, nil
+	}
 	if indexLib := l.GetIndexLibrary(); indexLib != nil {
 		var version *semver.Version
 		if indexLib.GetVersion() != "" {
@@ -494,6 +539,15 @@ func (l *ProfileLibraryReference) InternalUniqueIdentifier() string {
 	f.Assert(l.InstallDir == nil,
 		"InternalUniqueIdentifier should not be called for library references with an install directory")
 
+	if l.GitURL != nil {
+		id := "git-" + utils.SanitizeName(l.GitURL.Host+l.GitURL.Path+"#"+l.GitURL.Fragment)
+		if len(id) > 50 {
+			id = id[:50]
+		}
+		h := sha256.Sum256([]byte(l.GitURL.String()))
+		return id + "-" + hex.EncodeToString(h[:])[:8]
+	}
+
 	id := l.Library + "@" + l.Version.String()
 	h := sha256.Sum256([]byte(id))
 	res := fmt.Sprintf("%s_%s", id, hex.EncodeToString(h[:])[:16])

internal/arduino/sketch/profiles_test.go

@@ -60,15 +60,19 @@ func TestProjectFileLibraries(t *testing.T) {
 	require.NoError(t, err)
 	require.Len(t, proj.Profiles, 1)
 	prof := proj.Profiles[0]
-	require.Len(t, prof.Libraries, 4)
+	require.Len(t, prof.Libraries, 6)
 	require.Equal(t, "[email protected]", prof.Libraries[0].String())
 	require.Equal(t, "@dir:/path/to/system/lib", prof.Libraries[1].String())
 	require.Equal(t, "@dir:path/to/sketch/lib", prof.Libraries[2].String())
 	require.Equal(t, "[email protected] (dep)", prof.Libraries[3].String())
+	require.Equal(t, "@git:https://github.com/username/HelloWorld.git#v2.13", prof.Libraries[4].String())
+	require.Equal(t, "@git:https://github.com/username/HelloWorld.git#v2.14", prof.Libraries[5].String())
 	require.Equal(t, "FlashStorage_1.2.3_e525d7c96b27788f", prof.Libraries[0].InternalUniqueIdentifier())
 	require.Panics(t, func() { prof.Libraries[1].InternalUniqueIdentifier() })
 	require.Panics(t, func() { prof.Libraries[2].InternalUniqueIdentifier() })
 	require.Equal(t, "DependencyLib_2.3.4_ecde631facb47ae5", prof.Libraries[3].InternalUniqueIdentifier())
+	require.Equal(t, "git-github.com_username_HelloWorld.git_v2.13-0c146203", prof.Libraries[4].InternalUniqueIdentifier())
+	require.Equal(t, "git-github.com_username_HelloWorld.git_v2.14-49f5df7f", prof.Libraries[5].InternalUniqueIdentifier())
 
 	orig, err := sketchProj.ReadFile()
 	require.NoError(t, err)

internal/arduino/sketch/testdata/profiles/profile_with_libraries.yml

@@ -8,5 +8,7 @@ profiles:
       - dir: /path/to/system/lib
       - dir: path/to/sketch/lib
       - dependency: DependencyLib (2.3.4)
+      - git: https://github.com/username/HelloWorld.git#v2.13
+      - git: https://github.com/username/HelloWorld.git#v2.14
 
 default_profile: giga_any

internal/cli/feedback/result/rpc.go

@@ -1126,6 +1126,22 @@ func NewIndexUpdateReport_Status(r rpc.IndexUpdateReport_Status) IndexUpdateRepo
 	}
 }
 
+type ProfileLibraryReference_GitLibraryResult struct {
+	Url string `json:"url,omitempty"`
+}
+
+func NewProfileLibraryReference_GitLibraryResult(resp *rpc.ProfileLibraryReference_GitLibrary) *ProfileLibraryReference_GitLibraryResult {
+	return &ProfileLibraryReference_GitLibraryResult{
+		Url: resp.GetUrl(),
+	}
+}
+
+func (*ProfileLibraryReference_GitLibraryResult) isProfileLibraryReference() {}
+
+func (l *ProfileLibraryReference_GitLibraryResult) String() string {
+	return fmt.Sprintf("git: %s", l.Url)
+}
+
 type ProfileLibraryReference_LocalLibraryResult struct {
 	Path string `json:"path,omitempty"`
 }
@@ -1182,6 +1198,12 @@ func NewProfileLibraryReference(resp *rpc.ProfileLibraryReference) *ProfileLibra
 			Kind:    "index",
 		}
 	}
+	if lib := resp.GetGitLibrary(); lib != nil {
+		return &ProfileLibraryReference{
+			Library: NewProfileLibraryReference_GitLibraryResult(lib),
+			Kind:    "git",
+		}
+	}
 	if lib := resp.GetLocalLibrary(); lib != nil {
 		return &ProfileLibraryReference{
 			Library: NewProfileLibraryReference_LocalLibraryResult(lib),

internal/cli/feedback/result/rpc_test.go

@@ -238,6 +238,10 @@ func TestAllFieldAreMapped(t *testing.T) {
 	profileLibraryReference_IndexLibraryResult := result.NewProfileLibraryReference_IndexLibraryResult(profileLibraryReference_IndexLibraryRpc)
 	mustContainsAllPropertyOfRpcStruct(t, profileLibraryReference_IndexLibraryRpc, profileLibraryReference_IndexLibraryResult)
 
+	profileLibraryReference_GitLibraryRpc := &rpc.ProfileLibraryReference_GitLibrary{}
+	profileLibraryReference_GitLibraryResult := result.NewProfileLibraryReference_GitLibraryResult(profileLibraryReference_GitLibraryRpc)
+	mustContainsAllPropertyOfRpcStruct(t, profileLibraryReference_GitLibraryRpc, profileLibraryReference_GitLibraryResult)
+
 	profileLibraryReference_LocalLibraryRpc := &rpc.ProfileLibraryReference_LocalLibrary{}
 	profileLibraryReference_LocalLibraryResult := result.NewProfileLibraryReference_LocalLibraryResult(profileLibraryReference_LocalLibraryRpc)
 	mustContainsAllPropertyOfRpcStruct(t, profileLibraryReference_LocalLibraryRpc, profileLibraryReference_LocalLibraryResult)