chore: improve downloader

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/fatih/color v1.18.0
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/rjeczalik/notify v0.9.3
+	github.com/schollz/progressbar/v3 v3.19.0
 	github.com/sevlyar/go-daemon v0.1.6
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.9.1
@@ -20,7 +21,8 @@ require (
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/spf13/pflag v1.0.6 // indirect
-	github.com/stretchr/testify v1.8.4 // indirect
 	golang.org/x/sys v0.34.0 // indirect
 )

go.sum

@@ -1,3 +1,5 @@
+github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM=
+github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY=
 github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4=
 github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec=
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
@@ -19,11 +21,19 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
+github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rjeczalik/notify v0.9.3 h1:6rJAzHTGKXGj76sbRgDiDcYj/HniypXmSJo1SWakZeY=
 github.com/rjeczalik/notify v0.9.3/go.mod h1:gF3zSOrafR9DQEWSE8TjfI9NkooDxbyT4UgRGKZA0lc=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1IvohyTutOIFoc=
+github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
 github.com/sevlyar/go-daemon v0.1.6 h1:EUh1MDjEM4BI109Jign0EaknA2izkOyi0LV3ro3QQGs=
 github.com/sevlyar/go-daemon v0.1.6/go.mod h1:6dJpPatBT9eUwM5VCw9Bt6CdX9Tk6UWvhW3MebLDRKE=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
@@ -34,8 +44,8 @@ github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
 github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

util/downloader/download.go

@@ -1,16 +1,18 @@
 package downloader
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/abiosoft/colima/config"
 	"github.com/abiosoft/colima/environment"
 	"github.com/abiosoft/colima/util/shautil"
-	"github.com/abiosoft/colima/util/terminal"
 )
 
 type (
@@ -45,22 +47,18 @@ func DownloadToGuest(host hostActions, guest guestActions, r Request, filename s
 
 // Download downloads file at url and returns the location of the downloaded file.
 func Download(host hostActions, r Request) (string, error) {
-	d := downloader{
-		host: host,
-	}
+	d := downloader{}
 
 	if !d.hasCache(r.URL) {
 		if err := d.downloadFile(r); err != nil {
-			return "", fmt.Errorf("error downloading '%s': %w", r.URL, err)
+			return "", err
 		}
 	}
 
 	return CacheFilename(r.URL), nil
 }
 
-type downloader struct {
-	host hostActions
-}
+type downloader struct{}
 
 // CacheFilename returns the computed filename for the url.
 func CacheFilename(url string) string {
@@ -71,40 +69,85 @@ func (d downloader) cacheDownloadingFileName(url string) string {
 	return CacheFilename(url) + ".downloading"
 }
 
+func (d downloader) resumeInfoPath(url string) string {
+	return CacheFilename(url) + ".resume"
+}
+
 func (d downloader) downloadFile(r Request) (err error) {
-	// save to a temporary file initially before renaming to the desired file after successful download
-	// this prevents having a corrupt file
 	cacheDownloadingFilename := d.cacheDownloadingFileName(r.URL)
-	if err := d.host.RunQuiet("mkdir", "-p", filepath.Dir(cacheDownloadingFilename)); err != nil {
+
+	// create cache directory
+	cacheDir := filepath.Dir(cacheDownloadingFilename)
+	if err := os.MkdirAll(cacheDir, 0755); err != nil {
 		return fmt.Errorf("error preparing cache dir: %w", err)
 	}
 
-	// get rid of curl's initial progress bar by getting the redirect url directly.
-	downloadURL, err := d.host.RunOutput("curl", "-ILs", "-o", "/dev/null", "-w", "%{url_effective}", r.URL)
+	// check for existing partial download and resume info
+	var resumeInfo ResumeInfo
+	resumeInfoPath := d.resumeInfoPath(r.URL)
+	if data, err := os.ReadFile(resumeInfoPath); err == nil {
+		_ = json.Unmarshal(data, &resumeInfo)
+	}
+
+	// get existing file size for resume
+	var existingSize int64
+	if stat, err := os.Stat(cacheDownloadingFilename); err == nil {
+		existingSize = stat.Size()
+	}
+
+	// create HTTP client
+	client := NewHTTPClient()
+
+	// use a long timeout for large files (2 hours)
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Hour)
+	defer cancel()
+
+	// get final URL (follows redirects)
+	finalURL, err := client.GetFinalURL(ctx, r.URL)
 	if err != nil {
-		return fmt.Errorf("error retrieving redirect url: %w", err)
+		return fmt.Errorf("error resolving download URL '%s': %w", r.URL, err)
 	}
 
-	// ask curl to resume previous download if possible "-C -"
-	if err := d.host.RunInteractive("curl", "-L", "-#", "-C", "-", "-o", cacheDownloadingFilename, downloadURL); err != nil {
-		return err
+	// download the file
+	result, err := client.Download(ctx, DownloadOptions{
+		URL:            finalURL,
+		DestPath:       cacheDownloadingFilename,
+		ExpectedETag:   resumeInfo.ETag,
+		ResumeFromByte: existingSize,
+		ShowProgress:   true,
+	})
+	if err != nil {
+		// save resume info for next attempt if we have ETag
+		if result != nil && result.ETag != "" {
+			d.saveResumeInfo(r.URL, result.ETag, existingSize)
+		}
+		return fmt.Errorf("error downloading '%s': %w", path.Base(r.URL), err)
 	}
-	// clear curl progress line
-	terminal.ClearLine()
 
-	// validate download if sha is present
-	if r.SHA != nil {
-		if err := r.SHA.validateDownload(d.host, r.URL, cacheDownloadingFilename); err != nil {
+	// clean up resume info on successful download
+	_ = os.Remove(resumeInfoPath)
 
+	// validate download if SHA is present
+	if r.SHA != nil {
+		if err := r.SHA.validateDownload(r.URL, cacheDownloadingFilename); err != nil {
 			// move file to allow subsequent re-download
-			// error discarded, would not be actioned anyways
-			_ = d.host.RunQuiet("mv", cacheDownloadingFilename, cacheDownloadingFilename+".invalid")
-
+			_ = os.Rename(cacheDownloadingFilename, cacheDownloadingFilename+".invalid")
 			return fmt.Errorf("error validating SHA sum for '%s': %w", path.Base(r.URL), err)
 		}
 	}
 
-	return d.host.RunQuiet("mv", cacheDownloadingFilename, CacheFilename(r.URL))
+	// move completed download to final location
+	if err := os.Rename(cacheDownloadingFilename, CacheFilename(r.URL)); err != nil {
+		return fmt.Errorf("error finalizing download: %w", err)
+	}
+
+	return nil
+}
+
+func (d downloader) saveResumeInfo(url, etag string, bytesWritten int64) {
+	info := ResumeInfo{ETag: etag, BytesWritten: bytesWritten}
+	data, _ := json.Marshal(info)
+	_ = os.WriteFile(d.resumeInfoPath(url), data, 0644)
 }
 
 func (d downloader) hasCache(url string) bool {

util/downloader/errors.go

@@ -0,0 +1,121 @@
+package downloader
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"net/url"
+	"path"
+	"syscall"
+)
+
+// Sentinel errors for type checking
+var (
+	ErrNetworkConnection = errors.New("network connection error")
+	ErrHTTPStatus        = errors.New("HTTP error")
+	ErrResumeFailed      = errors.New("resume failed")
+	ErrSHAValidation     = errors.New("SHA validation failed")
+)
+
+// NetworkError wraps network-related errors with user-friendly messages
+type NetworkError struct {
+	Op  string // "connect", "resolve", "download"
+	URL string
+	Err error
+}
+
+func (e *NetworkError) Error() string {
+	return fmt.Sprintf("%s failed for '%s': %s", e.Op, e.URL, e.friendlyMessage())
+}
+
+func (e *NetworkError) Unwrap() error {
+	return e.Err
+}
+
+func (e *NetworkError) friendlyMessage() string {
+	// check for DNS resolution errors
+	var dnsErr *net.DNSError
+	if errors.As(e.Err, &dnsErr) {
+		return fmt.Sprintf("DNS lookup failed for host '%s'. Check your network connection or DNS settings", dnsErr.Name)
+	}
+
+	// check for connection refused
+	var opErr *net.OpError
+	if errors.As(e.Err, &opErr) {
+		if errors.Is(opErr.Err, syscall.ECONNREFUSED) {
+			return "connection refused. The server may be down or unreachable"
+		}
+		if opErr.Timeout() {
+			return "connection timed out. Check your network connection or try again later"
+		}
+	}
+
+	// check for URL parsing errors
+	var urlErr *url.Error
+	if errors.As(e.Err, &urlErr) {
+		if urlErr.Timeout() {
+			return "request timed out. The server may be slow or overloaded"
+		}
+	}
+
+	return e.Err.Error()
+}
+
+// HTTPStatusError represents HTTP error responses
+type HTTPStatusError struct {
+	StatusCode int
+	Status     string
+	URL        string
+}
+
+func (e *HTTPStatusError) Error() string {
+	switch e.StatusCode {
+	case 404:
+		return fmt.Sprintf("file not found at '%s'. The URL may be incorrect or the file may have been removed", e.URL)
+	case 403:
+		return fmt.Sprintf("access forbidden to '%s'. You may need authentication or the resource is restricted", e.URL)
+	case 401:
+		return fmt.Sprintf("authentication required for '%s'", e.URL)
+	case 500, 502, 503, 504:
+		return fmt.Sprintf("server error (%d) at '%s'. Try again later", e.StatusCode, e.URL)
+	case 416: // Range Not Satisfiable
+		return fmt.Sprintf("resume failed for '%s'. The server does not support the requested byte range", e.URL)
+	default:
+		return fmt.Sprintf("HTTP %d (%s) for '%s'", e.StatusCode, e.Status, e.URL)
+	}
+}
+
+func (e *HTTPStatusError) Unwrap() error {
+	return ErrHTTPStatus
+}
+
+// ResumeError indicates a failed resume attempt
+type ResumeError struct {
+	Reason string
+	URL    string
+}
+
+func (e *ResumeError) Error() string {
+	return fmt.Sprintf("cannot resume download of '%s': %s. Starting fresh download", path.Base(e.URL), e.Reason)
+}
+
+func (e *ResumeError) Unwrap() error {
+	return ErrResumeFailed
+}
+
+// SHAValidationError indicates checksum mismatch
+type SHAValidationError struct {
+	File     string
+	Expected string
+	Actual   string
+	Size     int // 256 or 512
+}
+
+func (e *SHAValidationError) Error() string {
+	return fmt.Sprintf("SHA%d checksum mismatch for '%s':\n  expected: %s\n  actual:   %s\nThe file may be corrupted or tampered with. Delete the cached file and retry",
+		e.Size, e.File, e.Expected, e.Actual)
+}
+
+func (e *SHAValidationError) Unwrap() error {
+	return ErrSHAValidation
+}