go/cipd/cipd.go

package cipd

/*
	Utilities for working with CIPD.
*/

//go:generate go run gen_versions.go

import (
	"context"
	"fmt"
	"io"
	"net/http"

	"go.chromium.org/luci/cipd/client/cipd"
	"go.chromium.org/luci/cipd/client/cipd/ensure"
	"go.chromium.org/luci/cipd/common"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
)

const (
	// CIPD server to use for obtaining packages.
	ServiceUrl = "https://chrome-infra-packages.appspot.com"

	// Platforms supported by CIPD.
	PlatformLinuxAmd64   = "linux-amd64"
	PlatformLinuxArm64   = "linux-arm64"
	PlatformMacAmd64     = "mac-amd64"
	PlatformWindows386   = "windows-386"
	PlatformWindowsAmd64 = "windows-amd64"

	// Placeholder for target platform.
	PlatformPlaceholder = "${platform}"

	// Template for Git CIPD package for a particular platform.
	pkgGitTmpl = "infra/3pp/tools/git/%s"
)

var (
	// CIPD package for the Go installation.
	PkgGo = MustGetPackage("skia/bots/go")

	// CIPD package containing the Google Protocol Buffer compiler.
	PkgProtoc = MustGetPackage("skia/bots/protoc")

	// CIPD packages required for using Git.
	PkgsGit = map[string][]*Package{
		PlatformLinuxAmd64: {
			MustGetPackage(fmt.Sprintf(pkgGitTmpl, PlatformLinuxAmd64)),
			MustGetPackage("infra/tools/git/${platform}"),
			MustGetPackage("infra/tools/luci/git-credential-luci/${platform}"),
		},
		PlatformLinuxArm64: {
			MustGetPackage(fmt.Sprintf(pkgGitTmpl, PlatformLinuxArm64)),
			MustGetPackage("infra/tools/git/${platform}"),
			MustGetPackage("infra/tools/luci/git-credential-luci/${platform}"),
		},
		PlatformMacAmd64: {
			MustGetPackage(fmt.Sprintf(pkgGitTmpl, PlatformMacAmd64)),
			MustGetPackage("infra/tools/git/${platform}"),
			MustGetPackage("infra/tools/luci/git-credential-luci/${platform}"),
		},
		PlatformWindows386: {
			MustGetPackage(fmt.Sprintf(pkgGitTmpl, PlatformWindows386)),
			MustGetPackage("infra/tools/git/${platform}"),
			MustGetPackage("infra/tools/luci/git-credential-luci/${platform}"),
		},
		PlatformWindowsAmd64: {
			MustGetPackage(fmt.Sprintf(pkgGitTmpl, PlatformWindowsAmd64)),
			MustGetPackage("infra/tools/git/${platform}"),
			MustGetPackage("infra/tools/luci/git-credential-luci/${platform}"),
		},
	}

	// CIPD packages required for using Python.
	PkgsPython = []*Package{
		MustGetPackage("infra/3pp/tools/cpython/${platform}"),
		MustGetPackage("infra/tools/luci/vpython/${platform}"),
	}
)

// VersionTag returns a CIPD version tag for the given version number.
func VersionTag(version string) string {
	return fmt.Sprintf("version:%s", version)
}

// Package describes a CIPD package.
type Package struct {
	// Name of the package.
	Name string `json:"name"`

	// Relative path within the root dir to install the package.
	Path string `json:"path"`

	// Version of the package. See the CIPD docs for valid version strings:
	// https://godoc.org/go.chromium.org/luci/cipd/common#ValidateInstanceVersion
	Version string `json:"version"`
}

func (p *Package) String() string {
	return fmt.Sprintf("%s:%s:%s", p.Path, p.Name, p.Version)
}

// PackageSlice is used for sorting packages by name.
type PackageSlice []*Package

func (s PackageSlice) Len() int           { return len(s) }
func (s PackageSlice) Less(i, j int) bool { return s[i].Name < s[j].Name }
func (s PackageSlice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

// GetPackage returns the definition for the package with the given name, or an
// error if the package does not exist in the registry.
func GetPackage(pkg string) (*Package, error) {
	rv, ok := PACKAGES[pkg]
	if !ok {
		return nil, skerr.Fmt("Unknown CIPD package %q", pkg)
	}
	return rv, nil
}

// MustGetPackage returns the definition for the package with the given name.
// Panics if the package does not exist in the registry.
func MustGetPackage(pkg string) *Package {
	rv, err := GetPackage(pkg)
	if err != nil {
		sklog.Fatal(err)
	}
	return rv
}

// Utility function that returns CIPD packages as slice of strings. Created for
// go/swarming, this can be removed when go/swarming has no more clients.
func GetStrCIPDPkgs(pkgs []*Package) []string {
	cipdPkgs := []string{}
	for _, p := range pkgs {
		cipdPkgs = append(cipdPkgs, p.String())
	}
	return cipdPkgs
}

// Run "cipd ensure" to get the correct packages in the given location. Note
// that any previously-installed packages in the given rootDir will be removed
// if not specified again.
func Ensure(ctx context.Context, c *http.Client, rootDir string, packages ...*Package) error {
	cipdClient, err := NewClient(c, rootDir)
	if err != nil {
		return fmt.Errorf("Failed to create CIPD client: %s", err)
	}
	return cipdClient.Ensure(ctx, packages...)
}

// ParseEnsureFile parses a CIPD ensure file and returns a slice of Packages.
func ParseEnsureFile(file string) ([]*Package, error) {
	var ensureFile *ensure.File
	if err := util.WithReadFile(file, func(r io.Reader) error {
		f, err := ensure.ParseFile(r)
		if err == nil {
			ensureFile = f
		}
		return err
	}); err != nil {
		return nil, skerr.Wrapf(err, "Failed to parse CIPD ensure file %s", file)
	}
	var rv []*Package
	for subdir, pkgSlice := range ensureFile.PackagesBySubdir {
		if subdir == "" {
			subdir = "."
		}
		for _, pkg := range pkgSlice {
			rv = append(rv, &Package{
				Path:    subdir,
				Name:    pkg.PackageTemplate,
				Version: pkg.UnresolvedVersion,
			})
		}
	}
	return rv, nil
}

// CIPDClient is the interface for interactions with the CIPD API.
type CIPDClient interface {
	cipd.Client

	// Ensure runs "cipd ensure" to get the correct packages in the given location. Note
	// that any previously-installed packages in the given rootDir will be removed
	// if not specified again.
	Ensure(ctx context.Context, packages ...*Package) error

	// Describe is a convenience wrapper around cipd.Client.DescribeInstance.
	Describe(ctx context.Context, pkg, instance string) (*cipd.InstanceDescription, error)
}

// Client is a struct used for interacting with the CIPD API.
type Client struct {
	cipd.Client
}

// NewClient returns a CIPD client.
func NewClient(c *http.Client, rootDir string) (*Client, error) {
	cipdClient, err := cipd.NewClient(cipd.ClientOptions{
		ServiceURL:          ServiceUrl,
		Root:                rootDir,
		AuthenticatedClient: c,
	})
	if err != nil {
		return nil, fmt.Errorf("Failed to create CIPD client: %s", err)
	}
	return &Client{cipdClient}, nil
}

func (c *Client) Ensure(ctx context.Context, packages ...*Package) error {
	pkgs := common.PinSliceBySubdir{}
	for _, pkg := range packages {
		pin, err := c.ResolveVersion(ctx, pkg.Name, pkg.Version)
		if err != nil {
			return fmt.Errorf("Failed to resolve package version %q @ %q: %s", pkg.Name, pkg.Version, err)
		}
		sklog.Infof("Installing version %s (from %s) of %s", pin.InstanceID, pkg.Version, pkg.Name)
		pkgs[pkg.Path] = common.PinSlice{pin}
	}
	// This means use as many threads as CPUs. (Prior to
	// https://chromium-review.googlesource.com/c/infra/luci/luci-go/+/1848212,
	// extracting the packages was always single-threaded.)
	const maxThreads = 0
	if _, err := c.EnsurePackages(ctx, pkgs, cipd.CheckPresence, maxThreads, false); err != nil {
		return fmt.Errorf("Failed to ensure packages: %s", err)
	}
	return nil
}

func (c *Client) Describe(ctx context.Context, pkg, instance string) (*cipd.InstanceDescription, error) {
	pin := common.Pin{
		PackageName: pkg,
		InstanceID:  instance,
	}
	opts := &cipd.DescribeInstanceOpts{
		DescribeRefs: true,
		DescribeTags: true,
	}
	return c.DescribeInstance(ctx, pin, opts)
}