autoroll/go/repo_manager/fuchsia_sdk_android_repo_manager.go

package repo_manager

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"

	"cloud.google.com/go/storage"
	"go.skia.org/infra/autoroll/go/codereview"
	"go.skia.org/infra/autoroll/go/strategy"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/fileutil"
	"go.skia.org/infra/go/gcs"
	"go.skia.org/infra/go/gerrit"
	"go.skia.org/infra/go/git"
	"go.skia.org/infra/go/gitiles"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/tar"
	"google.golang.org/api/option"
)

const (
	ANDROID_BP                       = "Android.bp"
	FUCHSIA_SDK_ANDROID_VERSION_FILE = "hash"
	FUCHSIA_UTIL                     = "fuchsia_util.py"
	GEN_SDK_BP                       = "gen_sdk_bp.py"
	GEN_SDK_BP_DIR                   = "scripts"
	SDK_DEST_PATH                    = "prebuilts/fuchsia_sdk"
)

type FuchsiaSDKAndroidRepoManagerConfig struct {
	FuchsiaSDKRepoManagerConfig
	GenSdkBpRepo string `json:"genSdkBpRepo"`
}

func (c *FuchsiaSDKAndroidRepoManagerConfig) Validate() error {
	if err := c.FuchsiaSDKRepoManagerConfig.Validate(); err != nil {
		return err
	}
	if c.GenSdkBpRepo == "" {
		return errors.New("GenSdkBpRepo is required.")
	}
	return nil
}

// fuchsiaSDKAndroidRepoManager is a RepoManager which rolls the Fuchsia SDK
// into Android. Unlike the fuchsiaSDKRepoManager, it actually unzips the
// contents of the SDK and checks them into the target repo. Additionally, it
// generates an Android.bp file.
type fuchsiaSDKAndroidRepoManager struct {
	*fuchsiaSDKRepoManager
	arm          *androidRepoManager
	genSdkBpRepo *git.Checkout
	parentRepo   *git.Checkout
}

// Return a fuchsiaSDKAndroidRepoManager instance.
func NewFuchsiaSDKAndroidRepoManager(ctx context.Context, c *FuchsiaSDKAndroidRepoManagerConfig, workdir string, g gerrit.GerritInterface, serverURL, gitcookiesPath string, authClient *http.Client, cr codereview.CodeReview, local bool) (RepoManager, error) {
	// We're not using the constructor for fuchsiaSDKRepoManager because we
	// need the NoCheckoutRepoManager to use the methods of this
	// implementation.
	if err := c.Validate(); err != nil {
		return nil, err
	}
	androidConfig := &AndroidRepoManagerConfig{
		CommonRepoManagerConfig: c.CommonRepoManagerConfig,
	}
	androidRM, err := NewAndroidRepoManager(ctx, androidConfig, workdir, g, serverURL, "<unused>", authClient, cr, local)
	if err != nil {
		return nil, err
	}
	arm := androidRM.(*androidRepoManager)
	arm.SetStrategy(strategy.StrategyRemoteHead(arm.childBranch, UPSTREAM_REMOTE_NAME, arm.childRepo))
	storageClient, err := storage.NewClient(ctx, option.WithHTTPClient(authClient))
	if err != nil {
		return nil, err
	}

	fsrm := &fuchsiaSDKRepoManager{
		gcsClient:         gcs.NewGCSClient(storageClient, FUCHSIA_SDK_GS_BUCKET),
		gsBucket:          FUCHSIA_SDK_GS_BUCKET,
		gsLatestPathLinux: "sdk/linux-amd64/LATEST_ARCHIVE",
		gsLatestPathMac:   "sdk/mac-amd64/LATEST_ARCHIVE",
		gsListPath:        "sdk",
		storageClient:     storageClient,
		versionFileLinux:  FUCHSIA_SDK_ANDROID_VERSION_FILE,
		versionFileMac:    "", // Ignored by this RepoManager.
	}
	parentRepo, err := git.NewCheckout(ctx, c.ParentRepo, workdir)
	if err != nil {
		return nil, err
	}
	genSdkBpRepo, err := git.NewCheckout(ctx, c.GenSdkBpRepo, workdir)
	if err != nil {
		return nil, err
	}
	rv := &fuchsiaSDKAndroidRepoManager{
		fuchsiaSDKRepoManager: fsrm,
		arm:                   arm,
		parentRepo:            parentRepo,
		genSdkBpRepo:          genSdkBpRepo,
	}
	ncrm, err := newNoCheckoutRepoManager(ctx, c.NoCheckoutRepoManagerConfig, workdir, g, serverURL, gitcookiesPath, authClient, cr, rv.buildCommitMessage, rv.updateHelper, local)
	if err != nil {
		return nil, err
	}
	rv.noCheckoutRepoManager = ncrm
	return rv, nil
}

// See documentation for noCheckoutRepoManagerUpdateHelperFunc.
func (rm *fuchsiaSDKAndroidRepoManager) updateHelper(ctx context.Context, strat strategy.NextRollStrategy, parentRepo *gitiles.Repo, baseCommit string) (string, string, int, map[string]string, error) {
	sklog.Info("Updating Android checkout...")
	if err := rm.arm.updateAndroidCheckout(ctx); err != nil {
		return "", "", 0, nil, err
	}

	sklog.Info("Finding next roll rev...")
	lastRollRev, nextRollRev, commitsNotRolled, _, err := rm.fuchsiaSDKRepoManager.updateHelper(ctx, strat, parentRepo, baseCommit)
	if err != nil {
		return "", "", 0, nil, err
	}

	if err := rm.parentRepo.Update(ctx); err != nil {
		return "", "", 0, nil, err
	}
	// Sync the parentRepo to baseCommit.
	if _, err := rm.parentRepo.Git(ctx, "reset", "--hard", baseCommit); err != nil {
		return "", "", 0, nil, err
	}
	if err := rm.genSdkBpRepo.Update(ctx); err != nil {
		return "", "", 0, nil, err
	}
	if _, err := rm.genSdkBpRepo.Git(ctx, "checkout", fmt.Sprintf("origin/%s", rm.parentBranch)); err != nil {
		return "", "", 0, nil, err
	}
	genSdkBpRepoHash, err := rm.genSdkBpRepo.RevParse(ctx, "HEAD")
	if err != nil {
		return "", "", 0, nil, err
	}
	sklog.Info("Reading old file contents...")
	oldContents, err := fileutil.ReadAllFilesRecursive(rm.parentRepo.Dir(), []string{".git"})
	if err != nil {
		return "", "", 0, nil, err
	}

	// Instead of simply rolling the version hash into a file, download and
	// unzip the SDK, and commit its contents.
	sdkDestPath := path.Join(rm.arm.workdir, SDK_DEST_PATH)
	if err := os.RemoveAll(sdkDestPath); err != nil {
		sklog.Warningf("Failed to remove SDK dest path %s: %s", sdkDestPath, err)
	}
	if err := os.MkdirAll(sdkDestPath, os.ModePerm); err != nil {
		return "", "", 0, nil, err
	}
	sdkGsPath := rm.gsListPath + "/linux-amd64/" + nextRollRev
	sklog.Infof("Downloading SDK from %s...", sdkGsPath)
	newContents := map[string][]byte{}
	r, err := rm.gcsClient.FileReader(ctx, sdkGsPath)
	if err != nil {
		return "", "", 0, nil, err
	}
	if err := tar.ReadGzipArchive(r, func(filename string, r io.Reader) error {
		b, err := ioutil.ReadAll(r)
		if err != nil {
			return err
		}
		newContents[filename] = b
		filePath := path.Join(sdkDestPath, filename)
		dir := path.Dir(filePath)
		if _, err := os.Stat(dir); os.IsNotExist(err) {
			if err := os.MkdirAll(dir, os.ModePerm); err != nil {
				return err
			}
		} else if err != nil {
			return err
		}
		return ioutil.WriteFile(filePath, b, os.ModePerm)
	}); err != nil {
		return "", "", 0, nil, fmt.Errorf("Failed to read archive: %s", err)
	}

	// Run the gen_sdk_bp.py script.
	genSdkBp := path.Join(rm.genSdkBpRepo.Dir(), GEN_SDK_BP_DIR, GEN_SDK_BP)
	sklog.Infof("Running %s at %s", genSdkBp, genSdkBpRepoHash)
	env := []string{fmt.Sprintf("ANDROID_BUILD_TOP=%s", rm.arm.workdir)}
	if _, err := exec.RunCommand(ctx, &exec.Command{
		Dir:        rm.genSdkBpRepo.Dir(),
		Name:       "python",
		Args:       []string{genSdkBp},
		Env:        env,
		InheritEnv: true,
		LogStdout:  true,
		LogStderr:  true,
	}); err != nil {
		return "", "", 0, nil, err
	}
	src := path.Join(rm.arm.workdir, rm.childPath, ANDROID_BP)
	b, err := ioutil.ReadFile(src)
	if err != nil {
		return "", "", 0, nil, err
	}
	newContents[ANDROID_BP] = b

	// Determine the contents of the next roll.
	nextRollContents := make(map[string]string, len(newContents))
	for f, contents := range newContents {
		if !bytes.Equal(oldContents[f], contents) {
			nextRollContents[f] = string(contents)
		}
	}
	for f, _ := range oldContents {
		if _, ok := newContents[f]; !ok {
			nextRollContents[f] = ""
		}
	}

	// Lastly, include the SDK version hash file.
	nextRollContents[rm.versionFileLinux] = nextRollRev
	sklog.Infof("Next roll modifies %d files.", len(nextRollContents))
	return lastRollRev, nextRollRev, commitsNotRolled, nextRollContents, nil
}

// See documentation for noCheckoutRepoManagerBuildCommitMessageFunc.
func (rm *fuchsiaSDKAndroidRepoManager) buildCommitMessage(from, to, serverURL, cqExtraTrybots string, emails []string) (string, error) {
	msg, err := rm.fuchsiaSDKRepoManager.buildCommitMessage(from, to, serverURL, cqExtraTrybots, emails)
	if err != nil {
		return "", err
	}
	msg += "\nExempt-From-Owner-Approval: The autoroll bot does not require owner approval."
	return msg, nil
}