scrap/go/scrap/scrap.go

// Package scrap defines the scrap types and functions on them.
package scrap

import (
	"bytes"
	"compress/gzip"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"path"
	"regexp"

	"cloud.google.com/go/storage"
	"go.skia.org/infra/go/gcs"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/go/util"
)

const maxScrapSize = 128 * 1024

var (
	ErrInvalidScrapType = errors.New("Invalid scrap type.")
	ErrInvalidScrapName = errors.New("Invalid scrap name.")
	ErrInvalidLanguage  = errors.New("Invalid language.")
	ErrInvalidHash      = errors.New("Invalid SHA256 hash.")
	ErrInvalidScrapSize = errors.New("Scrap is too large.")
)

// SHA256 is a SHA 256 hash encoded in hex.
type SHA256 string

// Type identifies the type of a scrap.
type Type string

const (
	// SVG scrap.
	SVG Type = "svg"

	// SKSL scrap.
	SKSL Type = "sksl"

	// Particle scrap.
	Particle Type = "particle"

	// UnknownType type of scrap.
	UnknownType Type = ""
)

// AllTypes is a slice of supported Types.
var AllTypes = []Type{SVG, SKSL, Particle}

// ToType converts a string to a Type, returning UnknownType if it is not a
// valid Type.
func ToType(s string) Type {
	for _, t := range AllTypes {
		if string(t) == s {
			return t
		}
	}
	return UnknownType
}

// validateType returns true if t is a valid Type.
func validateType(t Type) error {
	if ToType(string(t)) == UnknownType {
		return skerr.Wrapf(ErrInvalidScrapType, "got: %q", t)
	}
	return nil
}

// Lang is a programming language a scrap can be embedded in.
type Lang string

const (
	// CPP is the C++ language.
	CPP Lang = "cpp"

	// JS is the Javascript language.
	JS Lang = "js"

	// UnknownLang is an unknown language.
	UnknownLang Lang = ""
)

// AllLangs is the list of all supported Langs.
var AllLangs = []Lang{CPP, JS}

// ToLang converts a string to a Lang, returning UnknownLang if it not a valid
// Lang.
func ToLang(s string) Lang {
	for _, l := range AllLangs {
		if string(l) == s {
			return l
		}
	}
	return UnknownLang
}

// validateLang returns true if l is a valid Lang.
func validateLang(l Lang) error {
	if ToLang(string(l)) == UnknownLang {
		return skerr.Wrapf(ErrInvalidLanguage, "got: %q", l)
	}
	return nil
}

// MimeTypes for each scrap type when served raw.
var MimeTypes = map[Type]string{
	SVG:      "image/svg+xml",
	SKSL:     "text/plain",
	Particle: "application/json",
}

// SVGMetaData is metadata for SVG scraps.
type SVGMetaData struct {
}

// Uniform is a single uniform for an SkSL shader.
type Uniform struct {
	Name  string
	Value float64
}

// ChildShader is the scrap id of a single child shader along with the name that
// the uniform should have to access it.
type ChildShader struct {
	UniformName     string
	ScrapHashOrName string
}

// SKSLMetaData is metadata for SKSL scraps.
type SKSLMetaData struct {
	// Uniforms are all the inputs to the shader.
	Uniforms []float32

	// Child shaders. A slice because order is important when mapping uniform
	// names in code to child shaders passed to makeShaderWithChildren.
	Children []ChildShader
}

// ParticlesMetaData is metadata for Particle scraps.
type ParticlesMetaData struct {
}

// ScrapBody is the body of scrap stored in GCS and transported by the API.
type ScrapBody struct {
	Type Type
	Body string

	// Type specific metadata:
	SVGMetaData       *SVGMetaData       `json:",omitempty"`
	SKSLMetaData      *SKSLMetaData      `json:",omitempty"`
	ParticlesMetaData *ParticlesMetaData `json:",omitempty"`
}

// ScrapID contains the identity of a newly created scrap.
type ScrapID struct {
	Hash SHA256
}

// Name has information about a single named scrap.
type Name struct {
	Hash        SHA256
	Description string
}

// ScrapExchange handles reading and writing scraps.
type ScrapExchange interface {
	// Expand the given scrap into a full program in the given language and write
	// that code to the given io.Writer.
	Expand(ctx context.Context, t Type, hashOrName string, lang Lang, w io.Writer) error

	// LoadScrap loads a scrap. The 'name' can be either a hash, or if prefixed with
	// an "@" it is the name of scrap.
	LoadScrap(ctx context.Context, t Type, hashOrName string) (ScrapBody, error)

	// CreateScrap and return the hash by the ScrapID.
	CreateScrap(ctx context.Context, scrap ScrapBody) (ScrapID, error)

	// DeleteScrap and also delete the name if hashOrName is a name, which is indicated by
	// the prefix "@".
	DeleteScrap(ctx context.Context, t Type, hashOrName string) error

	// PutName creates or updates a name for a given scrap.
	PutName(ctx context.Context, t Type, name string, nameBody Name) error

	// GetName retrieves the hash for the given named scrap.
	GetName(ctx context.Context, t Type, name string) (Name, error)

	// DeleteName removes the name for the given named scrap.
	DeleteName(ctx context.Context, t Type, name string) error

	// ListNames lists all the known names for a given type of scrap.
	ListNames(ctx context.Context, t Type) ([]string, error)
}

// scrapExchange implements ScrapExchange.
type scrapExchange struct {
	client    gcs.GCSClient
	templates templateMap
}

// New returns a new instance of ScrapExchange.
func New(client gcs.GCSClient) (*scrapExchange, error) {
	tmpl, err := loadTemplates()
	if err != nil {
		return nil, skerr.Wrapf(err, "Failed to parse templates")
	}

	return &scrapExchange{
		client:    client,
		templates: tmpl,
	}, nil
}

var validName = regexp.MustCompile("^@[0-9a-zA-Z-_]+$")

// validateName returns true if s is a valid name for a scrap.
func validateName(s string) error {
	if !validName.MatchString(s) {
		return skerr.Wrapf(ErrInvalidScrapName, "got: %q", s)
	}
	return nil
}

var validSHA256Hash = regexp.MustCompile("^[0-9a-f]{64}$")

// isValidHash returns true if s is a valid SHA256 hash.
func isValidHash(s SHA256) bool {
	return validSHA256Hash.MatchString(string(s))
}

// validate returns an error if the given scrap is invalid.
//
// validate may also modify the scrap to make it valid, so call this before
// calculating the hash for a scrap.
func (s *scrapExchange) validate(scrap *ScrapBody) error {
	if err := validateType(scrap.Type); err != nil {
		return err
	}
	return nil
}

// Expand the given scrap into a full program in the given language and write
// that code to the given io.Writer.
func (s *scrapExchange) Expand(ctx context.Context, t Type, hashOrName string, lang Lang, w io.Writer) error {
	if err := validateLang(lang); err != nil {
		return err
	}
	scrapBody, err := s.LoadScrap(ctx, t, hashOrName)
	if err != nil {
		return skerr.Wrapf(err, "Failed to load scrap.")
	}
	// TODO(jcgregorio) Add helpers to the template to allow recursively loading child scraps.
	err = s.templates[lang][t].Execute(w, scrapBody)
	if err != nil {
		return skerr.Wrapf(err, "Failed to expand template.")
	}

	return nil
}

// LoadScrap loads a scrap. The 'name' can be either a hash, or if prefixed with
// an "@" it is the name of scrap.
func (s *scrapExchange) LoadScrap(ctx context.Context, t Type, hashOrName string) (ScrapBody, error) {
	var ret ScrapBody

	if err := validateType(t); err != nil {
		return ret, err
	}

	var hash SHA256
	if err := validateName(hashOrName); err == nil {
		name := hashOrName
		nameBody, err := s.GetName(ctx, t, name)
		if err != nil {
			return ret, skerr.Wrapf(err, "Failed to get hash of name to load.")
		}
		hash = nameBody.Hash
	} else {
		hash = SHA256(hashOrName)
	}
	if !isValidHash(hash) {
		return ret, skerr.Wrap(ErrInvalidHash)
	}

	rc, err := s.client.FileReader(ctx, fmt.Sprintf("scraps/%s/%s", t, hash))
	if err != nil {
		return ret, skerr.Wrapf(err, "Failed to open scrap.")
	}
	defer util.Close(rc)
	if err := json.NewDecoder(rc).Decode(&ret); err != nil {
		return ret, skerr.Wrapf(err, "Failed to decode scrap.")
	}
	return ret, nil
}

// CreateScrap and return the hash by the ScrapID.
func (s *scrapExchange) CreateScrap(ctx context.Context, scrap ScrapBody) (ScrapID, error) {
	var ret ScrapID
	if err := s.validate(&scrap); err != nil {
		return ret, skerr.Wrapf(err, "Invalid scrap.")
	}
	var b bytes.Buffer
	if err := json.NewEncoder(&b).Encode(scrap); err != nil {
		return ret, skerr.Wrapf(err, "Failed to JSON encode scrap.")
	}
	if b.Len() > maxScrapSize {
		return ret, ErrInvalidScrapSize
	}
	unencodedBody := b.Bytes()
	hashAsByteArray := sha256.Sum256(unencodedBody)
	hash := hex.EncodeToString(hashAsByteArray[:])
	ret.Hash = SHA256(hash)
	w := s.client.FileWriter(ctx, fmt.Sprintf("scraps/%s/%s", scrap.Type, hash), gcs.FileWriteOptions{
		ContentEncoding: "gzip",
		ContentType:     "application/json",
	})
	zw := gzip.NewWriter(w)
	_, err := zw.Write(unencodedBody)
	if err != nil {
		return ret, skerr.Wrapf(err, "Failed to write JSON body.")
	}
	if err := zw.Close(); err != nil {
		return ret, skerr.Wrapf(err, "Failed to close gzip writer.")
	}
	if err := w.Close(); err != nil {
		return ret, skerr.Wrapf(err, "Failed to close GCS Storage writer.")
	}
	return ret, nil
}

// DeleteScrap and also delete the name if hashOrName is a name, which is indicated by
// the prefix "@".
func (s *scrapExchange) DeleteScrap(ctx context.Context, t Type, hashOrName string) error {
	if err := validateType(t); err != nil {
		return err
	}

	var hash SHA256
	if err := validateName(hashOrName); err == nil {
		name := hashOrName
		nameBody, err := s.GetName(ctx, t, name)
		if err != nil {
			return skerr.Wrapf(err, "Failed to get hash of name to delete.")
		}
		err = s.DeleteName(ctx, t, name)
		if err != nil {
			return skerr.Wrapf(err, "Failed to delete name.")
		}
		hash = nameBody.Hash
	} else {
		hash = SHA256(hashOrName)
	}

	if !isValidHash(hash) {
		return skerr.Wrap(ErrInvalidHash)
	}

	err := s.client.DeleteFile(ctx, fmt.Sprintf("scraps/%s/%s", t, hash))
	if err != nil {
		return skerr.Wrapf(err, "Failed to delete hash.")
	}
	return nil
}

// PutName creates or updates a name for a given scrap.
func (s *scrapExchange) PutName(ctx context.Context, t Type, name string, nameBody Name) error {
	if err := validateType(t); err != nil {
		return err
	}
	if err := validateName(name); err != nil {
		return err
	}
	if !isValidHash(nameBody.Hash) {
		return skerr.Wrap(ErrInvalidHash)
	}
	w := s.client.FileWriter(ctx, fmt.Sprintf("names/%s/%s", t, name), gcs.FileWriteOptions{
		ContentType: "application/json",
	})
	if err := json.NewEncoder(w).Encode(nameBody); err != nil {
		return skerr.Wrapf(err, "Failed to encode JSON.")
	}
	if err := w.Close(); err != nil {
		return skerr.Wrapf(err, "Failed to close GCS Storage writer.")
	}

	return nil
}

// GetName retrieves the hash for the given named scrap.
func (s *scrapExchange) GetName(ctx context.Context, t Type, name string) (Name, error) {
	var ret Name

	if err := validateType(t); err != nil {
		return ret, err
	}
	if err := validateName(name); err != nil {
		return ret, err
	}

	rc, err := s.client.FileReader(ctx, fmt.Sprintf("names/%s/%s", t, name))
	if err != nil {
		return ret, skerr.Wrapf(err, "Failed to open name.")
	}
	defer util.Close(rc)
	if err := json.NewDecoder(rc).Decode(&ret); err != nil {
		return ret, skerr.Wrapf(err, "Failed to decode body.")
	}
	return ret, nil
}

// DeleteName removes the name for the given named scrap.
func (s *scrapExchange) DeleteName(ctx context.Context, t Type, name string) error {
	if err := validateType(t); err != nil {
		return err
	}
	if err := validateName(name); err != nil {
		return err
	}

	err := s.client.DeleteFile(ctx, fmt.Sprintf("names/%s/%s", t, name))
	if err != nil {
		return skerr.Wrapf(err, "Failed to delete name.")
	}
	return nil
}

// ListNames lists all the known names for a given type of scrap.
func (s *scrapExchange) ListNames(ctx context.Context, t Type) ([]string, error) {
	ret := []string{}

	if err := validateType(t); err != nil {
		return nil, err
	}

	err := s.client.AllFilesInDirectory(ctx, fmt.Sprintf("names/%s/", t), func(item *storage.ObjectAttrs) {
		// GCS always uses forward slashes.
		name := path.Base(item.Name)
		ret = append(ret, name)
	})
	if err != nil {
		return nil, skerr.Wrapf(err, "Failed to read directory.")
	}

	return ret, nil
}

// Confirm that scrapExchange implements the ScrapExchange interface.
var _ ScrapExchange = (*scrapExchange)(nil)