docserverk/go/docset/docset.go - buildbot - Git at Google

 // docset keeps track of checkouts of a repository of Markdown documents.
 package docset

 /*

   DocSets work around a `workDir` which is presumed to be a directory with enough space
   to handle many checkouts of the Markdown repository. It has the following structure:

   {workDir}
     /primary/ - The primary checkout of the Markdown repository.
     /patches/
        {issue_id}-{patchset_id}/ - A patched checkout of the Markdown respository.

   Each repo should have a directory /site that contains all the documentation in Markdown
   and associated files such as PNG images, For example:

   site
   ├── dev
   │   ├── contrib
   │   │   ├── codereviews.md
   │   │   └── index.md
   │   └── index.md
   ├── index.md
   ├── logo.png
   ├── roles.md
   ├── roles.png
   ├── user
   │   ├── api.md
   │   ├── download.md
   │   ├── index.md
   │   ├── issue-tracker.md
   │   └── quick
   │       ├── index.md
   │       └── linux.md
   └── xtra
       └── index.md

 */

 import (
 	"bufio"
 	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/mail"
 	"os"
 	"path"
 	"path/filepath"
 	"regexp"
 	"sort"
 	"strconv"
 	"strings"
 	"sync"
 	"text/template"
 	"time"

 	"github.com/golang/groupcache/lru"
 	"go.skia.org/infra/docserverk/go/config"
 	"go.skia.org/infra/go/auth"
 	"go.skia.org/infra/go/gerrit"
 	"go.skia.org/infra/go/git"
 	"go.skia.org/infra/go/git/gitinfo"
 	"go.skia.org/infra/go/httputils"
 	"go.skia.org/infra/go/sklog"
 	"go.skia.org/infra/go/util"
 )

 const (
 	MARKDOWN_CACHE_SIZE = 100
 )

 var (
 	gc *gerrit.Gerrit

 	IssueCommittedErr = errors.New("The requested issue is merged.")

 	// markdownHeader matches the hashes that appear at the beginning of a
 	// header.
 	markdownHeader = regexp.MustCompile(`^#+\ `)
 )

 func Init(local bool) error {
 	ts, err := auth.NewDefaultTokenSource(local, auth.SCOPE_GERRIT)
 	if err != nil {
 		return err
 	}
 	client := httputils.DefaultClientConfig().WithTokenSource(ts).With2xxOnly().Client()
 	gc, err = gerrit.NewGerrit(gerrit.GERRIT_SKIA_URL, client)
 	return err
 }

 // DocSet is a single checked out repository of Markdown documents.
 type DocSet struct {
 	// repoDir is the directory the repo is checked out into.
 	repoDir string
 	// navigation is the HTML formatted navigation structure for the given repo.
 	navigation string

 	// A site map served to the Google crawler.
 	siteMap string

 	cache *lru.Cache

 	mutex sync.Mutex
 }

 // newDocSet does the core of the work for both NewDocSet and NewDocSetForIssue.
 //
 // The repo is checked out somewhere under workDir.
 // If a valid issue and patchset are supplied then the repo will be patched with that CL.
 // If refresh is true then the git repo will be periodically refreshed (git pull).
 func newDocSet(ctx context.Context, workDir, repo string, issue, patchset int64, refresh bool) (*DocSet, error) {
 	primaryDir := filepath.Join(workDir, "primary")
 	issueDir := filepath.Join(workDir, "patches", fmt.Sprintf("%d-%d", issue, patchset))
 	repoDir := primaryDir
 	if issue > 0 {
 		repoDir = issueDir
 		if _, err := os.Stat(issueDir); err == nil {
 			d := &DocSet{
 				repoDir: repoDir,
 			}
 			d.BuildNavigation()
 			return d, nil
 		}
 	}

 	if issue > 0 {
 		info, err := gc.GetIssueProperties(ctx, issue)
 		if err != nil {
 			return nil, fmt.Errorf("Failed to load issue info: %s", err)
 		}
 		if info.Committed {
 			return nil, IssueCommittedErr
 		}
 	}
 	var gi *gitinfo.GitInfo
 	var err error
 	if issue > 0 {
 		gi, err = gitinfo.CloneOrUpdate(ctx, primaryDir, repoDir, false)
 	} else {
 		gi, err = gitinfo.CloneOrUpdate(ctx, repo, repoDir, false)
 	}
 	if err != nil {
 		return nil, fmt.Errorf("Failed to CloneOrUpdate repo %q: %s", repo, err)
 	}

 	if issue > 0 {
 		// Run a git fetch for the branch where gerrit stores patches.
 		//
 		//  refs/changes/46/4546/1
 		//                |  |   |
 		//                |  |   +-> Patch set.
 		//                |  |
 		//                |  +-> Issue ID.
 		//                |
 		//                +-> Last two digits of Issue ID.

 		issuePostfix := issue % 100
 		output, err := git.GitDir(repoDir).Git(ctx, "fetch", repo, fmt.Sprintf("refs/changes/%02d/%d/%d", issuePostfix, issue, patchset))
 		if err != nil {
 			return nil, fmt.Errorf("Failed to execute Git %q: %s", output, err)
 		}
 		err = gi.Checkout(ctx, "FETCH_HEAD")
 		if err != nil {
 			return nil, fmt.Errorf("Failed to CloneOrUpdate repo %q: %s", repo, err)
 		}
 	}
 	d := &DocSet{
 		repoDir: repoDir,
 	}
 	d.BuildNavigation()
 	if refresh {
 		go func() {
 			for range time.Tick(config.REFRESH) {
 				util.LogErr(gi.Update(ctx, true, false))
 				d.BuildNavigation()
 			}
 		}()
 	}
 	return d, nil
 }

 // NewPreviewDocSet creates a new DocSet, one that is not refreshed.
 func NewPreviewDocSet() (*DocSet, error) {
 	// Start from cwd and move up until you find a .git file, then use that dir as repoDir.
 	dir, err := os.Getwd()
 	if err != nil {
 		return nil, fmt.Errorf("Can't find cwd: %s", err)
 	}
 	for {
 		if _, err := os.Stat(filepath.Join(dir, ".git")); os.IsNotExist(err) {
 			dir = path.Dir(dir)
 		} else {
 			break
 		}
 		if dir == "/" || dir == "." {
 			return nil, fmt.Errorf("docserver --preview must be run from within the Git repo.")
 		}
 	}
 	d := &DocSet{
 		repoDir: dir,
 	}
 	d.BuildNavigation()
 	d.cache = nil
 	return d, nil
 }

 // NewDocSet creates a new DocSet, one that is periodically refreshed.
 func NewDocSet(ctx context.Context, workDir, repo string) (*DocSet, error) {
 	return newDocSet(ctx, workDir, repo, -1, -1, true)
 }

 // NewDocSetForIssue creates a new DocSet patched to the latest patch level of
 // the given issue.
 //
 // The returned DocSet is not periodically refreshed.
 func NewDocSetForIssue(ctx context.Context, workDir, repo string, issue int64) (*DocSet, error) {
 	info, err := gc.GetIssueProperties(ctx, issue)
 	if err != nil {
 		return nil, fmt.Errorf("Failed to load issue info: %s", err)
 	}
 	patchset := int64(len(info.Revisions))
 	if patchset == 0 {
 		return nil, fmt.Errorf("Failed to find a patchset for issue %d.", issue)
 	}
 	addr, err := mail.ParseAddress(info.Owner.Email)
 	if err != nil {
 		return nil, fmt.Errorf("CL contains invalid author email: %s", err)
 	}
 	domain := strings.Split(addr.Address, "@")[1]
 	if !util.In(domain, config.WHITELIST) {
 		return nil, fmt.Errorf("User is not authorized to test docset CLs.")
 	}
 	return newDocSet(ctx, workDir, repo, issue, patchset, false)
 }

 // RawFilename returns the absolute filename for the file associated with the
 // given url.
 //
 // The bool returned will be true if the url identifies a raw resource, such as
 // a PNG, as opposed to a Markdown file that should be processed.
 func (d *DocSet) RawFilename(url string) (string, bool, error) {
 	startFilename := filepath.Join(d.repoDir, config.REPO_SUBDIR, url)
 	endFilename, err := findFile(startFilename)
 	return endFilename, (startFilename == endFilename), err
 }

 // Body returns the contents of the given filename.
 func (d *DocSet) Body(filename string) ([]byte, error) {
 	d.mutex.Lock()
 	defer d.mutex.Unlock()
 	var err error = nil
 	var body interface{} = nil
 	ok := false
 	if d.cache != nil {
 		body, ok = d.cache.Get(filename)
 	}
 	if !ok {
 		body, err = ioutil.ReadFile(filename)
 		if err == nil {
 			if d.cache != nil {
 				d.cache.Add(filename, body)
 			}
 		}
 	}
 	return body.([]byte), err
 }

 // hasPrefix returns true if p is a prefix of a.
 func hasPrefix(a, p []string) bool {
 	if len(p) > len(a) {
 		return false
 	}
 	for i, s := range p {
 		if s != a[i] {
 			return false
 		}
 	}
 	return true
 }

 // diff determines how many levels of ul's we need to push and pop.
 func diff(current, next []string) (int, int) {
 	// Start by popping off values from the end of 'next' until we get a prefix
 	// of 'current', which may be the empty list. Use that to calculate how many
 	// </ul>'s we need to emit.
 	end := 1
 	for i := 0; i <= len(next); i++ {
 		if hasPrefix(current, next[:len(next)-i]) {
 			end = len(current) - (len(next) - i)
 			break
 		}
 	}
 	// If we are just adding a file in a new directory then don't end the list.
 	if len(current) > 0 && len(next) > 0 && end == 1 && current[len(current)-1] == "" {
 		end = 0
 	}
 	// We are always going to begin a new list.
 	begin := 1
 	// Unless we are adding in a file in the same directory, in which case do nothing.
 	if len(current) > 0 && len(next) > 0 && end == 1 && current[len(current)-1] != "" && next[len(next)-1] != "" {
 		end = 0
 		begin = 0
 	}

 	return end, begin
 }

 // siteMapTemplate is a self-refrential template used to recursively expand over node tree.
 var siteMapTemplate = template.Must(template.New("SITENODE").Parse(`https://skia.org{{.Index.URL}}
 {{range .Files}}https://skia.org{{.URL}}
 {{end}}{{range .Dirs}}{{template "SITENODE" .}}{{end}}`))

 // nodeToSite converts the node to a sitemap.
 func nodeToSite(n *node, depth int) string {
 	b := &bytes.Buffer{}
 	if err := siteMapTemplate.Execute(b, n); err != nil {
 		sklog.Errorf("Failed to expand: %s", err)
 		return ""
 	}
 	return b.String()
 }

 // navTemplate is a self-refrential template used to recursively expand over node tree.
 var navTemplate = template.Must(template.New("NODE").Parse(`
 <li><a data-path="{{.Index.URL}}" href="{{.Index.URL}}">{{.Index.Name}}</a></li>
 <ul class=files>
   {{range .Files}}
     <li><a data-path="{{.URL}}" href="{{.URL}}">{{.Name}}</a></li>
   {{end}}
 </ul>
 <ul class="dirs depth{{.Index.Depth}}">
   {{range .Dirs}}
     {{template "NODE" .}}
   {{end}}
 </ul>`))

 // buildSiteMapconverts a slice of navEntry's into an HTML formatted
 // site map.
 func buildSiteMap(n *node) string {
 	return nodeToSite(n, 1)
 }

 // nodeToHTML converts the node to HTML, keeping track of depth for pretty printing the output.
 func nodeToHTML(n *node, depth int) string {
 	b := &bytes.Buffer{}
 	if err := navTemplate.Execute(b, n); err != nil {
 		sklog.Errorf("Failed to expand: %s", err)
 		return ""
 	}
 	return b.String()
 }

 // metadata is the struct for deserializing JSON found in METADATA files.
 type metadata struct {
 	DirOrder  []string `json:"dirOrder"`
 	FileOrder []string `json:"fileOrder"`
 }

 // buildNavString converts a slice of navEntry's into an HTML formatted
 // navigation structure.
 func buildNavString(n *node) string {
 	return "\n<ul class=depth0>\n" + nodeToHTML(n, 1) + "</ul>\n"
 }

 // node is a single directory of site docs.
 type node struct {
 	Index navEntry
 	Dirs  []*node
 	Files []*navEntry
 }

 // nodeSlice is for sorting nodes.
 type nodeSlice []*node

 func (p nodeSlice) Len() int           { return len(p) }
 func (p nodeSlice) Less(i, j int) bool { return p[i].Index.URL < p[j].Index.URL }
 func (p nodeSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

 // walk the directory tree below root and populate a tree stucture of nodes.
 func walk(root, path string) (*node, error) {
 	ret := &node{
 		Dirs:  []*node{},
 		Files: []*navEntry{},
 	}

 	// for each directory fill in the navEntry for index.md
 	rel, _ := filepath.Rel(root, path)
 	rel = filepath.Clean("/" + rel)
 	ret.Index = navEntry{
 		URL:  rel,
 		Name: readTitle(filepath.Join(path, "index.md"), rel),
 	}
 	m := &metadata{
 		DirOrder:  []string{},
 		FileOrder: []string{},
 	}

 	// populate all the other files
 	f, err := os.Open(path)
 	if err != nil {
 		return nil, err
 	}
 	allFiles, err := f.Readdir(-1)
 	if err != nil {
 		return nil, err
 	}
 	for _, fi := range allFiles {
 		// The contents are either files or directories.
 		if fi.IsDir() {
 			n, err := walk(root, filepath.Join(path, fi.Name()))
 			if err != nil {
 				return nil, err
 			}
 			ret.Dirs = append(ret.Dirs, n)
 		} else if fi.Name() != "index.md" && strings.HasSuffix(fi.Name(), ".md") {
 			fileRel := filepath.Clean(rel + "/" + fi.Name()[:len(fi.Name())-3])
 			ret.Files = append(ret.Files, &navEntry{
 				URL:  fileRel,
 				Name: readTitle(filepath.Join(path, fi.Name()), fileRel),
 			})
 		} else if fi.Name() == "METADATA" {
 			// Load JSON found in METADATA.
 			metaPath := filepath.Join(path, fi.Name())
 			f, err := os.Open(metaPath)
 			if err != nil {
 				sklog.Warningf("Failed to open %q: %s", metaPath, err)
 				continue
 			}
 			dec := json.NewDecoder(f)
 			if err := dec.Decode(m); err != nil {
 				sklog.Warningf("Failed to decode %q: %s", metaPath, err)
 			}
 		}
 	}

 	// Sort dirs and files, use METADATA if available.
 	// Pick out the matches in the order they appear, then sort the rest.
 	// Yes, this is O(n^2), but for very small n.
 	sortedDirs := []*node{}
 	for _, name := range m.DirOrder {
 		for i, n := range ret.Dirs {
 			if name == filepath.Base(n.Index.URL) {
 				sortedDirs = append(sortedDirs, n)
 				ret.Dirs = append(ret.Dirs[:i], ret.Dirs[i+1:]...)
 				break
 			}
 		}
 	}
 	sort.Sort(nodeSlice(ret.Dirs))
 	ret.Dirs = append(sortedDirs, ret.Dirs...)

 	sortedFiles := []*navEntry{}
 	for _, name := range m.FileOrder {
 		for i, n := range ret.Files {
 			if name == filepath.Base(n.URL) {
 				sortedFiles = append(sortedFiles, n)
 				ret.Files = append(ret.Files[:i], ret.Files[i+1:]...)
 				break
 			}
 		}
 	}
 	sort.Sort(navEntrySlice(ret.Files))
 	ret.Files = append(sortedFiles, ret.Files...)

 	return ret, nil
 }

 func printnode(n *node, depth int) {
 	sklog.Infof("Node: %*s%#v\n", depth*2, "", n.Index)
 	for _, f := range n.Files {
 		sklog.Infof("File: %*s%#v\n", (depth+1)*2, "", *f)
 	}
 	for _, d := range n.Dirs {
 		printnode(d, depth+1)
 	}
 }

 func addDepth(n *node, depth int) {
 	n.Index.Depth = depth
 	for _, d := range n.Dirs {
 		addDepth(d, depth+1)
 	}
 }

 // BuildNavigation builds the Navigation for the DocSet.
 func (d *DocSet) BuildNavigation() {
 	// Walk the directory tree to build the navigation menu.
 	root := filepath.Join(d.repoDir, config.REPO_SUBDIR)
 	node, _ := walk(root, root)
 	addDepth(node, 1)
 	s := buildNavString(node)
 	sm := buildSiteMap(node)
 	d.mutex.Lock()
 	defer d.mutex.Unlock()
 	d.cache = lru.New(MARKDOWN_CACHE_SIZE)
 	d.navigation = s
 	d.siteMap = sm
 }

 // Navigation returns the HTML formatted navigation.
 func (d *DocSet) Navigation() string {
 	d.mutex.Lock()
 	defer d.mutex.Unlock()
 	return d.navigation
 }

 // SiteMap returns the txt formatted site map.
 func (d *DocSet) SiteMap() string {
 	d.mutex.Lock()
 	defer d.mutex.Unlock()
 	return d.siteMap
 }

 // issueAndPatch is a regex for extracting the issue number from a directory name
 // that is formatted like {issue_id}-{pathset_id}.
 var issueAndPatch = regexp.MustCompile("([0-9]+)-[0-9]+")

 // StartCleaner is a process that periodically checks the status of every issue
 // that has been previewed and removes all the local files for closed issues.
 func StartCleaner(workDir string) {
 	sklog.Info("Starting Cleaner")
 	for range time.Tick(config.REFRESH) {
 		matches, err := filepath.Glob(workDir + "/patches/*")
 		sklog.Infof("Matches: %v", matches)
 		if err != nil {
 			sklog.Errorf("Failed to retrieve list of patched checkouts: %s", err)
 			continue
 		}
 		for _, filename := range matches {
 			_, file := filepath.Split(filename)
 			sklog.Info(file)
 			m := issueAndPatch.FindStringSubmatch(file)
 			if len(m) < 2 {
 				continue
 			}
 			issue, err := strconv.ParseInt(m[1], 10, 64)
 			if err != nil {
 				sklog.Errorf("Failed to parse %q as int: %s", m[1], err)
 				continue
 			}
 			info, err := gc.GetIssueProperties(context.TODO(), issue)
 			// Delete closed and missing issues.
 			if err != nil || info.Committed {
 				if err := os.RemoveAll(filename); err != nil {
 					sklog.Errorf("Failed to remove %q: %s", filename, err)
 				}
 			}
 		}
 	}
 }

 // findFile takes a filename guess and turns into real name of a file.
 //
 // Look for the given file, if it exists then serve it raw with a guess at the
 // content type. Otherwise append ".md" and return it as processed markdown.
 // If it is a directory append "index.md" and return that.
 func findFile(filename string) (string, error) {
 	if stat, err := os.Stat(filename); err == nil {
 		if stat.IsDir() {
 			return findFile(filepath.Join(filename, "./index.md"))
 		} else {
 			return filename, nil
 		}
 	} else {
 		if filepath.Ext(filename) == ".md" {
 			return filename, err
 		} else {
 			return findFile(filename + ".md")
 		}
 	}
 }

 // pathOf returns a '/' terminated path for the given filename.
 func pathOf(s string) string {
 	if s[len(s)-1] == '/' {
 		return s
 	}
 	parts := strings.Split(s, "/")
 	if len(parts) > 0 {
 		parts = parts[:len(parts)-1]
 	}

 	ret := strings.Join(parts, "/")
 	if len(ret) == 0 || ret[len(ret)-1] != '/' {
 		ret += "/"
 	}
 	return ret
 }

 // navEntry is a single directory entry in the Markdown repo.
 type navEntry struct {
 	Depth int
 	URL   string
 	Name  string
 }

 // navEntrySlice is a utility type for sorting navEntry's.
 type navEntrySlice []*navEntry

 func (p navEntrySlice) Len() int { return len(p) }
 func (p navEntrySlice) Less(i, j int) bool {
 	if pathOf(p[i].URL) < pathOf(p[j].URL) {
 		return true
 	}
 	if pathOf(p[i].URL) == pathOf(p[j].URL) {
 		return p[i].URL < p[j].URL
 	}
 	return false
 }
 func (p navEntrySlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

 // readTitle reads the first line from a Markdown file.
 func readTitle(filename, def string) string {
 	f, err := os.Open(filename)
 	if err != nil {
 		sklog.Warningf("Failed to open file %s: %s", filename, err)
 		return def
 	}
 	defer util.Close(f)
 	reader := bufio.NewReader(f)
 	title, err := reader.ReadString('\n')
 	if err != nil {
 		sklog.Warningf("Failed to read title %s: %s", filename, err)
 	}
 	if strings.HasPrefix(title, "#") {
 		title = markdownHeader.ReplaceAllString(title, "")
 	}
 	title = strings.TrimSpace(title)
 	return title
 }
	// docset keeps track of checkouts of a repository of Markdown documents.
	package docset

	/*

	DocSets work around a `workDir` which is presumed to be a directory with enough space
	to handle many checkouts of the Markdown repository. It has the following structure:

	{workDir}
	/primary/ - The primary checkout of the Markdown repository.
	/patches/
	{issue_id}-{patchset_id}/ - A patched checkout of the Markdown respository.

	Each repo should have a directory /site that contains all the documentation in Markdown
	and associated files such as PNG images, For example:

	site
	├── dev
	│ ├── contrib
	│ │ ├── codereviews.md
	│ │ └── index.md
	│ └── index.md
	├── index.md
	├── logo.png
	├── roles.md
	├── roles.png
	├── user
	│ ├── api.md
	│ ├── download.md
	│ ├── index.md
	│ ├── issue-tracker.md
	│ └── quick
	│ ├── index.md
	│ └── linux.md
	└── xtra
	└── index.md

	*/

	import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/mail"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"text/template"
	"time"

	"github.com/golang/groupcache/lru"
	"go.skia.org/infra/docserverk/go/config"
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/gerrit"
	"go.skia.org/infra/go/git"
	"go.skia.org/infra/go/git/gitinfo"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	)

	const (
	MARKDOWN_CACHE_SIZE = 100
	)

	var (
	gc *gerrit.Gerrit

	IssueCommittedErr = errors.New("The requested issue is merged.")

	// markdownHeader matches the hashes that appear at the beginning of a
	// header.
	markdownHeader = regexp.MustCompile(`^#+\ `)
	)

	func Init(local bool) error {
	ts, err := auth.NewDefaultTokenSource(local, auth.SCOPE_GERRIT)
	if err != nil {
	return err
	}
	client := httputils.DefaultClientConfig().WithTokenSource(ts).With2xxOnly().Client()
	gc, err = gerrit.NewGerrit(gerrit.GERRIT_SKIA_URL, client)
	return err
	}

	// DocSet is a single checked out repository of Markdown documents.
	type DocSet struct {
	// repoDir is the directory the repo is checked out into.
	repoDir string
	// navigation is the HTML formatted navigation structure for the given repo.
	navigation string

	// A site map served to the Google crawler.
	siteMap string

	cache *lru.Cache

	mutex sync.Mutex
	}

	// newDocSet does the core of the work for both NewDocSet and NewDocSetForIssue.
	//
	// The repo is checked out somewhere under workDir.
	// If a valid issue and patchset are supplied then the repo will be patched with that CL.
	// If refresh is true then the git repo will be periodically refreshed (git pull).
	func newDocSet(ctx context.Context, workDir, repo string, issue, patchset int64, refresh bool) (*DocSet, error) {
	primaryDir := filepath.Join(workDir, "primary")
	issueDir := filepath.Join(workDir, "patches", fmt.Sprintf("%d-%d", issue, patchset))
	repoDir := primaryDir
	if issue > 0 {
	repoDir = issueDir
	if _, err := os.Stat(issueDir); err == nil {
	d := &DocSet{
	repoDir: repoDir,
	}
	d.BuildNavigation()
	return d, nil
	}
	}

	if issue > 0 {
	info, err := gc.GetIssueProperties(ctx, issue)
	if err != nil {
	return nil, fmt.Errorf("Failed to load issue info: %s", err)
	}
	if info.Committed {
	return nil, IssueCommittedErr
	}
	}
	var gi *gitinfo.GitInfo
	var err error
	if issue > 0 {
	gi, err = gitinfo.CloneOrUpdate(ctx, primaryDir, repoDir, false)
	} else {
	gi, err = gitinfo.CloneOrUpdate(ctx, repo, repoDir, false)
	}
	if err != nil {
	return nil, fmt.Errorf("Failed to CloneOrUpdate repo %q: %s", repo, err)
	}

	if issue > 0 {
	// Run a git fetch for the branch where gerrit stores patches.
	//
	// refs/changes/46/4546/1
	// \| \| \|
	// \| \| +-> Patch set.
	// \| \|
	// \| +-> Issue ID.
	// \|
	// +-> Last two digits of Issue ID.

	issuePostfix := issue % 100
	output, err := git.GitDir(repoDir).Git(ctx, "fetch", repo, fmt.Sprintf("refs/changes/%02d/%d/%d", issuePostfix, issue, patchset))
	if err != nil {
	return nil, fmt.Errorf("Failed to execute Git %q: %s", output, err)
	}
	err = gi.Checkout(ctx, "FETCH_HEAD")
	if err != nil {
	return nil, fmt.Errorf("Failed to CloneOrUpdate repo %q: %s", repo, err)
	}
	}
	d := &DocSet{
	repoDir: repoDir,
	}
	d.BuildNavigation()
	if refresh {
	go func() {
	for range time.Tick(config.REFRESH) {
	util.LogErr(gi.Update(ctx, true, false))
	d.BuildNavigation()
	}
	}()
	}
	return d, nil
	}

	// NewPreviewDocSet creates a new DocSet, one that is not refreshed.
	func NewPreviewDocSet() (*DocSet, error) {
	// Start from cwd and move up until you find a .git file, then use that dir as repoDir.
	dir, err := os.Getwd()
	if err != nil {
	return nil, fmt.Errorf("Can't find cwd: %s", err)
	}
	for {
	if _, err := os.Stat(filepath.Join(dir, ".git")); os.IsNotExist(err) {
	dir = path.Dir(dir)
	} else {
	break
	}
	if dir == "/" \|\| dir == "." {
	return nil, fmt.Errorf("docserver --preview must be run from within the Git repo.")
	}
	}
	d := &DocSet{
	repoDir: dir,
	}
	d.BuildNavigation()
	d.cache = nil
	return d, nil
	}

	// NewDocSet creates a new DocSet, one that is periodically refreshed.
	func NewDocSet(ctx context.Context, workDir, repo string) (*DocSet, error) {
	return newDocSet(ctx, workDir, repo, -1, -1, true)
	}

	// NewDocSetForIssue creates a new DocSet patched to the latest patch level of
	// the given issue.
	//
	// The returned DocSet is not periodically refreshed.
	func NewDocSetForIssue(ctx context.Context, workDir, repo string, issue int64) (*DocSet, error) {
	info, err := gc.GetIssueProperties(ctx, issue)
	if err != nil {
	return nil, fmt.Errorf("Failed to load issue info: %s", err)
	}
	patchset := int64(len(info.Revisions))
	if patchset == 0 {
	return nil, fmt.Errorf("Failed to find a patchset for issue %d.", issue)
	}
	addr, err := mail.ParseAddress(info.Owner.Email)
	if err != nil {
	return nil, fmt.Errorf("CL contains invalid author email: %s", err)
	}
	domain := strings.Split(addr.Address, "@")[1]
	if !util.In(domain, config.WHITELIST) {
	return nil, fmt.Errorf("User is not authorized to test docset CLs.")
	}
	return newDocSet(ctx, workDir, repo, issue, patchset, false)
	}

	// RawFilename returns the absolute filename for the file associated with the
	// given url.
	//
	// The bool returned will be true if the url identifies a raw resource, such as
	// a PNG, as opposed to a Markdown file that should be processed.
	func (d *DocSet) RawFilename(url string) (string, bool, error) {
	startFilename := filepath.Join(d.repoDir, config.REPO_SUBDIR, url)
	endFilename, err := findFile(startFilename)
	return endFilename, (startFilename == endFilename), err
	}

	// Body returns the contents of the given filename.
	func (d *DocSet) Body(filename string) ([]byte, error) {
	d.mutex.Lock()
	defer d.mutex.Unlock()
	var err error = nil
	var body interface{} = nil
	ok := false
	if d.cache != nil {
	body, ok = d.cache.Get(filename)
	}
	if !ok {
	body, err = ioutil.ReadFile(filename)
	if err == nil {
	if d.cache != nil {
	d.cache.Add(filename, body)
	}
	}
	}
	return body.([]byte), err
	}

	// hasPrefix returns true if p is a prefix of a.
	func hasPrefix(a, p []string) bool {
	if len(p) > len(a) {
	return false
	}
	for i, s := range p {
	if s != a[i] {
	return false
	}
	}
	return true
	}

	// diff determines how many levels of ul's we need to push and pop.
	func diff(current, next []string) (int, int) {
	// Start by popping off values from the end of 'next' until we get a prefix
	// of 'current', which may be the empty list. Use that to calculate how many
	// </ul>'s we need to emit.
	end := 1
	for i := 0; i <= len(next); i++ {
	if hasPrefix(current, next[:len(next)-i]) {
	end = len(current) - (len(next) - i)
	break
	}
	}
	// If we are just adding a file in a new directory then don't end the list.
	if len(current) > 0 && len(next) > 0 && end == 1 && current[len(current)-1] == "" {
	end = 0
	}
	// We are always going to begin a new list.
	begin := 1
	// Unless we are adding in a file in the same directory, in which case do nothing.
	if len(current) > 0 && len(next) > 0 && end == 1 && current[len(current)-1] != "" && next[len(next)-1] != "" {
	end = 0
	begin = 0
	}

	return end, begin
	}

	// siteMapTemplate is a self-refrential template used to recursively expand over node tree.
	var siteMapTemplate = template.Must(template.New("SITENODE").Parse(`https://skia.org{{.Index.URL}}
	{{range .Files}}https://skia.org{{.URL}}
	{{end}}{{range .Dirs}}{{template "SITENODE" .}}{{end}}`))

	// nodeToSite converts the node to a sitemap.
	func nodeToSite(n *node, depth int) string {
	b := &bytes.Buffer{}
	if err := siteMapTemplate.Execute(b, n); err != nil {
	sklog.Errorf("Failed to expand: %s", err)
	return ""
	}
	return b.String()
	}

	// navTemplate is a self-refrential template used to recursively expand over node tree.
	var navTemplate = template.Must(template.New("NODE").Parse(`
	<li><a data-path="{{.Index.URL}}" href="{{.Index.URL}}">{{.Index.Name}}</a></li>
	<ul class=files>
	{{range .Files}}
	<li><a data-path="{{.URL}}" href="{{.URL}}">{{.Name}}</a></li>
	{{end}}
	</ul>
	<ul class="dirs depth{{.Index.Depth}}">
	{{range .Dirs}}
	{{template "NODE" .}}
	{{end}}
	</ul>`))

	// buildSiteMapconverts a slice of navEntry's into an HTML formatted
	// site map.
	func buildSiteMap(n *node) string {
	return nodeToSite(n, 1)
	}

	// nodeToHTML converts the node to HTML, keeping track of depth for pretty printing the output.
	func nodeToHTML(n *node, depth int) string {
	b := &bytes.Buffer{}
	if err := navTemplate.Execute(b, n); err != nil {
	sklog.Errorf("Failed to expand: %s", err)
	return ""
	}
	return b.String()
	}

	// metadata is the struct for deserializing JSON found in METADATA files.
	type metadata struct {
	DirOrder []string `json:"dirOrder"`
	FileOrder []string `json:"fileOrder"`
	}

	// buildNavString converts a slice of navEntry's into an HTML formatted
	// navigation structure.
	func buildNavString(n *node) string {
	return "\n<ul class=depth0>\n" + nodeToHTML(n, 1) + "</ul>\n"
	}

	// node is a single directory of site docs.
	type node struct {
	Index navEntry
	Dirs []*node
	Files []*navEntry
	}

	// nodeSlice is for sorting nodes.
	type nodeSlice []*node

	func (p nodeSlice) Len() int { return len(p) }
	func (p nodeSlice) Less(i, j int) bool { return p[i].Index.URL < p[j].Index.URL }
	func (p nodeSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

	// walk the directory tree below root and populate a tree stucture of nodes.
	func walk(root, path string) (*node, error) {
	ret := &node{
	Dirs: []*node{},
	Files: []*navEntry{},
	}

	// for each directory fill in the navEntry for index.md
	rel, _ := filepath.Rel(root, path)
	rel = filepath.Clean("/" + rel)
	ret.Index = navEntry{
	URL: rel,
	Name: readTitle(filepath.Join(path, "index.md"), rel),
	}
	m := &metadata{
	DirOrder: []string{},
	FileOrder: []string{},
	}

	// populate all the other files
	f, err := os.Open(path)
	if err != nil {
	return nil, err
	}
	allFiles, err := f.Readdir(-1)
	if err != nil {
	return nil, err
	}
	for _, fi := range allFiles {
	// The contents are either files or directories.
	if fi.IsDir() {
	n, err := walk(root, filepath.Join(path, fi.Name()))
	if err != nil {
	return nil, err
	}
	ret.Dirs = append(ret.Dirs, n)
	} else if fi.Name() != "index.md" && strings.HasSuffix(fi.Name(), ".md") {
	fileRel := filepath.Clean(rel + "/" + fi.Name()[:len(fi.Name())-3])
	ret.Files = append(ret.Files, &navEntry{
	URL: fileRel,
	Name: readTitle(filepath.Join(path, fi.Name()), fileRel),
	})
	} else if fi.Name() == "METADATA" {
	// Load JSON found in METADATA.
	metaPath := filepath.Join(path, fi.Name())
	f, err := os.Open(metaPath)
	if err != nil {
	sklog.Warningf("Failed to open %q: %s", metaPath, err)
	continue
	}
	dec := json.NewDecoder(f)
	if err := dec.Decode(m); err != nil {
	sklog.Warningf("Failed to decode %q: %s", metaPath, err)
	}
	}
	}

	// Sort dirs and files, use METADATA if available.
	// Pick out the matches in the order they appear, then sort the rest.
	// Yes, this is O(n^2), but for very small n.
	sortedDirs := []*node{}
	for _, name := range m.DirOrder {
	for i, n := range ret.Dirs {
	if name == filepath.Base(n.Index.URL) {
	sortedDirs = append(sortedDirs, n)
	ret.Dirs = append(ret.Dirs[:i], ret.Dirs[i+1:]...)
	break
	}
	}
	}
	sort.Sort(nodeSlice(ret.Dirs))
	ret.Dirs = append(sortedDirs, ret.Dirs...)

	sortedFiles := []*navEntry{}
	for _, name := range m.FileOrder {
	for i, n := range ret.Files {
	if name == filepath.Base(n.URL) {
	sortedFiles = append(sortedFiles, n)
	ret.Files = append(ret.Files[:i], ret.Files[i+1:]...)
	break
	}
	}
	}
	sort.Sort(navEntrySlice(ret.Files))
	ret.Files = append(sortedFiles, ret.Files...)

	return ret, nil
	}

	func printnode(n *node, depth int) {
	sklog.Infof("Node: %s%#v\n", depth2, "", n.Index)
	for _, f := range n.Files {
	sklog.Infof("File: %s%#v\n", (depth+1)2, "", *f)
	}
	for _, d := range n.Dirs {
	printnode(d, depth+1)
	}
	}

	func addDepth(n *node, depth int) {
	n.Index.Depth = depth
	for _, d := range n.Dirs {
	addDepth(d, depth+1)
	}
	}

	// BuildNavigation builds the Navigation for the DocSet.
	func (d *DocSet) BuildNavigation() {
	// Walk the directory tree to build the navigation menu.
	root := filepath.Join(d.repoDir, config.REPO_SUBDIR)
	node, _ := walk(root, root)
	addDepth(node, 1)
	s := buildNavString(node)
	sm := buildSiteMap(node)
	d.mutex.Lock()
	defer d.mutex.Unlock()
	d.cache = lru.New(MARKDOWN_CACHE_SIZE)
	d.navigation = s
	d.siteMap = sm
	}

	// Navigation returns the HTML formatted navigation.
	func (d *DocSet) Navigation() string {
	d.mutex.Lock()
	defer d.mutex.Unlock()
	return d.navigation
	}

	// SiteMap returns the txt formatted site map.
	func (d *DocSet) SiteMap() string {
	d.mutex.Lock()
	defer d.mutex.Unlock()
	return d.siteMap
	}

	// issueAndPatch is a regex for extracting the issue number from a directory name
	// that is formatted like {issue_id}-{pathset_id}.
	var issueAndPatch = regexp.MustCompile("([0-9]+)-[0-9]+")

	// StartCleaner is a process that periodically checks the status of every issue
	// that has been previewed and removes all the local files for closed issues.
	func StartCleaner(workDir string) {
	sklog.Info("Starting Cleaner")
	for range time.Tick(config.REFRESH) {
	matches, err := filepath.Glob(workDir + "/patches/*")
	sklog.Infof("Matches: %v", matches)
	if err != nil {
	sklog.Errorf("Failed to retrieve list of patched checkouts: %s", err)
	continue
	}
	for _, filename := range matches {
	_, file := filepath.Split(filename)
	sklog.Info(file)
	m := issueAndPatch.FindStringSubmatch(file)
	if len(m) < 2 {
	continue
	}
	issue, err := strconv.ParseInt(m[1], 10, 64)
	if err != nil {
	sklog.Errorf("Failed to parse %q as int: %s", m[1], err)
	continue
	}
	info, err := gc.GetIssueProperties(context.TODO(), issue)
	// Delete closed and missing issues.
	if err != nil \|\| info.Committed {
	if err := os.RemoveAll(filename); err != nil {
	sklog.Errorf("Failed to remove %q: %s", filename, err)
	}
	}
	}
	}
	}

	// findFile takes a filename guess and turns into real name of a file.
	//
	// Look for the given file, if it exists then serve it raw with a guess at the
	// content type. Otherwise append ".md" and return it as processed markdown.
	// If it is a directory append "index.md" and return that.
	func findFile(filename string) (string, error) {
	if stat, err := os.Stat(filename); err == nil {
	if stat.IsDir() {
	return findFile(filepath.Join(filename, "./index.md"))
	} else {
	return filename, nil
	}
	} else {
	if filepath.Ext(filename) == ".md" {
	return filename, err
	} else {
	return findFile(filename + ".md")
	}
	}
	}

	// pathOf returns a '/' terminated path for the given filename.
	func pathOf(s string) string {
	if s[len(s)-1] == '/' {
	return s
	}
	parts := strings.Split(s, "/")
	if len(parts) > 0 {
	parts = parts[:len(parts)-1]
	}

	ret := strings.Join(parts, "/")
	if len(ret) == 0 \|\| ret[len(ret)-1] != '/' {
	ret += "/"
	}
	return ret
	}

	// navEntry is a single directory entry in the Markdown repo.
	type navEntry struct {
	Depth int
	URL string
	Name string
	}

	// navEntrySlice is a utility type for sorting navEntry's.
	type navEntrySlice []*navEntry

	func (p navEntrySlice) Len() int { return len(p) }
	func (p navEntrySlice) Less(i, j int) bool {
	if pathOf(p[i].URL) < pathOf(p[j].URL) {
	return true
	}
	if pathOf(p[i].URL) == pathOf(p[j].URL) {
	return p[i].URL < p[j].URL
	}
	return false
	}
	func (p navEntrySlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

	// readTitle reads the first line from a Markdown file.
	func readTitle(filename, def string) string {
	f, err := os.Open(filename)
	if err != nil {
	sklog.Warningf("Failed to open file %s: %s", filename, err)
	return def
	}
	defer util.Close(f)
	reader := bufio.NewReader(f)
	title, err := reader.ReadString('\n')
	if err != nil {
	sklog.Warningf("Failed to read title %s: %s", filename, err)
	}
	if strings.HasPrefix(title, "#") {
	title = markdownHeader.ReplaceAllString(title, "")
	}
	title = strings.TrimSpace(title)
	return title
	}