Add a new log server that runs on port 8001. Also move all the logs into /tmp/glog so that we don't inadvertently serve up some random things stored in /tmp. BUG=skia: R=mtklein@google.com Author: jcgregorio@google.com Review URL: https://codereview.chromium.org/390953003
diff --git a/perf/server/Makefile b/perf/server/Makefile index 855dbcd..2431e1b 100644 --- a/perf/server/Makefile +++ b/perf/server/Makefile
@@ -14,6 +14,10 @@ ingest: src/ingest/ingest.go src/ingest/ingest_main.go go build src/ingest/ingest.go src/ingest/ingest_main.go +logs: + GOPATH=$(GOPATH):`pwd` go build -v logserver + + tool: src/ingest/ingest.go GOPATH=$(GOPATH):`pwd` go build -v tiletool
diff --git a/perf/server/setup/continue_install b/perf/server/setup/continue_install index 77268ad..00c0c4a 100644 --- a/perf/server/setup/continue_install +++ b/perf/server/setup/continue_install
@@ -29,6 +29,7 @@ git clone https://skia.googlesource.com/skia fi +mkdir=/tmp/glog mkdir=$HOME/golib export GOROOT=$HOME/go export GOPATH=$HOME/golib @@ -40,3 +41,4 @@ make make tile make ingest +make logs
diff --git a/perf/server/setup/perf_setup.sh b/perf/server/setup/perf_setup.sh index 946edef..6d79289 100755 --- a/perf/server/setup/perf_setup.sh +++ b/perf/server/setup/perf_setup.sh
@@ -18,6 +18,8 @@ sudo chmod 744 /etc/init.d/tilebuilder sudo cp sys/ingester_init /etc/init.d/ingester sudo chmod 744 /etc/init.d/ingester +sudo cp sys/logserver_init /etc/init.d/logserver +sudo chmod 744 /etc/init.d/logserver sudo cp sys/perf_monit /etc/monit/conf.d/perf sudo cp sys/perf_squid /etc/squid3/squid.conf # Confirm that monit is happy. @@ -26,3 +28,4 @@ sudo /etc/init.d/perf restart sudo /etc/init.d/tilebuilder restart sudo /etc/init.d/ingester restart +sudo /etc/init.d/logserver restart
diff --git a/perf/server/setup/sys/ingester_init b/perf/server/setup/sys/ingester_init index 124435b..3a22021 100644 --- a/perf/server/setup/sys/ingester_init +++ b/perf/server/setup/sys/ingester_init
@@ -1,12 +1,12 @@ #! /bin/sh ### BEGIN INIT INFO -# Provides: perf +# Provides: ingester # Required-Start: $remote_fs $syslog # Required-Stop: $remote_fs $syslog # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: Start perf. -# Description: Web server for Skia performance monitoring. +# Description: Moves Google Storage data into BigQuery. ### END INIT INFO # Author: Joe Gregorio <jcgregorio@google.com> (copied by Kelvin Ly) @@ -22,7 +22,7 @@ DESC="The Skia JSON ingester application." NAME=ingest DAEMON=/home/perf/buildbot/perf/server/$NAME -DAEMON_ARGS="--log_only=true --oauth=false" +DAEMON_ARGS="--log_only=true --oauth=false --log_dir=/tmp/glog" PIDFILE=/var/run/$NAME.pid SCRIPTNAME=/etc/init.d/$NAME
diff --git a/perf/server/setup/sys/logserver_init b/perf/server/setup/sys/logserver_init new file mode 100644 index 0000000..8876493 --- /dev/null +++ b/perf/server/setup/sys/logserver_init
@@ -0,0 +1,159 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: logserver +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Start perf. +# Description: Serves the log file directory via HTTP. +### END INIT INFO + +# Author: Joe Gregorio <jcgregorio@google.com> +# +# Copied from /etc/init.d/skeleton and modified only the following +# environment variables and updated the start-stop-daemon calls +# in do_start() to add --make-pidfile, --background, and --chuid. + +# Do NOT "set -e" + +# PATH should only include /usr/* if it runs after the mountnfs.sh script +PATH=/sbin:/usr/sbin:/bin:/usr/bin +DESC="The Skia log server application." +NAME=logserver +DAEMON=/home/perf/buildbot/perf/server/$NAME +DAEMON_ARGS="" +PIDFILE=/var/run/$NAME.pid +SCRIPTNAME=/etc/init.d/$NAME + +# Exit if the package is not installed +[ -x "$DAEMON" ] || exit 0 + +# Read configuration variable file if it is present +[ -r /etc/default/$NAME ] && . /etc/default/$NAME + +# Load the VERBOSE setting and other rcS variables +. /lib/init/vars.sh + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.2-14) to ensure that this file is present +# and status_of_proc is working. +. /lib/lsb/init-functions + +# +# Function that starts the daemon/service +# +do_start() +{ + # Return + # 0 if daemon has been started + # 1 if daemon was already running + # 2 if daemon could not be started + start-stop-daemon --start --pidfile $PIDFILE --exec $DAEMON --make-pidfile --background --chuid perf --test > /dev/null \ + || return 1 + start-stop-daemon --start --pidfile $PIDFILE --exec $DAEMON --make-pidfile --background --chuid perf --exec $DAEMON -- \ + $DAEMON_ARGS \ + || return 2 + # Add code here, if necessary, that waits for the process to be ready + # to handle requests from services started subsequently which depend + # on this one. As a last resort, sleep for some time. +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + # Return + # 0 if daemon has been stopped + # 1 if daemon was already stopped + # 2 if daemon could not be stopped + # other if a failure occurred + start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME + RETVAL="$?" + [ "$RETVAL" = 2 ] && return 2 + # Wait for children to finish too if this is a daemon that forks + # and if the daemon is only ever run from this initscript. + # If the above conditions are not satisfied then add some other code + # that waits for the process to drop all resources that could be + # needed by services started subsequently. A last resort is to + # sleep for some time. + start-stop-daemon --stop --quiet --oknodo --retry=0/30/KILL/5 --exec $DAEMON + [ "$?" = 2 ] && return 2 + # Many daemons don't delete their pidfiles when they exit. + rm -f $PIDFILE + return "$RETVAL" +} + +# +# Function that sends a SIGHUP to the daemon/service +# +do_reload() { + # + # If the daemon can reload its configuration without + # restarting (for example, when it is sent a SIGHUP), + # then implement that here. + # + start-stop-daemon --stop --signal 1 --quiet --pidfile $PIDFILE --name $NAME + return 0 +} + +case "$1" in + start) + [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; + 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; + esac + ;; + stop) + [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; + 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; + esac + ;; + status) + status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $? + ;; + #reload|force-reload) + # + # If do_reload() is not implemented then leave this commented out + # and leave 'force-reload' as an alias for 'restart'. + # + #log_daemon_msg "Reloading $DESC" "$NAME" + #do_reload + #log_end_msg $? + #;; + restart|force-reload) + # + # If the "reload" option is implemented then remove the + # 'force-reload' alias + # + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + case "$?" in + 0|1) + do_start + case "$?" in + 0) log_end_msg 0 ;; + 1) log_end_msg 1 ;; # Old process is still running + *) log_end_msg 1 ;; # Failed to start + esac + ;; + *) + # Failed to stop + log_end_msg 1 + ;; + esac + ;; + *) + #echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2 + echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac + +:
diff --git a/perf/server/setup/sys/perf_init b/perf/server/setup/sys/perf_init index fd15793..f9052af 100644 --- a/perf/server/setup/sys/perf_init +++ b/perf/server/setup/sys/perf_init
@@ -22,7 +22,7 @@ DESC="The Skia perf application." NAME=perf DAEMON=/home/perf/buildbot/perf/server/$NAME -DAEMON_ARGS="--oauth=false --git_repo_dir=/home/perf/skia" +DAEMON_ARGS="--oauth=false --git_repo_dir=/home/perf/skia --log_dir=/tmp/glog" PIDFILE=/var/run/$NAME.pid SCRIPTNAME=/etc/init.d/$NAME
diff --git a/perf/server/setup/sys/tilebuilder_init b/perf/server/setup/sys/tilebuilder_init index 4865afa..a5d256e 100644 --- a/perf/server/setup/sys/tilebuilder_init +++ b/perf/server/setup/sys/tilebuilder_init
@@ -1,12 +1,12 @@ #! /bin/sh ### BEGIN INIT INFO -# Provides: perf +# Provides: tilebuilder # Required-Start: $remote_fs $syslog # Required-Stop: $remote_fs $syslog # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: Start perf. -# Description: Web server for Skia performance monitoring. +# Description: Builds tiles from BigQuery data.. ### END INIT INFO # Author: Joe Gregorio <jcgregorio@google.com> @@ -22,7 +22,7 @@ DESC="The Skia tilebuilder application." NAME=tilebuilder DAEMON=/home/perf/buildbot/perf/server/$NAME -DAEMON_ARGS="--oauth=false --tile_dir=/home/perf/tileStore" +DAEMON_ARGS="--oauth=false --tile_dir=/home/perf/tileStore --log_dir=/tmp/glog" PIDFILE=/var/run/$NAME.pid SCRIPTNAME=/etc/init.d/$NAME
diff --git a/perf/server/src/logserver/main.go b/perf/server/src/logserver/main.go new file mode 100644 index 0000000..7440e6d --- /dev/null +++ b/perf/server/src/logserver/main.go
@@ -0,0 +1,111 @@ +// Application that serves up the contents of /tmp/glog via HTTP, giving access +// to logs w/o needing to SSH into the server. +package main + +import ( + "flag" + "fmt" + "html/template" + "net/http" + "net/url" + "os" + "path" + "sort" + "strings" + + "github.com/golang/glog" +) + +var port = flag.String("port", ":8001", "HTTP service address (e.g., ':8001')") + +// FileServer returns a handler that serves HTTP requests +// with the contents of the file system rooted at root. +// +// To use the operating system's file system implementation, +// use http.Dir: +// +// http.Handle("/", FileServer(http.Dir("/tmp"))) +// +// Differs from net/http FileServer by making directory listings better. +func FileServer(root http.FileSystem) http.Handler { + return &fileHandler{root} +} + +type fileHandler struct { + root http.FileSystem +} + +func (f *fileHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + upath := r.URL.Path + if !strings.HasPrefix(upath, "/") { + upath = "/" + upath + r.URL.Path = upath + } + serveFile(w, r, f.root, path.Clean(upath)) +} + +// FileInfoSlice is for sorting. +type FileInfoSlice []os.FileInfo + +func (p FileInfoSlice) Len() int { return len(p) } +func (p FileInfoSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } +func (p FileInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +func dirList(w http.ResponseWriter, f http.File) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + fmt.Fprintf(w, "<pre>\n") + for { + dirs, err := f.Readdir(10000) + sort.Sort(FileInfoSlice(dirs)) + if err != nil || len(dirs) == 0 { + break + } + for _, d := range dirs { + name := d.Name() + if d.IsDir() { + name += "/" + } + url := url.URL{Path: name} + fmt.Fprintf(w, "%s <a href=\"%s\">%s</a>\n", d.ModTime(), url.String(), template.HTMLEscapeString(name)) + } + } + fmt.Fprintf(w, "</pre>\n") +} + +func serveFile(w http.ResponseWriter, r *http.Request, fs http.FileSystem, name string) { + f, err := fs.Open(name) + if err != nil { + http.NotFound(w, r) + return + } + defer f.Close() + + d, err1 := f.Stat() + if err1 != nil { + http.NotFound(w, r) + return + } + + url := r.URL.Path + if d.IsDir() { + if url[len(url)-1] != '/' { + w.Header().Set("Location", path.Base(url)+"/") + w.WriteHeader(http.StatusMovedPermanently) + return + } + } + + if d.IsDir() { + dirList(w, f) + return + } + + http.ServeContent(w, r, d.Name(), d.ModTime(), f) +} + +func main() { + flag.Parse() + + http.Handle("/", http.StripPrefix("/", FileServer(http.Dir("/tmp/glog")))) + glog.Fatal(http.ListenAndServe(*port, nil)) +}