fiddle - Restore apoptosis.
Somehow this got lost in a previous CL.
Also record what state pods are in.
Bug: skia:
Change-Id: I6be67414e885297e053df1208df0e17329e19c4f
Reviewed-on: https://skia-review.googlesource.com/c/buildbot/+/201378
Reviewed-by: Ravi Mistry <rmistry@google.com>
Commit-Queue: Joe Gregorio <jcgregorio@google.com>
diff --git a/fiddlek/go/fiddler/main.go b/fiddlek/go/fiddler/main.go
index 945d2a9..fffec40 100644
--- a/fiddlek/go/fiddler/main.go
+++ b/fiddlek/go/fiddler/main.go
@@ -132,6 +132,18 @@
return
}
+ // Apoptosis.
+ _, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go func() {
+ select {
+ case <-time.Tick(*apoptosis):
+ sklog.Fatal("Exceeded total allowed runtime.")
+ case <-ctx.Done():
+ sklog.Info("Exited cleanly.")
+ }
+ }()
+
// Compile draw.cpp into 'fiddle'.
if err := ioutil.WriteFile(filepath.Join(*checkout, "tools", "fiddle", "draw.cpp"), []byte(request.Code), 0644); err != nil {
res.Execute.Errors = fmt.Sprintf("Failed to write draw.cpp: %s", err)
@@ -141,7 +153,6 @@
setState(types.COMPILING)
- // TODO(jcgregorio) Put a timeout here.
buildResults, err := build(ctx, *checkout, filepath.Join(*fiddleRoot, "depot_tools", "ninja"), "-C", "out/Static")
buildLogs := strings.Split(buildResults, "\n")
sklog.Info("BuildLog")
@@ -330,7 +341,6 @@
InheritEnv: true,
Stdout: &stdout,
Stderr: &stderr,
- Timeout: 30 * time.Second,
}
if err := exec.Run(ctx, runCmd); err != nil {
sklog.Errorf("Failed to run: %s", err)
diff --git a/fiddlek/go/runner/runner.go b/fiddlek/go/runner/runner.go
index 8c51440..210f9fa 100644
--- a/fiddlek/go/runner/runner.go
+++ b/fiddlek/go/runner/runner.go
@@ -329,30 +329,30 @@
return ret
}
-func singlePodVersion(client *http.Client, address string) (string, bool) {
+func singlePodVersion(client *http.Client, address string) (string, types.State, bool) {
rootURL := fmt.Sprintf("http://%s:8000", address)
req, err := http.NewRequest("GET", rootURL, nil)
if err != nil {
sklog.Infof("Failed to create request for fiddler status: %s", err)
- return "", false
+ return "", types.ERROR, false
}
// Pods come and go, so don't keep the connection alive.
req.Close = true
resp, err := client.Do(req)
if err != nil {
sklog.Infof("Failed to request fiddler status: %s", err)
- return "", false
+ return "", types.ERROR, false
}
defer util.Close(resp.Body)
var fiddlerResp types.FiddlerMainResponse
if err := json.NewDecoder(resp.Body).Decode(&fiddlerResp); err != nil {
sklog.Warningf("Failed to read status body: %s", err)
- return "", false
+ return "", types.ERROR, false
}
if fiddlerResp.State == types.IDLE {
- return fiddlerResp.Version, true
+ return fiddlerResp.Version, fiddlerResp.State, true
}
- return "", false
+ return "", fiddlerResp.State, false
}
func (r *Runner) metricsSingleStep() {
@@ -361,10 +361,16 @@
versions := map[string]int{}
ips := r.podIPs()
fastClient := httputils.NewFastTimeoutClient()
+ for _, st := range types.AllStates {
+ metrics2.GetCounter("fiddler_pod_state", map[string]string{"state": string(st)}).Reset()
+ }
for _, address := range ips {
- if ver, ok := singlePodVersion(fastClient, address); ok {
+ if ver, st, ok := singlePodVersion(fastClient, address); ok {
idleCount += 1
versions[ver] += 1
+ metrics2.GetCounter("fiddler_pod_state", map[string]string{"state": string(st)}).Inc(1)
+ } else {
+ metrics2.GetCounter("fiddler_pod_state", map[string]string{"state": string(st)}).Inc(1)
}
}
// Report the version that appears the most. Usually there will only be one
diff --git a/fiddlek/go/types/types.go b/fiddlek/go/types/types.go
index 38bc538..3740f51 100644
--- a/fiddlek/go/types/types.go
+++ b/fiddlek/go/types/types.go
@@ -161,6 +161,11 @@
WRITING State = "writing"
COMPILING State = "compiling"
RUNNING State = "running"
+ ERROR State = "error"
+)
+
+var (
+ AllStates = []State{IDLE, WRITING, COMPILING, RUNNING, ERROR}
)
// FiddlerMainResponse is the JSON that fiddler responds with for a request to "/".
