Add recovery steps for scaling and disabling cpu cores.
Bug: b/265965381
Change-Id: I59f787474f7bc8c4021ae347e128dec59fce133a
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/631997
Reviewed-by: Eric Boren <borenet@google.com>
Commit-Queue: Joe Gregorio <jcgregorio@google.com>
diff --git a/infra/bots/recipe_modules/flavor/android.py b/infra/bots/recipe_modules/flavor/android.py
index ddf9f29..963bf2b 100644
--- a/infra/bots/recipe_modules/flavor/android.py
+++ b/infra/bots/recipe_modules/flavor/android.py
@@ -79,6 +79,53 @@
"Nexus5x": 600000000,
}
+ def _wait_for_device(self, title, attempt):
+ self.m.run(self.m.step,
+ 'adb kill-server after failure of \'%s\' (attempt %d)' % (
+ title, attempt),
+ cmd=[self.ADB_BINARY, 'kill-server'],
+ infra_step=True, timeout=30, abort_on_failure=False,
+ fail_build_on_failure=False)
+ self.m.run(self.m.step,
+ 'wait for device after failure of \'%s\' (attempt %d)' % (
+ title, attempt),
+ cmd=[self.ADB_BINARY, 'wait-for-device'], infra_step=True,
+ timeout=180, abort_on_failure=False,
+ fail_build_on_failure=False)
+ self.m.run(self.m.step,
+ 'adb devices -l after failure of \'%s\' (attempt %d)' % (
+ title, attempt),
+ cmd=[self.ADB_BINARY, 'devices', '-l'],
+ infra_step=True, timeout=30, abort_on_failure=False,
+ fail_build_on_failure=False)
+ self.m.run(self.m.step,
+ 'adb reboot device after failure of \'%s\' (attempt %d)' % (
+ title, attempt),
+ cmd=[self.ADB_BINARY, 'reboot'],
+ infra_step=True, timeout=30, abort_on_failure=False,
+ fail_build_on_failure=False)
+ self.m.run(self.m.step,
+ 'wait for device after failure of \'%s\' (attempt %d)' % (
+ title, attempt),
+ cmd=[
+ self.ADB_BINARY, 'wait-for-device', 'shell',
+ # Wait until the boot is actually complete.
+ # https://android.stackexchange.com/a/164050
+ 'while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done',
+ ],
+ timeout=180, abort_on_failure=False,
+ fail_build_on_failure=False)
+ device = self.m.vars.builder_cfg.get('model')
+ if (device in self.cant_root): # pragma: nocover
+ return
+ self.m.run(self.m.step,
+ 'adb root',
+ cmd=[
+ self.ADB_BINARY, 'root'
+ ],
+ timeout=180, abort_on_failure=False,
+ fail_build_on_failure=False)
+
def _adb(self, title, *cmd, **kwargs):
# The only non-infra adb steps (dm / nanobench) happen to not use _adb().
if 'infra_step' not in kwargs:
@@ -89,51 +136,7 @@
attempts = kwargs.pop('attempts', 3)
def wait_for_device(attempt):
- self.m.run(self.m.step,
- 'adb kill-server after failure of \'%s\' (attempt %d)' % (
- title, attempt),
- cmd=[self.ADB_BINARY, 'kill-server'],
- infra_step=True, timeout=30, abort_on_failure=False,
- fail_build_on_failure=False)
- self.m.run(self.m.step,
- 'wait for device after failure of \'%s\' (attempt %d)' % (
- title, attempt),
- cmd=[self.ADB_BINARY, 'wait-for-device'], infra_step=True,
- timeout=180, abort_on_failure=False,
- fail_build_on_failure=False)
- self.m.run(self.m.step,
- 'adb devices -l after failure of \'%s\' (attempt %d)' % (
- title, attempt),
- cmd=[self.ADB_BINARY, 'devices', '-l'],
- infra_step=True, timeout=30, abort_on_failure=False,
- fail_build_on_failure=False)
- self.m.run(self.m.step,
- 'adb reboot device after failure of \'%s\' (attempt %d)' % (
- title, attempt),
- cmd=[self.ADB_BINARY, 'reboot'],
- infra_step=True, timeout=30, abort_on_failure=False,
- fail_build_on_failure=False)
- self.m.run(self.m.step,
- 'wait for device after failure of \'%s\' (attempt %d)' % (
- title, attempt),
- cmd=[
- self.ADB_BINARY, 'wait-for-device', 'shell',
- # Wait until the boot is actually complete.
- # https://android.stackexchange.com/a/164050
- 'while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done',
- ],
- timeout=180, abort_on_failure=False,
- fail_build_on_failure=False)
- device = self.m.vars.builder_cfg.get('model')
- if (device in self.cant_root): # pragma: nocover
- return
- self.m.run(self.m.step,
- 'adb root',
- cmd=[
- self.ADB_BINARY, 'root'
- ],
- timeout=180, abort_on_failure=False,
- fail_build_on_failure=False)
+ return self._wait_for_device(title, attempt)
with self.m.context(cwd=self.m.path['start_dir'].join('skia')):
with self.m.env({'ADB_VENDOR_KEYS': self.ADB_PUB_KEY}):
@@ -286,6 +289,10 @@
msg = 'Disabling'
if value:
msg = 'Enabling'
+
+ def wait_for_device(attempt):
+ return self._wait_for_device("set cpu online", attempt) # pragma: nocover
+
self.m.run.with_retry(self.m.python.inline,
'%s CPU %d' % (msg, cpu),
3, # attempts
@@ -322,11 +329,16 @@
""",
args = [self.ADB_BINARY, cpu, value],
infra_step=True,
+ between_attempts_fn=wait_for_device,
timeout=30)
def _scale_cpu(self, cpu, target_percent):
self._ever_ran_adb = True
+
+ def wait_for_device(attempt):
+ return self._wait_for_device("scale cpu", attempt)
+
self.m.run.with_retry(self.m.python.inline,
'Scale CPU %d to %f' % (cpu, target_percent),
3, # attempts
@@ -387,6 +399,7 @@
""",
args = [self.ADB_BINARY, str(target_percent), cpu],
infra_step=True,
+ between_attempts_fn=wait_for_device,
timeout=30)
diff --git a/infra/bots/recipe_modules/flavor/examples/full.expected/cpu_scale_failed.json b/infra/bots/recipe_modules/flavor/examples/full.expected/cpu_scale_failed.json
index 95167d9..78ad8da 100644
--- a/infra/bots/recipe_modules/flavor/examples/full.expected/cpu_scale_failed.json
+++ b/infra/bots/recipe_modules/flavor/examples/full.expected/cpu_scale_failed.json
@@ -330,6 +330,85 @@
},
{
"cmd": [
+ "/usr/bin/adb.1.0.35",
+ "kill-server"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "adb kill-server after failure of 'scale cpu' (attempt 1)",
+ "timeout": 30
+ },
+ {
+ "cmd": [
+ "/usr/bin/adb.1.0.35",
+ "wait-for-device"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "wait for device after failure of 'scale cpu' (attempt 1)",
+ "timeout": 180
+ },
+ {
+ "cmd": [
+ "/usr/bin/adb.1.0.35",
+ "devices",
+ "-l"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "adb devices -l after failure of 'scale cpu' (attempt 1)",
+ "timeout": 30
+ },
+ {
+ "cmd": [
+ "/usr/bin/adb.1.0.35",
+ "reboot"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "adb reboot device after failure of 'scale cpu' (attempt 1)",
+ "timeout": 30
+ },
+ {
+ "cmd": [
+ "/usr/bin/adb.1.0.35",
+ "wait-for-device",
+ "shell",
+ "while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "name": "wait for device after failure of 'scale cpu' (attempt 1) (2)",
+ "timeout": 180
+ },
+ {
+ "cmd": [
+ "/usr/bin/adb.1.0.35",
+ "root"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "name": "adb root",
+ "timeout": 180
+ },
+ {
+ "cmd": [
"python",
"\nimport os\nimport subprocess\nimport sys\nimport time\nADB = sys.argv[1]\ntarget_percent = float(sys.argv[2])\ncpu = int(sys.argv[3])\nlog = subprocess.check_output([ADB, 'root']).decode('utf-8')\n# check for message like 'adbd cannot run as root in production builds'\nprint(log)\nif 'cannot' in log:\n raise Exception('adb root failed')\n\nroot = '/sys/devices/system/cpu/cpu%d/cpufreq' %cpu\n\n# All devices we test on give a list of their available frequencies.\navailable_freqs = subprocess.check_output([ADB, 'shell',\n 'cat %s/scaling_available_frequencies' % root]).decode('utf-8')\n\n# Check for message like '/system/bin/sh: file not found'\nif available_freqs and '/system/bin/sh' not in available_freqs:\n available_freqs = sorted(\n int(i) for i in available_freqs.strip().split())\nelse:\n raise Exception('Could not get list of available frequencies: %s' %\n available_freqs)\n\nmaxfreq = available_freqs[-1]\ntarget = int(round(maxfreq * target_percent))\nfreq = maxfreq\nfor f in reversed(available_freqs):\n if f <= target:\n freq = f\n break\n\nprint('Setting frequency to %d' % freq)\n\n# If scaling_max_freq is lower than our attempted setting, it won't take.\n# We must set min first, because if we try to set max to be less than min\n# (which sometimes happens after certain devices reboot) it returns a\n# perplexing permissions error.\nsubprocess.check_call([ADB, 'shell', 'echo 0 > '\n '%s/scaling_min_freq' % root])\nsubprocess.check_call([ADB, 'shell', 'echo %d > '\n '%s/scaling_max_freq' % (freq, root)])\nsubprocess.check_call([ADB, 'shell', 'echo %d > '\n '%s/scaling_setspeed' % (freq, root)])\ntime.sleep(5)\nactual_freq = subprocess.check_output([ADB, 'shell', 'cat '\n '%s/scaling_cur_freq' % root]).decode('utf-8').strip()\nif actual_freq != str(freq):\n raise Exception('(actual, expected) (%s, %d)'\n % (actual_freq, freq))\n",
"/usr/bin/adb.1.0.35",
@@ -404,6 +483,85 @@
},
{
"cmd": [
+ "/usr/bin/adb.1.0.35",
+ "kill-server"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "adb kill-server after failure of 'scale cpu' (attempt 2)",
+ "timeout": 30
+ },
+ {
+ "cmd": [
+ "/usr/bin/adb.1.0.35",
+ "wait-for-device"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "wait for device after failure of 'scale cpu' (attempt 2)",
+ "timeout": 180
+ },
+ {
+ "cmd": [
+ "/usr/bin/adb.1.0.35",
+ "devices",
+ "-l"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "adb devices -l after failure of 'scale cpu' (attempt 2)",
+ "timeout": 30
+ },
+ {
+ "cmd": [
+ "/usr/bin/adb.1.0.35",
+ "reboot"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "adb reboot device after failure of 'scale cpu' (attempt 2)",
+ "timeout": 30
+ },
+ {
+ "cmd": [
+ "/usr/bin/adb.1.0.35",
+ "wait-for-device",
+ "shell",
+ "while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "name": "wait for device after failure of 'scale cpu' (attempt 2) (2)",
+ "timeout": 180
+ },
+ {
+ "cmd": [
+ "/usr/bin/adb.1.0.35",
+ "root"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "name": "adb root (2)",
+ "timeout": 180
+ },
+ {
+ "cmd": [
"python",
"\nimport os\nimport subprocess\nimport sys\nimport time\nADB = sys.argv[1]\ntarget_percent = float(sys.argv[2])\ncpu = int(sys.argv[3])\nlog = subprocess.check_output([ADB, 'root']).decode('utf-8')\n# check for message like 'adbd cannot run as root in production builds'\nprint(log)\nif 'cannot' in log:\n raise Exception('adb root failed')\n\nroot = '/sys/devices/system/cpu/cpu%d/cpufreq' %cpu\n\n# All devices we test on give a list of their available frequencies.\navailable_freqs = subprocess.check_output([ADB, 'shell',\n 'cat %s/scaling_available_frequencies' % root]).decode('utf-8')\n\n# Check for message like '/system/bin/sh: file not found'\nif available_freqs and '/system/bin/sh' not in available_freqs:\n available_freqs = sorted(\n int(i) for i in available_freqs.strip().split())\nelse:\n raise Exception('Could not get list of available frequencies: %s' %\n available_freqs)\n\nmaxfreq = available_freqs[-1]\ntarget = int(round(maxfreq * target_percent))\nfreq = maxfreq\nfor f in reversed(available_freqs):\n if f <= target:\n freq = f\n break\n\nprint('Setting frequency to %d' % freq)\n\n# If scaling_max_freq is lower than our attempted setting, it won't take.\n# We must set min first, because if we try to set max to be less than min\n# (which sometimes happens after certain devices reboot) it returns a\n# perplexing permissions error.\nsubprocess.check_call([ADB, 'shell', 'echo 0 > '\n '%s/scaling_min_freq' % root])\nsubprocess.check_call([ADB, 'shell', 'echo %d > '\n '%s/scaling_max_freq' % (freq, root)])\nsubprocess.check_call([ADB, 'shell', 'echo %d > '\n '%s/scaling_setspeed' % (freq, root)])\ntime.sleep(5)\nactual_freq = subprocess.check_output([ADB, 'shell', 'cat '\n '%s/scaling_cur_freq' % root]).decode('utf-8').strip()\nif actual_freq != str(freq):\n raise Exception('(actual, expected) (%s, %d)'\n % (actual_freq, freq))\n",
"/usr/bin/adb.1.0.35",
@@ -479,6 +637,85 @@
{
"cmd": [
"/usr/bin/adb.1.0.35",
+ "kill-server"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "adb kill-server after failure of 'scale cpu' (attempt 3)",
+ "timeout": 30
+ },
+ {
+ "cmd": [
+ "/usr/bin/adb.1.0.35",
+ "wait-for-device"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "wait for device after failure of 'scale cpu' (attempt 3)",
+ "timeout": 180
+ },
+ {
+ "cmd": [
+ "/usr/bin/adb.1.0.35",
+ "devices",
+ "-l"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "adb devices -l after failure of 'scale cpu' (attempt 3)",
+ "timeout": 30
+ },
+ {
+ "cmd": [
+ "/usr/bin/adb.1.0.35",
+ "reboot"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "adb reboot device after failure of 'scale cpu' (attempt 3)",
+ "timeout": 30
+ },
+ {
+ "cmd": [
+ "/usr/bin/adb.1.0.35",
+ "wait-for-device",
+ "shell",
+ "while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "name": "wait for device after failure of 'scale cpu' (attempt 3) (2)",
+ "timeout": 180
+ },
+ {
+ "cmd": [
+ "/usr/bin/adb.1.0.35",
+ "root"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "name": "adb root (3)",
+ "timeout": 180
+ },
+ {
+ "cmd": [
+ "/usr/bin/adb.1.0.35",
"reboot"
],
"env": {
diff --git a/infra/bots/recipe_modules/flavor/examples/full.expected/cpu_scale_failed_golo.json b/infra/bots/recipe_modules/flavor/examples/full.expected/cpu_scale_failed_golo.json
index 95cf268..e45858b 100644
--- a/infra/bots/recipe_modules/flavor/examples/full.expected/cpu_scale_failed_golo.json
+++ b/infra/bots/recipe_modules/flavor/examples/full.expected/cpu_scale_failed_golo.json
@@ -330,6 +330,85 @@
},
{
"cmd": [
+ "/opt/infra-android/tools/adb",
+ "kill-server"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "adb kill-server after failure of 'scale cpu' (attempt 1)",
+ "timeout": 30
+ },
+ {
+ "cmd": [
+ "/opt/infra-android/tools/adb",
+ "wait-for-device"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "wait for device after failure of 'scale cpu' (attempt 1)",
+ "timeout": 180
+ },
+ {
+ "cmd": [
+ "/opt/infra-android/tools/adb",
+ "devices",
+ "-l"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "adb devices -l after failure of 'scale cpu' (attempt 1)",
+ "timeout": 30
+ },
+ {
+ "cmd": [
+ "/opt/infra-android/tools/adb",
+ "reboot"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "adb reboot device after failure of 'scale cpu' (attempt 1)",
+ "timeout": 30
+ },
+ {
+ "cmd": [
+ "/opt/infra-android/tools/adb",
+ "wait-for-device",
+ "shell",
+ "while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "name": "wait for device after failure of 'scale cpu' (attempt 1) (2)",
+ "timeout": 180
+ },
+ {
+ "cmd": [
+ "/opt/infra-android/tools/adb",
+ "root"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "name": "adb root",
+ "timeout": 180
+ },
+ {
+ "cmd": [
"python",
"\nimport os\nimport subprocess\nimport sys\nimport time\nADB = sys.argv[1]\ntarget_percent = float(sys.argv[2])\ncpu = int(sys.argv[3])\nlog = subprocess.check_output([ADB, 'root']).decode('utf-8')\n# check for message like 'adbd cannot run as root in production builds'\nprint(log)\nif 'cannot' in log:\n raise Exception('adb root failed')\n\nroot = '/sys/devices/system/cpu/cpu%d/cpufreq' %cpu\n\n# All devices we test on give a list of their available frequencies.\navailable_freqs = subprocess.check_output([ADB, 'shell',\n 'cat %s/scaling_available_frequencies' % root]).decode('utf-8')\n\n# Check for message like '/system/bin/sh: file not found'\nif available_freqs and '/system/bin/sh' not in available_freqs:\n available_freqs = sorted(\n int(i) for i in available_freqs.strip().split())\nelse:\n raise Exception('Could not get list of available frequencies: %s' %\n available_freqs)\n\nmaxfreq = available_freqs[-1]\ntarget = int(round(maxfreq * target_percent))\nfreq = maxfreq\nfor f in reversed(available_freqs):\n if f <= target:\n freq = f\n break\n\nprint('Setting frequency to %d' % freq)\n\n# If scaling_max_freq is lower than our attempted setting, it won't take.\n# We must set min first, because if we try to set max to be less than min\n# (which sometimes happens after certain devices reboot) it returns a\n# perplexing permissions error.\nsubprocess.check_call([ADB, 'shell', 'echo 0 > '\n '%s/scaling_min_freq' % root])\nsubprocess.check_call([ADB, 'shell', 'echo %d > '\n '%s/scaling_max_freq' % (freq, root)])\nsubprocess.check_call([ADB, 'shell', 'echo %d > '\n '%s/scaling_setspeed' % (freq, root)])\ntime.sleep(5)\nactual_freq = subprocess.check_output([ADB, 'shell', 'cat '\n '%s/scaling_cur_freq' % root]).decode('utf-8').strip()\nif actual_freq != str(freq):\n raise Exception('(actual, expected) (%s, %d)'\n % (actual_freq, freq))\n",
"/opt/infra-android/tools/adb",
@@ -404,6 +483,85 @@
},
{
"cmd": [
+ "/opt/infra-android/tools/adb",
+ "kill-server"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "adb kill-server after failure of 'scale cpu' (attempt 2)",
+ "timeout": 30
+ },
+ {
+ "cmd": [
+ "/opt/infra-android/tools/adb",
+ "wait-for-device"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "wait for device after failure of 'scale cpu' (attempt 2)",
+ "timeout": 180
+ },
+ {
+ "cmd": [
+ "/opt/infra-android/tools/adb",
+ "devices",
+ "-l"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "adb devices -l after failure of 'scale cpu' (attempt 2)",
+ "timeout": 30
+ },
+ {
+ "cmd": [
+ "/opt/infra-android/tools/adb",
+ "reboot"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "adb reboot device after failure of 'scale cpu' (attempt 2)",
+ "timeout": 30
+ },
+ {
+ "cmd": [
+ "/opt/infra-android/tools/adb",
+ "wait-for-device",
+ "shell",
+ "while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "name": "wait for device after failure of 'scale cpu' (attempt 2) (2)",
+ "timeout": 180
+ },
+ {
+ "cmd": [
+ "/opt/infra-android/tools/adb",
+ "root"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "name": "adb root (2)",
+ "timeout": 180
+ },
+ {
+ "cmd": [
"python",
"\nimport os\nimport subprocess\nimport sys\nimport time\nADB = sys.argv[1]\ntarget_percent = float(sys.argv[2])\ncpu = int(sys.argv[3])\nlog = subprocess.check_output([ADB, 'root']).decode('utf-8')\n# check for message like 'adbd cannot run as root in production builds'\nprint(log)\nif 'cannot' in log:\n raise Exception('adb root failed')\n\nroot = '/sys/devices/system/cpu/cpu%d/cpufreq' %cpu\n\n# All devices we test on give a list of their available frequencies.\navailable_freqs = subprocess.check_output([ADB, 'shell',\n 'cat %s/scaling_available_frequencies' % root]).decode('utf-8')\n\n# Check for message like '/system/bin/sh: file not found'\nif available_freqs and '/system/bin/sh' not in available_freqs:\n available_freqs = sorted(\n int(i) for i in available_freqs.strip().split())\nelse:\n raise Exception('Could not get list of available frequencies: %s' %\n available_freqs)\n\nmaxfreq = available_freqs[-1]\ntarget = int(round(maxfreq * target_percent))\nfreq = maxfreq\nfor f in reversed(available_freqs):\n if f <= target:\n freq = f\n break\n\nprint('Setting frequency to %d' % freq)\n\n# If scaling_max_freq is lower than our attempted setting, it won't take.\n# We must set min first, because if we try to set max to be less than min\n# (which sometimes happens after certain devices reboot) it returns a\n# perplexing permissions error.\nsubprocess.check_call([ADB, 'shell', 'echo 0 > '\n '%s/scaling_min_freq' % root])\nsubprocess.check_call([ADB, 'shell', 'echo %d > '\n '%s/scaling_max_freq' % (freq, root)])\nsubprocess.check_call([ADB, 'shell', 'echo %d > '\n '%s/scaling_setspeed' % (freq, root)])\ntime.sleep(5)\nactual_freq = subprocess.check_output([ADB, 'shell', 'cat '\n '%s/scaling_cur_freq' % root]).decode('utf-8').strip()\nif actual_freq != str(freq):\n raise Exception('(actual, expected) (%s, %d)'\n % (actual_freq, freq))\n",
"/opt/infra-android/tools/adb",
@@ -479,6 +637,85 @@
{
"cmd": [
"/opt/infra-android/tools/adb",
+ "kill-server"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "adb kill-server after failure of 'scale cpu' (attempt 3)",
+ "timeout": 30
+ },
+ {
+ "cmd": [
+ "/opt/infra-android/tools/adb",
+ "wait-for-device"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "wait for device after failure of 'scale cpu' (attempt 3)",
+ "timeout": 180
+ },
+ {
+ "cmd": [
+ "/opt/infra-android/tools/adb",
+ "devices",
+ "-l"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "adb devices -l after failure of 'scale cpu' (attempt 3)",
+ "timeout": 30
+ },
+ {
+ "cmd": [
+ "/opt/infra-android/tools/adb",
+ "reboot"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "adb reboot device after failure of 'scale cpu' (attempt 3)",
+ "timeout": 30
+ },
+ {
+ "cmd": [
+ "/opt/infra-android/tools/adb",
+ "wait-for-device",
+ "shell",
+ "while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "name": "wait for device after failure of 'scale cpu' (attempt 3) (2)",
+ "timeout": 180
+ },
+ {
+ "cmd": [
+ "/opt/infra-android/tools/adb",
+ "root"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "name": "adb root (3)",
+ "timeout": 180
+ },
+ {
+ "cmd": [
+ "/opt/infra-android/tools/adb",
"reboot"
],
"env": {
diff --git a/infra/bots/recipe_modules/flavor/examples/full.expected/cpu_scale_failed_once.json b/infra/bots/recipe_modules/flavor/examples/full.expected/cpu_scale_failed_once.json
index d438ddc..9996d38 100644
--- a/infra/bots/recipe_modules/flavor/examples/full.expected/cpu_scale_failed_once.json
+++ b/infra/bots/recipe_modules/flavor/examples/full.expected/cpu_scale_failed_once.json
@@ -330,6 +330,85 @@
},
{
"cmd": [
+ "/opt/infra-android/tools/adb",
+ "kill-server"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "adb kill-server after failure of 'scale cpu' (attempt 1)",
+ "timeout": 30
+ },
+ {
+ "cmd": [
+ "/opt/infra-android/tools/adb",
+ "wait-for-device"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "wait for device after failure of 'scale cpu' (attempt 1)",
+ "timeout": 180
+ },
+ {
+ "cmd": [
+ "/opt/infra-android/tools/adb",
+ "devices",
+ "-l"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "adb devices -l after failure of 'scale cpu' (attempt 1)",
+ "timeout": 30
+ },
+ {
+ "cmd": [
+ "/opt/infra-android/tools/adb",
+ "reboot"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "infra_step": true,
+ "name": "adb reboot device after failure of 'scale cpu' (attempt 1)",
+ "timeout": 30
+ },
+ {
+ "cmd": [
+ "/opt/infra-android/tools/adb",
+ "wait-for-device",
+ "shell",
+ "while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "name": "wait for device after failure of 'scale cpu' (attempt 1) (2)",
+ "timeout": 180
+ },
+ {
+ "cmd": [
+ "/opt/infra-android/tools/adb",
+ "root"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "name": "adb root",
+ "timeout": 180
+ },
+ {
+ "cmd": [
"python",
"\nimport os\nimport subprocess\nimport sys\nimport time\nADB = sys.argv[1]\ntarget_percent = float(sys.argv[2])\ncpu = int(sys.argv[3])\nlog = subprocess.check_output([ADB, 'root']).decode('utf-8')\n# check for message like 'adbd cannot run as root in production builds'\nprint(log)\nif 'cannot' in log:\n raise Exception('adb root failed')\n\nroot = '/sys/devices/system/cpu/cpu%d/cpufreq' %cpu\n\n# All devices we test on give a list of their available frequencies.\navailable_freqs = subprocess.check_output([ADB, 'shell',\n 'cat %s/scaling_available_frequencies' % root]).decode('utf-8')\n\n# Check for message like '/system/bin/sh: file not found'\nif available_freqs and '/system/bin/sh' not in available_freqs:\n available_freqs = sorted(\n int(i) for i in available_freqs.strip().split())\nelse:\n raise Exception('Could not get list of available frequencies: %s' %\n available_freqs)\n\nmaxfreq = available_freqs[-1]\ntarget = int(round(maxfreq * target_percent))\nfreq = maxfreq\nfor f in reversed(available_freqs):\n if f <= target:\n freq = f\n break\n\nprint('Setting frequency to %d' % freq)\n\n# If scaling_max_freq is lower than our attempted setting, it won't take.\n# We must set min first, because if we try to set max to be less than min\n# (which sometimes happens after certain devices reboot) it returns a\n# perplexing permissions error.\nsubprocess.check_call([ADB, 'shell', 'echo 0 > '\n '%s/scaling_min_freq' % root])\nsubprocess.check_call([ADB, 'shell', 'echo %d > '\n '%s/scaling_max_freq' % (freq, root)])\nsubprocess.check_call([ADB, 'shell', 'echo %d > '\n '%s/scaling_setspeed' % (freq, root)])\ntime.sleep(5)\nactual_freq = subprocess.check_output([ADB, 'shell', 'cat '\n '%s/scaling_cur_freq' % root]).decode('utf-8').strip()\nif actual_freq != str(freq):\n raise Exception('(actual, expected) (%s, %d)'\n % (actual_freq, freq))\n",
"/opt/infra-android/tools/adb",
