| #!/usr/bin/env python |
| # |
| # Copyright 2016 Google Inc. |
| # |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| |
| """Create the SKP asset.""" |
| |
| |
| from __future__ import print_function |
| import argparse |
| from distutils.dir_util import copy_tree |
| import os |
| import shutil |
| import subprocess |
| import sys |
| import tempfile |
| |
| FILE_DIR = os.path.dirname(os.path.abspath(__file__)) |
| INFRA_BOTS_DIR = os.path.realpath(os.path.join(FILE_DIR, os.pardir, os.pardir)) |
| sys.path.insert(0, INFRA_BOTS_DIR) |
| import utils |
| |
| |
| BROWSER_EXECUTABLE_ENV_VAR = 'SKP_BROWSER_EXECUTABLE' |
| CHROME_SRC_PATH_ENV_VAR = 'SKP_CHROME_SRC_PATH' |
| UPLOAD_TO_PARTNER_BUCKET_ENV_VAR = 'SKP_UPLOAD_TO_PARTNER_BUCKET' |
| DM_PATH_ENV_VAR = 'DM_PATH' |
| |
| SKIA_TOOLS = os.path.join(INFRA_BOTS_DIR, os.pardir, os.pardir, 'tools') |
| PRIVATE_SKPS_GS = 'gs://skia-skps/private/skps' |
| |
| |
| def getenv(key): |
| val = os.environ.get(key) |
| if not val: |
| print(('Environment variable %s not set; you should run this via ' |
| 'create_and_upload.py.' % key), file=sys.stderr) |
| sys.exit(1) |
| return val |
| |
| |
| def get_flutter_skps(target_dir): |
| """Creates SKPs using Flutter's skp_generator tool. |
| |
| Documentation is at https://github.com/flutter/tests/tree/master/skp_generator |
| """ |
| with utils.tmp_dir(): |
| print('Retrieving Flutter SKPs...') |
| utils.git_clone('https://github.com/flutter/tests.git', '.') |
| os.chdir('skp_generator') |
| subprocess.check_call(['bash', 'build.sh']) |
| # Fix invalid SKP file names. |
| for f in os.listdir('skps'): |
| original_file_name = os.path.splitext(f)[0] |
| new_file_name = ''.join([x if x.isalnum() else "_" |
| for x in original_file_name]) |
| if new_file_name != original_file_name: |
| os.rename(os.path.join('skps', f), |
| os.path.join('skps', new_file_name + '.skp')) |
| copy_tree('skps', target_dir) |
| print('Done retrieving Flutter SKPs.') |
| |
| |
| def create_asset(chrome_src_path, browser_executable, target_dir, |
| upload_to_partner_bucket, dm_path): |
| """Create the SKP asset. |
| |
| Creates the asset from 3 sources: |
| 1. From Flutter's skp_generator tool. |
| 2. The web pages defined in the tools/skp/page_sets/ directory. |
| 3. Any private SKPs stored in $PRIVATE_SKPS_GS after running dm on |
| them (see below). |
| |
| The script runs the following cmd on the non-generated SKPs stored in |
| $PRIVATE_SKPS_GS - |
| `dm --config skp -w newskps/ --skps oldskps/ --src skp` |
| The cmd updates the version stored in the SKPs so that the versions in |
| them do not eventually become unsupported. |
| """ |
| browser_executable = os.path.realpath(browser_executable) |
| chrome_src_path = os.path.realpath(chrome_src_path) |
| dm_path = os.path.realpath(dm_path) |
| target_dir = os.path.realpath(target_dir) |
| |
| if not os.path.exists(target_dir): |
| os.makedirs(target_dir) |
| |
| # 1. Flutter SKPs |
| get_flutter_skps(target_dir) |
| |
| # 2. Skia's SKPs from tools/skp/page_sets/ |
| with utils.tmp_dir(): |
| if os.environ.get('CHROME_HEADLESS'): |
| print('Starting xvfb') |
| # Start Xvfb if running on a bot. |
| try: |
| xvfb_proc = subprocess.Popen([ |
| 'sudo', 'Xvfb', ':0', '-screen', '0', '1280x1024x24']) |
| except Exception: |
| # It is ok if the above command fails, it just means that DISPLAY=:0 |
| # is already up. |
| xvfb_proc = None |
| |
| print('Running webpages_playback to generate SKPs...') |
| webpages_playback_cmd = [ |
| 'python', '-u', os.path.join(SKIA_TOOLS, 'skp', 'webpages_playback.py'), |
| '--page_sets', 'all', |
| '--browser_executable', browser_executable, |
| '--non-interactive', |
| '--output_dir', os.getcwd(), |
| '--chrome_src_path', chrome_src_path, |
| ] |
| if upload_to_partner_bucket: |
| webpages_playback_cmd.append('--upload_to_partner_bucket') |
| print('Running webpages_playback command:\n$ %s' % |
| ' '.join(webpages_playback_cmd)) |
| try: |
| subprocess.check_call(webpages_playback_cmd) |
| finally: |
| if xvfb_proc: |
| try: |
| xvfb_proc.kill() |
| except OSError as e: |
| print('Failed to kill xvfb process via Popen.kill();' |
| ' attempting `sudo kill`...') |
| try: |
| subprocess.check_call(['sudo', 'kill', '-9', str(xvfb_proc.pid)]) |
| except subprocess.CalledProcessError as e: |
| print('Failed to kill xvfb process via `sudo kill`;' |
| 'this may cause a hang.') |
| # Clean up any leftover browser instances. This can happen if there are |
| # telemetry crashes, processes are not always cleaned up appropriately by |
| # the webpagereplay and telemetry frameworks. |
| procs = subprocess.check_output(['ps', 'ax']).decode() |
| for line in procs.splitlines(): |
| if browser_executable in line: |
| print(line) |
| pid = line.strip().split(' ')[0] |
| if pid != str(os.getpid()) and not 'python' in line: |
| print('Kill browser process %s' % str(pid)) |
| try: |
| subprocess.check_call(['kill', '-9', str(pid)]) |
| except subprocess.CalledProcessError as e: |
| print(e) |
| else: |
| pass |
| if 'Xvfb' in line: |
| print(line) |
| pid = line.strip().split(' ')[0] |
| print('Kill Xvfb process %s' % str(pid)) |
| try: |
| subprocess.check_call(['sudo', 'kill', '-9', str(pid)]) |
| except subprocess.CalledProcessError as e: |
| print(e) |
| |
| src = os.path.join(os.getcwd(), 'playback', 'skps') |
| for f in os.listdir(src): |
| if f.endswith('.skp'): |
| shutil.copyfile(os.path.join(src, f), os.path.join(target_dir, f)) |
| print('Done running webpages_playback.') |
| |
| # 3. Copy over private SKPs from Google storage into the target_dir. |
| old_skps_dir = tempfile.mkdtemp() |
| new_skps_dir = tempfile.mkdtemp() |
| print('Copying non-generated SKPs from private GCS bucket...') |
| subprocess.check_call([ |
| 'gsutil', 'cp', os.path.join(PRIVATE_SKPS_GS, '*'), old_skps_dir]) |
| print('Updating non-generated SKP versions') |
| subprocess.check_call([ |
| dm_path, |
| '--config', 'skp', |
| '-w', new_skps_dir, |
| '--skps', old_skps_dir, |
| '--src', 'skp']) |
| |
| # DM creates artifacts in 2 directory level- one for the config and one for |
| # the output type. In this case it ends up creating ${temp_dir}/skp/skp/ |
| for f in os.listdir(os.path.join(new_skps_dir, 'skp', 'skp')): |
| if f.endswith('.skp'): |
| # Files generated by DM add a suffix based on the output type. |
| # For "xyz.skp" DM will create "xyz.skp.skp". We strip out |
| # the final ".skp" here. |
| shutil.copyfile( |
| os.path.join(new_skps_dir, 'skp', 'skp', f), |
| os.path.join(target_dir, f.replace('.skp.skp', '.skp'))) |
| shutil.rmtree(old_skps_dir) |
| shutil.rmtree(new_skps_dir) |
| |
| |
| def main(): |
| parser = argparse.ArgumentParser() |
| parser.add_argument('--target_dir', '-t', required=True) |
| args = parser.parse_args() |
| |
| # Obtain flags from create_and_upload via environment variables, since |
| # this script is called via `sk` and not directly. |
| chrome_src_path = getenv(CHROME_SRC_PATH_ENV_VAR) |
| browser_executable = getenv(BROWSER_EXECUTABLE_ENV_VAR) |
| upload_to_partner_bucket = getenv(UPLOAD_TO_PARTNER_BUCKET_ENV_VAR) == '1' |
| dm_path = getenv(DM_PATH_ENV_VAR) |
| |
| create_asset(chrome_src_path, browser_executable, args.target_dir, |
| upload_to_partner_bucket, dm_path) |
| |
| |
| if __name__ == '__main__': |
| main() |