| # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| |
| """Miscellaneous utilities needed by the Skia buildbot master.""" |
| |
| |
| import difflib |
| import httplib2 |
| import os |
| import re |
| |
| # requires Google APIs client library for Python; see |
| # https://code.google.com/p/google-api-python-client/wiki/Installation |
| from apiclient.discovery import build |
| from buildbot.scheduler import Dependent |
| from buildbot.scheduler import Scheduler |
| from buildbot.schedulers import timed |
| from buildbot.schedulers.filter import ChangeFilter |
| from config_private import TRY_SVN_BASEURL |
| from master import try_job_svn |
| from master import try_job_rietveld |
| from master.builders_pools import BuildersPools |
| from oauth2client.client import SignedJwtAssertionCredentials |
| |
| import builder_name_schema |
| import config_private |
| import os |
| import skia_vars |
| import subprocess |
| |
| |
| GATEKEEPER_NAME = 'GateKeeper' |
| |
| TRY_SCHEDULER_SVN = 'skia_try_svn' |
| TRY_SCHEDULER_RIETVELD = 'skia_try_rietveld' |
| TRY_SCHEDULERS = [TRY_SCHEDULER_SVN, TRY_SCHEDULER_RIETVELD] |
| TRY_SCHEDULERS_STR = '|'.join(TRY_SCHEDULERS) |
| |
| |
| def GetListFromEnvVar(name, splitstring=','): |
| """ Returns contents of an environment variable, as a list. |
| |
| If the environment variable is unset or set to empty-string, this returns |
| an empty list. |
| |
| name: string; name of the environment variable to read |
| splitstring: string with which to split the env var into list items |
| """ |
| unsplit = os.environ.get(name, None) |
| if unsplit: |
| return unsplit.split(',') |
| else: |
| return [] |
| |
| |
| def StringDiff(expected, actual): |
| """ Returns the diff between two multiline strings, as a multiline string.""" |
| return ''.join(difflib.unified_diff(expected.splitlines(1), |
| actual.splitlines(1))) |
| |
| |
| def _IndentStr(indent): |
| return ' ' * (indent + 1) |
| |
| |
| def ToString(obj, indent=0): |
| """ Returns a string representation of the given object. This differs from the |
| built-in string function in that it does not give memory locations. |
| |
| obj: the object to print. |
| indent: integer; the current indent level. |
| """ |
| if isinstance(obj, list): |
| return _ListToString(obj, indent) |
| elif isinstance(obj, dict): |
| return _DictToString(obj, indent) |
| elif isinstance(obj, tuple): |
| return _ListToString(obj, indent) |
| elif isinstance(obj, str): |
| return '\'%s\'' % obj |
| elif obj is None: |
| return 'None' |
| else: |
| return '<Object>' |
| |
| |
| def _ListToString(list_var, indent): |
| if not list_var: |
| return '[]' |
| indent_str = _IndentStr(indent) |
| val = '[\n' |
| indent += 1 |
| val += ''.join(['%s%s,\n' % (indent_str, ToString(elem, indent)) \ |
| for elem in list_var]) |
| indent -= 1 |
| indent_str = _IndentStr(indent - 1) |
| val += indent_str + ']' |
| return val |
| |
| |
| def _DictToString(d, indent): |
| if not d: |
| return '{}' |
| indent_str = _IndentStr(indent) |
| val = '{\n' |
| indent += 1 |
| val += ''.join(['%s%s: %s,\n' % (indent_str, ToString(k, indent), |
| ToString(d[k], indent)) \ |
| for k in sorted(d.keys())]) |
| indent -= 1 |
| indent_str = _IndentStr(indent - 1) |
| val += indent_str + '}' |
| return val |
| |
| |
| def FixGitSvnEmail(addr): |
| """ Git-svn tacks a git-svn-id onto email addresses. This function removes it. |
| |
| For example, "skia.buildbots@gmail.com@2bbb7eff-a529-9590-31e7-b0007b416f81" |
| becomes, "skia.buildbots@gmail.com". Addresses containing a single '@' will be |
| unchanged. |
| """ |
| return '@'.join(addr.split('@')[:2]) |
| |
| |
| class SkiaChangeFilter(ChangeFilter): |
| """Skia specific subclass of ChangeFilter.""" |
| |
| def __init__(self, builders, **kwargs): |
| self._builders = builders |
| ChangeFilter.__init__(self, **kwargs) |
| |
| def filter_change(self, change): |
| """Overrides ChangeFilter.filter_change to pass builders to filter_fn. |
| |
| The code has been copied from |
| http://buildbot.net/buildbot/docs/0.8.3/reference/buildbot.schedulers.filter-pysrc.html#ChangeFilter |
| with one change: We pass a sequence of builders to the filter function. |
| """ |
| if self.filter_fn is not None and not self.filter_fn(change, |
| self._builders): |
| return False |
| for (filt_list, filt_re, filt_fn, chg_attr) in self.checks: |
| chg_val = getattr(change, chg_attr, '') |
| if filt_list is not None and chg_val not in filt_list: |
| return False |
| if filt_re is not None and ( |
| chg_val is None or not filt_re.match(chg_val)): |
| return False |
| if filt_fn is not None and not filt_fn(chg_val): |
| return False |
| return True |
| |
| |
| def _AssertValidString(var, varName='[unknown]'): |
| """Raises an exception if a var is not a valid string. |
| |
| A string is considered valid if it is not None, is not the empty string and is |
| not just whitespace. |
| |
| Args: |
| var: the variable to validate |
| varName: name of the variable, for error reporting |
| """ |
| if not isinstance(var, str): |
| raise Exception('variable "%s" is not a string' % varName) |
| if not var: |
| raise Exception('variable "%s" is empty' % varName) |
| if var.isspace(): |
| raise Exception('variable "%s" is whitespace' % varName) |
| |
| |
| def _AssertValidStringList(var, varName='[unknown]'): |
| """Raises an exception if var is not a list of valid strings. |
| |
| A list is considered valid if it is either empty or if it contains at |
| least one item and each item it contains is also a valid string. |
| |
| Args: |
| var: the variable to validate |
| varName: name of the variable, for error reporting |
| """ |
| if not isinstance(var, list): |
| raise Exception('variable "%s" is not a list' % varName) |
| for index, item in zip(range(len(var)), var): |
| _AssertValidString(item, '%s[%d]' % (varName, index)) |
| |
| |
| def FileBug(summary, description, owner=None, ccs=None, labels=None): |
| """Files a bug to the Skia issue tracker. |
| |
| Args: |
| summary: a single-line string to use as the issue summary |
| description: a multiline string to use as the issue description |
| owner: email address of the issue owner (as a string), or None if unknown |
| ccs: email addresses (list of strings) to CC on the bug |
| labels: labels (list of strings) to apply to the bug |
| |
| Returns: |
| A representation of the issue tracker issue that was filed or raises an |
| exception if there was a problem. |
| """ |
| project_id = 'skia' # This is the project name: skia |
| key_file = 'key.p12' # Key file from the API console, renamed to key.p12 |
| service_acct = ('352371350305-b3u8jq5sotdh964othi9ntg9d0pelu77' |
| '@developer.gserviceaccount.com') # Created with the key |
| result = {} |
| if not ccs: |
| ccs = [] |
| if not labels: |
| labels = [] |
| |
| if owner is not None: # owner can be None |
| _AssertValidString(owner, 'owner') |
| _AssertValidString(summary, 'summary') |
| _AssertValidString(description, 'description') |
| _AssertValidStringList(ccs, 'ccs') |
| _AssertValidStringList(labels, 'labels') |
| |
| f = file(key_file, 'rb') |
| key = f.read() |
| f.close() |
| |
| # Create an httplib2.Http object to handle the HTTP requests and authorize |
| # it with the credentials. |
| credentials = SignedJwtAssertionCredentials( |
| service_acct, |
| key, |
| scope='https://www.googleapis.com/auth/projecthosting') |
| http = httplib2.Http() |
| http = credentials.authorize(http) |
| |
| service = build("projecthosting", "v2", http=http) |
| |
| # Insert a new issue into the project. |
| body = { |
| 'summary': summary, |
| 'description': description |
| } |
| |
| insertparams = { |
| 'projectId': project_id, |
| 'sendEmail': 'true' |
| } |
| |
| if owner is not None: |
| owner_value = { |
| 'name': owner |
| } |
| body['owner'] = owner_value |
| |
| cc_values = [] |
| for cc in ccs: |
| cc_values.append({'name': cc}) |
| body['cc'] = cc_values |
| |
| body['labels'] = labels |
| |
| insertparams['body'] = body |
| |
| request = service.issues().insert(**insertparams) |
| result = request.execute() |
| |
| return result |
| |
| |
| # Skip buildbot runs of a CL if its commit log message contains the following |
| # substring. |
| SKIP_BUILDBOT_SUBSTRING = '(SkipBuildbotRuns)' |
| |
| # If the below regex is found in a CL's commit log message, only run the |
| # builders specified therein. |
| RUN_BUILDERS_REGEX = '\(RunBuilders:(.+)\)' |
| RUN_BUILDERS_RE_COMPILED = re.compile(RUN_BUILDERS_REGEX) |
| |
| |
| def CapWordsToUnderscores(string): |
| """ Converts a string containing capitalized words to one in which all |
| characters are lowercase and words are separated by underscores. |
| |
| Examples: |
| 'NexusS' becomes 'nexus_s' |
| 'Nexus10' becomes 'nexus_10' |
| |
| string: string; string to manipulate. |
| """ |
| name_parts = [] |
| for part in re.split('(\d+)', string): |
| if re.match('(\d+)', part): |
| name_parts.append(part) |
| else: |
| name_parts.extend(re.findall('[A-Z][a-z]*', part)) |
| return '_'.join([part.lower() for part in name_parts]) |
| |
| |
| def UnderscoresToCapWords(string): |
| """ Converts a string lowercase words separated by underscores to one in which |
| words are capitalized and not separated by underscores. |
| |
| Examples: |
| 'nexus_s' becomes 'NexusS' |
| 'nexus_10' becomes 'Nexus10' |
| |
| string: string; string to manipulate. |
| """ |
| name_parts = string.split('_') |
| return ''.join([part.title() for part in name_parts]) |
| |
| |
| # Since we can't modify the existing Helper class, we subclass it here, |
| # overriding the necessary parts to get things working as we want. |
| class SkiaHelper(object): |
| def __init__(self, defaults): |
| self._defaults = defaults |
| self._builders = [] |
| self._factories = {} |
| self._schedulers = {} |
| |
| def Builder(self, name, factory, gatekeeper=None, scheduler=None, |
| builddir=None, auto_reboot=False, notify_on_missing=False): |
| # Override the category with the first two parts of the builder name. |
| name_parts = name.split(builder_name_schema.BUILDER_NAME_SEP) |
| category = name_parts[0] |
| subcategory = name_parts[1] if len(name_parts) > 1 else 'default' |
| full_category = '|'.join((category, subcategory)) |
| self._builders.append({'name': name, |
| 'factory': factory, |
| 'gatekeeper': gatekeeper, |
| 'schedulers': scheduler.split('|'), |
| 'builddir': builddir, |
| 'category': full_category, |
| 'auto_reboot': auto_reboot, |
| 'notify_on_missing': notify_on_missing}) |
| |
| def PeriodicScheduler(self, name, minute=0, hour='*', dayOfMonth='*', |
| month='*', dayOfWeek='*'): |
| """Helper method for the Periodic scheduler.""" |
| if name in self._schedulers: |
| raise ValueError('Scheduler %s already exists' % name) |
| self._schedulers[name] = {'type': 'PeriodicScheduler', |
| 'builders': [], |
| 'minute': minute, |
| 'hour': hour, |
| 'dayOfMonth': dayOfMonth, |
| 'month': month, |
| 'dayOfWeek': dayOfWeek} |
| |
| def Dependent(self, name, parent): |
| if name in self._schedulers: |
| raise ValueError('Scheduler %s already exists' % name) |
| self._schedulers[name] = {'type': 'Dependent', |
| 'parent': parent, |
| 'builders': []} |
| |
| def Factory(self, name, factory): |
| if name in self._factories: |
| raise ValueError('Factory %s already exists' % name) |
| self._factories[name] = factory |
| |
| def Scheduler(self, name, treeStableTimer=60, categories=None): |
| if name in self._schedulers: |
| raise ValueError('Scheduler %s already exists' % name) |
| self._schedulers[name] = {'type': 'Scheduler', |
| 'treeStableTimer': treeStableTimer, |
| 'builders': [], |
| 'categories': categories} |
| |
| def TryJobSubversion(self, name): |
| """ Adds a Subversion-based try scheduler. """ |
| if name in self._schedulers: |
| raise ValueError('Scheduler %s already exists' % name) |
| self._schedulers[name] = {'type': 'TryJobSubversion', 'builders': []} |
| |
| def TryJobRietveld(self, name): |
| """ Adds a Rietveld-based try scheduler. """ |
| if name in self._schedulers: |
| raise ValueError('Scheduler %s already exists' % name) |
| self._schedulers[name] = {'type': 'TryJobRietveld', 'builders': []} |
| |
| def Update(self, c): |
| for builder in self._builders: |
| # Update the schedulers with the builder. |
| schedulers = builder['schedulers'] |
| if schedulers: |
| for scheduler in schedulers: |
| self._schedulers[scheduler]['builders'].append(builder['name']) |
| |
| # Construct the category. |
| categories = [] |
| if builder.get('category', None): |
| categories.append(builder['category']) |
| if builder.get('gatekeeper', None): |
| categories.extend(builder['gatekeeper'].split('|')) |
| category = '|'.join(categories) |
| |
| # Append the builder to the list. |
| new_builder = {'name': builder['name'], |
| 'factory': self._factories[builder['factory']], |
| 'category': category, |
| 'auto_reboot': builder['auto_reboot']} |
| if builder['builddir']: |
| new_builder['builddir'] = builder['builddir'] |
| c['builders'].append(new_builder) |
| |
| c['builders'].sort(key=lambda builder: builder['name']) |
| |
| # Process the main schedulers. |
| for s_name in self._schedulers: |
| scheduler = self._schedulers[s_name] |
| if scheduler['type'] == 'Scheduler': |
| def filter_fn(change, builders): |
| """Filters out if change.comments contains certain keywords. |
| |
| The change is filtered out if the commit message contains: |
| * SKIP_BUILDBOT_SUBSTRING or |
| * RUN_BUILDERS_REGEX when the scheduler does not contain any of the |
| specified builders |
| |
| Args: |
| change: An instance of changes.Change. |
| builders: Sequence of strings. The builders that are run by this |
| scheduler. |
| |
| Returns: |
| If the change should be filtered out (i.e. not run by the buildbot |
| code) then False is returned else True is returned. |
| """ |
| if SKIP_BUILDBOT_SUBSTRING in change.comments: |
| return False |
| match_obj = RUN_BUILDERS_RE_COMPILED.search(change.comments) |
| if builders and match_obj: |
| for builder_to_run in match_obj.group(1).split(','): |
| if builder_to_run.strip() in builders: |
| break |
| else: |
| return False |
| return True |
| |
| skia_change_filter = SkiaChangeFilter( |
| builders=scheduler['builders'], |
| branch=skia_vars.GetGlobalVariable('master_branch_name'), |
| filter_fn=filter_fn) |
| |
| instance = Scheduler(name=s_name, |
| treeStableTimer=scheduler['treeStableTimer'], |
| builderNames=scheduler['builders'], |
| change_filter=skia_change_filter) |
| c['schedulers'].append(instance) |
| self._schedulers[s_name]['instance'] = instance |
| |
| # Process the periodic schedulers. |
| for s_name in self._schedulers: |
| scheduler = self._schedulers[s_name] |
| if scheduler['type'] == 'PeriodicScheduler': |
| instance = timed.Nightly( |
| name=s_name, |
| branch=skia_vars.GetGlobalVariable('master_branch_name'), |
| builderNames=scheduler['builders'], |
| minute=scheduler['minute'], |
| hour=scheduler['hour'], |
| dayOfMonth=scheduler['dayOfMonth'], |
| month=scheduler['month'], |
| dayOfWeek=scheduler['dayOfWeek']) |
| c['schedulers'].append(instance) |
| self._schedulers[s_name]['instance'] = instance |
| |
| # Process the Rietveld-based try schedulers. |
| for s_name in self._schedulers: |
| scheduler = self._schedulers[s_name] |
| if scheduler['type'] == 'TryJobRietveld': |
| pools = BuildersPools(s_name) |
| pools[s_name].extend(scheduler['builders']) |
| instance = try_job_rietveld.TryJobRietveld( |
| name=s_name, |
| pools=pools, |
| last_good_urls={'skia': None}, |
| code_review_sites={'skia': config_private.CODE_REVIEW_SITE}, |
| project='skia') |
| c['schedulers'].append(instance) |
| self._schedulers[s_name]['instance'] = instance |
| |
| # Process the svn-based try schedulers. |
| for s_name in self._schedulers: |
| scheduler = self._schedulers[s_name] |
| if scheduler['type'] == 'TryJobSubversion': |
| pools = BuildersPools(s_name) |
| pools[s_name].extend(scheduler['builders']) |
| instance = try_job_svn.TryJobSubversion( |
| name=s_name, |
| svn_url=TRY_SVN_BASEURL, |
| last_good_urls={'skia': None}, |
| code_review_sites={'skia': config_private.CODE_REVIEW_SITE}, |
| pools=pools) |
| c['schedulers'].append(instance) |
| self._schedulers[s_name]['instance'] = instance |
| |
| # Process the dependent schedulers. |
| for s_name in self._schedulers: |
| scheduler = self._schedulers[s_name] |
| if scheduler['type'] == 'Dependent': |
| instance = Dependent( |
| s_name, |
| self._schedulers[scheduler['parent']]['instance'], |
| scheduler['builders']) |
| c['schedulers'].append(instance) |
| self._schedulers[s_name]['instance'] = instance |
| |
| |
| def CanMergeBuildRequests(req1, req2): |
| """ Determine whether or not two BuildRequests can be merged. Note that the |
| call to buildbot.sourcestamp.SourceStamp.canBeMergedWith() is conspicuously |
| missing. This is because that method verifies that: |
| 1. req1.source.repository == req2.source.repository |
| 2. req1.source.project == req2.source.project |
| 3. req1.source.branch == req2.source.branch |
| 4. req1.patch == None and req2.patch = None |
| 5. (req1.source.changes and req2.source.changes) or \ |
| (not req1.source.changes and not req2.source.changes and \ |
| req1.source.revision == req2.source.revision) |
| |
| Of the above, we want 1, 2, 3, and 5. |
| Instead of 4, we want to make sure that neither request is a Trybot request. |
| """ |
| # Verify that the repositories are the same (#1 above). |
| if req1.source.repository != req2.source.repository: |
| return False |
| |
| # Verify that the projects are the same (#2 above). |
| if req1.source.project != req2.source.project: |
| return False |
| |
| # Verify that the branches are the same (#3 above). |
| if req1.source.branch != req2.source.branch: |
| return False |
| |
| # If either is a try request, don't merge (#4 above). |
| if (builder_name_schema.IsTrybot(req1.buildername) or |
| builder_name_schema.IsTrybot(req2.buildername)): |
| return False |
| |
| # Verify that either: both requests are associated with changes OR neither |
| # request is associated with a change but the revisions match (#5 above). |
| if req1.source.changes and not req2.source.changes: |
| return False |
| if not req1.source.changes and req2.source.changes: |
| return False |
| if not (req1.source.changes and req2.source.changes): |
| if req1.source.revision != req2.source.revision: |
| return False |
| |
| return True |
| |
| |
| def get_current_revision(): |
| """Obtain the checked-out buildbot code revision.""" |
| checkout_dir = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir) |
| if os.path.isdir(os.path.join(checkout_dir, '.git')): |
| return subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip() |
| elif os.path.isdir(os.path.join(checkout_dir, '.svn')): |
| return subprocess.check_output(['svnversion', '.']).strip() |
| raise Exception('Unable to determine version control system.') |
