# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.


""" Monkeypatches to override upstream code. """


from buildbot.changes.gitpoller import GitPoller
from buildbot.process.properties import Properties
from buildbot.schedulers.trysched import BadJobfile
from buildbot.status.builder import EXCEPTION, FAILURE
from buildbot.status.web import base as webstatus_base
from buildbot.status.web.status_json import BuilderJsonResource
from buildbot.status.web.status_json import BuildersJsonResource
from buildbot.status.web.status_json import ChangeSourcesJsonResource
from buildbot.status.web.status_json import JsonResource
from buildbot.status.web.status_json import JsonStatusResource
from buildbot.status.web.status_json import MetricsJsonResource
from buildbot.status.web.status_json import ProjectJsonResource
from buildbot.status.web.status_json import SlavesJsonResource
from master import build_utils
from master import chromium_notifier
from master import gatekeeper
from master import try_job_base
from master import try_job_rietveld
from master import try_job_svn
from master.try_job_base import text_to_dict
from twisted.internet import defer
from twisted.python import log
from twisted.web import server
from webstatus import builder_statuses

import builder_name_schema
import config_private
import json
import master_revision
import re
import slave_hosts_cfg
import slaves_cfg
import skia_vars
import utils


# The following users are allowed to run trybots even though they do not have
# accounts in google.com or chromium.org
TRYBOTS_REQUESTER_WHITELIST = [
    'henrik.smiding@intel.com',
    'kkinnunen@nvidia.com',
    'cdalton@nvidia.com',
    'ravimist@gmail.com'
]


################################################################################
############################# Trybot Monkeypatches #############################
################################################################################


@defer.deferredGenerator
def SubmitTryJobChanges(self, changes):
  """ Override of SVNPoller.submit_changes:
  http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/master/try_job_svn.py?view=markup

  We modify it so that the patch file url is added to the build properties.
  This allows the slave to download the patch directly rather than receiving
  it from the master.
  """
  for chdict in changes:
    # pylint: disable=E1101
    parsed = self.parent.parse_options(text_to_dict(chdict['comments']))

    # 'fix' revision.
    # LKGR must be known before creating the change object.
    wfd = defer.waitForDeferred(self.parent.get_lkgr(parsed))
    yield wfd
    wfd.getResult()

    wfd = defer.waitForDeferred(self.master.addChange(
      author=','.join(parsed['email']),
      revision=parsed['revision'],
      comments='',
      properties={'patch_file_url': chdict['repository'] + '/' + \
                      chdict['files'][0]}))
    yield wfd
    change = wfd.getResult()

    self.parent.addChangeInner(chdict['files'], parsed, change.number)

try_job_svn.SVNPoller.submit_changes = SubmitTryJobChanges


def TryJobCreateBuildset(self, ssid, parsed_job):
  """ Override of TryJobBase.create_buildset:
  http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/master/try_job_base.py?view=markup

  We modify it to verify that the requested builders are in the builder pool for
  this try scheduler. This prevents try requests from running on builders which
  are not registered as trybots. This apparently isn't a problem for Chromium
  since they use a separate try master.
  """
  log.msg('Creating try job(s) %s' % ssid)
  result = None
  for builder in parsed_job['bot']:
    if builder in self.pools[self.name]:
      result = self.addBuildsetForSourceStamp(ssid=ssid,
          reason=parsed_job['name'],
          external_idstring=parsed_job['name'],
          builderNames=[builder],
          properties=self.get_props(builder, parsed_job))
    else:
      log.msg('Scheduler: %s rejecting try job for builder: %s not in %s' % (
                  self.name,
                  builder,
                  self.pools[self.name]))
  log.msg('Returning buildset: %s' % result)
  return result

try_job_base.TryJobBase.create_buildset = TryJobCreateBuildset


def HtmlResourceRender(self, request):
  """ Override of buildbot.status.web.base.HtmlResource.render:
  http://src.chromium.org/viewvc/chrome/trunk/tools/build/third_party/buildbot_8_4p1/buildbot/status/web/base.py?view=markup

  We modify it to pass additional variables on to the web status pages, and
  remove the "if False" section.
  """
  # tell the WebStatus about the HTTPChannel that got opened, so they
  # can close it if we get reconfigured and the WebStatus goes away.
  # They keep a weakref to this, since chances are good that it will be
  # closed by the browser or by us before we get reconfigured. See
  # ticket #102 for details.
  if hasattr(request, "channel"):
    # web.distrib.Request has no .channel
    request.site.buildbot_service.registerChannel(request.channel)

  ctx = self.getContext(request)

  ############################## Added by borenet ##############################
  status = self.getStatus(request)
  all_builders = status.getBuilderNames()
  all_full_category_names = set()
  all_categories = set()
  all_subcategories = set()
  subcategories_by_category = {}
  for builder_name in all_builders:
    category_full = status.getBuilder(builder_name).category or 'default'
    all_full_category_names.add(category_full)
    category_split = category_full.split('|')
    category = category_split[0]
    subcategory = category_split[1] if len(category_split) > 1 else 'default'
    all_categories.add(category)
    all_subcategories.add(subcategory)
    if not subcategories_by_category.get(category):
      subcategories_by_category[category] = []
    if not subcategory in subcategories_by_category[category]:
      subcategories_by_category[category].append(subcategory)

  ctx['tree_status_baseurl'] = \
      skia_vars.GetGlobalVariable('tree_status_baseurl')

  ctx['all_full_category_names'] = sorted(list(all_full_category_names))
  ctx['all_categories'] = sorted(list(all_categories))
  ctx['all_subcategories'] = sorted(list(all_subcategories))
  ctx['subcategories_by_category'] = subcategories_by_category
  ctx['default_refresh'] = \
      skia_vars.GetGlobalVariable('default_webstatus_refresh')
  ctx['skia_repo'] = config_private.SKIA_GIT_URL

  active_master = config_private.Master.get_active_master()
  ctx['internal_port'] = active_master.master_port
  ctx['external_port'] = active_master.master_port_alt
  ctx['title_url'] = config_private.Master.Skia.project_url
  ctx['slave_hosts_cfg'] = slave_hosts_cfg.SLAVE_HOSTS
  ctx['slaves_cfg'] = slaves_cfg.SLAVES

  ctx['active_master_name'] = active_master.project_name
  ctx['master_revision'] = utils.get_current_revision()
  ctx['master_running_revision'] = active_master.running_revision
  ctx['master_launch_datetime'] = active_master.launch_datetime
  ctx['is_internal_view'] = request.host.port == ctx['internal_port']
  ctx['masters'] = []
  for master in config_private.Master.valid_masters:
    ctx['masters'].append({
      'name': master.project_name,
      'host': master.master_host,
      'internal_port': master.master_port,
      'external_port': master.master_port_alt,
    })
  ##############################################################################

  d = defer.maybeDeferred(lambda : self.content(request, ctx))
  def handle(data):
    if isinstance(data, unicode):
      data = data.encode("utf-8")
    request.setHeader("content-type", self.contentType)
    if request.method == "HEAD":
      request.setHeader("content-length", len(data))
      return ''
    return data
  d.addCallback(handle)
  def ok(data):
    request.write(data)
    request.finish()
  def fail(f):
    request.processingFailed(f)
    return None # processingFailed will log this for us
  d.addCallbacks(ok, fail)
  return server.NOT_DONE_YET

webstatus_base.HtmlResource.render = HtmlResourceRender


class TryBuildersJsonResource(JsonResource):
  """ Clone of buildbot.status.web.status_json.BuildersJsonResource:
  http://src.chromium.org/viewvc/chrome/trunk/tools/build/third_party/buildbot_8_4p1/buildbot/status/web/status_json.py?view=markup

  We add filtering to display only the try builders.
  """
  help = """List of all the try builders defined on a master."""
  pageTitle = 'Builders'

  def __init__(self, status, include_only_cq_trybots=False):
    JsonResource.__init__(self, status)
    for builder_name in self.status.getBuilderNames():
      if builder_name_schema.IsTrybot(builder_name) and (
          not include_only_cq_trybots or builder_name in slaves_cfg.CQ_TRYBOTS):
        self.putChild(builder_name,
                      BuilderJsonResource(status,
                                          status.getBuilder(builder_name)))


class CQRequiredStepsJsonResource(JsonResource):
  help = 'List the steps which cannot fail on the commit queue.'
  pageTitle = 'CQ Required Steps'

  def asDict(self, request):
    return {'cq_required_steps':
                skia_vars.GetGlobalVariable('cq_required_steps')}


def JsonStatusResourceInit(self, status):
  """ Override of buildbot.status.web.status_json.JsonStatusResource.__init__:
  http://src.chromium.org/viewvc/chrome/trunk/tools/build/third_party/buildbot_8_4p1/buildbot/status/web/status_json.py?view=markup

  We add trybots, cqtrybots, cq_required_steps (details below).
  """
  JsonResource.__init__(self, status)
  self.level = 1
  self.putChild('builders', BuildersJsonResource(status))
  self.putChild('change_sources', ChangeSourcesJsonResource(status))
  self.putChild('project', ProjectJsonResource(status))
  self.putChild('slaves', SlavesJsonResource(status))
  self.putChild('metrics', MetricsJsonResource(status))

  ############################## Added by borenet ##############################
  # Added to address: https://code.google.com/p/skia/issues/detail?id=1134
  self.putChild('trybots', TryBuildersJsonResource(status))
  ##############################################################################
  ############################## Added by rmistry ##############################
  # Added to have a place to get the list of trybots run by the CQ.
  self.putChild('cqtrybots',
                TryBuildersJsonResource(status, include_only_cq_trybots=True))
  ##############################################################################
  ############################## Added by borenet ##############################
  # Added to have a place to get the list of steps which cannot fail on the CQ.
  self.putChild('cq_required_steps', CQRequiredStepsJsonResource(status))
  ##############################################################################

  ############################## Added by borenet ##############################
  # Added to have a way to determine which code revision the master is running.
  self.putChild('master_revision',
                master_revision.MasterCheckedOutRevisionJsonResource(status))
  running_rev = config_private.Master.get_active_master().running_revision
  self.putChild('master_running_revision',
                master_revision.MasterRunningRevisionJsonResource(
                    status=status, running_revision=running_rev))

  # This page gives the result of the most recent build for each builder.
  self.putChild('builder_statuses',
                builder_statuses.BuilderStatusesJsonResource(status))
  ##############################################################################

  # This needs to be called before the first HelpResource().body call.
  self.hackExamples()

JsonStatusResource.__init__ = JsonStatusResourceInit


@defer.deferredGenerator
def TryJobRietveldSubmitJobs(self, jobs):
  """ Override of master.try_job_rietveld.TryJobRietveld.SubmitJobs:
  http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/master/try_job_rietveld.py?view=markup

  We modify it to include "baseurl" as a build property.
  """
  log.msg('TryJobRietveld.SubmitJobs: %s' % json.dumps(jobs, indent=2))
  for job in jobs:
    try:
      # Gate the try job on the user that requested the job, not the one that
      # authored the CL.
      # pylint: disable=W0212
      ########################## Added by rmistry ##########################
      if (job.get('requester') and not job['requester'].endswith('@google.com')
          and not job['requester'].endswith('@chromium.org')
          and not job['requester'] in TRYBOTS_REQUESTER_WHITELIST):
        # Reject the job only if the requester has an email not ending in
        # google.com or chromium.org
        raise BadJobfile(
            'TryJobRietveld rejecting job from %s' % job['requester'])
      ######################################################################
      ########################## Added by borenet ##########################
      if not (job.get('baseurl') and
              config_private.Master.Skia.project_name.lower() in
                  job['baseurl']):
        raise BadJobfile('TryJobRietveld rejecting job with unknown baseurl: %s'
                         % job.get('baseurl'))
      ######################################################################
      if job['email'] != job['requester']:
        # Note the fact the try job was requested by someone else in the
        # 'reason'.
        job['reason'] = job.get('reason') or ''
        if job['reason']:
          job['reason'] += '; '
        job['reason'] += "This CL was triggered by %s" % job['requester']

      options = {
          'bot': {job['builder']: job['tests']},
          'email': [job['email']],
          'project': [self._project],
          'try_job_key': job['key'],
      }
      # Transform some properties as is expected by parse_options().
      for key in (
          ########################## Added by borenet ##########################
          'baseurl',
          ######################################################################
          'name', 'user', 'root', 'reason', 'clobber', 'patchset',
          'issue', 'requester', 'revision'):
        options[key] = [job[key]]

      # Now cleanup the job dictionary and submit it.
      cleaned_job = self.parse_options(options)

      wfd = defer.waitForDeferred(self.get_lkgr(cleaned_job))
      yield wfd
      wfd.getResult()

      wfd = defer.waitForDeferred(self.master.addChange(
          author=','.join(cleaned_job['email']),
          # TODO(maruel): Get patchset properties to get the list of files.
          # files=[],
          revision=cleaned_job['revision'],
          comments=''))
      yield wfd
      changeids = [wfd.getResult().number]

      wfd = defer.waitForDeferred(self.SubmitJob(cleaned_job, changeids))
      yield wfd
      wfd.getResult()
    except BadJobfile, e:
      # We need to mark it as failed otherwise it'll stay in the pending
      # state. Simulate a buildFinished event on the build.
      if not job.get('key'):
        log.err(
            'Got %s for issue %s but not key, not updating Rietveld' %
            (e, job.get('issue')))
        continue
      log.err(
          'Got %s for issue %s, updating Rietveld' % (e, job.get('issue')))
      for service in self.master.services:
        if service.__class__.__name__ == 'TryServerHttpStatusPush':
          # pylint: disable=W0212,W0612
          build = {
            'properties': [
              ('buildername', job.get('builder'), None),
              ('buildnumber', -1, None),
              ('issue', job['issue'], None),
              ('patchset', job['patchset'], None),
              ('project', self._project, None),
              ('revision', '', None),
              ('slavename', '', None),
              ('try_job_key', job['key'], None),
            ],
            'reason': job.get('reason', ''),
            # Use EXCEPTION until SKIPPED results in a non-green try job
            # results on Rietveld.
            'results': EXCEPTION,
          }
          ########################## Added by rmistry #########################
          # Do not update Rietveld to mark the try job request as failed.
          # See https://code.google.com/p/chromium/issues/detail?id=224014 for
          # more context.
          # service.push('buildFinished', build=build)
          #####################################################################
          break

try_job_rietveld.TryJobRietveld.SubmitJobs = TryJobRietveldSubmitJobs


def TryJobBaseGetProps(self, builder, options):
  """ Override of try_job_base.TryJobBase.get_props:
  http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/master/try_job_base.py?view=markup

  We modify it to add "baseurl".
  """
  keys = (
############################### Added by borenet ###############################
    'baseurl',
################################################################################
    'clobber',
    'issue',
    'patchset',
    'requester',
    'rietveld',
    'root',
    'try_job_key',
  )
  # All these settings have no meaning when False or not set, so don't set
  # them in that case.
  properties = dict((i, options[i]) for i in keys if options.get(i))
  properties['testfilter'] = options['bot'].get(builder, None)
  # pylint: disable=W0212
  props = Properties()
  props.updateFromProperties(self.properties)
  props.update(properties, self._PROPERTY_SOURCE)
  return props

try_job_base.TryJobBase.get_props = TryJobBaseGetProps


def TryJobRietveldConstructor(
    self, name, pools, properties=None, last_good_urls=None,
    code_review_sites=None, project=None):
  try_job_base.TryJobBase.__init__(self, name, pools, properties,
                                   last_good_urls, code_review_sites)
  # pylint: disable=W0212
  endpoint = self._GetRietveldEndPointForProject(code_review_sites, project)
############################### Added by rmistry ###############################
  # rmistry: Adding '&master=tryserver.skia' to the endpoint to help filter the
  # number of pending try patchsets returned. More details are in
  # https://code.google.com/p/skia/issues/detail?id=2659
  endpoint += '&master=tryserver.skia'
  # rmistry: Increased the polling time from 10 seconds to 1 min because 10
  # seconds is too short for us. The RietveldPoller stops working if the time is
  # too short.
  # pylint: disable=W0212
  self._poller = try_job_rietveld._RietveldPoller(endpoint, interval=60)
################################################################################
  # pylint: disable=W0212
  self._valid_users = try_job_rietveld._ValidUserPoller(interval=12 * 60 * 60)
  self._project = project
  log.msg('TryJobRietveld created, get_pending_endpoint=%s '
          'project=%s' % (endpoint, project))

try_job_rietveld.TryJobRietveld.__init__ = TryJobRietveldConstructor


class SkiaGateKeeper(gatekeeper.GateKeeper):

  def isInterestingBuilder(self, builder_status):
    """ Override of gatekeeper.GateKeeper.isInterestingBuilder:
    http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/master/gatekeeper.py?view=markup

    We modify it to actually check whether the builder should be considered by
    the GateKeeper, as indicated in its category name.
    """

    ret = (not builder_name_schema.IsTrybot(builder_status.getName()) and
           chromium_notifier.ChromiumNotifier.isInterestingBuilder(self,
               builder_status))
    log.msg('[gatekeeper-debug2] ======================')
    log.msg('[gatekeeper-debug2] is not trybot: %s' % (
        not builder_name_schema.IsTrybot(builder_status.getName())))
    log.msg('[gatekeeper-debug2] isInterestingBuilder: %s' % (
        chromium_notifier.ChromiumNotifier.isInterestingBuilder(
            self, builder_status)))
    log.msg('[gatekeeper-debug2] builder_status.getName(): %s' % (
        builder_status.getName()))
    log.msg('[gatekeeper-debug2] ret: %s' % ret)
    log.msg('[gatekeeper-debug2] ======================')
    return ret

  def isInterestingStep(self, build_status, step_status, results):
    """ Override of gatekeeper.GateKeeper.isInterestingStep:
    http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/master/gatekeeper.py?view=markup

    We modify it to comment out the SVN revision comparision to determine if the
    current build is older because Skia uses commit hashes.
    """
    # If we have not failed, or are not interested in this builder,
    # then we have nothing to do.
    if results[0] != FAILURE:
      return False

    # Check if the slave is still alive. We should not close the tree for
    # inactive slaves.
    slave_name = build_status.getSlavename()
    if slave_name in self.master_status.getSlaveNames():
      # @type self.master_status: L{buildbot.status.builder.Status}
      # @type self.parent: L{buildbot.master.BuildMaster}
      # @rtype getSlave(): L{buildbot.status.builder.SlaveStatus}
      slave_status = self.master_status.getSlave(slave_name)
      if slave_status and not slave_status.isConnected():
        log.msg('[gatekeeper] Slave %s was disconnected, '
                'not closing the tree' % slave_name)
        return False

    # If the previous build step failed with the same result, we don't care
    # about this step.
    previous_build_status = build_status.getPreviousBuild()
    if previous_build_status:
      step_name = self.getName(step_status)
      step_type = self.getGenericName(step_name)
      previous_steps = [step for step in previous_build_status.getSteps()
                        if self.getGenericName(self.getName(step)) == step_type]
      if len(previous_steps) == 1:
        if previous_steps[0].getResults()[0] == FAILURE:
          log.msg('[gatekeeper] Slave %s failed, but previously failed on '
                  'the same step (%s). So not closing tree.' % (
                      (step_name, slave_name)))
          return False
      else:
        log.msg('[gatekeeper] len(previous_steps) == %d which is weird' %
                len(previous_steps))

    # If check_revisions=False that means that the tree closure request is
    # coming from nightly scheduled bots, that need not necessarily have the
    # revision info.
    if not self.check_revisions:
      return True

    # If we don't have a version stamp nor a blame list, then this is most
    # likely a build started manually, and we don't want to close the
    # tree.
    latest_revision = build_utils.getLatestRevision(build_status)
    if not latest_revision or not build_status.getResponsibleUsers():
      log.msg('[gatekeeper] Slave %s failed, but no version stamp, '
              'so skipping.' % slave_name)
      return False

    # If the tree is open, we don't want to close it again for the same
    # revision, or an earlier one in case the build that just finished is a
    # slow one and we already fixed the problem and manually opened the tree.
    ############################### Added by rmistry ###########################
    # rmistry: Commenting out the below SVN revision comparision because Skia
    # uses commit hashes.
    # TODO(rmistry): Figure out how to ensure that previous builds do not close
    # the tree again.
    #
    # if latest_revision <= self._last_closure_revision:
    #   log.msg('[gatekeeper] Slave %s failed, but we already closed it '
    #           'for a previous revision (old=%s, new=%s)' % (
    #               slave_name, str(self._last_closure_revision),
    #               str(latest_revision)))
    #   return False
    ###########################################################################

    log.msg('[gatekeeper] Decided to close tree because of slave %s '
            'on revision %s' % (slave_name, str(latest_revision)))

    # Up to here, in theory we'd check if the tree is closed but this is too
    # slow to check here. Instead, take a look only when we want to close the
    # tree.
    return True


# Fix try_job_base.TryJobBase._EMAIL_VALIDATOR to handle *.info. This was fixed
# in https://codereview.chromium.org/216293005 but we need this monkeypatch to
# pick it up without a DEPS roll.
try_job_base.TryJobBase._EMAIL_VALIDATOR = re.compile(
    r'[a-zA-Z0-9][a-zA-Z0-9\.\+\-\_]*@[a-zA-Z0-9\.\-]+\.[a-zA-Z]{2,}$')


# Add logging to GitPoller._stop_on_failure
def _stop_on_failure(self, f):
  "utility method to stop the service when a failure occurs"
  log.err('GitPoller stopping due to failure: %s' % str(f))
  if self.running:
    d = defer.maybeDeferred(lambda : self.stopService())
    d.addErrback(log.err, 'while stopping broken GitPoller service')
  return f

GitPoller._stop_on_failure = _stop_on_failure