depot_tools/recipes/recipe_modules/bot_update/resources/bot_update.py

#!/usr/bin/env python
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# TODO(hinoka): Use logging.

import cStringIO
import codecs
import copy
import ctypes
import json
import optparse
import os
import pprint
import random
import re
import subprocess
import sys
import tempfile
import threading
import time
import urllib2
import urlparse
import uuid

import os.path as path

# How many bytes at a time to read from pipes.
BUF_SIZE = 256


# Define a bunch of directory paths.
# Relative to the current working directory.
CURRENT_DIR = path.abspath(os.getcwd())
BUILDER_DIR = path.dirname(CURRENT_DIR)

# Relative to this script's filesystem path.
THIS_DIR = path.dirname(path.abspath(__file__))

DEPOT_TOOLS_DIR = path.abspath(path.join(THIS_DIR, '..', '..', '..', '..'))

CHROMIUM_GIT_HOST = 'https://chromium.googlesource.com'
CHROMIUM_SRC_URL = CHROMIUM_GIT_HOST + '/chromium/src.git'

BRANCH_HEADS_REFSPEC = '+refs/branch-heads/*'
TAGS_REFSPEC = '+refs/tags/*'

# Regular expression that matches a single commit footer line.
COMMIT_FOOTER_ENTRY_RE = re.compile(r'([^:]+):\s*(.*)')

# Footer metadata keys for regular and gsubtreed mirrored commit positions.
COMMIT_POSITION_FOOTER_KEY = 'Cr-Commit-Position'
COMMIT_ORIGINAL_POSITION_FOOTER_KEY = 'Cr-Original-Commit-Position'

# Regular expression to parse gclient's revinfo entries.
REVINFO_RE = re.compile(r'^([^:]+):\s+([^@]+)@(.+)$')


# Copied from scripts/recipes/chromium.py.
GOT_REVISION_MAPPINGS = {
    CHROMIUM_SRC_URL: {
        'got_revision': 'src/',
        'got_nacl_revision': 'src/native_client/',
        'got_swarm_client_revision': 'src/tools/swarm_client/',
        'got_swarming_client_revision': 'src/tools/swarming_client/',
        'got_v8_revision': 'src/v8/',
        'got_webkit_revision': 'src/third_party/WebKit/',
        'got_webrtc_revision': 'src/third_party/webrtc/',
    }
}


GCLIENT_TEMPLATE = """solutions = %(solutions)s

cache_dir = r%(cache_dir)s
%(target_os)s
%(target_os_only)s
"""


# How many times to try before giving up.
ATTEMPTS = 5

GIT_CACHE_PATH = path.join(DEPOT_TOOLS_DIR, 'git_cache.py')
GCLIENT_PATH = path.join(DEPOT_TOOLS_DIR, 'gclient.py')

# If there is less than 100GB of disk space on the system, then we do
# a shallow checkout.
SHALLOW_CLONE_THRESHOLD = 100 * 1024 * 1024 * 1024


class SubprocessFailed(Exception):
  def __init__(self, message, code, output):
    Exception.__init__(self, message)
    self.code = code
    self.output = output


class PatchFailed(SubprocessFailed):
  pass


class GclientSyncFailed(SubprocessFailed):
  pass


class InvalidDiff(Exception):
  pass


RETRY = object()
OK = object()
FAIL = object()


class PsPrinter(object):
  def __init__(self, interval=300):
    self.interval = interval
    self.active = sys.platform.startswith('linux2')
    self.thread = None

  @staticmethod
  def print_pstree():
    """Debugging function used to print "ps auxwwf" for stuck processes."""
    subprocess.call(['ps', 'auxwwf'])

  def poke(self):
    if self.active:
      self.cancel()
      self.thread = threading.Timer(self.interval, self.print_pstree)
      self.thread.start()

  def cancel(self):
    if self.active and self.thread is not None:
      self.thread.cancel()
      self.thread = None


def call(*args, **kwargs):  # pragma: no cover
  """Interactive subprocess call."""
  kwargs['stdout'] = subprocess.PIPE
  kwargs['stderr'] = subprocess.STDOUT
  kwargs.setdefault('bufsize', BUF_SIZE)
  cwd = kwargs.get('cwd', os.getcwd())
  result_fn = kwargs.pop('result_fn', lambda code, out: RETRY if code else OK)
  stdin_data = kwargs.pop('stdin_data', None)
  tries = kwargs.pop('tries', ATTEMPTS)
  if stdin_data:
    kwargs['stdin'] = subprocess.PIPE
  out = cStringIO.StringIO()
  new_env = kwargs.get('env', {})
  env = copy.copy(os.environ)
  env.update(new_env)
  kwargs['env'] = env
  attempt = 0
  for attempt in range(1, tries + 1):
    attempt_msg = ' (attempt #%d)' % attempt if attempt else ''
    if new_env:
      print '===Injecting Environment Variables==='
      for k, v in sorted(new_env.items()):
        print '%s: %s' % (k, v)
    print '===Running %s%s===' % (' '.join(args), attempt_msg)
    print 'In directory: %s' % cwd
    start_time = time.time()
    proc = subprocess.Popen(args, **kwargs)
    if stdin_data:
      proc.stdin.write(stdin_data)
      proc.stdin.close()
    psprinter = PsPrinter()
    # This is here because passing 'sys.stdout' into stdout for proc will
    # produce out of order output.
    hanging_cr = False
    while True:
      psprinter.poke()
      buf = proc.stdout.read(BUF_SIZE)
      if not buf:
        break
      if hanging_cr:
        buf = '\r' + buf
      hanging_cr = buf.endswith('\r')
      if hanging_cr:
        buf = buf[:-1]
      buf = buf.replace('\r\n', '\n').replace('\r', '\n')
      sys.stdout.write(buf)
      out.write(buf)
    if hanging_cr:
      sys.stdout.write('\n')
      out.write('\n')
    psprinter.cancel()

    code = proc.wait()
    elapsed_time = ((time.time() - start_time) / 60.0)
    outval = out.getvalue()
    result = result_fn(code, outval)
    if result in (FAIL, RETRY):
      print '===Failed in %.1f mins===' % elapsed_time
      print
    else:
      print '===Succeeded in %.1f mins===' % elapsed_time
      print
      return outval
    if result is FAIL:
      break
    if result is RETRY and attempt < tries:
      sleep_backoff = 4 ** attempt
      sleep_time = random.randint(sleep_backoff, int(sleep_backoff * 1.2))
      print '===backing off, sleeping for %d secs===' % sleep_time
      time.sleep(sleep_time)

  raise SubprocessFailed('%s failed with code %d in %s after %d attempts.' %
                         (' '.join(args), code, cwd, attempt),
                         code, outval)


def git(*args, **kwargs):  # pragma: no cover
  """Wrapper around call specifically for Git commands."""
  if args and args[0] == 'cache':
    # Rewrite "git cache" calls into "python git_cache.py".
    cmd = (sys.executable, '-u', GIT_CACHE_PATH) + args[1:]
  else:
    git_executable = 'git'
    # On windows, subprocess doesn't fuzzy-match 'git' to 'git.bat', so we
    # have to do it explicitly. This is better than passing shell=True.
    if sys.platform.startswith('win'):
      git_executable += '.bat'
    cmd = (git_executable,) + args
  return call(*cmd, **kwargs)


def get_gclient_spec(solutions, target_os, target_os_only, git_cache_dir):
  return GCLIENT_TEMPLATE % {
      'solutions': pprint.pformat(solutions, indent=4),
      'cache_dir': '"%s"' % git_cache_dir,
      'target_os': ('\ntarget_os=%s' % target_os) if target_os else '',
      'target_os_only': '\ntarget_os_only=%s' % target_os_only
  }


def solutions_printer(solutions):
  """Prints gclient solution to stdout."""
  print 'Gclient Solutions'
  print '================='
  for solution in solutions:
    name = solution.get('name')
    url = solution.get('url')
    print '%s (%s)' % (name, url)
    if solution.get('deps_file'):
      print '  Dependencies file is %s' % solution['deps_file']
    if 'managed' in solution:
      print '  Managed mode is %s' % ('ON' if solution['managed'] else 'OFF')
    custom_vars = solution.get('custom_vars')
    if custom_vars:
      print '  Custom Variables:'
      for var_name, var_value in sorted(custom_vars.iteritems()):
        print '    %s = %s' % (var_name, var_value)
    custom_deps = solution.get('custom_deps')
    if 'custom_deps' in solution:
      print '  Custom Dependencies:'
      for deps_name, deps_value in sorted(custom_deps.iteritems()):
        if deps_value:
          print '    %s -> %s' % (deps_name, deps_value)
        else:
          print '    %s: Ignore' % deps_name
    for k, v in solution.iteritems():
      # Print out all the keys we don't know about.
      if k in ['name', 'url', 'deps_file', 'custom_vars', 'custom_deps',
               'managed']:
        continue
      print '  %s is %s' % (k, v)
    print


def modify_solutions(input_solutions):
  """Modifies urls in solutions to point at Git repos.

  returns: new solution dictionary
  """
  assert input_solutions
  solutions = copy.deepcopy(input_solutions)
  for solution in solutions:
    original_url = solution['url']
    parsed_url = urlparse.urlparse(original_url)
    parsed_path = parsed_url.path

    solution['managed'] = False
    # We don't want gclient to be using a safesync URL. Instead it should
    # using the lkgr/lkcr branch/tags.
    if 'safesync_url' in solution:
      print 'Removing safesync url %s from %s' % (solution['safesync_url'],
                                                  parsed_path)
      del solution['safesync_url']

  return solutions


def remove(target):
  """Remove a target by moving it into build.dead."""
  dead_folder = path.join(BUILDER_DIR, 'build.dead')
  if not path.exists(dead_folder):
    os.makedirs(dead_folder)
  dest = path.join(dead_folder, uuid.uuid4().hex)
  print 'Marking for removal %s => %s' % (target, dest)
  try:
    os.rename(target, dest)
  except Exception as e:
    print 'Error renaming %s to %s: %s' % (target, dest, str(e))
    raise


def ensure_no_checkout(dir_names):
  """Ensure that there is no undesired checkout under build/."""
  build_dir = os.getcwd()
  has_checkout = any(path.exists(path.join(build_dir, dir_name, '.git'))
                     for dir_name in dir_names)
  if has_checkout:
    for filename in os.listdir(build_dir):
      deletion_target = path.join(build_dir, filename)
      print '.git detected in checkout, deleting %s...' % deletion_target,
      remove(deletion_target)
      print 'done'


def call_gclient(*args, **kwargs):
  """Run the "gclient.py" tool with the supplied arguments.

  Args:
    args: command-line arguments to pass to gclient.
    kwargs: keyword arguments to pass to call.
  """
  cmd = [sys.executable, '-u', GCLIENT_PATH]
  cmd.extend(args)
  return call(*cmd, **kwargs)


def gclient_configure(solutions, target_os, target_os_only, git_cache_dir):
  """Should do the same thing as gclient --spec='...'."""
  with codecs.open('.gclient', mode='w', encoding='utf-8') as f:
    f.write(get_gclient_spec(
        solutions, target_os, target_os_only, git_cache_dir))


def gclient_sync(
    with_branch_heads, with_tags, shallow, revisions, break_repo_locks):
  # We just need to allocate a filename.
  fd, gclient_output_file = tempfile.mkstemp(suffix='.json')
  os.close(fd)

  args = ['sync', '--verbose', '--reset', '--force',
         '--ignore_locks', '--output-json', gclient_output_file,
         '--nohooks', '--noprehooks', '--delete_unversioned_trees']
  if with_branch_heads:
    args += ['--with_branch_heads']
  if with_tags:
    args += ['--with_tags']
  if shallow:
    args += ['--shallow']
  if break_repo_locks:
    args += ['--break_repo_locks']
  for name, revision in sorted(revisions.iteritems()):
    if revision.upper() == 'HEAD':
      revision = 'origin/master'
    args.extend(['--revision', '%s@%s' % (name, revision)])

  try:
    call_gclient(*args, tries=1)
  except SubprocessFailed as e:
    # Throw a GclientSyncFailed exception so we can catch this independently.
    raise GclientSyncFailed(e.message, e.code, e.output)
  else:
    with open(gclient_output_file) as f:
      return json.load(f)
  finally:
    os.remove(gclient_output_file)


def gclient_revinfo():
  return call_gclient('revinfo', '-a') or ''


def create_manifest():
  manifest = {}
  output = gclient_revinfo()
  for line in output.strip().splitlines():
    match = REVINFO_RE.match(line.strip())
    if match:
      manifest[match.group(1)] = {
        'repository': match.group(2),
        'revision': match.group(3),
      }
    else:
      print "WARNING: Couldn't match revinfo line:\n%s" % line
  return manifest


def get_commit_message_footer_map(message):
  """Returns: (dict) A dictionary of commit message footer entries.
  """
  footers = {}

  # Extract the lines in the footer block.
  lines = []
  for line in message.strip().splitlines():
    line = line.strip()
    if len(line) == 0:
      del lines[:]
      continue
    lines.append(line)

  # Parse the footer
  for line in lines:
    m = COMMIT_FOOTER_ENTRY_RE.match(line)
    if not m:
      # If any single line isn't valid, continue anyway for compatibility with
      # Gerrit (which itself uses JGit for this).
      continue
    footers[m.group(1)] = m.group(2).strip()
  return footers


def get_commit_message_footer(message, key):
  """Returns: (str/None) The footer value for 'key', or None if none was found.
  """
  return get_commit_message_footer_map(message).get(key)


# Derived from:
# http://code.activestate.com/recipes/577972-disk-usage/?in=user-4178764
def get_total_disk_space():
  cwd = os.getcwd()
  # Windows is the only platform that doesn't support os.statvfs, so
  # we need to special case this.
  if sys.platform.startswith('win'):
    _, total, free = (ctypes.c_ulonglong(), ctypes.c_ulonglong(), \
                      ctypes.c_ulonglong())
    if sys.version_info >= (3,) or isinstance(cwd, unicode):
      fn = ctypes.windll.kernel32.GetDiskFreeSpaceExW
    else:
      fn = ctypes.windll.kernel32.GetDiskFreeSpaceExA
    ret = fn(cwd, ctypes.byref(_), ctypes.byref(total), ctypes.byref(free))
    if ret == 0:
      # WinError() will fetch the last error code.
      raise ctypes.WinError()
    return (total.value, free.value)

  else:
    st = os.statvfs(cwd)
    free = st.f_bavail * st.f_frsize
    total = st.f_blocks * st.f_frsize
    return (total, free)


def get_target_revision(folder_name, git_url, revisions):
  normalized_name = folder_name.strip('/')
  if normalized_name in revisions:
    return revisions[normalized_name]
  if git_url in revisions:
    return revisions[git_url]
  return None


def force_revision(folder_name, revision):
  split_revision = revision.split(':', 1)
  branch = 'master'
  if len(split_revision) == 2:
    # Support for "branch:revision" syntax.
    branch, revision = split_revision

  if revision and revision.upper() != 'HEAD':
    git('checkout', '--force', revision, cwd=folder_name, tries=1)
  else:
    ref = branch if branch.startswith('refs/') else 'origin/%s' % branch
    git('checkout', '--force', ref, cwd=folder_name, tries=1)


def is_broken_repo_dir(repo_dir):
  # Treat absence of 'config' as a signal of a partially deleted repo.
  return not path.exists(os.path.join(repo_dir, '.git', 'config'))


def _maybe_break_locks(checkout_path):
  """This removes all .lock files from this repo's .git directory.

  In particular, this will cleanup index.lock files, as well as ref lock
  files.
  """
  git_dir = os.path.join(checkout_path, '.git')
  for dirpath, _, filenames in os.walk(git_dir):
    for filename in filenames:
      if filename.endswith('.lock'):
        to_break = os.path.join(dirpath, filename)
        print 'breaking lock: %s' % to_break
        try:
          os.remove(to_break)
        except OSError as ex:
          print 'FAILED to break lock: %s: %s' % (to_break, ex)
          raise


def git_checkout(solutions, revisions, shallow, refs, git_cache_dir):
  build_dir = os.getcwd()
  # Before we do anything, break all git_cache locks.
  if path.isdir(git_cache_dir):
    git('cache', 'unlock', '-vv', '--force', '--all',
        '--cache-dir', git_cache_dir)
    for item in os.listdir(git_cache_dir):
      filename = os.path.join(git_cache_dir, item)
      if item.endswith('.lock'):
        raise Exception('%s exists after cache unlock' % filename)
  first_solution = True
  for sln in solutions:
    # Just in case we're hitting a different git server than the one from
    # which the target revision was polled, we retry some.
    done = False
    deadline = time.time() + 60  # One minute (5 tries with exp. backoff).
    tries = 0
    while not done:
      name = sln['name']
      url = sln['url']
      if url == CHROMIUM_SRC_URL or url + '.git' == CHROMIUM_SRC_URL:
        # Experiments show there's little to be gained from
        # a shallow clone of src.
        shallow = False
      sln_dir = path.join(build_dir, name)
      s = ['--shallow'] if shallow else []
      populate_cmd = (['cache', 'populate', '--ignore_locks', '-v',
                       '--cache-dir', git_cache_dir] + s + [url])
      for ref in refs:
        populate_cmd.extend(['--ref', ref])
      git(*populate_cmd)
      mirror_dir = git(
          'cache', 'exists', '--quiet',
          '--cache-dir', git_cache_dir, url).strip()
      clone_cmd = (
          'clone', '--no-checkout', '--local', '--shared', mirror_dir, sln_dir)

      try:
        # If repo deletion was aborted midway, it may have left .git in broken
        # state.
        if path.exists(sln_dir) and is_broken_repo_dir(sln_dir):
          print 'Git repo %s appears to be broken, removing it' % sln_dir
          remove(sln_dir)

        # Use "tries=1", since we retry manually in this loop.
        if not path.isdir(sln_dir):
          git(*clone_cmd, tries=1)
        else:
          git('remote', 'set-url', 'origin', mirror_dir, cwd=sln_dir, tries=1)
          git('fetch', 'origin', cwd=sln_dir, tries=1)
        for ref in refs:
          refspec = '%s:%s' % (ref, ref.lstrip('+'))
          git('fetch', 'origin', refspec, cwd=sln_dir, tries=1)

        # Windows sometimes has trouble deleting files.
        # This can make git commands that rely on locks fail.
        # Try a few times in case Windows has trouble again (and again).
        if sys.platform.startswith('win'):
          tries = 3
          while tries:
            try:
              _maybe_break_locks(sln_dir)
              break
            except Exception:
              tries -= 1

        revision = get_target_revision(name, url, revisions) or 'HEAD'
        force_revision(sln_dir, revision)
        done = True
      except SubprocessFailed as e:
        # Exited abnormally, theres probably something wrong.
        print 'Something failed: %s.' % str(e)

        if time.time() > deadline:
          overrun = time.time() - deadline
          print 'Ran %s seconds past deadline. Aborting.' % overrun
          raise

        # Lets wipe the checkout and try again.
        tries += 1
        sleep_secs = 2**tries
        print 'waiting %s seconds and trying again...' % sleep_secs
        time.sleep(sleep_secs)
        remove(sln_dir)

    git('clean', '-dff', cwd=sln_dir)

    if first_solution:
      git_ref = git('log', '--format=%H', '--max-count=1',
                    cwd=sln_dir).strip()
    first_solution = False
  return git_ref


def _download(url):
  """Fetch url and return content, with retries for flake."""
  for attempt in xrange(ATTEMPTS):
    try:
      return urllib2.urlopen(url).read()
    except Exception:
      if attempt == ATTEMPTS - 1:
        raise


def apply_rietveld_issue(issue, patchset, root, server, _rev_map, _revision,
                         email_file, key_file, oauth2_file,
                         whitelist=None, blacklist=None):
  apply_issue_bin = ('apply_issue.bat' if sys.platform.startswith('win')
                     else 'apply_issue')
  cmd = [apply_issue_bin,
         # The patch will be applied on top of this directory.
         '--root_dir', root,
         # Tell apply_issue how to fetch the patch.
         '--issue', issue,
         '--server', server,
         # Always run apply_issue.py, otherwise it would see update.flag
         # and then bail out.
         '--force',
         # Don't run gclient sync when it sees a DEPS change.
         '--ignore_deps',
  ]
  # Use an oauth key or json file if specified.
  if oauth2_file:
    cmd.extend(['--auth-refresh-token-json', oauth2_file])
  elif email_file and key_file:
    cmd.extend(['--email-file', email_file, '--private-key-file', key_file])
  else:
    cmd.append('--no-auth')

  if patchset:
    cmd.extend(['--patchset', patchset])
  if whitelist:
    for item in whitelist:
      cmd.extend(['--whitelist', item])
  elif blacklist:
    for item in blacklist:
      cmd.extend(['--blacklist', item])

  # TODO(kjellander): Remove this hack when http://crbug.com/611808 is fixed.
  if root == path.join('src', 'third_party', 'webrtc'):
    cmd.extend(['--extra_patchlevel=1'])

  # Only try once, since subsequent failures hide the real failure.
  try:
    call(*cmd, tries=1)
  except SubprocessFailed as e:
    raise PatchFailed(e.message, e.code, e.output)

def apply_gerrit_ref(gerrit_repo, gerrit_ref, root, gerrit_reset,
                     gerrit_rebase_patch_ref):
  gerrit_repo = gerrit_repo or 'origin'
  assert gerrit_ref
  base_rev = git('rev-parse', 'HEAD', cwd=root).strip()

  print '===Applying gerrit ref==='
  print 'Repo is %r @ %r, ref is %r, root is %r' % (
      gerrit_repo, base_rev, gerrit_ref, root)
  # TODO(tandrii): move the fix below to common rietveld/gerrit codepath.
  # Speculative fix: prior bot_update run with Rietveld patch may leave git
  # index with unmerged paths. bot_update calls 'checkout --force xyz' thus
  # ignoring such paths, but potentially never cleaning them up. The following
  # command will do so. See http://crbug.com/692067.
  git('reset', '--hard', cwd=root)
  try:
    git('retry', 'fetch', gerrit_repo, gerrit_ref, cwd=root, tries=1)
    git('checkout', 'FETCH_HEAD', cwd=root)

    if gerrit_rebase_patch_ref:
      print '===Rebasing==='
      # git rebase requires a branch to operate on.
      temp_branch_name = 'tmp/' + uuid.uuid4().hex
      try:
        ok = False
        git('checkout', '-b', temp_branch_name, cwd=root)
        try:
          git('rebase', base_rev, cwd=root, tries=1)
        except SubprocessFailed:
          # Abort the rebase since there were failures.
          git('rebase', '--abort', cwd=root)
          raise

        # Get off of the temporary branch since it can't be deleted otherwise.
        cur_rev = git('rev-parse', 'HEAD', cwd=root).strip()
        git('checkout', cur_rev, cwd=root)
        git('branch', '-D', temp_branch_name, cwd=root)
        ok = True
      finally:
        if not ok:
          # Get off of the temporary branch since it can't be deleted otherwise.
          git('checkout', base_rev, cwd=root)
          git('branch', '-D', temp_branch_name, cwd=root)

    if gerrit_reset:
      git('reset', '--soft', base_rev, cwd=root)
  except SubprocessFailed as e:
    raise PatchFailed(e.message, e.code, e.output)


def get_commit_position(git_path, revision='HEAD'):
  """Dumps the 'git' log for a specific revision and parses out the commit
  position.

  If a commit position metadata key is found, its value will be returned.
  """
  # TODO(iannucci): Use git-footers for this.
  git_log = git('log', '--format=%B', '-n1', revision, cwd=git_path)
  footer_map = get_commit_message_footer_map(git_log)

  # Search for commit position metadata
  value = (footer_map.get(COMMIT_POSITION_FOOTER_KEY) or
           footer_map.get(COMMIT_ORIGINAL_POSITION_FOOTER_KEY))
  if value:
    return value
  return None


def parse_got_revision(gclient_output, got_revision_mapping):
  """Translate git gclient revision mapping to build properties."""
  properties = {}
  solutions_output = {
      # Make sure path always ends with a single slash.
      '%s/' % path.rstrip('/') : solution_output for path, solution_output
      in gclient_output['solutions'].iteritems()
  }
  for property_name, dir_name in got_revision_mapping.iteritems():
    # Make sure dir_name always ends with a single slash.
    dir_name = '%s/' % dir_name.rstrip('/')
    if dir_name not in solutions_output:
      continue
    solution_output = solutions_output[dir_name]
    if solution_output.get('scm') is None:
      # This is an ignored DEPS, so the output got_revision should be 'None'.
      revision = commit_position = None
    else:
      # Since we are using .DEPS.git, everything had better be git.
      assert solution_output.get('scm') == 'git'
      revision = git('rev-parse', 'HEAD', cwd=dir_name).strip()
      commit_position = get_commit_position(dir_name)

    properties[property_name] = revision
    if commit_position:
      properties['%s_cp' % property_name] = commit_position

  return properties


def emit_json(out_file, did_run, gclient_output=None, **kwargs):
  """Write run information into a JSON file."""
  output = {}
  output.update(gclient_output if gclient_output else {})
  output.update({'did_run': did_run})
  output.update(kwargs)
  with open(out_file, 'wb') as f:
    f.write(json.dumps(output, sort_keys=True))


def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
                    patch_root, issue, patchset, rietveld_server, gerrit_repo,
                    gerrit_ref, gerrit_rebase_patch_ref, revision_mapping,
                    apply_issue_email_file, apply_issue_key_file,
                    apply_issue_oauth2_file, shallow, refs, git_cache_dir,
                    gerrit_reset):
  # Get a checkout of each solution, without DEPS or hooks.
  # Calling git directly because there is no way to run Gclient without
  # invoking DEPS.
  print 'Fetching Git checkout'

  git_ref = git_checkout(solutions, revisions, shallow, refs, git_cache_dir)

  print '===Processing patch solutions==='
  already_patched = []
  patch_root = patch_root or ''
  applied_gerrit_patch = False
  print 'Patch root is %r' % patch_root
  for solution in solutions:
    print 'Processing solution %r' % solution['name']
    if (patch_root == solution['name'] or
        solution['name'].startswith(patch_root + '/')):
      relative_root = solution['name'][len(patch_root) + 1:]
      target = '/'.join([relative_root, 'DEPS']).lstrip('/')
      print '  relative root is %r, target is %r' % (relative_root, target)
      if issue:
        apply_rietveld_issue(issue, patchset, patch_root, rietveld_server,
                             revision_mapping, git_ref, apply_issue_email_file,
                             apply_issue_key_file, apply_issue_oauth2_file,
                             whitelist=[target])
        already_patched.append(target)
      elif gerrit_ref:
        apply_gerrit_ref(gerrit_repo, gerrit_ref, patch_root, gerrit_reset,
                         gerrit_rebase_patch_ref)
        applied_gerrit_patch = True

  # Ensure our build/ directory is set up with the correct .gclient file.
  gclient_configure(solutions, target_os, target_os_only, git_cache_dir)

  # Windows sometimes has trouble deleting files. This can make git commands
  # that rely on locks fail.
  break_repo_locks = True if sys.platform.startswith('win') else False
  # We want to pass all non-solution revisions into the gclient sync call.
  solution_dirs = {sln['name'] for sln in solutions}
  gc_revisions = {
      dirname: rev for dirname, rev in revisions.iteritems()
      if dirname not in solution_dirs}
  # Gclient sometimes ignores "unmanaged": "False" in the gclient solution
  # if --revision <anything> is passed (for example, for subrepos).
  # This forces gclient to always treat solutions deps as unmanaged.
  for solution_name in list(solution_dirs):
    gc_revisions[solution_name] = 'unmanaged'
  # Let gclient do the DEPS syncing.
  # The branch-head refspec is a special case because its possible Chrome
  # src, which contains the branch-head refspecs, is DEPSed in.
  gclient_output = gclient_sync(
      BRANCH_HEADS_REFSPEC in refs,
      TAGS_REFSPEC in refs,
      shallow,
      gc_revisions,
      break_repo_locks)

  # Now that gclient_sync has finished, we should revert any .DEPS.git so that
  # presubmit doesn't complain about it being modified.
  if git('ls-files', '.DEPS.git', cwd=first_sln).strip():
    git('checkout', 'HEAD', '--', '.DEPS.git', cwd=first_sln)

  # Apply the rest of the patch here (sans DEPS)
  if issue:
    apply_rietveld_issue(issue, patchset, patch_root, rietveld_server,
                         revision_mapping, git_ref, apply_issue_email_file,
                         apply_issue_key_file, apply_issue_oauth2_file,
                         blacklist=already_patched)
  elif gerrit_ref and not applied_gerrit_patch:
    # If gerrit_ref was for solution's main repository, it has already been
    # applied above. This chunk is executed only for patches to DEPS-ed in
    # git repositories.
    apply_gerrit_ref(gerrit_repo, gerrit_ref, patch_root, gerrit_reset,
                     gerrit_rebase_patch_ref)

  # Reset the deps_file point in the solutions so that hooks get run properly.
  for sln in solutions:
    sln['deps_file'] = sln.get('deps_file', 'DEPS').replace('.DEPS.git', 'DEPS')
  gclient_configure(solutions, target_os, target_os_only, git_cache_dir)

  return gclient_output


def parse_revisions(revisions, root):
  """Turn a list of revision specs into a nice dictionary.

  We will always return a dict with {root: something}.  By default if root
  is unspecified, or if revisions is [], then revision will be assigned 'HEAD'
  """
  results = {root.strip('/'): 'HEAD'}
  expanded_revisions = []
  for revision in revisions:
    # Allow rev1,rev2,rev3 format.
    # TODO(hinoka): Delete this when webkit switches to recipes.
    expanded_revisions.extend(revision.split(','))
  for revision in expanded_revisions:
    split_revision = revision.split('@')
    if len(split_revision) == 1:
      # This is just a plain revision, set it as the revision for root.
      results[root] = split_revision[0]
    elif len(split_revision) == 2:
      # This is an alt_root@revision argument.
      current_root, current_rev = split_revision

      parsed_root = urlparse.urlparse(current_root)
      if parsed_root.scheme in ['http', 'https']:
        # We want to normalize git urls into .git urls.
        normalized_root = 'https://%s/%s' % (parsed_root.netloc,
                                             parsed_root.path)
        if not normalized_root.endswith('.git'):
          normalized_root = '%s.git' % normalized_root
      elif parsed_root.scheme:
        print 'WARNING: Unrecognized scheme %s, ignoring' % parsed_root.scheme
        continue
      else:
        # This is probably a local path.
        normalized_root = current_root.strip('/')

      results[normalized_root] = current_rev
    else:
      print ('WARNING: %r is not recognized as a valid revision specification,'
             'skipping' % revision)
  return results


def parse_args():
  parse = optparse.OptionParser()

  parse.add_option('--issue', help='Issue number to patch from.')
  parse.add_option('--patchset',
                   help='Patchset from issue to patch from, if applicable.')
  parse.add_option('--apply_issue_email_file',
                   help='--email-file option passthrough for apply_patch.py.')
  parse.add_option('--apply_issue_key_file',
                   help='--private-key-file option passthrough for '
                        'apply_patch.py.')
  parse.add_option('--apply_issue_oauth2_file',
                   help='--auth-refresh-token-json option passthrough for '
                        'apply_patch.py.')
  parse.add_option('--root', dest='patch_root',
                   help='DEPRECATED: Use --patch_root.')
  parse.add_option('--patch_root', help='Directory to patch on top of.')
  parse.add_option('--rietveld_server',
                   default='codereview.chromium.org',
                   help='Rietveld server.')
  parse.add_option('--gerrit_repo',
                   help='Gerrit repository to pull the ref from.')
  parse.add_option('--gerrit_ref', help='Gerrit ref to apply.')
  parse.add_option('--gerrit_no_rebase_patch_ref', action='store_true',
                   help='Bypass rebase of Gerrit patch ref after checkout.')
  parse.add_option('--gerrit_no_reset', action='store_true',
                   help='Bypass calling reset after applying a gerrit ref.')
  parse.add_option('--specs', help='Gcilent spec.')
  parse.add_option('--revision_mapping_file',
                   help=('Path to a json file of the form '
                         '{"property_name": "path/to/repo/"}'))
  parse.add_option('--revision', action='append', default=[],
                   help='Revision to check out. Can be any form of git ref. '
                        'Can prepend root@<rev> to specify which repository, '
                        'where root is either a filesystem path or git https '
                        'url. To specify Tip of Tree, set rev to HEAD. ')
  # TODO(machenbach): Remove the flag when all uses have been removed.
  parse.add_option('--output_manifest', action='store_true',
                   help=('Deprecated.'))
  parse.add_option('--clobber', action='store_true',
                   help='Delete checkout first, always')
  parse.add_option('--output_json',
                   help='Output JSON information into a specified file')
  parse.add_option('--no_shallow', action='store_true',
                   help='Bypass disk detection and never shallow clone. '
                        'Does not override the --shallow flag')
  parse.add_option('--refs', action='append',
                   help='Also fetch this refspec for the main solution(s). '
                        'Eg. +refs/branch-heads/*')
  parse.add_option('--with_branch_heads', action='store_true',
                    help='Always pass --with_branch_heads to gclient.  This '
                          'does the same thing as --refs +refs/branch-heads/*')
  parse.add_option('--with_tags', action='store_true',
                    help='Always pass --with_tags to gclient.  This '
                          'does the same thing as --refs +refs/tags/*')
  parse.add_option('--git-cache-dir', help='Path to git cache directory.')


  options, args = parse.parse_args()

  if not options.git_cache_dir:
    parse.error('--git-cache-dir is required')

  if not options.refs:
    options.refs = []

  if options.with_branch_heads:
    options.refs.append(BRANCH_HEADS_REFSPEC)
    del options.with_branch_heads

  if options.with_tags:
    options.refs.append(TAGS_REFSPEC)
    del options.with_tags

  try:
    if not options.revision_mapping_file:
      parse.error('--revision_mapping_file is required')

    with open(options.revision_mapping_file, 'r') as f:
      options.revision_mapping = json.load(f)
  except Exception as e:
    print (
        'WARNING: Caught execption while parsing revision_mapping*: %s'
        % (str(e),)
    )

  # Because we print CACHE_DIR out into a .gclient file, and then later run
  # eval() on it, backslashes need to be escaped, otherwise "E:\b\build" gets
  # parsed as "E:[\x08][\x08]uild".
  if sys.platform.startswith('win'):
    options.git_cache_dir = options.git_cache_dir.replace('\\', '\\\\')

  return options, args


def prepare(options, git_slns, active):
  """Prepares the target folder before we checkout."""
  dir_names = [sln.get('name') for sln in git_slns if 'name' in sln]
  if options.clobber:
    ensure_no_checkout(dir_names)
  # Make sure we tell recipes that we didn't run if the script exits here.
  emit_json(options.output_json, did_run=active)

  # Do a shallow checkout if the disk is less than 100GB.
  total_disk_space, free_disk_space = get_total_disk_space()
  total_disk_space_gb = int(total_disk_space / (1024 * 1024 * 1024))
  used_disk_space_gb = int((total_disk_space - free_disk_space)
                           / (1024 * 1024 * 1024))
  percent_used = int(used_disk_space_gb * 100 / total_disk_space_gb)
  step_text = '[%dGB/%dGB used (%d%%)]' % (used_disk_space_gb,
                                           total_disk_space_gb,
                                           percent_used)
  if not options.output_json:
    print '@@@STEP_TEXT@%s@@@' % step_text
  shallow = (total_disk_space < SHALLOW_CLONE_THRESHOLD
             and not options.no_shallow)

  # The first solution is where the primary DEPS file resides.
  first_sln = dir_names[0]

  # Split all the revision specifications into a nice dict.
  print 'Revisions: %s' % options.revision
  revisions = parse_revisions(options.revision, first_sln)
  print 'Fetching Git checkout at %s@%s' % (first_sln, revisions[first_sln])
  return revisions, step_text, shallow


def checkout(options, git_slns, specs, revisions, step_text, shallow):
  print 'Checking git version...'
  ver = git('version').strip()
  print 'Using %s' % ver

  first_sln = git_slns[0]['name']
  dir_names = [sln.get('name') for sln in git_slns if 'name' in sln]
  try:
    # Outer try is for catching patch failures and exiting gracefully.
    # Inner try is for catching gclient failures and retrying gracefully.
    try:
      checkout_parameters = dict(
          # First, pass in the base of what we want to check out.
          solutions=git_slns,
          revisions=revisions,
          first_sln=first_sln,

          # Also, target os variables for gclient.
          target_os=specs.get('target_os', []),
          target_os_only=specs.get('target_os_only', False),

          # Then, pass in information about how to patch.
          patch_root=options.patch_root,
          issue=options.issue,
          patchset=options.patchset,
          rietveld_server=options.rietveld_server,
          gerrit_repo=options.gerrit_repo,
          gerrit_ref=options.gerrit_ref,
          gerrit_rebase_patch_ref=not options.gerrit_no_rebase_patch_ref,
          revision_mapping=options.revision_mapping,
          apply_issue_email_file=options.apply_issue_email_file,
          apply_issue_key_file=options.apply_issue_key_file,
          apply_issue_oauth2_file=options.apply_issue_oauth2_file,

          # Finally, extra configurations such as shallowness of the clone.
          shallow=shallow,
          refs=options.refs,
          git_cache_dir=options.git_cache_dir,
          gerrit_reset=not options.gerrit_no_reset)
      gclient_output = ensure_checkout(**checkout_parameters)
    except GclientSyncFailed:
      print 'We failed gclient sync, lets delete the checkout and retry.'
      ensure_no_checkout(dir_names)
      gclient_output = ensure_checkout(**checkout_parameters)
  except PatchFailed as e:
    if options.output_json:
      # Tell recipes information such as root, got_revision, etc.
      emit_json(options.output_json,
                did_run=True,
                root=first_sln,
                patch_apply_return_code=e.code,
                patch_root=options.patch_root,
                patch_failure=True,
                failed_patch_body=e.output,
                step_text='%s PATCH FAILED' % step_text,
                fixed_revisions=revisions)
    raise

  # Take care of got_revisions outputs.
  revision_mapping = GOT_REVISION_MAPPINGS.get(git_slns[0]['url'], {})
  if options.revision_mapping:
    revision_mapping.update(options.revision_mapping)

  # If the repo is not in the default GOT_REVISION_MAPPINGS and no
  # revision_mapping were specified on the command line then
  # default to setting 'got_revision' based on the first solution.
  if not revision_mapping:
    revision_mapping['got_revision'] = first_sln

  got_revisions = parse_got_revision(gclient_output, revision_mapping)

  if not got_revisions:
    # TODO(hinoka): We should probably bail out here, but in the interest
    # of giving mis-configured bots some time to get fixed use a dummy
    # revision here.
    got_revisions = { 'got_revision': 'BOT_UPDATE_NO_REV_FOUND' }
    #raise Exception('No got_revision(s) found in gclient output')

  if options.output_json:
    # Tell recipes information such as root, got_revision, etc.
    emit_json(options.output_json,
              did_run=True,
              root=first_sln,
              patch_root=options.patch_root,
              step_text=step_text,
              fixed_revisions=revisions,
              properties=got_revisions,
              manifest=create_manifest())


def print_debug_info():
  print "Debugging info:"
  debug_params = {
    'CURRENT_DIR': CURRENT_DIR,
    'BUILDER_DIR': BUILDER_DIR,
    'THIS_DIR': THIS_DIR,
    'DEPOT_TOOLS_DIR': DEPOT_TOOLS_DIR,
  }
  for k, v in sorted(debug_params.iteritems()):
    print "%s: %r" % (k, v)


def main():
  # Get inputs.
  options, _ = parse_args()

  # Check if this script should activate or not.
  active = True

  # Print a helpful message to tell developers whats going on with this step.
  print_debug_info()

  # Parse, manipulate, and print the gclient solutions.
  specs = {}
  exec(options.specs, specs)
  orig_solutions = specs.get('solutions', [])
  git_slns = modify_solutions(orig_solutions)

  solutions_printer(git_slns)

  try:
    # Dun dun dun, the main part of bot_update.
    revisions, step_text, shallow = prepare(options, git_slns, active)
    checkout(options, git_slns, specs, revisions, step_text, shallow)

  except PatchFailed as e:
    # Return a specific non-zero exit code for patch failure (because it is
    # a failure), but make it different than other failures to distinguish
    # between infra failures (independent from patch author), and patch
    # failures (that patch author can fix). However, PatchFailure due to
    # download patch failure is still an infra problem.
    if e.code == 3:
      # Patch download problem.
      return 87
    # Genuine patch problem.
    return 88


if __name__ == '__main__':
  sys.exit(main())