Create "git cache" command.

The git cache command is a central place to manage a machine's git cache.

It provides two subcommands:
* populate -- creates or updates the cache of a given repository
* exists -- prints the path to the cache of a repo, if it exists

Gclient, deps2git, bot_update, and any other tools that touch the cache will
be able to use this command to make sure that everyone is interacting with
the cache in the same way.

R=hinoka@chromium.org, iannucci@chromium.org
BUG=339168

Review URL: https://codereview.chromium.org/164823002

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@253007 0039d316-1c4b-4281-b951-d872f2087c98

gclient_scm.py

@@ -12,7 +12,6 @@ import re
 import sys
 import tempfile
 import threading
-import time
 import traceback
 import urlparse
 
@@ -143,43 +142,6 @@ class SCMWrapper(object):
     return getattr(self, command)(options, args, file_list)
 
 
-class GitFilter(object):
-  """A filter_fn implementation for quieting down git output messages.
-
-  Allows a custom function to skip certain lines (predicate), and will throttle
-  the output of percentage completed lines to only output every X seconds.
-  """
-  PERCENT_RE = re.compile('.* ([0-9]{1,2})% .*')
-
-  def __init__(self, time_throttle=0, predicate=None):
-    """
-    Args:
-      time_throttle (int): GitFilter will throttle 'noisy' output (such as the
-        XX% complete messages) to only be printed at least |time_throttle|
-        seconds apart.
-      predicate (f(line)): An optional function which is invoked for every line.
-        The line will be skipped if predicate(line) returns False.
-    """
-    self.last_time = 0
-    self.time_throttle = time_throttle
-    self.predicate = predicate
-
-  def __call__(self, line):
-    # git uses an escape sequence to clear the line; elide it.
-    esc = line.find(unichr(033))
-    if esc > -1:
-      line = line[:esc]
-    if self.predicate and not self.predicate(line):
-      return
-    now = time.time()
-    match = self.PERCENT_RE.match(line)
-    if not match:
-      self.last_time = 0
-    if (now - self.last_time) >= self.time_throttle:
-      self.last_time = now
-      print line
-
-
 class GitWrapper(SCMWrapper):
   """Wrapper for Git"""
   name = 'git'
@@ -1064,7 +1026,7 @@ class GitWrapper(SCMWrapper):
     kwargs.setdefault('cwd', self.checkout_path)
     git_filter = not options.verbose
     if git_filter:
-      kwargs['filter_fn'] = GitFilter(kwargs.get('filter_fn'))
+      kwargs['filter_fn'] = gclient_utils.GitFilter(kwargs.get('filter_fn'))
       kwargs.setdefault('print_stdout', False)
       # Don't prompt for passwords; just fail quickly and noisily.
       # By default, git will use an interactive terminal prompt when a username/

gclient_utils.py

@@ -491,6 +491,43 @@ def CheckCallAndFilter(args, stdout=None, filter_fn=None,
       rv, args, kwargs.get('cwd', None), None, None)
 
 
+class GitFilter(object):
+  """A filter_fn implementation for quieting down git output messages.
+
+  Allows a custom function to skip certain lines (predicate), and will throttle
+  the output of percentage completed lines to only output every X seconds.
+  """
+  PERCENT_RE = re.compile('.* ([0-9]{1,2})% .*')
+
+  def __init__(self, time_throttle=0, predicate=None):
+    """
+    Args:
+      time_throttle (int): GitFilter will throttle 'noisy' output (such as the
+        XX% complete messages) to only be printed at least |time_throttle|
+        seconds apart.
+      predicate (f(line)): An optional function which is invoked for every line.
+        The line will be skipped if predicate(line) returns False.
+    """
+    self.last_time = 0
+    self.time_throttle = time_throttle
+    self.predicate = predicate
+
+  def __call__(self, line):
+    # git uses an escape sequence to clear the line; elide it.
+    esc = line.find(unichr(033))
+    if esc > -1:
+      line = line[:esc]
+    if self.predicate and not self.predicate(line):
+      return
+    now = time.time()
+    match = self.PERCENT_RE.match(line)
+    if not match:
+      self.last_time = 0
+    if (now - self.last_time) >= self.time_throttle:
+      self.last_time = now
+      print line
+
+
 def FindGclientRoot(from_dir, filename='.gclient'):
   """Tries to find the gclient root."""
   real_from_dir = os.path.realpath(from_dir)

git-cache

@@ -0,0 +1,26 @@
+#!/bin/sh
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# git-cache -- a git-command for managing local caches of remote repositories.
+
+# Test if this script is running under a MSys install.  If it is, we will
+# hardcode the path to Python where possible.
+OUTPUT="$(uname | grep 'MINGW')"
+MINGW=$?
+
+if [ $MINGW = 0 ]; then
+  base_dir="${0%\\*}"
+else
+  base_dir=$(dirname "$0")
+fi
+
+# Uncomment this line if you never use gclient.
+# "$base_dir"/update_depot_tools
+
+if [ -e "$base_dir/python.bat" -a $MINGW = 0 ]; then
+  PYTHONDONTWRITEBYTECODE=1 cmd.exe //c "$base_dir\\python.bat" "$base_dir\\git_cache.py" "$@"
+else
+  PYTHONDONTWRITEBYTECODE=1 exec "$base_dir/git_cache.py" "$@"
+fi

git_cache.py

@@ -0,0 +1,303 @@
+#!/usr/bin/env python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""A git command for managing a local cache of git repositories."""
+
+import errno
+import logging
+import optparse
+import os
+import tempfile
+import subprocess
+import sys
+import urlparse
+
+import gclient_utils
+import subcommand
+
+
+GIT_EXECUTABLE = 'git.bat' if sys.platform.startswith('win') else 'git'
+
+
+def NormalizeUrl(url):
+  """Convert a git url to a normalized form."""
+  parsed = urlparse.urlparse(url)
+  norm_url = 'https://' + parsed.netloc + parsed.path
+  if not norm_url.endswith('.git'):
+    norm_url += '.git'
+  return norm_url
+
+
+def UrlToCacheDir(url):
+  """Convert a git url to a normalized form for the cache dir path."""
+  parsed = urlparse.urlparse(url)
+  norm_url = parsed.netloc + parsed.path
+  if norm_url.endswith('.git'):
+    norm_url = norm_url[:-len('.git')]
+  return norm_url.replace('-', '--').replace('/', '-').lower()
+
+
+def RunGit(cmd, **kwargs):
+  """Run git in a subprocess."""
+  kwargs.setdefault('cwd', os.getcwd())
+  if kwargs.get('filter_fn'):
+    kwargs['filter_fn'] = gclient_utils.GitFilter(kwargs.get('filter_fn'))
+    kwargs.setdefault('print_stdout', False)
+    env = kwargs.get('env') or kwargs.setdefault('env', os.environ.copy())
+    env.setdefault('GIT_ASKPASS', 'true')
+    env.setdefault('SSH_ASKPASS', 'true')
+  else:
+    kwargs.setdefault('print_stdout', True)
+  stdout = kwargs.get('stdout', sys.stdout)
+  print >> stdout, 'running "git %s" in "%s"' % (' '.join(cmd), kwargs['cwd'])
+  gclient_utils.CheckCallAndFilter([GIT_EXECUTABLE] + cmd, **kwargs)
+
+
+class LockError(Exception):
+  pass
+
+
+class Lockfile(object):
+  """Class to represent a cross-platform process-specific lockfile."""
+
+  def __init__(self, path):
+    self.path = os.path.abspath(path)
+    self.lockfile = self.path + ".lock"
+    self.pid = os.getpid()
+
+  def _read_pid(self):
+    """Read the pid stored in the lockfile.
+
+    Note: This method is potentially racy. By the time it returns the lockfile
+    may have been unlocked, removed, or stolen by some other process.
+    """
+    try:
+      with open(self.lockfile, 'r') as f:
+        pid = int(f.readline().strip())
+    except (IOError, ValueError):
+      pid = None
+    return pid
+
+  def _make_lockfile(self):
+    """Safely creates a lockfile containing the current pid."""
+    open_flags = (os.O_CREAT | os.O_EXCL | os.O_WRONLY)
+    fd = os.open(self.lockfile, open_flags, 0o644)
+    f = os.fdopen(fd, 'w')
+    print >> f, self.pid
+    f.close()
+
+  def _remove_lockfile(self):
+    """Delete the lockfile. Complains (implicitly) if it doesn't exist."""
+    os.remove(self.lockfile)
+
+  def lock(self):
+    """Acquire the lock.
+
+    Note: This is a NON-BLOCKING FAIL-FAST operation.
+    Do. Or do not. There is no try.
+    """
+    try:
+      self._make_lockfile()
+    except OSError as e:
+      if e.errno == errno.EEXIST:
+        raise LockError("%s is already locked" % self.path)
+      else:
+        raise LockError("Failed to create %s (err %s)" % (self.path, e.errno))
+
+  def unlock(self):
+    """Release the lock."""
+    if not self.is_locked():
+      raise LockError("%s is not locked" % self.path)
+    if not self.i_am_locking():
+      raise LockError("%s is locked, but not by me" % self.path)
+    self._remove_lockfile()
+
+  def break_lock(self):
+    """Remove the lock, even if it was created by someone else."""
+    try:
+      self._remove_lockfile()
+      return True
+    except OSError as exc:
+      if exc.errno == errno.ENOENT:
+        return False
+      else:
+        raise
+
+  def is_locked(self):
+    """Test if the file is locked by anyone.
+
+    Note: This method is potentially racy. By the time it returns the lockfile
+    may have been unlocked, removed, or stolen by some other process.
+    """
+    return os.path.exists(self.lockfile)
+
+  def i_am_locking(self):
+    """Test if the file is locked by this process."""
+    return self.is_locked() and self.pid == self._read_pid()
+
+  def __enter__(self):
+    self.lock()
+    return self
+
+  def __exit__(self, *_exc):
+    self.unlock()
+
+
+@subcommand.usage('[url of repo to check for caching]')
+def CMDexists(parser, args):
+  """Check to see if there already is a cache of the given repo."""
+  options, args = parser.parse_args(args)
+  if not len(args) == 1:
+    parser.error('git cache exists only takes exactly one repo url.')
+  url = args[0]
+  repo_dir = os.path.join(options.cache_dir, UrlToCacheDir(url))
+  flag_file = os.path.join(repo_dir, 'config')
+  if os.path.isdir(repo_dir) and os.path.isfile(flag_file):
+    print repo_dir
+    return 0
+  return 1
+
+
+@subcommand.usage('[url of repo to add to or update in cache]')
+def CMDpopulate(parser, args):
+  """Ensure that the cache has all up-to-date objects for the given repo."""
+  parser.add_option('--depth', type='int',
+                    help='Only cache DEPTH commits of history')
+  parser.add_option('--shallow', '-s', action='store_true',
+                    help='Only cache 10000 commits of history')
+  parser.add_option('--ref', action='append',
+                    help='Specify additional refs to be fetched')
+  options, args = parser.parse_args(args)
+  if options.shallow and not options.depth:
+    options.depth = 10000
+  if not len(args) == 1:
+    parser.error('git cache populate only takes exactly one repo url.')
+  url = args[0]
+
+  gclient_utils.safe_makedirs(options.cache_dir)
+  repo_dir = os.path.join(options.cache_dir, UrlToCacheDir(url))
+
+  v = []
+  filter_fn = lambda l: '[up to date]' not in l
+  if options.verbose:
+    v = ['-v', '--progress']
+    filter_fn = None
+
+  d = []
+  if options.depth:
+    d = ['--depth', '%d' % options.depth]
+
+  def _config(directory):
+    RunGit(['config', 'core.deltaBaseCacheLimit', '2g'],
+           cwd=directory)
+    RunGit(['config', 'remote.origin.url', NormalizeUrl(url)],
+           cwd=directory)
+    RunGit(['config', '--replace-all', 'remote.origin.fetch',
+            '+refs/heads/*:refs/heads/*'],
+           cwd=directory)
+    for ref in options.ref or []:
+      ref = ref.rstrip('/')
+      refspec = '+refs/%s/*:refs/%s/*' % (ref, ref)
+      RunGit(['config', '--add', 'remote.origin.fetch', refspec],
+             cwd=directory)
+
+  with Lockfile(repo_dir):
+    # Setup from scratch if the repo is new or is in a bad state.
+    if not os.path.exists(os.path.join(repo_dir, 'config')):
+      gclient_utils.rmtree(repo_dir)
+      tempdir = tempfile.mkdtemp(suffix=UrlToCacheDir(url),
+                                 dir=options.cache_dir)
+      RunGit(['init', '--bare'], cwd=tempdir)
+      _config(tempdir)
+      fetch_cmd = ['fetch'] + v + d + ['--tags', 'origin']
+      RunGit(fetch_cmd, filter_fn=filter_fn, cwd=tempdir, retry=True)
+      os.rename(tempdir, repo_dir)
+    else:
+      _config(repo_dir)
+      if options.depth and os.path.exists(os.path.join(repo_dir, 'shallow')):
+        logging.warn('Shallow fetch requested, but repo cache already exists.')
+      fetch_cmd = ['fetch'] + v + ['--update-shallow', '--tags', 'origin']
+      RunGit(fetch_cmd, filter_fn=filter_fn, cwd=repo_dir, retry=True)
+
+
+@subcommand.usage('[url of repo to unlock, or -a|--all]')
+def CMDunlock(parser, args):
+  """Unlock one or all repos if their lock files are still around."""
+  parser.add_option('--force', '-f', action='store_true',
+                    help='Actually perform the action')
+  parser.add_option('--all', '-a', action='store_true',
+                    help='Unlock all repository caches')
+  options, args = parser.parse_args(args)
+  if len(args) > 1 or (len(args) == 0 and not options.all):
+    parser.error('git cache unlock takes exactly one repo url, or --all')
+
+  if not options.all:
+    url = args[0]
+    repo_dirs = [os.path.join(options.cache_dir, UrlToCacheDir(url))]
+  else:
+    repo_dirs = [path for path in os.listdir(options.cache_dir)
+                 if os.path.isdir(path)]
+  lockfiles = [repo_dir + '.lock' for repo_dir in repo_dirs
+               if os.path.exists(repo_dir + '.lock')]
+
+  if not options.force:
+    parser.error('git cache unlock requires -f|--force to do anything. '
+                 'Refusing to unlock the following repo caches: '
+                 ', '.join(lockfiles))
+
+  unlocked = []
+  untouched = []
+  for repo_dir in repo_dirs:
+    lf = Lockfile(repo_dir)
+    if lf.break_lock():
+      unlocked.append(repo_dir)
+    else:
+      untouched.append(repo_dir)
+
+  if unlocked:
+    logging.info('Broke locks on these caches: %s' % unlocked)
+  if untouched:
+    logging.debug('Did not touch these caches: %s' % untouched)
+
+
+class OptionParser(optparse.OptionParser):
+  """Wrapper class for OptionParser to handle global options."""
+
+  def __init__(self, *args, **kwargs):
+    optparse.OptionParser.__init__(self, *args, prog='git cache', **kwargs)
+    self.add_option('-c', '--cache-dir',
+                    help='Path to the directory containing the cache')
+    self.add_option('-v', '--verbose', action='count', default=0,
+                    help='Increase verbosity (can be passed multiple times)')
+
+  def parse_args(self, args=None, values=None):
+    options, args = optparse.OptionParser.parse_args(self, args, values)
+
+    try:
+      global_cache_dir = subprocess.check_output(
+          [GIT_EXECUTABLE, 'config', '--global', 'cache.cachepath']).strip()
+      if options.cache_dir:
+        logging.warn('Overriding globally-configured cache directory.')
+      else:
+        options.cache_dir = global_cache_dir
+    except subprocess.CalledProcessError:
+      if not options.cache_dir:
+        self.error('No cache directory specified on command line '
+                   'or in cache.cachepath.')
+    options.cache_dir = os.path.abspath(options.cache_dir)
+
+    levels = [logging.WARNING, logging.INFO, logging.DEBUG]
+    logging.basicConfig(level=levels[min(options.verbose, len(levels) - 1)])
+
+    return options, args
+
+
+def main(argv):
+  dispatcher = subcommand.CommandDispatcher(__name__)
+  return dispatcher.execute(OptionParser(), argv)
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1:]))

tests/gclient_utils_test.py

@@ -32,7 +32,7 @@ class GclientUtilsUnittest(GclientUtilBase):
         'CheckCallAndFilterAndHeader', 'Error', 'ExecutionQueue', 'FileRead',
         'FileWrite', 'FindFileUpwards', 'FindGclientRoot',
         'GetGClientRootAndEntries', 'GetEditor', 'GetExeSuffix',
-        'GetMacWinOrLinux', 'IsDateRevision', 'MakeDateRevision',
+        'GetMacWinOrLinux', 'GitFilter', 'IsDateRevision', 'MakeDateRevision',
         'MakeFileAutoFlush', 'MakeFileAnnotated', 'PathDifference',
         'ParseCodereviewSettingsContent', 'NumLocalCpus', 'PrintableObject',
         'RETRY_INITIAL_SLEEP', 'RETRY_MAX', 'RunEditor', 'GCLIENT_CHILDREN',