[no-sync] Add a skip_sync_revisions and process it before running deps.

Bug: 1339472
Change-Id: I429489f7deea035c47e27e32ada858fb8a827643
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/3735487
Reviewed-by: Gavin Mak <gavinmak@google.com>
Commit-Queue: Joanna Wang <jojwang@chromium.org>

gclient.py

@@ -131,6 +131,9 @@ DEPOT_TOOLS_DIR = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
 UNSET_CACHE_DIR = object()
 
 
+PREVIOUS_CUSTOM_VARS = 'GCLIENT_PREVIOUS_CUSTOM_VARS'
+
+
 class GNException(Exception):
   pass
 
@@ -896,6 +899,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
     return bad_deps
 
   def FuzzyMatchUrl(self, candidates):
+    # type: (Union[Mapping[str, str], Collection[str]]) -> Optional[str]
     """Attempts to find this dependency in the list of candidates.
 
     It looks first for the URL of this dependency in the list of
@@ -917,12 +921,9 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
     """
     if self.url:
       origin, _ = gclient_utils.SplitUrlRevision(self.url)
-      if origin in candidates:
+      match = gclient_utils.FuzzyMatchRepo(origin, candidates)
-        return origin
+      if match:
-      if origin.endswith('.git') and origin[:-len('.git')] in candidates:
+        return match
-        return origin[:-len('.git')]
-      if origin + '.git' in candidates:
-        return origin + '.git'
     if self.name in candidates:
       return self.name
     return None
@@ -1564,8 +1565,9 @@ it or fix the checkout.
 
   @staticmethod
   def LoadCurrentConfig(options):
+    # type: (optparse.Values) -> GClient
     """Searches for and loads a .gclient file relative to the current working
-    dir. Returns a GClient object."""
+    dir."""
     if options.spec:
       client = GClient('.', options)
       client.SetConfig(options.spec)
@@ -1593,6 +1595,16 @@ it or fix the checkout.
               options.revisions[0],
               ', '.join(s.name for s in client.dependencies[1:])),
           file=sys.stderr)
+
+    if any('@' not in r for r in options.skip_sync_revisions):
+      raise gclient_utils.Error(
+          "You must specify the full solution name like --revision src@abc")
+    skip_sync_names = [rev.split('@')[0] for rev in options.skip_sync_revisions]
+    sol_names = [s.name for s in client.dependencies]
+    if any(name not in sol_names for name in skip_sync_names):
+      raise gclient_utils.Error(
+          "--skip_sync_revisions are only allowed for solutions.")
+
     return client
 
   def SetDefaultConfig(self, solution_name, deps_file, solution_url,
@@ -1646,6 +1658,51 @@ it or fix the checkout.
       gclient_utils.SyntaxErrorToError(filename, e)
     return scope.get('entries', {})
 
+  def _EnforceSkipSyncRevisions(self, patch_refs):
+    # type: (Mapping[str, str]) -> Mapping[str, str]
+    """Checks for and enforces revisions for skipping deps syncing."""
+    if not self._options.skip_sync_revisions:
+      return {}
+
+    # Current `self.dependencies` only contain solutions. If a patch_ref is
+    # not for a solution, then it is for a solution's dependency or recursed
+    # dependency which we cannot support with skip_sync_revisions.
+    if patch_refs:
+      unclaimed_prs = []
+      candidates = []
+      for dep in self.dependencies:
+        origin, _ = gclient_utils.SplitUrlRevision(dep.url)
+        candidates.extend([origin, dep.name])
+      for patch_repo in patch_refs:
+        if not gclient_utils.FuzzyMatchRepo(patch_repo, candidates):
+          unclaimed_prs.append(patch_repo)
+      if unclaimed_prs:
+        print(
+            'Ignoring all --skip-sync-revisions. It cannot be used when there '
+            'are --patch-refs flags for non-solution dependencies. To skip '
+            'syncing remove patch_refs for: \n%s' % '\n'.join(unclaimed_prs))
+        return {}
+
+    # We cannot skip syncing if there are custom_vars that differ from the
+    # previous run's custom_vars.
+    previous_custom_vars = json.loads(os.environ.get(PREVIOUS_CUSTOM_VARS,
+                                                     '{}'))
+    cvs_by_name = {s.name: s.custom_vars for s in self.dependencies}
+    skip_sync_revisions = {}
+    for revision in self._options.skip_sync_revisions:
+      name, rev = revision.split('@', 1)
+      previous_vars = previous_custom_vars.get(name, {})
+      if previous_vars == cvs_by_name.get(name):
+        skip_sync_revisions[name] = rev
+      else:
+        print('--skip-sync-revisions cannot be used for solutions where '
+              'custom_vars is different from custom_vars of the last run on '
+              'this machine.\nRemoving skip_sync_revision for:\n'
+              'solution: %s, current: %r, previous: %r.' %
+              (name, cvs_by_name.get(name), previous_vars))
+    return skip_sync_revisions
+
+  # TODO(crbug.com/1340695): Remove handling revisions without '@'.
   def _EnforceRevisions(self):
     """Checks for revision overrides."""
     revision_overrides = {}
@@ -1665,6 +1722,7 @@ it or fix the checkout.
     return revision_overrides
 
   def _EnforcePatchRefsAndBranches(self):
+    # type: () -> Tuple[Mapping[str, str], Mapping[str, str]]
     """Checks for patch refs."""
     patch_refs = {}
     target_branches = {}
@@ -1822,6 +1880,9 @@ it or fix the checkout.
 
     if command == 'update':
       patch_refs, target_branches = self._EnforcePatchRefsAndBranches()
+      # TODO(crbug.com/1339472): Pass skip_sync_revisions to flush()
+      _skip_sync_revisions = self._EnforceSkipSyncRevisions(patch_refs)
+
     # Disable progress for non-tty stdout.
     should_show_progress = (
         setup_color.IS_TTY and not self._options.verbose and progress)
@@ -1875,6 +1936,11 @@ it or fix the checkout.
         pm = Progress('Running hooks', 1)
       self.RunHooksRecursively(self._options, pm)
 
+    # Store custom_vars on disk to compare in the next run.
+    custom_vars = {}
+    for dep in self.dependencies:
+      custom_vars[dep.name] = dep.custom_vars
+    os.environ[PREVIOUS_CUSTOM_VARS] = json.dumps(sorted(custom_vars))
 
     return 0
 
@@ -3193,6 +3259,8 @@ class OptionParser(optparse.OptionParser):
     if not hasattr(options, 'revisions'):
       # GClient.RunOnDeps expects it even if not applicable.
       options.revisions = []
+    if not hasattr(options, 'skip_sync_revisions'):
+      options.skip_sync_revisions = []
     if not hasattr(options, 'head'):
       options.head = None
     if not hasattr(options, 'nohooks'):

gclient_utils.py

@@ -95,6 +95,27 @@ def AddWarning(msg):
   _WARNINGS.append(msg)
 
 
+def FuzzyMatchRepo(repo, candidates):
+  # type: (str, Union[Collection[str], Mapping[str, Any]]) -> Optional[str]
+  """Attempts to find a representation of repo in the candidates.
+
+  Args:
+    repo: a string representation of a repo in the form of a url or the
+      name and path of the solution it represents.
+    candidates: The candidates to look through which may contain `repo` in
+      in any of the forms mentioned above.
+  Returns:
+    The matching string, if any, which may be in a different form from `repo`.
+  """
+  if repo in candidates:
+    return repo
+  if repo.endswith('.git') and repo[:-len('.git')] in candidates:
+    return repo[:-len('.git')]
+  if repo + '.git' in candidates:
+    return repo + '.git'
+  return None
+
+
 def SplitUrlRevision(url):
   """Splits url and returns a two-tuple: url, rev"""
   if url.startswith('ssh:'):

tests/gclient_test.py

@@ -9,6 +9,7 @@ See gclient_smoketest.py for integration tests.
 """
 
 import copy
+import json
 import logging
 import ntpath
 import os
@@ -79,6 +80,9 @@ class GclientTest(trial_dir.TestCase):
     gclient.gclient_scm.GitWrapper = SCMMock
     SCMMock.unit_test = self
 
+    mock.patch('os.environ', {}).start()
+    self.addCleanup(mock.patch.stopall)
+
   def tearDown(self):
     self.assertEqual([], self._get_processed())
     gclient.gclient_scm.GitWrapper = self._old_createscm
@@ -1435,6 +1439,96 @@ class GclientTest(trial_dir.TestCase):
     foo_sol = obj.dependencies[0]
     self.assertEqual('foo', foo_sol.FuzzyMatchUrl(['foo']))
 
+  def testLoadCurrentConfig_SkipSyncRevisions(self):
+    """Invalid skip_sync_revisions should raise an error."""
+    write(
+        '.gclient', 'solutions = [\n'
+        '  { "name": "foo", "url": "https://example.com/foo",\n'
+        '    "deps_file" : ".DEPS.git",\n'
+        '  },\n'
+        ']')
+    write(
+        os.path.join('foo', 'DEPS'), 'deps = {\n'
+        '  "bar": "https://example.com/bar.git@bar_version",\n'
+        '}')
+    options, _ = gclient.OptionParser().parse_args([])
+
+    options.skip_sync_revisions = ['1234']
+    with self.assertRaises(gclient_utils.Error):
+      gclient.GClient.LoadCurrentConfig(options)
+
+    options.skip_sync_revisions = ['notasolution@12345']
+    with self.assertRaises(gclient_utils.Error):
+      gclient.GClient.LoadCurrentConfig(options)
+
+  def testEnforceSkipSyncRevisions_DepsPatchRefs(self):
+    """Patch_refs for any deps removes all skip_sync_revisions."""
+    write(
+        '.gclient', 'solutions = [\n'
+        '  { "name": "foo", "url": "https://example.com/foo",\n'
+        '    "deps_file" : ".DEPS.git",\n'
+        '  },\n'
+        ']')
+    write(
+        os.path.join('foo', 'DEPS'), 'deps = {\n'
+        '  "bar": "https://example.com/bar.git@bar_version",\n'
+        '}')
+    options, _ = gclient.OptionParser().parse_args([])
+    options.skip_sync_revisions = ['foo@1234']
+    client = gclient.GClient.LoadCurrentConfig(options)
+    patch_refs = {'foo': '1222', 'somedeps': '1111'}
+    self.assertEqual({}, client._EnforceSkipSyncRevisions(patch_refs))
+
+  def testEnforceSkipSyncRevisions_CustomVars(self):
+    """Changes in a sol's custom_vars removes its revisions."""
+    write(
+        '.gclient', 'solutions = [\n'
+        '  { "name": "samevars", "url": "https://example.com/foo",\n'
+        '    "deps_file" : ".DEPS.git",\n'
+        '    "custom_vars" : { "checkout_foo": "true" },\n'
+        '  },\n'
+        '  { "name": "diffvars", "url": "https://example.com/chicken",\n'
+        '    "deps_file" : ".DEPS.git",\n'
+        '    "custom_vars" : { "checkout_chicken": "true" },\n'
+        '  },\n'
+        '  { "name": "novars", "url": "https://example.com/cow",\n'
+        '    "deps_file" : ".DEPS.git",\n'
+        '  },\n'
+        ']')
+    write(
+        os.path.join('samevars', 'DEPS'), 'deps = {\n'
+        '  "bar": "https://example.com/bar.git@bar_version",\n'
+        '}')
+    write(
+        os.path.join('diffvars', 'DEPS'), 'deps = {\n'
+        '  "moo": "https://example.com/moo.git@moo_version",\n'
+        '}')
+    write(
+        os.path.join('novars', 'DEPS'), 'deps = {\n'
+        '  "poo": "https://example.com/poo.git@poo_version",\n'
+        '}')
+
+    previous_custom_vars = {
+        'samevars': {
+            'checkout_foo': 'true'
+        },
+        'diffvars': {
+            'checkout_chicken': 'false'
+        },
+    }
+    os.environ[gclient.PREVIOUS_CUSTOM_VARS] = json.dumps(previous_custom_vars)
+    options, _ = gclient.OptionParser().parse_args([])
+
+    patch_refs = {'samevars': '1222'}
+    options.skip_sync_revisions = [
+        'samevars@10001', 'diffvars@10002', 'novars@10003'
+    ]
+    expected_skip_sync_revisions = {'samevars': '10001', 'novars': '10003'}
+
+    client = gclient.GClient.LoadCurrentConfig(options)
+    self.assertEqual(expected_skip_sync_revisions,
+                     client._EnforceSkipSyncRevisions(patch_refs))
+
 
 class MergeVarsTest(unittest.TestCase):