diff --git a/gclient.py b/gclient.py index 833ffd9737..47109b9312 100755 --- a/gclient.py +++ b/gclient.py @@ -131,6 +131,9 @@ DEPOT_TOOLS_DIR = os.path.dirname(os.path.abspath(os.path.realpath(__file__))) UNSET_CACHE_DIR = object() +PREVIOUS_CUSTOM_VARS = 'GCLIENT_PREVIOUS_CUSTOM_VARS' + + class GNException(Exception): pass @@ -896,6 +899,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): return bad_deps def FuzzyMatchUrl(self, candidates): + # type: (Union[Mapping[str, str], Collection[str]]) -> Optional[str] """Attempts to find this dependency in the list of candidates. It looks first for the URL of this dependency in the list of @@ -917,12 +921,9 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): """ if self.url: origin, _ = gclient_utils.SplitUrlRevision(self.url) - if origin in candidates: - return origin - if origin.endswith('.git') and origin[:-len('.git')] in candidates: - return origin[:-len('.git')] - if origin + '.git' in candidates: - return origin + '.git' + match = gclient_utils.FuzzyMatchRepo(origin, candidates) + if match: + return match if self.name in candidates: return self.name return None @@ -1564,8 +1565,9 @@ it or fix the checkout. @staticmethod def LoadCurrentConfig(options): + # type: (optparse.Values) -> GClient """Searches for and loads a .gclient file relative to the current working - dir. Returns a GClient object.""" + dir.""" if options.spec: client = GClient('.', options) client.SetConfig(options.spec) @@ -1593,6 +1595,16 @@ it or fix the checkout. options.revisions[0], ', '.join(s.name for s in client.dependencies[1:])), file=sys.stderr) + + if any('@' not in r for r in options.skip_sync_revisions): + raise gclient_utils.Error( + "You must specify the full solution name like --revision src@abc") + skip_sync_names = [rev.split('@')[0] for rev in options.skip_sync_revisions] + sol_names = [s.name for s in client.dependencies] + if any(name not in sol_names for name in skip_sync_names): + raise gclient_utils.Error( + "--skip_sync_revisions are only allowed for solutions.") + return client def SetDefaultConfig(self, solution_name, deps_file, solution_url, @@ -1646,6 +1658,51 @@ it or fix the checkout. gclient_utils.SyntaxErrorToError(filename, e) return scope.get('entries', {}) + def _EnforceSkipSyncRevisions(self, patch_refs): + # type: (Mapping[str, str]) -> Mapping[str, str] + """Checks for and enforces revisions for skipping deps syncing.""" + if not self._options.skip_sync_revisions: + return {} + + # Current `self.dependencies` only contain solutions. If a patch_ref is + # not for a solution, then it is for a solution's dependency or recursed + # dependency which we cannot support with skip_sync_revisions. + if patch_refs: + unclaimed_prs = [] + candidates = [] + for dep in self.dependencies: + origin, _ = gclient_utils.SplitUrlRevision(dep.url) + candidates.extend([origin, dep.name]) + for patch_repo in patch_refs: + if not gclient_utils.FuzzyMatchRepo(patch_repo, candidates): + unclaimed_prs.append(patch_repo) + if unclaimed_prs: + print( + 'Ignoring all --skip-sync-revisions. It cannot be used when there ' + 'are --patch-refs flags for non-solution dependencies. To skip ' + 'syncing remove patch_refs for: \n%s' % '\n'.join(unclaimed_prs)) + return {} + + # We cannot skip syncing if there are custom_vars that differ from the + # previous run's custom_vars. + previous_custom_vars = json.loads(os.environ.get(PREVIOUS_CUSTOM_VARS, + '{}')) + cvs_by_name = {s.name: s.custom_vars for s in self.dependencies} + skip_sync_revisions = {} + for revision in self._options.skip_sync_revisions: + name, rev = revision.split('@', 1) + previous_vars = previous_custom_vars.get(name, {}) + if previous_vars == cvs_by_name.get(name): + skip_sync_revisions[name] = rev + else: + print('--skip-sync-revisions cannot be used for solutions where ' + 'custom_vars is different from custom_vars of the last run on ' + 'this machine.\nRemoving skip_sync_revision for:\n' + 'solution: %s, current: %r, previous: %r.' % + (name, cvs_by_name.get(name), previous_vars)) + return skip_sync_revisions + + # TODO(crbug.com/1340695): Remove handling revisions without '@'. def _EnforceRevisions(self): """Checks for revision overrides.""" revision_overrides = {} @@ -1665,6 +1722,7 @@ it or fix the checkout. return revision_overrides def _EnforcePatchRefsAndBranches(self): + # type: () -> Tuple[Mapping[str, str], Mapping[str, str]] """Checks for patch refs.""" patch_refs = {} target_branches = {} @@ -1822,6 +1880,9 @@ it or fix the checkout. if command == 'update': patch_refs, target_branches = self._EnforcePatchRefsAndBranches() + # TODO(crbug.com/1339472): Pass skip_sync_revisions to flush() + _skip_sync_revisions = self._EnforceSkipSyncRevisions(patch_refs) + # Disable progress for non-tty stdout. should_show_progress = ( setup_color.IS_TTY and not self._options.verbose and progress) @@ -1875,6 +1936,11 @@ it or fix the checkout. pm = Progress('Running hooks', 1) self.RunHooksRecursively(self._options, pm) + # Store custom_vars on disk to compare in the next run. + custom_vars = {} + for dep in self.dependencies: + custom_vars[dep.name] = dep.custom_vars + os.environ[PREVIOUS_CUSTOM_VARS] = json.dumps(sorted(custom_vars)) return 0 @@ -3193,6 +3259,8 @@ class OptionParser(optparse.OptionParser): if not hasattr(options, 'revisions'): # GClient.RunOnDeps expects it even if not applicable. options.revisions = [] + if not hasattr(options, 'skip_sync_revisions'): + options.skip_sync_revisions = [] if not hasattr(options, 'head'): options.head = None if not hasattr(options, 'nohooks'): diff --git a/gclient_utils.py b/gclient_utils.py index bbf273d114..af085fa97d 100644 --- a/gclient_utils.py +++ b/gclient_utils.py @@ -95,6 +95,27 @@ def AddWarning(msg): _WARNINGS.append(msg) +def FuzzyMatchRepo(repo, candidates): + # type: (str, Union[Collection[str], Mapping[str, Any]]) -> Optional[str] + """Attempts to find a representation of repo in the candidates. + + Args: + repo: a string representation of a repo in the form of a url or the + name and path of the solution it represents. + candidates: The candidates to look through which may contain `repo` in + in any of the forms mentioned above. + Returns: + The matching string, if any, which may be in a different form from `repo`. + """ + if repo in candidates: + return repo + if repo.endswith('.git') and repo[:-len('.git')] in candidates: + return repo[:-len('.git')] + if repo + '.git' in candidates: + return repo + '.git' + return None + + def SplitUrlRevision(url): """Splits url and returns a two-tuple: url, rev""" if url.startswith('ssh:'): diff --git a/tests/gclient_test.py b/tests/gclient_test.py index 01ec1f32fe..794c0a6912 100755 --- a/tests/gclient_test.py +++ b/tests/gclient_test.py @@ -9,6 +9,7 @@ See gclient_smoketest.py for integration tests. """ import copy +import json import logging import ntpath import os @@ -79,6 +80,9 @@ class GclientTest(trial_dir.TestCase): gclient.gclient_scm.GitWrapper = SCMMock SCMMock.unit_test = self + mock.patch('os.environ', {}).start() + self.addCleanup(mock.patch.stopall) + def tearDown(self): self.assertEqual([], self._get_processed()) gclient.gclient_scm.GitWrapper = self._old_createscm @@ -1435,6 +1439,96 @@ class GclientTest(trial_dir.TestCase): foo_sol = obj.dependencies[0] self.assertEqual('foo', foo_sol.FuzzyMatchUrl(['foo'])) + def testLoadCurrentConfig_SkipSyncRevisions(self): + """Invalid skip_sync_revisions should raise an error.""" + write( + '.gclient', 'solutions = [\n' + ' { "name": "foo", "url": "https://example.com/foo",\n' + ' "deps_file" : ".DEPS.git",\n' + ' },\n' + ']') + write( + os.path.join('foo', 'DEPS'), 'deps = {\n' + ' "bar": "https://example.com/bar.git@bar_version",\n' + '}') + options, _ = gclient.OptionParser().parse_args([]) + + options.skip_sync_revisions = ['1234'] + with self.assertRaises(gclient_utils.Error): + gclient.GClient.LoadCurrentConfig(options) + + options.skip_sync_revisions = ['notasolution@12345'] + with self.assertRaises(gclient_utils.Error): + gclient.GClient.LoadCurrentConfig(options) + + def testEnforceSkipSyncRevisions_DepsPatchRefs(self): + """Patch_refs for any deps removes all skip_sync_revisions.""" + write( + '.gclient', 'solutions = [\n' + ' { "name": "foo", "url": "https://example.com/foo",\n' + ' "deps_file" : ".DEPS.git",\n' + ' },\n' + ']') + write( + os.path.join('foo', 'DEPS'), 'deps = {\n' + ' "bar": "https://example.com/bar.git@bar_version",\n' + '}') + options, _ = gclient.OptionParser().parse_args([]) + options.skip_sync_revisions = ['foo@1234'] + client = gclient.GClient.LoadCurrentConfig(options) + patch_refs = {'foo': '1222', 'somedeps': '1111'} + self.assertEqual({}, client._EnforceSkipSyncRevisions(patch_refs)) + + def testEnforceSkipSyncRevisions_CustomVars(self): + """Changes in a sol's custom_vars removes its revisions.""" + write( + '.gclient', 'solutions = [\n' + ' { "name": "samevars", "url": "https://example.com/foo",\n' + ' "deps_file" : ".DEPS.git",\n' + ' "custom_vars" : { "checkout_foo": "true" },\n' + ' },\n' + ' { "name": "diffvars", "url": "https://example.com/chicken",\n' + ' "deps_file" : ".DEPS.git",\n' + ' "custom_vars" : { "checkout_chicken": "true" },\n' + ' },\n' + ' { "name": "novars", "url": "https://example.com/cow",\n' + ' "deps_file" : ".DEPS.git",\n' + ' },\n' + ']') + write( + os.path.join('samevars', 'DEPS'), 'deps = {\n' + ' "bar": "https://example.com/bar.git@bar_version",\n' + '}') + write( + os.path.join('diffvars', 'DEPS'), 'deps = {\n' + ' "moo": "https://example.com/moo.git@moo_version",\n' + '}') + write( + os.path.join('novars', 'DEPS'), 'deps = {\n' + ' "poo": "https://example.com/poo.git@poo_version",\n' + '}') + + previous_custom_vars = { + 'samevars': { + 'checkout_foo': 'true' + }, + 'diffvars': { + 'checkout_chicken': 'false' + }, + } + os.environ[gclient.PREVIOUS_CUSTOM_VARS] = json.dumps(previous_custom_vars) + options, _ = gclient.OptionParser().parse_args([]) + + patch_refs = {'samevars': '1222'} + options.skip_sync_revisions = [ + 'samevars@10001', 'diffvars@10002', 'novars@10003' + ] + expected_skip_sync_revisions = {'samevars': '10001', 'novars': '10003'} + + client = gclient.GClient.LoadCurrentConfig(options) + self.assertEqual(expected_skip_sync_revisions, + client._EnforceSkipSyncRevisions(patch_refs)) + class MergeVarsTest(unittest.TestCase):