diff --git a/.style.yapf b/.style.yapf index 24681e21f7..4741fb4f3b 100644 --- a/.style.yapf +++ b/.style.yapf @@ -1,4 +1,3 @@ [style] based_on_style = pep8 -indent_width = 2 column_limit = 80 diff --git a/PRESUBMIT.py b/PRESUBMIT.py index 0d164419c0..63d3cdaf5a 100644 --- a/PRESUBMIT.py +++ b/PRESUBMIT.py @@ -1,7 +1,6 @@ # Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Top-level presubmit script for depot tools. See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts for @@ -31,149 +30,164 @@ TEST_TIMEOUT_S = 330 # 5m 30s def CheckPylint(input_api, output_api): - """Gather all the pylint logic into one place to make it self-contained.""" - files_to_check = [ - r'^[^/]*\.py$', - r'^testing_support/[^/]*\.py$', - r'^tests/[^/]*\.py$', - r'^recipe_modules/.*\.py$', # Allow recursive search in recipe modules. - ] - files_to_skip = list(input_api.DEFAULT_FILES_TO_SKIP) - if os.path.exists('.gitignore'): - with open('.gitignore', encoding='utf-8') as fh: - lines = [l.strip() for l in fh.readlines()] - files_to_skip.extend([fnmatch.translate(l) for l in lines if - l and not l.startswith('#')]) - if os.path.exists('.git/info/exclude'): - with open('.git/info/exclude', encoding='utf-8') as fh: - lines = [l.strip() for l in fh.readlines()] - files_to_skip.extend([fnmatch.translate(l) for l in lines if - l and not l.startswith('#')]) - disabled_warnings = [ - 'R0401', # Cyclic import - 'W0613', # Unused argument - 'C0415', # import-outside-toplevel - 'R1710', # inconsistent-return-statements - 'E1101', # no-member - 'E1120', # no-value-for-parameter - 'R1708', # stop-iteration-return - 'W1510', # subprocess-run-check - # Checks which should be re-enabled after Python 2 support is removed. - 'R0205', # useless-object-inheritance - 'R1725', # super-with-arguments - 'W0707', # raise-missing-from - 'W1113', # keyword-arg-before-vararg - ] - return input_api.RunTests(input_api.canned_checks.GetPylint( - input_api, - output_api, - files_to_check=files_to_check, - files_to_skip=files_to_skip, - disabled_warnings=disabled_warnings, - version='2.7'), parallel=False) + """Gather all the pylint logic into one place to make it self-contained.""" + files_to_check = [ + r'^[^/]*\.py$', + r'^testing_support/[^/]*\.py$', + r'^tests/[^/]*\.py$', + r'^recipe_modules/.*\.py$', # Allow recursive search in recipe modules. + ] + files_to_skip = list(input_api.DEFAULT_FILES_TO_SKIP) + if os.path.exists('.gitignore'): + with open('.gitignore', encoding='utf-8') as fh: + lines = [l.strip() for l in fh.readlines()] + files_to_skip.extend([ + fnmatch.translate(l) for l in lines + if l and not l.startswith('#') + ]) + if os.path.exists('.git/info/exclude'): + with open('.git/info/exclude', encoding='utf-8') as fh: + lines = [l.strip() for l in fh.readlines()] + files_to_skip.extend([ + fnmatch.translate(l) for l in lines + if l and not l.startswith('#') + ]) + disabled_warnings = [ + 'R0401', # Cyclic import + 'W0613', # Unused argument + 'C0415', # import-outside-toplevel + 'R1710', # inconsistent-return-statements + 'E1101', # no-member + 'E1120', # no-value-for-parameter + 'R1708', # stop-iteration-return + 'W1510', # subprocess-run-check + # Checks which should be re-enabled after Python 2 support is removed. + 'R0205', # useless-object-inheritance + 'R1725', # super-with-arguments + 'W0707', # raise-missing-from + 'W1113', # keyword-arg-before-vararg + ] + return input_api.RunTests(input_api.canned_checks.GetPylint( + input_api, + output_api, + files_to_check=files_to_check, + files_to_skip=files_to_skip, + disabled_warnings=disabled_warnings, + version='2.7'), + parallel=False) def CheckRecipes(input_api, output_api): - file_filter = lambda x: x.LocalPath() == 'infra/config/recipes.cfg' - return input_api.canned_checks.CheckJsonParses(input_api, output_api, - file_filter=file_filter) + file_filter = lambda x: x.LocalPath() == 'infra/config/recipes.cfg' + return input_api.canned_checks.CheckJsonParses(input_api, + output_api, + file_filter=file_filter) def CheckUsePython3(input_api, output_api): - results = [] + results = [] - if sys.version_info.major != 3: - results.append( - output_api.PresubmitError( - 'Did not use Python3 for //tests/PRESUBMIT.py.')) + if sys.version_info.major != 3: + results.append( + output_api.PresubmitError( + 'Did not use Python3 for //tests/PRESUBMIT.py.')) - return results + return results def CheckJsonFiles(input_api, output_api): - return input_api.canned_checks.CheckJsonParses( - input_api, output_api) + return input_api.canned_checks.CheckJsonParses(input_api, output_api) def CheckUnitTestsOnCommit(input_api, output_api): - """ Do not run integration tests on upload since they are way too slow.""" + """ Do not run integration tests on upload since they are way too slow.""" - input_api.SetTimeout(TEST_TIMEOUT_S) + input_api.SetTimeout(TEST_TIMEOUT_S) - # Run only selected tests on Windows. - test_to_run_list = [r'.*test\.py$'] - tests_to_skip_list = [] - if input_api.platform.startswith(('cygwin', 'win32')): - print('Warning: skipping most unit tests on Windows') - tests_to_skip_list.extend([ - r'.*auth_test\.py$', - r'.*git_common_test\.py$', - r'.*git_hyper_blame_test\.py$', - r'.*git_map_test\.py$', - r'.*ninjalog_uploader_test\.py$', - r'.*recipes_test\.py$', - ]) + # Run only selected tests on Windows. + test_to_run_list = [r'.*test\.py$'] + tests_to_skip_list = [] + if input_api.platform.startswith(('cygwin', 'win32')): + print('Warning: skipping most unit tests on Windows') + tests_to_skip_list.extend([ + r'.*auth_test\.py$', + r'.*git_common_test\.py$', + r'.*git_hyper_blame_test\.py$', + r'.*git_map_test\.py$', + r'.*ninjalog_uploader_test\.py$', + r'.*recipes_test\.py$', + ]) - tests = input_api.canned_checks.GetUnitTestsInDirectory( - input_api, - output_api, - 'tests', - files_to_check=test_to_run_list, - files_to_skip=tests_to_skip_list) + tests = input_api.canned_checks.GetUnitTestsInDirectory( + input_api, + output_api, + 'tests', + files_to_check=test_to_run_list, + files_to_skip=tests_to_skip_list) - return input_api.RunTests(tests) + return input_api.RunTests(tests) def CheckCIPDManifest(input_api, output_api): - # Validate CIPD manifests. - root = input_api.os_path.normpath( - input_api.os_path.abspath(input_api.PresubmitLocalPath())) - rel_file = lambda rel: input_api.os_path.join(root, rel) - cipd_manifests = set(rel_file(input_api.os_path.join(*x)) for x in ( - ('cipd_manifest.txt',), - ('bootstrap', 'manifest.txt'), - ('bootstrap', 'manifest_bleeding_edge.txt'), + # Validate CIPD manifests. + root = input_api.os_path.normpath( + input_api.os_path.abspath(input_api.PresubmitLocalPath())) + rel_file = lambda rel: input_api.os_path.join(root, rel) + cipd_manifests = set( + rel_file(input_api.os_path.join(*x)) for x in ( + ('cipd_manifest.txt', ), + ('bootstrap', 'manifest.txt'), + ('bootstrap', 'manifest_bleeding_edge.txt'), - # Also generate a file for the cipd client itself. - ('cipd_client_version',), - )) - affected_manifests = input_api.AffectedFiles( - include_deletes=False, - file_filter=lambda x: - input_api.os_path.normpath(x.AbsoluteLocalPath()) in cipd_manifests) - tests = [] - for path in affected_manifests: - path = path.AbsoluteLocalPath() - if path.endswith('.txt'): - tests.append(input_api.canned_checks.CheckCIPDManifest( - input_api, output_api, path=path)) - else: - pkg = 'infra/tools/cipd/${platform}' - ver = input_api.ReadFile(path) - tests.append(input_api.canned_checks.CheckCIPDManifest( - input_api, output_api, - content=CIPD_CLIENT_ENSURE_FILE_TEMPLATE % (pkg, ver))) - tests.append(input_api.canned_checks.CheckCIPDClientDigests( - input_api, output_api, client_version_file=path)) + # Also generate a file for the cipd client itself. + ( + 'cipd_client_version', ), + )) + affected_manifests = input_api.AffectedFiles( + include_deletes=False, + file_filter=lambda x: input_api.os_path.normpath(x.AbsoluteLocalPath() + ) in cipd_manifests) + tests = [] + for path in affected_manifests: + path = path.AbsoluteLocalPath() + if path.endswith('.txt'): + tests.append( + input_api.canned_checks.CheckCIPDManifest(input_api, + output_api, + path=path)) + else: + pkg = 'infra/tools/cipd/${platform}' + ver = input_api.ReadFile(path) + tests.append( + input_api.canned_checks.CheckCIPDManifest( + input_api, + output_api, + content=CIPD_CLIENT_ENSURE_FILE_TEMPLATE % (pkg, ver))) + tests.append( + input_api.canned_checks.CheckCIPDClientDigests( + input_api, output_api, client_version_file=path)) - return input_api.RunTests(tests) + return input_api.RunTests(tests) def CheckOwnersFormat(input_api, output_api): - return input_api.canned_checks.CheckOwnersFormat(input_api, output_api) + return input_api.canned_checks.CheckOwnersFormat(input_api, output_api) def CheckOwnersOnUpload(input_api, output_api): - return input_api.canned_checks.CheckOwners(input_api, output_api, - allow_tbr=False) + return input_api.canned_checks.CheckOwners(input_api, + output_api, + allow_tbr=False) + def CheckDoNotSubmitOnCommit(input_api, output_api): - return input_api.canned_checks.CheckDoNotSubmit(input_api, output_api) + return input_api.canned_checks.CheckDoNotSubmit(input_api, output_api) def CheckPatchFormatted(input_api, output_api): - # TODO(https://crbug.com/979330) If clang-format is fixed for non-chromium - # repos, remove check_clang_format=False so that proto files can be formatted - return input_api.canned_checks.CheckPatchFormatted(input_api, - output_api, - check_clang_format=False) + # TODO(https://crbug.com/979330) If clang-format is fixed for non-chromium + # repos, remove check_clang_format=False so that proto files can be + # formatted + return input_api.canned_checks.CheckPatchFormatted(input_api, + output_api, + check_clang_format=False) diff --git a/auth.py b/auth.py index b01664b56c..5696a7ac5c 100644 --- a/auth.py +++ b/auth.py @@ -1,7 +1,6 @@ # Copyright 2015 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Google OAuth2 related functions.""" from __future__ import print_function @@ -16,6 +15,8 @@ import os import subprocess2 +# TODO: Should fix these warnings. +# pylint: disable=line-too-long # This is what most GAE apps require for authentication. OAUTH_SCOPE_EMAIL = 'https://www.googleapis.com/auth/userinfo.email' @@ -27,89 +28,87 @@ OAUTH_SCOPES = OAUTH_SCOPE_EMAIL # Mockable datetime.datetime.utcnow for testing. def datetime_now(): - return datetime.datetime.utcnow() + return datetime.datetime.utcnow() # OAuth access token with its expiration time (UTC datetime or None if unknown). -class AccessToken(collections.namedtuple('AccessToken', [ - 'token', - 'expires_at', - ])): - - def needs_refresh(self): - """True if this AccessToken should be refreshed.""" - if self.expires_at is not None: - # Allow 30s of clock skew between client and backend. - return datetime_now() + datetime.timedelta(seconds=30) >= self.expires_at - # Token without expiration time never expires. - return False +class AccessToken( + collections.namedtuple('AccessToken', [ + 'token', + 'expires_at', + ])): + def needs_refresh(self): + """True if this AccessToken should be refreshed.""" + if self.expires_at is not None: + # Allow 30s of clock skew between client and backend. + return datetime_now() + datetime.timedelta( + seconds=30) >= self.expires_at + # Token without expiration time never expires. + return False class LoginRequiredError(Exception): - """Interaction with the user is required to authenticate.""" - - def __init__(self, scopes=OAUTH_SCOPE_EMAIL): - msg = ( - 'You are not logged in. Please login first by running:\n' - ' luci-auth login -scopes %s' % scopes) - super(LoginRequiredError, self).__init__(msg) + """Interaction with the user is required to authenticate.""" + def __init__(self, scopes=OAUTH_SCOPE_EMAIL): + msg = ('You are not logged in. Please login first by running:\n' + ' luci-auth login -scopes %s' % scopes) + super(LoginRequiredError, self).__init__(msg) def has_luci_context_local_auth(): - """Returns whether LUCI_CONTEXT should be used for ambient authentication.""" - ctx_path = os.environ.get('LUCI_CONTEXT') - if not ctx_path: - return False - try: - with open(ctx_path) as f: - loaded = json.load(f) - except (OSError, IOError, ValueError): - return False - return loaded.get('local_auth', {}).get('default_account_id') is not None + """Returns whether LUCI_CONTEXT should be used for ambient authentication.""" + ctx_path = os.environ.get('LUCI_CONTEXT') + if not ctx_path: + return False + try: + with open(ctx_path) as f: + loaded = json.load(f) + except (OSError, IOError, ValueError): + return False + return loaded.get('local_auth', {}).get('default_account_id') is not None class Authenticator(object): - """Object that knows how to refresh access tokens when needed. + """Object that knows how to refresh access tokens when needed. Args: scopes: space separated oauth scopes. Defaults to OAUTH_SCOPE_EMAIL. """ + def __init__(self, scopes=OAUTH_SCOPE_EMAIL): + self._access_token = None + self._scopes = scopes - def __init__(self, scopes=OAUTH_SCOPE_EMAIL): - self._access_token = None - self._scopes = scopes - - def has_cached_credentials(self): - """Returns True if credentials can be obtained. + def has_cached_credentials(self): + """Returns True if credentials can be obtained. If returns False, get_access_token() later will probably ask for interactive login by raising LoginRequiredError. If returns True, get_access_token() won't ask for interactive login. """ - return bool(self._get_luci_auth_token()) + return bool(self._get_luci_auth_token()) - def get_access_token(self): - """Returns AccessToken, refreshing it if necessary. + def get_access_token(self): + """Returns AccessToken, refreshing it if necessary. Raises: LoginRequiredError if user interaction is required. """ - if self._access_token and not self._access_token.needs_refresh(): - return self._access_token + if self._access_token and not self._access_token.needs_refresh(): + return self._access_token - # Token expired or missing. Maybe some other process already updated it, - # reload from the cache. - self._access_token = self._get_luci_auth_token() - if self._access_token and not self._access_token.needs_refresh(): - return self._access_token + # Token expired or missing. Maybe some other process already updated it, + # reload from the cache. + self._access_token = self._get_luci_auth_token() + if self._access_token and not self._access_token.needs_refresh(): + return self._access_token - # Nope, still expired. Needs user interaction. - logging.error('Failed to create access token') - raise LoginRequiredError(self._scopes) + # Nope, still expired. Needs user interaction. + logging.error('Failed to create access token') + raise LoginRequiredError(self._scopes) - def authorize(self, http): - """Monkey patches authentication logic of httplib2.Http instance. + def authorize(self, http): + """Monkey patches authentication logic of httplib2.Http instance. The modified http.request method will add authentication headers to each request. @@ -120,46 +119,53 @@ class Authenticator(object): Returns: A modified instance of http that was passed in. """ - # Adapted from oauth2client.OAuth2Credentials.authorize. - request_orig = http.request + # Adapted from oauth2client.OAuth2Credentials.authorize. + request_orig = http.request - @functools.wraps(request_orig) - def new_request( - uri, method='GET', body=None, headers=None, - redirections=httplib2.DEFAULT_MAX_REDIRECTS, - connection_type=None): - headers = (headers or {}).copy() - headers['Authorization'] = 'Bearer %s' % self.get_access_token().token - return request_orig( - uri, method, body, headers, redirections, connection_type) + @functools.wraps(request_orig) + def new_request(uri, + method='GET', + body=None, + headers=None, + redirections=httplib2.DEFAULT_MAX_REDIRECTS, + connection_type=None): + headers = (headers or {}).copy() + headers['Authorization'] = 'Bearer %s' % self.get_access_token( + ).token + return request_orig(uri, method, body, headers, redirections, + connection_type) - http.request = new_request - return http + http.request = new_request + return http - ## Private methods. + ## Private methods. - def _run_luci_auth_login(self): - """Run luci-auth login. + def _run_luci_auth_login(self): + """Run luci-auth login. Returns: AccessToken with credentials. """ - logging.debug('Running luci-auth login') - subprocess2.check_call(['luci-auth', 'login', '-scopes', self._scopes]) - return self._get_luci_auth_token() + logging.debug('Running luci-auth login') + subprocess2.check_call(['luci-auth', 'login', '-scopes', self._scopes]) + return self._get_luci_auth_token() - def _get_luci_auth_token(self): - logging.debug('Running luci-auth token') - try: - out, err = subprocess2.check_call_out( - ['luci-auth', 'token', '-scopes', self._scopes, '-json-output', '-'], - stdout=subprocess2.PIPE, stderr=subprocess2.PIPE) - logging.debug('luci-auth token stderr:\n%s', err) - token_info = json.loads(out) - return AccessToken( - token_info['token'], - datetime.datetime.utcfromtimestamp(token_info['expiry'])) - except subprocess2.CalledProcessError as e: - # subprocess2.CalledProcessError.__str__ nicely formats stdout/stderr. - logging.error('luci-auth token failed: %s', e) - return None + def _get_luci_auth_token(self): + logging.debug('Running luci-auth token') + try: + out, err = subprocess2.check_call_out([ + 'luci-auth', 'token', '-scopes', self._scopes, '-json-output', + '-' + ], + stdout=subprocess2.PIPE, + stderr=subprocess2.PIPE) + logging.debug('luci-auth token stderr:\n%s', err) + token_info = json.loads(out) + return AccessToken( + token_info['token'], + datetime.datetime.utcfromtimestamp(token_info['expiry'])) + except subprocess2.CalledProcessError as e: + # subprocess2.CalledProcessError.__str__ nicely formats + # stdout/stderr. + logging.error('luci-auth token failed: %s', e) + return None diff --git a/autoninja.py b/autoninja.py index d4f8c0c0f2..202c5740ab 100755 --- a/autoninja.py +++ b/autoninja.py @@ -2,7 +2,6 @@ # Copyright (c) 2017 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """ This script (intended to be invoked by autoninja or autoninja.bat) detects whether a build is accelerated using a service like goma. If so, it runs with a @@ -19,256 +18,273 @@ import subprocess import sys if sys.platform == 'darwin': - import resource + import resource SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) def main(args): - # The -t tools are incompatible with -j - t_specified = False - j_specified = False - offline = False - output_dir = '.' - input_args = args - # On Windows the autoninja.bat script passes along the arguments enclosed in - # double quotes. This prevents multiple levels of parsing of the special '^' - # characters needed when compiling a single file but means that this script - # gets called with a single argument containing all of the actual arguments, - # separated by spaces. When this case is detected we need to do argument - # splitting ourselves. This means that arguments containing actual spaces are - # not supported by autoninja, but that is not a real limitation. - if (sys.platform.startswith('win') and len(args) == 2 - and input_args[1].count(' ') > 0): - input_args = args[:1] + args[1].split() + # The -t tools are incompatible with -j + t_specified = False + j_specified = False + offline = False + output_dir = '.' + input_args = args + # On Windows the autoninja.bat script passes along the arguments enclosed in + # double quotes. This prevents multiple levels of parsing of the special '^' + # characters needed when compiling a single file but means that this script + # gets called with a single argument containing all of the actual arguments, + # separated by spaces. When this case is detected we need to do argument + # splitting ourselves. This means that arguments containing actual spaces + # are not supported by autoninja, but that is not a real limitation. + if (sys.platform.startswith('win') and len(args) == 2 + and input_args[1].count(' ') > 0): + input_args = args[:1] + args[1].split() - # Ninja uses getopt_long, which allow to intermix non-option arguments. - # To leave non supported parameters untouched, we do not use getopt. - for index, arg in enumerate(input_args[1:]): - if arg.startswith('-j'): - j_specified = True - if arg.startswith('-t'): - t_specified = True - if arg == '-C': - # + 1 to get the next argument and +1 because we trimmed off input_args[0] - output_dir = input_args[index + 2] - elif arg.startswith('-C'): - # Support -Cout/Default - output_dir = arg[2:] - elif arg in ('-o', '--offline'): - offline = True - elif arg == '-h': - print('autoninja: Use -o/--offline to temporary disable goma.', - file=sys.stderr) - print(file=sys.stderr) + # Ninja uses getopt_long, which allow to intermix non-option arguments. + # To leave non supported parameters untouched, we do not use getopt. + for index, arg in enumerate(input_args[1:]): + if arg.startswith('-j'): + j_specified = True + if arg.startswith('-t'): + t_specified = True + if arg == '-C': + # + 1 to get the next argument and +1 because we trimmed off + # input_args[0] + output_dir = input_args[index + 2] + elif arg.startswith('-C'): + # Support -Cout/Default + output_dir = arg[2:] + elif arg in ('-o', '--offline'): + offline = True + elif arg == '-h': + print('autoninja: Use -o/--offline to temporary disable goma.', + file=sys.stderr) + print(file=sys.stderr) - use_goma = False - use_remoteexec = False - use_rbe = False - use_siso = False - - # Attempt to auto-detect remote build acceleration. We support gn-based - # builds, where we look for args.gn in the build tree, and cmake-based builds - # where we look for rules.ninja. - if os.path.exists(os.path.join(output_dir, 'args.gn')): - with open(os.path.join(output_dir, 'args.gn')) as file_handle: - for line in file_handle: - # use_goma, use_remoteexec, or use_rbe will activate build acceleration. - # - # This test can match multi-argument lines. Examples of this are: - # is_debug=false use_goma=true is_official_build=false - # use_goma=false# use_goma=true This comment is ignored - # - # Anything after a comment is not consider a valid argument. - line_without_comment = line.split('#')[0] - if re.search(r'(^|\s)(use_goma)\s*=\s*true($|\s)', - line_without_comment): - use_goma = True - continue - if re.search(r'(^|\s)(use_remoteexec)\s*=\s*true($|\s)', - line_without_comment): - use_remoteexec = True - continue - if re.search(r'(^|\s)(use_rbe)\s*=\s*true($|\s)', line_without_comment): - use_rbe = True - continue - if re.search(r'(^|\s)(use_siso)\s*=\s*true($|\s)', - line_without_comment): - use_siso = True - continue - - siso_marker = os.path.join(output_dir, '.siso_deps') - if use_siso: - ninja_marker = os.path.join(output_dir, '.ninja_log') - # autosiso generates a .ninja_log file so the mere existence of a - # .ninja_log file doesn't imply that a ninja build was done. However if - # there is a .ninja_log but no .siso_deps then that implies a ninja build. - if os.path.exists(ninja_marker) and not os.path.exists(siso_marker): - return ('echo Run gn clean before switching from ninja to siso in %s' % - output_dir) - siso = ['autosiso'] if use_remoteexec else ['siso', 'ninja'] - if sys.platform.startswith('win'): - # An explicit 'call' is needed to make sure the invocation of autosiso - # returns to autoninja.bat, and the command prompt title gets reset. - siso = ['call'] + siso - return ' '.join(siso + input_args[1:]) - - if os.path.exists(siso_marker): - return ('echo Run gn clean before switching from siso to ninja in %s' % - output_dir) - - else: - for relative_path in [ - '', # GN keeps them in the root of output_dir - 'CMakeFiles' - ]: - path = os.path.join(output_dir, relative_path, 'rules.ninja') - if os.path.exists(path): - with open(path) as file_handle: - for line in file_handle: - if re.match(r'^\s*command\s*=\s*\S+gomacc', line): - use_goma = True - break - - # Strip -o/--offline so ninja doesn't see them. - input_args = [arg for arg in input_args if arg not in ('-o', '--offline')] - - # If GOMA_DISABLED is set to "true", "t", "yes", "y", or "1" - # (case-insensitive) then gomacc will use the local compiler instead of doing - # a goma compile. This is convenient if you want to briefly disable goma. It - # avoids having to rebuild the world when transitioning between goma/non-goma - # builds. However, it is not as fast as doing a "normal" non-goma build - # because an extra process is created for each compile step. Checking this - # environment variable ensures that autoninja uses an appropriate -j value in - # this situation. - goma_disabled_env = os.environ.get('GOMA_DISABLED', '0').lower() - if offline or goma_disabled_env in ['true', 't', 'yes', 'y', '1']: use_goma = False + use_remoteexec = False + use_rbe = False + use_siso = False - if use_goma: - gomacc_file = 'gomacc.exe' if sys.platform.startswith('win') else 'gomacc' - goma_dir = os.environ.get('GOMA_DIR', os.path.join(SCRIPT_DIR, '.cipd_bin')) - gomacc_path = os.path.join(goma_dir, gomacc_file) - # Don't invoke gomacc if it doesn't exist. - if os.path.exists(gomacc_path): - # Check to make sure that goma is running. If not, don't start the build. - status = subprocess.call([gomacc_path, 'port'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=False) - if status == 1: - print('Goma is not running. Use "goma_ctl ensure_start" to start it.', - file=sys.stderr) - if sys.platform.startswith('win'): - # Set an exit code of 1 in the batch file. - print('cmd "/c exit 1"') - else: - # Set an exit code of 1 by executing 'false' in the bash script. - print('false') - sys.exit(1) + # Attempt to auto-detect remote build acceleration. We support gn-based + # builds, where we look for args.gn in the build tree, and cmake-based + # builds where we look for rules.ninja. + if os.path.exists(os.path.join(output_dir, 'args.gn')): + with open(os.path.join(output_dir, 'args.gn')) as file_handle: + for line in file_handle: + # use_goma, use_remoteexec, or use_rbe will activate build + # acceleration. + # + # This test can match multi-argument lines. Examples of this + # are: is_debug=false use_goma=true is_official_build=false + # use_goma=false# use_goma=true This comment is ignored + # + # Anything after a comment is not consider a valid argument. + line_without_comment = line.split('#')[0] + if re.search(r'(^|\s)(use_goma)\s*=\s*true($|\s)', + line_without_comment): + use_goma = True + continue + if re.search(r'(^|\s)(use_remoteexec)\s*=\s*true($|\s)', + line_without_comment): + use_remoteexec = True + continue + if re.search(r'(^|\s)(use_rbe)\s*=\s*true($|\s)', + line_without_comment): + use_rbe = True + continue + if re.search(r'(^|\s)(use_siso)\s*=\s*true($|\s)', + line_without_comment): + use_siso = True + continue - # A large build (with or without goma) tends to hog all system resources. - # Launching the ninja process with 'nice' priorities improves this situation. - prefix_args = [] - if (sys.platform.startswith('linux') - and os.environ.get('NINJA_BUILD_IN_BACKGROUND', '0') == '1'): - # nice -10 is process priority 10 lower than default 0 - # ionice -c 3 is IO priority IDLE - prefix_args = ['nice'] + ['-10'] + siso_marker = os.path.join(output_dir, '.siso_deps') + if use_siso: + ninja_marker = os.path.join(output_dir, '.ninja_log') + # autosiso generates a .ninja_log file so the mere existence of a + # .ninja_log file doesn't imply that a ninja build was done. However + # if there is a .ninja_log but no .siso_deps then that implies a + # ninja build. + if os.path.exists(ninja_marker) and not os.path.exists(siso_marker): + return ( + 'echo Run gn clean before switching from ninja to siso in ' + '%s' % output_dir) + siso = ['autosiso'] if use_remoteexec else ['siso', 'ninja'] + if sys.platform.startswith('win'): + # An explicit 'call' is needed to make sure the invocation of + # autosiso returns to autoninja.bat, and the command prompt + # title gets reset. + siso = ['call'] + siso + return ' '.join(siso + input_args[1:]) - # Tell goma or reclient to do local compiles. On Windows these environment - # variables are set by the wrapper batch file. - offline_env = ['RBE_remote_disabled=1', 'GOMA_DISABLED=1' - ] if offline and not sys.platform.startswith('win') else [] + if os.path.exists(siso_marker): + return ( + 'echo Run gn clean before switching from siso to ninja in %s' % + output_dir) - # On macOS, the default limit of open file descriptors is too low (256). - # This causes a large j value to result in 'Too many open files' errors. - # Check whether the limit can be raised to a large enough value. If yes, - # use `ulimit -n .... &&` as a prefix to increase the limit when running - # ninja. - if sys.platform == 'darwin': - wanted_limit = 200000 # Large enough to avoid any risk of exhaustion. - fileno_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) - if fileno_limit <= wanted_limit: - try: - resource.setrlimit(resource.RLIMIT_NOFILE, (wanted_limit, hard_limit)) - except Exception as _: - pass - fileno_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) - if fileno_limit >= wanted_limit: - prefix_args = ['ulimit', '-n', f'{wanted_limit}', '&&'] + offline_env - offline_env = [] - - - # Call ninja.py so that it can find ninja binary installed by DEPS or one in - # PATH. - ninja_path = os.path.join(SCRIPT_DIR, 'ninja.py') - # If using remoteexec, use ninja_reclient.py which wraps ninja.py with - # starting and stopping reproxy. - if use_remoteexec: - ninja_path = os.path.join(SCRIPT_DIR, 'ninja_reclient.py') - - args = offline_env + prefix_args + [sys.executable, ninja_path - ] + input_args[1:] - - num_cores = multiprocessing.cpu_count() - if not j_specified and not t_specified: - if not offline and (use_goma or use_remoteexec or use_rbe): - args.append('-j') - default_core_multiplier = 80 - if platform.machine() in ('x86_64', 'AMD64'): - # Assume simultaneous multithreading and therefore half as many cores as - # logical processors. - num_cores //= 2 - - core_multiplier = int( - os.environ.get('NINJA_CORE_MULTIPLIER', default_core_multiplier)) - j_value = num_cores * core_multiplier - - core_limit = int(os.environ.get('NINJA_CORE_LIMIT', j_value)) - j_value = min(j_value, core_limit) - - if sys.platform.startswith('win'): - # On windows, j value higher than 1000 does not improve build - # performance. - j_value = min(j_value, 1000) - elif sys.platform == 'darwin': - # If the number of open file descriptors is large enough (or it can be - # raised to a large enough value), then set j value to 1000. This limit - # comes from ninja which is limited to at most FD_SETSIZE (1024) open - # file descriptors (using 1000 leave a bit of head room). - # - # If the number of open file descriptors cannot be raised, then use a - # j value of 200 which is the maximum value that reliably work with - # the default limit of 256. - if fileno_limit >= wanted_limit: - j_value = min(j_value, 1000) - else: - j_value = min(j_value, 200) - - args.append('%d' % j_value) else: - j_value = num_cores - # Ninja defaults to |num_cores + 2| - j_value += int(os.environ.get('NINJA_CORE_ADDITION', '2')) - args.append('-j') - args.append('%d' % j_value) + for relative_path in [ + '', # GN keeps them in the root of output_dir + 'CMakeFiles' + ]: + path = os.path.join(output_dir, relative_path, 'rules.ninja') + if os.path.exists(path): + with open(path) as file_handle: + for line in file_handle: + if re.match(r'^\s*command\s*=\s*\S+gomacc', line): + use_goma = True + break - # On Windows, fully quote the path so that the command processor doesn't think - # the whole output is the command. - # On Linux and Mac, if people put depot_tools in directories with ' ', - # shell would misunderstand ' ' as a path separation. - # TODO(yyanagisawa): provide proper quoting for Windows. - # see https://cs.chromium.org/chromium/src/tools/mb/mb.py - for i in range(len(args)): - if (i == 0 and sys.platform.startswith('win')) or ' ' in args[i]: - args[i] = '"%s"' % args[i].replace('"', '\\"') + # Strip -o/--offline so ninja doesn't see them. + input_args = [arg for arg in input_args if arg not in ('-o', '--offline')] - if os.environ.get('NINJA_SUMMARIZE_BUILD', '0') == '1': - args += ['-d', 'stats'] + # If GOMA_DISABLED is set to "true", "t", "yes", "y", or "1" + # (case-insensitive) then gomacc will use the local compiler instead of + # doing a goma compile. This is convenient if you want to briefly disable + # goma. It avoids having to rebuild the world when transitioning between + # goma/non-goma builds. However, it is not as fast as doing a "normal" + # non-goma build because an extra process is created for each compile step. + # Checking this environment variable ensures that autoninja uses an + # appropriate -j value in this situation. + goma_disabled_env = os.environ.get('GOMA_DISABLED', '0').lower() + if offline or goma_disabled_env in ['true', 't', 'yes', 'y', '1']: + use_goma = False - return ' '.join(args) + if use_goma: + gomacc_file = 'gomacc.exe' if sys.platform.startswith( + 'win') else 'gomacc' + goma_dir = os.environ.get('GOMA_DIR', + os.path.join(SCRIPT_DIR, '.cipd_bin')) + gomacc_path = os.path.join(goma_dir, gomacc_file) + # Don't invoke gomacc if it doesn't exist. + if os.path.exists(gomacc_path): + # Check to make sure that goma is running. If not, don't start the + # build. + status = subprocess.call([gomacc_path, 'port'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False) + if status == 1: + print( + 'Goma is not running. Use "goma_ctl ensure_start" to start ' + 'it.', + file=sys.stderr) + if sys.platform.startswith('win'): + # Set an exit code of 1 in the batch file. + print('cmd "/c exit 1"') + else: + # Set an exit code of 1 by executing 'false' in the bash + # script. + print('false') + sys.exit(1) + + # A large build (with or without goma) tends to hog all system resources. + # Launching the ninja process with 'nice' priorities improves this + # situation. + prefix_args = [] + if (sys.platform.startswith('linux') + and os.environ.get('NINJA_BUILD_IN_BACKGROUND', '0') == '1'): + # nice -10 is process priority 10 lower than default 0 + # ionice -c 3 is IO priority IDLE + prefix_args = ['nice'] + ['-10'] + + # Tell goma or reclient to do local compiles. On Windows these environment + # variables are set by the wrapper batch file. + offline_env = ['RBE_remote_disabled=1', 'GOMA_DISABLED=1' + ] if offline and not sys.platform.startswith('win') else [] + + # On macOS, the default limit of open file descriptors is too low (256). + # This causes a large j value to result in 'Too many open files' errors. + # Check whether the limit can be raised to a large enough value. If yes, + # use `ulimit -n .... &&` as a prefix to increase the limit when running + # ninja. + if sys.platform == 'darwin': + wanted_limit = 200000 # Large enough to avoid any risk of exhaustion. + fileno_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) + if fileno_limit <= wanted_limit: + try: + resource.setrlimit(resource.RLIMIT_NOFILE, + (wanted_limit, hard_limit)) + except Exception as _: + pass + fileno_limit, hard_limit = resource.getrlimit( + resource.RLIMIT_NOFILE) + if fileno_limit >= wanted_limit: + prefix_args = ['ulimit', '-n', f'{wanted_limit}', '&&' + ] + offline_env + offline_env = [] + + # Call ninja.py so that it can find ninja binary installed by DEPS or one in + # PATH. + ninja_path = os.path.join(SCRIPT_DIR, 'ninja.py') + # If using remoteexec, use ninja_reclient.py which wraps ninja.py with + # starting and stopping reproxy. + if use_remoteexec: + ninja_path = os.path.join(SCRIPT_DIR, 'ninja_reclient.py') + + args = offline_env + prefix_args + [sys.executable, ninja_path + ] + input_args[1:] + + num_cores = multiprocessing.cpu_count() + if not j_specified and not t_specified: + if not offline and (use_goma or use_remoteexec or use_rbe): + args.append('-j') + default_core_multiplier = 80 + if platform.machine() in ('x86_64', 'AMD64'): + # Assume simultaneous multithreading and therefore half as many + # cores as logical processors. + num_cores //= 2 + + core_multiplier = int( + os.environ.get('NINJA_CORE_MULTIPLIER', + default_core_multiplier)) + j_value = num_cores * core_multiplier + + core_limit = int(os.environ.get('NINJA_CORE_LIMIT', j_value)) + j_value = min(j_value, core_limit) + + if sys.platform.startswith('win'): + # On windows, j value higher than 1000 does not improve build + # performance. + j_value = min(j_value, 1000) + elif sys.platform == 'darwin': + # If the number of open file descriptors is large enough (or it + # can be raised to a large enough value), then set j value to + # 1000. This limit comes from ninja which is limited to at most + # FD_SETSIZE (1024) open file descriptors (using 1000 leave a + # bit of head room). + # + # If the number of open file descriptors cannot be raised, then + # use a j value of 200 which is the maximum value that reliably + # work with the default limit of 256. + if fileno_limit >= wanted_limit: + j_value = min(j_value, 1000) + else: + j_value = min(j_value, 200) + + args.append('%d' % j_value) + else: + j_value = num_cores + # Ninja defaults to |num_cores + 2| + j_value += int(os.environ.get('NINJA_CORE_ADDITION', '2')) + args.append('-j') + args.append('%d' % j_value) + + # On Windows, fully quote the path so that the command processor doesn't + # think the whole output is the command. On Linux and Mac, if people put + # depot_tools in directories with ' ', shell would misunderstand ' ' as a + # path separation. TODO(yyanagisawa): provide proper quoting for Windows. + # see https://cs.chromium.org/chromium/src/tools/mb/mb.py + for i in range(len(args)): + if (i == 0 and sys.platform.startswith('win')) or ' ' in args[i]: + args[i] = '"%s"' % args[i].replace('"', '\\"') + + if os.environ.get('NINJA_SUMMARIZE_BUILD', '0') == '1': + args += ['-d', 'stats'] + + return ' '.join(args) if __name__ == '__main__': - print(main(sys.argv)) + print(main(sys.argv)) diff --git a/autosiso.py b/autosiso.py index 3088aa23eb..d73ef36a77 100755 --- a/autosiso.py +++ b/autosiso.py @@ -18,53 +18,53 @@ import siso def _use_remoteexec(argv): - out_dir = reclient_helper.find_ninja_out_dir(argv) - gn_args_path = os.path.join(out_dir, 'args.gn') - if not os.path.exists(gn_args_path): + out_dir = reclient_helper.find_ninja_out_dir(argv) + gn_args_path = os.path.join(out_dir, 'args.gn') + if not os.path.exists(gn_args_path): + return False + with open(gn_args_path) as f: + for line in f: + line_without_comment = line.split('#')[0] + if re.search(r'(^|\s)use_remoteexec\s*=\s*true($|\s)', + line_without_comment): + return True return False - with open(gn_args_path) as f: - for line in f: - line_without_comment = line.split('#')[0] - if re.search(r'(^|\s)use_remoteexec\s*=\s*true($|\s)', - line_without_comment): - return True - return False def main(argv): - # On Windows the autosiso.bat script passes along the arguments enclosed in - # double quotes. This prevents multiple levels of parsing of the special '^' - # characters needed when compiling a single file but means that this script - # gets called with a single argument containing all of the actual arguments, - # separated by spaces. When this case is detected we need to do argument - # splitting ourselves. This means that arguments containing actual spaces are - # not supported by autoninja, but that is not a real limitation. - if (sys.platform.startswith('win') and len(argv) == 2 - and argv[1].count(' ') > 0): - argv = argv[:1] + argv[1].split() + # On Windows the autosiso.bat script passes along the arguments enclosed in + # double quotes. This prevents multiple levels of parsing of the special '^' + # characters needed when compiling a single file but means that this script + # gets called with a single argument containing all of the actual arguments, + # separated by spaces. When this case is detected we need to do argument + # splitting ourselves. This means that arguments containing actual spaces + # are not supported by autoninja, but that is not a real limitation. + if (sys.platform.startswith('win') and len(argv) == 2 + and argv[1].count(' ') > 0): + argv = argv[:1] + argv[1].split() - if not _use_remoteexec(argv): - print( - "`use_remoteexec=true` is not detected.\n" - "Please run `siso` command directly.", - file=sys.stderr) - return 1 + if not _use_remoteexec(argv): + print( + "`use_remoteexec=true` is not detected.\n" + "Please run `siso` command directly.", + file=sys.stderr) + return 1 - with reclient_helper.build_context(argv, 'autosiso') as ret_code: - if ret_code: - return ret_code - argv = [ - argv[0], - 'ninja', - # Do not authenticate when using Reproxy. - '-project=', - '-reapi_instance=', - ] + argv[1:] - return siso.main(argv) + with reclient_helper.build_context(argv, 'autosiso') as ret_code: + if ret_code: + return ret_code + argv = [ + argv[0], + 'ninja', + # Do not authenticate when using Reproxy. + '-project=', + '-reapi_instance=', + ] + argv[1:] + return siso.main(argv) if __name__ == '__main__': - try: - sys.exit(main(sys.argv)) - except KeyboardInterrupt: - sys.exit(1) + try: + sys.exit(main(sys.argv)) + except KeyboardInterrupt: + sys.exit(1) diff --git a/bazel.py b/bazel.py index 8fbd77854e..e3c9eb3150 100755 --- a/bazel.py +++ b/bazel.py @@ -25,29 +25,29 @@ from typing import List, Optional def _find_bazel_cros() -> Optional[Path]: - """Find the bazel launcher for ChromiumOS.""" - cwd = Path.cwd() - for parent in itertools.chain([cwd], cwd.parents): - bazel_launcher = parent / "chromite" / "bin" / "bazel" - if bazel_launcher.exists(): - return bazel_launcher - return None + """Find the bazel launcher for ChromiumOS.""" + cwd = Path.cwd() + for parent in itertools.chain([cwd], cwd.parents): + bazel_launcher = parent / "chromite" / "bin" / "bazel" + if bazel_launcher.exists(): + return bazel_launcher + return None def _find_next_bazel_in_path() -> Optional[Path]: - """The fallback method: search the remainder of PATH for bazel.""" - # Remove depot_tools from PATH if present. - depot_tools = Path(__file__).resolve().parent - path_env = os.environ.get("PATH", os.defpath) - search_paths = [] - for path in path_env.split(os.pathsep): - if Path(path).resolve() != depot_tools: - search_paths.append(path) - new_path_env = os.pathsep.join(search_paths) - bazel = shutil.which("bazel", path=new_path_env) - if bazel: - return Path(bazel) - return None + """The fallback method: search the remainder of PATH for bazel.""" + # Remove depot_tools from PATH if present. + depot_tools = Path(__file__).resolve().parent + path_env = os.environ.get("PATH", os.defpath) + search_paths = [] + for path in path_env.split(os.pathsep): + if Path(path).resolve() != depot_tools: + search_paths.append(path) + new_path_env = os.pathsep.join(search_paths) + bazel = shutil.which("bazel", path=new_path_env) + if bazel: + return Path(bazel) + return None # All functions used to search for Bazel (in order of search). @@ -71,15 +71,15 @@ it's actually installed.""" def main(argv: List[str]) -> int: - """Main.""" - for search_func in _SEARCH_FUNCTIONS: - bazel = search_func() - if bazel: - os.execv(bazel, [str(bazel), *argv]) + """Main.""" + for search_func in _SEARCH_FUNCTIONS: + bazel = search_func() + if bazel: + os.execv(bazel, [str(bazel), *argv]) - print(_FIND_FAILURE_MSG, file=sys.stderr) - return 1 + print(_FIND_FAILURE_MSG, file=sys.stderr) + return 1 if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) + sys.exit(main(sys.argv[1:])) diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index 39665881a2..7afa2b237c 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -17,7 +17,6 @@ import subprocess import sys import tempfile - THIS_DIR = os.path.abspath(os.path.dirname(__file__)) ROOT_DIR = os.path.abspath(os.path.join(THIS_DIR, '..')) @@ -29,26 +28,31 @@ BAT_EXT = '.bat' if IS_WIN else '' # Top-level stubs to generate that fall through to executables within the Git # directory. WIN_GIT_STUBS = { - 'git.bat': 'cmd\\git.exe', - 'gitk.bat': 'cmd\\gitk.exe', - 'ssh.bat': 'usr\\bin\\ssh.exe', - 'ssh-keygen.bat': 'usr\\bin\\ssh-keygen.exe', + 'git.bat': 'cmd\\git.exe', + 'gitk.bat': 'cmd\\gitk.exe', + 'ssh.bat': 'usr\\bin\\ssh.exe', + 'ssh-keygen.bat': 'usr\\bin\\ssh-keygen.exe', } # Accumulated template parameters for generated stubs. -class Template(collections.namedtuple('Template', ( - 'PYTHON_RELDIR', 'PYTHON_BIN_RELDIR', 'PYTHON_BIN_RELDIR_UNIX', - 'PYTHON3_BIN_RELDIR', 'PYTHON3_BIN_RELDIR_UNIX', 'GIT_BIN_RELDIR', - 'GIT_BIN_RELDIR_UNIX', 'GIT_PROGRAM', - ))): +class Template( + collections.namedtuple('Template', ( + 'PYTHON_RELDIR', + 'PYTHON_BIN_RELDIR', + 'PYTHON_BIN_RELDIR_UNIX', + 'PYTHON3_BIN_RELDIR', + 'PYTHON3_BIN_RELDIR_UNIX', + 'GIT_BIN_RELDIR', + 'GIT_BIN_RELDIR_UNIX', + 'GIT_PROGRAM', + ))): + @classmethod + def empty(cls): + return cls(**{k: None for k in cls._fields}) - @classmethod - def empty(cls): - return cls(**{k: None for k in cls._fields}) - - def maybe_install(self, name, dst_path): - """Installs template |name| to |dst_path| if it has changed. + def maybe_install(self, name, dst_path): + """Installs template |name| to |dst_path| if it has changed. This loads the template |name| from THIS_DIR, resolves template parameters, and installs it to |dst_path|. See `maybe_update` for more information. @@ -59,14 +63,14 @@ class Template(collections.namedtuple('Template', ( Returns (bool): True if |dst_path| was updated, False otherwise. """ - template_path = os.path.join(THIS_DIR, name) - with open(template_path, 'r', encoding='utf8') as fd: - t = string.Template(fd.read()) - return maybe_update(t.safe_substitute(self._asdict()), dst_path) + template_path = os.path.join(THIS_DIR, name) + with open(template_path, 'r', encoding='utf8') as fd: + t = string.Template(fd.read()) + return maybe_update(t.safe_substitute(self._asdict()), dst_path) def maybe_update(content, dst_path): - """Writes |content| to |dst_path| if |dst_path| does not already match. + """Writes |content| to |dst_path| if |dst_path| does not already match. This function will ensure that there is a file at |dst_path| containing |content|. If |dst_path| already exists and contains |content|, no operation @@ -79,22 +83,22 @@ def maybe_update(content, dst_path): Returns (bool): True if |dst_path| was updated, False otherwise. """ - # If the path already exists and matches the new content, refrain from writing - # a new one. - if os.path.exists(dst_path): - with open(dst_path, 'r', encoding='utf-8') as fd: - if fd.read() == content: - return False + # If the path already exists and matches the new content, refrain from + # writing a new one. + if os.path.exists(dst_path): + with open(dst_path, 'r', encoding='utf-8') as fd: + if fd.read() == content: + return False - logging.debug('Updating %r', dst_path) - with open(dst_path, 'w', encoding='utf-8') as fd: - fd.write(content) - os.chmod(dst_path, 0o755) - return True + logging.debug('Updating %r', dst_path) + with open(dst_path, 'w', encoding='utf-8') as fd: + fd.write(content) + os.chmod(dst_path, 0o755) + return True def maybe_copy(src_path, dst_path): - """Writes the content of |src_path| to |dst_path| if needed. + """Writes the content of |src_path| to |dst_path| if needed. See `maybe_update` for more information. @@ -104,13 +108,13 @@ def maybe_copy(src_path, dst_path): Returns (bool): True if |dst_path| was updated, False otherwise. """ - with open(src_path, 'r', encoding='utf-8') as fd: - content = fd.read() - return maybe_update(content, dst_path) + with open(src_path, 'r', encoding='utf-8') as fd: + content = fd.read() + return maybe_update(content, dst_path) def call_if_outdated(stamp_path, stamp_version, fn): - """Invokes |fn| if the stamp at |stamp_path| doesn't match |stamp_version|. + """Invokes |fn| if the stamp at |stamp_path| doesn't match |stamp_version|. This can be used to keep a filesystem record of whether an operation has been performed. The record is stored at |stamp_path|. To invalidate a record, @@ -128,22 +132,22 @@ def call_if_outdated(stamp_path, stamp_version, fn): Returns (bool): True if an update occurred. """ - stamp_version = stamp_version.strip() - if os.path.isfile(stamp_path): - with open(stamp_path, 'r', encoding='utf-8') as fd: - current_version = fd.read().strip() - if current_version == stamp_version: - return False + stamp_version = stamp_version.strip() + if os.path.isfile(stamp_path): + with open(stamp_path, 'r', encoding='utf-8') as fd: + current_version = fd.read().strip() + if current_version == stamp_version: + return False - fn() + fn() - with open(stamp_path, 'w', encoding='utf-8') as fd: - fd.write(stamp_version) - return True + with open(stamp_path, 'w', encoding='utf-8') as fd: + fd.write(stamp_version) + return True def _in_use(path): - """Checks if a Windows file is in use. + """Checks if a Windows file is in use. When Windows is using an executable, it prevents other writers from modifying or deleting that executable. We can safely test for an in-use @@ -152,93 +156,93 @@ def _in_use(path): Returns (bool): True if the file was in use, False if not. """ - try: - with open(path, 'r+'): - return False - except IOError: - return True + try: + with open(path, 'r+'): + return False + except IOError: + return True def _toolchain_in_use(toolchain_path): - """Returns (bool): True if a toolchain rooted at |path| is in use. + """Returns (bool): True if a toolchain rooted at |path| is in use. """ - # Look for Python files that may be in use. - for python_dir in ( - os.path.join(toolchain_path, 'python', 'bin'), # CIPD - toolchain_path, # Legacy ZIP distributions. - ): - for component in ( - os.path.join(python_dir, 'python.exe'), - os.path.join(python_dir, 'DLLs', 'unicodedata.pyd'), + # Look for Python files that may be in use. + for python_dir in ( + os.path.join(toolchain_path, 'python', 'bin'), # CIPD + toolchain_path, # Legacy ZIP distributions. + ): + for component in ( + os.path.join(python_dir, 'python.exe'), + os.path.join(python_dir, 'DLLs', 'unicodedata.pyd'), ): - if os.path.isfile(component) and _in_use(component): - return True - # Look for Pytho:n 3 files that may be in use. - python_dir = os.path.join(toolchain_path, 'python3', 'bin') - for component in ( - os.path.join(python_dir, 'python3.exe'), - os.path.join(python_dir, 'DLLs', 'unicodedata.pyd'), - ): - if os.path.isfile(component) and _in_use(component): - return True - return False - + if os.path.isfile(component) and _in_use(component): + return True + # Look for Pytho:n 3 files that may be in use. + python_dir = os.path.join(toolchain_path, 'python3', 'bin') + for component in ( + os.path.join(python_dir, 'python3.exe'), + os.path.join(python_dir, 'DLLs', 'unicodedata.pyd'), + ): + if os.path.isfile(component) and _in_use(component): + return True + return False def _check_call(argv, stdin_input=None, **kwargs): - """Wrapper for subprocess.check_call that adds logging.""" - logging.info('running %r', argv) - if stdin_input is not None: - kwargs['stdin'] = subprocess.PIPE - proc = subprocess.Popen(argv, **kwargs) - proc.communicate(input=stdin_input) - if proc.returncode: - raise subprocess.CalledProcessError(proc.returncode, argv, None) + """Wrapper for subprocess.check_call that adds logging.""" + logging.info('running %r', argv) + if stdin_input is not None: + kwargs['stdin'] = subprocess.PIPE + proc = subprocess.Popen(argv, **kwargs) + proc.communicate(input=stdin_input) + if proc.returncode: + raise subprocess.CalledProcessError(proc.returncode, argv, None) def _safe_rmtree(path): - if not os.path.exists(path): - return + if not os.path.exists(path): + return - def _make_writable_and_remove(path): - st = os.stat(path) - new_mode = st.st_mode | 0o200 - if st.st_mode == new_mode: - return False - try: - os.chmod(path, new_mode) - os.remove(path) - return True - except Exception: - return False + def _make_writable_and_remove(path): + st = os.stat(path) + new_mode = st.st_mode | 0o200 + if st.st_mode == new_mode: + return False + try: + os.chmod(path, new_mode) + os.remove(path) + return True + except Exception: + return False - def _on_error(function, path, excinfo): - if not _make_writable_and_remove(path): - logging.warning('Failed to %s: %s (%s)', function, path, excinfo) + def _on_error(function, path, excinfo): + if not _make_writable_and_remove(path): + logging.warning('Failed to %s: %s (%s)', function, path, excinfo) - shutil.rmtree(path, onerror=_on_error) + shutil.rmtree(path, onerror=_on_error) def clean_up_old_installations(skip_dir): - """Removes Python installations other than |skip_dir|. + """Removes Python installations other than |skip_dir|. This includes an "in-use" check against the "python.exe" in a given directory to avoid removing Python executables that are currently ruinning. We need this because our Python bootstrap may be run after (and by) other software that is using the bootstrapped Python! """ - root_contents = os.listdir(ROOT_DIR) - for f in ('win_tools-*_bin', 'python27*_bin', 'git-*_bin', 'bootstrap-*_bin'): - for entry in fnmatch.filter(root_contents, f): - full_entry = os.path.join(ROOT_DIR, entry) - if full_entry == skip_dir or not os.path.isdir(full_entry): - continue + root_contents = os.listdir(ROOT_DIR) + for f in ('win_tools-*_bin', 'python27*_bin', 'git-*_bin', + 'bootstrap-*_bin'): + for entry in fnmatch.filter(root_contents, f): + full_entry = os.path.join(ROOT_DIR, entry) + if full_entry == skip_dir or not os.path.isdir(full_entry): + continue - logging.info('Cleaning up old installation %r', entry) - if not _toolchain_in_use(full_entry): - _safe_rmtree(full_entry) - else: - logging.info('Toolchain at %r is in-use; skipping', full_entry) + logging.info('Cleaning up old installation %r', entry) + if not _toolchain_in_use(full_entry): + _safe_rmtree(full_entry) + else: + logging.info('Toolchain at %r is in-use; skipping', full_entry) # Version of "git_postprocess" system configuration (see |git_postprocess|). @@ -246,111 +250,110 @@ GIT_POSTPROCESS_VERSION = '2' def git_get_mingw_dir(git_directory): - """Returns (str) The "mingw" directory in a Git installation, or None.""" - for candidate in ('mingw64', 'mingw32'): - mingw_dir = os.path.join(git_directory, candidate) - if os.path.isdir(mingw_dir): - return mingw_dir - return None + """Returns (str) The "mingw" directory in a Git installation, or None.""" + for candidate in ('mingw64', 'mingw32'): + mingw_dir = os.path.join(git_directory, candidate) + if os.path.isdir(mingw_dir): + return mingw_dir + return None def git_postprocess(template, git_directory): - # Update depot_tools files for "git help " - mingw_dir = git_get_mingw_dir(git_directory) - if mingw_dir: - docsrc = os.path.join(ROOT_DIR, 'man', 'html') - git_docs_dir = os.path.join(mingw_dir, 'share', 'doc', 'git-doc') - for name in os.listdir(docsrc): - maybe_copy( - os.path.join(docsrc, name), - os.path.join(git_docs_dir, name)) - else: - logging.info('Could not find mingw directory for %r.', git_directory) + # Update depot_tools files for "git help " + mingw_dir = git_get_mingw_dir(git_directory) + if mingw_dir: + docsrc = os.path.join(ROOT_DIR, 'man', 'html') + git_docs_dir = os.path.join(mingw_dir, 'share', 'doc', 'git-doc') + for name in os.listdir(docsrc): + maybe_copy(os.path.join(docsrc, name), + os.path.join(git_docs_dir, name)) + else: + logging.info('Could not find mingw directory for %r.', git_directory) - # Create Git templates and configure its base layout. - for stub_name, relpath in WIN_GIT_STUBS.items(): - stub_template = template._replace(GIT_PROGRAM=relpath) - stub_template.maybe_install( - 'git.template.bat', - os.path.join(ROOT_DIR, stub_name)) + # Create Git templates and configure its base layout. + for stub_name, relpath in WIN_GIT_STUBS.items(): + stub_template = template._replace(GIT_PROGRAM=relpath) + stub_template.maybe_install('git.template.bat', + os.path.join(ROOT_DIR, stub_name)) - # Set-up our system configuration environment. The following set of - # parameters is versioned by "GIT_POSTPROCESS_VERSION". If they change, - # update "GIT_POSTPROCESS_VERSION" accordingly. - def configure_git_system(): - git_bat_path = os.path.join(ROOT_DIR, 'git.bat') - _check_call([git_bat_path, 'config', '--system', 'core.autocrlf', 'false']) - _check_call([git_bat_path, 'config', '--system', 'core.filemode', 'false']) - _check_call([git_bat_path, 'config', '--system', 'core.preloadindex', - 'true']) - _check_call([git_bat_path, 'config', '--system', 'core.fscache', 'true']) - _check_call([git_bat_path, 'config', '--system', 'protocol.version', '2']) + # Set-up our system configuration environment. The following set of + # parameters is versioned by "GIT_POSTPROCESS_VERSION". If they change, + # update "GIT_POSTPROCESS_VERSION" accordingly. + def configure_git_system(): + git_bat_path = os.path.join(ROOT_DIR, 'git.bat') + _check_call( + [git_bat_path, 'config', '--system', 'core.autocrlf', 'false']) + _check_call( + [git_bat_path, 'config', '--system', 'core.filemode', 'false']) + _check_call( + [git_bat_path, 'config', '--system', 'core.preloadindex', 'true']) + _check_call( + [git_bat_path, 'config', '--system', 'core.fscache', 'true']) + _check_call( + [git_bat_path, 'config', '--system', 'protocol.version', '2']) - call_if_outdated( - os.path.join(git_directory, '.git_postprocess'), - GIT_POSTPROCESS_VERSION, - configure_git_system) + call_if_outdated(os.path.join(git_directory, '.git_postprocess'), + GIT_POSTPROCESS_VERSION, configure_git_system) def main(argv): - parser = argparse.ArgumentParser() - parser.add_argument('--verbose', action='store_true') - parser.add_argument('--bootstrap-name', required=True, - help='The directory of the Python installation.') - args = parser.parse_args(argv) + parser = argparse.ArgumentParser() + parser.add_argument('--verbose', action='store_true') + parser.add_argument('--bootstrap-name', + required=True, + help='The directory of the Python installation.') + args = parser.parse_args(argv) - logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARN) + logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARN) - template = Template.empty()._replace( - PYTHON_RELDIR=os.path.join(args.bootstrap_name, 'python'), - PYTHON_BIN_RELDIR=os.path.join(args.bootstrap_name, 'python', 'bin'), - PYTHON_BIN_RELDIR_UNIX=posixpath.join( - args.bootstrap_name, 'python', 'bin'), - PYTHON3_BIN_RELDIR=os.path.join(args.bootstrap_name, 'python3', 'bin'), - PYTHON3_BIN_RELDIR_UNIX=posixpath.join( - args.bootstrap_name, 'python3', 'bin'), - GIT_BIN_RELDIR=os.path.join(args.bootstrap_name, 'git'), - GIT_BIN_RELDIR_UNIX=posixpath.join(args.bootstrap_name, 'git')) + template = Template.empty()._replace( + PYTHON_RELDIR=os.path.join(args.bootstrap_name, 'python'), + PYTHON_BIN_RELDIR=os.path.join(args.bootstrap_name, 'python', 'bin'), + PYTHON_BIN_RELDIR_UNIX=posixpath.join(args.bootstrap_name, 'python', + 'bin'), + PYTHON3_BIN_RELDIR=os.path.join(args.bootstrap_name, 'python3', 'bin'), + PYTHON3_BIN_RELDIR_UNIX=posixpath.join(args.bootstrap_name, 'python3', + 'bin'), + GIT_BIN_RELDIR=os.path.join(args.bootstrap_name, 'git'), + GIT_BIN_RELDIR_UNIX=posixpath.join(args.bootstrap_name, 'git')) - bootstrap_dir = os.path.join(ROOT_DIR, args.bootstrap_name) + bootstrap_dir = os.path.join(ROOT_DIR, args.bootstrap_name) - # Clean up any old Python and Git installations. - clean_up_old_installations(bootstrap_dir) + # Clean up any old Python and Git installations. + clean_up_old_installations(bootstrap_dir) - if IS_WIN: - git_postprocess(template, os.path.join(bootstrap_dir, 'git')) - templates = [ - ('git-bash.template.sh', 'git-bash', ROOT_DIR), - ('python27.bat', 'python.bat', ROOT_DIR), - ('python3.bat', 'python3.bat', ROOT_DIR), - ] - for src_name, dst_name, dst_dir in templates: - # Re-evaluate and regenerate our root templated files. - template.maybe_install(src_name, os.path.join(dst_dir, dst_name)) + if IS_WIN: + git_postprocess(template, os.path.join(bootstrap_dir, 'git')) + templates = [ + ('git-bash.template.sh', 'git-bash', ROOT_DIR), + ('python27.bat', 'python.bat', ROOT_DIR), + ('python3.bat', 'python3.bat', ROOT_DIR), + ] + for src_name, dst_name, dst_dir in templates: + # Re-evaluate and regenerate our root templated files. + template.maybe_install(src_name, os.path.join(dst_dir, dst_name)) - # Emit our Python bin depot-tools-relative directory. This is read by - # python.bat, python3.bat, vpython[.bat] and vpython3[.bat] to identify the - # path of the current Python installation. - # - # We use this indirection so that upgrades can change this pointer to - # redirect "python.bat" to a new Python installation. We can't just update - # "python.bat" because batch file executions reload the batch file and seek - # to the previous cursor in between every command, so changing the batch - # file contents could invalidate any existing executions. - # - # The intention is that the batch file itself never needs to change when - # switching Python versions. + # Emit our Python bin depot-tools-relative directory. This is read by + # python.bat, python3.bat, vpython[.bat] and vpython3[.bat] to identify the + # path of the current Python installation. + # + # We use this indirection so that upgrades can change this pointer to + # redirect "python.bat" to a new Python installation. We can't just update + # "python.bat" because batch file executions reload the batch file and seek + # to the previous cursor in between every command, so changing the batch + # file contents could invalidate any existing executions. + # + # The intention is that the batch file itself never needs to change when + # switching Python versions. - maybe_update( - template.PYTHON_BIN_RELDIR, - os.path.join(ROOT_DIR, 'python_bin_reldir.txt')) + maybe_update(template.PYTHON_BIN_RELDIR, + os.path.join(ROOT_DIR, 'python_bin_reldir.txt')) - maybe_update( - template.PYTHON3_BIN_RELDIR, - os.path.join(ROOT_DIR, 'python3_bin_reldir.txt')) + maybe_update(template.PYTHON3_BIN_RELDIR, + os.path.join(ROOT_DIR, 'python3_bin_reldir.txt')) - return 0 + return 0 if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) + sys.exit(main(sys.argv[1:])) diff --git a/breakpad.py b/breakpad.py index 6d4dd1626d..cef1b3ee54 100644 --- a/breakpad.py +++ b/breakpad.py @@ -1,7 +1,6 @@ # Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """This file remains here because of multiple find_depot_tools.py scripts that attempt to import it as a way to find depot_tools. diff --git a/clang_format.py b/clang_format.py index 4e7b1baadd..bfe3b4b47e 100755 --- a/clang_format.py +++ b/clang_format.py @@ -2,7 +2,6 @@ # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Redirects to the version of clang-format checked into the Chrome tree. clang-format binaries are pulled down from Google Cloud Storage whenever you @@ -18,77 +17,81 @@ import sys class NotFoundError(Exception): - """A file could not be found.""" - def __init__(self, e): - Exception.__init__(self, - 'Problem while looking for clang-format in Chromium source tree:\n' - '%s' % e) + """A file could not be found.""" + def __init__(self, e): + Exception.__init__( + self, + 'Problem while looking for clang-format in Chromium source tree:\n' + '%s' % e) def FindClangFormatToolInChromiumTree(): - """Return a path to the clang-format executable, or die trying.""" - primary_solution_path = gclient_paths.GetPrimarySolutionPath() - if primary_solution_path: - bin_path = os.path.join(primary_solution_path, 'third_party', - 'clang-format', - 'clang-format' + gclient_paths.GetExeSuffix()) - if os.path.exists(bin_path): - return bin_path + """Return a path to the clang-format executable, or die trying.""" + primary_solution_path = gclient_paths.GetPrimarySolutionPath() + if primary_solution_path: + bin_path = os.path.join(primary_solution_path, 'third_party', + 'clang-format', + 'clang-format' + gclient_paths.GetExeSuffix()) + if os.path.exists(bin_path): + return bin_path - bin_path = gclient_paths.GetBuildtoolsPlatformBinaryPath() - if not bin_path: - raise NotFoundError( - 'Could not find checkout in any parent of the current path.\n' - 'Set CHROMIUM_BUILDTOOLS_PATH to use outside of a chromium checkout.') + bin_path = gclient_paths.GetBuildtoolsPlatformBinaryPath() + if not bin_path: + raise NotFoundError( + 'Could not find checkout in any parent of the current path.\n' + 'Set CHROMIUM_BUILDTOOLS_PATH to use outside of a chromium ' + 'checkout.') - tool_path = os.path.join(bin_path, - 'clang-format' + gclient_paths.GetExeSuffix()) - if not os.path.exists(tool_path): - raise NotFoundError('File does not exist: %s' % tool_path) - return tool_path + tool_path = os.path.join(bin_path, + 'clang-format' + gclient_paths.GetExeSuffix()) + if not os.path.exists(tool_path): + raise NotFoundError('File does not exist: %s' % tool_path) + return tool_path def FindClangFormatScriptInChromiumTree(script_name): - """Return a path to a clang-format helper script, or die trying.""" - primary_solution_path = gclient_paths.GetPrimarySolutionPath() - if primary_solution_path: - script_path = os.path.join(primary_solution_path, 'third_party', - 'clang-format', 'script', script_name) - if os.path.exists(script_path): - return script_path + """Return a path to a clang-format helper script, or die trying.""" + primary_solution_path = gclient_paths.GetPrimarySolutionPath() + if primary_solution_path: + script_path = os.path.join(primary_solution_path, 'third_party', + 'clang-format', 'script', script_name) + if os.path.exists(script_path): + return script_path - tools_path = gclient_paths.GetBuildtoolsPath() - if not tools_path: - raise NotFoundError( - 'Could not find checkout in any parent of the current path.\n', - 'Set CHROMIUM_BUILDTOOLS_PATH to use outside of a chromium checkout.') + tools_path = gclient_paths.GetBuildtoolsPath() + if not tools_path: + raise NotFoundError( + 'Could not find checkout in any parent of the current path.\n', + 'Set CHROMIUM_BUILDTOOLS_PATH to use outside of a chromium ' + 'checkout.') - script_path = os.path.join(tools_path, 'clang_format', 'script', script_name) - if not os.path.exists(script_path): - raise NotFoundError('File does not exist: %s' % script_path) - return script_path + script_path = os.path.join(tools_path, 'clang_format', 'script', + script_name) + if not os.path.exists(script_path): + raise NotFoundError('File does not exist: %s' % script_path) + return script_path def main(args): - try: - tool = FindClangFormatToolInChromiumTree() - except NotFoundError as e: - sys.stderr.write("%s\n" % str(e)) - return 1 + try: + tool = FindClangFormatToolInChromiumTree() + except NotFoundError as e: + sys.stderr.write("%s\n" % str(e)) + return 1 - # Add some visibility to --help showing where the tool lives, since this - # redirection can be a little opaque. - help_syntax = ('-h', '--help', '-help', '-help-list', '--help-list') - if any(match in args for match in help_syntax): - print( - '\nDepot tools redirects you to the clang-format at:\n %s\n' % tool) + # Add some visibility to --help showing where the tool lives, since this + # redirection can be a little opaque. + help_syntax = ('-h', '--help', '-help', '-help-list', '--help-list') + if any(match in args for match in help_syntax): + print('\nDepot tools redirects you to the clang-format at:\n %s\n' % + tool) - return subprocess.call([tool] + args) + return subprocess.call([tool] + args) if __name__ == '__main__': - try: - sys.exit(main(sys.argv[1:])) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(1) + try: + sys.exit(main(sys.argv[1:])) + except KeyboardInterrupt: + sys.stderr.write('interrupted\n') + sys.exit(1) diff --git a/compile_single_file.py b/compile_single_file.py index c143fed55d..b766e0073b 100644 --- a/compile_single_file.py +++ b/compile_single_file.py @@ -10,64 +10,64 @@ import os import subprocess import sys - DEPOT_TOOLS_DIR = os.path.dirname(os.path.realpath(__file__)) + # This function is inspired from the one in src/tools/vim/ninja-build.vim in the # Chromium repository. def path_to_source_root(path): - """Returns the absolute path to the chromium source root.""" - candidate = os.path.dirname(path) - # This is a list of directories that need to identify the src directory. The - # shorter it is, the more likely it's wrong (checking for just - # "build/common.gypi" would find "src/v8" for files below "src/v8", as - # "src/v8/build/common.gypi" exists). The longer it is, the more likely it is - # to break when we rename directories. - fingerprints = ['chrome', 'net', 'v8', 'build', 'skia'] - while candidate and not all( - os.path.isdir(os.path.join(candidate, fp)) for fp in fingerprints): - new_candidate = os.path.dirname(candidate) - if new_candidate == candidate: - raise Exception("Couldn't find source-dir from %s" % path) - candidate = os.path.dirname(candidate) - return candidate + """Returns the absolute path to the chromium source root.""" + candidate = os.path.dirname(path) + # This is a list of directories that need to identify the src directory. The + # shorter it is, the more likely it's wrong (checking for just + # "build/common.gypi" would find "src/v8" for files below "src/v8", as + # "src/v8/build/common.gypi" exists). The longer it is, the more likely it + # is to break when we rename directories. + fingerprints = ['chrome', 'net', 'v8', 'build', 'skia'] + while candidate and not all( + os.path.isdir(os.path.join(candidate, fp)) for fp in fingerprints): + new_candidate = os.path.dirname(candidate) + if new_candidate == candidate: + raise Exception("Couldn't find source-dir from %s" % path) + candidate = os.path.dirname(candidate) + return candidate def main(): - parser = argparse.ArgumentParser() - parser.add_argument( - '--file-path', - help='The file path, could be absolute or relative to the current ' - 'directory.', - required=True) - parser.add_argument( - '--build-dir', - help='The build directory, relative to the source directory.', - required=True) + parser = argparse.ArgumentParser() + parser.add_argument( + '--file-path', + help='The file path, could be absolute or relative to the current ' + 'directory.', + required=True) + parser.add_argument( + '--build-dir', + help='The build directory, relative to the source directory.', + required=True) - options = parser.parse_args() + options = parser.parse_args() - src_dir = path_to_source_root(os.path.abspath(options.file_path)) - abs_build_dir = os.path.join(src_dir, options.build_dir) - src_relpath = os.path.relpath(options.file_path, abs_build_dir) + src_dir = path_to_source_root(os.path.abspath(options.file_path)) + abs_build_dir = os.path.join(src_dir, options.build_dir) + src_relpath = os.path.relpath(options.file_path, abs_build_dir) - print('Building %s' % options.file_path) + print('Building %s' % options.file_path) - carets = '^' - if sys.platform == 'win32': - # The caret character has to be escaped on Windows as it's an escape - # character. - carets = '^^' + carets = '^' + if sys.platform == 'win32': + # The caret character has to be escaped on Windows as it's an escape + # character. + carets = '^^' - command = [ - 'python3', - os.path.join(DEPOT_TOOLS_DIR, 'ninja.py'), '-C', abs_build_dir, - '%s%s' % (src_relpath, carets) - ] - # |shell| should be set to True on Windows otherwise the carets characters - # get dropped from the command line. - return subprocess.call(command, shell=sys.platform=='win32') + command = [ + 'python3', + os.path.join(DEPOT_TOOLS_DIR, 'ninja.py'), '-C', abs_build_dir, + '%s%s' % (src_relpath, carets) + ] + # |shell| should be set to True on Windows otherwise the carets characters + # get dropped from the command line. + return subprocess.call(command, shell=sys.platform == 'win32') if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/cpplint.py b/cpplint.py index 62662bf5ad..a8a8761243 100755 --- a/cpplint.py +++ b/cpplint.py @@ -29,7 +29,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # pylint: skip-file - """Does google-lint on c++ files. The goal of this script is to identify places in the code that *may* @@ -54,7 +53,6 @@ import string import sys import unicodedata - _USAGE = r""" Syntax: cpplint.py [--verbose=#] [--output=vs7] [--filter=-x,+y,...] [--counting=total|toplevel|detailed] [--root=subdir] @@ -244,14 +242,14 @@ _ERROR_CATEGORIES = [ 'whitespace/semicolon', 'whitespace/tab', 'whitespace/todo', - ] +] # These error categories are no longer enforced by cpplint, but for backwards- # compatibility they may still appear in NOLINT comments. _LEGACY_ERROR_CATEGORIES = [ 'readability/streams', 'readability/function', - ] +] # The default state of the category filter. This is overridden by the --filter= # flag. By default all errors are on, so only add here categories that should be @@ -262,12 +260,12 @@ _DEFAULT_FILTERS = ['-build/include_alpha'] # The default list of categories suppressed for C (not C++) files. _DEFAULT_C_SUPPRESSED_CATEGORIES = [ 'readability/casting', - ] +] # The default list of categories suppressed for Linux Kernel files. _DEFAULT_KERNEL_SUPPRESSED_CATEGORIES = [ 'whitespace/tab', - ] +] # We used to check for high-bit characters, but after much discussion we # decided those were OK, as long as they were in UTF-8 and didn't represent @@ -416,7 +414,7 @@ _CPP_HEADERS = frozenset([ 'cuchar', 'cwchar', 'cwctype', - ]) +]) # List of functions from . See [meta.type.synop] _TYPE_TRAITS = [ @@ -644,18 +642,16 @@ _TYPE_TRAITS = [ _TYPE_TRAITS_RE = re.compile(r'\b::(?:' + ('|'.join(_TYPE_TRAITS)) + ')<') # Type names -_TYPES = re.compile( - r'^(?:' - # [dcl.type.simple] - r'(char(16_t|32_t)?)|wchar_t|' - r'bool|short|int|long|signed|unsigned|float|double|' - # [support.types] - r'(ptrdiff_t|size_t|max_align_t|nullptr_t)|' - # [cstdint.syn] - r'(u?int(_fast|_least)?(8|16|32|64)_t)|' - r'(u?int(max|ptr)_t)|' - r')$') - +_TYPES = re.compile(r'^(?:' + # [dcl.type.simple] + r'(char(16_t|32_t)?)|wchar_t|' + r'bool|short|int|long|signed|unsigned|float|double|' + # [support.types] + r'(ptrdiff_t|size_t|max_align_t|nullptr_t)|' + # [cstdint.syn] + r'(u?int(_fast|_least)?(8|16|32|64)_t)|' + r'(u?int(max|ptr)_t)|' + r')$') # These headers are excluded from [build/include], [build/include_directory], # and [build/include_order] checks: @@ -674,27 +670,28 @@ _EMPTY_CONDITIONAL_BODY_PATTERN = re.compile(r'^\s*$', re.DOTALL) # Assertion macros. These are defined in base/logging.h and # testing/base/public/gunit.h. _CHECK_MACROS = [ - 'DCHECK', 'CHECK', - 'EXPECT_TRUE', 'ASSERT_TRUE', - 'EXPECT_FALSE', 'ASSERT_FALSE', - ] + 'DCHECK', + 'CHECK', + 'EXPECT_TRUE', + 'ASSERT_TRUE', + 'EXPECT_FALSE', + 'ASSERT_FALSE', +] # Replacement macros for CHECK/DCHECK/EXPECT_TRUE/EXPECT_FALSE _CHECK_REPLACEMENT = dict([(m, {}) for m in _CHECK_MACROS]) -for op, replacement in [('==', 'EQ'), ('!=', 'NE'), - ('>=', 'GE'), ('>', 'GT'), +for op, replacement in [('==', 'EQ'), ('!=', 'NE'), ('>=', 'GE'), ('>', 'GT'), ('<=', 'LE'), ('<', 'LT')]: - _CHECK_REPLACEMENT['DCHECK'][op] = 'DCHECK_%s' % replacement - _CHECK_REPLACEMENT['CHECK'][op] = 'CHECK_%s' % replacement - _CHECK_REPLACEMENT['EXPECT_TRUE'][op] = 'EXPECT_%s' % replacement - _CHECK_REPLACEMENT['ASSERT_TRUE'][op] = 'ASSERT_%s' % replacement + _CHECK_REPLACEMENT['DCHECK'][op] = 'DCHECK_%s' % replacement + _CHECK_REPLACEMENT['CHECK'][op] = 'CHECK_%s' % replacement + _CHECK_REPLACEMENT['EXPECT_TRUE'][op] = 'EXPECT_%s' % replacement + _CHECK_REPLACEMENT['ASSERT_TRUE'][op] = 'ASSERT_%s' % replacement -for op, inv_replacement in [('==', 'NE'), ('!=', 'EQ'), - ('>=', 'LT'), ('>', 'LE'), - ('<=', 'GT'), ('<', 'GE')]: - _CHECK_REPLACEMENT['EXPECT_FALSE'][op] = 'EXPECT_%s' % inv_replacement - _CHECK_REPLACEMENT['ASSERT_FALSE'][op] = 'ASSERT_%s' % inv_replacement +for op, inv_replacement in [('==', 'NE'), ('!=', 'EQ'), ('>=', 'LT'), + ('>', 'LE'), ('<=', 'GT'), ('<', 'GE')]: + _CHECK_REPLACEMENT['EXPECT_FALSE'][op] = 'EXPECT_%s' % inv_replacement + _CHECK_REPLACEMENT['ASSERT_FALSE'][op] = 'ASSERT_%s' % inv_replacement # Alternative tokens and their replacements. For full list, see section 2.5 # Alternative tokens [lex.digraph] in the C++ standard. @@ -713,7 +710,7 @@ _ALT_TOKEN_REPLACEMENT = { 'xor_eq': '^=', 'not': '!', 'not_eq': '!=' - } +} # Compile regular expression that matches all the above keywords. The "[ =()]" # bit is meant to avoid matching these keywords outside of boolean expressions. @@ -723,7 +720,6 @@ _ALT_TOKEN_REPLACEMENT = { _ALT_TOKEN_REPLACEMENT_PATTERN = re.compile( r'[ =()](' + ('|'.join(_ALT_TOKEN_REPLACEMENT.keys())) + r')(?=[ (]|$)') - # These constants define types of headers for use with # _IncludeState.CheckNextIncludeOrder(). _C_SYS_HEADER = 1 @@ -733,10 +729,10 @@ _POSSIBLE_MY_HEADER = 4 _OTHER_HEADER = 5 # These constants define the current inline assembly state -_NO_ASM = 0 # Outside of inline assembly block -_INSIDE_ASM = 1 # Inside inline assembly block -_END_ASM = 2 # Last line of inline assembly block -_BLOCK_ASM = 3 # The whole block is an inline assembly block +_NO_ASM = 0 # Outside of inline assembly block +_INSIDE_ASM = 1 # Inside inline assembly block +_END_ASM = 2 # Last line of inline assembly block +_BLOCK_ASM = 3 # The whole block is an inline assembly block # Match start of assembly blocks _MATCH_ASM = re.compile(r'^\s*(?:asm|_asm|__asm|__asm__)' @@ -779,7 +775,7 @@ _global_error_suppressions = {} def ParseNolintSuppressions(filename, raw_line, linenum, error): - """Updates the global list of line error-suppressions. + """Updates the global list of line error-suppressions. Parses any NOLINT comments on the current line, updating the global error_suppressions store. Reports an error if the NOLINT comment @@ -791,27 +787,28 @@ def ParseNolintSuppressions(filename, raw_line, linenum, error): linenum: int, the number of the current line. error: function, an error handler. """ - matched = Search(r'\bNOLINT(NEXTLINE)?\b(\([^)]+\))?', raw_line) - if matched: - if matched.group(1): - suppressed_line = linenum + 1 - else: - suppressed_line = linenum - category = matched.group(2) - if category in (None, '(*)'): # => "suppress all" - _error_suppressions.setdefault(None, set()).add(suppressed_line) - else: - if category.startswith('(') and category.endswith(')'): - category = category[1:-1] - if category in _ERROR_CATEGORIES: - _error_suppressions.setdefault(category, set()).add(suppressed_line) - elif category not in _LEGACY_ERROR_CATEGORIES: - error(filename, linenum, 'readability/nolint', 5, - 'Unknown NOLINT error category: %s' % category) + matched = Search(r'\bNOLINT(NEXTLINE)?\b(\([^)]+\))?', raw_line) + if matched: + if matched.group(1): + suppressed_line = linenum + 1 + else: + suppressed_line = linenum + category = matched.group(2) + if category in (None, '(*)'): # => "suppress all" + _error_suppressions.setdefault(None, set()).add(suppressed_line) + else: + if category.startswith('(') and category.endswith(')'): + category = category[1:-1] + if category in _ERROR_CATEGORIES: + _error_suppressions.setdefault(category, + set()).add(suppressed_line) + elif category not in _LEGACY_ERROR_CATEGORIES: + error(filename, linenum, 'readability/nolint', 5, + 'Unknown NOLINT error category: %s' % category) def ProcessGlobalSuppresions(lines): - """Updates the list of global error suppressions. + """Updates the list of global error suppressions. Parses any lint directives in the file that have global effect. @@ -819,23 +816,23 @@ def ProcessGlobalSuppresions(lines): lines: An array of strings, each representing a line of the file, with the last element being empty if the file is terminated with a newline. """ - for line in lines: - if _SEARCH_C_FILE.search(line): - for category in _DEFAULT_C_SUPPRESSED_CATEGORIES: - _global_error_suppressions[category] = True - if _SEARCH_KERNEL_FILE.search(line): - for category in _DEFAULT_KERNEL_SUPPRESSED_CATEGORIES: - _global_error_suppressions[category] = True + for line in lines: + if _SEARCH_C_FILE.search(line): + for category in _DEFAULT_C_SUPPRESSED_CATEGORIES: + _global_error_suppressions[category] = True + if _SEARCH_KERNEL_FILE.search(line): + for category in _DEFAULT_KERNEL_SUPPRESSED_CATEGORIES: + _global_error_suppressions[category] = True def ResetNolintSuppressions(): - """Resets the set of NOLINT suppressions to empty.""" - _error_suppressions.clear() - _global_error_suppressions.clear() + """Resets the set of NOLINT suppressions to empty.""" + _error_suppressions.clear() + _global_error_suppressions.clear() def IsErrorSuppressedByNolint(category, linenum): - """Returns true if the specified error category is suppressed on this line. + """Returns true if the specified error category is suppressed on this line. Consults the global error_suppressions map populated by ParseNolintSuppressions/ProcessGlobalSuppresions/ResetNolintSuppressions. @@ -847,23 +844,23 @@ def IsErrorSuppressedByNolint(category, linenum): bool, True iff the error should be suppressed due to a NOLINT comment or global suppression. """ - return (_global_error_suppressions.get(category, False) or - linenum in _error_suppressions.get(category, set()) or - linenum in _error_suppressions.get(None, set())) + return (_global_error_suppressions.get(category, False) + or linenum in _error_suppressions.get(category, set()) + or linenum in _error_suppressions.get(None, set())) def Match(pattern, s): - """Matches the string with the pattern, caching the compiled regexp.""" - # The regexp compilation caching is inlined in both Match and Search for - # performance reasons; factoring it out into a separate function turns out - # to be noticeably expensive. - if pattern not in _regexp_compile_cache: - _regexp_compile_cache[pattern] = sre_compile.compile(pattern) - return _regexp_compile_cache[pattern].match(s) + """Matches the string with the pattern, caching the compiled regexp.""" + # The regexp compilation caching is inlined in both Match and Search for + # performance reasons; factoring it out into a separate function turns out + # to be noticeably expensive. + if pattern not in _regexp_compile_cache: + _regexp_compile_cache[pattern] = sre_compile.compile(pattern) + return _regexp_compile_cache[pattern].match(s) def ReplaceAll(pattern, rep, s): - """Replaces instances of pattern in a string with a replacement. + """Replaces instances of pattern in a string with a replacement. The compiled regex is kept in a cache shared by Match and Search. @@ -875,25 +872,25 @@ def ReplaceAll(pattern, rep, s): Returns: string with replacements made (or original string if no replacements) """ - if pattern not in _regexp_compile_cache: - _regexp_compile_cache[pattern] = sre_compile.compile(pattern) - return _regexp_compile_cache[pattern].sub(rep, s) + if pattern not in _regexp_compile_cache: + _regexp_compile_cache[pattern] = sre_compile.compile(pattern) + return _regexp_compile_cache[pattern].sub(rep, s) def Search(pattern, s): - """Searches the string for the pattern, caching the compiled regexp.""" - if pattern not in _regexp_compile_cache: - _regexp_compile_cache[pattern] = sre_compile.compile(pattern) - return _regexp_compile_cache[pattern].search(s) + """Searches the string for the pattern, caching the compiled regexp.""" + if pattern not in _regexp_compile_cache: + _regexp_compile_cache[pattern] = sre_compile.compile(pattern) + return _regexp_compile_cache[pattern].search(s) def _IsSourceExtension(s): - """File extension (excluding dot) matches a source file extension.""" - return s in ('c', 'cc', 'cpp', 'cxx') + """File extension (excluding dot) matches a source file extension.""" + return s in ('c', 'cc', 'cpp', 'cxx') class _IncludeState(object): - """Tracks line numbers for includes, and the order in which includes appear. + """Tracks line numbers for includes, and the order in which includes appear. include_list contains list of lists of (header, line number) pairs. It's a lists of lists rather than just one flat list to make it @@ -904,35 +901,35 @@ class _IncludeState(object): raise an _IncludeError with an appropriate error message. """ - # self._section will move monotonically through this set. If it ever - # needs to move backwards, CheckNextIncludeOrder will raise an error. - _INITIAL_SECTION = 0 - _MY_H_SECTION = 1 - _C_SECTION = 2 - _CPP_SECTION = 3 - _OTHER_H_SECTION = 4 + # self._section will move monotonically through this set. If it ever + # needs to move backwards, CheckNextIncludeOrder will raise an error. + _INITIAL_SECTION = 0 + _MY_H_SECTION = 1 + _C_SECTION = 2 + _CPP_SECTION = 3 + _OTHER_H_SECTION = 4 - _TYPE_NAMES = { - _C_SYS_HEADER: 'C system header', - _CPP_SYS_HEADER: 'C++ system header', - _LIKELY_MY_HEADER: 'header this file implements', - _POSSIBLE_MY_HEADER: 'header this file may implement', - _OTHER_HEADER: 'other header', - } - _SECTION_NAMES = { - _INITIAL_SECTION: "... nothing. (This can't be an error.)", - _MY_H_SECTION: 'a header this file implements', - _C_SECTION: 'C system header', - _CPP_SECTION: 'C++ system header', - _OTHER_H_SECTION: 'other header', - } + _TYPE_NAMES = { + _C_SYS_HEADER: 'C system header', + _CPP_SYS_HEADER: 'C++ system header', + _LIKELY_MY_HEADER: 'header this file implements', + _POSSIBLE_MY_HEADER: 'header this file may implement', + _OTHER_HEADER: 'other header', + } + _SECTION_NAMES = { + _INITIAL_SECTION: "... nothing. (This can't be an error.)", + _MY_H_SECTION: 'a header this file implements', + _C_SECTION: 'C system header', + _CPP_SECTION: 'C++ system header', + _OTHER_H_SECTION: 'other header', + } - def __init__(self): - self.include_list = [[]] - self.ResetSection('') + def __init__(self): + self.include_list = [[]] + self.ResetSection('') - def FindHeader(self, header): - """Check if a header has already been included. + def FindHeader(self, header): + """Check if a header has already been included. Args: header: header to check. @@ -940,35 +937,35 @@ class _IncludeState(object): Line number of previous occurrence, or -1 if the header has not been seen before. """ - for section_list in self.include_list: - for f in section_list: - if f[0] == header: - return f[1] - return -1 + for section_list in self.include_list: + for f in section_list: + if f[0] == header: + return f[1] + return -1 - def ResetSection(self, directive): - """Reset section checking for preprocessor directive. + def ResetSection(self, directive): + """Reset section checking for preprocessor directive. Args: directive: preprocessor directive (e.g. "if", "else"). """ - # The name of the current section. - self._section = self._INITIAL_SECTION - # The path of last found header. - self._last_header = '' + # The name of the current section. + self._section = self._INITIAL_SECTION + # The path of last found header. + self._last_header = '' - # Update list of includes. Note that we never pop from the - # include list. - if directive in ('if', 'ifdef', 'ifndef'): - self.include_list.append([]) - elif directive in ('else', 'elif'): - self.include_list[-1] = [] + # Update list of includes. Note that we never pop from the + # include list. + if directive in ('if', 'ifdef', 'ifndef'): + self.include_list.append([]) + elif directive in ('else', 'elif'): + self.include_list[-1] = [] - def SetLastHeader(self, header_path): - self._last_header = header_path + def SetLastHeader(self, header_path): + self._last_header = header_path - def CanonicalizeAlphabeticalOrder(self, header_path): - """Returns a path canonicalized for alphabetical comparison. + def CanonicalizeAlphabeticalOrder(self, header_path): + """Returns a path canonicalized for alphabetical comparison. - replaces "-" with "_" so they both cmp the same. - removes '-inl' since we don't require them to be after the main header. @@ -980,10 +977,10 @@ class _IncludeState(object): Returns: Canonicalized path. """ - return header_path.replace('-inl.h', '.h').replace('-', '_').lower() + return header_path.replace('-inl.h', '.h').replace('-', '_').lower() - def IsInAlphabeticalOrder(self, clean_lines, linenum, header_path): - """Check if a header is in alphabetical order with the previous header. + def IsInAlphabeticalOrder(self, clean_lines, linenum, header_path): + """Check if a header is in alphabetical order with the previous header. Args: clean_lines: A CleansedLines instance containing the file. @@ -993,18 +990,19 @@ class _IncludeState(object): Returns: Returns true if the header is in alphabetical order. """ - # If previous section is different from current section, _last_header will - # be reset to empty string, so it's always less than current header. - # - # If previous line was a blank line, assume that the headers are - # intentionally sorted the way they are. - if (self._last_header > header_path and - Match(r'^\s*#\s*include\b', clean_lines.elided[linenum - 1])): - return False - return True + # If previous section is different from current section, _last_header + # will be reset to empty string, so it's always less than current + # header. + # + # If previous line was a blank line, assume that the headers are + # intentionally sorted the way they are. + if (self._last_header > header_path and Match( + r'^\s*#\s*include\b', clean_lines.elided[linenum - 1])): + return False + return True - def CheckNextIncludeOrder(self, header_type): - """Returns a non-empty error message if the next header is out of order. + def CheckNextIncludeOrder(self, header_type): + """Returns a non-empty error message if the next header is out of order. This function also updates the internal state to be ready to check the next include. @@ -1017,80 +1015,79 @@ class _IncludeState(object): error message describing what's wrong. """ - error_message = ('Found %s after %s' % - (self._TYPE_NAMES[header_type], - self._SECTION_NAMES[self._section])) + error_message = ( + 'Found %s after %s' % + (self._TYPE_NAMES[header_type], self._SECTION_NAMES[self._section])) - last_section = self._section + last_section = self._section - if header_type == _C_SYS_HEADER: - if self._section <= self._C_SECTION: - self._section = self._C_SECTION - else: - self._last_header = '' - return error_message - elif header_type == _CPP_SYS_HEADER: - if self._section <= self._CPP_SECTION: - self._section = self._CPP_SECTION - else: - self._last_header = '' - return error_message - elif header_type == _LIKELY_MY_HEADER: - if self._section <= self._MY_H_SECTION: - self._section = self._MY_H_SECTION - else: - self._section = self._OTHER_H_SECTION - elif header_type == _POSSIBLE_MY_HEADER: - if self._section <= self._MY_H_SECTION: - self._section = self._MY_H_SECTION - else: - # This will always be the fallback because we're not sure - # enough that the header is associated with this file. - self._section = self._OTHER_H_SECTION - else: - assert header_type == _OTHER_HEADER - self._section = self._OTHER_H_SECTION + if header_type == _C_SYS_HEADER: + if self._section <= self._C_SECTION: + self._section = self._C_SECTION + else: + self._last_header = '' + return error_message + elif header_type == _CPP_SYS_HEADER: + if self._section <= self._CPP_SECTION: + self._section = self._CPP_SECTION + else: + self._last_header = '' + return error_message + elif header_type == _LIKELY_MY_HEADER: + if self._section <= self._MY_H_SECTION: + self._section = self._MY_H_SECTION + else: + self._section = self._OTHER_H_SECTION + elif header_type == _POSSIBLE_MY_HEADER: + if self._section <= self._MY_H_SECTION: + self._section = self._MY_H_SECTION + else: + # This will always be the fallback because we're not sure + # enough that the header is associated with this file. + self._section = self._OTHER_H_SECTION + else: + assert header_type == _OTHER_HEADER + self._section = self._OTHER_H_SECTION - if last_section != self._section: - self._last_header = '' + if last_section != self._section: + self._last_header = '' - return '' + return '' class _CppLintState(object): - """Maintains module-wide state..""" + """Maintains module-wide state..""" + def __init__(self): + self.verbose_level = 1 # global setting. + self.error_count = 0 # global count of reported errors + # filters to apply when emitting error messages + self.filters = _DEFAULT_FILTERS[:] + # backup of filter list. Used to restore the state after each file. + self._filters_backup = self.filters[:] + self.counting = 'total' # In what way are we counting errors? + self.errors_by_category = {} # string to int dict storing error counts - def __init__(self): - self.verbose_level = 1 # global setting. - self.error_count = 0 # global count of reported errors - # filters to apply when emitting error messages - self.filters = _DEFAULT_FILTERS[:] - # backup of filter list. Used to restore the state after each file. - self._filters_backup = self.filters[:] - self.counting = 'total' # In what way are we counting errors? - self.errors_by_category = {} # string to int dict storing error counts + # output format: + # "emacs" - format that emacs can parse (default) + # "vs7" - format that Microsoft Visual Studio 7 can parse + self.output_format = 'emacs' - # output format: - # "emacs" - format that emacs can parse (default) - # "vs7" - format that Microsoft Visual Studio 7 can parse - self.output_format = 'emacs' + def SetOutputFormat(self, output_format): + """Sets the output format for errors.""" + self.output_format = output_format - def SetOutputFormat(self, output_format): - """Sets the output format for errors.""" - self.output_format = output_format + def SetVerboseLevel(self, level): + """Sets the module's verbosity, and returns the previous setting.""" + last_verbose_level = self.verbose_level + self.verbose_level = level + return last_verbose_level - def SetVerboseLevel(self, level): - """Sets the module's verbosity, and returns the previous setting.""" - last_verbose_level = self.verbose_level - self.verbose_level = level - return last_verbose_level + def SetCountingStyle(self, counting_style): + """Sets the module's counting options.""" + self.counting = counting_style - def SetCountingStyle(self, counting_style): - """Sets the module's counting options.""" - self.counting = counting_style - - def SetFilters(self, filters): - """Sets the error-message filters. + def SetFilters(self, filters): + """Sets the error-message filters. These filters are applied when deciding whether to emit a given error message. @@ -1103,86 +1100,88 @@ class _CppLintState(object): ValueError: The comma-separated filters did not all start with '+' or '-'. E.g. "-,+whitespace,-whitespace/indent,whitespace/badfilter" """ - # Default filters always have less priority than the flag ones. - self.filters = _DEFAULT_FILTERS[:] - self.AddFilters(filters) + # Default filters always have less priority than the flag ones. + self.filters = _DEFAULT_FILTERS[:] + self.AddFilters(filters) - def AddFilters(self, filters): - """ Adds more filters to the existing list of error-message filters. """ - for filt in filters.split(','): - clean_filt = filt.strip() - if clean_filt: - self.filters.append(clean_filt) - for filt in self.filters: - if not (filt.startswith('+') or filt.startswith('-')): - raise ValueError('Every filter in --filters must start with + or -' - ' (%s does not)' % filt) + def AddFilters(self, filters): + """ Adds more filters to the existing list of error-message filters. """ + for filt in filters.split(','): + clean_filt = filt.strip() + if clean_filt: + self.filters.append(clean_filt) + for filt in self.filters: + if not (filt.startswith('+') or filt.startswith('-')): + raise ValueError( + 'Every filter in --filters must start with + or -' + ' (%s does not)' % filt) - def BackupFilters(self): - """ Saves the current filter list to backup storage.""" - self._filters_backup = self.filters[:] + def BackupFilters(self): + """ Saves the current filter list to backup storage.""" + self._filters_backup = self.filters[:] - def RestoreFilters(self): - """ Restores filters previously backed up.""" - self.filters = self._filters_backup[:] + def RestoreFilters(self): + """ Restores filters previously backed up.""" + self.filters = self._filters_backup[:] - def ResetErrorCounts(self): - """Sets the module's error statistic back to zero.""" - self.error_count = 0 - self.errors_by_category = {} + def ResetErrorCounts(self): + """Sets the module's error statistic back to zero.""" + self.error_count = 0 + self.errors_by_category = {} - def IncrementErrorCount(self, category): - """Bumps the module's error statistic.""" - self.error_count += 1 - if self.counting in ('toplevel', 'detailed'): - if self.counting != 'detailed': - category = category.split('/')[0] - if category not in self.errors_by_category: - self.errors_by_category[category] = 0 - self.errors_by_category[category] += 1 + def IncrementErrorCount(self, category): + """Bumps the module's error statistic.""" + self.error_count += 1 + if self.counting in ('toplevel', 'detailed'): + if self.counting != 'detailed': + category = category.split('/')[0] + if category not in self.errors_by_category: + self.errors_by_category[category] = 0 + self.errors_by_category[category] += 1 + + def PrintErrorCounts(self): + """Print a summary of errors by category, and the total.""" + for category, count in self.errors_by_category.items(): + sys.stderr.write('Category \'%s\' errors found: %d\n' % + (category, count)) + sys.stderr.write('Total errors found: %d\n' % self.error_count) - def PrintErrorCounts(self): - """Print a summary of errors by category, and the total.""" - for category, count in self.errors_by_category.items(): - sys.stderr.write('Category \'%s\' errors found: %d\n' % - (category, count)) - sys.stderr.write('Total errors found: %d\n' % self.error_count) _cpplint_state = _CppLintState() def _OutputFormat(): - """Gets the module's output format.""" - return _cpplint_state.output_format + """Gets the module's output format.""" + return _cpplint_state.output_format def _SetOutputFormat(output_format): - """Sets the module's output format.""" - _cpplint_state.SetOutputFormat(output_format) + """Sets the module's output format.""" + _cpplint_state.SetOutputFormat(output_format) def _VerboseLevel(): - """Returns the module's verbosity setting.""" - return _cpplint_state.verbose_level + """Returns the module's verbosity setting.""" + return _cpplint_state.verbose_level def _SetVerboseLevel(level): - """Sets the module's verbosity, and returns the previous setting.""" - return _cpplint_state.SetVerboseLevel(level) + """Sets the module's verbosity, and returns the previous setting.""" + return _cpplint_state.SetVerboseLevel(level) def _SetCountingStyle(level): - """Sets the module's counting options.""" - _cpplint_state.SetCountingStyle(level) + """Sets the module's counting options.""" + _cpplint_state.SetCountingStyle(level) def _Filters(): - """Returns the module's list of output filters, as a list.""" - return _cpplint_state.filters + """Returns the module's list of output filters, as a list.""" + return _cpplint_state.filters def _SetFilters(filters): - """Sets the module's error-message filters. + """Sets the module's error-message filters. These filters are applied when deciding whether to emit a given error message. @@ -1191,10 +1190,11 @@ def _SetFilters(filters): filters: A string of comma-separated filters (eg "whitespace/indent"). Each filter should start with + or -; else we die. """ - _cpplint_state.SetFilters(filters) + _cpplint_state.SetFilters(filters) + def _AddFilters(filters): - """Adds more filter overrides. + """Adds more filter overrides. Unlike _SetFilters, this function does not reset the current list of filters available. @@ -1203,96 +1203,100 @@ def _AddFilters(filters): filters: A string of comma-separated filters (eg "whitespace/indent"). Each filter should start with + or -; else we die. """ - _cpplint_state.AddFilters(filters) + _cpplint_state.AddFilters(filters) + def _BackupFilters(): - """ Saves the current filter list to backup storage.""" - _cpplint_state.BackupFilters() + """ Saves the current filter list to backup storage.""" + _cpplint_state.BackupFilters() + def _RestoreFilters(): - """ Restores filters previously backed up.""" - _cpplint_state.RestoreFilters() + """ Restores filters previously backed up.""" + _cpplint_state.RestoreFilters() + class _FunctionState(object): - """Tracks current function name and the number of lines in its body.""" + """Tracks current function name and the number of lines in its body.""" - _NORMAL_TRIGGER = 250 # for --v=0, 500 for --v=1, etc. - _TEST_TRIGGER = 400 # about 50% more than _NORMAL_TRIGGER. + _NORMAL_TRIGGER = 250 # for --v=0, 500 for --v=1, etc. + _TEST_TRIGGER = 400 # about 50% more than _NORMAL_TRIGGER. - def __init__(self): - self.in_a_function = False - self.lines_in_function = 0 - self.current_function = '' + def __init__(self): + self.in_a_function = False + self.lines_in_function = 0 + self.current_function = '' - def Begin(self, function_name): - """Start analyzing function body. + def Begin(self, function_name): + """Start analyzing function body. Args: function_name: The name of the function being tracked. """ - self.in_a_function = True - self.lines_in_function = 0 - self.current_function = function_name + self.in_a_function = True + self.lines_in_function = 0 + self.current_function = function_name - def Count(self): - """Count line in current function body.""" - if self.in_a_function: - self.lines_in_function += 1 + def Count(self): + """Count line in current function body.""" + if self.in_a_function: + self.lines_in_function += 1 - def Check(self, error, filename, linenum): - """Report if too many lines in function body. + def Check(self, error, filename, linenum): + """Report if too many lines in function body. Args: error: The function to call with any errors found. filename: The name of the current file. linenum: The number of the line to check. """ - if not self.in_a_function: - return + if not self.in_a_function: + return - if Match(r'T(EST|est)', self.current_function): - base_trigger = self._TEST_TRIGGER - else: - base_trigger = self._NORMAL_TRIGGER - trigger = base_trigger * 2**_VerboseLevel() + if Match(r'T(EST|est)', self.current_function): + base_trigger = self._TEST_TRIGGER + else: + base_trigger = self._NORMAL_TRIGGER + trigger = base_trigger * 2**_VerboseLevel() - if self.lines_in_function > trigger: - error_level = int(math.log(self.lines_in_function / base_trigger, 2)) - # 50 => 0, 100 => 1, 200 => 2, 400 => 3, 800 => 4, 1600 => 5, ... - if error_level > 5: - error_level = 5 - error(filename, linenum, 'readability/fn_size', error_level, - 'Small and focused functions are preferred:' - ' %s has %d non-comment lines' - ' (error triggered by exceeding %d lines).' % ( - self.current_function, self.lines_in_function, trigger)) + if self.lines_in_function > trigger: + error_level = int(math.log(self.lines_in_function / base_trigger, + 2)) + # 50 => 0, 100 => 1, 200 => 2, 400 => 3, 800 => 4, 1600 => 5, ... + if error_level > 5: + error_level = 5 + error( + filename, linenum, 'readability/fn_size', error_level, + 'Small and focused functions are preferred:' + ' %s has %d non-comment lines' + ' (error triggered by exceeding %d lines).' % + (self.current_function, self.lines_in_function, trigger)) - def End(self): - """Stop analyzing function body.""" - self.in_a_function = False + def End(self): + """Stop analyzing function body.""" + self.in_a_function = False class _IncludeError(Exception): - """Indicates a problem with the include order in a file.""" - pass + """Indicates a problem with the include order in a file.""" + pass class FileInfo(object): - """Provides utility functions for filenames. + """Provides utility functions for filenames. FileInfo provides easy access to the components of a file's path relative to the project root. """ + def __init__(self, filename): + self._filename = filename - def __init__(self, filename): - self._filename = filename + def FullName(self): + """Make Windows paths like Unix.""" + return os.path.abspath(self._filename).replace('\\', '/') - def FullName(self): - """Make Windows paths like Unix.""" - return os.path.abspath(self._filename).replace('\\', '/') - - def RepositoryName(self): - r"""FullName after removing the local path to the repository. + def RepositoryName(self): + r"""FullName after removing the local path to the repository. If we have a real absolute path name here we can try to do something smart: detecting the root of the checkout and truncating /path/to/checkout from @@ -1301,47 +1305,48 @@ class FileInfo(object): people on different computers who have checked the source out to different locations won't see bogus errors. """ - fullname = self.FullName() + fullname = self.FullName() - if os.path.exists(fullname): - project_dir = os.path.dirname(fullname) + if os.path.exists(fullname): + project_dir = os.path.dirname(fullname) - if _project_root: - prefix = os.path.commonprefix([_project_root, project_dir]) - return fullname[len(prefix) + 1:] + if _project_root: + prefix = os.path.commonprefix([_project_root, project_dir]) + return fullname[len(prefix) + 1:] - if os.path.exists(os.path.join(project_dir, ".svn")): - # If there's a .svn file in the current directory, we recursively look - # up the directory tree for the top of the SVN checkout - root_dir = project_dir - one_up_dir = os.path.dirname(root_dir) - while os.path.exists(os.path.join(one_up_dir, ".svn")): - root_dir = os.path.dirname(root_dir) - one_up_dir = os.path.dirname(one_up_dir) + if os.path.exists(os.path.join(project_dir, ".svn")): + # If there's a .svn file in the current directory, we + # recursively look up the directory tree for the top of the SVN + # checkout + root_dir = project_dir + one_up_dir = os.path.dirname(root_dir) + while os.path.exists(os.path.join(one_up_dir, ".svn")): + root_dir = os.path.dirname(root_dir) + one_up_dir = os.path.dirname(one_up_dir) - prefix = os.path.commonprefix([root_dir, project_dir]) - return fullname[len(prefix) + 1:] + prefix = os.path.commonprefix([root_dir, project_dir]) + return fullname[len(prefix) + 1:] - # Not SVN <= 1.6? Try to find a git, hg, or svn top level directory by - # searching up from the current path. - root_dir = os.path.dirname(fullname) - while (root_dir != os.path.dirname(root_dir) and - not os.path.exists(os.path.join(root_dir, ".git")) and - not os.path.exists(os.path.join(root_dir, ".hg")) and - not os.path.exists(os.path.join(root_dir, ".svn"))): - root_dir = os.path.dirname(root_dir) + # Not SVN <= 1.6? Try to find a git, hg, or svn top level directory + # by searching up from the current path. + root_dir = os.path.dirname(fullname) + while (root_dir != os.path.dirname(root_dir) + and not os.path.exists(os.path.join(root_dir, ".git")) + and not os.path.exists(os.path.join(root_dir, ".hg")) + and not os.path.exists(os.path.join(root_dir, ".svn"))): + root_dir = os.path.dirname(root_dir) - if (os.path.exists(os.path.join(root_dir, ".git")) or - os.path.exists(os.path.join(root_dir, ".hg")) or - os.path.exists(os.path.join(root_dir, ".svn"))): - prefix = os.path.commonprefix([root_dir, project_dir]) - return fullname[len(prefix) + 1:] + if (os.path.exists(os.path.join(root_dir, ".git")) + or os.path.exists(os.path.join(root_dir, ".hg")) + or os.path.exists(os.path.join(root_dir, ".svn"))): + prefix = os.path.commonprefix([root_dir, project_dir]) + return fullname[len(prefix) + 1:] - # Don't know what to do; header guard warnings may be wrong... - return fullname + # Don't know what to do; header guard warnings may be wrong... + return fullname - def Split(self): - """Splits the file into the directory, basename, and extension. + def Split(self): + """Splits the file into the directory, basename, and extension. For 'chrome/browser/browser.cc', Split() would return ('chrome/browser', 'browser', '.cc') @@ -1350,57 +1355,57 @@ class FileInfo(object): A tuple of (directory, basename, extension). """ - googlename = self.RepositoryName() - project, rest = os.path.split(googlename) - return (project,) + os.path.splitext(rest) + googlename = self.RepositoryName() + project, rest = os.path.split(googlename) + return (project, ) + os.path.splitext(rest) - def BaseName(self): - """File base name - text after the final slash, before the final period.""" - return self.Split()[1] + def BaseName(self): + """File base name - text after the final slash, before the final period.""" + return self.Split()[1] - def Extension(self): - """File extension - text following the final period.""" - return self.Split()[2] + def Extension(self): + """File extension - text following the final period.""" + return self.Split()[2] - def NoExtension(self): - """File has no source file extension.""" - return '/'.join(self.Split()[0:2]) + def NoExtension(self): + """File has no source file extension.""" + return '/'.join(self.Split()[0:2]) - def IsSource(self): - """File has a source file extension.""" - return _IsSourceExtension(self.Extension()[1:]) + def IsSource(self): + """File has a source file extension.""" + return _IsSourceExtension(self.Extension()[1:]) def _ShouldPrintError(category, confidence, linenum): - """If confidence >= verbose, category passes filter and is not suppressed.""" + """If confidence >= verbose, category passes filter and is not suppressed.""" - # There are three ways we might decide not to print an error message: - # a "NOLINT(category)" comment appears in the source, - # the verbosity level isn't high enough, or the filters filter it out. - if IsErrorSuppressedByNolint(category, linenum): - return False + # There are three ways we might decide not to print an error message: + # a "NOLINT(category)" comment appears in the source, + # the verbosity level isn't high enough, or the filters filter it out. + if IsErrorSuppressedByNolint(category, linenum): + return False - if confidence < _cpplint_state.verbose_level: - return False + if confidence < _cpplint_state.verbose_level: + return False - is_filtered = False - for one_filter in _Filters(): - if one_filter.startswith('-'): - if category.startswith(one_filter[1:]): - is_filtered = True - elif one_filter.startswith('+'): - if category.startswith(one_filter[1:]): - is_filtered = False - else: - assert False # should have been checked for in SetFilter. - if is_filtered: - return False + is_filtered = False + for one_filter in _Filters(): + if one_filter.startswith('-'): + if category.startswith(one_filter[1:]): + is_filtered = True + elif one_filter.startswith('+'): + if category.startswith(one_filter[1:]): + is_filtered = False + else: + assert False # should have been checked for in SetFilter. + if is_filtered: + return False - return True + return True def Error(filename, linenum, category, confidence, message): - """Logs the fact we've found a lint error. + """Logs the fact we've found a lint error. We log where the error was found, and also our confidence in the error, that is, how certain we are this is a legitimate style regression, and @@ -1421,17 +1426,17 @@ def Error(filename, linenum, category, confidence, message): and 1 meaning that it could be a legitimate construct. message: The error message. """ - if _ShouldPrintError(category, confidence, linenum): - _cpplint_state.IncrementErrorCount(category) - if _cpplint_state.output_format == 'vs7': - sys.stderr.write('%s(%s): (cpplint) %s [%s] [%d]\n' % - (filename, linenum, message, category, confidence)) - elif _cpplint_state.output_format == 'eclipse': - sys.stderr.write('%s:%s: (cpplint) warning: %s [%s] [%d]\n' % - (filename, linenum, message, category, confidence)) - else: - sys.stderr.write('%s:%s: (cpplint) %s [%s] [%d]\n' % - (filename, linenum, message, category, confidence)) + if _ShouldPrintError(category, confidence, linenum): + _cpplint_state.IncrementErrorCount(category) + if _cpplint_state.output_format == 'vs7': + sys.stderr.write('%s(%s): (cpplint) %s [%s] [%d]\n' % + (filename, linenum, message, category, confidence)) + elif _cpplint_state.output_format == 'eclipse': + sys.stderr.write('%s:%s: (cpplint) warning: %s [%s] [%d]\n' % + (filename, linenum, message, category, confidence)) + else: + sys.stderr.write('%s:%s: (cpplint) %s [%s] [%d]\n' % + (filename, linenum, message, category, confidence)) # Matches standard C++ escape sequences per 2.13.2.3 of the C++ standard. @@ -1447,15 +1452,18 @@ _RE_PATTERN_C_COMMENTS = r'/\*(?:[^*]|\*(?!/))*\*/' # end of the line. Otherwise, we try to remove spaces from the right side, # if this doesn't work we try on left side but only if there's a non-character # on the right. -_RE_PATTERN_CLEANSE_LINE_C_COMMENTS = re.compile( - r'(\s*' + _RE_PATTERN_C_COMMENTS + r'\s*$|' + - _RE_PATTERN_C_COMMENTS + r'\s+|' + - r'\s+' + _RE_PATTERN_C_COMMENTS + r'(?=\W)|' + - _RE_PATTERN_C_COMMENTS + r')') +_RE_PATTERN_CLEANSE_LINE_C_COMMENTS = re.compile(r'(\s*' + + _RE_PATTERN_C_COMMENTS + + r'\s*$|' + + _RE_PATTERN_C_COMMENTS + + r'\s+|' + r'\s+' + + _RE_PATTERN_C_COMMENTS + + r'(?=\W)|' + + _RE_PATTERN_C_COMMENTS + r')') def IsCppString(line): - """Does line terminate so, that the next symbol is in string constant. + """Does line terminate so, that the next symbol is in string constant. This function does not consider single-line nor multi-line comments. @@ -1467,12 +1475,12 @@ def IsCppString(line): string constant. """ - line = line.replace(r'\\', 'XX') # after this, \\" does not match to \" - return ((line.count('"') - line.count(r'\"') - line.count("'\"'")) & 1) == 1 + line = line.replace(r'\\', 'XX') # after this, \\" does not match to \" + return ((line.count('"') - line.count(r'\"') - line.count("'\"'")) & 1) == 1 def CleanseRawStrings(raw_lines): - """Removes C++11 raw strings from lines. + """Removes C++11 raw strings from lines. Before: static const char kData[] = R"( @@ -1491,108 +1499,110 @@ def CleanseRawStrings(raw_lines): list of lines with C++11 raw strings replaced by empty strings. """ - delimiter = None - lines_without_raw_strings = [] - for line in raw_lines: - if delimiter: - # Inside a raw string, look for the end - end = line.find(delimiter) - if end >= 0: - # Found the end of the string, match leading space for this - # line and resume copying the original lines, and also insert - # a "" on the last line. - leading_space = Match(r'^(\s*)\S', line) - line = leading_space.group(1) + '""' + line[end + len(delimiter):] - delimiter = None - else: - # Haven't found the end yet, append a blank line. - line = '""' + delimiter = None + lines_without_raw_strings = [] + for line in raw_lines: + if delimiter: + # Inside a raw string, look for the end + end = line.find(delimiter) + if end >= 0: + # Found the end of the string, match leading space for this + # line and resume copying the original lines, and also insert + # a "" on the last line. + leading_space = Match(r'^(\s*)\S', line) + line = leading_space.group(1) + '""' + line[end + + len(delimiter):] + delimiter = None + else: + # Haven't found the end yet, append a blank line. + line = '""' - # Look for beginning of a raw string, and replace them with - # empty strings. This is done in a loop to handle multiple raw - # strings on the same line. - while delimiter is None: - # Look for beginning of a raw string. - # See 2.14.15 [lex.string] for syntax. - # - # Once we have matched a raw string, we check the prefix of the - # line to make sure that the line is not part of a single line - # comment. It's done this way because we remove raw strings - # before removing comments as opposed to removing comments - # before removing raw strings. This is because there are some - # cpplint checks that requires the comments to be preserved, but - # we don't want to check comments that are inside raw strings. - matched = Match(r'^(.*?)\b(?:R|u8R|uR|UR|LR)"([^\s\\()]*)\((.*)$', line) - if (matched and - not Match(r'^([^\'"]|\'(\\.|[^\'])*\'|"(\\.|[^"])*")*//', - matched.group(1))): - delimiter = ')' + matched.group(2) + '"' + # Look for beginning of a raw string, and replace them with + # empty strings. This is done in a loop to handle multiple raw + # strings on the same line. + while delimiter is None: + # Look for beginning of a raw string. + # See 2.14.15 [lex.string] for syntax. + # + # Once we have matched a raw string, we check the prefix of the + # line to make sure that the line is not part of a single line + # comment. It's done this way because we remove raw strings + # before removing comments as opposed to removing comments + # before removing raw strings. This is because there are some + # cpplint checks that requires the comments to be preserved, but + # we don't want to check comments that are inside raw strings. + matched = Match(r'^(.*?)\b(?:R|u8R|uR|UR|LR)"([^\s\\()]*)\((.*)$', + line) + if (matched and + not Match(r'^([^\'"]|\'(\\.|[^\'])*\'|"(\\.|[^"])*")*//', + matched.group(1))): + delimiter = ')' + matched.group(2) + '"' - end = matched.group(3).find(delimiter) - if end >= 0: - # Raw string ended on same line - line = (matched.group(1) + '""' + - matched.group(3)[end + len(delimiter):]) - delimiter = None - else: - # Start of a multi-line raw string - line = matched.group(1) + '""' - else: - break + end = matched.group(3).find(delimiter) + if end >= 0: + # Raw string ended on same line + line = (matched.group(1) + '""' + + matched.group(3)[end + len(delimiter):]) + delimiter = None + else: + # Start of a multi-line raw string + line = matched.group(1) + '""' + else: + break - lines_without_raw_strings.append(line) + lines_without_raw_strings.append(line) - # TODO(unknown): if delimiter is not None here, we might want to - # emit a warning for unterminated string. - return lines_without_raw_strings + # TODO(unknown): if delimiter is not None here, we might want to + # emit a warning for unterminated string. + return lines_without_raw_strings def FindNextMultiLineCommentStart(lines, lineix): - """Find the beginning marker for a multiline comment.""" - while lineix < len(lines): - if lines[lineix].strip().startswith('/*'): - # Only return this marker if the comment goes beyond this line - if lines[lineix].strip().find('*/', 2) < 0: - return lineix - lineix += 1 - return len(lines) + """Find the beginning marker for a multiline comment.""" + while lineix < len(lines): + if lines[lineix].strip().startswith('/*'): + # Only return this marker if the comment goes beyond this line + if lines[lineix].strip().find('*/', 2) < 0: + return lineix + lineix += 1 + return len(lines) def FindNextMultiLineCommentEnd(lines, lineix): - """We are inside a comment, find the end marker.""" - while lineix < len(lines): - if lines[lineix].strip().endswith('*/'): - return lineix - lineix += 1 - return len(lines) + """We are inside a comment, find the end marker.""" + while lineix < len(lines): + if lines[lineix].strip().endswith('*/'): + return lineix + lineix += 1 + return len(lines) def RemoveMultiLineCommentsFromRange(lines, begin, end): - """Clears a range of lines for multi-line comments.""" - # Having // dummy comments makes the lines non-empty, so we will not get - # unnecessary blank line warnings later in the code. - for i in range(begin, end): - lines[i] = '/**/' + """Clears a range of lines for multi-line comments.""" + # Having // dummy comments makes the lines non-empty, so we will not get + # unnecessary blank line warnings later in the code. + for i in range(begin, end): + lines[i] = '/**/' def RemoveMultiLineComments(filename, lines, error): - """Removes multiline (c-style) comments from lines.""" - lineix = 0 - while lineix < len(lines): - lineix_begin = FindNextMultiLineCommentStart(lines, lineix) - if lineix_begin >= len(lines): - return - lineix_end = FindNextMultiLineCommentEnd(lines, lineix_begin) - if lineix_end >= len(lines): - error(filename, lineix_begin + 1, 'readability/multiline_comment', 5, - 'Could not find end of multi-line comment') - return - RemoveMultiLineCommentsFromRange(lines, lineix_begin, lineix_end + 1) - lineix = lineix_end + 1 + """Removes multiline (c-style) comments from lines.""" + lineix = 0 + while lineix < len(lines): + lineix_begin = FindNextMultiLineCommentStart(lines, lineix) + if lineix_begin >= len(lines): + return + lineix_end = FindNextMultiLineCommentEnd(lines, lineix_begin) + if lineix_end >= len(lines): + error(filename, lineix_begin + 1, 'readability/multiline_comment', + 5, 'Could not find end of multi-line comment') + return + RemoveMultiLineCommentsFromRange(lines, lineix_begin, lineix_end + 1) + lineix = lineix_end + 1 def CleanseComments(line): - """Removes //-comments and single-line C-style /* */ comments. + """Removes //-comments and single-line C-style /* */ comments. Args: line: A line of C++ source. @@ -1600,15 +1610,15 @@ def CleanseComments(line): Returns: The line with single-line comments removed. """ - commentpos = line.find('//') - if commentpos != -1 and not IsCppString(line[:commentpos]): - line = line[:commentpos].rstrip() - # get rid of /* ... */ - return _RE_PATTERN_CLEANSE_LINE_C_COMMENTS.sub('', line) + commentpos = line.find('//') + if commentpos != -1 and not IsCppString(line[:commentpos]): + line = line[:commentpos].rstrip() + # get rid of /* ... */ + return _RE_PATTERN_CLEANSE_LINE_C_COMMENTS.sub('', line) class CleansedLines(object): - """Holds 4 copies of all lines with different preprocessing applied to them. + """Holds 4 copies of all lines with different preprocessing applied to them. 1) elided member contains lines without strings and comments. 2) lines member contains lines without comments. @@ -1617,26 +1627,26 @@ class CleansedLines(object): strings removed. All these members are of , and of the same length. """ + def __init__(self, lines): + self.elided = [] + self.lines = [] + self.raw_lines = lines + self.num_lines = len(lines) + self.lines_without_raw_strings = CleanseRawStrings(lines) + for linenum in range(len(self.lines_without_raw_strings)): + self.lines.append( + CleanseComments(self.lines_without_raw_strings[linenum])) + elided = self._CollapseStrings( + self.lines_without_raw_strings[linenum]) + self.elided.append(CleanseComments(elided)) - def __init__(self, lines): - self.elided = [] - self.lines = [] - self.raw_lines = lines - self.num_lines = len(lines) - self.lines_without_raw_strings = CleanseRawStrings(lines) - for linenum in range(len(self.lines_without_raw_strings)): - self.lines.append(CleanseComments( - self.lines_without_raw_strings[linenum])) - elided = self._CollapseStrings(self.lines_without_raw_strings[linenum]) - self.elided.append(CleanseComments(elided)) + def NumLines(self): + """Returns the number of lines represented.""" + return self.num_lines - def NumLines(self): - """Returns the number of lines represented.""" - return self.num_lines - - @staticmethod - def _CollapseStrings(elided): - """Collapses strings and chars on a line to simple "" or '' blocks. + @staticmethod + def _CollapseStrings(elided): + """Collapses strings and chars on a line to simple "" or '' blocks. We nix strings first so we're not fooled by text like '"http://"' @@ -1646,64 +1656,66 @@ class CleansedLines(object): Returns: The line with collapsed strings. """ - if _RE_PATTERN_INCLUDE.match(elided): - return elided + if _RE_PATTERN_INCLUDE.match(elided): + return elided - # Remove escaped characters first to make quote/single quote collapsing - # basic. Things that look like escaped characters shouldn't occur - # outside of strings and chars. - elided = _RE_PATTERN_CLEANSE_LINE_ESCAPES.sub('', elided) + # Remove escaped characters first to make quote/single quote collapsing + # basic. Things that look like escaped characters shouldn't occur + # outside of strings and chars. + elided = _RE_PATTERN_CLEANSE_LINE_ESCAPES.sub('', elided) - # Replace quoted strings and digit separators. Both single quotes - # and double quotes are processed in the same loop, otherwise - # nested quotes wouldn't work. - collapsed = '' - while True: - # Find the first quote character - match = Match(r'^([^\'"]*)([\'"])(.*)$', elided) - if not match: - collapsed += elided - break - head, quote, tail = match.groups() + # Replace quoted strings and digit separators. Both single quotes + # and double quotes are processed in the same loop, otherwise + # nested quotes wouldn't work. + collapsed = '' + while True: + # Find the first quote character + match = Match(r'^([^\'"]*)([\'"])(.*)$', elided) + if not match: + collapsed += elided + break + head, quote, tail = match.groups() - if quote == '"': - # Collapse double quoted strings - second_quote = tail.find('"') - if second_quote >= 0: - collapsed += head + '""' - elided = tail[second_quote + 1:] - else: - # Unmatched double quote, don't bother processing the rest - # of the line since this is probably a multiline string. - collapsed += elided - break - else: - # Found single quote, check nearby text to eliminate digit separators. - # - # There is no special handling for floating point here, because - # the integer/fractional/exponent parts would all be parsed - # correctly as long as there are digits on both sides of the - # separator. So we are fine as long as we don't see something - # like "0.'3" (gcc 4.9.0 will not allow this literal). - if Search(r'\b(?:0[bBxX]?|[1-9])[0-9a-fA-F]*$', head): - match_literal = Match(r'^((?:\'?[0-9a-zA-Z_])*)(.*)$', "'" + tail) - collapsed += head + match_literal.group(1).replace("'", '') - elided = match_literal.group(2) - else: - second_quote = tail.find('\'') - if second_quote >= 0: - collapsed += head + "''" - elided = tail[second_quote + 1:] - else: - # Unmatched single quote - collapsed += elided - break + if quote == '"': + # Collapse double quoted strings + second_quote = tail.find('"') + if second_quote >= 0: + collapsed += head + '""' + elided = tail[second_quote + 1:] + else: + # Unmatched double quote, don't bother processing the rest + # of the line since this is probably a multiline string. + collapsed += elided + break + else: + # Found single quote, check nearby text to eliminate digit + # separators. + # + # There is no special handling for floating point here, because + # the integer/fractional/exponent parts would all be parsed + # correctly as long as there are digits on both sides of the + # separator. So we are fine as long as we don't see something + # like "0.'3" (gcc 4.9.0 will not allow this literal). + if Search(r'\b(?:0[bBxX]?|[1-9])[0-9a-fA-F]*$', head): + match_literal = Match(r'^((?:\'?[0-9a-zA-Z_])*)(.*)$', + "'" + tail) + collapsed += head + match_literal.group(1).replace("'", '') + elided = match_literal.group(2) + else: + second_quote = tail.find('\'') + if second_quote >= 0: + collapsed += head + "''" + elided = tail[second_quote + 1:] + else: + # Unmatched single quote + collapsed += elided + break - return collapsed + return collapsed def FindEndOfExpressionInLine(line, startpos, stack): - """Find the position just after the end of current parenthesized expression. + """Find the position just after the end of current parenthesized expression. Args: line: a CleansedLines line. @@ -1715,73 +1727,74 @@ def FindEndOfExpressionInLine(line, startpos, stack): On finding an unclosed expression: (-1, None) Otherwise: (-1, new stack at end of this line) """ - for i in range(startpos, len(line)): - char = line[i] - if char in '([{': - # Found start of parenthesized expression, push to expression stack - stack.append(char) - elif char == '<': - # Found potential start of template argument list - if i > 0 and line[i - 1] == '<': - # Left shift operator - if stack and stack[-1] == '<': - stack.pop() - if not stack: - return (-1, None) - elif i > 0 and Search(r'\boperator\s*$', line[0:i]): - # operator<, don't add to stack - continue - else: - # Tentative start of template argument list - stack.append('<') - elif char in ')]}': - # Found end of parenthesized expression. - # - # If we are currently expecting a matching '>', the pending '<' - # must have been an operator. Remove them from expression stack. - while stack and stack[-1] == '<': - stack.pop() - if not stack: - return (-1, None) - if ((stack[-1] == '(' and char == ')') or - (stack[-1] == '[' and char == ']') or - (stack[-1] == '{' and char == '}')): - stack.pop() - if not stack: - return (i + 1, None) - else: - # Mismatched parentheses - return (-1, None) - elif char == '>': - # Found potential end of template argument list. + for i in range(startpos, len(line)): + char = line[i] + if char in '([{': + # Found start of parenthesized expression, push to expression stack + stack.append(char) + elif char == '<': + # Found potential start of template argument list + if i > 0 and line[i - 1] == '<': + # Left shift operator + if stack and stack[-1] == '<': + stack.pop() + if not stack: + return (-1, None) + elif i > 0 and Search(r'\boperator\s*$', line[0:i]): + # operator<, don't add to stack + continue + else: + # Tentative start of template argument list + stack.append('<') + elif char in ')]}': + # Found end of parenthesized expression. + # + # If we are currently expecting a matching '>', the pending '<' + # must have been an operator. Remove them from expression stack. + while stack and stack[-1] == '<': + stack.pop() + if not stack: + return (-1, None) + if ((stack[-1] == '(' and char == ')') + or (stack[-1] == '[' and char == ']') + or (stack[-1] == '{' and char == '}')): + stack.pop() + if not stack: + return (i + 1, None) + else: + # Mismatched parentheses + return (-1, None) + elif char == '>': + # Found potential end of template argument list. - # Ignore "->" and operator functions - if (i > 0 and - (line[i - 1] == '-' or Search(r'\boperator\s*$', line[0:i - 1]))): - continue + # Ignore "->" and operator functions + if (i > 0 and (line[i - 1] == '-' + or Search(r'\boperator\s*$', line[0:i - 1]))): + continue - # Pop the stack if there is a matching '<'. Otherwise, ignore - # this '>' since it must be an operator. - if stack: - if stack[-1] == '<': - stack.pop() - if not stack: - return (i + 1, None) - elif char == ';': - # Found something that look like end of statements. If we are currently - # expecting a '>', the matching '<' must have been an operator, since - # template argument list should not contain statements. - while stack and stack[-1] == '<': - stack.pop() - if not stack: - return (-1, None) + # Pop the stack if there is a matching '<'. Otherwise, ignore + # this '>' since it must be an operator. + if stack: + if stack[-1] == '<': + stack.pop() + if not stack: + return (i + 1, None) + elif char == ';': + # Found something that look like end of statements. If we are + # currently expecting a '>', the matching '<' must have been an + # operator, since template argument list should not contain + # statements. + while stack and stack[-1] == '<': + stack.pop() + if not stack: + return (-1, None) - # Did not find end of expression or unbalanced parentheses on this line - return (-1, stack) + # Did not find end of expression or unbalanced parentheses on this line + return (-1, stack) def CloseExpression(clean_lines, linenum, pos): - """If input points to ( or { or [ or <, finds the position that closes it. + """If input points to ( or { or [ or <, finds the position that closes it. If lines[linenum][pos] points to a '(' or '{' or '[' or '<', finds the linenum/pos that correspond to the closing of the expression. @@ -1803,29 +1816,29 @@ def CloseExpression(clean_lines, linenum, pos): 'cleansed' line at linenum. """ - line = clean_lines.elided[linenum] - if (line[pos] not in '({[<') or Match(r'<[<=]', line[pos:]): - return (line, clean_lines.NumLines(), -1) - - # Check first line - (end_pos, stack) = FindEndOfExpressionInLine(line, pos, []) - if end_pos > -1: - return (line, linenum, end_pos) - - # Continue scanning forward - while stack and linenum < clean_lines.NumLines() - 1: - linenum += 1 line = clean_lines.elided[linenum] - (end_pos, stack) = FindEndOfExpressionInLine(line, 0, stack) - if end_pos > -1: - return (line, linenum, end_pos) + if (line[pos] not in '({[<') or Match(r'<[<=]', line[pos:]): + return (line, clean_lines.NumLines(), -1) - # Did not find end of expression before end of file, give up - return (line, clean_lines.NumLines(), -1) + # Check first line + (end_pos, stack) = FindEndOfExpressionInLine(line, pos, []) + if end_pos > -1: + return (line, linenum, end_pos) + + # Continue scanning forward + while stack and linenum < clean_lines.NumLines() - 1: + linenum += 1 + line = clean_lines.elided[linenum] + (end_pos, stack) = FindEndOfExpressionInLine(line, 0, stack) + if end_pos > -1: + return (line, linenum, end_pos) + + # Did not find end of expression before end of file, give up + return (line, clean_lines.NumLines(), -1) def FindStartOfExpressionInLine(line, endpos, stack): - """Find position at the matching start of current expression. + """Find position at the matching start of current expression. This is almost the reverse of FindEndOfExpressionInLine, but note that the input position and returned position differs by 1. @@ -1840,69 +1853,68 @@ def FindStartOfExpressionInLine(line, endpos, stack): On finding an unclosed expression: (-1, None) Otherwise: (-1, new stack at beginning of this line) """ - i = endpos - while i >= 0: - char = line[i] - if char in ')]}': - # Found end of expression, push to expression stack - stack.append(char) - elif char == '>': - # Found potential end of template argument list. - # - # Ignore it if it's a "->" or ">=" or "operator>" - if (i > 0 and - (line[i - 1] == '-' or - Match(r'\s>=\s', line[i - 1:]) or - Search(r'\boperator\s*$', line[0:i]))): - i -= 1 - else: - stack.append('>') - elif char == '<': - # Found potential start of template argument list - if i > 0 and line[i - 1] == '<': - # Left shift operator - i -= 1 - else: - # If there is a matching '>', we can pop the expression stack. - # Otherwise, ignore this '<' since it must be an operator. - if stack and stack[-1] == '>': - stack.pop() - if not stack: - return (i, None) - elif char in '([{': - # Found start of expression. - # - # If there are any unmatched '>' on the stack, they must be - # operators. Remove those. - while stack and stack[-1] == '>': - stack.pop() - if not stack: - return (-1, None) - if ((char == '(' and stack[-1] == ')') or - (char == '[' and stack[-1] == ']') or - (char == '{' and stack[-1] == '}')): - stack.pop() - if not stack: - return (i, None) - else: - # Mismatched parentheses - return (-1, None) - elif char == ';': - # Found something that look like end of statements. If we are currently - # expecting a '<', the matching '>' must have been an operator, since - # template argument list should not contain statements. - while stack and stack[-1] == '>': - stack.pop() - if not stack: - return (-1, None) + i = endpos + while i >= 0: + char = line[i] + if char in ')]}': + # Found end of expression, push to expression stack + stack.append(char) + elif char == '>': + # Found potential end of template argument list. + # + # Ignore it if it's a "->" or ">=" or "operator>" + if (i > 0 and (line[i - 1] == '-' or Match(r'\s>=\s', line[i - 1:]) + or Search(r'\boperator\s*$', line[0:i]))): + i -= 1 + else: + stack.append('>') + elif char == '<': + # Found potential start of template argument list + if i > 0 and line[i - 1] == '<': + # Left shift operator + i -= 1 + else: + # If there is a matching '>', we can pop the expression stack. + # Otherwise, ignore this '<' since it must be an operator. + if stack and stack[-1] == '>': + stack.pop() + if not stack: + return (i, None) + elif char in '([{': + # Found start of expression. + # + # If there are any unmatched '>' on the stack, they must be + # operators. Remove those. + while stack and stack[-1] == '>': + stack.pop() + if not stack: + return (-1, None) + if ((char == '(' and stack[-1] == ')') + or (char == '[' and stack[-1] == ']') + or (char == '{' and stack[-1] == '}')): + stack.pop() + if not stack: + return (i, None) + else: + # Mismatched parentheses + return (-1, None) + elif char == ';': + # Found something that look like end of statements. If we are + # currently expecting a '<', the matching '>' must have been an + # operator, since template argument list should not contain + # statements. + while stack and stack[-1] == '>': + stack.pop() + if not stack: + return (-1, None) - i -= 1 + i -= 1 - return (-1, stack) + return (-1, stack) def ReverseCloseExpression(clean_lines, linenum, pos): - """If input points to ) or } or ] or >, finds the position that opens it. + """If input points to ) or } or ] or >, finds the position that opens it. If lines[linenum][pos] points to a ')' or '}' or ']' or '>', finds the linenum/pos that correspond to the opening of the expression. @@ -1918,42 +1930,44 @@ def ReverseCloseExpression(clean_lines, linenum, pos): we ignore strings and comments when matching; and the line we return is the 'cleansed' line at linenum. """ - line = clean_lines.elided[linenum] - if line[pos] not in ')}]>': - return (line, 0, -1) - - # Check last line - (start_pos, stack) = FindStartOfExpressionInLine(line, pos, []) - if start_pos > -1: - return (line, linenum, start_pos) - - # Continue scanning backward - while stack and linenum > 0: - linenum -= 1 line = clean_lines.elided[linenum] - (start_pos, stack) = FindStartOfExpressionInLine(line, len(line) - 1, stack) - if start_pos > -1: - return (line, linenum, start_pos) + if line[pos] not in ')}]>': + return (line, 0, -1) - # Did not find start of expression before beginning of file, give up - return (line, 0, -1) + # Check last line + (start_pos, stack) = FindStartOfExpressionInLine(line, pos, []) + if start_pos > -1: + return (line, linenum, start_pos) + + # Continue scanning backward + while stack and linenum > 0: + linenum -= 1 + line = clean_lines.elided[linenum] + (start_pos, + stack) = FindStartOfExpressionInLine(line, + len(line) - 1, stack) + if start_pos > -1: + return (line, linenum, start_pos) + + # Did not find start of expression before beginning of file, give up + return (line, 0, -1) def CheckForCopyright(filename, lines, error): - """Logs an error if no Copyright message appears at the top of the file.""" + """Logs an error if no Copyright message appears at the top of the file.""" - # We'll say it should occur by line 10. Don't forget there's a - # dummy line at the front. - for line in range(1, min(len(lines), 11)): - if re.search(r'Copyright', lines[line], re.I): break - else: # means no copyright line was found - error(filename, 0, 'legal/copyright', 5, - 'No copyright message found. ' - 'You should have a line: "Copyright [year] "') + # We'll say it should occur by line 10. Don't forget there's a + # dummy line at the front. + for line in range(1, min(len(lines), 11)): + if re.search(r'Copyright', lines[line], re.I): break + else: # means no copyright line was found + error( + filename, 0, 'legal/copyright', 5, 'No copyright message found. ' + 'You should have a line: "Copyright [year] "') def GetIndentLevel(line): - """Return the number of leading spaces in line. + """Return the number of leading spaces in line. Args: line: A string to check. @@ -1961,15 +1975,15 @@ def GetIndentLevel(line): Returns: An integer count of leading spaces, possibly zero. """ - indent = Match(r'^( *)\S', line) - if indent: - return len(indent.group(1)) - else: - return 0 + indent = Match(r'^( *)\S', line) + if indent: + return len(indent.group(1)) + else: + return 0 def PathSplitToList(path): - """Returns the path split into a list by the separator. + """Returns the path split into a list by the separator. Args: path: An absolute or relative path (e.g. '/a/b/c/' or '../a') @@ -1977,25 +1991,25 @@ def PathSplitToList(path): Returns: A list of path components (e.g. ['a', 'b', 'c]). """ - lst = [] - while True: - (head, tail) = os.path.split(path) - if head == path: # absolute paths end - lst.append(head) - break - if tail == path: # relative paths end - lst.append(tail) - break + lst = [] + while True: + (head, tail) = os.path.split(path) + if head == path: # absolute paths end + lst.append(head) + break + if tail == path: # relative paths end + lst.append(tail) + break - path = head - lst.append(tail) + path = head + lst.append(tail) - lst.reverse() - return lst + lst.reverse() + return lst def GetHeaderGuardCPPVariable(filename): - """Returns the CPP variable that should be used as a header guard. + """Returns the CPP variable that should be used as a header guard. Args: filename: The name of a C++ header file. @@ -2006,73 +2020,77 @@ def GetHeaderGuardCPPVariable(filename): """ - # Restores original filename in case that cpplint is invoked from Emacs's - # flymake. - filename = re.sub(r'_flymake\.h$', '.h', filename) - filename = re.sub(r'/\.flymake/([^/]*)$', r'/\1', filename) - # Replace 'c++' with 'cpp'. - filename = filename.replace('C++', 'cpp').replace('c++', 'cpp') + # Restores original filename in case that cpplint is invoked from Emacs's + # flymake. + filename = re.sub(r'_flymake\.h$', '.h', filename) + filename = re.sub(r'/\.flymake/([^/]*)$', r'/\1', filename) + # Replace 'c++' with 'cpp'. + filename = filename.replace('C++', 'cpp').replace('c++', 'cpp') - fileinfo = FileInfo(filename) - file_path_from_root = fileinfo.RepositoryName() + fileinfo = FileInfo(filename) + file_path_from_root = fileinfo.RepositoryName() - def FixupPathFromRoot(): - if _root_debug: - sys.stderr.write("\n_root fixup, _root = '%s', repository name = '%s'\n" - % (_root, fileinfo.RepositoryName())) + def FixupPathFromRoot(): + if _root_debug: + sys.stderr.write( + "\n_root fixup, _root = '%s', repository name = '%s'\n" % + (_root, fileinfo.RepositoryName())) - # Process the file path with the --root flag if it was set. - if not _root: - if _root_debug: - sys.stderr.write("_root unspecified\n") - return file_path_from_root + # Process the file path with the --root flag if it was set. + if not _root: + if _root_debug: + sys.stderr.write("_root unspecified\n") + return file_path_from_root - def StripListPrefix(lst, prefix): - # f(['x', 'y'], ['w, z']) -> None (not a valid prefix) - if lst[:len(prefix)] != prefix: - return None - # f(['a, 'b', 'c', 'd'], ['a', 'b']) -> ['c', 'd'] - return lst[(len(prefix)):] + def StripListPrefix(lst, prefix): + # f(['x', 'y'], ['w, z']) -> None (not a valid prefix) + if lst[:len(prefix)] != prefix: + return None + # f(['a, 'b', 'c', 'd'], ['a', 'b']) -> ['c', 'd'] + return lst[(len(prefix)):] - # root behavior: - # --root=subdir , lstrips subdir from the header guard - maybe_path = StripListPrefix(PathSplitToList(file_path_from_root), - PathSplitToList(_root)) + # root behavior: + # --root=subdir , lstrips subdir from the header guard + maybe_path = StripListPrefix(PathSplitToList(file_path_from_root), + PathSplitToList(_root)) - if _root_debug: - sys.stderr.write(("_root lstrip (maybe_path=%s, file_path_from_root=%s," + - " _root=%s)\n") % (maybe_path, file_path_from_root, _root)) + if _root_debug: + sys.stderr.write( + ("_root lstrip (maybe_path=%s, file_path_from_root=%s," + + " _root=%s)\n") % (maybe_path, file_path_from_root, _root)) - if maybe_path: - return os.path.join(*maybe_path) + if maybe_path: + return os.path.join(*maybe_path) - # --root=.. , will prepend the outer directory to the header guard - full_path = fileinfo.FullName() - # adapt slashes for windows - root_abspath = os.path.abspath(_root).replace('\\', '/') + # --root=.. , will prepend the outer directory to the header guard + full_path = fileinfo.FullName() + # adapt slashes for windows + root_abspath = os.path.abspath(_root).replace('\\', '/') - maybe_path = StripListPrefix(PathSplitToList(full_path), - PathSplitToList(root_abspath)) + maybe_path = StripListPrefix(PathSplitToList(full_path), + PathSplitToList(root_abspath)) - if _root_debug: - sys.stderr.write(("_root prepend (maybe_path=%s, full_path=%s, " + - "root_abspath=%s)\n") % (maybe_path, full_path, root_abspath)) + if _root_debug: + sys.stderr.write( + ("_root prepend (maybe_path=%s, full_path=%s, " + + "root_abspath=%s)\n") % (maybe_path, full_path, root_abspath)) - if maybe_path: - return os.path.join(*maybe_path) + if maybe_path: + return os.path.join(*maybe_path) - if _root_debug: - sys.stderr.write("_root ignore, returning %s\n" % (file_path_from_root)) + if _root_debug: + sys.stderr.write("_root ignore, returning %s\n" % + (file_path_from_root)) - # --root=FAKE_DIR is ignored - return file_path_from_root + # --root=FAKE_DIR is ignored + return file_path_from_root - file_path_from_root = FixupPathFromRoot() - return re.sub(r'[^a-zA-Z0-9]', '_', file_path_from_root).upper() + '_' + file_path_from_root = FixupPathFromRoot() + return re.sub(r'[^a-zA-Z0-9]', '_', file_path_from_root).upper() + '_' def CheckForHeaderGuard(filename, clean_lines, error): - """Checks that the file contains a header guard. + """Checks that the file contains a header guard. Logs an error if no #ifndef header guard is present. For other headers, checks that the full pathname is used. @@ -2083,119 +2101,123 @@ def CheckForHeaderGuard(filename, clean_lines, error): error: The function to call with any errors found. """ - # Don't check for header guards if there are error suppression - # comments somewhere in this file. - # - # Because this is silencing a warning for a nonexistent line, we - # only support the very specific NOLINT(build/header_guard) syntax, - # and not the general NOLINT or NOLINT(*) syntax. - raw_lines = clean_lines.lines_without_raw_strings - for i in raw_lines: - if Search(r'//\s*NOLINT\(build/header_guard\)', i): - return + # Don't check for header guards if there are error suppression + # comments somewhere in this file. + # + # Because this is silencing a warning for a nonexistent line, we + # only support the very specific NOLINT(build/header_guard) syntax, + # and not the general NOLINT or NOLINT(*) syntax. + raw_lines = clean_lines.lines_without_raw_strings + for i in raw_lines: + if Search(r'//\s*NOLINT\(build/header_guard\)', i): + return - cppvar = GetHeaderGuardCPPVariable(filename) + cppvar = GetHeaderGuardCPPVariable(filename) - ifndef = '' - ifndef_linenum = 0 - define = '' - endif = '' - endif_linenum = 0 - for linenum, line in enumerate(raw_lines): - linesplit = line.split() - if len(linesplit) >= 2: - # find the first occurrence of #ifndef and #define, save arg - if not ifndef and linesplit[0] == '#ifndef': - # set ifndef to the header guard presented on the #ifndef line. - ifndef = linesplit[1] - ifndef_linenum = linenum - if not define and linesplit[0] == '#define': - define = linesplit[1] - # find the last occurrence of #endif, save entire line - if line.startswith('#endif'): - endif = line - endif_linenum = linenum + ifndef = '' + ifndef_linenum = 0 + define = '' + endif = '' + endif_linenum = 0 + for linenum, line in enumerate(raw_lines): + linesplit = line.split() + if len(linesplit) >= 2: + # find the first occurrence of #ifndef and #define, save arg + if not ifndef and linesplit[0] == '#ifndef': + # set ifndef to the header guard presented on the #ifndef line. + ifndef = linesplit[1] + ifndef_linenum = linenum + if not define and linesplit[0] == '#define': + define = linesplit[1] + # find the last occurrence of #endif, save entire line + if line.startswith('#endif'): + endif = line + endif_linenum = linenum - if not ifndef or not define or ifndef != define: - error(filename, 0, 'build/header_guard', 5, - 'No #ifndef header guard found, suggested CPP variable is: %s' % - cppvar) - return + if not ifndef or not define or ifndef != define: + error( + filename, 0, 'build/header_guard', 5, + 'No #ifndef header guard found, suggested CPP variable is: %s' % + cppvar) + return - # The guard should be PATH_FILE_H_, but we also allow PATH_FILE_H__ - # for backward compatibility. - if ifndef != cppvar: - error_level = 0 - if ifndef != cppvar + '_': - error_level = 5 + # The guard should be PATH_FILE_H_, but we also allow PATH_FILE_H__ + # for backward compatibility. + if ifndef != cppvar: + error_level = 0 + if ifndef != cppvar + '_': + error_level = 5 - ParseNolintSuppressions(filename, raw_lines[ifndef_linenum], ifndef_linenum, + ParseNolintSuppressions(filename, raw_lines[ifndef_linenum], + ifndef_linenum, error) + error(filename, ifndef_linenum, 'build/header_guard', error_level, + '#ifndef header guard has wrong style, please use: %s' % cppvar) + + # Check for "//" comments on endif line. + ParseNolintSuppressions(filename, raw_lines[endif_linenum], endif_linenum, error) - error(filename, ifndef_linenum, 'build/header_guard', error_level, - '#ifndef header guard has wrong style, please use: %s' % cppvar) - - # Check for "//" comments on endif line. - ParseNolintSuppressions(filename, raw_lines[endif_linenum], endif_linenum, - error) - match = Match(r'#endif\s*//\s*' + cppvar + r'(_)?\b', endif) - if match: - if match.group(1) == '_': - # Issue low severity warning for deprecated double trailing underscore - error(filename, endif_linenum, 'build/header_guard', 0, - '#endif line should be "#endif // %s"' % cppvar) - return - - # Didn't find the corresponding "//" comment. If this file does not - # contain any "//" comments at all, it could be that the compiler - # only wants "/**/" comments, look for those instead. - no_single_line_comments = True - for i in range(1, len(raw_lines) - 1): - line = raw_lines[i] - if Match(r'^(?:(?:\'(?:\.|[^\'])*\')|(?:"(?:\.|[^"])*")|[^\'"])*//', line): - no_single_line_comments = False - break - - if no_single_line_comments: - match = Match(r'#endif\s*/\*\s*' + cppvar + r'(_)?\s*\*/', endif) + match = Match(r'#endif\s*//\s*' + cppvar + r'(_)?\b', endif) if match: - if match.group(1) == '_': - # Low severity warning for double trailing underscore - error(filename, endif_linenum, 'build/header_guard', 0, - '#endif line should be "#endif /* %s */"' % cppvar) - return + if match.group(1) == '_': + # Issue low severity warning for deprecated double trailing + # underscore + error(filename, endif_linenum, 'build/header_guard', 0, + '#endif line should be "#endif // %s"' % cppvar) + return - # Didn't find anything - error(filename, endif_linenum, 'build/header_guard', 5, - '#endif line should be "#endif // %s"' % cppvar) + # Didn't find the corresponding "//" comment. If this file does not + # contain any "//" comments at all, it could be that the compiler + # only wants "/**/" comments, look for those instead. + no_single_line_comments = True + for i in range(1, len(raw_lines) - 1): + line = raw_lines[i] + if Match(r'^(?:(?:\'(?:\.|[^\'])*\')|(?:"(?:\.|[^"])*")|[^\'"])*//', + line): + no_single_line_comments = False + break + + if no_single_line_comments: + match = Match(r'#endif\s*/\*\s*' + cppvar + r'(_)?\s*\*/', endif) + if match: + if match.group(1) == '_': + # Low severity warning for double trailing underscore + error(filename, endif_linenum, 'build/header_guard', 0, + '#endif line should be "#endif /* %s */"' % cppvar) + return + + # Didn't find anything + error(filename, endif_linenum, 'build/header_guard', 5, + '#endif line should be "#endif // %s"' % cppvar) def CheckHeaderFileIncluded(filename, include_state, error): - """Logs an error if a .cc file does not include its header.""" + """Logs an error if a .cc file does not include its header.""" - # Do not check test files - fileinfo = FileInfo(filename) - if Search(_TEST_FILE_SUFFIX, fileinfo.BaseName()): - return - - headerfile = filename[0:len(filename) - len(fileinfo.Extension())] + '.h' - if not os.path.exists(headerfile): - return - headername = FileInfo(headerfile).RepositoryName() - first_include = 0 - for section_list in include_state.include_list: - for f in section_list: - if headername in f[0] or f[0] in headername: + # Do not check test files + fileinfo = FileInfo(filename) + if Search(_TEST_FILE_SUFFIX, fileinfo.BaseName()): return - if not first_include: - first_include = f[1] - error(filename, first_include, 'build/include', 5, - '%s should include its header file %s' % (fileinfo.RepositoryName(), - headername)) + headerfile = filename[0:len(filename) - len(fileinfo.Extension())] + '.h' + if not os.path.exists(headerfile): + return + headername = FileInfo(headerfile).RepositoryName() + first_include = 0 + for section_list in include_state.include_list: + for f in section_list: + if headername in f[0] or f[0] in headername: + return + if not first_include: + first_include = f[1] + + error( + filename, first_include, 'build/include', 5, + '%s should include its header file %s' % + (fileinfo.RepositoryName(), headername)) def CheckForBadCharacters(filename, lines, error): - """Logs an error for each line containing bad characters. + """Logs an error for each line containing bad characters. Two kinds of bad characters: @@ -2211,16 +2233,19 @@ def CheckForBadCharacters(filename, lines, error): lines: An array of strings, each representing a line of the file. error: The function to call with any errors found. """ - for linenum, line in enumerate(lines): - if u'\ufffd' in line: - error(filename, linenum, 'readability/utf8', 5, - 'Line contains invalid UTF-8 (or Unicode replacement character).') - if '\0' in line: - error(filename, linenum, 'readability/nul', 5, 'Line contains NUL byte.') + for linenum, line in enumerate(lines): + if u'\ufffd' in line: + error( + filename, linenum, 'readability/utf8', 5, + 'Line contains invalid UTF-8 (or Unicode replacement character).' + ) + if '\0' in line: + error(filename, linenum, 'readability/nul', 5, + 'Line contains NUL byte.') def CheckForNewlineAtEOF(filename, lines, error): - """Logs an error if there is no newline char at the end of the file. + """Logs an error if there is no newline char at the end of the file. Args: filename: The name of the current file. @@ -2228,17 +2253,18 @@ def CheckForNewlineAtEOF(filename, lines, error): error: The function to call with any errors found. """ - # The array lines() was created by adding two newlines to the - # original file (go figure), then splitting on \n. - # To verify that the file ends in \n, we just have to make sure the - # last-but-two element of lines() exists and is empty. - if len(lines) < 3 or lines[-2]: - error(filename, len(lines) - 2, 'whitespace/ending_newline', 5, - 'Could not find a newline character at the end of the file.') + # The array lines() was created by adding two newlines to the + # original file (go figure), then splitting on \n. + # To verify that the file ends in \n, we just have to make sure the + # last-but-two element of lines() exists and is empty. + if len(lines) < 3 or lines[-2]: + error(filename, + len(lines) - 2, 'whitespace/ending_newline', 5, + 'Could not find a newline character at the end of the file.') def CheckForMultilineCommentsAndStrings(filename, clean_lines, linenum, error): - """Logs an error if we see /* ... */ or "..." that extend past one line. + """Logs an error if we see /* ... */ or "..." that extend past one line. /* ... */ comments are legit inside macros, for one line. Otherwise, we prefer // comments, so it's ok to warn about the @@ -2254,25 +2280,27 @@ def CheckForMultilineCommentsAndStrings(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] + line = clean_lines.elided[linenum] - # Remove all \\ (escaped backslashes) from the line. They are OK, and the - # second (escaped) slash may trigger later \" detection erroneously. - line = line.replace('\\\\', '') + # Remove all \\ (escaped backslashes) from the line. They are OK, and the + # second (escaped) slash may trigger later \" detection erroneously. + line = line.replace('\\\\', '') - if line.count('/*') > line.count('*/'): - error(filename, linenum, 'readability/multiline_comment', 5, - 'Complex multi-line /*...*/-style comment found. ' - 'Lint may give bogus warnings. ' - 'Consider replacing these with //-style comments, ' - 'with #if 0...#endif, ' - 'or with more clearly structured multi-line comments.') + if line.count('/*') > line.count('*/'): + error( + filename, linenum, 'readability/multiline_comment', 5, + 'Complex multi-line /*...*/-style comment found. ' + 'Lint may give bogus warnings. ' + 'Consider replacing these with //-style comments, ' + 'with #if 0...#endif, ' + 'or with more clearly structured multi-line comments.') - if (line.count('"') - line.count('\\"')) % 2: - error(filename, linenum, 'readability/multiline_string', 5, - 'Multi-line string ("...") found. This lint script doesn\'t ' - 'do well with such strings, and may give bogus warnings. ' - 'Use C++11 raw strings or concatenation instead.') + if (line.count('"') - line.count('\\"')) % 2: + error( + filename, linenum, 'readability/multiline_string', 5, + 'Multi-line string ("...") found. This lint script doesn\'t ' + 'do well with such strings, and may give bogus warnings. ' + 'Use C++11 raw strings or concatenation instead.') # (non-threadsafe name, thread-safe alternative, validation pattern) @@ -2299,14 +2327,13 @@ _THREADING_LIST = ( ('gmtime(', 'gmtime_r(', _UNSAFE_FUNC_PREFIX + r'gmtime\([^)]+\)'), ('localtime(', 'localtime_r(', _UNSAFE_FUNC_PREFIX + r'localtime\([^)]+\)'), ('rand(', 'rand_r(', _UNSAFE_FUNC_PREFIX + r'rand\(\)'), - ('strtok(', 'strtok_r(', - _UNSAFE_FUNC_PREFIX + r'strtok\([^)]+\)'), + ('strtok(', 'strtok_r(', _UNSAFE_FUNC_PREFIX + r'strtok\([^)]+\)'), ('ttyname(', 'ttyname_r(', _UNSAFE_FUNC_PREFIX + r'ttyname\([^)]+\)'), - ) +) def CheckPosixThreading(filename, clean_lines, linenum, error): - """Checks for calls to thread-unsafe functions. + """Checks for calls to thread-unsafe functions. Much code has been originally written without consideration of multi-threading. Also, engineers are relying on their old experience; @@ -2320,19 +2347,19 @@ def CheckPosixThreading(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - for single_thread_func, multithread_safe_func, pattern in _THREADING_LIST: - # Additional pattern matching check to confirm that this is the - # function we are looking for - if Search(pattern, line): - error(filename, linenum, 'runtime/threadsafe_fn', 2, - 'Consider using ' + multithread_safe_func + - '...) instead of ' + single_thread_func + - '...) for improved thread safety.') + line = clean_lines.elided[linenum] + for single_thread_func, multithread_safe_func, pattern in _THREADING_LIST: + # Additional pattern matching check to confirm that this is the + # function we are looking for + if Search(pattern, line): + error( + filename, linenum, 'runtime/threadsafe_fn', 2, + 'Consider using ' + multithread_safe_func + '...) instead of ' + + single_thread_func + '...) for improved thread safety.') def CheckVlogArguments(filename, clean_lines, linenum, error): - """Checks that VLOG() is only used for defining a logging level. + """Checks that VLOG() is only used for defining a logging level. For example, VLOG(2) is correct. VLOG(INFO), VLOG(WARNING), VLOG(ERROR), and VLOG(FATAL) are not. @@ -2343,20 +2370,21 @@ def CheckVlogArguments(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - if Search(r'\bVLOG\((INFO|ERROR|WARNING|DFATAL|FATAL)\)', line): - error(filename, linenum, 'runtime/vlog', 5, - 'VLOG() should be used with numeric verbosity level. ' - 'Use LOG() if you want symbolic severity levels.') + line = clean_lines.elided[linenum] + if Search(r'\bVLOG\((INFO|ERROR|WARNING|DFATAL|FATAL)\)', line): + error( + filename, linenum, 'runtime/vlog', 5, + 'VLOG() should be used with numeric verbosity level. ' + 'Use LOG() if you want symbolic severity levels.') + # Matches invalid increment: *count++, which moves pointer instead of # incrementing a value. -_RE_PATTERN_INVALID_INCREMENT = re.compile( - r'^\s*\*\w+(\+\+|--);') +_RE_PATTERN_INVALID_INCREMENT = re.compile(r'^\s*\*\w+(\+\+|--);') def CheckInvalidIncrement(filename, clean_lines, linenum, error): - """Checks for invalid increment *count++. + """Checks for invalid increment *count++. For example following function: void increment_counter(int* count) { @@ -2371,38 +2399,38 @@ def CheckInvalidIncrement(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - if _RE_PATTERN_INVALID_INCREMENT.match(line): - error(filename, linenum, 'runtime/invalid_increment', 5, - 'Changing pointer instead of value (or unused value of operator*).') + line = clean_lines.elided[linenum] + if _RE_PATTERN_INVALID_INCREMENT.match(line): + error( + filename, linenum, 'runtime/invalid_increment', 5, + 'Changing pointer instead of value (or unused value of operator*).') def IsMacroDefinition(clean_lines, linenum): - if Search(r'^#define', clean_lines[linenum]): - return True + if Search(r'^#define', clean_lines[linenum]): + return True - if linenum > 0 and Search(r'\\$', clean_lines[linenum - 1]): - return True + if linenum > 0 and Search(r'\\$', clean_lines[linenum - 1]): + return True - return False + return False def IsForwardClassDeclaration(clean_lines, linenum): - return Match(r'^\s*(\btemplate\b)*.*class\s+\w+;\s*$', clean_lines[linenum]) + return Match(r'^\s*(\btemplate\b)*.*class\s+\w+;\s*$', clean_lines[linenum]) class _BlockInfo(object): - """Stores information about a generic block of code.""" + """Stores information about a generic block of code.""" + def __init__(self, linenum, seen_open_brace): + self.starting_linenum = linenum + self.seen_open_brace = seen_open_brace + self.open_parentheses = 0 + self.inline_asm = _NO_ASM + self.check_namespace_indentation = False - def __init__(self, linenum, seen_open_brace): - self.starting_linenum = linenum - self.seen_open_brace = seen_open_brace - self.open_parentheses = 0 - self.inline_asm = _NO_ASM - self.check_namespace_indentation = False - - def CheckBegin(self, filename, clean_lines, linenum, error): - """Run checks that applies to text up to the opening brace. + def CheckBegin(self, filename, clean_lines, linenum, error): + """Run checks that applies to text up to the opening brace. This is mostly for checking the text after the class identifier and the "{", usually where the base class is specified. For other @@ -2414,10 +2442,10 @@ class _BlockInfo(object): linenum: The number of the line to check. error: The function to call with any errors found. """ - pass + pass - def CheckEnd(self, filename, clean_lines, linenum, error): - """Run checks that applies to text after the closing brace. + def CheckEnd(self, filename, clean_lines, linenum, error): + """Run checks that applies to text after the closing brace. This is mostly used for checking end of namespace comments. @@ -2427,10 +2455,10 @@ class _BlockInfo(object): linenum: The number of the line to check. error: The function to call with any errors found. """ - pass + pass - def IsBlockInfo(self): - """Returns true if this block is a _BlockInfo. + def IsBlockInfo(self): + """Returns true if this block is a _BlockInfo. This is convenient for verifying that an object is an instance of a _BlockInfo, but not an instance of any of the derived classes. @@ -2438,229 +2466,230 @@ class _BlockInfo(object): Returns: True for this class, False for derived classes. """ - return self.__class__ == _BlockInfo + return self.__class__ == _BlockInfo class _ExternCInfo(_BlockInfo): - """Stores information about an 'extern "C"' block.""" - - def __init__(self, linenum): - _BlockInfo.__init__(self, linenum, True) + """Stores information about an 'extern "C"' block.""" + def __init__(self, linenum): + _BlockInfo.__init__(self, linenum, True) class _ClassInfo(_BlockInfo): - """Stores information about a class.""" + """Stores information about a class.""" + def __init__(self, name, class_or_struct, clean_lines, linenum): + _BlockInfo.__init__(self, linenum, False) + self.name = name + self.is_derived = False + self.check_namespace_indentation = True + if class_or_struct == 'struct': + self.access = 'public' + self.is_struct = True + else: + self.access = 'private' + self.is_struct = False - def __init__(self, name, class_or_struct, clean_lines, linenum): - _BlockInfo.__init__(self, linenum, False) - self.name = name - self.is_derived = False - self.check_namespace_indentation = True - if class_or_struct == 'struct': - self.access = 'public' - self.is_struct = True - else: - self.access = 'private' - self.is_struct = False + # Remember initial indentation level for this class. Using raw_lines + # here instead of elided to account for leading comments. + self.class_indent = GetIndentLevel(clean_lines.raw_lines[linenum]) - # Remember initial indentation level for this class. Using raw_lines here - # instead of elided to account for leading comments. - self.class_indent = GetIndentLevel(clean_lines.raw_lines[linenum]) + # Try to find the end of the class. This will be confused by things + # like: class A { } *x = { ... + # + # But it's still good enough for CheckSectionSpacing. + self.last_line = 0 + depth = 0 + for i in range(linenum, clean_lines.NumLines()): + line = clean_lines.elided[i] + depth += line.count('{') - line.count('}') + if not depth: + self.last_line = i + break - # Try to find the end of the class. This will be confused by things like: - # class A { - # } *x = { ... - # - # But it's still good enough for CheckSectionSpacing. - self.last_line = 0 - depth = 0 - for i in range(linenum, clean_lines.NumLines()): - line = clean_lines.elided[i] - depth += line.count('{') - line.count('}') - if not depth: - self.last_line = i - break + def CheckBegin(self, filename, clean_lines, linenum, error): + # Look for a bare ':' + if Search('(^|[^:]):($|[^:])', clean_lines.elided[linenum]): + self.is_derived = True - def CheckBegin(self, filename, clean_lines, linenum, error): - # Look for a bare ':' - if Search('(^|[^:]):($|[^:])', clean_lines.elided[linenum]): - self.is_derived = True + def CheckEnd(self, filename, clean_lines, linenum, error): + # If there is a DISALLOW macro, it should appear near the end of + # the class. + seen_last_thing_in_class = False + for i in range(linenum - 1, self.starting_linenum, -1): + match = Search( + r'\b(DISALLOW_COPY_AND_ASSIGN|DISALLOW_IMPLICIT_CONSTRUCTORS)\(' + + self.name + r'\)', clean_lines.elided[i]) + if match: + if seen_last_thing_in_class: + error( + filename, i, 'readability/constructors', 3, + match.group(1) + + ' should be the last thing in the class') + break - def CheckEnd(self, filename, clean_lines, linenum, error): - # If there is a DISALLOW macro, it should appear near the end of - # the class. - seen_last_thing_in_class = False - for i in range(linenum - 1, self.starting_linenum, -1): - match = Search( - r'\b(DISALLOW_COPY_AND_ASSIGN|DISALLOW_IMPLICIT_CONSTRUCTORS)\(' + - self.name + r'\)', - clean_lines.elided[i]) - if match: - if seen_last_thing_in_class: - error(filename, i, 'readability/constructors', 3, - match.group(1) + ' should be the last thing in the class') - break + if not Match(r'^\s*$', clean_lines.elided[i]): + seen_last_thing_in_class = True - if not Match(r'^\s*$', clean_lines.elided[i]): - seen_last_thing_in_class = True - - # Check that closing brace is aligned with beginning of the class. - # Only do this if the closing brace is indented by only whitespaces. - # This means we will not check single-line class definitions. - indent = Match(r'^( *)\}', clean_lines.elided[linenum]) - if indent and len(indent.group(1)) != self.class_indent: - if self.is_struct: - parent = 'struct ' + self.name - else: - parent = 'class ' + self.name - error(filename, linenum, 'whitespace/indent', 3, - 'Closing brace should be aligned with beginning of %s' % parent) + # Check that closing brace is aligned with beginning of the class. + # Only do this if the closing brace is indented by only whitespaces. + # This means we will not check single-line class definitions. + indent = Match(r'^( *)\}', clean_lines.elided[linenum]) + if indent and len(indent.group(1)) != self.class_indent: + if self.is_struct: + parent = 'struct ' + self.name + else: + parent = 'class ' + self.name + error( + filename, linenum, 'whitespace/indent', 3, + 'Closing brace should be aligned with beginning of %s' % parent) class _NamespaceInfo(_BlockInfo): - """Stores information about a namespace.""" + """Stores information about a namespace.""" + def __init__(self, name, linenum): + _BlockInfo.__init__(self, linenum, False) + self.name = name or '' + self.check_namespace_indentation = True - def __init__(self, name, linenum): - _BlockInfo.__init__(self, linenum, False) - self.name = name or '' - self.check_namespace_indentation = True + def CheckEnd(self, filename, clean_lines, linenum, error): + """Check end of namespace comments.""" + line = clean_lines.raw_lines[linenum] - def CheckEnd(self, filename, clean_lines, linenum, error): - """Check end of namespace comments.""" - line = clean_lines.raw_lines[linenum] + # Check how many lines is enclosed in this namespace. Don't issue + # warning for missing namespace comments if there aren't enough + # lines. However, do apply checks if there is already an end of + # namespace comment and it's incorrect. + # + # TODO(unknown): We always want to check end of namespace comments + # if a namespace is large, but sometimes we also want to apply the + # check if a short namespace contained nontrivial things (something + # other than forward declarations). There is currently no logic on + # deciding what these nontrivial things are, so this check is + # triggered by namespace size only, which works most of the time. + if (linenum - self.starting_linenum < 10 + and not Match(r'^\s*};*\s*(//|/\*).*\bnamespace\b', line)): + return - # Check how many lines is enclosed in this namespace. Don't issue - # warning for missing namespace comments if there aren't enough - # lines. However, do apply checks if there is already an end of - # namespace comment and it's incorrect. - # - # TODO(unknown): We always want to check end of namespace comments - # if a namespace is large, but sometimes we also want to apply the - # check if a short namespace contained nontrivial things (something - # other than forward declarations). There is currently no logic on - # deciding what these nontrivial things are, so this check is - # triggered by namespace size only, which works most of the time. - if (linenum - self.starting_linenum < 10 - and not Match(r'^\s*};*\s*(//|/\*).*\bnamespace\b', line)): - return - - # Look for matching comment at end of namespace. - # - # Note that we accept C style "/* */" comments for terminating - # namespaces, so that code that terminate namespaces inside - # preprocessor macros can be cpplint clean. - # - # We also accept stuff like "// end of namespace ." with the - # period at the end. - # - # Besides these, we don't accept anything else, otherwise we might - # get false negatives when existing comment is a substring of the - # expected namespace. - if self.name: - # Named namespace - if not Match((r'^\s*};*\s*(//|/\*).*\bnamespace\s+' + - re.escape(self.name) + r'[\*/\.\\\s]*$'), - line): - error(filename, linenum, 'readability/namespace', 5, - 'Namespace should be terminated with "// namespace %s"' % - self.name) - else: - # Anonymous namespace - if not Match(r'^\s*};*\s*(//|/\*).*\bnamespace[\*/\.\\\s]*$', line): - # If "// namespace anonymous" or "// anonymous namespace (more text)", - # mention "// anonymous namespace" as an acceptable form - if Match(r'^\s*}.*\b(namespace anonymous|anonymous namespace)\b', line): - error(filename, linenum, 'readability/namespace', 5, - 'Anonymous namespace should be terminated with "// namespace"' - ' or "// anonymous namespace"') + # Look for matching comment at end of namespace. + # + # Note that we accept C style "/* */" comments for terminating + # namespaces, so that code that terminate namespaces inside + # preprocessor macros can be cpplint clean. + # + # We also accept stuff like "// end of namespace ." with the + # period at the end. + # + # Besides these, we don't accept anything else, otherwise we might + # get false negatives when existing comment is a substring of the + # expected namespace. + if self.name: + # Named namespace + if not Match((r'^\s*};*\s*(//|/\*).*\bnamespace\s+' + + re.escape(self.name) + r'[\*/\.\\\s]*$'), line): + error( + filename, linenum, 'readability/namespace', 5, + 'Namespace should be terminated with "// namespace %s"' % + self.name) else: - error(filename, linenum, 'readability/namespace', 5, - 'Anonymous namespace should be terminated with "// namespace"') + # Anonymous namespace + if not Match(r'^\s*};*\s*(//|/\*).*\bnamespace[\*/\.\\\s]*$', line): + # If "// namespace anonymous" or "// anonymous namespace (more + # text)", mention "// anonymous namespace" as an acceptable form + if Match( + r'^\s*}.*\b(namespace anonymous|anonymous namespace)\b', + line): + error( + filename, linenum, 'readability/namespace', 5, + 'Anonymous namespace should be terminated with "// namespace"' + ' or "// anonymous namespace"') + else: + error( + filename, linenum, 'readability/namespace', 5, + 'Anonymous namespace should be terminated with "// namespace"' + ) class _PreprocessorInfo(object): - """Stores checkpoints of nesting stacks when #if/#else is seen.""" + """Stores checkpoints of nesting stacks when #if/#else is seen.""" + def __init__(self, stack_before_if): + # The entire nesting stack before #if + self.stack_before_if = stack_before_if - def __init__(self, stack_before_if): - # The entire nesting stack before #if - self.stack_before_if = stack_before_if + # The entire nesting stack up to #else + self.stack_before_else = [] - # The entire nesting stack up to #else - self.stack_before_else = [] - - # Whether we have already seen #else or #elif - self.seen_else = False + # Whether we have already seen #else or #elif + self.seen_else = False class NestingState(object): - """Holds states related to parsing braces.""" + """Holds states related to parsing braces.""" + def __init__(self): + # Stack for tracking all braces. An object is pushed whenever we + # see a "{", and popped when we see a "}". Only 3 types of + # objects are possible: + # - _ClassInfo: a class or struct. + # - _NamespaceInfo: a namespace. + # - _BlockInfo: some other type of block. + self.stack = [] - def __init__(self): - # Stack for tracking all braces. An object is pushed whenever we - # see a "{", and popped when we see a "}". Only 3 types of - # objects are possible: - # - _ClassInfo: a class or struct. - # - _NamespaceInfo: a namespace. - # - _BlockInfo: some other type of block. - self.stack = [] + # Top of the previous stack before each Update(). + # + # Because the nesting_stack is updated at the end of each line, we + # had to do some convoluted checks to find out what is the current + # scope at the beginning of the line. This check is simplified by + # saving the previous top of nesting stack. + # + # We could save the full stack, but we only need the top. Copying + # the full nesting stack would slow down cpplint by ~10%. + self.previous_stack_top = [] - # Top of the previous stack before each Update(). - # - # Because the nesting_stack is updated at the end of each line, we - # had to do some convoluted checks to find out what is the current - # scope at the beginning of the line. This check is simplified by - # saving the previous top of nesting stack. - # - # We could save the full stack, but we only need the top. Copying - # the full nesting stack would slow down cpplint by ~10%. - self.previous_stack_top = [] + # Stack of _PreprocessorInfo objects. + self.pp_stack = [] - # Stack of _PreprocessorInfo objects. - self.pp_stack = [] - - def SeenOpenBrace(self): - """Check if we have seen the opening brace for the innermost block. + def SeenOpenBrace(self): + """Check if we have seen the opening brace for the innermost block. Returns: True if we have seen the opening brace, False if the innermost block is still expecting an opening brace. """ - return (not self.stack) or self.stack[-1].seen_open_brace + return (not self.stack) or self.stack[-1].seen_open_brace - def InNamespaceBody(self): - """Check if we are currently one level inside a namespace body. + def InNamespaceBody(self): + """Check if we are currently one level inside a namespace body. Returns: True if top of the stack is a namespace block, False otherwise. """ - return self.stack and isinstance(self.stack[-1], _NamespaceInfo) + return self.stack and isinstance(self.stack[-1], _NamespaceInfo) - def InExternC(self): - """Check if we are currently one level inside an 'extern "C"' block. + def InExternC(self): + """Check if we are currently one level inside an 'extern "C"' block. Returns: True if top of the stack is an extern block, False otherwise. """ - return self.stack and isinstance(self.stack[-1], _ExternCInfo) + return self.stack and isinstance(self.stack[-1], _ExternCInfo) - def InClassDeclaration(self): - """Check if we are currently one level inside a class or struct declaration. + def InClassDeclaration(self): + """Check if we are currently one level inside a class or struct declaration. Returns: True if top of the stack is a class/struct, False otherwise. """ - return self.stack and isinstance(self.stack[-1], _ClassInfo) + return self.stack and isinstance(self.stack[-1], _ClassInfo) - def InAsmBlock(self): - """Check if we are currently one level inside an inline ASM block. + def InAsmBlock(self): + """Check if we are currently one level inside an inline ASM block. Returns: True if the top of the stack is a block containing inline ASM. """ - return self.stack and self.stack[-1].inline_asm != _NO_ASM + return self.stack and self.stack[-1].inline_asm != _NO_ASM - def InTemplateArgumentList(self, clean_lines, linenum, pos): - """Check if current position is inside template argument list. + def InTemplateArgumentList(self, clean_lines, linenum, pos): + """Check if current position is inside template argument list. Args: clean_lines: A CleansedLines instance containing the file. @@ -2669,50 +2698,52 @@ class NestingState(object): Returns: True if (linenum, pos) is inside template arguments. """ - while linenum < clean_lines.NumLines(): - # Find the earliest character that might indicate a template argument - line = clean_lines.elided[linenum] - match = Match(r'^[^{};=\[\]\.<>]*(.)', line[pos:]) - if not match: - linenum += 1 - pos = 0 - continue - token = match.group(1) - pos += len(match.group(0)) + while linenum < clean_lines.NumLines(): + # Find the earliest character that might indicate a template + # argument + line = clean_lines.elided[linenum] + match = Match(r'^[^{};=\[\]\.<>]*(.)', line[pos:]) + if not match: + linenum += 1 + pos = 0 + continue + token = match.group(1) + pos += len(match.group(0)) - # These things do not look like template argument list: - # class Suspect { - # class Suspect x; } - if token in ('{', '}', ';'): return False + # These things do not look like template argument list: + # class Suspect { + # class Suspect x; } + if token in ('{', '}', ';'): return False - # These things look like template argument list: - # template - # template - # template - # template - if token in ('>', '=', '[', ']', '.'): return True + # These things look like template argument list: + # template + # template + # template + # template + if token in ('>', '=', '[', ']', '.'): return True - # Check if token is an unmatched '<'. - # If not, move on to the next character. - if token != '<': - pos += 1 - if pos >= len(line): - linenum += 1 - pos = 0 - continue + # Check if token is an unmatched '<'. + # If not, move on to the next character. + if token != '<': + pos += 1 + if pos >= len(line): + linenum += 1 + pos = 0 + continue - # We can't be sure if we just find a single '<', and need to - # find the matching '>'. - (_, end_line, end_pos) = CloseExpression(clean_lines, linenum, pos - 1) - if end_pos < 0: - # Not sure if template argument list or syntax error in file + # We can't be sure if we just find a single '<', and need to + # find the matching '>'. + (_, end_line, end_pos) = CloseExpression(clean_lines, linenum, + pos - 1) + if end_pos < 0: + # Not sure if template argument list or syntax error in file + return False + linenum = end_line + pos = end_pos return False - linenum = end_line - pos = end_pos - return False - def UpdatePreprocessor(self, line): - """Update preprocessor stack. + def UpdatePreprocessor(self, line): + """Update preprocessor stack. We need to handle preprocessors due to classes like this: #ifdef SWIG @@ -2732,44 +2763,46 @@ class NestingState(object): Args: line: current line to check. """ - if Match(r'^\s*#\s*(if|ifdef|ifndef)\b', line): - # Beginning of #if block, save the nesting stack here. The saved - # stack will allow us to restore the parsing state in the #else case. - self.pp_stack.append(_PreprocessorInfo(copy.deepcopy(self.stack))) - elif Match(r'^\s*#\s*(else|elif)\b', line): - # Beginning of #else block - if self.pp_stack: - if not self.pp_stack[-1].seen_else: - # This is the first #else or #elif block. Remember the - # whole nesting stack up to this point. This is what we - # keep after the #endif. - self.pp_stack[-1].seen_else = True - self.pp_stack[-1].stack_before_else = copy.deepcopy(self.stack) + if Match(r'^\s*#\s*(if|ifdef|ifndef)\b', line): + # Beginning of #if block, save the nesting stack here. The saved + # stack will allow us to restore the parsing state in the #else + # case. + self.pp_stack.append(_PreprocessorInfo(copy.deepcopy(self.stack))) + elif Match(r'^\s*#\s*(else|elif)\b', line): + # Beginning of #else block + if self.pp_stack: + if not self.pp_stack[-1].seen_else: + # This is the first #else or #elif block. Remember the + # whole nesting stack up to this point. This is what we + # keep after the #endif. + self.pp_stack[-1].seen_else = True + self.pp_stack[-1].stack_before_else = copy.deepcopy( + self.stack) - # Restore the stack to how it was before the #if - self.stack = copy.deepcopy(self.pp_stack[-1].stack_before_if) - else: - # TODO(unknown): unexpected #else, issue warning? - pass - elif Match(r'^\s*#\s*endif\b', line): - # End of #if or #else blocks. - if self.pp_stack: - # If we saw an #else, we will need to restore the nesting - # stack to its former state before the #else, otherwise we - # will just continue from where we left off. - if self.pp_stack[-1].seen_else: - # Here we can just use a shallow copy since we are the last - # reference to it. - self.stack = self.pp_stack[-1].stack_before_else - # Drop the corresponding #if - self.pp_stack.pop() - else: - # TODO(unknown): unexpected #endif, issue warning? - pass + # Restore the stack to how it was before the #if + self.stack = copy.deepcopy(self.pp_stack[-1].stack_before_if) + else: + # TODO(unknown): unexpected #else, issue warning? + pass + elif Match(r'^\s*#\s*endif\b', line): + # End of #if or #else blocks. + if self.pp_stack: + # If we saw an #else, we will need to restore the nesting + # stack to its former state before the #else, otherwise we + # will just continue from where we left off. + if self.pp_stack[-1].seen_else: + # Here we can just use a shallow copy since we are the last + # reference to it. + self.stack = self.pp_stack[-1].stack_before_else + # Drop the corresponding #if + self.pp_stack.pop() + else: + # TODO(unknown): unexpected #endif, issue warning? + pass - # TODO(unknown): Update() is too long, but we will refactor later. - def Update(self, filename, clean_lines, linenum, error): - """Update nesting state with current line. + # TODO(unknown): Update() is too long, but we will refactor later. + def Update(self, filename, clean_lines, linenum, error): + """Update nesting state with current line. Args: filename: The name of the current file. @@ -2777,198 +2810,201 @@ class NestingState(object): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] + line = clean_lines.elided[linenum] - # Remember top of the previous nesting stack. - # - # The stack is always pushed/popped and not modified in place, so - # we can just do a shallow copy instead of copy.deepcopy. Using - # deepcopy would slow down cpplint by ~28%. - if self.stack: - self.previous_stack_top = self.stack[-1] - else: - self.previous_stack_top = None - - # Update pp_stack - self.UpdatePreprocessor(line) - - # Count parentheses. This is to avoid adding struct arguments to - # the nesting stack. - if self.stack: - inner_block = self.stack[-1] - depth_change = line.count('(') - line.count(')') - inner_block.open_parentheses += depth_change - - # Also check if we are starting or ending an inline assembly block. - if inner_block.inline_asm in (_NO_ASM, _END_ASM): - if (depth_change != 0 and - inner_block.open_parentheses == 1 and - _MATCH_ASM.match(line)): - # Enter assembly block - inner_block.inline_asm = _INSIDE_ASM - else: - # Not entering assembly block. If previous line was _END_ASM, - # we will now shift to _NO_ASM state. - inner_block.inline_asm = _NO_ASM - elif (inner_block.inline_asm == _INSIDE_ASM and - inner_block.open_parentheses == 0): - # Exit assembly block - inner_block.inline_asm = _END_ASM - - # Consume namespace declaration at the beginning of the line. Do - # this in a loop so that we catch same line declarations like this: - # namespace proto2 { namespace bridge { class MessageSet; } } - while True: - # Match start of namespace. The "\b\s*" below catches namespace - # declarations even if it weren't followed by a whitespace, this - # is so that we don't confuse our namespace checker. The - # missing spaces will be flagged by CheckSpacing. - namespace_decl_match = Match(r'^\s*namespace\b\s*([:\w]+)?(.*)$', line) - if not namespace_decl_match: - break - - new_namespace = _NamespaceInfo(namespace_decl_match.group(1), linenum) - self.stack.append(new_namespace) - - line = namespace_decl_match.group(2) - if line.find('{') != -1: - new_namespace.seen_open_brace = True - line = line[line.find('{') + 1:] - - # Look for a class declaration in whatever is left of the line - # after parsing namespaces. The regexp accounts for decorated classes - # such as in: - # class LOCKABLE API Object { - # }; - class_decl_match = Match( - r'^(\s*(?:template\s*<[\w\s<>,:]*>\s*)?' - r'(class|struct)\s+(?:[A-Z0-9_]+\s+)*(\w+(?:::\w+)*))' - r'(.*)$', line) - if (class_decl_match and - (not self.stack or self.stack[-1].open_parentheses == 0)): - # We do not want to accept classes that are actually template arguments: - # template , - # template class Ignore3> - # void Function() {}; - # - # To avoid template argument cases, we scan forward and look for - # an unmatched '>'. If we see one, assume we are inside a - # template argument list. - end_declaration = len(class_decl_match.group(1)) - if not self.InTemplateArgumentList(clean_lines, linenum, end_declaration): - self.stack.append(_ClassInfo( - class_decl_match.group(3), class_decl_match.group(2), - clean_lines, linenum)) - line = class_decl_match.group(4) - - # If we have not yet seen the opening brace for the innermost block, - # run checks here. - if not self.SeenOpenBrace(): - self.stack[-1].CheckBegin(filename, clean_lines, linenum, error) - - # Update access control if we are inside a class/struct - if self.stack and isinstance(self.stack[-1], _ClassInfo): - classinfo = self.stack[-1] - access_match = Match( - r'^(.*)\b(public|private|protected|signals)(\s+(?:slots\s*)?)?' - r':(?:[^:]|$)', - line) - if access_match: - classinfo.access = access_match.group(2) - - # Check that access keywords are indented +1 space. Skip this - # check if the keywords are not preceded by whitespaces. - indent = access_match.group(1) - if (len(indent) != classinfo.class_indent + 1 and - Match(r'^\s*$', indent)): - if classinfo.is_struct: - parent = 'struct ' + classinfo.name - else: - parent = 'class ' + classinfo.name - slots = '' - if access_match.group(3): - slots = access_match.group(3) - error(filename, linenum, 'whitespace/indent', 3, - '%s%s: should be indented +1 space inside %s' % ( - access_match.group(2), slots, parent)) - - # Consume braces or semicolons from what's left of the line - while True: - # Match first brace, semicolon, or closed parenthesis. - matched = Match(r'^[^{;)}]*([{;)}])(.*)$', line) - if not matched: - break - - token = matched.group(1) - if token == '{': - # If namespace or class hasn't seen a opening brace yet, mark - # namespace/class head as complete. Push a new block onto the - # stack otherwise. - if not self.SeenOpenBrace(): - self.stack[-1].seen_open_brace = True - elif Match(r'^extern\s*"[^"]*"\s*\{', line): - self.stack.append(_ExternCInfo(linenum)) - else: - self.stack.append(_BlockInfo(linenum, True)) - if _MATCH_ASM.match(line): - self.stack[-1].inline_asm = _BLOCK_ASM - - elif token == ';' or token == ')': - # If we haven't seen an opening brace yet, but we already saw - # a semicolon, this is probably a forward declaration. Pop - # the stack for these. + # Remember top of the previous nesting stack. # - # Similarly, if we haven't seen an opening brace yet, but we - # already saw a closing parenthesis, then these are probably - # function arguments with extra "class" or "struct" keywords. - # Also pop these stack for these. - if not self.SeenOpenBrace(): - self.stack.pop() - else: # token == '}' - # Perform end of block checks and pop the stack. + # The stack is always pushed/popped and not modified in place, so + # we can just do a shallow copy instead of copy.deepcopy. Using + # deepcopy would slow down cpplint by ~28%. if self.stack: - self.stack[-1].CheckEnd(filename, clean_lines, linenum, error) - self.stack.pop() - line = matched.group(2) + self.previous_stack_top = self.stack[-1] + else: + self.previous_stack_top = None - def InnermostClass(self): - """Get class info on the top of the stack. + # Update pp_stack + self.UpdatePreprocessor(line) + + # Count parentheses. This is to avoid adding struct arguments to + # the nesting stack. + if self.stack: + inner_block = self.stack[-1] + depth_change = line.count('(') - line.count(')') + inner_block.open_parentheses += depth_change + + # Also check if we are starting or ending an inline assembly block. + if inner_block.inline_asm in (_NO_ASM, _END_ASM): + if (depth_change != 0 and inner_block.open_parentheses == 1 + and _MATCH_ASM.match(line)): + # Enter assembly block + inner_block.inline_asm = _INSIDE_ASM + else: + # Not entering assembly block. If previous line was + # _END_ASM, we will now shift to _NO_ASM state. + inner_block.inline_asm = _NO_ASM + elif (inner_block.inline_asm == _INSIDE_ASM + and inner_block.open_parentheses == 0): + # Exit assembly block + inner_block.inline_asm = _END_ASM + + # Consume namespace declaration at the beginning of the line. Do + # this in a loop so that we catch same line declarations like this: + # namespace proto2 { namespace bridge { class MessageSet; } } + while True: + # Match start of namespace. The "\b\s*" below catches namespace + # declarations even if it weren't followed by a whitespace, this + # is so that we don't confuse our namespace checker. The + # missing spaces will be flagged by CheckSpacing. + namespace_decl_match = Match(r'^\s*namespace\b\s*([:\w]+)?(.*)$', + line) + if not namespace_decl_match: + break + + new_namespace = _NamespaceInfo(namespace_decl_match.group(1), + linenum) + self.stack.append(new_namespace) + + line = namespace_decl_match.group(2) + if line.find('{') != -1: + new_namespace.seen_open_brace = True + line = line[line.find('{') + 1:] + + # Look for a class declaration in whatever is left of the line + # after parsing namespaces. The regexp accounts for decorated classes + # such as in: + # class LOCKABLE API Object { + # }; + class_decl_match = Match( + r'^(\s*(?:template\s*<[\w\s<>,:]*>\s*)?' + r'(class|struct)\s+(?:[A-Z0-9_]+\s+)*(\w+(?:::\w+)*))' + r'(.*)$', line) + if (class_decl_match + and (not self.stack or self.stack[-1].open_parentheses == 0)): + # We do not want to accept classes that are actually template + # arguments: template , + # template class Ignore3> void Function() {}; + # + # To avoid template argument cases, we scan forward and look for + # an unmatched '>'. If we see one, assume we are inside a + # template argument list. + end_declaration = len(class_decl_match.group(1)) + if not self.InTemplateArgumentList(clean_lines, linenum, + end_declaration): + self.stack.append( + _ClassInfo(class_decl_match.group(3), + class_decl_match.group(2), clean_lines, linenum)) + line = class_decl_match.group(4) + + # If we have not yet seen the opening brace for the innermost block, + # run checks here. + if not self.SeenOpenBrace(): + self.stack[-1].CheckBegin(filename, clean_lines, linenum, error) + + # Update access control if we are inside a class/struct + if self.stack and isinstance(self.stack[-1], _ClassInfo): + classinfo = self.stack[-1] + access_match = Match( + r'^(.*)\b(public|private|protected|signals)(\s+(?:slots\s*)?)?' + r':(?:[^:]|$)', line) + if access_match: + classinfo.access = access_match.group(2) + + # Check that access keywords are indented +1 space. Skip this + # check if the keywords are not preceded by whitespaces. + indent = access_match.group(1) + if (len(indent) != classinfo.class_indent + 1 + and Match(r'^\s*$', indent)): + if classinfo.is_struct: + parent = 'struct ' + classinfo.name + else: + parent = 'class ' + classinfo.name + slots = '' + if access_match.group(3): + slots = access_match.group(3) + error( + filename, linenum, 'whitespace/indent', 3, + '%s%s: should be indented +1 space inside %s' % + (access_match.group(2), slots, parent)) + + # Consume braces or semicolons from what's left of the line + while True: + # Match first brace, semicolon, or closed parenthesis. + matched = Match(r'^[^{;)}]*([{;)}])(.*)$', line) + if not matched: + break + + token = matched.group(1) + if token == '{': + # If namespace or class hasn't seen a opening brace yet, mark + # namespace/class head as complete. Push a new block onto the + # stack otherwise. + if not self.SeenOpenBrace(): + self.stack[-1].seen_open_brace = True + elif Match(r'^extern\s*"[^"]*"\s*\{', line): + self.stack.append(_ExternCInfo(linenum)) + else: + self.stack.append(_BlockInfo(linenum, True)) + if _MATCH_ASM.match(line): + self.stack[-1].inline_asm = _BLOCK_ASM + + elif token == ';' or token == ')': + # If we haven't seen an opening brace yet, but we already saw + # a semicolon, this is probably a forward declaration. Pop + # the stack for these. + # + # Similarly, if we haven't seen an opening brace yet, but we + # already saw a closing parenthesis, then these are probably + # function arguments with extra "class" or "struct" keywords. + # Also pop these stack for these. + if not self.SeenOpenBrace(): + self.stack.pop() + else: # token == '}' + # Perform end of block checks and pop the stack. + if self.stack: + self.stack[-1].CheckEnd(filename, clean_lines, linenum, + error) + self.stack.pop() + line = matched.group(2) + + def InnermostClass(self): + """Get class info on the top of the stack. Returns: A _ClassInfo object if we are inside a class, or None otherwise. """ - for i in range(len(self.stack), 0, -1): - classinfo = self.stack[i - 1] - if isinstance(classinfo, _ClassInfo): - return classinfo - return None + for i in range(len(self.stack), 0, -1): + classinfo = self.stack[i - 1] + if isinstance(classinfo, _ClassInfo): + return classinfo + return None - def CheckCompletedBlocks(self, filename, error): - """Checks that all classes and namespaces have been completely parsed. + def CheckCompletedBlocks(self, filename, error): + """Checks that all classes and namespaces have been completely parsed. Call this when all lines in a file have been processed. Args: filename: The name of the current file. error: The function to call with any errors found. """ - # Note: This test can result in false positives if #ifdef constructs - # get in the way of brace matching. See the testBuildClass test in - # cpplint_unittest.py for an example of this. - for obj in self.stack: - if isinstance(obj, _ClassInfo): - error(filename, obj.starting_linenum, 'build/class', 5, - 'Failed to find complete declaration of class %s' % - obj.name) - elif isinstance(obj, _NamespaceInfo): - error(filename, obj.starting_linenum, 'build/namespaces', 5, - 'Failed to find complete declaration of namespace %s' % - obj.name) + # Note: This test can result in false positives if #ifdef constructs + # get in the way of brace matching. See the testBuildClass test in + # cpplint_unittest.py for an example of this. + for obj in self.stack: + if isinstance(obj, _ClassInfo): + error( + filename, obj.starting_linenum, 'build/class', 5, + 'Failed to find complete declaration of class %s' % + obj.name) + elif isinstance(obj, _NamespaceInfo): + error( + filename, obj.starting_linenum, 'build/namespaces', 5, + 'Failed to find complete declaration of namespace %s' % + obj.name) -def CheckForNonStandardConstructs(filename, clean_lines, linenum, - nesting_state, error): - r"""Logs an error if we see certain non-ANSI constructs ignored by gcc-2. +def CheckForNonStandardConstructs(filename, clean_lines, linenum, nesting_state, + error): + r"""Logs an error if we see certain non-ANSI constructs ignored by gcc-2. Complain about several constructs which gcc-2 accepts, but which are not standard C++. Warning about these in lint is one way to ease the @@ -2995,139 +3031,145 @@ def CheckForNonStandardConstructs(filename, clean_lines, linenum, filename, line number, error level, and message """ - # Remove comments from the line, but leave in strings for now. - line = clean_lines.lines[linenum] + # Remove comments from the line, but leave in strings for now. + line = clean_lines.lines[linenum] - if Search(r'printf\s*\(.*".*%[-+ ]?\d*q', line): - error(filename, linenum, 'runtime/printf_format', 3, - '%q in format strings is deprecated. Use %ll instead.') + if Search(r'printf\s*\(.*".*%[-+ ]?\d*q', line): + error(filename, linenum, 'runtime/printf_format', 3, + '%q in format strings is deprecated. Use %ll instead.') - if Search(r'printf\s*\(.*".*%\d+\$', line): - error(filename, linenum, 'runtime/printf_format', 2, - '%N$ formats are unconventional. Try rewriting to avoid them.') + if Search(r'printf\s*\(.*".*%\d+\$', line): + error(filename, linenum, 'runtime/printf_format', 2, + '%N$ formats are unconventional. Try rewriting to avoid them.') - # Remove escaped backslashes before looking for undefined escapes. - line = line.replace('\\\\', '') + # Remove escaped backslashes before looking for undefined escapes. + line = line.replace('\\\\', '') - if Search(r'("|\').*\\(%|\[|\(|{)', line): - error(filename, linenum, 'build/printf_format', 3, - '%, [, (, and { are undefined character escapes. Unescape them.') + if Search(r'("|\').*\\(%|\[|\(|{)', line): + error( + filename, linenum, 'build/printf_format', 3, + '%, [, (, and { are undefined character escapes. Unescape them.') - # For the rest, work with both comments and strings removed. - line = clean_lines.elided[linenum] + # For the rest, work with both comments and strings removed. + line = clean_lines.elided[linenum] - if Search(r'\b(const|volatile|void|char|short|int|long' + if Search( + r'\b(const|volatile|void|char|short|int|long' r'|float|double|signed|unsigned' r'|schar|u?int8|u?int16|u?int32|u?int64)' - r'\s+(register|static|extern|typedef)\b', - line): - error(filename, linenum, 'build/storage_class', 5, - 'Storage-class specifier (static, extern, typedef, etc) should be ' - 'at the beginning of the declaration.') + r'\s+(register|static|extern|typedef)\b', line): + error( + filename, linenum, 'build/storage_class', 5, + 'Storage-class specifier (static, extern, typedef, etc) should be ' + 'at the beginning of the declaration.') - if Match(r'\s*#\s*endif\s*[^/\s]+', line): - error(filename, linenum, 'build/endif_comment', 5, - 'Uncommented text after #endif is non-standard. Use a comment.') + if Match(r'\s*#\s*endif\s*[^/\s]+', line): + error(filename, linenum, 'build/endif_comment', 5, + 'Uncommented text after #endif is non-standard. Use a comment.') - if Match(r'\s*class\s+(\w+\s*::\s*)+\w+\s*;', line): - error(filename, linenum, 'build/forward_decl', 5, - 'Inner-style forward declarations are invalid. Remove this line.') + if Match(r'\s*class\s+(\w+\s*::\s*)+\w+\s*;', line): + error( + filename, linenum, 'build/forward_decl', 5, + 'Inner-style forward declarations are invalid. Remove this line.') - if Search(r'(\w+|[+-]?\d+(\.\d*)?)\s*(<|>)\?=?\s*(\w+|[+-]?\d+)(\.\d*)?', - line): - error(filename, linenum, 'build/deprecated', 3, - '>? and )\?=?\s*(\w+|[+-]?\d+)(\.\d*)?', + line): + error( + filename, linenum, 'build/deprecated', 3, + '>? and ))?' - # r'\s*const\s*' + type_name + '\s*&\s*\w+\s*;' - error(filename, linenum, 'runtime/member_string_references', 2, - 'const string& members are dangerous. It is much better to use ' - 'alternatives, such as pointers or simple constants.') + if Search(r'^\s*const\s*string\s*&\s*\w+\s*;', line): + # TODO(unknown): Could it be expanded safely to arbitrary references, + # without triggering too many false positives? The first + # attempt triggered 5 warnings for mostly benign code in the regtest, + # hence the restriction. Here's the original regexp, for the reference: + # type_name = r'\w+((\s*::\s*\w+)|(\s*<\s*\w+?\s*>))?' r'\s*const\s*' + + # type_name + '\s*&\s*\w+\s*;' + error( + filename, linenum, 'runtime/member_string_references', 2, + 'const string& members are dangerous. It is much better to use ' + 'alternatives, such as pointers or simple constants.') - # Everything else in this function operates on class declarations. - # Return early if the top of the nesting stack is not a class, or if - # the class head is not completed yet. - classinfo = nesting_state.InnermostClass() - if not classinfo or not classinfo.seen_open_brace: - return + # Everything else in this function operates on class declarations. + # Return early if the top of the nesting stack is not a class, or if + # the class head is not completed yet. + classinfo = nesting_state.InnermostClass() + if not classinfo or not classinfo.seen_open_brace: + return - # The class may have been declared with namespace or classname qualifiers. - # The constructor and destructor will not have those qualifiers. - base_classname = classinfo.name.split('::')[-1] + # The class may have been declared with namespace or classname qualifiers. + # The constructor and destructor will not have those qualifiers. + base_classname = classinfo.name.split('::')[-1] - # Look for single-argument constructors that aren't marked explicit. - # Technically a valid construct, but against style. - explicit_constructor_match = Match( - r'\s+(?:(?:inline|constexpr)\s+)*(explicit\s+)?' - r'(?:(?:inline|constexpr)\s+)*%s\s*' - r'\(((?:[^()]|\([^()]*\))*)\)' - % re.escape(base_classname), - line) + # Look for single-argument constructors that aren't marked explicit. + # Technically a valid construct, but against style. + explicit_constructor_match = Match( + r'\s+(?:(?:inline|constexpr)\s+)*(explicit\s+)?' + r'(?:(?:inline|constexpr)\s+)*%s\s*' + r'\(((?:[^()]|\([^()]*\))*)\)' % re.escape(base_classname), line) - if explicit_constructor_match: - is_marked_explicit = explicit_constructor_match.group(1) + if explicit_constructor_match: + is_marked_explicit = explicit_constructor_match.group(1) - if not explicit_constructor_match.group(2): - constructor_args = [] - else: - constructor_args = explicit_constructor_match.group(2).split(',') + if not explicit_constructor_match.group(2): + constructor_args = [] + else: + constructor_args = explicit_constructor_match.group(2).split(',') - # collapse arguments so that commas in template parameter lists and function - # argument parameter lists don't split arguments in two - i = 0 - while i < len(constructor_args): - constructor_arg = constructor_args[i] - while (constructor_arg.count('<') > constructor_arg.count('>') or - constructor_arg.count('(') > constructor_arg.count(')')): - constructor_arg += ',' + constructor_args[i + 1] - del constructor_args[i + 1] - constructor_args[i] = constructor_arg - i += 1 + # collapse arguments so that commas in template parameter lists and + # function argument parameter lists don't split arguments in two + i = 0 + while i < len(constructor_args): + constructor_arg = constructor_args[i] + while (constructor_arg.count('<') > constructor_arg.count('>') + or constructor_arg.count('(') > constructor_arg.count(')')): + constructor_arg += ',' + constructor_args[i + 1] + del constructor_args[i + 1] + constructor_args[i] = constructor_arg + i += 1 - defaulted_args = [arg for arg in constructor_args if '=' in arg] - noarg_constructor = (not constructor_args or # empty arg list - # 'void' arg specifier - (len(constructor_args) == 1 and - constructor_args[0].strip() == 'void')) - onearg_constructor = ((len(constructor_args) == 1 and # exactly one arg - not noarg_constructor) or - # all but at most one arg defaulted - (len(constructor_args) >= 1 and - not noarg_constructor and - len(defaulted_args) >= len(constructor_args) - 1)) - initializer_list_constructor = bool( - onearg_constructor and - Search(r'\bstd\s*::\s*initializer_list\b', constructor_args[0])) - copy_constructor = bool( - onearg_constructor and - Match(r'(const\s+)?%s(\s*<[^>]*>)?(\s+const)?\s*(?:<\w+>\s*)?&' - % re.escape(base_classname), constructor_args[0].strip())) + defaulted_args = [arg for arg in constructor_args if '=' in arg] + noarg_constructor = ( + not constructor_args or # empty arg list + # 'void' arg specifier + (len(constructor_args) == 1 + and constructor_args[0].strip() == 'void')) + onearg_constructor = ( + ( + len(constructor_args) == 1 and # exactly one arg + not noarg_constructor) or + # all but at most one arg defaulted + (len(constructor_args) >= 1 and not noarg_constructor + and len(defaulted_args) >= len(constructor_args) - 1)) + initializer_list_constructor = bool( + onearg_constructor + and Search(r'\bstd\s*::\s*initializer_list\b', constructor_args[0])) + copy_constructor = bool(onearg_constructor and Match( + r'(const\s+)?%s(\s*<[^>]*>)?(\s+const)?\s*(?:<\w+>\s*)?&' % + re.escape(base_classname), constructor_args[0].strip())) - if (not is_marked_explicit and - onearg_constructor and - not initializer_list_constructor and - not copy_constructor): - if defaulted_args: - error(filename, linenum, 'runtime/explicit', 5, - 'Constructors callable with one argument ' - 'should be marked explicit.') - else: - error(filename, linenum, 'runtime/explicit', 5, - 'Single-parameter constructors should be marked explicit.') - elif is_marked_explicit and not onearg_constructor: - if noarg_constructor: - error(filename, linenum, 'runtime/explicit', 5, - 'Zero-parameter constructors should not be marked explicit.') + if (not is_marked_explicit and onearg_constructor + and not initializer_list_constructor and not copy_constructor): + if defaulted_args: + error( + filename, linenum, 'runtime/explicit', 5, + 'Constructors callable with one argument ' + 'should be marked explicit.') + else: + error( + filename, linenum, 'runtime/explicit', 5, + 'Single-parameter constructors should be marked explicit.') + elif is_marked_explicit and not onearg_constructor: + if noarg_constructor: + error( + filename, linenum, 'runtime/explicit', 5, + 'Zero-parameter constructors should not be marked explicit.' + ) def CheckSpacingForFunctionCall(filename, clean_lines, linenum, error): - """Checks for the correctness of various spacing around function calls. + """Checks for the correctness of various spacing around function calls. Args: filename: The name of the current file. @@ -3135,76 +3177,78 @@ def CheckSpacingForFunctionCall(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] + line = clean_lines.elided[linenum] - # Since function calls often occur inside if/for/while/switch - # expressions - which have their own, more liberal conventions - we - # first see if we should be looking inside such an expression for a - # function call, to which we can apply more strict standards. - fncall = line # if there's no control flow construct, look at whole line - for pattern in (r'\bif\s*(?:constexpr\s*)?\((.*)\)\s*{', - r'\bfor\s*\((.*)\)\s*{', - r'\bwhile\s*\((.*)\)\s*[{;]', - r'\bswitch\s*\((.*)\)\s*{'): - match = Search(pattern, line) - if match: - fncall = match.group(1) # look inside the parens for function calls - break + # Since function calls often occur inside if/for/while/switch + # expressions - which have their own, more liberal conventions - we + # first see if we should be looking inside such an expression for a + # function call, to which we can apply more strict standards. + fncall = line # if there's no control flow construct, look at whole line + for pattern in (r'\bif\s*(?:constexpr\s*)?\((.*)\)\s*{', + r'\bfor\s*\((.*)\)\s*{', r'\bwhile\s*\((.*)\)\s*[{;]', + r'\bswitch\s*\((.*)\)\s*{'): + match = Search(pattern, line) + if match: + fncall = match.group(1) # look inside the parens for function calls + break - # Except in if/for/while/switch, there should never be space - # immediately inside parens (eg "f( 3, 4 )"). We make an exception - # for nested parens ( (a+b) + c ). Likewise, there should never be - # a space before a ( when it's a function argument. I assume it's a - # function argument when the char before the whitespace is legal in - # a function name (alnum + _) and we're not starting a macro. Also ignore - # pointers and references to arrays and functions coz they're too tricky: - # we use a very simple way to recognize these: - # " (something)(maybe-something)" or - # " (something)(maybe-something," or - # " (something)[something]" - # Note that we assume the contents of [] to be short enough that - # they'll never need to wrap. - if ( # Ignore control structures. - not Search(r'\b(if|for|while|switch|return|new|delete|catch|sizeof)\b', - fncall) and - # Ignore pointers/references to functions. - not Search(r' \([^)]+\)\([^)]*(\)|,$)', fncall) and - # Ignore pointers/references to arrays. - not Search(r' \([^)]+\)\[[^\]]+\]', fncall)): - if Search(r'\w\s*\(\s(?!\s*\\$)', fncall): # a ( used for a fn call - error(filename, linenum, 'whitespace/parens', 4, - 'Extra space after ( in function call') - elif Search(r'\(\s+(?!(\s*\\)|\()', fncall): - error(filename, linenum, 'whitespace/parens', 2, - 'Extra space after (') - if (Search(r'\w\s+\(', fncall) and - not Search(r'_{0,2}asm_{0,2}\s+_{0,2}volatile_{0,2}\s+\(', fncall) and - not Search(r'#\s*define|typedef|__except|using\s+\w+\s*=', fncall) and - not Search(r'\w\s+\((\w+::)*\*\w+\)\(', fncall) and - not Search(r'\bcase\s+\(', fncall)): - # TODO(unknown): Space after an operator function seem to be a common - # error, silence those for now by restricting them to highest verbosity. - if Search(r'\boperator_*\b', line): - error(filename, linenum, 'whitespace/parens', 0, - 'Extra space before ( in function call') - else: - error(filename, linenum, 'whitespace/parens', 4, - 'Extra space before ( in function call') - # If the ) is followed only by a newline or a { + newline, assume it's - # part of a control statement (if/while/etc), and don't complain - if Search(r'[^)]\s+\)\s*[^{\s]', fncall): - # If the closing parenthesis is preceded by only whitespaces, - # try to give a more descriptive error message. - if Search(r'^\s+\)', fncall): - error(filename, linenum, 'whitespace/parens', 2, - 'Closing ) should be moved to the previous line') - else: - error(filename, linenum, 'whitespace/parens', 2, - 'Extra space before )') + # Except in if/for/while/switch, there should never be space + # immediately inside parens (eg "f( 3, 4 )"). We make an exception + # for nested parens ( (a+b) + c ). Likewise, there should never be + # a space before a ( when it's a function argument. I assume it's a + # function argument when the char before the whitespace is legal in + # a function name (alnum + _) and we're not starting a macro. Also ignore + # pointers and references to arrays and functions coz they're too tricky: + # we use a very simple way to recognize these: + # " (something)(maybe-something)" or + # " (something)(maybe-something," or + # " (something)[something]" + # Note that we assume the contents of [] to be short enough that + # they'll never need to wrap. + if ( # Ignore control structures. + not Search( + r'\b(if|for|while|switch|return|new|delete|catch|sizeof)\b', + fncall) and + # Ignore pointers/references to functions. + not Search(r' \([^)]+\)\([^)]*(\)|,$)', fncall) and + # Ignore pointers/references to arrays. + not Search(r' \([^)]+\)\[[^\]]+\]', fncall)): + if Search(r'\w\s*\(\s(?!\s*\\$)', fncall): # a ( used for a fn call + error(filename, linenum, 'whitespace/parens', 4, + 'Extra space after ( in function call') + elif Search(r'\(\s+(?!(\s*\\)|\()', fncall): + error(filename, linenum, 'whitespace/parens', 2, + 'Extra space after (') + if (Search(r'\w\s+\(', fncall) and not Search( + r'_{0,2}asm_{0,2}\s+_{0,2}volatile_{0,2}\s+\(', fncall) + and not Search(r'#\s*define|typedef|__except|using\s+\w+\s*=', + fncall) + and not Search(r'\w\s+\((\w+::)*\*\w+\)\(', fncall) + and not Search(r'\bcase\s+\(', fncall)): + # TODO(unknown): Space after an operator function seem to be a + # common error, silence those for now by restricting them to highest + # verbosity. + if Search(r'\boperator_*\b', line): + error(filename, linenum, 'whitespace/parens', 0, + 'Extra space before ( in function call') + else: + error(filename, linenum, 'whitespace/parens', 4, + 'Extra space before ( in function call') + # If the ) is followed only by a newline or a { + newline, assume it's + # part of a control statement (if/while/etc), and don't complain + if Search(r'[^)]\s+\)\s*[^{\s]', fncall): + # If the closing parenthesis is preceded by only whitespaces, + # try to give a more descriptive error message. + if Search(r'^\s+\)', fncall): + error(filename, linenum, 'whitespace/parens', 2, + 'Closing ) should be moved to the previous line') + else: + error(filename, linenum, 'whitespace/parens', 2, + 'Extra space before )') def IsBlankLine(line): - """Returns true if the given line is blank. + """Returns true if the given line is blank. We consider a line to be blank if the line is empty or consists of only white spaces. @@ -3215,26 +3259,26 @@ def IsBlankLine(line): Returns: True, if the given line is blank. """ - return not line or line.isspace() + return not line or line.isspace() def CheckForNamespaceIndentation(filename, nesting_state, clean_lines, line, error): - is_namespace_indent_item = ( - len(nesting_state.stack) > 1 and - nesting_state.stack[-1].check_namespace_indentation and - isinstance(nesting_state.previous_stack_top, _NamespaceInfo) and - nesting_state.previous_stack_top == nesting_state.stack[-2]) + is_namespace_indent_item = ( + len(nesting_state.stack) > 1 + and nesting_state.stack[-1].check_namespace_indentation + and isinstance(nesting_state.previous_stack_top, _NamespaceInfo) + and nesting_state.previous_stack_top == nesting_state.stack[-2]) - if ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item, - clean_lines.elided, line): - CheckItemIndentationInNamespace(filename, clean_lines.elided, - line, error) + if ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item, + clean_lines.elided, line): + CheckItemIndentationInNamespace(filename, clean_lines.elided, line, + error) -def CheckForFunctionLengths(filename, clean_lines, linenum, - function_state, error): - """Reports for long function bodies. +def CheckForFunctionLengths(filename, clean_lines, linenum, function_state, + error): + """Reports for long function bodies. For an overview why this is done, see: https://google.github.io/styleguide/cppguide.html#Write_Short_Functions @@ -3255,49 +3299,51 @@ def CheckForFunctionLengths(filename, clean_lines, linenum, function_state: Current function name and lines in body so far. error: The function to call with any errors found. """ - lines = clean_lines.lines - line = lines[linenum] - joined_line = '' + lines = clean_lines.lines + line = lines[linenum] + joined_line = '' - starting_func = False - regexp = r'(\w(\w|::|\*|\&|\s)*)\(' # decls * & space::name( ... - match_result = Match(regexp, line) - if match_result: - # If the name is all caps and underscores, figure it's a macro and - # ignore it, unless it's TEST or TEST_F. - function_name = match_result.group(1).split()[-1] - if function_name == 'TEST' or function_name == 'TEST_F' or ( - not Match(r'[A-Z_0-9]+$', function_name)): - starting_func = True + starting_func = False + regexp = r'(\w(\w|::|\*|\&|\s)*)\(' # decls * & space::name( ... + match_result = Match(regexp, line) + if match_result: + # If the name is all caps and underscores, figure it's a macro and + # ignore it, unless it's TEST or TEST_F. + function_name = match_result.group(1).split()[-1] + if function_name == 'TEST' or function_name == 'TEST_F' or (not Match( + r'[A-Z_0-9]+$', function_name)): + starting_func = True - if starting_func: - body_found = False - for start_linenum in range(linenum, clean_lines.NumLines()): - start_line = lines[start_linenum] - joined_line += ' ' + start_line.lstrip() - if Search(r'(;|})', start_line): # Declarations and trivial functions - body_found = True - break # ... ignore - elif Search(r'{', start_line): - body_found = True - function = Search(r'((\w|:)*)\(', line).group(1) - if Match(r'TEST', function): # Handle TEST... macros - parameter_regexp = Search(r'(\(.*\))', joined_line) - if parameter_regexp: # Ignore bad syntax - function += parameter_regexp.group(1) - else: - function += '()' - function_state.Begin(function) - break - if not body_found: - # No body for the function (or evidence of a non-function) was found. - error(filename, linenum, 'readability/fn_size', 5, - 'Lint failed to find start of function body.') - elif Match(r'^\}\s*$', line): # function end - function_state.Check(error, filename, linenum) - function_state.End() - elif not Match(r'^\s*$', line): - function_state.Count() # Count non-blank/non-comment lines. + if starting_func: + body_found = False + for start_linenum in range(linenum, clean_lines.NumLines()): + start_line = lines[start_linenum] + joined_line += ' ' + start_line.lstrip() + if Search(r'(;|})', + start_line): # Declarations and trivial functions + body_found = True + break # ... ignore + elif Search(r'{', start_line): + body_found = True + function = Search(r'((\w|:)*)\(', line).group(1) + if Match(r'TEST', function): # Handle TEST... macros + parameter_regexp = Search(r'(\(.*\))', joined_line) + if parameter_regexp: # Ignore bad syntax + function += parameter_regexp.group(1) + else: + function += '()' + function_state.Begin(function) + break + if not body_found: + # No body for the function (or evidence of a non-function) was + # found. + error(filename, linenum, 'readability/fn_size', 5, + 'Lint failed to find start of function body.') + elif Match(r'^\}\s*$', line): # function end + function_state.Check(error, filename, linenum) + function_state.End() + elif not Match(r'^\s*$', line): + function_state.Count() # Count non-blank/non-comment lines. _RE_PATTERN_TODO = re.compile( @@ -3305,7 +3351,7 @@ _RE_PATTERN_TODO = re.compile( def CheckComment(line, filename, linenum, next_line_start, error): - """Checks for common mistakes in comments. + """Checks for common mistakes in comments. Args: line: The line in question. @@ -3314,52 +3360,55 @@ def CheckComment(line, filename, linenum, next_line_start, error): next_line_start: The first non-whitespace column of the next line. error: The function to call with any errors found. """ - commentpos = line.find('//') - if commentpos != -1: - # Check if the // may be in quotes. If so, ignore it - if re.sub(r'\\.', '', line[0:commentpos]).count('"') % 2 == 0: - # Allow one space for new scopes, two spaces otherwise: - if (not (Match(r'^.*{ *//', line) and next_line_start == commentpos) and - ((commentpos >= 1 and - line[commentpos-1] not in string.whitespace) or - (commentpos >= 2 and - line[commentpos-2] not in string.whitespace))): - error(filename, linenum, 'whitespace/comments', 2, - 'At least two spaces is best between code and comments') + commentpos = line.find('//') + if commentpos != -1: + # Check if the // may be in quotes. If so, ignore it + if re.sub(r'\\.', '', line[0:commentpos]).count('"') % 2 == 0: + # Allow one space for new scopes, two spaces otherwise: + if (not (Match(r'^.*{ *//', line) and next_line_start == commentpos) + and ((commentpos >= 1 + and line[commentpos - 1] not in string.whitespace) or + (commentpos >= 2 + and line[commentpos - 2] not in string.whitespace))): + error(filename, linenum, 'whitespace/comments', 2, + 'At least two spaces is best between code and comments') - # Checks for common mistakes in TODO comments. - comment = line[commentpos:] - match = _RE_PATTERN_TODO.match(comment) - if match: - # One whitespace is correct; zero whitespace is handled elsewhere. - leading_whitespace = match.group(1) - if len(leading_whitespace) > 1: - error(filename, linenum, 'whitespace/todo', 2, - 'Too many spaces before TODO') + # Checks for common mistakes in TODO comments. + comment = line[commentpos:] + match = _RE_PATTERN_TODO.match(comment) + if match: + # One whitespace is correct; zero whitespace is handled + # elsewhere. + leading_whitespace = match.group(1) + if len(leading_whitespace) > 1: + error(filename, linenum, 'whitespace/todo', 2, + 'Too many spaces before TODO') - username = match.group(2) - if not username: - error(filename, linenum, 'readability/todo', 2, - 'Missing username in TODO; it should look like ' - '"// TODO(my_username): Stuff."') + username = match.group(2) + if not username: + error( + filename, linenum, 'readability/todo', 2, + 'Missing username in TODO; it should look like ' + '"// TODO(my_username): Stuff."') - middle_whitespace = match.group(3) - # Comparisons made explicit for correctness -- pylint: disable=g-explicit-bool-comparison - if middle_whitespace != ' ' and middle_whitespace != '': - error(filename, linenum, 'whitespace/todo', 2, - 'TODO(my_username) should be followed by a space') + middle_whitespace = match.group(3) + # Comparisons made explicit for correctness -- pylint: + # disable=g-explicit-bool-comparison + if middle_whitespace != ' ' and middle_whitespace != '': + error(filename, linenum, 'whitespace/todo', 2, + 'TODO(my_username) should be followed by a space') - # If the comment contains an alphanumeric character, there - # should be a space somewhere between it and the // unless - # it's a /// or //! Doxygen comment. - if (Match(r'//[^ ]*\w', comment) and - not Match(r'(///|//\!)(\s+|$)', comment)): - error(filename, linenum, 'whitespace/comments', 4, - 'Should have a space between // and comment') + # If the comment contains an alphanumeric character, there + # should be a space somewhere between it and the // unless + # it's a /// or //! Doxygen comment. + if (Match(r'//[^ ]*\w', comment) + and not Match(r'(///|//\!)(\s+|$)', comment)): + error(filename, linenum, 'whitespace/comments', 4, + 'Should have a space between // and comment') def CheckSpacing(filename, clean_lines, linenum, nesting_state, error): - """Checks for the correctness of various spacing issues in the code. + """Checks for the correctness of various spacing issues in the code. Things we check for: spaces around operators, spaces after if/for/while/switch, no spaces around parens in function calls, two @@ -3376,121 +3425,121 @@ def CheckSpacing(filename, clean_lines, linenum, nesting_state, error): error: The function to call with any errors found. """ - # Don't use "elided" lines here, otherwise we can't check commented lines. - # Don't want to use "raw" either, because we don't want to check inside C++11 - # raw strings, - raw = clean_lines.lines_without_raw_strings - line = raw[linenum] + # Don't use "elided" lines here, otherwise we can't check commented lines. + # Don't want to use "raw" either, because we don't want to check inside + # C++11 raw strings, + raw = clean_lines.lines_without_raw_strings + line = raw[linenum] - # Before nixing comments, check if the line is blank for no good - # reason. This includes the first line after a block is opened, and - # blank lines at the end of a function (ie, right before a line like '}' - # - # Skip all the blank line checks if we are immediately inside a - # namespace body. In other words, don't issue blank line warnings - # for this block: - # namespace { - # - # } - # - # A warning about missing end of namespace comments will be issued instead. - # - # Also skip blank line checks for 'extern "C"' blocks, which are formatted - # like namespaces. - if (IsBlankLine(line) and - not nesting_state.InNamespaceBody() and - not nesting_state.InExternC()): - elided = clean_lines.elided - prev_line = elided[linenum - 1] - prevbrace = prev_line.rfind('{') - # TODO(unknown): Don't complain if line before blank line, and line after, - # both start with alnums and are indented the same amount. - # This ignores whitespace at the start of a namespace block - # because those are not usually indented. - if prevbrace != -1 and prev_line[prevbrace:].find('}') == -1: - # OK, we have a blank line at the start of a code block. Before we - # complain, we check if it is an exception to the rule: The previous - # non-empty line has the parameters of a function header that are indented - # 4 spaces (because they did not fit in a 80 column line when placed on - # the same line as the function name). We also check for the case where - # the previous line is indented 6 spaces, which may happen when the - # initializers of a constructor do not fit into a 80 column line. - exception = False - if Match(r' {6}\w', prev_line): # Initializer list? - # We are looking for the opening column of initializer list, which - # should be indented 4 spaces to cause 6 space indentation afterwards. - search_position = linenum-2 - while (search_position >= 0 - and Match(r' {6}\w', elided[search_position])): - search_position -= 1 - exception = (search_position >= 0 - and elided[search_position][:5] == ' :') - else: - # Search for the function arguments or an initializer list. We use a - # simple heuristic here: If the line is indented 4 spaces; and we have a - # closing paren, without the opening paren, followed by an opening brace - # or colon (for initializer lists) we assume that it is the last line of - # a function header. If we have a colon indented 4 spaces, it is an - # initializer list. - exception = (Match(r' {4}\w[^\(]*\)\s*(const\s*)?(\{\s*$|:)', - prev_line) - or Match(r' {4}:', prev_line)) - - if not exception: - error(filename, linenum, 'whitespace/blank_line', 2, - 'Redundant blank line at the start of a code block ' - 'should be deleted.') - # Ignore blank lines at the end of a block in a long if-else - # chain, like this: - # if (condition1) { - # // Something followed by a blank line + # Before nixing comments, check if the line is blank for no good + # reason. This includes the first line after a block is opened, and + # blank lines at the end of a function (ie, right before a line like '}' + # + # Skip all the blank line checks if we are immediately inside a + # namespace body. In other words, don't issue blank line warnings + # for this block: + # namespace { # - # } else if (condition2) { - # // Something else # } + # + # A warning about missing end of namespace comments will be issued instead. + # + # Also skip blank line checks for 'extern "C"' blocks, which are formatted + # like namespaces. + if (IsBlankLine(line) and not nesting_state.InNamespaceBody() + and not nesting_state.InExternC()): + elided = clean_lines.elided + prev_line = elided[linenum - 1] + prevbrace = prev_line.rfind('{') + # TODO(unknown): Don't complain if line before blank line, and line + # after, both start with alnums and are indented the same amount. This + # ignores whitespace at the start of a namespace block because those are + # not usually indented. + if prevbrace != -1 and prev_line[prevbrace:].find('}') == -1: + # OK, we have a blank line at the start of a code block. Before we + # complain, we check if it is an exception to the rule: The previous + # non-empty line has the parameters of a function header that are + # indented 4 spaces (because they did not fit in a 80 column line + # when placed on the same line as the function name). We also check + # for the case where the previous line is indented 6 spaces, which + # may happen when the initializers of a constructor do not fit into + # a 80 column line. + exception = False + if Match(r' {6}\w', prev_line): # Initializer list? + # We are looking for the opening column of initializer list, + # which should be indented 4 spaces to cause 6 space indentation + # afterwards. + search_position = linenum - 2 + while (search_position >= 0 + and Match(r' {6}\w', elided[search_position])): + search_position -= 1 + exception = (search_position >= 0 + and elided[search_position][:5] == ' :') + else: + # Search for the function arguments or an initializer list. We + # use a simple heuristic here: If the line is indented 4 spaces; + # and we have a closing paren, without the opening paren, + # followed by an opening brace or colon (for initializer lists) + # we assume that it is the last line of a function header. If + # we have a colon indented 4 spaces, it is an initializer list. + exception = (Match(r' {4}\w[^\(]*\)\s*(const\s*)?(\{\s*$|:)', + prev_line) or Match(r' {4}:', prev_line)) + + if not exception: + error( + filename, linenum, 'whitespace/blank_line', 2, + 'Redundant blank line at the start of a code block ' + 'should be deleted.') + # Ignore blank lines at the end of a block in a long if-else + # chain, like this: + # if (condition1) { + # // Something followed by a blank line + # + # } else if (condition2) { + # // Something else + # } + if linenum + 1 < clean_lines.NumLines(): + next_line = raw[linenum + 1] + if (next_line and Match(r'\s*}', next_line) + and next_line.find('} else ') == -1): + error( + filename, linenum, 'whitespace/blank_line', 3, + 'Redundant blank line at the end of a code block ' + 'should be deleted.') + + matched = Match(r'\s*(public|protected|private):', prev_line) + if matched: + error(filename, linenum, 'whitespace/blank_line', 3, + 'Do not leave a blank line after "%s:"' % matched.group(1)) + + # Next, check comments + next_line_start = 0 if linenum + 1 < clean_lines.NumLines(): - next_line = raw[linenum + 1] - if (next_line - and Match(r'\s*}', next_line) - and next_line.find('} else ') == -1): - error(filename, linenum, 'whitespace/blank_line', 3, - 'Redundant blank line at the end of a code block ' - 'should be deleted.') + next_line = raw[linenum + 1] + next_line_start = len(next_line) - len(next_line.lstrip()) + CheckComment(line, filename, linenum, next_line_start, error) - matched = Match(r'\s*(public|protected|private):', prev_line) - if matched: - error(filename, linenum, 'whitespace/blank_line', 3, - 'Do not leave a blank line after "%s:"' % matched.group(1)) + # get rid of comments and strings + line = clean_lines.elided[linenum] - # Next, check comments - next_line_start = 0 - if linenum + 1 < clean_lines.NumLines(): - next_line = raw[linenum + 1] - next_line_start = len(next_line) - len(next_line.lstrip()) - CheckComment(line, filename, linenum, next_line_start, error) + # You shouldn't have spaces before your brackets, except maybe after + # 'delete []', 'return []() {};', 'auto [abc, ...] = ...;' or in the case of + # c++ attributes like 'class [[clang::lto_visibility_public]] MyClass'. + if (Search(r'\w\s+\[', line) + and not Search(r'(?:auto&?|delete|return)\s+\[', line) + and not Search(r'\s+\[\[', line)): + error(filename, linenum, 'whitespace/braces', 5, 'Extra space before [') - # get rid of comments and strings - line = clean_lines.elided[linenum] - - # You shouldn't have spaces before your brackets, except maybe after - # 'delete []', 'return []() {};', 'auto [abc, ...] = ...;' or in the case of - # c++ attributes like 'class [[clang::lto_visibility_public]] MyClass'. - if (Search(r'\w\s+\[', line) - and not Search(r'(?:auto&?|delete|return)\s+\[', line) - and not Search(r'\s+\[\[', line)): - error(filename, linenum, 'whitespace/braces', 5, - 'Extra space before [') - - # In range-based for, we wanted spaces before and after the colon, but - # not around "::" tokens that might appear. - if (Search(r'for *\(.*[^:]:[^: ]', line) or - Search(r'for *\(.*[^: ]:[^:]', line)): - error(filename, linenum, 'whitespace/forcolon', 2, - 'Missing space around colon in range-based for loop') + # In range-based for, we wanted spaces before and after the colon, but + # not around "::" tokens that might appear. + if (Search(r'for *\(.*[^:]:[^: ]', line) + or Search(r'for *\(.*[^: ]:[^:]', line)): + error(filename, linenum, 'whitespace/forcolon', 2, + 'Missing space around colon in range-based for loop') def CheckOperatorSpacing(filename, clean_lines, linenum, error): - """Checks for horizontal spacing around operators. + """Checks for horizontal spacing around operators. Args: filename: The name of the current file. @@ -3498,114 +3547,114 @@ def CheckOperatorSpacing(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] + line = clean_lines.elided[linenum] - # Don't try to do spacing checks for operator methods. Do this by - # replacing the troublesome characters with something else, - # preserving column position for all other characters. - # - # The replacement is done repeatedly to avoid false positives from - # operators that call operators. - while True: - match = Match(r'^(.*\boperator\b)(\S+)(\s*\(.*)$', line) + # Don't try to do spacing checks for operator methods. Do this by + # replacing the troublesome characters with something else, + # preserving column position for all other characters. + # + # The replacement is done repeatedly to avoid false positives from + # operators that call operators. + while True: + match = Match(r'^(.*\boperator\b)(\S+)(\s*\(.*)$', line) + if match: + line = match.group(1) + ('_' * len(match.group(2))) + match.group(3) + else: + break + + # We allow no-spaces around = within an if: "if ( (a=Foo()) == 0 )". + # Otherwise not. Note we only check for non-spaces on *both* sides; + # sometimes people put non-spaces on one side when aligning ='s among + # many lines (not that this is behavior that I approve of...) + if ((Search(r'[\w.]=', line) or Search(r'=[\w.]', line)) + and not Search(r'\b(if|while|for) ', line) + # Operators taken from [lex.operators] in C++11 standard. + and not Search(r'(>=|<=|==|!=|&=|\^=|\|=|\+=|\*=|\/=|\%=)', line) + and not Search(r'operator=', line)): + error(filename, linenum, 'whitespace/operators', 4, + 'Missing spaces around =') + + # It's ok not to have spaces around binary operators like + - * /, but if + # there's too little whitespace, we get concerned. It's hard to tell, + # though, so we punt on this one for now. TODO. + + # You should always have whitespace around binary operators. + # + # Check <= and >= first to avoid false positives with < and >, then + # check non-include lines for spacing around < and >. + # + # If the operator is followed by a comma, assume it's be used in a + # macro context and don't do any checks. This avoids false + # positives. + # + # Note that && is not included here. This is because there are too + # many false positives due to RValue references. + match = Search(r'[^<>=!\s](==|!=|<=|>=|\|\|)[^<>=!\s,;\)]', line) if match: - line = match.group(1) + ('_' * len(match.group(2))) + match.group(3) - else: - break - - # We allow no-spaces around = within an if: "if ( (a=Foo()) == 0 )". - # Otherwise not. Note we only check for non-spaces on *both* sides; - # sometimes people put non-spaces on one side when aligning ='s among - # many lines (not that this is behavior that I approve of...) - if ((Search(r'[\w.]=', line) or - Search(r'=[\w.]', line)) - and not Search(r'\b(if|while|for) ', line) - # Operators taken from [lex.operators] in C++11 standard. - and not Search(r'(>=|<=|==|!=|&=|\^=|\|=|\+=|\*=|\/=|\%=)', line) - and not Search(r'operator=', line)): - error(filename, linenum, 'whitespace/operators', 4, - 'Missing spaces around =') - - # It's ok not to have spaces around binary operators like + - * /, but if - # there's too little whitespace, we get concerned. It's hard to tell, - # though, so we punt on this one for now. TODO. - - # You should always have whitespace around binary operators. - # - # Check <= and >= first to avoid false positives with < and >, then - # check non-include lines for spacing around < and >. - # - # If the operator is followed by a comma, assume it's be used in a - # macro context and don't do any checks. This avoids false - # positives. - # - # Note that && is not included here. This is because there are too - # many false positives due to RValue references. - match = Search(r'[^<>=!\s](==|!=|<=|>=|\|\|)[^<>=!\s,;\)]', line) - if match: - error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around %s' % match.group(1)) - elif not Match(r'#.*include', line): - # Look for < that is not surrounded by spaces. This is only - # triggered if both sides are missing spaces, even though - # technically should should flag if at least one side is missing a - # space. This is done to avoid some false positives with shifts. - match = Match(r'^(.*[^\s<])<[^\s=<,]', line) - if match: - (_, _, end_pos) = CloseExpression( - clean_lines, linenum, len(match.group(1))) - if end_pos <= -1: error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around <') + 'Missing spaces around %s' % match.group(1)) + elif not Match(r'#.*include', line): + # Look for < that is not surrounded by spaces. This is only + # triggered if both sides are missing spaces, even though + # technically should should flag if at least one side is missing a + # space. This is done to avoid some false positives with shifts. + match = Match(r'^(.*[^\s<])<[^\s=<,]', line) + if match: + (_, _, end_pos) = CloseExpression(clean_lines, linenum, + len(match.group(1))) + if end_pos <= -1: + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around <') - # Look for > that is not surrounded by spaces. Similar to the - # above, we only trigger if both sides are missing spaces to avoid - # false positives with shifts. - match = Match(r'^(.*[^-\s>])>[^\s=>,]', line) - if match: - (_, _, start_pos) = ReverseCloseExpression( - clean_lines, linenum, len(match.group(1))) - if start_pos <= -1: + # Look for > that is not surrounded by spaces. Similar to the + # above, we only trigger if both sides are missing spaces to avoid + # false positives with shifts. + match = Match(r'^(.*[^-\s>])>[^\s=>,]', line) + if match: + (_, _, start_pos) = ReverseCloseExpression(clean_lines, linenum, + len(match.group(1))) + if start_pos <= -1: + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around >') + + # We allow no-spaces around << when used like this: 10<<20, but + # not otherwise (particularly, not when used as streams) + # + # We also allow operators following an opening parenthesis, since + # those tend to be macros that deal with operators. + match = Search( + r'(operator|[^\s(<])(?:L|UL|LL|ULL|l|ul|ll|ull)?<<([^\s,=<])', line) + if (match and not (match.group(1).isdigit() and match.group(2).isdigit()) + and not (match.group(1) == 'operator' and match.group(2) == ';')): error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around >') + 'Missing spaces around <<') - # We allow no-spaces around << when used like this: 10<<20, but - # not otherwise (particularly, not when used as streams) - # - # We also allow operators following an opening parenthesis, since - # those tend to be macros that deal with operators. - match = Search(r'(operator|[^\s(<])(?:L|UL|LL|ULL|l|ul|ll|ull)?<<([^\s,=<])', line) - if (match and not (match.group(1).isdigit() and match.group(2).isdigit()) and - not (match.group(1) == 'operator' and match.group(2) == ';')): - error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around <<') + # We allow no-spaces around >> for almost anything. This is because + # C++11 allows ">>" to close nested templates, which accounts for + # most cases when ">>" is not followed by a space. + # + # We still warn on ">>" followed by alpha character, because that is + # likely due to ">>" being used for right shifts, e.g.: + # value >> alpha + # + # When ">>" is used to close templates, the alphanumeric letter that + # follows would be part of an identifier, and there should still be + # a space separating the template type and the identifier. + # type> alpha + match = Search(r'>>[a-zA-Z_]', line) + if match: + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around >>') - # We allow no-spaces around >> for almost anything. This is because - # C++11 allows ">>" to close nested templates, which accounts for - # most cases when ">>" is not followed by a space. - # - # We still warn on ">>" followed by alpha character, because that is - # likely due to ">>" being used for right shifts, e.g.: - # value >> alpha - # - # When ">>" is used to close templates, the alphanumeric letter that - # follows would be part of an identifier, and there should still be - # a space separating the template type and the identifier. - # type> alpha - match = Search(r'>>[a-zA-Z_]', line) - if match: - error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around >>') - - # There shouldn't be space around unary operators - match = Search(r'(!\s|~\s|[\s]--[\s;]|[\s]\+\+[\s;])', line) - if match: - error(filename, linenum, 'whitespace/operators', 4, - 'Extra space for operator %s' % match.group(1)) + # There shouldn't be space around unary operators + match = Search(r'(!\s|~\s|[\s]--[\s;]|[\s]\+\+[\s;])', line) + if match: + error(filename, linenum, 'whitespace/operators', 4, + 'Extra space for operator %s' % match.group(1)) def CheckParenthesisSpacing(filename, clean_lines, linenum, error): - """Checks for horizontal spacing around parentheses. + """Checks for horizontal spacing around parentheses. Args: filename: The name of the current file. @@ -3613,37 +3662,38 @@ def CheckParenthesisSpacing(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] + line = clean_lines.elided[linenum] - # No spaces after an if, while, switch, or for - match = Search(r' (if\(|for\(|while\(|switch\()', line) - if match: - error(filename, linenum, 'whitespace/parens', 5, - 'Missing space before ( in %s' % match.group(1)) - - # For if/for/while/switch, the left and right parens should be - # consistent about how many spaces are inside the parens, and - # there should either be zero or one spaces inside the parens. - # We don't want: "if ( foo)" or "if ( foo )". - # Exception: "for ( ; foo; bar)" and "for (foo; bar; )" are allowed. - match = Search(r'\b(if|for|while|switch)\s*' - r'\(([ ]*)(.).*[^ ]+([ ]*)\)\s*{\s*$', - line) - if match: - if len(match.group(2)) != len(match.group(4)): - if not (match.group(3) == ';' and - len(match.group(2)) == 1 + len(match.group(4)) or - not match.group(2) and Search(r'\bfor\s*\(.*; \)', line)): + # No spaces after an if, while, switch, or for + match = Search(r' (if\(|for\(|while\(|switch\()', line) + if match: error(filename, linenum, 'whitespace/parens', 5, - 'Mismatching spaces inside () in %s' % match.group(1)) - if len(match.group(2)) not in [0, 1]: - error(filename, linenum, 'whitespace/parens', 5, - 'Should have zero or one spaces inside ( and ) in %s' % - match.group(1)) + 'Missing space before ( in %s' % match.group(1)) + + # For if/for/while/switch, the left and right parens should be + # consistent about how many spaces are inside the parens, and + # there should either be zero or one spaces inside the parens. + # We don't want: "if ( foo)" or "if ( foo )". + # Exception: "for ( ; foo; bar)" and "for (foo; bar; )" are allowed. + match = Search( + r'\b(if|for|while|switch)\s*' + r'\(([ ]*)(.).*[^ ]+([ ]*)\)\s*{\s*$', line) + if match: + if len(match.group(2)) != len(match.group(4)): + if not (match.group(3) == ';' + and len(match.group(2)) == 1 + len(match.group(4)) or + not match.group(2) and Search(r'\bfor\s*\(.*; \)', line)): + error(filename, linenum, 'whitespace/parens', 5, + 'Mismatching spaces inside () in %s' % match.group(1)) + if len(match.group(2)) not in [0, 1]: + error( + filename, linenum, 'whitespace/parens', 5, + 'Should have zero or one spaces inside ( and ) in %s' % + match.group(1)) def CheckCommaSpacing(filename, clean_lines, linenum, error): - """Checks for horizontal spacing near commas and semicolons. + """Checks for horizontal spacing near commas and semicolons. Args: filename: The name of the current file. @@ -3651,35 +3701,35 @@ def CheckCommaSpacing(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - raw = clean_lines.lines_without_raw_strings - line = clean_lines.elided[linenum] + raw = clean_lines.lines_without_raw_strings + line = clean_lines.elided[linenum] - # You should always have a space after a comma (either as fn arg or operator) - # - # This does not apply when the non-space character following the - # comma is another comma, since the only time when that happens is - # for empty macro arguments. - # - # We run this check in two passes: first pass on elided lines to - # verify that lines contain missing whitespaces, second pass on raw - # lines to confirm that those missing whitespaces are not due to - # elided comments. - if (Search(r',[^,\s]', ReplaceAll(r'\boperator\s*,\s*\(', 'F(', line)) and - Search(r',[^,\s]', raw[linenum])): - error(filename, linenum, 'whitespace/comma', 3, - 'Missing space after ,') + # You should always have a space after a comma (either as fn arg or + # operator) + # + # This does not apply when the non-space character following the + # comma is another comma, since the only time when that happens is + # for empty macro arguments. + # + # We run this check in two passes: first pass on elided lines to + # verify that lines contain missing whitespaces, second pass on raw + # lines to confirm that those missing whitespaces are not due to + # elided comments. + if (Search(r',[^,\s]', ReplaceAll(r'\boperator\s*,\s*\(', 'F(', line)) + and Search(r',[^,\s]', raw[linenum])): + error(filename, linenum, 'whitespace/comma', 3, 'Missing space after ,') - # You should always have a space after a semicolon - # except for few corner cases - # TODO(unknown): clarify if 'if (1) { return 1;}' is requires one more - # space after ; - if Search(r';[^\s};\\)/]', line): - error(filename, linenum, 'whitespace/semicolon', 3, - 'Missing space after ;') + # You should always have a space after a semicolon + # except for few corner cases + # TODO(unknown): clarify if 'if (1) { return 1;}' is requires one more + # space after ; + if Search(r';[^\s};\\)/]', line): + error(filename, linenum, 'whitespace/semicolon', 3, + 'Missing space after ;') def _IsType(clean_lines, nesting_state, expr): - """Check if expression looks like a type name, returns true if so. + """Check if expression looks like a type name, returns true if so. Args: clean_lines: A CleansedLines instance containing the file. @@ -3689,60 +3739,61 @@ def _IsType(clean_lines, nesting_state, expr): Returns: True, if token looks like a type. """ - # Keep only the last token in the expression - last_word = Match(r'^.*(\b\S+)$', expr) - if last_word: - token = last_word.group(1) - else: - token = expr + # Keep only the last token in the expression + last_word = Match(r'^.*(\b\S+)$', expr) + if last_word: + token = last_word.group(1) + else: + token = expr - # Match native types and stdint types - if _TYPES.match(token): - return True - - # Try a bit harder to match templated types. Walk up the nesting - # stack until we find something that resembles a typename - # declaration for what we are looking for. - typename_pattern = (r'\b(?:typename|class|struct)\s+' + re.escape(token) + - r'\b') - block_index = len(nesting_state.stack) - 1 - while block_index >= 0: - if isinstance(nesting_state.stack[block_index], _NamespaceInfo): - return False - - # Found where the opening brace is. We want to scan from this - # line up to the beginning of the function, minus a few lines. - # template - # class C - # : public ... { // start scanning here - last_line = nesting_state.stack[block_index].starting_linenum - - next_block_start = 0 - if block_index > 0: - next_block_start = nesting_state.stack[block_index - 1].starting_linenum - first_line = last_line - while first_line >= next_block_start: - if clean_lines.elided[first_line].find('template') >= 0: - break - first_line -= 1 - if first_line < next_block_start: - # Didn't find any "template" keyword before reaching the next block, - # there are probably no template things to check for this block - block_index -= 1 - continue - - # Look for typename in the specified range - for i in range(first_line, last_line + 1, 1): - if Search(typename_pattern, clean_lines.elided[i]): + # Match native types and stdint types + if _TYPES.match(token): return True - block_index -= 1 - return False + # Try a bit harder to match templated types. Walk up the nesting + # stack until we find something that resembles a typename + # declaration for what we are looking for. + typename_pattern = (r'\b(?:typename|class|struct)\s+' + re.escape(token) + + r'\b') + block_index = len(nesting_state.stack) - 1 + while block_index >= 0: + if isinstance(nesting_state.stack[block_index], _NamespaceInfo): + return False + + # Found where the opening brace is. We want to scan from this + # line up to the beginning of the function, minus a few lines. + # template + # class C + # : public ... { // start scanning here + last_line = nesting_state.stack[block_index].starting_linenum + + next_block_start = 0 + if block_index > 0: + next_block_start = nesting_state.stack[block_index - + 1].starting_linenum + first_line = last_line + while first_line >= next_block_start: + if clean_lines.elided[first_line].find('template') >= 0: + break + first_line -= 1 + if first_line < next_block_start: + # Didn't find any "template" keyword before reaching the next block, + # there are probably no template things to check for this block + block_index -= 1 + continue + + # Look for typename in the specified range + for i in range(first_line, last_line + 1, 1): + if Search(typename_pattern, clean_lines.elided[i]): + return True + block_index -= 1 + + return False def CheckBracesSpacing(filename, clean_lines, linenum, nesting_state, error): - """Checks for horizontal spacing near commas. + """Checks for horizontal spacing near commas. Args: filename: The name of the current file. @@ -3752,86 +3803,88 @@ def CheckBracesSpacing(filename, clean_lines, linenum, nesting_state, error): the current stack of nested blocks being parsed. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] + line = clean_lines.elided[linenum] - # Except after an opening paren, or after another opening brace (in case of - # an initializer list, for instance), you should have spaces before your - # braces when they are delimiting blocks, classes, namespaces etc. - # And since you should never have braces at the beginning of a line, - # this is an easy test. Except that braces used for initialization don't - # follow the same rule; we often don't want spaces before those. - match = Match(r'^(.*[^ ({>]){', line) + # Except after an opening paren, or after another opening brace (in case of + # an initializer list, for instance), you should have spaces before your + # braces when they are delimiting blocks, classes, namespaces etc. + # And since you should never have braces at the beginning of a line, + # this is an easy test. Except that braces used for initialization don't + # follow the same rule; we often don't want spaces before those. + match = Match(r'^(.*[^ ({>]){', line) - if match: - # Try a bit harder to check for brace initialization. This - # happens in one of the following forms: - # Constructor() : initializer_list_{} { ... } - # Constructor{}.MemberFunction() - # Type variable{}; - # FunctionCall(type{}, ...); - # LastArgument(..., type{}); - # LOG(INFO) << type{} << " ..."; - # map_of_type[{...}] = ...; - # ternary = expr ? new type{} : nullptr; - # OuterTemplate{}> - # - # We check for the character following the closing brace, and - # silence the warning if it's one of those listed above, i.e. - # "{.;,)<>]:". - # - # To account for nested initializer list, we allow any number of - # closing braces up to "{;,)<". We can't simply silence the - # warning on first sight of closing brace, because that would - # cause false negatives for things that are not initializer lists. - # Silence this: But not this: - # Outer{ if (...) { - # Inner{...} if (...){ // Missing space before { - # }; } - # - # There is a false negative with this approach if people inserted - # spurious semicolons, e.g. "if (cond){};", but we will catch the - # spurious semicolon with a separate check. - leading_text = match.group(1) - (endline, endlinenum, endpos) = CloseExpression( - clean_lines, linenum, len(match.group(1))) - trailing_text = '' - if endpos > -1: - trailing_text = endline[endpos:] - for offset in range(endlinenum + 1, - min(endlinenum + 3, clean_lines.NumLines() - 1)): - trailing_text += clean_lines.elided[offset] - # We also suppress warnings for `uint64_t{expression}` etc., as the style - # guide recommends brace initialization for integral types to avoid - # overflow/truncation. - if (not Match(r'^[\s}]*[{.;,)<>\]:]', trailing_text) - and not _IsType(clean_lines, nesting_state, leading_text)): - error(filename, linenum, 'whitespace/braces', 5, - 'Missing space before {') + if match: + # Try a bit harder to check for brace initialization. This + # happens in one of the following forms: + # Constructor() : initializer_list_{} { ... } + # Constructor{}.MemberFunction() + # Type variable{}; + # FunctionCall(type{}, ...); + # LastArgument(..., type{}); + # LOG(INFO) << type{} << " ..."; + # map_of_type[{...}] = ...; + # ternary = expr ? new type{} : nullptr; + # OuterTemplate{}> + # + # We check for the character following the closing brace, and + # silence the warning if it's one of those listed above, i.e. + # "{.;,)<>]:". + # + # To account for nested initializer list, we allow any number of + # closing braces up to "{;,)<". We can't simply silence the + # warning on first sight of closing brace, because that would + # cause false negatives for things that are not initializer lists. + # Silence this: But not this: + # Outer{ if (...) { + # Inner{...} if (...){ // Missing space before { + # }; } + # + # There is a false negative with this approach if people inserted + # spurious semicolons, e.g. "if (cond){};", but we will catch the + # spurious semicolon with a separate check. + leading_text = match.group(1) + (endline, endlinenum, endpos) = CloseExpression(clean_lines, linenum, + len(match.group(1))) + trailing_text = '' + if endpos > -1: + trailing_text = endline[endpos:] + for offset in range(endlinenum + 1, + min(endlinenum + 3, + clean_lines.NumLines() - 1)): + trailing_text += clean_lines.elided[offset] + # We also suppress warnings for `uint64_t{expression}` etc., as the + # style guide recommends brace initialization for integral types to + # avoid overflow/truncation. + if (not Match(r'^[\s}]*[{.;,)<>\]:]', trailing_text) + and not _IsType(clean_lines, nesting_state, leading_text)): + error(filename, linenum, 'whitespace/braces', 5, + 'Missing space before {') - # Make sure '} else {' has spaces. - if Search(r'}else', line): - error(filename, linenum, 'whitespace/braces', 5, - 'Missing space before else') + # Make sure '} else {' has spaces. + if Search(r'}else', line): + error(filename, linenum, 'whitespace/braces', 5, + 'Missing space before else') - # You shouldn't have a space before a semicolon at the end of the line. - # There's a special case for "for" since the style guide allows space before - # the semicolon there. - if Search(r':\s*;\s*$', line): - error(filename, linenum, 'whitespace/semicolon', 5, - 'Semicolon defining empty statement. Use {} instead.') - elif Search(r'^\s*;\s*$', line): - error(filename, linenum, 'whitespace/semicolon', 5, - 'Line contains only semicolon. If this should be an empty statement, ' - 'use {} instead.') - elif (Search(r'\s+;\s*$', line) and - not Search(r'\bfor\b', line)): - error(filename, linenum, 'whitespace/semicolon', 5, - 'Extra space before last semicolon. If this should be an empty ' - 'statement, use {} instead.') + # You shouldn't have a space before a semicolon at the end of the line. + # There's a special case for "for" since the style guide allows space before + # the semicolon there. + if Search(r':\s*;\s*$', line): + error(filename, linenum, 'whitespace/semicolon', 5, + 'Semicolon defining empty statement. Use {} instead.') + elif Search(r'^\s*;\s*$', line): + error( + filename, linenum, 'whitespace/semicolon', 5, + 'Line contains only semicolon. If this should be an empty statement, ' + 'use {} instead.') + elif (Search(r'\s+;\s*$', line) and not Search(r'\bfor\b', line)): + error( + filename, linenum, 'whitespace/semicolon', 5, + 'Extra space before last semicolon. If this should be an empty ' + 'statement, use {} instead.') def IsDecltype(clean_lines, linenum, column): - """Check if the token ending on (linenum, column) is decltype(). + """Check if the token ending on (linenum, column) is decltype(). Args: clean_lines: A CleansedLines instance containing the file. @@ -3840,16 +3893,16 @@ def IsDecltype(clean_lines, linenum, column): Returns: True if this token is decltype() expression, False otherwise. """ - (text, _, start_col) = ReverseCloseExpression(clean_lines, linenum, column) - if start_col < 0: + (text, _, start_col) = ReverseCloseExpression(clean_lines, linenum, column) + if start_col < 0: + return False + if Search(r'\bdecltype\s*$', text[0:start_col]): + return True return False - if Search(r'\bdecltype\s*$', text[0:start_col]): - return True - return False def CheckSectionSpacing(filename, clean_lines, class_info, linenum, error): - """Checks for additional blank line issues related to sections. + """Checks for additional blank line issues related to sections. Currently the only thing checked here is blank line before protected/private. @@ -3860,51 +3913,54 @@ def CheckSectionSpacing(filename, clean_lines, class_info, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - # Skip checks if the class is small, where small means 25 lines or less. - # 25 lines seems like a good cutoff since that's the usual height of - # terminals, and any class that can't fit in one screen can't really - # be considered "small". - # - # Also skip checks if we are on the first line. This accounts for - # classes that look like - # class Foo { public: ... }; - # - # If we didn't find the end of the class, last_line would be zero, - # and the check will be skipped by the first condition. - if (class_info.last_line - class_info.starting_linenum <= 24 or - linenum <= class_info.starting_linenum): - return + # Skip checks if the class is small, where small means 25 lines or less. + # 25 lines seems like a good cutoff since that's the usual height of + # terminals, and any class that can't fit in one screen can't really + # be considered "small". + # + # Also skip checks if we are on the first line. This accounts for + # classes that look like + # class Foo { public: ... }; + # + # If we didn't find the end of the class, last_line would be zero, + # and the check will be skipped by the first condition. + if (class_info.last_line - class_info.starting_linenum <= 24 + or linenum <= class_info.starting_linenum): + return - matched = Match(r'\s*(public|protected|private):', clean_lines.lines[linenum]) - if matched: - # Issue warning if the line before public/protected/private was - # not a blank line, but don't do this if the previous line contains - # "class" or "struct". This can happen two ways: - # - We are at the beginning of the class. - # - We are forward-declaring an inner class that is semantically - # private, but needed to be public for implementation reasons. - # Also ignores cases where the previous line ends with a backslash as can be - # common when defining classes in C macros. - prev_line = clean_lines.lines[linenum - 1] - if (not IsBlankLine(prev_line) and - not Search(r'\b(class|struct)\b', prev_line) and - not Search(r'\\$', prev_line)): - # Try a bit harder to find the beginning of the class. This is to - # account for multi-line base-specifier lists, e.g.: - # class Derived - # : public Base { - end_class_head = class_info.starting_linenum - for i in range(class_info.starting_linenum, linenum): - if Search(r'\{\s*$', clean_lines.lines[i]): - end_class_head = i - break - if end_class_head < linenum - 1: - error(filename, linenum, 'whitespace/blank_line', 3, - '"%s:" should be preceded by a blank line' % matched.group(1)) + matched = Match(r'\s*(public|protected|private):', + clean_lines.lines[linenum]) + if matched: + # Issue warning if the line before public/protected/private was + # not a blank line, but don't do this if the previous line contains + # "class" or "struct". This can happen two ways: + # - We are at the beginning of the class. + # - We are forward-declaring an inner class that is semantically + # private, but needed to be public for implementation reasons. + # Also ignores cases where the previous line ends with a backslash as + # can be common when defining classes in C macros. + prev_line = clean_lines.lines[linenum - 1] + if (not IsBlankLine(prev_line) + and not Search(r'\b(class|struct)\b', prev_line) + and not Search(r'\\$', prev_line)): + # Try a bit harder to find the beginning of the class. This is to + # account for multi-line base-specifier lists, e.g.: + # class Derived + # : public Base { + end_class_head = class_info.starting_linenum + for i in range(class_info.starting_linenum, linenum): + if Search(r'\{\s*$', clean_lines.lines[i]): + end_class_head = i + break + if end_class_head < linenum - 1: + error( + filename, linenum, 'whitespace/blank_line', 3, + '"%s:" should be preceded by a blank line' % + matched.group(1)) def GetPreviousNonBlankLine(clean_lines, linenum): - """Return the most recent non-blank line and its line number. + """Return the most recent non-blank line and its line number. Args: clean_lines: A CleansedLines instance containing the file contents. @@ -3917,17 +3973,17 @@ def GetPreviousNonBlankLine(clean_lines, linenum): if this is the first non-blank line. """ - prevlinenum = linenum - 1 - while prevlinenum >= 0: - prevline = clean_lines.elided[prevlinenum] - if not IsBlankLine(prevline): # if not a blank line... - return (prevline, prevlinenum) - prevlinenum -= 1 - return ('', -1) + prevlinenum = linenum - 1 + while prevlinenum >= 0: + prevline = clean_lines.elided[prevlinenum] + if not IsBlankLine(prevline): # if not a blank line... + return (prevline, prevlinenum) + prevlinenum -= 1 + return ('', -1) def CheckBraces(filename, clean_lines, linenum, error): - """Looks for misplaced braces (e.g. at the end of line). + """Looks for misplaced braces (e.g. at the end of line). Args: filename: The name of the current file. @@ -3936,116 +3992,128 @@ def CheckBraces(filename, clean_lines, linenum, error): error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] # get rid of comments and strings + line = clean_lines.elided[linenum] # get rid of comments and strings - if Match(r'\s*{\s*$', line): - # We allow an open brace to start a line in the case where someone is using - # braces in a block to explicitly create a new scope, which is commonly used - # to control the lifetime of stack-allocated variables. Braces are also - # used for brace initializers inside function calls. We don't detect this - # perfectly: we just don't complain if the last non-whitespace character on - # the previous non-blank line is ',', ';', ':', '(', '{', or '}', or if the - # previous line starts a preprocessor block. We also allow a brace on the - # following line if it is part of an array initialization and would not fit - # within the 80 character limit of the preceding line. - prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] - if (not Search(r'[,;:}{(]\s*$', prevline) and not Match(r'\s*#', prevline) - and not (len(prevline) > _line_length - 2 and '[]' in prevline)): - error(filename, linenum, 'whitespace/braces', 4, - '{ should almost always be at the end of the previous line') + if Match(r'\s*{\s*$', line): + # We allow an open brace to start a line in the case where someone is + # using braces in a block to explicitly create a new scope, which is + # commonly used to control the lifetime of stack-allocated variables. + # Braces are also used for brace initializers inside function calls. We + # don't detect this perfectly: we just don't complain if the last + # non-whitespace character on the previous non-blank line is ',', ';', + # ':', '(', '{', or '}', or if the previous line starts a preprocessor + # block. We also allow a brace on the following line if it is part of an + # array initialization and would not fit within the 80 character limit + # of the preceding line. + prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] + if (not Search(r'[,;:}{(]\s*$', prevline) + and not Match(r'\s*#', prevline) and + not (len(prevline) > _line_length - 2 and '[]' in prevline)): + error(filename, linenum, 'whitespace/braces', 4, + '{ should almost always be at the end of the previous line') - # An else clause should be on the same line as the preceding closing brace. - if Match(r'\s*else\b\s*(?:if\b|\{|$)', line): - prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] - if Match(r'\s*}\s*$', prevline): - error(filename, linenum, 'whitespace/newline', 4, - 'An else should appear on the same line as the preceding }') + # An else clause should be on the same line as the preceding closing brace. + if Match(r'\s*else\b\s*(?:if\b|\{|$)', line): + prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] + if Match(r'\s*}\s*$', prevline): + error(filename, linenum, 'whitespace/newline', 4, + 'An else should appear on the same line as the preceding }') - # If braces come on one side of an else, they should be on both. - # However, we have to worry about "else if" that spans multiple lines! - if Search(r'else if\s*(?:constexpr\s*)?\(', line): # could be multi-line if - brace_on_left = bool(Search(r'}\s*else if\s*(?:constexpr\s*)?\(', line)) - # find the ( after the if - pos = line.find('else if') - pos = line.find('(', pos) - if pos > 0: - (endline, _, endpos) = CloseExpression(clean_lines, linenum, pos) - brace_on_right = endline[endpos:].find('{') != -1 - if brace_on_left != brace_on_right: # must be brace after if + # If braces come on one side of an else, they should be on both. + # However, we have to worry about "else if" that spans multiple lines! + if Search(r'else if\s*(?:constexpr\s*)?\(', line): # could be multi-line if + brace_on_left = bool(Search(r'}\s*else if\s*(?:constexpr\s*)?\(', line)) + # find the ( after the if + pos = line.find('else if') + pos = line.find('(', pos) + if pos > 0: + (endline, _, endpos) = CloseExpression(clean_lines, linenum, pos) + brace_on_right = endline[endpos:].find('{') != -1 + if brace_on_left != brace_on_right: # must be brace after if + error( + filename, linenum, 'readability/braces', 5, + 'If an else has a brace on one side, it should have it on both' + ) + elif Search(r'}\s*else[^{]*$', line) or Match(r'[^}]*else\s*{', line): error(filename, linenum, 'readability/braces', 5, 'If an else has a brace on one side, it should have it on both') - elif Search(r'}\s*else[^{]*$', line) or Match(r'[^}]*else\s*{', line): - error(filename, linenum, 'readability/braces', 5, - 'If an else has a brace on one side, it should have it on both') - # Likewise, an else should never have the else clause on the same line - if Search(r'\belse [^\s{]', line) and not Search(r'\belse if\b', line): - error(filename, linenum, 'whitespace/newline', 4, - 'Else clause should never be on same line as else (use 2 lines)') + # Likewise, an else should never have the else clause on the same line + if Search(r'\belse [^\s{]', line) and not Search(r'\belse if\b', line): + error(filename, linenum, 'whitespace/newline', 4, + 'Else clause should never be on same line as else (use 2 lines)') - # In the same way, a do/while should never be on one line - if Match(r'\s*do [^\s{]', line): - error(filename, linenum, 'whitespace/newline', 4, - 'do/while clauses should not be on a single line') + # In the same way, a do/while should never be on one line + if Match(r'\s*do [^\s{]', line): + error(filename, linenum, 'whitespace/newline', 4, + 'do/while clauses should not be on a single line') - # Check single-line if/else bodies. The style guide says 'curly braces are not - # required for single-line statements'. We additionally allow multi-line, - # single statements, but we reject anything with more than one semicolon in - # it. This means that the first semicolon after the if should be at the end of - # its line, and the line after that should have an indent level equal to or - # lower than the if. We also check for ambiguous if/else nesting without - # braces. - if_else_match = Search(r'\b(if\s*(?:constexpr\s*)?\(|else\b)', line) - if if_else_match and not Match(r'\s*#', line): - if_indent = GetIndentLevel(line) - endline, endlinenum, endpos = line, linenum, if_else_match.end() - if_match = Search(r'\bif\s*(?:constexpr\s*)?\(', line) - if if_match: - # This could be a multiline if condition, so find the end first. - pos = if_match.end() - 1 - (endline, endlinenum, endpos) = CloseExpression(clean_lines, linenum, pos) - # Check for an opening brace, either directly after the if or on the next - # line. If found, this isn't a single-statement conditional. - if (not Match(r'\s*{', endline[endpos:]) - and not (Match(r'\s*$', endline[endpos:]) - and endlinenum < (len(clean_lines.elided) - 1) - and Match(r'\s*{', clean_lines.elided[endlinenum + 1]))): - while (endlinenum < len(clean_lines.elided) - and ';' not in clean_lines.elided[endlinenum][endpos:]): - endlinenum += 1 - endpos = 0 - if endlinenum < len(clean_lines.elided): - endline = clean_lines.elided[endlinenum] - # We allow a mix of whitespace and closing braces (e.g. for one-liner - # methods) and a single \ after the semicolon (for macros) - endpos = endline.find(';') - if not Match(r';[\s}]*(\\?)$', endline[endpos:]): - # Semicolon isn't the last character, there's something trailing. - # Output a warning if the semicolon is not contained inside - # a lambda expression. - if not Match(r'^[^{};]*\[[^\[\]]*\][^{}]*\{[^{}]*\}\s*\)*[;,]\s*$', - endline): - error(filename, linenum, 'readability/braces', 4, - 'If/else bodies with multiple statements require braces') - elif endlinenum < len(clean_lines.elided) - 1: - # Make sure the next line is dedented - next_line = clean_lines.elided[endlinenum + 1] - next_indent = GetIndentLevel(next_line) - # With ambiguous nested if statements, this will error out on the - # if that *doesn't* match the else, regardless of whether it's the - # inner one or outer one. - if (if_match and Match(r'\s*else\b', next_line) - and next_indent != if_indent): - error(filename, linenum, 'readability/braces', 4, - 'Else clause should be indented at the same level as if. ' - 'Ambiguous nested if/else chains require braces.') - elif next_indent > if_indent: - error(filename, linenum, 'readability/braces', 4, - 'If/else bodies with multiple statements require braces') + # Check single-line if/else bodies. The style guide says 'curly braces are + # not required for single-line statements'. We additionally allow + # multi-line, single statements, but we reject anything with more than one + # semicolon in it. This means that the first semicolon after the if should + # be at the end of its line, and the line after that should have an indent + # level equal to or lower than the if. We also check for ambiguous if/else + # nesting without braces. + if_else_match = Search(r'\b(if\s*(?:constexpr\s*)?\(|else\b)', line) + if if_else_match and not Match(r'\s*#', line): + if_indent = GetIndentLevel(line) + endline, endlinenum, endpos = line, linenum, if_else_match.end() + if_match = Search(r'\bif\s*(?:constexpr\s*)?\(', line) + if if_match: + # This could be a multiline if condition, so find the end first. + pos = if_match.end() - 1 + (endline, endlinenum, + endpos) = CloseExpression(clean_lines, linenum, pos) + # Check for an opening brace, either directly after the if or on the + # next line. If found, this isn't a single-statement conditional. + if (not Match(r'\s*{', endline[endpos:]) + and not (Match(r'\s*$', endline[endpos:]) and endlinenum < + (len(clean_lines.elided) - 1) and Match( + r'\s*{', clean_lines.elided[endlinenum + 1]))): + while (endlinenum < len(clean_lines.elided) + and ';' not in clean_lines.elided[endlinenum][endpos:]): + endlinenum += 1 + endpos = 0 + if endlinenum < len(clean_lines.elided): + endline = clean_lines.elided[endlinenum] + # We allow a mix of whitespace and closing braces (e.g. for + # one-liner methods) and a single \ after the semicolon (for + # macros) + endpos = endline.find(';') + if not Match(r';[\s}]*(\\?)$', endline[endpos:]): + # Semicolon isn't the last character, there's something + # trailing. Output a warning if the semicolon is not + # contained inside a lambda expression. + if not Match( + r'^[^{};]*\[[^\[\]]*\][^{}]*\{[^{}]*\}\s*\)*[;,]\s*$', + endline): + error( + filename, linenum, 'readability/braces', 4, + 'If/else bodies with multiple statements require braces' + ) + elif endlinenum < len(clean_lines.elided) - 1: + # Make sure the next line is dedented + next_line = clean_lines.elided[endlinenum + 1] + next_indent = GetIndentLevel(next_line) + # With ambiguous nested if statements, this will error out + # on the if that *doesn't* match the else, regardless of + # whether it's the inner one or outer one. + if (if_match and Match(r'\s*else\b', next_line) + and next_indent != if_indent): + error( + filename, linenum, 'readability/braces', 4, + 'Else clause should be indented at the same level as if. ' + 'Ambiguous nested if/else chains require braces.') + elif next_indent > if_indent: + error( + filename, linenum, 'readability/braces', 4, + 'If/else bodies with multiple statements require braces' + ) def CheckTrailingSemicolon(filename, clean_lines, linenum, error): - """Looks for redundant trailing semicolon. + """Looks for redundant trailing semicolon. Args: filename: The name of the current file. @@ -4054,143 +4122,143 @@ def CheckTrailingSemicolon(filename, clean_lines, linenum, error): error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] + line = clean_lines.elided[linenum] - # Block bodies should not be followed by a semicolon. Due to C++11 - # brace initialization and C++20 concepts, there are more places - # where semicolons are required than not. Places that are - # recognized as true positives are listed below. - # - # 1. Some flavor of block following closing parenthesis: - # for (;;) {}; - # while (...) {}; - # switch (...) {}; - # Function(...) {}; - # if (...) {}; - # if (...) else if (...) {}; - # - # 2. else block: - # if (...) else {}; - # - # 3. const member function: - # Function(...) const {}; - # - # 4. Block following some statement: - # x = 42; - # {}; - # - # 5. Block at the beginning of a function: - # Function(...) { - # {}; - # } - # - # Note that naively checking for the preceding "{" will also match - # braces inside multi-dimensional arrays, but this is fine since - # that expression will not contain semicolons. - # - # 6. Block following another block: - # while (true) {} - # {}; - # - # 7. End of namespaces: - # namespace {}; - # - # These semicolons seems far more common than other kinds of - # redundant semicolons, possibly due to people converting classes - # to namespaces. For now we do not warn for this case. - # - # Try matching case 1 first. - match = Match(r'^(.*\)\s*)\{', line) - if match: - # Matched closing parenthesis (case 1). Check the token before the - # matching opening parenthesis, and don't warn if it looks like a - # macro. This avoids these false positives: - # - macro that defines a base class - # - multi-line macro that defines a base class - # - macro that defines the whole class-head + # Block bodies should not be followed by a semicolon. Due to C++11 + # brace initialization and C++20 concepts, there are more places + # where semicolons are required than not. Places that are + # recognized as true positives are listed below. # - # But we still issue warnings for macros that we know are safe to - # warn, specifically: - # - TEST, TEST_F, TEST_P, MATCHER, MATCHER_P - # - TYPED_TEST - # - INTERFACE_DEF - # - EXCLUSIVE_LOCKS_REQUIRED, SHARED_LOCKS_REQUIRED, LOCKS_EXCLUDED: + # 1. Some flavor of block following closing parenthesis: + # for (;;) {}; + # while (...) {}; + # switch (...) {}; + # Function(...) {}; + # if (...) {}; + # if (...) else if (...) {}; # - # We implement an allowlist of safe macros instead of a blocklist of - # unsafe macros, even though the latter appears less frequently in - # google code and would have been easier to implement. This is because - # the downside for getting the allowlist wrong means some extra - # semicolons, while the downside for getting the blocklist wrong - # would result in compile errors. + # 2. else block: + # if (...) else {}; # - # In addition to macros, we also don't want to warn on - # - Compound literals - # - Lambdas - # - alignas specifier with anonymous structs - # - decltype - # - Type casts with parentheses, e.g.: var = (Type){value}; - # - Return type casts with parentheses, e.g.: return (Type){value}; - # - Function pointers with initializer list, e.g.: int (*f)(){}; - # - Requires expression, e.g. C = requires(){}; - closing_brace_pos = match.group(1).rfind(')') - opening_parenthesis = ReverseCloseExpression( - clean_lines, linenum, closing_brace_pos) - if opening_parenthesis[2] > -1: - line_prefix = opening_parenthesis[0][0:opening_parenthesis[2]] - macro = Search(r'\b([A-Z_][A-Z0-9_]*)\s*$', line_prefix) - func = Match(r'^(.*\])\s*$', line_prefix) - if ((macro and macro.group(1) not in - ('TEST', 'TEST_F', 'MATCHER', 'MATCHER_P', 'TYPED_TEST', - 'EXCLUSIVE_LOCKS_REQUIRED', 'SHARED_LOCKS_REQUIRED', - 'LOCKS_EXCLUDED', 'INTERFACE_DEF')) - or (func and not Search(r'\boperator\s*\[\s*\]', func.group(1))) - or Search(r'\b(?:struct|union)\s+alignas\s*$', line_prefix) - or Search(r'\b(decltype|requires)$', line_prefix) - or Search(r'(?:\s+=|\breturn)\s*$', line_prefix) - or (Match(r'^\s*$', line_prefix) and Search( - r'(?:\s+=|\breturn)\s*$', clean_lines.elided[linenum - 1])) - or Search(r'\(\*\w+\)$', line_prefix)): - match = None - if (match and - opening_parenthesis[1] > 1 and - Search(r'\]\s*$', clean_lines.elided[opening_parenthesis[1] - 1])): - # Multi-line lambda-expression - match = None + # 3. const member function: + # Function(...) const {}; + # + # 4. Block following some statement: + # x = 42; + # {}; + # + # 5. Block at the beginning of a function: + # Function(...) { + # {}; + # } + # + # Note that naively checking for the preceding "{" will also match + # braces inside multi-dimensional arrays, but this is fine since + # that expression will not contain semicolons. + # + # 6. Block following another block: + # while (true) {} + # {}; + # + # 7. End of namespaces: + # namespace {}; + # + # These semicolons seems far more common than other kinds of + # redundant semicolons, possibly due to people converting classes + # to namespaces. For now we do not warn for this case. + # + # Try matching case 1 first. + match = Match(r'^(.*\)\s*)\{', line) + if match: + # Matched closing parenthesis (case 1). Check the token before the + # matching opening parenthesis, and don't warn if it looks like a + # macro. This avoids these false positives: + # - macro that defines a base class + # - multi-line macro that defines a base class + # - macro that defines the whole class-head + # + # But we still issue warnings for macros that we know are safe to + # warn, specifically: + # - TEST, TEST_F, TEST_P, MATCHER, MATCHER_P + # - TYPED_TEST + # - INTERFACE_DEF + # - EXCLUSIVE_LOCKS_REQUIRED, SHARED_LOCKS_REQUIRED, LOCKS_EXCLUDED: + # + # We implement an allowlist of safe macros instead of a blocklist of + # unsafe macros, even though the latter appears less frequently in + # google code and would have been easier to implement. This is because + # the downside for getting the allowlist wrong means some extra + # semicolons, while the downside for getting the blocklist wrong + # would result in compile errors. + # + # In addition to macros, we also don't want to warn on + # - Compound literals + # - Lambdas + # - alignas specifier with anonymous structs + # - decltype + # - Type casts with parentheses, e.g.: var = (Type){value}; + # - Return type casts with parentheses, e.g.: return (Type){value}; + # - Function pointers with initializer list, e.g.: int (*f)(){}; + # - Requires expression, e.g. C = requires(){}; + closing_brace_pos = match.group(1).rfind(')') + opening_parenthesis = ReverseCloseExpression(clean_lines, linenum, + closing_brace_pos) + if opening_parenthesis[2] > -1: + line_prefix = opening_parenthesis[0][0:opening_parenthesis[2]] + macro = Search(r'\b([A-Z_][A-Z0-9_]*)\s*$', line_prefix) + func = Match(r'^(.*\])\s*$', line_prefix) + if ((macro and macro.group(1) + not in ('TEST', 'TEST_F', 'MATCHER', 'MATCHER_P', 'TYPED_TEST', + 'EXCLUSIVE_LOCKS_REQUIRED', 'SHARED_LOCKS_REQUIRED', + 'LOCKS_EXCLUDED', 'INTERFACE_DEF')) or + (func and not Search(r'\boperator\s*\[\s*\]', func.group(1))) + or Search(r'\b(?:struct|union)\s+alignas\s*$', line_prefix) + or Search(r'\b(decltype|requires)$', line_prefix) + or Search(r'(?:\s+=|\breturn)\s*$', line_prefix) or + (Match(r'^\s*$', line_prefix) and Search( + r'(?:\s+=|\breturn)\s*$', clean_lines.elided[linenum - 1])) + or Search(r'\(\*\w+\)$', line_prefix)): + match = None + if (match and opening_parenthesis[1] > 1 and Search( + r'\]\s*$', clean_lines.elided[opening_parenthesis[1] - 1])): + # Multi-line lambda-expression + match = None - else: - # Try matching cases 2-3. - match = Match(r'^(.*(?:else|\)\s*const)\s*)\{', line) - if not match: - # Try matching cases 4-6. These are always matched on separate lines. - # - # Note that we can't simply concatenate the previous line to the - # current line and do a single match, otherwise we may output - # duplicate warnings for the blank line case: - # if (cond) { - # // blank line - # } - prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] - if prevline and Search(r'[;{}]\s*$', prevline): - match = Match(r'^(\s*)\{', line) + else: + # Try matching cases 2-3. + match = Match(r'^(.*(?:else|\)\s*const)\s*)\{', line) + if not match: + # Try matching cases 4-6. These are always matched on separate + # lines. + # + # Note that we can't simply concatenate the previous line to the + # current line and do a single match, otherwise we may output + # duplicate warnings for the blank line case: + # if (cond) { + # // blank line + # } + prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] + if prevline and Search(r'[;{}]\s*$', prevline): + match = Match(r'^(\s*)\{', line) - # Check matching closing brace - if match: - (endline, endlinenum, endpos) = CloseExpression( - clean_lines, linenum, len(match.group(1))) - if endpos > -1 and Match(r'^\s*;', endline[endpos:]): - # Current {} pair is eligible for semicolon check, and we have found - # the redundant semicolon, output warning here. - # - # Note: because we are scanning forward for opening braces, and - # outputting warnings for the matching closing brace, if there are - # nested blocks with trailing semicolons, we will get the error - # messages in reversed order. - error(filename, endlinenum, 'readability/braces', 4, - "You don't need a ; after a }") + # Check matching closing brace + if match: + (endline, endlinenum, endpos) = CloseExpression(clean_lines, linenum, + len(match.group(1))) + if endpos > -1 and Match(r'^\s*;', endline[endpos:]): + # Current {} pair is eligible for semicolon check, and we have found + # the redundant semicolon, output warning here. + # + # Note: because we are scanning forward for opening braces, and + # outputting warnings for the matching closing brace, if there are + # nested blocks with trailing semicolons, we will get the error + # messages in reversed order. + error(filename, endlinenum, 'readability/braces', 4, + "You don't need a ; after a }") def CheckEmptyBlockBody(filename, clean_lines, linenum, error): - """Look for empty loop/conditional body with only a single semicolon. + """Look for empty loop/conditional body with only a single semicolon. Args: filename: The name of the current file. @@ -4199,102 +4267,115 @@ def CheckEmptyBlockBody(filename, clean_lines, linenum, error): error: The function to call with any errors found. """ - # Search for loop keywords at the beginning of the line. Because only - # whitespaces are allowed before the keywords, this will also ignore most - # do-while-loops, since those lines should start with closing brace. - # - # We also check "if" blocks here, since an empty conditional block - # is likely an error. - line = clean_lines.elided[linenum] - matched = Match(r'\s*(for|while|if)\s*\(', line) - if matched: - # Find the end of the conditional expression. - (end_line, end_linenum, end_pos) = CloseExpression( - clean_lines, linenum, line.find('(')) + # Search for loop keywords at the beginning of the line. Because only + # whitespaces are allowed before the keywords, this will also ignore most + # do-while-loops, since those lines should start with closing brace. + # + # We also check "if" blocks here, since an empty conditional block + # is likely an error. + line = clean_lines.elided[linenum] + matched = Match(r'\s*(for|while|if)\s*\(', line) + if matched: + # Find the end of the conditional expression. + (end_line, end_linenum, + end_pos) = CloseExpression(clean_lines, linenum, line.find('(')) - # Output warning if what follows the condition expression is a semicolon. - # No warning for all other cases, including whitespace or newline, since we - # have a separate check for semicolons preceded by whitespace. - if end_pos >= 0 and Match(r';', end_line[end_pos:]): - if matched.group(1) == 'if': - error(filename, end_linenum, 'whitespace/empty_conditional_body', 5, - 'Empty conditional bodies should use {}') - else: - error(filename, end_linenum, 'whitespace/empty_loop_body', 5, - 'Empty loop bodies should use {} or continue') + # Output warning if what follows the condition expression is a + # semicolon. No warning for all other cases, including whitespace or + # newline, since we have a separate check for semicolons preceded by + # whitespace. + if end_pos >= 0 and Match(r';', end_line[end_pos:]): + if matched.group(1) == 'if': + error(filename, end_linenum, + 'whitespace/empty_conditional_body', 5, + 'Empty conditional bodies should use {}') + else: + error(filename, end_linenum, 'whitespace/empty_loop_body', 5, + 'Empty loop bodies should use {} or continue') - # Check for if statements that have completely empty bodies (no comments) - # and no else clauses. - if end_pos >= 0 and matched.group(1) == 'if': - # Find the position of the opening { for the if statement. - # Return without logging an error if it has no brackets. - opening_linenum = end_linenum - opening_line_fragment = end_line[end_pos:] - # Loop until EOF or find anything that's not whitespace or opening {. - while not Search(r'^\s*\{', opening_line_fragment): - if Search(r'^(?!\s*$)', opening_line_fragment): - # Conditional has no brackets. - return - opening_linenum += 1 - if opening_linenum == len(clean_lines.elided): - # Couldn't find conditional's opening { or any code before EOF. - return - opening_line_fragment = clean_lines.elided[opening_linenum] - # Set opening_line (opening_line_fragment may not be entire opening line). - opening_line = clean_lines.elided[opening_linenum] + # Check for if statements that have completely empty bodies (no + # comments) and no else clauses. + if end_pos >= 0 and matched.group(1) == 'if': + # Find the position of the opening { for the if statement. + # Return without logging an error if it has no brackets. + opening_linenum = end_linenum + opening_line_fragment = end_line[end_pos:] + # Loop until EOF or find anything that's not whitespace or opening + # {. + while not Search(r'^\s*\{', opening_line_fragment): + if Search(r'^(?!\s*$)', opening_line_fragment): + # Conditional has no brackets. + return + opening_linenum += 1 + if opening_linenum == len(clean_lines.elided): + # Couldn't find conditional's opening { or any code before + # EOF. + return + opening_line_fragment = clean_lines.elided[opening_linenum] + # Set opening_line (opening_line_fragment may not be entire opening + # line). + opening_line = clean_lines.elided[opening_linenum] - # Find the position of the closing }. - opening_pos = opening_line_fragment.find('{') - if opening_linenum == end_linenum: - # We need to make opening_pos relative to the start of the entire line. - opening_pos += end_pos - (closing_line, closing_linenum, closing_pos) = CloseExpression( - clean_lines, opening_linenum, opening_pos) - if closing_pos < 0: - return + # Find the position of the closing }. + opening_pos = opening_line_fragment.find('{') + if opening_linenum == end_linenum: + # We need to make opening_pos relative to the start of the + # entire line. + opening_pos += end_pos + (closing_line, closing_linenum, + closing_pos) = CloseExpression(clean_lines, opening_linenum, + opening_pos) + if closing_pos < 0: + return - # Now construct the body of the conditional. This consists of the portion - # of the opening line after the {, all lines until the closing line, - # and the portion of the closing line before the }. - if (clean_lines.raw_lines[opening_linenum] != - CleanseComments(clean_lines.raw_lines[opening_linenum])): - # Opening line ends with a comment, so conditional isn't empty. - return - if closing_linenum > opening_linenum: - # Opening line after the {. Ignore comments here since we checked above. - body = list(opening_line[opening_pos+1:]) - # All lines until closing line, excluding closing line, with comments. - body.extend(clean_lines.raw_lines[opening_linenum+1:closing_linenum]) - # Closing line before the }. Won't (and can't) have comments. - body.append(clean_lines.elided[closing_linenum][:closing_pos-1]) - body = '\n'.join(body) - else: - # If statement has brackets and fits on a single line. - body = opening_line[opening_pos+1:closing_pos-1] + # Now construct the body of the conditional. This consists of the + # portion of the opening line after the {, all lines until the + # closing line, and the portion of the closing line before the }. + if (clean_lines.raw_lines[opening_linenum] != CleanseComments( + clean_lines.raw_lines[opening_linenum])): + # Opening line ends with a comment, so conditional isn't empty. + return + if closing_linenum > opening_linenum: + # Opening line after the {. Ignore comments here since we + # checked above. + body = list(opening_line[opening_pos + 1:]) + # All lines until closing line, excluding closing line, with + # comments. + body.extend(clean_lines.raw_lines[opening_linenum + + 1:closing_linenum]) + # Closing line before the }. Won't (and can't) have comments. + body.append(clean_lines.elided[closing_linenum][:closing_pos - + 1]) + body = '\n'.join(body) + else: + # If statement has brackets and fits on a single line. + body = opening_line[opening_pos + 1:closing_pos - 1] - # Check if the body is empty - if not _EMPTY_CONDITIONAL_BODY_PATTERN.search(body): - return - # The body is empty. Now make sure there's not an else clause. - current_linenum = closing_linenum - current_line_fragment = closing_line[closing_pos:] - # Loop until EOF or find anything that's not whitespace or else clause. - while Search(r'^\s*$|^(?=\s*else)', current_line_fragment): - if Search(r'^(?=\s*else)', current_line_fragment): - # Found an else clause, so don't log an error. - return - current_linenum += 1 - if current_linenum == len(clean_lines.elided): - break - current_line_fragment = clean_lines.elided[current_linenum] + # Check if the body is empty + if not _EMPTY_CONDITIONAL_BODY_PATTERN.search(body): + return + # The body is empty. Now make sure there's not an else clause. + current_linenum = closing_linenum + current_line_fragment = closing_line[closing_pos:] + # Loop until EOF or find anything that's not whitespace or else + # clause. + while Search(r'^\s*$|^(?=\s*else)', current_line_fragment): + if Search(r'^(?=\s*else)', current_line_fragment): + # Found an else clause, so don't log an error. + return + current_linenum += 1 + if current_linenum == len(clean_lines.elided): + break + current_line_fragment = clean_lines.elided[current_linenum] - # The body is empty and there's no else clause until EOF or other code. - error(filename, end_linenum, 'whitespace/empty_if_body', 4, - ('If statement had no body and no else clause')) + # The body is empty and there's no else clause until EOF or other + # code. + error(filename, end_linenum, 'whitespace/empty_if_body', 4, + ('If statement had no body and no else clause')) def FindCheckMacro(line): - """Find a replaceable CHECK-like macro. + """Find a replaceable CHECK-like macro. Args: line: line to search on. @@ -4302,22 +4383,22 @@ def FindCheckMacro(line): (macro name, start position), or (None, -1) if no replaceable macro is found. """ - for macro in _CHECK_MACROS: - i = line.find(macro) - if i >= 0: - # Find opening parenthesis. Do a regular expression match here - # to make sure that we are matching the expected CHECK macro, as - # opposed to some other macro that happens to contain the CHECK - # substring. - matched = Match(r'^(.*\b' + macro + r'\s*)\(', line) - if not matched: - continue - return (macro, len(matched.group(1))) - return (None, -1) + for macro in _CHECK_MACROS: + i = line.find(macro) + if i >= 0: + # Find opening parenthesis. Do a regular expression match here + # to make sure that we are matching the expected CHECK macro, as + # opposed to some other macro that happens to contain the CHECK + # substring. + matched = Match(r'^(.*\b' + macro + r'\s*)\(', line) + if not matched: + continue + return (macro, len(matched.group(1))) + return (None, -1) def CheckCheck(filename, clean_lines, linenum, error): - """Checks the use of CHECK and EXPECT macros. + """Checks the use of CHECK and EXPECT macros. Args: filename: The name of the current file. @@ -4326,116 +4407,117 @@ def CheckCheck(filename, clean_lines, linenum, error): error: The function to call with any errors found. """ - # Decide the set of replacement macros that should be suggested - lines = clean_lines.elided - (check_macro, start_pos) = FindCheckMacro(lines[linenum]) - if not check_macro: - return - - # Find end of the boolean expression by matching parentheses - (last_line, end_line, end_pos) = CloseExpression( - clean_lines, linenum, start_pos) - if end_pos < 0: - return - - # If the check macro is followed by something other than a - # semicolon, assume users will log their own custom error messages - # and don't suggest any replacements. - if not Match(r'\s*;', last_line[end_pos:]): - return - - if linenum == end_line: - expression = lines[linenum][start_pos + 1:end_pos - 1] - else: - expression = lines[linenum][start_pos + 1:] - for i in range(linenum + 1, end_line): - expression += lines[i] - expression += last_line[0:end_pos - 1] - - # Parse expression so that we can take parentheses into account. - # This avoids false positives for inputs like "CHECK((a < 4) == b)", - # which is not replaceable by CHECK_LE. - lhs = '' - rhs = '' - operator = None - while expression: - matched = Match(r'^\s*(<<|<<=|>>|>>=|->\*|->|&&|\|\||' - r'==|!=|>=|>|<=|<|\()(.*)$', expression) - if matched: - token = matched.group(1) - if token == '(': - # Parenthesized operand - expression = matched.group(2) - (end, _) = FindEndOfExpressionInLine(expression, 0, ['(']) - if end < 0: - return # Unmatched parenthesis - lhs += '(' + expression[0:end] - expression = expression[end:] - elif token in ('&&', '||'): - # Logical and/or operators. This means the expression - # contains more than one term, for example: - # CHECK(42 < a && a < b); - # - # These are not replaceable with CHECK_LE, so bail out early. + # Decide the set of replacement macros that should be suggested + lines = clean_lines.elided + (check_macro, start_pos) = FindCheckMacro(lines[linenum]) + if not check_macro: return - elif token in ('<<', '<<=', '>>', '>>=', '->*', '->'): - # Non-relational operator - lhs += token - expression = matched.group(2) - else: - # Relational operator - operator = token - rhs = matched.group(2) - break + + # Find end of the boolean expression by matching parentheses + (last_line, end_line, end_pos) = CloseExpression(clean_lines, linenum, + start_pos) + if end_pos < 0: + return + + # If the check macro is followed by something other than a + # semicolon, assume users will log their own custom error messages + # and don't suggest any replacements. + if not Match(r'\s*;', last_line[end_pos:]): + return + + if linenum == end_line: + expression = lines[linenum][start_pos + 1:end_pos - 1] else: - # Unparenthesized operand. Instead of appending to lhs one character - # at a time, we do another regular expression match to consume several - # characters at once if possible. Trivial benchmark shows that this - # is more efficient when the operands are longer than a single - # character, which is generally the case. - matched = Match(r'^([^-=!<>()&|]+)(.*)$', expression) - if not matched: - matched = Match(r'^(\s*\S)(.*)$', expression) - if not matched: - break - lhs += matched.group(1) - expression = matched.group(2) + expression = lines[linenum][start_pos + 1:] + for i in range(linenum + 1, end_line): + expression += lines[i] + expression += last_line[0:end_pos - 1] - # Only apply checks if we got all parts of the boolean expression - if not (lhs and operator and rhs): - return + # Parse expression so that we can take parentheses into account. + # This avoids false positives for inputs like "CHECK((a < 4) == b)", + # which is not replaceable by CHECK_LE. + lhs = '' + rhs = '' + operator = None + while expression: + matched = Match( + r'^\s*(<<|<<=|>>|>>=|->\*|->|&&|\|\||' + r'==|!=|>=|>|<=|<|\()(.*)$', expression) + if matched: + token = matched.group(1) + if token == '(': + # Parenthesized operand + expression = matched.group(2) + (end, _) = FindEndOfExpressionInLine(expression, 0, ['(']) + if end < 0: + return # Unmatched parenthesis + lhs += '(' + expression[0:end] + expression = expression[end:] + elif token in ('&&', '||'): + # Logical and/or operators. This means the expression + # contains more than one term, for example: + # CHECK(42 < a && a < b); + # + # These are not replaceable with CHECK_LE, so bail out early. + return + elif token in ('<<', '<<=', '>>', '>>=', '->*', '->'): + # Non-relational operator + lhs += token + expression = matched.group(2) + else: + # Relational operator + operator = token + rhs = matched.group(2) + break + else: + # Unparenthesized operand. Instead of appending to lhs one + # character at a time, we do another regular expression match to + # consume several characters at once if possible. Trivial benchmark + # shows that this is more efficient when the operands are longer + # than a single character, which is generally the case. + matched = Match(r'^([^-=!<>()&|]+)(.*)$', expression) + if not matched: + matched = Match(r'^(\s*\S)(.*)$', expression) + if not matched: + break + lhs += matched.group(1) + expression = matched.group(2) - # Check that rhs do not contain logical operators. We already know - # that lhs is fine since the loop above parses out && and ||. - if rhs.find('&&') > -1 or rhs.find('||') > -1: - return + # Only apply checks if we got all parts of the boolean expression + if not (lhs and operator and rhs): + return - # At least one of the operands must be a constant literal. This is - # to avoid suggesting replacements for unprintable things like - # CHECK(variable != iterator) - # - # The following pattern matches decimal, hex integers, strings, and - # characters (in that order). - lhs = lhs.strip() - rhs = rhs.strip() - match_constant = r'^([-+]?(\d+|0[xX][0-9a-fA-F]+)[lLuU]{0,3}|".*"|\'.*\')$' - if Match(match_constant, lhs) or Match(match_constant, rhs): - # Note: since we know both lhs and rhs, we can provide a more - # descriptive error message like: - # Consider using CHECK_EQ(x, 42) instead of CHECK(x == 42) - # Instead of: - # Consider using CHECK_EQ instead of CHECK(a == b) + # Check that rhs do not contain logical operators. We already know + # that lhs is fine since the loop above parses out && and ||. + if rhs.find('&&') > -1 or rhs.find('||') > -1: + return + + # At least one of the operands must be a constant literal. This is + # to avoid suggesting replacements for unprintable things like + # CHECK(variable != iterator) # - # We are still keeping the less descriptive message because if lhs - # or rhs gets long, the error message might become unreadable. - error(filename, linenum, 'readability/check', 2, - 'Consider using %s instead of %s(a %s b)' % ( - _CHECK_REPLACEMENT[check_macro][operator], - check_macro, operator)) + # The following pattern matches decimal, hex integers, strings, and + # characters (in that order). + lhs = lhs.strip() + rhs = rhs.strip() + match_constant = r'^([-+]?(\d+|0[xX][0-9a-fA-F]+)[lLuU]{0,3}|".*"|\'.*\')$' + if Match(match_constant, lhs) or Match(match_constant, rhs): + # Note: since we know both lhs and rhs, we can provide a more + # descriptive error message like: + # Consider using CHECK_EQ(x, 42) instead of CHECK(x == 42) + # Instead of: + # Consider using CHECK_EQ instead of CHECK(a == b) + # + # We are still keeping the less descriptive message because if lhs + # or rhs gets long, the error message might become unreadable. + error( + filename, linenum, 'readability/check', 2, + 'Consider using %s instead of %s(a %s b)' % + (_CHECK_REPLACEMENT[check_macro][operator], check_macro, operator)) def CheckAltTokens(filename, clean_lines, linenum, error): - """Check alternative keywords being used in boolean expressions. + """Check alternative keywords being used in boolean expressions. Args: filename: The name of the current file. @@ -4443,32 +4525,33 @@ def CheckAltTokens(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] + line = clean_lines.elided[linenum] - # Avoid preprocessor lines - if Match(r'^\s*#', line): - return + # Avoid preprocessor lines + if Match(r'^\s*#', line): + return - # Last ditch effort to avoid multi-line comments. This will not help - # if the comment started before the current line or ended after the - # current line, but it catches most of the false positives. At least, - # it provides a way to workaround this warning for people who use - # multi-line comments in preprocessor macros. - # - # TODO(unknown): remove this once cpplint has better support for - # multi-line comments. - if line.find('/*') >= 0 or line.find('*/') >= 0: - return + # Last ditch effort to avoid multi-line comments. This will not help + # if the comment started before the current line or ended after the + # current line, but it catches most of the false positives. At least, + # it provides a way to workaround this warning for people who use + # multi-line comments in preprocessor macros. + # + # TODO(unknown): remove this once cpplint has better support for + # multi-line comments. + if line.find('/*') >= 0 or line.find('*/') >= 0: + return - for match in _ALT_TOKEN_REPLACEMENT_PATTERN.finditer(line): - error(filename, linenum, 'readability/alt_tokens', 2, - 'Use operator %s instead of %s' % ( - _ALT_TOKEN_REPLACEMENT[match.group(1)], match.group(1))) + for match in _ALT_TOKEN_REPLACEMENT_PATTERN.finditer(line): + error( + filename, linenum, 'readability/alt_tokens', 2, + 'Use operator %s instead of %s' % + (_ALT_TOKEN_REPLACEMENT[match.group(1)], match.group(1))) def CheckStyle(filename, clean_lines, linenum, file_extension, nesting_state, error): - """Checks rules from the 'C++ style rules' section of cppguide.html. + """Checks rules from the 'C++ style rules' section of cppguide.html. Most of these rules are hard to test (naming, comment style), but we do what we can. In particular we check for 2-space indents, line lengths, @@ -4484,104 +4567,105 @@ def CheckStyle(filename, clean_lines, linenum, file_extension, nesting_state, error: The function to call with any errors found. """ - # Don't use "elided" lines here, otherwise we can't check commented lines. - # Don't want to use "raw" either, because we don't want to check inside C++11 - # raw strings, - raw_lines = clean_lines.lines_without_raw_strings - line = raw_lines[linenum] - prev = raw_lines[linenum - 1] if linenum > 0 else '' + # Don't use "elided" lines here, otherwise we can't check commented lines. + # Don't want to use "raw" either, because we don't want to check inside + # C++11 raw strings, + raw_lines = clean_lines.lines_without_raw_strings + line = raw_lines[linenum] + prev = raw_lines[linenum - 1] if linenum > 0 else '' - if line.find('\t') != -1: - error(filename, linenum, 'whitespace/tab', 1, - 'Tab found; better to use spaces') + if line.find('\t') != -1: + error(filename, linenum, 'whitespace/tab', 1, + 'Tab found; better to use spaces') - # One or three blank spaces at the beginning of the line is weird; it's - # hard to reconcile that with 2-space indents. - # NOTE: here are the conditions rob pike used for his tests. Mine aren't - # as sophisticated, but it may be worth becoming so: RLENGTH==initial_spaces - # if(RLENGTH > 20) complain = 0; - # if(match($0, " +(error|private|public|protected):")) complain = 0; - # if(match(prev, "&& *$")) complain = 0; - # if(match(prev, "\\|\\| *$")) complain = 0; - # if(match(prev, "[\",=><] *$")) complain = 0; - # if(match($0, " <<")) complain = 0; - # if(match(prev, " +for \\(")) complain = 0; - # if(prevodd && match(prevprev, " +for \\(")) complain = 0; - scope_or_label_pattern = r'\s*\w+\s*:\s*\\?$' - classinfo = nesting_state.InnermostClass() - initial_spaces = 0 - cleansed_line = clean_lines.elided[linenum] - while initial_spaces < len(line) and line[initial_spaces] == ' ': - initial_spaces += 1 - # There are certain situations we allow one space, notably for - # section labels, and also lines containing multi-line raw strings. - # We also don't check for lines that look like continuation lines - # (of lines ending in double quotes, commas, equals, or angle brackets) - # because the rules for how to indent those are non-trivial. - if (not Search(r'[",=><] *$', prev) and - (initial_spaces == 1 or initial_spaces == 3) and - not Match(scope_or_label_pattern, cleansed_line) and - not (clean_lines.raw_lines[linenum] != line and - Match(r'^\s*""', line))): - error(filename, linenum, 'whitespace/indent', 3, - 'Weird number of spaces at line-start. ' - 'Are you using a 2-space indent?') + # One or three blank spaces at the beginning of the line is weird; it's + # hard to reconcile that with 2-space indents. + # NOTE: here are the conditions rob pike used for his tests. Mine aren't + # as sophisticated, but it may be worth becoming so: + # RLENGTH==initial_spaces if(RLENGTH > 20) complain = 0; if(match($0, " + # +(error|private|public|protected):")) complain = 0; if(match(prev, "&& + # *$")) complain = 0; if(match(prev, "\\|\\| *$")) complain = 0; + # if(match(prev, "[\",=><] *$")) complain = 0; if(match($0, " <<")) complain + # = 0; if(match(prev, " +for \\(")) complain = 0; if(prevodd && + # match(prevprev, " +for \\(")) complain = 0; + scope_or_label_pattern = r'\s*\w+\s*:\s*\\?$' + classinfo = nesting_state.InnermostClass() + initial_spaces = 0 + cleansed_line = clean_lines.elided[linenum] + while initial_spaces < len(line) and line[initial_spaces] == ' ': + initial_spaces += 1 + # There are certain situations we allow one space, notably for + # section labels, and also lines containing multi-line raw strings. + # We also don't check for lines that look like continuation lines + # (of lines ending in double quotes, commas, equals, or angle brackets) + # because the rules for how to indent those are non-trivial. + if (not Search(r'[",=><] *$', prev) + and (initial_spaces == 1 or initial_spaces == 3) + and not Match(scope_or_label_pattern, cleansed_line) + and not (clean_lines.raw_lines[linenum] != line + and Match(r'^\s*""', line))): + error( + filename, linenum, 'whitespace/indent', 3, + 'Weird number of spaces at line-start. ' + 'Are you using a 2-space indent?') - if line and line[-1].isspace(): - error(filename, linenum, 'whitespace/end_of_line', 4, - 'Line ends in whitespace. Consider deleting these extra spaces.') + if line and line[-1].isspace(): + error( + filename, linenum, 'whitespace/end_of_line', 4, + 'Line ends in whitespace. Consider deleting these extra spaces.') - # Check if the line is a header guard. - is_header_guard = False - if file_extension == 'h': - cppvar = GetHeaderGuardCPPVariable(filename) - if (line.startswith('#ifndef %s' % cppvar) or - line.startswith('#define %s' % cppvar) or - line.startswith('#endif // %s' % cppvar)): - is_header_guard = True - # #include lines and header guards can be long, since there's no clean way to - # split them. - # - # URLs can be long too. It's possible to split these, but it makes them - # harder to cut&paste. - # - # The "$Id:...$" comment may also get very long without it being the - # developers fault. - if (not line.startswith('#include') and not is_header_guard and - not Match(r'^\s*//.*http(s?)://\S*$', line) and - not Match(r'^\s*//\s*[^\s]*$', line) and - not Match(r'^// \$Id:.*#[0-9]+ \$$', line)): - if len(line) > _line_length: - error(filename, linenum, 'whitespace/line_length', 2, - 'Lines should be <= %i characters long' % _line_length) + # Check if the line is a header guard. + is_header_guard = False + if file_extension == 'h': + cppvar = GetHeaderGuardCPPVariable(filename) + if (line.startswith('#ifndef %s' % cppvar) + or line.startswith('#define %s' % cppvar) + or line.startswith('#endif // %s' % cppvar)): + is_header_guard = True + # #include lines and header guards can be long, since there's no clean way + # to split them. + # + # URLs can be long too. It's possible to split these, but it makes them + # harder to cut&paste. + # + # The "$Id:...$" comment may also get very long without it being the + # developers fault. + if (not line.startswith('#include') and not is_header_guard + and not Match(r'^\s*//.*http(s?)://\S*$', line) + and not Match(r'^\s*//\s*[^\s]*$', line) + and not Match(r'^// \$Id:.*#[0-9]+ \$$', line)): + if len(line) > _line_length: + error(filename, linenum, 'whitespace/line_length', 2, + 'Lines should be <= %i characters long' % _line_length) - if (cleansed_line.count(';') > 1 and - # for loops are allowed two ;'s (and may run over two lines). - cleansed_line.find('for') == -1 and - (GetPreviousNonBlankLine(clean_lines, linenum)[0].find('for') == -1 or - GetPreviousNonBlankLine(clean_lines, linenum)[0].find(';') != -1) and - # It's ok to have many commands in a switch case that fits in 1 line - not ((cleansed_line.find('case ') != -1 or - cleansed_line.find('default:') != -1) and - cleansed_line.find('break;') != -1)): - error(filename, linenum, 'whitespace/newline', 0, - 'More than one command on the same line') + if (cleansed_line.count(';') > 1 and + # for loops are allowed two ;'s (and may run over two lines). + cleansed_line.find('for') == -1 and + (GetPreviousNonBlankLine(clean_lines, linenum)[0].find('for') == -1 + or GetPreviousNonBlankLine(clean_lines, linenum)[0].find(';') != -1) + and + # It's ok to have many commands in a switch case that fits in 1 line + not ((cleansed_line.find('case ') != -1 + or cleansed_line.find('default:') != -1) + and cleansed_line.find('break;') != -1)): + error(filename, linenum, 'whitespace/newline', 0, + 'More than one command on the same line') - # Some more style checks - CheckBraces(filename, clean_lines, linenum, error) - CheckTrailingSemicolon(filename, clean_lines, linenum, error) - CheckEmptyBlockBody(filename, clean_lines, linenum, error) - CheckSpacing(filename, clean_lines, linenum, nesting_state, error) - CheckOperatorSpacing(filename, clean_lines, linenum, error) - CheckParenthesisSpacing(filename, clean_lines, linenum, error) - CheckCommaSpacing(filename, clean_lines, linenum, error) - CheckBracesSpacing(filename, clean_lines, linenum, nesting_state, error) - CheckSpacingForFunctionCall(filename, clean_lines, linenum, error) - CheckCheck(filename, clean_lines, linenum, error) - CheckAltTokens(filename, clean_lines, linenum, error) - classinfo = nesting_state.InnermostClass() - if classinfo: - CheckSectionSpacing(filename, clean_lines, classinfo, linenum, error) + # Some more style checks + CheckBraces(filename, clean_lines, linenum, error) + CheckTrailingSemicolon(filename, clean_lines, linenum, error) + CheckEmptyBlockBody(filename, clean_lines, linenum, error) + CheckSpacing(filename, clean_lines, linenum, nesting_state, error) + CheckOperatorSpacing(filename, clean_lines, linenum, error) + CheckParenthesisSpacing(filename, clean_lines, linenum, error) + CheckCommaSpacing(filename, clean_lines, linenum, error) + CheckBracesSpacing(filename, clean_lines, linenum, nesting_state, error) + CheckSpacingForFunctionCall(filename, clean_lines, linenum, error) + CheckCheck(filename, clean_lines, linenum, error) + CheckAltTokens(filename, clean_lines, linenum, error) + classinfo = nesting_state.InnermostClass() + if classinfo: + CheckSectionSpacing(filename, clean_lines, classinfo, linenum, error) _RE_PATTERN_INCLUDE = re.compile(r'^\s*#\s*include\s*([<"])([^>"]*)[>"].*$') @@ -4594,7 +4678,7 @@ _RE_FIRST_COMPONENT = re.compile(r'^[^-_.]+') def _DropCommonSuffixes(filename): - """Drops common suffixes like _test.cc or -inl.h from filename. + """Drops common suffixes like _test.cc or -inl.h from filename. For example: >>> _DropCommonSuffixes('foo/foo-inl.h') @@ -4612,16 +4696,16 @@ def _DropCommonSuffixes(filename): Returns: The filename with the common suffix removed. """ - for suffix in ('test.cc', 'regtest.cc', 'unittest.cc', - 'inl.h', 'impl.h', 'internal.h'): - if (filename.endswith(suffix) and len(filename) > len(suffix) and - filename[-len(suffix) - 1] in ('-', '_')): - return filename[:-len(suffix) - 1] - return os.path.splitext(filename)[0] + for suffix in ('test.cc', 'regtest.cc', 'unittest.cc', 'inl.h', 'impl.h', + 'internal.h'): + if (filename.endswith(suffix) and len(filename) > len(suffix) + and filename[-len(suffix) - 1] in ('-', '_')): + return filename[:-len(suffix) - 1] + return os.path.splitext(filename)[0] def _ClassifyInclude(fileinfo, include, is_system): - """Figures out what kind of header 'include' is. + """Figures out what kind of header 'include' is. Args: fileinfo: The current file cpplint is running over. A FileInfo instance. @@ -4644,44 +4728,43 @@ def _ClassifyInclude(fileinfo, include, is_system): >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'foo/bar.h', False) _OTHER_HEADER """ - # This is a list of all standard c++ header files, except - # those already checked for above. - is_cpp_h = include in _CPP_HEADERS + # This is a list of all standard c++ header files, except + # those already checked for above. + is_cpp_h = include in _CPP_HEADERS - if is_system: - if is_cpp_h: - return _CPP_SYS_HEADER - else: - return _C_SYS_HEADER + if is_system: + if is_cpp_h: + return _CPP_SYS_HEADER + else: + return _C_SYS_HEADER - # If the target file and the include we're checking share a - # basename when we drop common extensions, and the include - # lives in . , then it's likely to be owned by the target file. - target_dir, target_base = ( - os.path.split(_DropCommonSuffixes(fileinfo.RepositoryName()))) - include_dir, include_base = os.path.split(_DropCommonSuffixes(include)) - if target_base == include_base and ( - include_dir == target_dir or - include_dir == os.path.normpath(target_dir + '/../public')): - return _LIKELY_MY_HEADER + # If the target file and the include we're checking share a + # basename when we drop common extensions, and the include + # lives in . , then it's likely to be owned by the target file. + target_dir, target_base = (os.path.split( + _DropCommonSuffixes(fileinfo.RepositoryName()))) + include_dir, include_base = os.path.split(_DropCommonSuffixes(include)) + if target_base == include_base and ( + include_dir == target_dir + or include_dir == os.path.normpath(target_dir + '/../public')): + return _LIKELY_MY_HEADER - # If the target and include share some initial basename - # component, it's possible the target is implementing the - # include, so it's allowed to be first, but we'll never - # complain if it's not there. - target_first_component = _RE_FIRST_COMPONENT.match(target_base) - include_first_component = _RE_FIRST_COMPONENT.match(include_base) - if (target_first_component and include_first_component and - target_first_component.group(0) == - include_first_component.group(0)): - return _POSSIBLE_MY_HEADER - - return _OTHER_HEADER + # If the target and include share some initial basename + # component, it's possible the target is implementing the + # include, so it's allowed to be first, but we'll never + # complain if it's not there. + target_first_component = _RE_FIRST_COMPONENT.match(target_base) + include_first_component = _RE_FIRST_COMPONENT.match(include_base) + if (target_first_component and include_first_component + and target_first_component.group(0) + == include_first_component.group(0)): + return _POSSIBLE_MY_HEADER + return _OTHER_HEADER def CheckIncludeLine(filename, clean_lines, linenum, include_state, error): - """Check rules that are applicable to #include lines. + """Check rules that are applicable to #include lines. Strings on #include lines are NOT removed from elided line, to make certain tasks easier. However, to prevent false positives, checks @@ -4694,68 +4777,70 @@ def CheckIncludeLine(filename, clean_lines, linenum, include_state, error): include_state: An _IncludeState instance in which the headers are inserted. error: The function to call with any errors found. """ - fileinfo = FileInfo(filename) - line = clean_lines.lines[linenum] + fileinfo = FileInfo(filename) + line = clean_lines.lines[linenum] - # "include" should use the new style "foo/bar.h" instead of just "bar.h" - # Only do this check if the included header follows google naming - # conventions. If not, assume that it's a 3rd party API that - # requires special include conventions. - # - # We also make an exception for Lua headers, which follow google - # naming convention but not the include convention. - match = Match(r'#include\s*"([^/]+\.h)"', line) - if match and not _THIRD_PARTY_HEADERS_PATTERN.match(match.group(1)): - error(filename, linenum, 'build/include_directory', 4, - 'Include the directory when naming .h files') + # "include" should use the new style "foo/bar.h" instead of just "bar.h" + # Only do this check if the included header follows google naming + # conventions. If not, assume that it's a 3rd party API that + # requires special include conventions. + # + # We also make an exception for Lua headers, which follow google + # naming convention but not the include convention. + match = Match(r'#include\s*"([^/]+\.h)"', line) + if match and not _THIRD_PARTY_HEADERS_PATTERN.match(match.group(1)): + error(filename, linenum, 'build/include_directory', 4, + 'Include the directory when naming .h files') - # we shouldn't include a file more than once. actually, there are a - # handful of instances where doing so is okay, but in general it's - # not. - match = _RE_PATTERN_INCLUDE.search(line) - if match: - include = match.group(2) - is_system = (match.group(1) == '<') - duplicate_line = include_state.FindHeader(include) - if duplicate_line >= 0: - error(filename, linenum, 'build/include', 4, - '"%s" already included at %s:%s' % - (include, filename, duplicate_line)) - elif (include.endswith('.cc') and - os.path.dirname(fileinfo.RepositoryName()) != os.path.dirname(include)): - error(filename, linenum, 'build/include', 4, - 'Do not include .cc files from other packages') - elif not _THIRD_PARTY_HEADERS_PATTERN.match(include): - include_state.include_list[-1].append((include, linenum)) - - # We want to ensure that headers appear in the right order: - # 1) for foo.cc, foo.h (preferred location) - # 2) c system files - # 3) cpp system files - # 4) for foo.cc, foo.h (deprecated location) - # 5) other google headers - # - # We classify each include statement as one of those 5 types - # using a number of techniques. The include_state object keeps - # track of the highest type seen, and complains if we see a - # lower type after that. - error_message = include_state.CheckNextIncludeOrder( - _ClassifyInclude(fileinfo, include, is_system)) - if error_message: - error(filename, linenum, 'build/include_order', 4, - '%s. Should be: %s.h, c system, c++ system, other.' % - (error_message, fileinfo.BaseName())) - canonical_include = include_state.CanonicalizeAlphabeticalOrder(include) - if not include_state.IsInAlphabeticalOrder( - clean_lines, linenum, canonical_include): - error(filename, linenum, 'build/include_alpha', 4, - 'Include "%s" not in alphabetical order' % include) - include_state.SetLastHeader(canonical_include) + # we shouldn't include a file more than once. actually, there are a + # handful of instances where doing so is okay, but in general it's + # not. + match = _RE_PATTERN_INCLUDE.search(line) + if match: + include = match.group(2) + is_system = (match.group(1) == '<') + duplicate_line = include_state.FindHeader(include) + if duplicate_line >= 0: + error( + filename, linenum, 'build/include', 4, + '"%s" already included at %s:%s' % + (include, filename, duplicate_line)) + elif (include.endswith('.cc') and os.path.dirname( + fileinfo.RepositoryName()) != os.path.dirname(include)): + error(filename, linenum, 'build/include', 4, + 'Do not include .cc files from other packages') + elif not _THIRD_PARTY_HEADERS_PATTERN.match(include): + include_state.include_list[-1].append((include, linenum)) + # We want to ensure that headers appear in the right order: + # 1) for foo.cc, foo.h (preferred location) + # 2) c system files + # 3) cpp system files + # 4) for foo.cc, foo.h (deprecated location) + # 5) other google headers + # + # We classify each include statement as one of those 5 types + # using a number of techniques. The include_state object keeps + # track of the highest type seen, and complains if we see a + # lower type after that. + error_message = include_state.CheckNextIncludeOrder( + _ClassifyInclude(fileinfo, include, is_system)) + if error_message: + error( + filename, linenum, 'build/include_order', 4, + '%s. Should be: %s.h, c system, c++ system, other.' % + (error_message, fileinfo.BaseName())) + canonical_include = include_state.CanonicalizeAlphabeticalOrder( + include) + if not include_state.IsInAlphabeticalOrder(clean_lines, linenum, + canonical_include): + error(filename, linenum, 'build/include_alpha', 4, + 'Include "%s" not in alphabetical order' % include) + include_state.SetLastHeader(canonical_include) def _GetTextInside(text, start_pattern): - r"""Retrieves all the text between matching open and close parentheses. + r"""Retrieves all the text between matching open and close parentheses. Given a string of lines and a regular expression string, retrieve all the text following the expression and between opening punctuation symbols like @@ -4774,40 +4859,40 @@ def _GetTextInside(text, start_pattern): The extracted text. None if either the opening string or ending punctuation could not be found. """ - # TODO(unknown): Audit cpplint.py to see what places could be profitably - # rewritten to use _GetTextInside (and use inferior regexp matching today). + # TODO(unknown): Audit cpplint.py to see what places could be profitably + # rewritten to use _GetTextInside (and use inferior regexp matching today). - # Give opening punctuations to get the matching close-punctuations. - matching_punctuation = {'(': ')', '{': '}', '[': ']'} - closing_punctuation = set(matching_punctuation.values()) + # Give opening punctuations to get the matching close-punctuations. + matching_punctuation = {'(': ')', '{': '}', '[': ']'} + closing_punctuation = set(matching_punctuation.values()) - # Find the position to start extracting text. - match = re.search(start_pattern, text, re.M) - if not match: # start_pattern not found in text. - return None - start_position = match.end(0) + # Find the position to start extracting text. + match = re.search(start_pattern, text, re.M) + if not match: # start_pattern not found in text. + return None + start_position = match.end(0) - assert start_position > 0, ( - 'start_pattern must ends with an opening punctuation.') - assert text[start_position - 1] in matching_punctuation, ( - 'start_pattern must ends with an opening punctuation.') - # Stack of closing punctuations we expect to have in text after position. - punctuation_stack = [matching_punctuation[text[start_position - 1]]] - position = start_position - while punctuation_stack and position < len(text): - if text[position] == punctuation_stack[-1]: - punctuation_stack.pop() - elif text[position] in closing_punctuation: - # A closing punctuation without matching opening punctuations. - return None - elif text[position] in matching_punctuation: - punctuation_stack.append(matching_punctuation[text[position]]) - position += 1 - if punctuation_stack: - # Opening punctuations left without matching close-punctuations. - return None - # punctuations match. - return text[start_position:position - 1] + assert start_position > 0, ( + 'start_pattern must ends with an opening punctuation.') + assert text[start_position - 1] in matching_punctuation, ( + 'start_pattern must ends with an opening punctuation.') + # Stack of closing punctuations we expect to have in text after position. + punctuation_stack = [matching_punctuation[text[start_position - 1]]] + position = start_position + while punctuation_stack and position < len(text): + if text[position] == punctuation_stack[-1]: + punctuation_stack.pop() + elif text[position] in closing_punctuation: + # A closing punctuation without matching opening punctuations. + return None + elif text[position] in matching_punctuation: + punctuation_stack.append(matching_punctuation[text[position]]) + position += 1 + if punctuation_stack: + # Opening punctuations left without matching close-punctuations. + return None + # punctuations match. + return text[start_position:position - 1] # Patterns for matching call-by-reference parameters. @@ -4826,22 +4911,23 @@ _RE_PATTERN_TYPE = ( r'\s*<(?:<(?:<[^<>]*>|[^<>])*>|[^<>])*>|' r'::)+') # A call-by-reference parameter ends with '& identifier'. -_RE_PATTERN_REF_PARAM = re.compile( - r'(' + _RE_PATTERN_TYPE + r'(?:\s*(?:\bconst\b|[*]))*\s*' - r'&\s*' + _RE_PATTERN_IDENT + r')\s*(?:=[^,()]+)?[,)]') +_RE_PATTERN_REF_PARAM = re.compile(r'(' + _RE_PATTERN_TYPE + + r'(?:\s*(?:\bconst\b|[*]))*\s*' + r'&\s*' + _RE_PATTERN_IDENT + + r')\s*(?:=[^,()]+)?[,)]') # A call-by-const-reference parameter either ends with 'const& identifier' # or looks like 'const type& identifier' when 'type' is atomic. -_RE_PATTERN_CONST_REF_PARAM = ( - r'(?:.*\s*\bconst\s*&\s*' + _RE_PATTERN_IDENT + - r'|const\s+' + _RE_PATTERN_TYPE + r'\s*&\s*' + _RE_PATTERN_IDENT + r')') +_RE_PATTERN_CONST_REF_PARAM = (r'(?:.*\s*\bconst\s*&\s*' + _RE_PATTERN_IDENT + + r'|const\s+' + _RE_PATTERN_TYPE + r'\s*&\s*' + + _RE_PATTERN_IDENT + r')') # Stream types. -_RE_PATTERN_REF_STREAM_PARAM = ( - r'(?:.*stream\s*&\s*' + _RE_PATTERN_IDENT + r')') +_RE_PATTERN_REF_STREAM_PARAM = (r'(?:.*stream\s*&\s*' + _RE_PATTERN_IDENT + + r')') -def CheckLanguage(filename, clean_lines, linenum, file_extension, - include_state, nesting_state, error): - """Checks rules from the 'C++ language rules' section of cppguide.html. +def CheckLanguage(filename, clean_lines, linenum, file_extension, include_state, + nesting_state, error): + """Checks rules from the 'C++ language rules' section of cppguide.html. Some of these rules are hard to test (function overloading, using uint32 inappropriately), but we do the best we can. @@ -4856,152 +4942,160 @@ def CheckLanguage(filename, clean_lines, linenum, file_extension, the current stack of nested blocks being parsed. error: The function to call with any errors found. """ - # If the line is empty or consists of entirely a comment, no need to - # check it. - line = clean_lines.elided[linenum] - if not line: - return + # If the line is empty or consists of entirely a comment, no need to + # check it. + line = clean_lines.elided[linenum] + if not line: + return - match = _RE_PATTERN_INCLUDE.search(line) - if match: - CheckIncludeLine(filename, clean_lines, linenum, include_state, error) - return - - # Reset include state across preprocessor directives. This is meant - # to silence warnings for conditional includes. - match = Match(r'^\s*#\s*(if|ifdef|ifndef|elif|else|endif)\b', line) - if match: - include_state.ResetSection(match.group(1)) - - # Make Windows paths like Unix. - fullname = os.path.abspath(filename).replace('\\', '/') - - # Perform other checks now that we are sure that this is not an include line - CheckCasts(filename, clean_lines, linenum, error) - CheckGlobalStatic(filename, clean_lines, linenum, error) - CheckPrintf(filename, clean_lines, linenum, error) - - if file_extension == 'h': - # TODO(unknown): check that 1-arg constructors are explicit. - # How to tell it's a constructor? - # (handled in CheckForNonStandardConstructs for now) - # TODO(unknown): check that classes declare or disable copy/assign - # (level 1 error) - pass - - # Check if people are using the verboten C basic types. The only exception - # we regularly allow is "unsigned short port" for port. - if Search(r'\bshort port\b', line): - if not Search(r'\bunsigned short port\b', line): - error(filename, linenum, 'runtime/int', 4, - 'Use "unsigned short" for ports, not "short"') - else: - match = Search(r'\b(short|long(?! +double)|long long)\b', line) + match = _RE_PATTERN_INCLUDE.search(line) if match: - error(filename, linenum, 'runtime/int', 4, - 'Use int16/int64/etc, rather than the C type %s' % match.group(1)) + CheckIncludeLine(filename, clean_lines, linenum, include_state, error) + return - # Check if some verboten operator overloading is going on - # TODO(unknown): catch out-of-line unary operator&: - # class X {}; - # int operator&(const X& x) { return 42; } // unary operator& - # The trick is it's hard to tell apart from binary operator&: - # class Y { int operator&(const Y& x) { return 23; } }; // binary operator& - if Search(r'\boperator\s*&\s*\(\s*\)', line): - error(filename, linenum, 'runtime/operator', 4, - 'Unary operator& is dangerous. Do not use it.') + # Reset include state across preprocessor directives. This is meant + # to silence warnings for conditional includes. + match = Match(r'^\s*#\s*(if|ifdef|ifndef|elif|else|endif)\b', line) + if match: + include_state.ResetSection(match.group(1)) - # Check for suspicious usage of "if" like - # } if (a == b) { - if Search(r'\}\s*if\s*(?:constexpr\s*)?\(', line): - error(filename, linenum, 'readability/braces', 4, - 'Did you mean "else if"? If not, start a new line for "if".') + # Make Windows paths like Unix. + fullname = os.path.abspath(filename).replace('\\', '/') - # Check for potential format string bugs like printf(foo). - # We constrain the pattern not to pick things like DocidForPrintf(foo). - # Not perfect but it can catch printf(foo.c_str()) and printf(foo->c_str()) - # TODO(unknown): Catch the following case. Need to change the calling - # convention of the whole function to process multiple line to handle it. - # printf( - # boy_this_is_a_really_long_variable_that_cannot_fit_on_the_prev_line); - printf_args = _GetTextInside(line, r'(?i)\b(string)?printf\s*\(') - if printf_args: - match = Match(r'([\w.\->()]+)$', printf_args) - if match and match.group(1) != '__VA_ARGS__': - function_name = re.search(r'\b((?:string)?printf)\s*\(', - line, re.I).group(1) - error(filename, linenum, 'runtime/printf', 4, - 'Potential format string bug. Do %s("%%s", %s) instead.' - % (function_name, match.group(1))) + # Perform other checks now that we are sure that this is not an include line + CheckCasts(filename, clean_lines, linenum, error) + CheckGlobalStatic(filename, clean_lines, linenum, error) + CheckPrintf(filename, clean_lines, linenum, error) - # Check for potential memset bugs like memset(buf, sizeof(buf), 0). - match = Search(r'memset\s*\(([^,]*),\s*([^,]*),\s*0\s*\)', line) - if match and not Match(r"^''|-?[0-9]+|0x[0-9A-Fa-f]$", match.group(2)): - error(filename, linenum, 'runtime/memset', 4, - 'Did you mean "memset(%s, 0, %s)"?' - % (match.group(1), match.group(2))) + if file_extension == 'h': + # TODO(unknown): check that 1-arg constructors are explicit. + # How to tell it's a constructor? + # (handled in CheckForNonStandardConstructs for now) + # TODO(unknown): check that classes declare or disable copy/assign + # (level 1 error) + pass - if Search(r'\busing namespace\b', line): - error(filename, linenum, 'build/namespaces', 5, - 'Do not use namespace using-directives. ' - 'Use using-declarations instead.') + # Check if people are using the verboten C basic types. The only exception + # we regularly allow is "unsigned short port" for port. + if Search(r'\bshort port\b', line): + if not Search(r'\bunsigned short port\b', line): + error(filename, linenum, 'runtime/int', 4, + 'Use "unsigned short" for ports, not "short"') + else: + match = Search(r'\b(short|long(?! +double)|long long)\b', line) + if match: + error( + filename, linenum, 'runtime/int', 4, + 'Use int16/int64/etc, rather than the C type %s' % + match.group(1)) - # Detect variable-length arrays. - match = Match(r'\s*(.+::)?(\w+) [a-z]\w*\[(.+)];', line) - if (match and match.group(2) != 'return' and match.group(2) != 'delete' and - match.group(3).find(']') == -1): - # Split the size using space and arithmetic operators as delimiters. - # If any of the resulting tokens are not compile time constants then - # report the error. - tokens = re.split(r'\s|\+|\-|\*|\/|<<|>>]', match.group(3)) - is_const = True - skip_next = False - for tok in tokens: - if skip_next: + # Check if some verboten operator overloading is going on + # TODO(unknown): catch out-of-line unary operator&: + # class X {}; + # int operator&(const X& x) { return 42; } // unary operator& + # The trick is it's hard to tell apart from binary operator&: + # class Y { int operator&(const Y& x) { return 23; } }; // binary operator& + if Search(r'\boperator\s*&\s*\(\s*\)', line): + error(filename, linenum, 'runtime/operator', 4, + 'Unary operator& is dangerous. Do not use it.') + + # Check for suspicious usage of "if" like + # } if (a == b) { + if Search(r'\}\s*if\s*(?:constexpr\s*)?\(', line): + error(filename, linenum, 'readability/braces', 4, + 'Did you mean "else if"? If not, start a new line for "if".') + + # Check for potential format string bugs like printf(foo). + # We constrain the pattern not to pick things like DocidForPrintf(foo). + # Not perfect but it can catch printf(foo.c_str()) and printf(foo->c_str()) + # TODO(unknown): Catch the following case. Need to change the calling + # convention of the whole function to process multiple line to handle it. + # printf( + # boy_this_is_a_really_long_variable_that_cannot_fit_on_the_prev_line); + printf_args = _GetTextInside(line, r'(?i)\b(string)?printf\s*\(') + if printf_args: + match = Match(r'([\w.\->()]+)$', printf_args) + if match and match.group(1) != '__VA_ARGS__': + function_name = re.search(r'\b((?:string)?printf)\s*\(', line, + re.I).group(1) + error( + filename, linenum, 'runtime/printf', 4, + 'Potential format string bug. Do %s("%%s", %s) instead.' % + (function_name, match.group(1))) + + # Check for potential memset bugs like memset(buf, sizeof(buf), 0). + match = Search(r'memset\s*\(([^,]*),\s*([^,]*),\s*0\s*\)', line) + if match and not Match(r"^''|-?[0-9]+|0x[0-9A-Fa-f]$", match.group(2)): + error( + filename, linenum, 'runtime/memset', 4, + 'Did you mean "memset(%s, 0, %s)"?' % + (match.group(1), match.group(2))) + + if Search(r'\busing namespace\b', line): + error( + filename, linenum, 'build/namespaces', 5, + 'Do not use namespace using-directives. ' + 'Use using-declarations instead.') + + # Detect variable-length arrays. + match = Match(r'\s*(.+::)?(\w+) [a-z]\w*\[(.+)];', line) + if (match and match.group(2) != 'return' and match.group(2) != 'delete' + and match.group(3).find(']') == -1): + # Split the size using space and arithmetic operators as delimiters. + # If any of the resulting tokens are not compile time constants then + # report the error. + tokens = re.split(r'\s|\+|\-|\*|\/|<<|>>]', match.group(3)) + is_const = True skip_next = False - continue + for tok in tokens: + if skip_next: + skip_next = False + continue - if Search(r'sizeof\(.+\)', tok): continue - if Search(r'arraysize\(\w+\)', tok): continue - if Search(r'base::size\(.+\)', tok): continue - if Search(r'std::size\(.+\)', tok): continue - if Search(r'std::extent<.+>', tok): continue + if Search(r'sizeof\(.+\)', tok): continue + if Search(r'arraysize\(\w+\)', tok): continue + if Search(r'base::size\(.+\)', tok): continue + if Search(r'std::size\(.+\)', tok): continue + if Search(r'std::extent<.+>', tok): continue - tok = tok.lstrip('(') - tok = tok.rstrip(')') - if not tok: continue - if Match(r'\d+', tok): continue - if Match(r'0[xX][0-9a-fA-F]+', tok): continue - if Match(r'k[A-Z0-9]\w*', tok): continue - if Match(r'(.+::)?k[A-Z0-9]\w*', tok): continue - if Match(r'(.+::)?[A-Z][A-Z0-9_]*', tok): continue - # A catch all for tricky sizeof cases, including 'sizeof expression', - # 'sizeof(*type)', 'sizeof(const type)', 'sizeof(struct StructName)' - # requires skipping the next token because we split on ' ' and '*'. - if tok.startswith('sizeof'): - skip_next = True - continue - is_const = False - break - if not is_const: - error(filename, linenum, 'runtime/arrays', 1, - 'Do not use variable-length arrays. Use an appropriately named ' - "('k' followed by CamelCase) compile-time constant for the size.") + tok = tok.lstrip('(') + tok = tok.rstrip(')') + if not tok: continue + if Match(r'\d+', tok): continue + if Match(r'0[xX][0-9a-fA-F]+', tok): continue + if Match(r'k[A-Z0-9]\w*', tok): continue + if Match(r'(.+::)?k[A-Z0-9]\w*', tok): continue + if Match(r'(.+::)?[A-Z][A-Z0-9_]*', tok): continue + # A catch all for tricky sizeof cases, including 'sizeof + # expression', 'sizeof(*type)', 'sizeof(const type)', 'sizeof(struct + # StructName)' requires skipping the next token because we split on + # ' ' and '*'. + if tok.startswith('sizeof'): + skip_next = True + continue + is_const = False + break + if not is_const: + error( + filename, linenum, 'runtime/arrays', 1, + 'Do not use variable-length arrays. Use an appropriately named ' + "('k' followed by CamelCase) compile-time constant for the size." + ) - # Check for use of unnamed namespaces in header files. Registration - # macros are typically OK, so we allow use of "namespace {" on lines - # that end with backslashes. - if (file_extension == 'h' - and Search(r'\bnamespace\s*{', line) - and line[-1] != '\\'): - error(filename, linenum, 'build/namespaces', 4, - 'Do not use unnamed namespaces in header files. See ' - 'https://google.github.io/styleguide/cppguide.html#Namespaces' - ' for more information.') + # Check for use of unnamed namespaces in header files. Registration + # macros are typically OK, so we allow use of "namespace {" on lines + # that end with backslashes. + if (file_extension == 'h' and Search(r'\bnamespace\s*{', line) + and line[-1] != '\\'): + error( + filename, linenum, 'build/namespaces', 4, + 'Do not use unnamed namespaces in header files. See ' + 'https://google.github.io/styleguide/cppguide.html#Namespaces' + ' for more information.') def CheckGlobalStatic(filename, clean_lines, linenum, error): - """Check for unsafe global or static objects. + """Check for unsafe global or static objects. Args: filename: The name of the current file. @@ -5009,60 +5103,60 @@ def CheckGlobalStatic(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] + line = clean_lines.elided[linenum] - # Match two lines at a time to support multiline declarations - if linenum + 1 < clean_lines.NumLines() and not Search(r'[;({]', line): - line += clean_lines.elided[linenum + 1].strip() + # Match two lines at a time to support multiline declarations + if linenum + 1 < clean_lines.NumLines() and not Search(r'[;({]', line): + line += clean_lines.elided[linenum + 1].strip() - # Check for people declaring static/global STL strings at the top level. - # This is dangerous because the C++ language does not guarantee that - # globals with constructors are initialized before the first access, and - # also because globals can be destroyed when some threads are still running. - # TODO(unknown): Generalize this to also find static unique_ptr instances. - # TODO(unknown): File bugs for clang-tidy to find these. - match = Match( - r'((?:|static +)(?:|const +))(?::*std::)?string( +const)? +' - r'([a-zA-Z0-9_:]+)\b(.*)', - line) + # Check for people declaring static/global STL strings at the top level. + # This is dangerous because the C++ language does not guarantee that + # globals with constructors are initialized before the first access, and + # also because globals can be destroyed when some threads are still running. + # TODO(unknown): Generalize this to also find static unique_ptr instances. + # TODO(unknown): File bugs for clang-tidy to find these. + match = Match( + r'((?:|static +)(?:|const +))(?::*std::)?string( +const)? +' + r'([a-zA-Z0-9_:]+)\b(.*)', line) - # Remove false positives: - # - String pointers (as opposed to values). - # string *pointer - # const string *pointer - # string const *pointer - # string *const pointer - # - # - Functions and template specializations. - # string Function(... - # string Class::Method(... - # - # - Operators. These are matched separately because operator names - # cross non-word boundaries, and trying to match both operators - # and functions at the same time would decrease accuracy of - # matching identifiers. - # string Class::operator*() - if (match and - not Search(r'\bstring\b(\s+const)?\s*[\*\&]\s*(const\s+)?\w', line) and - not Search(r'\boperator\W', line) and - not Match(r'\s*(<.*>)?(::[a-zA-Z0-9_]+)*\s*\(([^"]|$)', match.group(4))): - if Search(r'\bconst\b', line): - error(filename, linenum, 'runtime/string', 4, - 'For a static/global string constant, use a C style string ' - 'instead: "%schar%s %s[]".' % - (match.group(1), match.group(2) or '', match.group(3))) - else: - error(filename, linenum, 'runtime/string', 4, - 'Static/global string variables are not permitted.') + # Remove false positives: + # - String pointers (as opposed to values). + # string *pointer + # const string *pointer + # string const *pointer + # string *const pointer + # + # - Functions and template specializations. + # string Function(... + # string Class::Method(... + # + # - Operators. These are matched separately because operator names + # cross non-word boundaries, and trying to match both operators + # and functions at the same time would decrease accuracy of + # matching identifiers. + # string Class::operator*() + if (match and + not Search(r'\bstring\b(\s+const)?\s*[\*\&]\s*(const\s+)?\w', line) + and not Search(r'\boperator\W', line) and not Match( + r'\s*(<.*>)?(::[a-zA-Z0-9_]+)*\s*\(([^"]|$)', match.group(4))): + if Search(r'\bconst\b', line): + error( + filename, linenum, 'runtime/string', 4, + 'For a static/global string constant, use a C style string ' + 'instead: "%schar%s %s[]".' % + (match.group(1), match.group(2) or '', match.group(3))) + else: + error(filename, linenum, 'runtime/string', 4, + 'Static/global string variables are not permitted.') - if (Search(r'\b([A-Za-z0-9_]*_)\(\1\)', line) or - Search(r'\b([A-Za-z0-9_]*_)\(CHECK_NOTNULL\(\1\)\)', line)): - error(filename, linenum, 'runtime/init', 4, - 'You seem to be initializing a member variable with itself.') + if (Search(r'\b([A-Za-z0-9_]*_)\(\1\)', line) + or Search(r'\b([A-Za-z0-9_]*_)\(CHECK_NOTNULL\(\1\)\)', line)): + error(filename, linenum, 'runtime/init', 4, + 'You seem to be initializing a member variable with itself.') def CheckPrintf(filename, clean_lines, linenum, error): - """Check for printf related issues. + """Check for printf related issues. Args: filename: The name of the current file. @@ -5070,28 +5164,29 @@ def CheckPrintf(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] + line = clean_lines.elided[linenum] - # When snprintf is used, the second argument shouldn't be a literal. - match = Search(r'snprintf\s*\(([^,]*),\s*([0-9]*)\s*,', line) - if match and match.group(2) != '0': - # If 2nd arg is zero, snprintf is used to calculate size. - error(filename, linenum, 'runtime/printf', 3, - 'If you can, use sizeof(%s) instead of %s as the 2nd arg ' - 'to snprintf.' % (match.group(1), match.group(2))) + # When snprintf is used, the second argument shouldn't be a literal. + match = Search(r'snprintf\s*\(([^,]*),\s*([0-9]*)\s*,', line) + if match and match.group(2) != '0': + # If 2nd arg is zero, snprintf is used to calculate size. + error( + filename, linenum, 'runtime/printf', 3, + 'If you can, use sizeof(%s) instead of %s as the 2nd arg ' + 'to snprintf.' % (match.group(1), match.group(2))) - # Check if some verboten C functions are being used. - if Search(r'\bsprintf\s*\(', line): - error(filename, linenum, 'runtime/printf', 5, - 'Never use sprintf. Use snprintf instead.') - match = Search(r'\b(strcpy|strcat)\s*\(', line) - if match: - error(filename, linenum, 'runtime/printf', 4, - 'Almost always, snprintf is better than %s' % match.group(1)) + # Check if some verboten C functions are being used. + if Search(r'\bsprintf\s*\(', line): + error(filename, linenum, 'runtime/printf', 5, + 'Never use sprintf. Use snprintf instead.') + match = Search(r'\b(strcpy|strcat)\s*\(', line) + if match: + error(filename, linenum, 'runtime/printf', 4, + 'Almost always, snprintf is better than %s' % match.group(1)) def IsDerivedFunction(clean_lines, linenum): - """Check if current line contains an inherited function. + """Check if current line contains an inherited function. Args: clean_lines: A CleansedLines instance containing the file. @@ -5100,20 +5195,20 @@ def IsDerivedFunction(clean_lines, linenum): True if current line contains a function with "override" virt-specifier. """ - # Scan back a few lines for start of current function - for i in range(linenum, max(-1, linenum - 10), -1): - match = Match(r'^([^()]*\w+)\(', clean_lines.elided[i]) - if match: - # Look for "override" after the matching closing parenthesis - line, _, closing_paren = CloseExpression( - clean_lines, i, len(match.group(1))) - return (closing_paren >= 0 and - Search(r'\boverride\b', line[closing_paren:])) - return False + # Scan back a few lines for start of current function + for i in range(linenum, max(-1, linenum - 10), -1): + match = Match(r'^([^()]*\w+)\(', clean_lines.elided[i]) + if match: + # Look for "override" after the matching closing parenthesis + line, _, closing_paren = CloseExpression(clean_lines, i, + len(match.group(1))) + return (closing_paren >= 0 + and Search(r'\boverride\b', line[closing_paren:])) + return False def IsOutOfLineMethodDefinition(clean_lines, linenum): - """Check if current line contains an out-of-line method definition. + """Check if current line contains an out-of-line method definition. Args: clean_lines: A CleansedLines instance containing the file. @@ -5121,15 +5216,16 @@ def IsOutOfLineMethodDefinition(clean_lines, linenum): Returns: True if current line contains an out-of-line method definition. """ - # Scan back a few lines for start of current function - for i in range(linenum, max(-1, linenum - 10), -1): - if Match(r'^([^()]*\w+)\(', clean_lines.elided[i]): - return Match(r'^[^()]*\w+::\w+\(', clean_lines.elided[i]) is not None - return False + # Scan back a few lines for start of current function + for i in range(linenum, max(-1, linenum - 10), -1): + if Match(r'^([^()]*\w+)\(', clean_lines.elided[i]): + return Match(r'^[^()]*\w+::\w+\(', + clean_lines.elided[i]) is not None + return False def IsInitializerList(clean_lines, linenum): - """Check if current line is inside constructor initializer list. + """Check if current line is inside constructor initializer list. Args: clean_lines: A CleansedLines instance containing the file. @@ -5138,41 +5234,42 @@ def IsInitializerList(clean_lines, linenum): True if current line appears to be inside constructor initializer list, False otherwise. """ - for i in range(linenum, 1, -1): - line = clean_lines.elided[i] - if i == linenum: - remove_function_body = Match(r'^(.*)\{\s*$', line) - if remove_function_body: - line = remove_function_body.group(1) + for i in range(linenum, 1, -1): + line = clean_lines.elided[i] + if i == linenum: + remove_function_body = Match(r'^(.*)\{\s*$', line) + if remove_function_body: + line = remove_function_body.group(1) - if Search(r'\s:\s*\w+[({]', line): - # A lone colon tend to indicate the start of a constructor - # initializer list. It could also be a ternary operator, which - # also tend to appear in constructor initializer lists as - # opposed to parameter lists. - return True - if Search(r'\}\s*,\s*$', line): - # A closing brace followed by a comma is probably the end of a - # brace-initialized member in constructor initializer list. - return True - if Search(r'[{};]\s*$', line): - # Found one of the following: - # - A closing brace or semicolon, probably the end of the previous - # function. - # - An opening brace, probably the start of current class or namespace. - # - # Current line is probably not inside an initializer list since - # we saw one of those things without seeing the starting colon. - return False + if Search(r'\s:\s*\w+[({]', line): + # A lone colon tend to indicate the start of a constructor + # initializer list. It could also be a ternary operator, which + # also tend to appear in constructor initializer lists as + # opposed to parameter lists. + return True + if Search(r'\}\s*,\s*$', line): + # A closing brace followed by a comma is probably the end of a + # brace-initialized member in constructor initializer list. + return True + if Search(r'[{};]\s*$', line): + # Found one of the following: + # - A closing brace or semicolon, probably the end of the previous + # function. + # - An opening brace, probably the start of current class or + # namespace. + # + # Current line is probably not inside an initializer list since + # we saw one of those things without seeing the starting colon. + return False - # Got to the beginning of the file without seeing the start of - # constructor initializer list. - return False + # Got to the beginning of the file without seeing the start of + # constructor initializer list. + return False -def CheckForNonConstReference(filename, clean_lines, linenum, - nesting_state, error): - """Check for non-const references. +def CheckForNonConstReference(filename, clean_lines, linenum, nesting_state, + error): + """Check for non-const references. Separate from CheckLanguage since it scans backwards from current line, instead of scanning forward. @@ -5185,132 +5282,134 @@ def CheckForNonConstReference(filename, clean_lines, linenum, the current stack of nested blocks being parsed. error: The function to call with any errors found. """ - # Do nothing if there is no '&' on current line. - line = clean_lines.elided[linenum] - if '&' not in line: - return - - # If a function is inherited, current function doesn't have much of - # a choice, so any non-const references should not be blamed on - # derived function. - if IsDerivedFunction(clean_lines, linenum): - return - - # Don't warn on out-of-line method definitions, as we would warn on the - # in-line declaration, if it isn't marked with 'override'. - if IsOutOfLineMethodDefinition(clean_lines, linenum): - return - - # Long type names may be broken across multiple lines, usually in one - # of these forms: - # LongType - # ::LongTypeContinued &identifier - # LongType:: - # LongTypeContinued &identifier - # LongType< - # ...>::LongTypeContinued &identifier - # - # If we detected a type split across two lines, join the previous - # line to current line so that we can match const references - # accordingly. - # - # Note that this only scans back one line, since scanning back - # arbitrary number of lines would be expensive. If you have a type - # that spans more than 2 lines, please use a typedef. - if linenum > 1: - previous = None - if Match(r'\s*::(?:[\w<>]|::)+\s*&\s*\S', line): - # previous_line\n + ::current_line - previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+[\w<>])\s*$', - clean_lines.elided[linenum - 1]) - elif Match(r'\s*[a-zA-Z_]([\w<>]|::)+\s*&\s*\S', line): - # previous_line::\n + current_line - previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+::)\s*$', - clean_lines.elided[linenum - 1]) - if previous: - line = previous.group(1) + line.lstrip() - else: - # Check for templated parameter that is split across multiple lines - endpos = line.rfind('>') - if endpos > -1: - (_, startline, startpos) = ReverseCloseExpression( - clean_lines, linenum, endpos) - if startpos > -1 and startline < linenum: - # Found the matching < on an earlier line, collect all - # pieces up to current line. - line = '' - for i in range(startline, linenum + 1): - line += clean_lines.elided[i].strip() - - # Check for non-const references in function parameters. A single '&' may - # found in the following places: - # inside expression: binary & for bitwise AND - # inside expression: unary & for taking the address of something - # inside declarators: reference parameter - # We will exclude the first two cases by checking that we are not inside a - # function body, including one that was just introduced by a trailing '{'. - # TODO(unknown): Doesn't account for 'catch(Exception& e)' [rare]. - if (nesting_state.previous_stack_top and - not (isinstance(nesting_state.previous_stack_top, _ClassInfo) or - isinstance(nesting_state.previous_stack_top, _NamespaceInfo))): - # Not at toplevel, not within a class, and not within a namespace - return - - # Avoid initializer lists. We only need to scan back from the - # current line for something that starts with ':'. - # - # We don't need to check the current line, since the '&' would - # appear inside the second set of parentheses on the current line as - # opposed to the first set. - if linenum > 0: - for i in range(linenum - 1, max(0, linenum - 10), -1): - previous_line = clean_lines.elided[i] - if not Search(r'[),]\s*$', previous_line): - break - if Match(r'^\s*:\s+\S', previous_line): + # Do nothing if there is no '&' on current line. + line = clean_lines.elided[linenum] + if '&' not in line: return - # Avoid preprocessors - if Search(r'\\\s*$', line): - return - - # Avoid constructor initializer lists - if IsInitializerList(clean_lines, linenum): - return - - # We allow non-const references in a few standard places, like functions - # called "swap()" or iostream operators like "<<" or ">>". Do not check - # those function parameters. - # - # We also accept & in static_assert, which looks like a function but - # it's actually a declaration expression. - allowlisted_functions = (r'(?:[sS]wap(?:<\w:+>)?|' - r'operator\s*[<>][<>]|' - r'static_assert|COMPILE_ASSERT' - r')\s*\(') - if Search(allowlisted_functions, line): - return - elif not Search(r'\S+\([^)]*$', line): - # Don't see an allowlisted function on this line. Actually we - # didn't see any function name on this line, so this is likely a - # multi-line parameter list. Try a bit harder to catch this case. - for i in range(2): - if (linenum > i and - Search(allowlisted_functions, clean_lines.elided[linenum - i - 1])): + # If a function is inherited, current function doesn't have much of + # a choice, so any non-const references should not be blamed on + # derived function. + if IsDerivedFunction(clean_lines, linenum): return - decls = ReplaceAll(r'{[^}]*}', ' ', line) # exclude function body - for parameter in re.findall(_RE_PATTERN_REF_PARAM, decls): - if (not Match(_RE_PATTERN_CONST_REF_PARAM, parameter) and - not Match(_RE_PATTERN_REF_STREAM_PARAM, parameter)): - error(filename, linenum, 'runtime/references', 2, - 'Is this a non-const reference? ' - 'If so, make const or use a pointer: ' + - ReplaceAll(' *<', '<', parameter)) + # Don't warn on out-of-line method definitions, as we would warn on the + # in-line declaration, if it isn't marked with 'override'. + if IsOutOfLineMethodDefinition(clean_lines, linenum): + return + + # Long type names may be broken across multiple lines, usually in one + # of these forms: + # LongType + # ::LongTypeContinued &identifier + # LongType:: + # LongTypeContinued &identifier + # LongType< + # ...>::LongTypeContinued &identifier + # + # If we detected a type split across two lines, join the previous + # line to current line so that we can match const references + # accordingly. + # + # Note that this only scans back one line, since scanning back + # arbitrary number of lines would be expensive. If you have a type + # that spans more than 2 lines, please use a typedef. + if linenum > 1: + previous = None + if Match(r'\s*::(?:[\w<>]|::)+\s*&\s*\S', line): + # previous_line\n + ::current_line + previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+[\w<>])\s*$', + clean_lines.elided[linenum - 1]) + elif Match(r'\s*[a-zA-Z_]([\w<>]|::)+\s*&\s*\S', line): + # previous_line::\n + current_line + previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+::)\s*$', + clean_lines.elided[linenum - 1]) + if previous: + line = previous.group(1) + line.lstrip() + else: + # Check for templated parameter that is split across multiple lines + endpos = line.rfind('>') + if endpos > -1: + (_, startline, + startpos) = ReverseCloseExpression(clean_lines, linenum, + endpos) + if startpos > -1 and startline < linenum: + # Found the matching < on an earlier line, collect all + # pieces up to current line. + line = '' + for i in range(startline, linenum + 1): + line += clean_lines.elided[i].strip() + + # Check for non-const references in function parameters. A single '&' may + # found in the following places: + # inside expression: binary & for bitwise AND + # inside expression: unary & for taking the address of something + # inside declarators: reference parameter + # We will exclude the first two cases by checking that we are not inside a + # function body, including one that was just introduced by a trailing '{'. + # TODO(unknown): Doesn't account for 'catch(Exception& e)' [rare]. + if (nesting_state.previous_stack_top and + not (isinstance(nesting_state.previous_stack_top, _ClassInfo) or + isinstance(nesting_state.previous_stack_top, _NamespaceInfo))): + # Not at toplevel, not within a class, and not within a namespace + return + + # Avoid initializer lists. We only need to scan back from the + # current line for something that starts with ':'. + # + # We don't need to check the current line, since the '&' would + # appear inside the second set of parentheses on the current line as + # opposed to the first set. + if linenum > 0: + for i in range(linenum - 1, max(0, linenum - 10), -1): + previous_line = clean_lines.elided[i] + if not Search(r'[),]\s*$', previous_line): + break + if Match(r'^\s*:\s+\S', previous_line): + return + + # Avoid preprocessors + if Search(r'\\\s*$', line): + return + + # Avoid constructor initializer lists + if IsInitializerList(clean_lines, linenum): + return + + # We allow non-const references in a few standard places, like functions + # called "swap()" or iostream operators like "<<" or ">>". Do not check + # those function parameters. + # + # We also accept & in static_assert, which looks like a function but + # it's actually a declaration expression. + allowlisted_functions = (r'(?:[sS]wap(?:<\w:+>)?|' + r'operator\s*[<>][<>]|' + r'static_assert|COMPILE_ASSERT' + r')\s*\(') + if Search(allowlisted_functions, line): + return + elif not Search(r'\S+\([^)]*$', line): + # Don't see an allowlisted function on this line. Actually we + # didn't see any function name on this line, so this is likely a + # multi-line parameter list. Try a bit harder to catch this case. + for i in range(2): + if (linenum > i and Search(allowlisted_functions, + clean_lines.elided[linenum - i - 1])): + return + + decls = ReplaceAll(r'{[^}]*}', ' ', line) # exclude function body + for parameter in re.findall(_RE_PATTERN_REF_PARAM, decls): + if (not Match(_RE_PATTERN_CONST_REF_PARAM, parameter) + and not Match(_RE_PATTERN_REF_STREAM_PARAM, parameter)): + error( + filename, linenum, 'runtime/references', 2, + 'Is this a non-const reference? ' + 'If so, make const or use a pointer: ' + + ReplaceAll(' *<', '<', parameter)) def CheckCasts(filename, clean_lines, linenum, error): - """Various cast related checks. + """Various cast related checks. Args: filename: The name of the current file. @@ -5318,118 +5417,120 @@ def CheckCasts(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] + line = clean_lines.elided[linenum] - # Check to see if they're using an conversion function cast. - # I just try to capture the most common basic types, though there are more. - # Parameterless conversion functions, such as bool(), are allowed as they are - # probably a member operator declaration or default constructor. - match = Search( - r'(\bnew\s+(?:const\s+)?|\S<\s*(?:const\s+)?)?\b' - r'(int|float|double|bool|char|int32|uint32|int64|uint64)' - r'(\([^)].*)', line) - expecting_function = ExpectingFunctionArgs(clean_lines, linenum) - if match and not expecting_function: - matched_type = match.group(2) + # Check to see if they're using an conversion function cast. + # I just try to capture the most common basic types, though there are more. + # Parameterless conversion functions, such as bool(), are allowed as they + # are probably a member operator declaration or default constructor. + match = Search( + r'(\bnew\s+(?:const\s+)?|\S<\s*(?:const\s+)?)?\b' + r'(int|float|double|bool|char|int32|uint32|int64|uint64)' + r'(\([^)].*)', line) + expecting_function = ExpectingFunctionArgs(clean_lines, linenum) + if match and not expecting_function: + matched_type = match.group(2) - # matched_new_or_template is used to silence two false positives: - # - New operators - # - Template arguments with function types + # matched_new_or_template is used to silence two false positives: + # - New operators + # - Template arguments with function types + # + # For template arguments, we match on types immediately following + # an opening bracket without any spaces. This is a fast way to + # silence the common case where the function type is the first + # template argument. False negative with less-than comparison is + # avoided because those operators are usually followed by a space. + # + # function // bracket + no space = false positive + # value < double(42) // bracket + space = true positive + matched_new_or_template = match.group(1) + + # Avoid arrays by looking for brackets that come after the closing + # parenthesis. + if Match(r'\([^()]+\)\s*\[', match.group(3)): + return + + # Other things to ignore: + # - Function pointers + # - Casts to pointer types + # - Placement new + # - Alias declarations + matched_funcptr = match.group(3) + if (matched_new_or_template is None and not ( + matched_funcptr and + (Match(r'\((?:[^() ]+::\s*\*\s*)?[^() ]+\)\s*\(', matched_funcptr) + or matched_funcptr.startswith('(*)'))) + and not Match(r'\s*using\s+\S+\s*=\s*' + matched_type, line) + and not Search(r'new\(\S+\)\s*' + matched_type, line)): + error( + filename, linenum, 'readability/casting', 4, + 'Using deprecated casting style. ' + 'Use static_cast<%s>(...) instead' % matched_type) + + if not expecting_function: + CheckCStyleCast(filename, clean_lines, linenum, 'static_cast', + r'\((int|float|double|bool|char|u?int(16|32|64))\)', + error) + + # This doesn't catch all cases. Consider (const char * const)"hello". # - # For template arguments, we match on types immediately following - # an opening bracket without any spaces. This is a fast way to - # silence the common case where the function type is the first - # template argument. False negative with less-than comparison is - # avoided because those operators are usually followed by a space. - # - # function // bracket + no space = false positive - # value < double(42) // bracket + space = true positive - matched_new_or_template = match.group(1) - - # Avoid arrays by looking for brackets that come after the closing - # parenthesis. - if Match(r'\([^()]+\)\s*\[', match.group(3)): - return - - # Other things to ignore: - # - Function pointers - # - Casts to pointer types - # - Placement new - # - Alias declarations - matched_funcptr = match.group(3) - if (matched_new_or_template is None and - not (matched_funcptr and - (Match(r'\((?:[^() ]+::\s*\*\s*)?[^() ]+\)\s*\(', - matched_funcptr) or - matched_funcptr.startswith('(*)'))) and - not Match(r'\s*using\s+\S+\s*=\s*' + matched_type, line) and - not Search(r'new\(\S+\)\s*' + matched_type, line)): - error(filename, linenum, 'readability/casting', 4, - 'Using deprecated casting style. ' - 'Use static_cast<%s>(...) instead' % - matched_type) - - if not expecting_function: - CheckCStyleCast(filename, clean_lines, linenum, 'static_cast', - r'\((int|float|double|bool|char|u?int(16|32|64))\)', error) - - # This doesn't catch all cases. Consider (const char * const)"hello". - # - # (char *) "foo" should always be a const_cast (reinterpret_cast won't - # compile). - if CheckCStyleCast(filename, clean_lines, linenum, 'const_cast', - r'\((char\s?\*+\s?)\)\s*"', error): - pass - else: - # Check pointer casts for other than string constants - CheckCStyleCast(filename, clean_lines, linenum, 'reinterpret_cast', - r'\((\w+\s?\*+\s?)\)', error) - - # In addition, we look for people taking the address of a cast. This - # is dangerous -- casts can assign to temporaries, so the pointer doesn't - # point where you think. - # - # Some non-identifier character is required before the '&' for the - # expression to be recognized as a cast. These are casts: - # expression = &static_cast(temporary()); - # function(&(int*)(temporary())); - # - # This is not a cast: - # reference_type&(int* function_param); - match = Search( - r'(?:[^\w]&\(([^)*][^)]*)\)[\w(])|' - r'(?:[^\w]&(static|dynamic|down|reinterpret)_cast\b)', line) - if match: - # Try a better error message when the & is bound to something - # dereferenced by the casted pointer, as opposed to the casted - # pointer itself. - parenthesis_error = False - match = Match(r'^(.*&(?:static|dynamic|down|reinterpret)_cast\b)<', line) - if match: - _, y1, x1 = CloseExpression(clean_lines, linenum, len(match.group(1))) - if x1 >= 0 and clean_lines.elided[y1][x1] == '(': - _, y2, x2 = CloseExpression(clean_lines, y1, x1) - if x2 >= 0: - extended_line = clean_lines.elided[y2][x2:] - if y2 < clean_lines.NumLines() - 1: - extended_line += clean_lines.elided[y2 + 1] - if Match(r'\s*(?:->|\[)', extended_line): - parenthesis_error = True - - if parenthesis_error: - error(filename, linenum, 'readability/casting', 4, - ('Are you taking an address of something dereferenced ' - 'from a cast? Wrapping the dereferenced expression in ' - 'parentheses will make the binding more obvious')) + # (char *) "foo" should always be a const_cast (reinterpret_cast won't + # compile). + if CheckCStyleCast(filename, clean_lines, linenum, 'const_cast', + r'\((char\s?\*+\s?)\)\s*"', error): + pass else: - error(filename, linenum, 'runtime/casting', 4, - ('Are you taking an address of a cast? ' - 'This is dangerous: could be a temp var. ' - 'Take the address before doing the cast, rather than after')) + # Check pointer casts for other than string constants + CheckCStyleCast(filename, clean_lines, linenum, 'reinterpret_cast', + r'\((\w+\s?\*+\s?)\)', error) + + # In addition, we look for people taking the address of a cast. This + # is dangerous -- casts can assign to temporaries, so the pointer doesn't + # point where you think. + # + # Some non-identifier character is required before the '&' for the + # expression to be recognized as a cast. These are casts: + # expression = &static_cast(temporary()); + # function(&(int*)(temporary())); + # + # This is not a cast: + # reference_type&(int* function_param); + match = Search( + r'(?:[^\w]&\(([^)*][^)]*)\)[\w(])|' + r'(?:[^\w]&(static|dynamic|down|reinterpret)_cast\b)', line) + if match: + # Try a better error message when the & is bound to something + # dereferenced by the casted pointer, as opposed to the casted + # pointer itself. + parenthesis_error = False + match = Match(r'^(.*&(?:static|dynamic|down|reinterpret)_cast\b)<', + line) + if match: + _, y1, x1 = CloseExpression(clean_lines, linenum, + len(match.group(1))) + if x1 >= 0 and clean_lines.elided[y1][x1] == '(': + _, y2, x2 = CloseExpression(clean_lines, y1, x1) + if x2 >= 0: + extended_line = clean_lines.elided[y2][x2:] + if y2 < clean_lines.NumLines() - 1: + extended_line += clean_lines.elided[y2 + 1] + if Match(r'\s*(?:->|\[)', extended_line): + parenthesis_error = True + + if parenthesis_error: + error(filename, linenum, 'readability/casting', 4, + ('Are you taking an address of something dereferenced ' + 'from a cast? Wrapping the dereferenced expression in ' + 'parentheses will make the binding more obvious')) + else: + error(filename, linenum, 'runtime/casting', 4, + ('Are you taking an address of a cast? ' + 'This is dangerous: could be a temp var. ' + 'Take the address before doing the cast, rather than after')) def CheckCStyleCast(filename, clean_lines, linenum, cast_type, pattern, error): - """Checks for a C-style cast by looking for the pattern. + """Checks for a C-style cast by looking for the pattern. Args: filename: The name of the current file. @@ -5444,45 +5545,46 @@ def CheckCStyleCast(filename, clean_lines, linenum, cast_type, pattern, error): True if an error was emitted. False otherwise. """ - line = clean_lines.elided[linenum] - match = Search(pattern, line) - if not match: - return False + line = clean_lines.elided[linenum] + match = Search(pattern, line) + if not match: + return False - # Exclude lines with keywords that tend to look like casts - context = line[0:match.start(1) - 1] - if Match(r'.*\b(?:sizeof|alignof|alignas|[_A-Z][_A-Z0-9]*)\s*$', context): - return False + # Exclude lines with keywords that tend to look like casts + context = line[0:match.start(1) - 1] + if Match(r'.*\b(?:sizeof|alignof|alignas|[_A-Z][_A-Z0-9]*)\s*$', context): + return False - # Try expanding current context to see if we one level of - # parentheses inside a macro. - if linenum > 0: - for i in range(linenum - 1, max(0, linenum - 5), -1): - context = clean_lines.elided[i] + context - if Match(r'.*\b[_A-Z][_A-Z0-9]*\s*\((?:\([^()]*\)|[^()])*$', context): - return False + # Try expanding current context to see if we one level of + # parentheses inside a macro. + if linenum > 0: + for i in range(linenum - 1, max(0, linenum - 5), -1): + context = clean_lines.elided[i] + context + if Match(r'.*\b[_A-Z][_A-Z0-9]*\s*\((?:\([^()]*\)|[^()])*$', context): + return False - # operator++(int) and operator--(int) - if context.endswith(' operator++') or context.endswith(' operator--'): - return False + # operator++(int) and operator--(int) + if context.endswith(' operator++') or context.endswith(' operator--'): + return False - # A single unnamed argument for a function tends to look like old style cast. - # If we see those, don't issue warnings for deprecated casts. - remainder = line[match.end(0):] - if Match(r'^\s*(?:;|const\b|throw\b|final\b|override\b|[=>{),]|->)', - remainder): - return False + # A single unnamed argument for a function tends to look like old style + # cast. If we see those, don't issue warnings for deprecated casts. + remainder = line[match.end(0):] + if Match(r'^\s*(?:;|const\b|throw\b|final\b|override\b|[=>{),]|->)', + remainder): + return False - # At this point, all that should be left is actual casts. - error(filename, linenum, 'readability/casting', 4, + # At this point, all that should be left is actual casts. + error( + filename, linenum, 'readability/casting', 4, 'Using C-style cast. Use %s<%s>(...) instead' % (cast_type, match.group(1))) - return True + return True def ExpectingFunctionArgs(clean_lines, linenum): - """Checks whether where function type arguments are expected. + """Checks whether where function type arguments are expected. Args: clean_lines: A CleansedLines instance containing the file. @@ -5492,88 +5594,131 @@ def ExpectingFunctionArgs(clean_lines, linenum): True if the line at 'linenum' is inside something that expects arguments of function types. """ - line = clean_lines.elided[linenum] - return (Match(r'^\s*MOCK_(CONST_)?METHOD\d+(_T)?\(', line) - or _TYPE_TRAITS_RE.search(line) - or (linenum >= 2 and - (Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\((?:\S+,)?\s*$', - clean_lines.elided[linenum - 1]) - or Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\(\s*$', - clean_lines.elided[linenum - 2]) - or Search(r'\b(::function|base::FunctionRef)\s*\<\s*$', - clean_lines.elided[linenum - 1])))) + line = clean_lines.elided[linenum] + return (Match(r'^\s*MOCK_(CONST_)?METHOD\d+(_T)?\(', line) + or _TYPE_TRAITS_RE.search(line) + or (linenum >= 2 and + (Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\((?:\S+,)?\s*$', + clean_lines.elided[linenum - 1]) + or Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\(\s*$', + clean_lines.elided[linenum - 2]) + or Search(r'\b(::function|base::FunctionRef)\s*\<\s*$', + clean_lines.elided[linenum - 1])))) _HEADERS_CONTAINING_TEMPLATES = ( - ('', ('deque',)), - ('', ('unary_function', 'binary_function', - 'plus', 'minus', 'multiplies', 'divides', 'modulus', - 'negate', - 'equal_to', 'not_equal_to', 'greater', 'less', - 'greater_equal', 'less_equal', - 'logical_and', 'logical_or', 'logical_not', - 'unary_negate', 'not1', 'binary_negate', 'not2', - 'bind1st', 'bind2nd', - 'pointer_to_unary_function', - 'pointer_to_binary_function', - 'ptr_fun', - 'mem_fun_t', 'mem_fun', 'mem_fun1_t', 'mem_fun1_ref_t', - 'mem_fun_ref_t', - 'const_mem_fun_t', 'const_mem_fun1_t', - 'const_mem_fun_ref_t', 'const_mem_fun1_ref_t', - 'mem_fun_ref', - )), - ('', ('numeric_limits',)), - ('', ('list',)), - ('', ('map', 'multimap',)), + ('', ('deque', )), + ('', ( + 'unary_function', + 'binary_function', + 'plus', + 'minus', + 'multiplies', + 'divides', + 'modulus', + 'negate', + 'equal_to', + 'not_equal_to', + 'greater', + 'less', + 'greater_equal', + 'less_equal', + 'logical_and', + 'logical_or', + 'logical_not', + 'unary_negate', + 'not1', + 'binary_negate', + 'not2', + 'bind1st', + 'bind2nd', + 'pointer_to_unary_function', + 'pointer_to_binary_function', + 'ptr_fun', + 'mem_fun_t', + 'mem_fun', + 'mem_fun1_t', + 'mem_fun1_ref_t', + 'mem_fun_ref_t', + 'const_mem_fun_t', + 'const_mem_fun1_t', + 'const_mem_fun_ref_t', + 'const_mem_fun1_ref_t', + 'mem_fun_ref', + )), + ('', ('numeric_limits', )), + ('', ('list', )), + ('', ( + 'map', + 'multimap', + )), ('', ('allocator', 'make_shared', 'make_unique', 'shared_ptr', 'unique_ptr', 'weak_ptr')), - ('', ('queue', 'priority_queue',)), - ('', ('set', 'multiset',)), - ('', ('stack',)), - ('', ('char_traits', 'basic_string',)), - ('', ('tuple',)), + ('', ( + 'queue', + 'priority_queue', + )), + ('', ( + 'set', + 'multiset', + )), + ('', ('stack', )), + ('', ( + 'char_traits', + 'basic_string', + )), + ('', ('tuple', )), ('', ('unordered_map', 'unordered_multimap')), ('', ('unordered_set', 'unordered_multiset')), - ('', ('pair',)), - ('', ('vector',)), + ('', ('pair', )), + ('', ('vector', )), # gcc extensions. # Note: std::hash is their hash, ::hash is our hash - ('', ('hash_map', 'hash_multimap',)), - ('', ('hash_set', 'hash_multiset',)), - ('', ('slist',)), - ) + ('', ( + 'hash_map', + 'hash_multimap', + )), + ('', ( + 'hash_set', + 'hash_multiset', + )), + ('', ('slist', )), +) _HEADERS_MAYBE_TEMPLATES = ( - ('', ('copy', 'max', 'min', 'min_element', 'sort', - 'transform', - )), + ('', ( + 'copy', + 'max', + 'min', + 'min_element', + 'sort', + 'transform', + )), ('', ('forward', 'make_pair', 'move', 'swap')), - ) +) _RE_PATTERN_STRING = re.compile(r'\bstring\b') _re_pattern_headers_maybe_templates = [] for _header, _templates in _HEADERS_MAYBE_TEMPLATES: - for _template in _templates: - # Match max(..., ...), max(..., ...), but not foo->max or foo.max. - _re_pattern_headers_maybe_templates.append( - (re.compile(r'(?.])\b' + _template + r'(<.*?>)?\([^\)]'), _template, - _header)) + for _template in _templates: + # Match max(..., ...), max(..., ...), but not foo->max or foo.max. + _re_pattern_headers_maybe_templates.append( + (re.compile(r'(?.])\b' + _template + r'(<.*?>)?\([^\)]'), + _template, _header)) # Other scripts may reach in and modify this pattern. _re_pattern_templates = [] for _header, _templates in _HEADERS_CONTAINING_TEMPLATES: - for _template in _templates: - _re_pattern_templates.append( - (re.compile(r'(\<|\b)' + _template + r'\s*\<'), - _template + '<>', - _header)) + for _template in _templates: + _re_pattern_templates.append( + (re.compile(r'(\<|\b)' + _template + r'\s*\<'), _template + '<>', + _header)) def FilesBelongToSameModule(filename_cc, filename_h): - """Check if these two filenames belong to the same module. + """Check if these two filenames belong to the same module. The concept of a 'module' here is a as follows: foo.h, foo-inl.h, foo.cc, foo_test.cc and foo_unittest.cc belong to the @@ -5602,33 +5747,33 @@ def FilesBelongToSameModule(filename_cc, filename_h): string: the additional prefix needed to open the header file. """ - fileinfo = FileInfo(filename_cc) - if not fileinfo.IsSource(): - return (False, '') - filename_cc = filename_cc[:-len(fileinfo.Extension())] - matched_test_suffix = Search(_TEST_FILE_SUFFIX, fileinfo.BaseName()) - if matched_test_suffix: - filename_cc = filename_cc[:-len(matched_test_suffix.group(1))] - filename_cc = filename_cc.replace('/public/', '/') - filename_cc = filename_cc.replace('/internal/', '/') + fileinfo = FileInfo(filename_cc) + if not fileinfo.IsSource(): + return (False, '') + filename_cc = filename_cc[:-len(fileinfo.Extension())] + matched_test_suffix = Search(_TEST_FILE_SUFFIX, fileinfo.BaseName()) + if matched_test_suffix: + filename_cc = filename_cc[:-len(matched_test_suffix.group(1))] + filename_cc = filename_cc.replace('/public/', '/') + filename_cc = filename_cc.replace('/internal/', '/') - if not filename_h.endswith('.h'): - return (False, '') - filename_h = filename_h[:-len('.h')] - if filename_h.endswith('-inl'): - filename_h = filename_h[:-len('-inl')] - filename_h = filename_h.replace('/public/', '/') - filename_h = filename_h.replace('/internal/', '/') + if not filename_h.endswith('.h'): + return (False, '') + filename_h = filename_h[:-len('.h')] + if filename_h.endswith('-inl'): + filename_h = filename_h[:-len('-inl')] + filename_h = filename_h.replace('/public/', '/') + filename_h = filename_h.replace('/internal/', '/') - files_belong_to_same_module = filename_cc.endswith(filename_h) - common_path = '' - if files_belong_to_same_module: - common_path = filename_cc[:-len(filename_h)] - return files_belong_to_same_module, common_path + files_belong_to_same_module = filename_cc.endswith(filename_h) + common_path = '' + if files_belong_to_same_module: + common_path = filename_cc[:-len(filename_h)] + return files_belong_to_same_module, common_path def UpdateIncludeState(filename, include_dict, io=codecs): - """Fill up the include_dict with new includes found from the file. + """Fill up the include_dict with new includes found from the file. Args: filename: the name of the header to read. @@ -5638,37 +5783,40 @@ def UpdateIncludeState(filename, include_dict, io=codecs): Returns: True if a header was successfully added. False otherwise. """ - headerfile = None - try: - headerfile = io.open(filename, 'r', 'utf8', 'replace') - except IOError: - return False - linenum = 0 - for line in headerfile: - linenum += 1 - clean_line = CleanseComments(line) - match = _RE_PATTERN_INCLUDE.search(clean_line) - if match: - include = match.group(2) - include_dict.setdefault(include, linenum) - return True + headerfile = None + try: + headerfile = io.open(filename, 'r', 'utf8', 'replace') + except IOError: + return False + linenum = 0 + for line in headerfile: + linenum += 1 + clean_line = CleanseComments(line) + match = _RE_PATTERN_INCLUDE.search(clean_line) + if match: + include = match.group(2) + include_dict.setdefault(include, linenum) + return True def UpdateRequiredHeadersForLine(patterns, line, linenum, required): - for pattern, template, header in patterns: - matched = pattern.search(line) - if matched: - # Don't warn about IWYU in non-STL namespaces: - # (We check only the first match per line; good enough.) - prefix = line[:matched.start()] - if prefix.endswith('std::') or not prefix.endswith('::'): - required[header] = (linenum, template) - return required + for pattern, template, header in patterns: + matched = pattern.search(line) + if matched: + # Don't warn about IWYU in non-STL namespaces: + # (We check only the first match per line; good enough.) + prefix = line[:matched.start()] + if prefix.endswith('std::') or not prefix.endswith('::'): + required[header] = (linenum, template) + return required -def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error, +def CheckForIncludeWhatYouUse(filename, + clean_lines, + include_state, + error, io=codecs): - """Reports for missing stl includes. + """Reports for missing stl includes. This function will output warnings to make sure you are including the headers necessary for the stl containers and functions that you use. We only give one @@ -5684,85 +5832,87 @@ def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error, io: The IO factory to use to read the header file. Provided for unittest injection. """ - # A map of header name to linenumber and the template entity. - # Example of required: { '': (1219, 'less<>') } - required = {} - for linenum in range(clean_lines.NumLines()): - line = clean_lines.elided[linenum] - if not line or line[0] == '#': - continue + # A map of header name to linenumber and the template entity. + # Example of required: { '': (1219, 'less<>') } + required = {} + for linenum in range(clean_lines.NumLines()): + line = clean_lines.elided[linenum] + if not line or line[0] == '#': + continue - # String is special -- it is a non-templatized type in STL. - matched = _RE_PATTERN_STRING.search(line) - if matched: - # Don't warn about strings in non-STL namespaces: - # (We check only the first match per line; good enough.) - prefix = line[:matched.start()] - if prefix.endswith('std::') or not prefix.endswith('::'): - required[''] = (linenum, 'string') + # String is special -- it is a non-templatized type in STL. + matched = _RE_PATTERN_STRING.search(line) + if matched: + # Don't warn about strings in non-STL namespaces: + # (We check only the first match per line; good enough.) + prefix = line[:matched.start()] + if prefix.endswith('std::') or not prefix.endswith('::'): + required[''] = (linenum, 'string') - required = UpdateRequiredHeadersForLine(_re_pattern_headers_maybe_templates, - line, linenum, required) + required = UpdateRequiredHeadersForLine( + _re_pattern_headers_maybe_templates, line, linenum, required) - # The following function is just a speed up, no semantics are changed. - if not '<' in line: # Reduces the cpu time usage by skipping lines. - continue + # The following function is just a speed up, no semantics are changed. + if not '<' in line: # Reduces the cpu time usage by skipping lines. + continue - required = UpdateRequiredHeadersForLine(_re_pattern_templates, line, - linenum, required) + required = UpdateRequiredHeadersForLine(_re_pattern_templates, line, + linenum, required) - # The policy is that if you #include something in foo.h you don't need to - # include it again in foo.cc. Here, we will look at possible includes. - # Let's flatten the include_state include_list and copy it into a dictionary. - include_dict = dict([item for sublist in include_state.include_list - for item in sublist]) + # The policy is that if you #include something in foo.h you don't need to + # include it again in foo.cc. Here, we will look at possible includes. + # Let's flatten the include_state include_list and copy it into a + # dictionary. + include_dict = dict( + [item for sublist in include_state.include_list for item in sublist]) - # Did we find the header for this file (if any) and successfully load it? - header_found = False + # Did we find the header for this file (if any) and successfully load it? + header_found = False - # Use the absolute path so that matching works properly. - abs_filename = FileInfo(filename).FullName() + # Use the absolute path so that matching works properly. + abs_filename = FileInfo(filename).FullName() - # For Emacs's flymake. - # If cpplint is invoked from Emacs's flymake, a temporary file is generated - # by flymake and that file name might end with '_flymake.cc'. In that case, - # restore original file name here so that the corresponding header file can be - # found. - # e.g. If the file name is 'foo_flymake.cc', we should search for 'foo.h' - # instead of 'foo_flymake.h' - abs_filename = re.sub(r'_flymake\.cc$', '.cc', abs_filename) + # For Emacs's flymake. + # If cpplint is invoked from Emacs's flymake, a temporary file is generated + # by flymake and that file name might end with '_flymake.cc'. In that case, + # restore original file name here so that the corresponding header file can + # be found. e.g. If the file name is 'foo_flymake.cc', we should search for + # 'foo.h' instead of 'foo_flymake.h' + abs_filename = re.sub(r'_flymake\.cc$', '.cc', abs_filename) - # include_dict is modified during iteration, so we iterate over a copy of - # the keys. - header_keys = list(include_dict.keys()) - for header in header_keys: - (same_module, common_path) = FilesBelongToSameModule(abs_filename, header) - fullpath = common_path + header - if same_module and UpdateIncludeState(fullpath, include_dict, io): - header_found = True + # include_dict is modified during iteration, so we iterate over a copy of + # the keys. + header_keys = list(include_dict.keys()) + for header in header_keys: + (same_module, + common_path) = FilesBelongToSameModule(abs_filename, header) + fullpath = common_path + header + if same_module and UpdateIncludeState(fullpath, include_dict, io): + header_found = True - # If we can't find the header file for a .cc, assume it's because we don't - # know where to look. In that case we'll give up as we're not sure they - # didn't include it in the .h file. - # TODO(unknown): Do a better job of finding .h files so we are confident that - # not having the .h file means there isn't one. - if filename.endswith('.cc') and not header_found: - return + # If we can't find the header file for a .cc, assume it's because we don't + # know where to look. In that case we'll give up as we're not sure they + # didn't include it in the .h file. + # TODO(unknown): Do a better job of finding .h files so we are confident + # that not having the .h file means there isn't one. + if filename.endswith('.cc') and not header_found: + return - # All the lines have been processed, report the errors found. - for required_header_unstripped in required: - template = required[required_header_unstripped][1] - if required_header_unstripped.strip('<>"') not in include_dict: - error(filename, required[required_header_unstripped][0], - 'build/include_what_you_use', 4, - 'Add #include ' + required_header_unstripped + ' for ' + template) + # All the lines have been processed, report the errors found. + for required_header_unstripped in required: + template = required[required_header_unstripped][1] + if required_header_unstripped.strip('<>"') not in include_dict: + error( + filename, required[required_header_unstripped][0], + 'build/include_what_you_use', 4, 'Add #include ' + + required_header_unstripped + ' for ' + template) _RE_PATTERN_EXPLICIT_MAKEPAIR = re.compile(r'\bmake_pair\s*<') def CheckMakePairUsesDeduction(filename, clean_lines, linenum, error): - """Check that make_pair's template arguments are deduced. + """Check that make_pair's template arguments are deduced. G++ 4.6 in C++11 mode fails badly if make_pair's template arguments are specified explicitly, and such use isn't intended in any case. @@ -5773,17 +5923,21 @@ def CheckMakePairUsesDeduction(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - match = _RE_PATTERN_EXPLICIT_MAKEPAIR.search(line) - if match: - error(filename, linenum, 'build/explicit_make_pair', - 4, # 4 = high confidence - 'For C++11-compatibility, omit template arguments from make_pair' - ' OR use pair directly OR if appropriate, construct a pair directly') + line = clean_lines.elided[linenum] + match = _RE_PATTERN_EXPLICIT_MAKEPAIR.search(line) + if match: + error( + filename, + linenum, + 'build/explicit_make_pair', + 4, # 4 = high confidence + 'For C++11-compatibility, omit template arguments from make_pair' + ' OR use pair directly OR if appropriate, construct a pair directly' + ) def CheckRedundantVirtual(filename, clean_lines, linenum, error): - """Check if line contains a redundant "virtual" function-specifier. + """Check if line contains a redundant "virtual" function-specifier. Args: filename: The name of the current file. @@ -5791,63 +5945,64 @@ def CheckRedundantVirtual(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - # Look for "virtual" on current line. - line = clean_lines.elided[linenum] - virtual = Match(r'^(.*)(\bvirtual\b)(.*)$', line) - if not virtual: return + # Look for "virtual" on current line. + line = clean_lines.elided[linenum] + virtual = Match(r'^(.*)(\bvirtual\b)(.*)$', line) + if not virtual: return - # Ignore "virtual" keywords that are near access-specifiers. These - # are only used in class base-specifier and do not apply to member - # functions. - if (Search(r'\b(public|protected|private)\s+$', virtual.group(1)) or - Match(r'^\s+(public|protected|private)\b', virtual.group(3))): - return + # Ignore "virtual" keywords that are near access-specifiers. These + # are only used in class base-specifier and do not apply to member + # functions. + if (Search(r'\b(public|protected|private)\s+$', virtual.group(1)) + or Match(r'^\s+(public|protected|private)\b', virtual.group(3))): + return - # Ignore the "virtual" keyword from virtual base classes. Usually - # there is a column on the same line in these cases (virtual base - # classes are rare in google3 because multiple inheritance is rare). - if Match(r'^.*[^:]:[^:].*$', line): return + # Ignore the "virtual" keyword from virtual base classes. Usually + # there is a column on the same line in these cases (virtual base + # classes are rare in google3 because multiple inheritance is rare). + if Match(r'^.*[^:]:[^:].*$', line): return - # Look for the next opening parenthesis. This is the start of the - # parameter list (possibly on the next line shortly after virtual). - # TODO(unknown): doesn't work if there are virtual functions with - # decltype() or other things that use parentheses, but csearch suggests - # that this is rare. - end_col = -1 - end_line = -1 - start_col = len(virtual.group(2)) - for start_line in range(linenum, min(linenum + 3, clean_lines.NumLines())): - line = clean_lines.elided[start_line][start_col:] - parameter_list = Match(r'^([^(]*)\(', line) - if parameter_list: - # Match parentheses to find the end of the parameter list - (_, end_line, end_col) = CloseExpression( - clean_lines, start_line, start_col + len(parameter_list.group(1))) - break - start_col = 0 + # Look for the next opening parenthesis. This is the start of the + # parameter list (possibly on the next line shortly after virtual). + # TODO(unknown): doesn't work if there are virtual functions with + # decltype() or other things that use parentheses, but csearch suggests + # that this is rare. + end_col = -1 + end_line = -1 + start_col = len(virtual.group(2)) + for start_line in range(linenum, min(linenum + 3, clean_lines.NumLines())): + line = clean_lines.elided[start_line][start_col:] + parameter_list = Match(r'^([^(]*)\(', line) + if parameter_list: + # Match parentheses to find the end of the parameter list + (_, end_line, end_col) = CloseExpression( + clean_lines, start_line, + start_col + len(parameter_list.group(1))) + break + start_col = 0 - if end_col < 0: - return # Couldn't find end of parameter list, give up + if end_col < 0: + return # Couldn't find end of parameter list, give up - # Look for "override" or "final" after the parameter list - # (possibly on the next few lines). - for i in range(end_line, min(end_line + 3, clean_lines.NumLines())): - line = clean_lines.elided[i][end_col:] - match = Search(r'\b(override|final)\b', line) - if match: - error(filename, linenum, 'readability/inheritance', 4, - ('"virtual" is redundant since function is ' - 'already declared as "%s"' % match.group(1))) + # Look for "override" or "final" after the parameter list + # (possibly on the next few lines). + for i in range(end_line, min(end_line + 3, clean_lines.NumLines())): + line = clean_lines.elided[i][end_col:] + match = Search(r'\b(override|final)\b', line) + if match: + error(filename, linenum, 'readability/inheritance', 4, + ('"virtual" is redundant since function is ' + 'already declared as "%s"' % match.group(1))) - # Set end_col to check whole lines after we are done with the - # first line. - end_col = 0 - if Search(r'[^\w]\s*$', line): - break + # Set end_col to check whole lines after we are done with the + # first line. + end_col = 0 + if Search(r'[^\w]\s*$', line): + break def CheckRedundantOverrideOrFinal(filename, clean_lines, linenum, error): - """Check if line contains a redundant "override" or "final" virt-specifier. + """Check if line contains a redundant "override" or "final" virt-specifier. Args: filename: The name of the current file. @@ -5855,32 +6010,30 @@ def CheckRedundantOverrideOrFinal(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - # Look for closing parenthesis nearby. We need one to confirm where - # the declarator ends and where the virt-specifier starts to avoid - # false positives. - line = clean_lines.elided[linenum] - declarator_end = line.rfind(')') - if declarator_end >= 0: - fragment = line[declarator_end:] - else: - if linenum > 1 and clean_lines.elided[linenum - 1].rfind(')') >= 0: - fragment = line + # Look for closing parenthesis nearby. We need one to confirm where + # the declarator ends and where the virt-specifier starts to avoid + # false positives. + line = clean_lines.elided[linenum] + declarator_end = line.rfind(')') + if declarator_end >= 0: + fragment = line[declarator_end:] else: - return - - # Check that at most one of "override" or "final" is present, not both - if Search(r'\boverride\b', fragment) and Search(r'\bfinal\b', fragment): - error(filename, linenum, 'readability/inheritance', 4, - ('"override" is redundant since function is ' - 'already declared as "final"')) - + if linenum > 1 and clean_lines.elided[linenum - 1].rfind(')') >= 0: + fragment = line + else: + return + # Check that at most one of "override" or "final" is present, not both + if Search(r'\boverride\b', fragment) and Search(r'\bfinal\b', fragment): + error(filename, linenum, 'readability/inheritance', 4, + ('"override" is redundant since function is ' + 'already declared as "final"')) # Returns true if we are at a new block, and it is directly # inside of a namespace. def IsBlockInNameSpace(nesting_state, is_forward_declaration): - """Checks that the new block is directly in a namespace. + """Checks that the new block is directly in a namespace. Args: nesting_state: The _NestingState object that contains info about our state. @@ -5888,21 +6041,21 @@ def IsBlockInNameSpace(nesting_state, is_forward_declaration): Returns: Whether or not the new block is directly in a namespace. """ - if is_forward_declaration: - if len(nesting_state.stack) >= 1 and ( - isinstance(nesting_state.stack[-1], _NamespaceInfo)): - return True - else: - return False + if is_forward_declaration: + if len(nesting_state.stack) >= 1 and (isinstance( + nesting_state.stack[-1], _NamespaceInfo)): + return True + else: + return False - return (len(nesting_state.stack) > 1 and - nesting_state.stack[-1].check_namespace_indentation and - isinstance(nesting_state.stack[-2], _NamespaceInfo)) + return (len(nesting_state.stack) > 1 + and nesting_state.stack[-1].check_namespace_indentation + and isinstance(nesting_state.stack[-2], _NamespaceInfo)) def ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item, raw_lines_no_comments, linenum): - """This method determines if we should apply our namespace indentation check. + """This method determines if we should apply our namespace indentation check. Args: nesting_state: The current nesting state. @@ -5917,17 +6070,17 @@ def ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item, only works for classes and namespaces inside of a namespace. """ - is_forward_declaration = IsForwardClassDeclaration(raw_lines_no_comments, - linenum) + is_forward_declaration = IsForwardClassDeclaration(raw_lines_no_comments, + linenum) - if not (is_namespace_indent_item or is_forward_declaration): - return False + if not (is_namespace_indent_item or is_forward_declaration): + return False - # If we are in a macro, we do not want to check the namespace indentation. - if IsMacroDefinition(raw_lines_no_comments, linenum): - return False + # If we are in a macro, we do not want to check the namespace indentation. + if IsMacroDefinition(raw_lines_no_comments, linenum): + return False - return IsBlockInNameSpace(nesting_state, is_forward_declaration) + return IsBlockInNameSpace(nesting_state, is_forward_declaration) # Call this method if the line is directly inside of a namespace. @@ -5935,16 +6088,22 @@ def ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item, # an inner namespace, it cannot be indented. def CheckItemIndentationInNamespace(filename, raw_lines_no_comments, linenum, error): - line = raw_lines_no_comments[linenum] - if Match(r'^\s+', line): - error(filename, linenum, 'runtime/indentation_namespace', 4, - 'Do not indent within a namespace') + line = raw_lines_no_comments[linenum] + if Match(r'^\s+', line): + error(filename, linenum, 'runtime/indentation_namespace', 4, + 'Do not indent within a namespace') -def ProcessLine(filename, file_extension, clean_lines, line, - include_state, function_state, nesting_state, error, +def ProcessLine(filename, + file_extension, + clean_lines, + line, + include_state, + function_state, + nesting_state, + error, extra_check_functions=[]): - """Processes a single line in the file. + """Processes a single line in the file. Args: filename: Filename of the file that is being processed. @@ -5962,31 +6121,33 @@ def ProcessLine(filename, file_extension, clean_lines, line, run on each source line. Each function takes 4 arguments: filename, clean_lines, line, error """ - raw_lines = clean_lines.raw_lines - ParseNolintSuppressions(filename, raw_lines[line], line, error) - nesting_state.Update(filename, clean_lines, line, error) - CheckForNamespaceIndentation(filename, nesting_state, clean_lines, line, - error) - if nesting_state.InAsmBlock(): return - CheckForFunctionLengths(filename, clean_lines, line, function_state, error) - CheckForMultilineCommentsAndStrings(filename, clean_lines, line, error) - CheckStyle(filename, clean_lines, line, file_extension, nesting_state, error) - CheckLanguage(filename, clean_lines, line, file_extension, include_state, - nesting_state, error) - CheckForNonConstReference(filename, clean_lines, line, nesting_state, error) - CheckForNonStandardConstructs(filename, clean_lines, line, - nesting_state, error) - CheckVlogArguments(filename, clean_lines, line, error) - CheckPosixThreading(filename, clean_lines, line, error) - CheckInvalidIncrement(filename, clean_lines, line, error) - CheckMakePairUsesDeduction(filename, clean_lines, line, error) - CheckRedundantVirtual(filename, clean_lines, line, error) - CheckRedundantOverrideOrFinal(filename, clean_lines, line, error) - for check_fn in extra_check_functions: - check_fn(filename, clean_lines, line, error) + raw_lines = clean_lines.raw_lines + ParseNolintSuppressions(filename, raw_lines[line], line, error) + nesting_state.Update(filename, clean_lines, line, error) + CheckForNamespaceIndentation(filename, nesting_state, clean_lines, line, + error) + if nesting_state.InAsmBlock(): return + CheckForFunctionLengths(filename, clean_lines, line, function_state, error) + CheckForMultilineCommentsAndStrings(filename, clean_lines, line, error) + CheckStyle(filename, clean_lines, line, file_extension, nesting_state, + error) + CheckLanguage(filename, clean_lines, line, file_extension, include_state, + nesting_state, error) + CheckForNonConstReference(filename, clean_lines, line, nesting_state, error) + CheckForNonStandardConstructs(filename, clean_lines, line, nesting_state, + error) + CheckVlogArguments(filename, clean_lines, line, error) + CheckPosixThreading(filename, clean_lines, line, error) + CheckInvalidIncrement(filename, clean_lines, line, error) + CheckMakePairUsesDeduction(filename, clean_lines, line, error) + CheckRedundantVirtual(filename, clean_lines, line, error) + CheckRedundantOverrideOrFinal(filename, clean_lines, line, error) + for check_fn in extra_check_functions: + check_fn(filename, clean_lines, line, error) + def FlagCxx11Features(filename, clean_lines, linenum, error): - """Flag those c++11 features that we only allow in certain places. + """Flag those c++11 features that we only allow in certain places. Args: filename: The name of the current file. @@ -5994,51 +6155,53 @@ def FlagCxx11Features(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] + line = clean_lines.elided[linenum] - include = Match(r'\s*#\s*include\s+[<"]([^<"]+)[">]', line) + include = Match(r'\s*#\s*include\s+[<"]([^<"]+)[">]', line) - # Flag unapproved C++ TR1 headers. - if include and include.group(1).startswith('tr1/'): - error(filename, linenum, 'build/c++tr1', 5, - ('C++ TR1 headers such as <%s> are unapproved.') % include.group(1)) + # Flag unapproved C++ TR1 headers. + if include and include.group(1).startswith('tr1/'): + error(filename, linenum, 'build/c++tr1', 5, + ('C++ TR1 headers such as <%s> are unapproved.') % + include.group(1)) - # Flag unapproved C++11 headers. - if include and include.group(1) in ('cfenv', - 'condition_variable', - 'fenv.h', - 'future', - 'mutex', - 'thread', - 'chrono', - 'ratio', - 'regex', - 'system_error', - ): - error(filename, linenum, 'build/c++11', 5, - ('<%s> is an unapproved C++11 header.') % include.group(1)) + # Flag unapproved C++11 headers. + if include and include.group(1) in ( + 'cfenv', + 'condition_variable', + 'fenv.h', + 'future', + 'mutex', + 'thread', + 'chrono', + 'ratio', + 'regex', + 'system_error', + ): + error(filename, linenum, 'build/c++11', 5, + ('<%s> is an unapproved C++11 header.') % include.group(1)) - # The only place where we need to worry about C++11 keywords and library - # features in preprocessor directives is in macro definitions. - if Match(r'\s*#', line) and not Match(r'\s*#\s*define\b', line): return + # The only place where we need to worry about C++11 keywords and library + # features in preprocessor directives is in macro definitions. + if Match(r'\s*#', line) and not Match(r'\s*#\s*define\b', line): return - # These are classes and free functions. The classes are always - # mentioned as std::*, but we only catch the free functions if - # they're not found by ADL. They're alphabetical by header. - for top_name in ( - # type_traits - 'alignment_of', - 'aligned_union', - ): - if Search(r'\bstd::%s\b' % top_name, line): - error(filename, linenum, 'build/c++11', 5, - ('std::%s is an unapproved C++11 class or function. Send c-style ' - 'an example of where it would make your code more readable, and ' - 'they may let you use it.') % top_name) + # These are classes and free functions. The classes are always + # mentioned as std::*, but we only catch the free functions if + # they're not found by ADL. They're alphabetical by header. + for top_name in ( + # type_traits + 'alignment_of', + 'aligned_union', + ): + if Search(r'\bstd::%s\b' % top_name, line): + error(filename, linenum, 'build/c++11', 5, ( + 'std::%s is an unapproved C++11 class or function. Send c-style ' + 'an example of where it would make your code more readable, and ' + 'they may let you use it.') % top_name) def FlagCxx14Features(filename, clean_lines, linenum, error): - """Flag those C++14 features that we restrict. + """Flag those C++14 features that we restrict. Args: filename: The name of the current file. @@ -6046,19 +6209,22 @@ def FlagCxx14Features(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] + line = clean_lines.elided[linenum] - include = Match(r'\s*#\s*include\s+[<"]([^<"]+)[">]', line) + include = Match(r'\s*#\s*include\s+[<"]([^<"]+)[">]', line) - # Flag unapproved C++14 headers. - if include and include.group(1) in ('scoped_allocator', 'shared_mutex'): - error(filename, linenum, 'build/c++14', 5, - ('<%s> is an unapproved C++14 header.') % include.group(1)) + # Flag unapproved C++14 headers. + if include and include.group(1) in ('scoped_allocator', 'shared_mutex'): + error(filename, linenum, 'build/c++14', 5, + ('<%s> is an unapproved C++14 header.') % include.group(1)) -def ProcessFileData(filename, file_extension, lines, error, +def ProcessFileData(filename, + file_extension, + lines, + error, extra_check_functions=[]): - """Performs lint checks and reports any errors to the given error function. + """Performs lint checks and reports any errors to the given error function. Args: filename: Filename of the file that is being processed. @@ -6071,44 +6237,44 @@ def ProcessFileData(filename, file_extension, lines, error, run on each source line. Each function takes 4 arguments: filename, clean_lines, line, error """ - lines = (['// marker so line numbers and indices both start at 1'] + lines + - ['// marker so line numbers end in a known way']) + lines = (['// marker so line numbers and indices both start at 1'] + lines + + ['// marker so line numbers end in a known way']) - include_state = _IncludeState() - function_state = _FunctionState() - nesting_state = NestingState() + include_state = _IncludeState() + function_state = _FunctionState() + nesting_state = NestingState() - ResetNolintSuppressions() + ResetNolintSuppressions() - CheckForCopyright(filename, lines, error) - ProcessGlobalSuppresions(lines) - RemoveMultiLineComments(filename, lines, error) - clean_lines = CleansedLines(lines) + CheckForCopyright(filename, lines, error) + ProcessGlobalSuppresions(lines) + RemoveMultiLineComments(filename, lines, error) + clean_lines = CleansedLines(lines) - if file_extension == 'h': - CheckForHeaderGuard(filename, clean_lines, error) + if file_extension == 'h': + CheckForHeaderGuard(filename, clean_lines, error) - for line in range(clean_lines.NumLines()): - ProcessLine(filename, file_extension, clean_lines, line, - include_state, function_state, nesting_state, error, - extra_check_functions) - FlagCxx11Features(filename, clean_lines, line, error) - nesting_state.CheckCompletedBlocks(filename, error) + for line in range(clean_lines.NumLines()): + ProcessLine(filename, file_extension, clean_lines, line, include_state, + function_state, nesting_state, error, extra_check_functions) + FlagCxx11Features(filename, clean_lines, line, error) + nesting_state.CheckCompletedBlocks(filename, error) - CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error) + CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error) - # Check that the .cc file has included its header if it exists. - if _IsSourceExtension(file_extension): - CheckHeaderFileIncluded(filename, include_state, error) + # Check that the .cc file has included its header if it exists. + if _IsSourceExtension(file_extension): + CheckHeaderFileIncluded(filename, include_state, error) - # We check here rather than inside ProcessLine so that we see raw - # lines rather than "cleaned" lines. - CheckForBadCharacters(filename, lines, error) + # We check here rather than inside ProcessLine so that we see raw + # lines rather than "cleaned" lines. + CheckForBadCharacters(filename, lines, error) + + CheckForNewlineAtEOF(filename, lines, error) - CheckForNewlineAtEOF(filename, lines, error) def ProcessConfigOverrides(filename): - """ Loads the configuration files and processes the config overrides. + """ Loads the configuration files and processes the config overrides. Args: filename: The name of the file being processed by the linter. @@ -6117,74 +6283,77 @@ def ProcessConfigOverrides(filename): False if the current |filename| should not be processed further. """ - abs_filename = os.path.abspath(filename) - cfg_filters = [] - keep_looking = True - while keep_looking: - abs_path, base_name = os.path.split(abs_filename) - if not base_name: - break # Reached the root directory. + abs_filename = os.path.abspath(filename) + cfg_filters = [] + keep_looking = True + while keep_looking: + abs_path, base_name = os.path.split(abs_filename) + if not base_name: + break # Reached the root directory. - cfg_file = os.path.join(abs_path, "CPPLINT.cfg") - abs_filename = abs_path - if not os.path.isfile(cfg_file): - continue - - try: - with open(cfg_file) as file_handle: - for line in file_handle: - line, _, _ = line.partition('#') # Remove comments. - if not line.strip(): + cfg_file = os.path.join(abs_path, "CPPLINT.cfg") + abs_filename = abs_path + if not os.path.isfile(cfg_file): continue - name, _, val = line.partition('=') - name = name.strip() - val = val.strip() - if name == 'set noparent': - keep_looking = False - elif name == 'filter': - cfg_filters.append(val) - elif name == 'exclude_files': - # When matching exclude_files pattern, use the base_name of - # the current file name or the directory name we are processing. - # For example, if we are checking for lint errors in /foo/bar/baz.cc - # and we found the .cfg file at /foo/CPPLINT.cfg, then the config - # file's "exclude_files" filter is meant to be checked against "bar" - # and not "baz" nor "bar/baz.cc". - if base_name: - pattern = re.compile(val) - if pattern.match(base_name): - sys.stderr.write('Ignoring "%s": file excluded by "%s". ' - 'File path component "%s" matches ' - 'pattern "%s"\n' % - (filename, cfg_file, base_name, val)) - return False - elif name == 'linelength': - global _line_length - try: - _line_length = int(val) - except ValueError: - sys.stderr.write('Line length must be numeric.') - else: + try: + with open(cfg_file) as file_handle: + for line in file_handle: + line, _, _ = line.partition('#') # Remove comments. + if not line.strip(): + continue + + name, _, val = line.partition('=') + name = name.strip() + val = val.strip() + if name == 'set noparent': + keep_looking = False + elif name == 'filter': + cfg_filters.append(val) + elif name == 'exclude_files': + # When matching exclude_files pattern, use the base_name + # of the current file name or the directory name we are + # processing. For example, if we are checking for lint + # errors in /foo/bar/baz.cc and we found the .cfg file + # at /foo/CPPLINT.cfg, then the config file's + # "exclude_files" filter is meant to be checked against + # "bar" and not "baz" nor "bar/baz.cc". + if base_name: + pattern = re.compile(val) + if pattern.match(base_name): + sys.stderr.write( + 'Ignoring "%s": file excluded by "%s". ' + 'File path component "%s" matches ' + 'pattern "%s"\n' % + (filename, cfg_file, base_name, val)) + return False + elif name == 'linelength': + global _line_length + try: + _line_length = int(val) + except ValueError: + sys.stderr.write('Line length must be numeric.') + else: + sys.stderr.write( + 'Invalid configuration option (%s) in file %s\n' % + (name, cfg_file)) + + except IOError: sys.stderr.write( - 'Invalid configuration option (%s) in file %s\n' % - (name, cfg_file)) + "Skipping config file '%s': Can't open for reading\n" % + cfg_file) + keep_looking = False - except IOError: - sys.stderr.write( - "Skipping config file '%s': Can't open for reading\n" % cfg_file) - keep_looking = False + # Apply all the accumulated filters in reverse order (top-level directory + # config options having the least priority). + for filter in reversed(cfg_filters): + _AddFilters(filter) - # Apply all the accumulated filters in reverse order (top-level directory - # config options having the least priority). - for filter in reversed(cfg_filters): - _AddFilters(filter) - - return True + return True def ProcessFile(filename, vlevel, extra_check_functions=[]): - """Does google-lint on a single file. + """Does google-lint on a single file. Args: filename: The name of the file to parse. @@ -6197,104 +6366,104 @@ def ProcessFile(filename, vlevel, extra_check_functions=[]): arguments: filename, clean_lines, line, error """ - _SetVerboseLevel(vlevel) - _BackupFilters() + _SetVerboseLevel(vlevel) + _BackupFilters() - if not ProcessConfigOverrides(filename): - _RestoreFilters() - return + if not ProcessConfigOverrides(filename): + _RestoreFilters() + return - lf_lines = [] - crlf_lines = [] - try: - # Support the UNIX convention of using "-" for stdin. Note that - # we are not opening the file with universal newline support - # (which codecs doesn't support anyway), so the resulting lines do - # contain trailing '\r' characters if we are reading a file that - # has CRLF endings. - # If after the split a trailing '\r' is present, it is removed - # below. - if filename == '-': - lines = codecs.StreamReaderWriter(sys.stdin, - codecs.getreader('utf8'), - codecs.getwriter('utf8'), - 'replace').read().split('\n') + lf_lines = [] + crlf_lines = [] + try: + # Support the UNIX convention of using "-" for stdin. Note that + # we are not opening the file with universal newline support + # (which codecs doesn't support anyway), so the resulting lines do + # contain trailing '\r' characters if we are reading a file that + # has CRLF endings. + # If after the split a trailing '\r' is present, it is removed + # below. + if filename == '-': + lines = codecs.StreamReaderWriter(sys.stdin, + codecs.getreader('utf8'), + codecs.getwriter('utf8'), + 'replace').read().split('\n') + else: + with codecs.open(filename, 'r', 'utf8', 'replace') as stream: + lines = stream.read().split('\n') + + # Remove trailing '\r'. + # The -1 accounts for the extra trailing blank line we get from split() + for linenum in range(len(lines) - 1): + if lines[linenum].endswith('\r'): + lines[linenum] = lines[linenum].rstrip('\r') + crlf_lines.append(linenum + 1) + else: + lf_lines.append(linenum + 1) + + except IOError: + sys.stderr.write("Skipping input '%s': Can't open for reading\n" % + filename) + _RestoreFilters() + return + + # Note, if no dot is found, this will give the entire filename as the ext. + file_extension = filename[filename.rfind('.') + 1:] + + # When reading from stdin, the extension is unknown, so no cpplint tests + # should rely on the extension. + if filename != '-' and file_extension not in _valid_extensions: + sys.stderr.write('Ignoring %s; not a valid file name ' + '(%s)\n' % (filename, ', '.join(_valid_extensions))) else: - with codecs.open(filename, 'r', 'utf8', 'replace') as stream: - lines = stream.read().split('\n') + ProcessFileData(filename, file_extension, lines, Error, + extra_check_functions) - # Remove trailing '\r'. - # The -1 accounts for the extra trailing blank line we get from split() - for linenum in range(len(lines) - 1): - if lines[linenum].endswith('\r'): - lines[linenum] = lines[linenum].rstrip('\r') - crlf_lines.append(linenum + 1) - else: - lf_lines.append(linenum + 1) + # If end-of-line sequences are a mix of LF and CR-LF, issue + # warnings on the lines with CR. + # + # Don't issue any warnings if all lines are uniformly LF or CR-LF, + # since critique can handle these just fine, and the style guide + # doesn't dictate a particular end of line sequence. + # + # We can't depend on os.linesep to determine what the desired + # end-of-line sequence should be, since that will return the + # server-side end-of-line sequence. + if lf_lines and crlf_lines: + # Warn on every line with CR. An alternative approach might be to + # check whether the file is mostly CRLF or just LF, and warn on the + # minority, we bias toward LF here since most tools prefer LF. + for linenum in crlf_lines: + Error(filename, linenum, 'whitespace/newline', 1, + 'Unexpected \\r (^M) found; better to use only \\n') - except IOError: - sys.stderr.write( - "Skipping input '%s': Can't open for reading\n" % filename) _RestoreFilters() - return - - # Note, if no dot is found, this will give the entire filename as the ext. - file_extension = filename[filename.rfind('.') + 1:] - - # When reading from stdin, the extension is unknown, so no cpplint tests - # should rely on the extension. - if filename != '-' and file_extension not in _valid_extensions: - sys.stderr.write('Ignoring %s; not a valid file name ' - '(%s)\n' % (filename, ', '.join(_valid_extensions))) - else: - ProcessFileData(filename, file_extension, lines, Error, - extra_check_functions) - - # If end-of-line sequences are a mix of LF and CR-LF, issue - # warnings on the lines with CR. - # - # Don't issue any warnings if all lines are uniformly LF or CR-LF, - # since critique can handle these just fine, and the style guide - # doesn't dictate a particular end of line sequence. - # - # We can't depend on os.linesep to determine what the desired - # end-of-line sequence should be, since that will return the - # server-side end-of-line sequence. - if lf_lines and crlf_lines: - # Warn on every line with CR. An alternative approach might be to - # check whether the file is mostly CRLF or just LF, and warn on the - # minority, we bias toward LF here since most tools prefer LF. - for linenum in crlf_lines: - Error(filename, linenum, 'whitespace/newline', 1, - 'Unexpected \\r (^M) found; better to use only \\n') - - _RestoreFilters() def PrintUsage(message): - """Prints a brief usage string and exits, optionally with an error message. + """Prints a brief usage string and exits, optionally with an error message. Args: message: The optional error message. """ - sys.stderr.write(_USAGE) - if message: - sys.exit('\nFATAL ERROR: ' + message) - else: - sys.exit(1) + sys.stderr.write(_USAGE) + if message: + sys.exit('\nFATAL ERROR: ' + message) + else: + sys.exit(1) def PrintCategories(): - """Prints a list of all the error-categories used by error messages. + """Prints a list of all the error-categories used by error messages. These are the categories used to filter messages via --filter. """ - sys.stderr.write(''.join(' %s\n' % cat for cat in _ERROR_CATEGORIES)) - sys.exit(0) + sys.stderr.write(''.join(' %s\n' % cat for cat in _ERROR_CATEGORIES)) + sys.exit(0) def ParseArguments(args): - """Parses the command line arguments. + """Parses the command line arguments. This may set the output format and verbosity level as side-effects. @@ -6304,91 +6473,101 @@ def ParseArguments(args): Returns: The list of filenames to lint. """ - try: - (opts, filenames) = getopt.getopt(args, '', ['help', 'output=', 'verbose=', - 'headers=', # We understand but ignore headers. - 'counting=', - 'filter=', - 'root=', - 'linelength=', - 'extensions=', - 'project_root=', - 'repository=']) - except getopt.GetoptError as e: - PrintUsage('Invalid arguments: {}'.format(e)) + try: + (opts, filenames) = getopt.getopt( + args, + '', + [ + 'help', + 'output=', + 'verbose=', + 'headers=', # We understand but ignore headers. + 'counting=', + 'filter=', + 'root=', + 'linelength=', + 'extensions=', + 'project_root=', + 'repository=' + ]) + except getopt.GetoptError as e: + PrintUsage('Invalid arguments: {}'.format(e)) - verbosity = _VerboseLevel() - output_format = _OutputFormat() - filters = '' - counting_style = '' + verbosity = _VerboseLevel() + output_format = _OutputFormat() + filters = '' + counting_style = '' - for (opt, val) in opts: - if opt == '--help': - PrintUsage(None) - elif opt == '--output': - if val not in ('emacs', 'vs7', 'eclipse'): - PrintUsage('The only allowed output formats are emacs, vs7 and eclipse.') - output_format = val - elif opt == '--verbose': - verbosity = int(val) - elif opt == '--filter': - filters = val - if not filters: - PrintCategories() - elif opt == '--counting': - if val not in ('total', 'toplevel', 'detailed'): - PrintUsage('Valid counting options are total, toplevel, and detailed') - counting_style = val - elif opt == '--root': - global _root - _root = val - elif opt == '--project_root' or opt == "--repository": - global _project_root - _project_root = val - if not os.path.isabs(_project_root): - PrintUsage('Project root must be an absolute path.') - elif opt == '--linelength': - global _line_length - try: - _line_length = int(val) - except ValueError: - PrintUsage('Line length must be digits.') - elif opt == '--extensions': - global _valid_extensions - try: - _valid_extensions = set(val.split(',')) - except ValueError: - PrintUsage('Extensions must be comma separated list.') + for (opt, val) in opts: + if opt == '--help': + PrintUsage(None) + elif opt == '--output': + if val not in ('emacs', 'vs7', 'eclipse'): + PrintUsage( + 'The only allowed output formats are emacs, vs7 and eclipse.' + ) + output_format = val + elif opt == '--verbose': + verbosity = int(val) + elif opt == '--filter': + filters = val + if not filters: + PrintCategories() + elif opt == '--counting': + if val not in ('total', 'toplevel', 'detailed'): + PrintUsage( + 'Valid counting options are total, toplevel, and detailed') + counting_style = val + elif opt == '--root': + global _root + _root = val + elif opt == '--project_root' or opt == "--repository": + global _project_root + _project_root = val + if not os.path.isabs(_project_root): + PrintUsage('Project root must be an absolute path.') + elif opt == '--linelength': + global _line_length + try: + _line_length = int(val) + except ValueError: + PrintUsage('Line length must be digits.') + elif opt == '--extensions': + global _valid_extensions + try: + _valid_extensions = set(val.split(',')) + except ValueError: + PrintUsage('Extensions must be comma separated list.') - if not filenames: - PrintUsage('No files were specified.') + if not filenames: + PrintUsage('No files were specified.') - _SetOutputFormat(output_format) - _SetVerboseLevel(verbosity) - _SetFilters(filters) - _SetCountingStyle(counting_style) + _SetOutputFormat(output_format) + _SetVerboseLevel(verbosity) + _SetFilters(filters) + _SetCountingStyle(counting_style) - return filenames + return filenames def main(): - filenames = ParseArguments(sys.argv[1:]) + filenames = ParseArguments(sys.argv[1:]) - # Change stderr to write with replacement characters so we don't die - # if we try to print something containing non-ASCII characters. - # We use sys.stderr.buffer in Python 3, since StreamReaderWriter writes bytes - # to the specified stream. - sys.stderr = codecs.StreamReaderWriter( - getattr(sys.stderr, 'buffer', sys.stderr), - codecs.getreader('utf8'), codecs.getwriter('utf8'), 'replace') + # Change stderr to write with replacement characters so we don't die + # if we try to print something containing non-ASCII characters. + # We use sys.stderr.buffer in Python 3, since StreamReaderWriter writes + # bytes to the specified stream. + sys.stderr = codecs.StreamReaderWriter( + getattr(sys.stderr, 'buffer', sys.stderr), codecs.getreader('utf8'), + codecs.getwriter('utf8'), 'replace') - _cpplint_state.ResetErrorCounts() - for filename in filenames: - ProcessFile(filename, _cpplint_state.verbose_level) - _cpplint_state.PrintErrorCounts() + _cpplint_state.ResetErrorCounts() + for filename in filenames: + ProcessFile(filename, _cpplint_state.verbose_level) + _cpplint_state.PrintErrorCounts() - sys.exit(_cpplint_state.error_count > 0) + sys.exit(_cpplint_state.error_count > 0) if __name__ == '__main__': - main() + main() diff --git a/cpplint_chromium.py b/cpplint_chromium.py index 9318a8657c..8604763af3 100755 --- a/cpplint_chromium.py +++ b/cpplint_chromium.py @@ -36,7 +36,7 @@ _RE_PATTERN_POINTER_DECLARATION_WHITESPACE = re.compile( def CheckPointerDeclarationWhitespace(filename, clean_lines, linenum, error): - """Checks for Foo *foo declarations. + """Checks for Foo *foo declarations. Args: filename: The name of the current file. @@ -44,9 +44,10 @@ def CheckPointerDeclarationWhitespace(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - matched = _RE_PATTERN_POINTER_DECLARATION_WHITESPACE.match(line) - if matched: - error(filename, linenum, 'whitespace/declaration', 3, - 'Declaration has space between type name and %s in %s' % - (matched.group('pointer_operator'), matched.group(0).strip())) + line = clean_lines.elided[linenum] + matched = _RE_PATTERN_POINTER_DECLARATION_WHITESPACE.match(line) + if matched: + error( + filename, linenum, 'whitespace/declaration', 3, + 'Declaration has space between type name and %s in %s' % + (matched.group('pointer_operator'), matched.group(0).strip())) diff --git a/cros b/cros index 4874bf166c..a791f13acf 100755 --- a/cros +++ b/cros @@ -2,7 +2,6 @@ # Copyright 2011 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Wrapper for chromite tools. The script is intend to be symlinked to any number of chromite tools, attempts @@ -17,7 +16,6 @@ import pathlib import subprocess import sys - # Min version of Python that we *want*. We warn for older versions. MIN_PYTHON_VER_SOFT = (3, 8) # Min version of Python that we *require*. We abort for older versions. @@ -30,91 +28,95 @@ CIPD_CACHE_DIR = DEPOT_TOOLS_DIR / '.cipd_bin_cros_python2' def _FindChromite(path): - """Find the chromite dir in a repo, gclient, or submodule checkout.""" - path = os.path.abspath(path) - # Depending on the checkout type (whether repo chromeos or gclient chrome) - # Chromite lives in a different location. - roots = ( - ('.repo', 'chromite/.git'), - ('.gclient', 'src/third_party/chromite/.git'), - ('src/.gitmodules', 'src/third_party/chromite/.git'), - ) + """Find the chromite dir in a repo, gclient, or submodule checkout.""" + path = os.path.abspath(path) + # Depending on the checkout type (whether repo chromeos or gclient chrome) + # Chromite lives in a different location. + roots = ( + ('.repo', 'chromite/.git'), + ('.gclient', 'src/third_party/chromite/.git'), + ('src/.gitmodules', 'src/third_party/chromite/.git'), + ) - while path != '/': - for root, chromite_git_dir in roots: - if all(os.path.exists(os.path.join(path, x)) - for x in [root, chromite_git_dir]): - return os.path.dirname(os.path.join(path, chromite_git_dir)) - path = os.path.dirname(path) - return None + while path != '/': + for root, chromite_git_dir in roots: + if all( + os.path.exists(os.path.join(path, x)) + for x in [root, chromite_git_dir]): + return os.path.dirname(os.path.join(path, chromite_git_dir)) + path = os.path.dirname(path) + return None def _MissingErrorOut(target): - sys.stderr.write("""ERROR: Couldn't find the chromite tool %s. + sys.stderr.write("""ERROR: Couldn't find the chromite tool %s. Please change to a directory inside your ChromiumOS source tree and retry. If you need to setup a ChromiumOS source tree, see https://chromium.googlesource.com/chromiumos/docs/+/HEAD/developer_guide.md """ % target) - return 127 + return 127 def _CheckPythonVersion(): - """Verify active Python is new enough.""" - if sys.version_info >= MIN_PYTHON_VER_SOFT: - return + """Verify active Python is new enough.""" + if sys.version_info >= MIN_PYTHON_VER_SOFT: + return - progname = os.path.basename(sys.argv[0]) - print('%s: ChromiumOS requires Python-%s+, but "%s" is "%s"' % - (progname, '.'.join(str(x) for x in MIN_PYTHON_VER_SOFT), - sys.executable, sys.version.replace('\n', ' ')), - file=sys.stderr) - if sys.version_info < MIN_PYTHON_VER_HARD: - print('%s: fatal: giving up since Python is too old.' % (progname,), + progname = os.path.basename(sys.argv[0]) + print('%s: ChromiumOS requires Python-%s+, but "%s" is "%s"' % + (progname, '.'.join(str(x) for x in MIN_PYTHON_VER_SOFT), + sys.executable, sys.version.replace('\n', ' ')), file=sys.stderr) - sys.exit(1) + if sys.version_info < MIN_PYTHON_VER_HARD: + print('%s: fatal: giving up since Python is too old.' % (progname, ), + file=sys.stderr) + sys.exit(1) - print('warning: temporarily continuing anyways; you must upgrade soon to ' - 'maintain support.', file=sys.stderr) + print( + 'warning: temporarily continuing anyways; you must upgrade soon to ' + 'maintain support.', + file=sys.stderr) def _BootstrapVpython27(): - """Installs the vpython2.7 packages into the cipd cache directory.""" - subprocess.run(['cipd', 'ensure', - '-log-level', 'info', - '-ensure-file', - DEPOT_TOOLS_DIR / 'cipd_manifest_cros_python2.txt', - '-root', CIPD_CACHE_DIR], - check=True) + """Installs the vpython2.7 packages into the cipd cache directory.""" + subprocess.run([ + 'cipd', 'ensure', '-log-level', 'info', '-ensure-file', + DEPOT_TOOLS_DIR / 'cipd_manifest_cros_python2.txt', '-root', + CIPD_CACHE_DIR + ], + check=True) def main(): - _CheckPythonVersion() + _CheckPythonVersion() - chromite_dir = _FindChromite(os.getcwd()) - target = os.path.basename(sys.argv[0]) - if chromite_dir is None: - return _MissingErrorOut(target) + chromite_dir = _FindChromite(os.getcwd()) + target = os.path.basename(sys.argv[0]) + if chromite_dir is None: + return _MissingErrorOut(target) - path = os.path.join(chromite_dir, 'bin', target) + path = os.path.join(chromite_dir, 'bin', target) - # Check to see if this is a script requiring vpython2.7. - with open(path, 'rb') as fp: - shebang = next(fp).strip() - interpreter = shebang.split()[-1] - if interpreter in (b'python', b'python2', b'python2.7', b'vpython'): - _BootstrapVpython27() - vpython = CIPD_CACHE_DIR / 'vpython' - args = [vpython] - if interpreter != b'vpython': - args.extend(['-vpython-spec', DEPOT_TOOLS_DIR / 'cros_python2.vpython']) - args.append(path) - path = vpython - else: - args = [path] + # Check to see if this is a script requiring vpython2.7. + with open(path, 'rb') as fp: + shebang = next(fp).strip() + interpreter = shebang.split()[-1] + if interpreter in (b'python', b'python2', b'python2.7', b'vpython'): + _BootstrapVpython27() + vpython = CIPD_CACHE_DIR / 'vpython' + args = [vpython] + if interpreter != b'vpython': + args.extend( + ['-vpython-spec', DEPOT_TOOLS_DIR / 'cros_python2.vpython']) + args.append(path) + path = vpython + else: + args = [path] - os.execv(path, args + sys.argv[1:]) + os.execv(path, args + sys.argv[1:]) if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/detect_host_arch.py b/detect_host_arch.py index 4294800fe6..94e2572aad 100755 --- a/detect_host_arch.py +++ b/detect_host_arch.py @@ -2,7 +2,6 @@ # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Outputs host CPU architecture in format recognized by gyp.""" from __future__ import print_function @@ -10,55 +9,59 @@ from __future__ import print_function import platform import re + def HostArch(): - """Returns the host architecture with a predictable string.""" - host_arch = platform.machine().lower() - host_processor = platform.processor().lower() + """Returns the host architecture with a predictable string.""" + host_arch = platform.machine().lower() + host_processor = platform.processor().lower() - # Convert machine type to format recognized by gyp. - if re.match(r'i.86', host_arch) or host_arch == 'i86pc': - host_arch = 'x86' - elif host_arch in ['x86_64', 'amd64']: - host_arch = 'x64' - elif host_arch == 'arm64' or host_arch.startswith('aarch64'): - host_arch = 'arm64' - elif host_arch.startswith('arm'): - host_arch = 'arm' - elif host_arch.startswith('mips64'): - host_arch = 'mips64' - elif host_arch.startswith('mips'): - host_arch = 'mips' - elif host_arch.startswith('ppc') or host_processor == 'powerpc': - host_arch = 'ppc' - elif host_arch.startswith('s390'): - host_arch = 's390' - elif host_arch.startswith('riscv'): - host_arch = 'riscv64' + # Convert machine type to format recognized by gyp. + if re.match(r'i.86', host_arch) or host_arch == 'i86pc': + host_arch = 'x86' + elif host_arch in ['x86_64', 'amd64']: + host_arch = 'x64' + elif host_arch == 'arm64' or host_arch.startswith('aarch64'): + host_arch = 'arm64' + elif host_arch.startswith('arm'): + host_arch = 'arm' + elif host_arch.startswith('mips64'): + host_arch = 'mips64' + elif host_arch.startswith('mips'): + host_arch = 'mips' + elif host_arch.startswith('ppc') or host_processor == 'powerpc': + host_arch = 'ppc' + elif host_arch.startswith('s390'): + host_arch = 's390' + elif host_arch.startswith('riscv'): + host_arch = 'riscv64' - if host_arch == 'arm64': - host_platform = platform.architecture() - if len(host_platform) > 1: - if host_platform[1].lower() == 'windowspe': - # Special case for Windows on Arm: windows-386 packages no longer work - # so use the x64 emulation (this restricts us to Windows 11). Python - # 32-bit returns the host_arch as arm64, 64-bit does not. - return 'x64' + if host_arch == 'arm64': + host_platform = platform.architecture() + if len(host_platform) > 1: + if host_platform[1].lower() == 'windowspe': + # Special case for Windows on Arm: windows-386 packages no + # longer work so use the x64 emulation (this restricts us to + # Windows 11). Python 32-bit returns the host_arch as arm64, + # 64-bit does not. + return 'x64' - # platform.machine is based on running kernel. It's possible to use 64-bit - # kernel with 32-bit userland, e.g. to give linker slightly more memory. - # Distinguish between different userland bitness by querying - # the python binary. - if host_arch == 'x64' and platform.architecture()[0] == '32bit': - host_arch = 'x86' - if host_arch == 'arm64' and platform.architecture()[0] == '32bit': - host_arch = 'arm' + # platform.machine is based on running kernel. It's possible to use 64-bit + # kernel with 32-bit userland, e.g. to give linker slightly more memory. + # Distinguish between different userland bitness by querying + # the python binary. + if host_arch == 'x64' and platform.architecture()[0] == '32bit': + host_arch = 'x86' + if host_arch == 'arm64' and platform.architecture()[0] == '32bit': + host_arch = 'arm' + + return host_arch - return host_arch def DoMain(_): - """Hook to be called from gyp without starting a separate python + """Hook to be called from gyp without starting a separate python interpreter.""" - return HostArch() + return HostArch() + if __name__ == '__main__': - print(DoMain([])) + print(DoMain([])) diff --git a/download_from_google_storage.py b/download_from_google_storage.py index 1396f4a19e..ff114f07b8 100755 --- a/download_from_google_storage.py +++ b/download_from_google_storage.py @@ -21,14 +21,13 @@ import time import subprocess2 - # Env vars that tempdir can be gotten from; minimally, this # needs to match python's tempfile module and match normal # unix standards. _TEMPDIR_ENV_VARS = ('TMPDIR', 'TEMP', 'TMP') -GSUTIL_DEFAULT_PATH = os.path.join( - os.path.dirname(os.path.abspath(__file__)), 'gsutil.py') +GSUTIL_DEFAULT_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'gsutil.py') # Maps sys.platform to what we actually want to call them. PLATFORM_MAPPING = { 'cygwin': 'win', @@ -42,345 +41,368 @@ PLATFORM_MAPPING = { class InvalidFileError(IOError): - pass + pass class InvalidPlatformError(Exception): - pass + pass def GetNormalizedPlatform(): - """Returns the result of sys.platform accounting for cygwin. + """Returns the result of sys.platform accounting for cygwin. Under cygwin, this will always return "win32" like the native Python.""" - if sys.platform == 'cygwin': - return 'win32' - return sys.platform + if sys.platform == 'cygwin': + return 'win32' + return sys.platform + # Common utilities class Gsutil(object): - """Call gsutil with some predefined settings. This is a convenience object, + """Call gsutil with some predefined settings. This is a convenience object, and is also immutable. HACK: This object is used directly by the external script `/win_toolchain/get_toolchain_if_necessary.py` """ - MAX_TRIES = 5 - RETRY_BASE_DELAY = 5.0 - RETRY_DELAY_MULTIPLE = 1.3 - VPYTHON3 = ('vpython3.bat' - if GetNormalizedPlatform() == 'win32' else 'vpython3') + MAX_TRIES = 5 + RETRY_BASE_DELAY = 5.0 + RETRY_DELAY_MULTIPLE = 1.3 + VPYTHON3 = ('vpython3.bat' + if GetNormalizedPlatform() == 'win32' else 'vpython3') - def __init__(self, path, boto_path=None): - if not os.path.exists(path): - raise FileNotFoundError('GSUtil not found in %s' % path) - self.path = path - self.boto_path = boto_path + def __init__(self, path, boto_path=None): + if not os.path.exists(path): + raise FileNotFoundError('GSUtil not found in %s' % path) + self.path = path + self.boto_path = boto_path - def get_sub_env(self): - env = os.environ.copy() - if self.boto_path == os.devnull: - env['AWS_CREDENTIAL_FILE'] = '' - env['BOTO_CONFIG'] = '' - elif self.boto_path: - env['AWS_CREDENTIAL_FILE'] = self.boto_path - env['BOTO_CONFIG'] = self.boto_path + def get_sub_env(self): + env = os.environ.copy() + if self.boto_path == os.devnull: + env['AWS_CREDENTIAL_FILE'] = '' + env['BOTO_CONFIG'] = '' + elif self.boto_path: + env['AWS_CREDENTIAL_FILE'] = self.boto_path + env['BOTO_CONFIG'] = self.boto_path - if PLATFORM_MAPPING[sys.platform] != 'win': - env.update((x, "/tmp") for x in _TEMPDIR_ENV_VARS) + if PLATFORM_MAPPING[sys.platform] != 'win': + env.update((x, "/tmp") for x in _TEMPDIR_ENV_VARS) - return env + return env - def call(self, *args): - cmd = [self.VPYTHON3, self.path] - cmd.extend(args) - return subprocess2.call(cmd, env=self.get_sub_env()) + def call(self, *args): + cmd = [self.VPYTHON3, self.path] + cmd.extend(args) + return subprocess2.call(cmd, env=self.get_sub_env()) - def check_call(self, *args): - cmd = [self.VPYTHON3, self.path] - cmd.extend(args) - ((out, err), code) = subprocess2.communicate( - cmd, - stdout=subprocess2.PIPE, - stderr=subprocess2.PIPE, - env=self.get_sub_env()) + def check_call(self, *args): + cmd = [self.VPYTHON3, self.path] + cmd.extend(args) + ((out, err), code) = subprocess2.communicate(cmd, + stdout=subprocess2.PIPE, + stderr=subprocess2.PIPE, + env=self.get_sub_env()) - out = out.decode('utf-8', 'replace') - err = err.decode('utf-8', 'replace') + out = out.decode('utf-8', 'replace') + err = err.decode('utf-8', 'replace') - # Parse output. - status_code_match = re.search('status=([0-9]+)', err) - if status_code_match: - return (int(status_code_match.group(1)), out, err) - if ('ServiceException: 401 Anonymous' in err): - return (401, out, err) - if ('You are attempting to access protected data with ' - 'no configured credentials.' in err): - return (403, out, err) - if 'matched no objects' in err or 'No URLs matched' in err: - return (404, out, err) - return (code, out, err) + # Parse output. + status_code_match = re.search('status=([0-9]+)', err) + if status_code_match: + return (int(status_code_match.group(1)), out, err) + if ('ServiceException: 401 Anonymous' in err): + return (401, out, err) + if ('You are attempting to access protected data with ' + 'no configured credentials.' in err): + return (403, out, err) + if 'matched no objects' in err or 'No URLs matched' in err: + return (404, out, err) + return (code, out, err) - def check_call_with_retries(self, *args): - delay = self.RETRY_BASE_DELAY - for i in range(self.MAX_TRIES): - code, out, err = self.check_call(*args) - if not code or i == self.MAX_TRIES - 1: - break + def check_call_with_retries(self, *args): + delay = self.RETRY_BASE_DELAY + for i in range(self.MAX_TRIES): + code, out, err = self.check_call(*args) + if not code or i == self.MAX_TRIES - 1: + break - time.sleep(delay) - delay *= self.RETRY_DELAY_MULTIPLE + time.sleep(delay) + delay *= self.RETRY_DELAY_MULTIPLE - return code, out, err + return code, out, err def check_platform(target): - """Checks if any parent directory of target matches (win|mac|linux).""" - assert os.path.isabs(target) - root, target_name = os.path.split(target) - if not target_name: - return None - if target_name in ('linux', 'mac', 'win'): - return target_name - return check_platform(root) + """Checks if any parent directory of target matches (win|mac|linux).""" + assert os.path.isabs(target) + root, target_name = os.path.split(target) + if not target_name: + return None + if target_name in ('linux', 'mac', 'win'): + return target_name + return check_platform(root) def get_sha1(filename): - sha1 = hashlib.sha1() - with open(filename, 'rb') as f: - while True: - # Read in 1mb chunks, so it doesn't all have to be loaded into memory. - chunk = f.read(1024*1024) - if not chunk: - break - sha1.update(chunk) - return sha1.hexdigest() + sha1 = hashlib.sha1() + with open(filename, 'rb') as f: + while True: + # Read in 1mb chunks, so it doesn't all have to be loaded into + # memory. + chunk = f.read(1024 * 1024) + if not chunk: + break + sha1.update(chunk) + return sha1.hexdigest() # Download-specific code starts here + def enumerate_input(input_filename, directory, recursive, ignore_errors, output, sha1_file, auto_platform): - if sha1_file: - if not os.path.exists(input_filename): - if not ignore_errors: - raise FileNotFoundError( - '{} not found when attempting enumerate files to download.'.format( - input_filename)) - print('%s not found.' % input_filename, file=sys.stderr) - with open(input_filename, 'rb') as f: - sha1_match = re.match(b'^([A-Za-z0-9]{40})$', f.read(1024).rstrip()) - if sha1_match: - yield (sha1_match.groups(1)[0].decode('utf-8'), output) - return - if not ignore_errors: - raise InvalidFileError('No sha1 sum found in %s.' % input_filename) - print('No sha1 sum found in %s.' % input_filename, file=sys.stderr) - return - - if not directory: - yield (input_filename, output) - return - - for root, dirs, files in os.walk(input_filename): - if not recursive: - for item in dirs[:]: - dirs.remove(item) - else: - for exclude in ['.svn', '.git']: - if exclude in dirs: - dirs.remove(exclude) - for filename in files: - full_path = os.path.join(root, filename) - if full_path.endswith('.sha1'): - if auto_platform: - # Skip if the platform does not match. - target_platform = check_platform(os.path.abspath(full_path)) - if not target_platform: - err = ('--auto_platform passed in but no platform name found in ' - 'the path of %s' % full_path) + if sha1_file: + if not os.path.exists(input_filename): if not ignore_errors: - raise InvalidFileError(err) - print(err, file=sys.stderr) - continue - current_platform = PLATFORM_MAPPING[sys.platform] - if current_platform != target_platform: - continue + raise FileNotFoundError( + '{} not found when attempting enumerate files to download.'. + format(input_filename)) + print('%s not found.' % input_filename, file=sys.stderr) + with open(input_filename, 'rb') as f: + sha1_match = re.match(b'^([A-Za-z0-9]{40})$', f.read(1024).rstrip()) + if sha1_match: + yield (sha1_match.groups(1)[0].decode('utf-8'), output) + return + if not ignore_errors: + raise InvalidFileError('No sha1 sum found in %s.' % input_filename) + print('No sha1 sum found in %s.' % input_filename, file=sys.stderr) + return - with open(full_path, 'rb') as f: - sha1_match = re.match(b'^([A-Za-z0-9]{40})$', f.read(1024).rstrip()) - if sha1_match: - yield ( - sha1_match.groups(1)[0].decode('utf-8'), - full_path.replace('.sha1', '') - ) + if not directory: + yield (input_filename, output) + return + + for root, dirs, files in os.walk(input_filename): + if not recursive: + for item in dirs[:]: + dirs.remove(item) else: - if not ignore_errors: - raise InvalidFileError('No sha1 sum found in %s.' % filename) - print('No sha1 sum found in %s.' % filename, file=sys.stderr) + for exclude in ['.svn', '.git']: + if exclude in dirs: + dirs.remove(exclude) + for filename in files: + full_path = os.path.join(root, filename) + if full_path.endswith('.sha1'): + if auto_platform: + # Skip if the platform does not match. + target_platform = check_platform(os.path.abspath(full_path)) + if not target_platform: + err = ('--auto_platform passed in but no platform name ' + 'found in the path of %s' % full_path) + if not ignore_errors: + raise InvalidFileError(err) + print(err, file=sys.stderr) + continue + current_platform = PLATFORM_MAPPING[sys.platform] + if current_platform != target_platform: + continue + + with open(full_path, 'rb') as f: + sha1_match = re.match(b'^([A-Za-z0-9]{40})$', + f.read(1024).rstrip()) + if sha1_match: + yield (sha1_match.groups(1)[0].decode('utf-8'), + full_path.replace('.sha1', '')) + else: + if not ignore_errors: + raise InvalidFileError('No sha1 sum found in %s.' % + filename) + print('No sha1 sum found in %s.' % filename, + file=sys.stderr) def _validate_tar_file(tar, prefix): - def _validate(tarinfo): - """Returns false if the tarinfo is something we explicitly forbid.""" - if tarinfo.issym() or tarinfo.islnk(): - return False - if ('../' in tarinfo.name or - '..\\' in tarinfo.name or - not tarinfo.name.startswith(prefix)): - return False - return True - return all(map(_validate, tar.getmembers())) + def _validate(tarinfo): + """Returns false if the tarinfo is something we explicitly forbid.""" + if tarinfo.issym() or tarinfo.islnk(): + return False + if ('../' in tarinfo.name or '..\\' in tarinfo.name + or not tarinfo.name.startswith(prefix)): + return False + return True -def _downloader_worker_thread(thread_num, q, force, base_url, - gsutil, out_q, ret_codes, verbose, extract, + return all(map(_validate, tar.getmembers())) + + +def _downloader_worker_thread(thread_num, + q, + force, + base_url, + gsutil, + out_q, + ret_codes, + verbose, + extract, delete=True): - while True: - input_sha1_sum, output_filename = q.get() - if input_sha1_sum is None: - return - extract_dir = None - if extract: - if not output_filename.endswith('.tar.gz'): - out_q.put('%d> Error: %s is not a tar.gz archive.' % ( - thread_num, output_filename)) - ret_codes.put((1, '%s is not a tar.gz archive.' % (output_filename))) - continue - extract_dir = output_filename[:-len('.tar.gz')] - if os.path.exists(output_filename) and not force: - skip = get_sha1(output_filename) == input_sha1_sum - if extract: - # Additional condition for extract: - # 1) extract_dir must exist - # 2) .tmp flag file mustn't exist - if not os.path.exists(extract_dir): - out_q.put('%d> Extract dir %s does not exist, re-downloading...' % - (thread_num, extract_dir)) - skip = False - # .tmp file is created just before extraction and removed just after - # extraction. If such file exists, it means the process was terminated - # mid-extraction and therefore needs to be extracted again. - elif os.path.exists(extract_dir + '.tmp'): - out_q.put('%d> Detected tmp flag file for %s, ' - 're-downloading...' % (thread_num, output_filename)) - skip = False - if skip: - continue + while True: + input_sha1_sum, output_filename = q.get() + if input_sha1_sum is None: + return + extract_dir = None + if extract: + if not output_filename.endswith('.tar.gz'): + out_q.put('%d> Error: %s is not a tar.gz archive.' % + (thread_num, output_filename)) + ret_codes.put( + (1, '%s is not a tar.gz archive.' % (output_filename))) + continue + extract_dir = output_filename[:-len('.tar.gz')] + if os.path.exists(output_filename) and not force: + skip = get_sha1(output_filename) == input_sha1_sum + if extract: + # Additional condition for extract: + # 1) extract_dir must exist + # 2) .tmp flag file mustn't exist + if not os.path.exists(extract_dir): + out_q.put( + '%d> Extract dir %s does not exist, re-downloading...' % + (thread_num, extract_dir)) + skip = False + # .tmp file is created just before extraction and removed just + # after extraction. If such file exists, it means the process + # was terminated mid-extraction and therefore needs to be + # extracted again. + elif os.path.exists(extract_dir + '.tmp'): + out_q.put('%d> Detected tmp flag file for %s, ' + 're-downloading...' % + (thread_num, output_filename)) + skip = False + if skip: + continue - file_url = '%s/%s' % (base_url, input_sha1_sum) + file_url = '%s/%s' % (base_url, input_sha1_sum) - try: - if delete: - os.remove(output_filename) # Delete the file if it exists already. - except OSError: - if os.path.exists(output_filename): - out_q.put('%d> Warning: deleting %s failed.' % ( - thread_num, output_filename)) - if verbose: - out_q.put('%d> Downloading %s@%s...' % ( - thread_num, output_filename, input_sha1_sum)) - code, _, err = gsutil.check_call('cp', file_url, output_filename) - if code != 0: - if code == 404: - out_q.put('%d> File %s for %s does not exist, skipping.' % ( - thread_num, file_url, output_filename)) - ret_codes.put((1, 'File %s for %s does not exist.' % ( - file_url, output_filename))) - elif code == 401: - out_q.put( - """%d> Failed to fetch file %s for %s due to unauthorized access, - skipping. Try running `gsutil.py config`.""" % - (thread_num, file_url, output_filename)) - ret_codes.put( - (1, 'Failed to fetch file %s for %s due to unauthorized access.' % - (file_url, output_filename))) - else: - # Other error, probably auth related (bad ~/.boto, etc). - out_q.put('%d> Failed to fetch file %s for %s, skipping. [Err: %s]' % - (thread_num, file_url, output_filename, err)) - ret_codes.put((code, 'Failed to fetch file %s for %s. [Err: %s]' % - (file_url, output_filename, err))) - continue - - remote_sha1 = get_sha1(output_filename) - if remote_sha1 != input_sha1_sum: - msg = ('%d> ERROR remote sha1 (%s) does not match expected sha1 (%s).' % - (thread_num, remote_sha1, input_sha1_sum)) - out_q.put(msg) - ret_codes.put((20, msg)) - continue - - if extract: - if not tarfile.is_tarfile(output_filename): - out_q.put('%d> Error: %s is not a tar.gz archive.' % ( - thread_num, output_filename)) - ret_codes.put((1, '%s is not a tar.gz archive.' % (output_filename))) - continue - with tarfile.open(output_filename, 'r:gz') as tar: - dirname = os.path.dirname(os.path.abspath(output_filename)) - # If there are long paths inside the tarball we can get extraction - # errors on windows due to the 260 path length limit (this includes - # pwd). Use the extended path syntax. - if sys.platform == 'win32': - dirname = '\\\\?\\%s' % dirname - if not _validate_tar_file(tar, os.path.basename(extract_dir)): - out_q.put('%d> Error: %s contains files outside %s.' % ( - thread_num, output_filename, extract_dir)) - ret_codes.put((1, '%s contains invalid entries.' % (output_filename))) - continue - if os.path.exists(extract_dir): - try: - shutil.rmtree(extract_dir) - out_q.put('%d> Removed %s...' % (thread_num, extract_dir)) - except OSError: - out_q.put('%d> Warning: Can\'t delete: %s' % ( - thread_num, extract_dir)) - ret_codes.put((1, 'Can\'t delete %s.' % (extract_dir))) + try: + if delete: + os.remove( + output_filename) # Delete the file if it exists already. + except OSError: + if os.path.exists(output_filename): + out_q.put('%d> Warning: deleting %s failed.' % + (thread_num, output_filename)) + if verbose: + out_q.put('%d> Downloading %s@%s...' % + (thread_num, output_filename, input_sha1_sum)) + code, _, err = gsutil.check_call('cp', file_url, output_filename) + if code != 0: + if code == 404: + out_q.put('%d> File %s for %s does not exist, skipping.' % + (thread_num, file_url, output_filename)) + ret_codes.put((1, 'File %s for %s does not exist.' % + (file_url, output_filename))) + elif code == 401: + out_q.put( + '%d> Failed to fetch file %s for %s due to unauthorized ' + 'access, skipping. Try running `gsutil.py config`.' % + (thread_num, file_url, output_filename)) + ret_codes.put(( + 1, + 'Failed to fetch file %s for %s due to unauthorized access.' + % (file_url, output_filename))) + else: + # Other error, probably auth related (bad ~/.boto, etc). + out_q.put( + '%d> Failed to fetch file %s for %s, skipping. [Err: %s]' % + (thread_num, file_url, output_filename, err)) + ret_codes.put( + (code, 'Failed to fetch file %s for %s. [Err: %s]' % + (file_url, output_filename, err))) continue - out_q.put('%d> Extracting %d entries from %s to %s' % - (thread_num, len(tar.getmembers()),output_filename, - extract_dir)) - with open(extract_dir + '.tmp', 'a'): - tar.extractall(path=dirname) - os.remove(extract_dir + '.tmp') - # Set executable bit. - if sys.platform == 'cygwin': - # Under cygwin, mark all files as executable. The executable flag in - # Google Storage will not be set when uploading from Windows, so if - # this script is running under cygwin and we're downloading an - # executable, it will be unrunnable from inside cygwin without this. - st = os.stat(output_filename) - os.chmod(output_filename, st.st_mode | stat.S_IEXEC) - elif sys.platform != 'win32': - # On non-Windows platforms, key off of the custom header - # "x-goog-meta-executable". - code, out, err = gsutil.check_call('stat', file_url) - if code != 0: - out_q.put('%d> %s' % (thread_num, err)) - ret_codes.put((code, err)) - elif re.search(r'executable:\s*1', out): - st = os.stat(output_filename) - os.chmod(output_filename, st.st_mode | stat.S_IEXEC) + + remote_sha1 = get_sha1(output_filename) + if remote_sha1 != input_sha1_sum: + msg = ( + '%d> ERROR remote sha1 (%s) does not match expected sha1 (%s).' + % (thread_num, remote_sha1, input_sha1_sum)) + out_q.put(msg) + ret_codes.put((20, msg)) + continue + + if extract: + if not tarfile.is_tarfile(output_filename): + out_q.put('%d> Error: %s is not a tar.gz archive.' % + (thread_num, output_filename)) + ret_codes.put( + (1, '%s is not a tar.gz archive.' % (output_filename))) + continue + with tarfile.open(output_filename, 'r:gz') as tar: + dirname = os.path.dirname(os.path.abspath(output_filename)) + # If there are long paths inside the tarball we can get + # extraction errors on windows due to the 260 path length limit + # (this includes pwd). Use the extended path syntax. + if sys.platform == 'win32': + dirname = '\\\\?\\%s' % dirname + if not _validate_tar_file(tar, os.path.basename(extract_dir)): + out_q.put('%d> Error: %s contains files outside %s.' % + (thread_num, output_filename, extract_dir)) + ret_codes.put( + (1, '%s contains invalid entries.' % (output_filename))) + continue + if os.path.exists(extract_dir): + try: + shutil.rmtree(extract_dir) + out_q.put('%d> Removed %s...' % + (thread_num, extract_dir)) + except OSError: + out_q.put('%d> Warning: Can\'t delete: %s' % + (thread_num, extract_dir)) + ret_codes.put((1, 'Can\'t delete %s.' % (extract_dir))) + continue + out_q.put('%d> Extracting %d entries from %s to %s' % + (thread_num, len( + tar.getmembers()), output_filename, extract_dir)) + with open(extract_dir + '.tmp', 'a'): + tar.extractall(path=dirname) + os.remove(extract_dir + '.tmp') + # Set executable bit. + if sys.platform == 'cygwin': + # Under cygwin, mark all files as executable. The executable flag in + # Google Storage will not be set when uploading from Windows, so if + # this script is running under cygwin and we're downloading an + # executable, it will be unrunnable from inside cygwin without this. + st = os.stat(output_filename) + os.chmod(output_filename, st.st_mode | stat.S_IEXEC) + elif sys.platform != 'win32': + # On non-Windows platforms, key off of the custom header + # "x-goog-meta-executable". + code, out, err = gsutil.check_call('stat', file_url) + if code != 0: + out_q.put('%d> %s' % (thread_num, err)) + ret_codes.put((code, err)) + elif re.search(r'executable:\s*1', out): + st = os.stat(output_filename) + os.chmod(output_filename, st.st_mode | stat.S_IEXEC) class PrinterThread(threading.Thread): - def __init__(self, output_queue): - super(PrinterThread, self).__init__() - self.output_queue = output_queue - self.did_print_anything = False + def __init__(self, output_queue): + super(PrinterThread, self).__init__() + self.output_queue = output_queue + self.did_print_anything = False - def run(self): - while True: - line = self.output_queue.get() - # It's plausible we want to print empty lines: Explicit `is None`. - if line is None: - break - self.did_print_anything = True - print(line) + def run(self): + while True: + line = self.output_queue.get() + # It's plausible we want to print empty lines: Explicit `is None`. + if line is None: + break + self.did_print_anything = True + print(line) def _data_exists(input_sha1_sum, output_filename, extract): - """Returns True if the data exists locally and matches the sha1. + """Returns True if the data exists locally and matches the sha1. This conservatively returns False for error cases. @@ -394,247 +416,284 @@ def _data_exists(input_sha1_sum, output_filename, extract): the target directory already exists. The content of the target directory is not checked. """ - extract_dir = None - if extract: - if not output_filename.endswith('.tar.gz'): - # This will cause an error later. Conservativly return False to not bail - # out too early. - return False - extract_dir = output_filename[:-len('.tar.gz')] - if os.path.exists(output_filename): - if not extract or os.path.exists(extract_dir): - if get_sha1(output_filename) == input_sha1_sum: - return True - return False + extract_dir = None + if extract: + if not output_filename.endswith('.tar.gz'): + # This will cause an error later. Conservativly return False to not + # bail out too early. + return False + extract_dir = output_filename[:-len('.tar.gz')] + if os.path.exists(output_filename): + if not extract or os.path.exists(extract_dir): + if get_sha1(output_filename) == input_sha1_sum: + return True + return False -def download_from_google_storage( - input_filename, base_url, gsutil, num_threads, directory, recursive, - force, output, ignore_errors, sha1_file, verbose, auto_platform, extract): +def download_from_google_storage(input_filename, base_url, gsutil, num_threads, + directory, recursive, force, output, + ignore_errors, sha1_file, verbose, + auto_platform, extract): - # Tuples of sha1s and paths. - input_data = list(enumerate_input( - input_filename, directory, recursive, ignore_errors, output, sha1_file, - auto_platform)) + # Tuples of sha1s and paths. + input_data = list( + enumerate_input(input_filename, directory, recursive, ignore_errors, + output, sha1_file, auto_platform)) - # Sequentially check for the most common case and see if we can bail out - # early before making any slow calls to gsutil. - if not force and all( - _data_exists(sha1, path, extract) for sha1, path in input_data): - return 0 + # Sequentially check for the most common case and see if we can bail out + # early before making any slow calls to gsutil. + if not force and all( + _data_exists(sha1, path, extract) for sha1, path in input_data): + return 0 - # Call this once to ensure gsutil's update routine is called only once. Only - # needs to be done if we'll process input data in parallel, which can lead to - # a race in gsutil's self-update on the first call. Note, this causes a - # network call, therefore any fast bailout should be done before this point. - if len(input_data) > 1: - gsutil.check_call('version') + # Call this once to ensure gsutil's update routine is called only once. Only + # needs to be done if we'll process input data in parallel, which can lead + # to a race in gsutil's self-update on the first call. Note, this causes a + # network call, therefore any fast bailout should be done before this point. + if len(input_data) > 1: + gsutil.check_call('version') - # Start up all the worker threads. - all_threads = [] - download_start = time.time() - stdout_queue = queue.Queue() - work_queue = queue.Queue() - ret_codes = queue.Queue() - ret_codes.put((0, None)) - for thread_num in range(num_threads): - t = threading.Thread( - target=_downloader_worker_thread, - args=[thread_num, work_queue, force, base_url, - gsutil, stdout_queue, ret_codes, verbose, extract]) - t.daemon = True - t.start() - all_threads.append(t) - printer_thread = PrinterThread(stdout_queue) - printer_thread.daemon = True - printer_thread.start() + # Start up all the worker threads. + all_threads = [] + download_start = time.time() + stdout_queue = queue.Queue() + work_queue = queue.Queue() + ret_codes = queue.Queue() + ret_codes.put((0, None)) + for thread_num in range(num_threads): + t = threading.Thread(target=_downloader_worker_thread, + args=[ + thread_num, work_queue, force, base_url, + gsutil, stdout_queue, ret_codes, verbose, + extract + ]) + t.daemon = True + t.start() + all_threads.append(t) + printer_thread = PrinterThread(stdout_queue) + printer_thread.daemon = True + printer_thread.start() - # Populate our work queue. - for sha1, path in input_data: - work_queue.put((sha1, path)) - for _ in all_threads: - work_queue.put((None, None)) # Used to tell worker threads to stop. + # Populate our work queue. + for sha1, path in input_data: + work_queue.put((sha1, path)) + for _ in all_threads: + work_queue.put((None, None)) # Used to tell worker threads to stop. - # Wait for all downloads to finish. - for t in all_threads: - t.join() - stdout_queue.put(None) - printer_thread.join() + # Wait for all downloads to finish. + for t in all_threads: + t.join() + stdout_queue.put(None) + printer_thread.join() - # See if we ran into any errors. - max_ret_code = 0 - for ret_code, message in ret_codes.queue: - max_ret_code = max(ret_code, max_ret_code) - if message: - print(message, file=sys.stderr) + # See if we ran into any errors. + max_ret_code = 0 + for ret_code, message in ret_codes.queue: + max_ret_code = max(ret_code, max_ret_code) + if message: + print(message, file=sys.stderr) - # Only print summary if any work was done. - if printer_thread.did_print_anything: - print('Downloading %d files took %1f second(s)' % - (len(input_data), time.time() - download_start)) - return max_ret_code + # Only print summary if any work was done. + if printer_thread.did_print_anything: + print('Downloading %d files took %1f second(s)' % + (len(input_data), time.time() - download_start)) + return max_ret_code def main(args): - usage = ('usage: %prog [options] target\n' - 'Target must be:\n' - ' (default) a sha1 sum ([A-Za-z0-9]{40}).\n' - ' (-s or --sha1_file) a .sha1 file, containing a sha1 sum on ' - 'the first line.\n' - ' (-d or --directory) A directory to scan for .sha1 files.') - parser = optparse.OptionParser(usage) - parser.add_option('-o', '--output', - help='Specify the output file name. Defaults to: ' - '(a) Given a SHA1 hash, the name is the SHA1 hash. ' - '(b) Given a .sha1 file or directory, the name will ' - 'match (.*).sha1.') - parser.add_option('-b', '--bucket', - help='Google Storage bucket to fetch from.') - parser.add_option('-e', '--boto', - help='Specify a custom boto file.') - parser.add_option('-c', '--no_resume', action='store_true', - help='DEPRECATED: Resume download if file is ' - 'partially downloaded.') - parser.add_option('-f', '--force', action='store_true', - help='Force download even if local file exists.') - parser.add_option('-i', '--ignore_errors', action='store_true', - help='Don\'t throw error if we find an invalid .sha1 file.') - parser.add_option('-r', '--recursive', action='store_true', - help='Scan folders recursively for .sha1 files. ' - 'Must be used with -d/--directory') - parser.add_option('-t', '--num_threads', default=1, type='int', - help='Number of downloader threads to run.') - parser.add_option('-d', '--directory', action='store_true', - help='The target is a directory. ' - 'Cannot be used with -s/--sha1_file.') - parser.add_option('-s', '--sha1_file', action='store_true', - help='The target is a file containing a sha1 sum. ' - 'Cannot be used with -d/--directory.') - parser.add_option('-g', '--config', action='store_true', - help='Alias for "gsutil config". Run this if you want ' - 'to initialize your saved Google Storage ' - 'credentials. This will create a read-only ' - 'credentials file in ~/.boto.depot_tools.') - parser.add_option('-n', '--no_auth', action='store_true', - help='Skip auth checking. Use if it\'s known that the ' - 'target bucket is a public bucket.') - parser.add_option('-p', '--platform', - help='A regular expression that is compared against ' - 'Python\'s sys.platform. If this option is specified, ' - 'the download will happen only if there is a match.') - parser.add_option('-a', '--auto_platform', - action='store_true', - help='Detects if any parent folder of the target matches ' - '(linux|mac|win). If so, the script will only ' - 'process files that are in the paths that ' - 'that matches the current platform.') - parser.add_option('-u', '--extract', - action='store_true', - help='Extract a downloaded tar.gz file. ' - 'Leaves the tar.gz file around for sha1 verification' - 'If a directory with the same name as the tar.gz ' - 'file already exists, is deleted (to get a ' - 'clean state in case of update.)') - parser.add_option('-v', '--verbose', action='store_true', default=True, - help='DEPRECATED: Defaults to True. Use --no-verbose ' - 'to suppress.') - parser.add_option('-q', '--quiet', action='store_false', dest='verbose', - help='Suppresses diagnostic and progress information.') + usage = ('usage: %prog [options] target\n' + 'Target must be:\n' + ' (default) a sha1 sum ([A-Za-z0-9]{40}).\n' + ' (-s or --sha1_file) a .sha1 file, containing a sha1 sum on ' + 'the first line.\n' + ' (-d or --directory) A directory to scan for .sha1 files.') + parser = optparse.OptionParser(usage) + parser.add_option('-o', + '--output', + help='Specify the output file name. Defaults to: ' + '(a) Given a SHA1 hash, the name is the SHA1 hash. ' + '(b) Given a .sha1 file or directory, the name will ' + 'match (.*).sha1.') + parser.add_option('-b', + '--bucket', + help='Google Storage bucket to fetch from.') + parser.add_option('-e', '--boto', help='Specify a custom boto file.') + parser.add_option('-c', + '--no_resume', + action='store_true', + help='DEPRECATED: Resume download if file is ' + 'partially downloaded.') + parser.add_option('-f', + '--force', + action='store_true', + help='Force download even if local file exists.') + parser.add_option( + '-i', + '--ignore_errors', + action='store_true', + help='Don\'t throw error if we find an invalid .sha1 file.') + parser.add_option('-r', + '--recursive', + action='store_true', + help='Scan folders recursively for .sha1 files. ' + 'Must be used with -d/--directory') + parser.add_option('-t', + '--num_threads', + default=1, + type='int', + help='Number of downloader threads to run.') + parser.add_option('-d', + '--directory', + action='store_true', + help='The target is a directory. ' + 'Cannot be used with -s/--sha1_file.') + parser.add_option('-s', + '--sha1_file', + action='store_true', + help='The target is a file containing a sha1 sum. ' + 'Cannot be used with -d/--directory.') + parser.add_option('-g', + '--config', + action='store_true', + help='Alias for "gsutil config". Run this if you want ' + 'to initialize your saved Google Storage ' + 'credentials. This will create a read-only ' + 'credentials file in ~/.boto.depot_tools.') + parser.add_option('-n', + '--no_auth', + action='store_true', + help='Skip auth checking. Use if it\'s known that the ' + 'target bucket is a public bucket.') + parser.add_option('-p', + '--platform', + help='A regular expression that is compared against ' + 'Python\'s sys.platform. If this option is specified, ' + 'the download will happen only if there is a match.') + parser.add_option('-a', + '--auto_platform', + action='store_true', + help='Detects if any parent folder of the target matches ' + '(linux|mac|win). If so, the script will only ' + 'process files that are in the paths that ' + 'that matches the current platform.') + parser.add_option('-u', + '--extract', + action='store_true', + help='Extract a downloaded tar.gz file. ' + 'Leaves the tar.gz file around for sha1 verification' + 'If a directory with the same name as the tar.gz ' + 'file already exists, is deleted (to get a ' + 'clean state in case of update.)') + parser.add_option('-v', + '--verbose', + action='store_true', + default=True, + help='DEPRECATED: Defaults to True. Use --no-verbose ' + 'to suppress.') + parser.add_option('-q', + '--quiet', + action='store_false', + dest='verbose', + help='Suppresses diagnostic and progress information.') - (options, args) = parser.parse_args() + (options, args) = parser.parse_args() - # Make sure we should run at all based on platform matching. - if options.platform: - if options.auto_platform: - parser.error('--platform can not be specified with --auto_platform') - if not re.match(options.platform, GetNormalizedPlatform()): - if options.verbose: - print('The current platform doesn\'t match "%s", skipping.' % - options.platform) - return 0 + # Make sure we should run at all based on platform matching. + if options.platform: + if options.auto_platform: + parser.error('--platform can not be specified with --auto_platform') + if not re.match(options.platform, GetNormalizedPlatform()): + if options.verbose: + print('The current platform doesn\'t match "%s", skipping.' % + options.platform) + return 0 - # Set the boto file to /dev/null if we don't need auth. - if options.no_auth: - if (set(('http_proxy', 'https_proxy')).intersection( - env.lower() for env in os.environ) and - 'NO_AUTH_BOTO_CONFIG' not in os.environ): - print('NOTICE: You have PROXY values set in your environment, but gsutil' - 'in depot_tools does not (yet) obey them.', - file=sys.stderr) - print('Also, --no_auth prevents the normal BOTO_CONFIG environment' - 'variable from being used.', - file=sys.stderr) - print('To use a proxy in this situation, please supply those settings' - 'in a .boto file pointed to by the NO_AUTH_BOTO_CONFIG environment' - 'variable.', - file=sys.stderr) - options.boto = os.environ.get('NO_AUTH_BOTO_CONFIG', os.devnull) + # Set the boto file to /dev/null if we don't need auth. + if options.no_auth: + if (set( + ('http_proxy', 'https_proxy')).intersection(env.lower() + for env in os.environ) + and 'NO_AUTH_BOTO_CONFIG' not in os.environ): + print( + 'NOTICE: You have PROXY values set in your environment, but ' + 'gsutil in depot_tools does not (yet) obey them.', + file=sys.stderr) + print( + 'Also, --no_auth prevents the normal BOTO_CONFIG environment ' + 'variable from being used.', + file=sys.stderr) + print( + 'To use a proxy in this situation, please supply those ' + 'settings in a .boto file pointed to by the ' + 'NO_AUTH_BOTO_CONFIG environment variable.', + file=sys.stderr) + options.boto = os.environ.get('NO_AUTH_BOTO_CONFIG', os.devnull) - # Make sure gsutil exists where we expect it to. - if os.path.exists(GSUTIL_DEFAULT_PATH): - gsutil = Gsutil(GSUTIL_DEFAULT_PATH, - boto_path=options.boto) - else: - parser.error('gsutil not found in %s, bad depot_tools checkout?' % - GSUTIL_DEFAULT_PATH) - - # Passing in -g/--config will run our copy of GSUtil, then quit. - if options.config: - print('===Note from depot_tools===') - print('If you do not have a project ID, enter "0" when asked for one.') - print('===End note from depot_tools===') - print() - gsutil.check_call('version') - return gsutil.call('config') - - if not args: - parser.error('Missing target.') - if len(args) > 1: - parser.error('Too many targets.') - if not options.bucket: - parser.error('Missing bucket. Specify bucket with --bucket.') - if options.sha1_file and options.directory: - parser.error('Both --directory and --sha1_file are specified, ' - 'can only specify one.') - if options.recursive and not options.directory: - parser.error('--recursive specified but --directory not specified.') - if options.output and options.directory: - parser.error('--directory is specified, so --output has no effect.') - if (not (options.sha1_file or options.directory) - and options.auto_platform): - parser.error('--auto_platform must be specified with either ' - '--sha1_file or --directory') - - input_filename = args[0] - - # Set output filename if not specified. - if not options.output and not options.directory: - if not options.sha1_file: - # Target is a sha1 sum, so output filename would also be the sha1 sum. - options.output = input_filename - elif options.sha1_file: - # Target is a .sha1 file. - if not input_filename.endswith('.sha1'): - parser.error('--sha1_file is specified, but the input filename ' - 'does not end with .sha1, and no --output is specified. ' - 'Either make sure the input filename has a .sha1 ' - 'extension, or specify --output.') - options.output = input_filename[:-5] + # Make sure gsutil exists where we expect it to. + if os.path.exists(GSUTIL_DEFAULT_PATH): + gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto) else: - parser.error('Unreachable state.') + parser.error('gsutil not found in %s, bad depot_tools checkout?' % + GSUTIL_DEFAULT_PATH) - base_url = 'gs://%s' % options.bucket + # Passing in -g/--config will run our copy of GSUtil, then quit. + if options.config: + print('===Note from depot_tools===') + print('If you do not have a project ID, enter "0" when asked for one.') + print('===End note from depot_tools===') + print() + gsutil.check_call('version') + return gsutil.call('config') - try: - return download_from_google_storage( - input_filename, base_url, gsutil, options.num_threads, options.directory, - options.recursive, options.force, options.output, options.ignore_errors, - options.sha1_file, options.verbose, options.auto_platform, - options.extract) - except FileNotFoundError as e: - print("Fatal error: {}".format(e)) - return 1 + if not args: + parser.error('Missing target.') + if len(args) > 1: + parser.error('Too many targets.') + if not options.bucket: + parser.error('Missing bucket. Specify bucket with --bucket.') + if options.sha1_file and options.directory: + parser.error('Both --directory and --sha1_file are specified, ' + 'can only specify one.') + if options.recursive and not options.directory: + parser.error('--recursive specified but --directory not specified.') + if options.output and options.directory: + parser.error('--directory is specified, so --output has no effect.') + if (not (options.sha1_file or options.directory) and options.auto_platform): + parser.error('--auto_platform must be specified with either ' + '--sha1_file or --directory') + + input_filename = args[0] + + # Set output filename if not specified. + if not options.output and not options.directory: + if not options.sha1_file: + # Target is a sha1 sum, so output filename would also be the sha1 + # sum. + options.output = input_filename + elif options.sha1_file: + # Target is a .sha1 file. + if not input_filename.endswith('.sha1'): + parser.error( + '--sha1_file is specified, but the input filename ' + 'does not end with .sha1, and no --output is specified. ' + 'Either make sure the input filename has a .sha1 ' + 'extension, or specify --output.') + options.output = input_filename[:-5] + else: + parser.error('Unreachable state.') + + base_url = 'gs://%s' % options.bucket + + try: + return download_from_google_storage( + input_filename, base_url, gsutil, options.num_threads, + options.directory, options.recursive, options.force, options.output, + options.ignore_errors, options.sha1_file, options.verbose, + options.auto_platform, options.extract) + except FileNotFoundError as e: + print("Fatal error: {}".format(e)) + return 1 if __name__ == '__main__': - sys.exit(main(sys.argv)) + sys.exit(main(sys.argv)) diff --git a/fetch.py b/fetch.py index 80f8927d0f..fcfdf781f3 100755 --- a/fetch.py +++ b/fetch.py @@ -2,7 +2,6 @@ # Copyright (c) 2013 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """ Tool to perform checkouts in one easy command line! @@ -31,14 +30,14 @@ import git_common from distutils import spawn - SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__)) + ################################################# # Checkout class definitions. ################################################# class Checkout(object): - """Base class for implementing different types of checkouts. + """Base class for implementing different types of checkouts. Attributes: |base|: the absolute path of the directory in which this script is run. @@ -47,212 +46,230 @@ class Checkout(object): |root|: the directory into which the checkout will be performed, as returned by the config. This is a relative path from |base|. """ - def __init__(self, options, spec, root): - self.base = os.getcwd() - self.options = options - self.spec = spec - self.root = root + def __init__(self, options, spec, root): + self.base = os.getcwd() + self.options = options + self.spec = spec + self.root = root - def exists(self): - """Check does this checkout already exist on desired location""" + def exists(self): + """Check does this checkout already exist on desired location""" - def init(self): - pass + def init(self): + pass - def run(self, cmd, return_stdout=False, **kwargs): - print('Running: %s' % (' '.join(pipes.quote(x) for x in cmd))) - if self.options.dry_run: - return '' - if return_stdout: - return subprocess.check_output(cmd, **kwargs).decode() + def run(self, cmd, return_stdout=False, **kwargs): + print('Running: %s' % (' '.join(pipes.quote(x) for x in cmd))) + if self.options.dry_run: + return '' + if return_stdout: + return subprocess.check_output(cmd, **kwargs).decode() - try: - subprocess.check_call(cmd, **kwargs) - except subprocess.CalledProcessError as e: - # If the subprocess failed, it likely emitted its own distress message - # already - don't scroll that message off the screen with a stack trace - # from this program as well. Emit a terse message and bail out here; - # otherwise a later step will try doing more work and may hide the - # subprocess message. - print('Subprocess failed with return code %d.' % e.returncode) - sys.exit(e.returncode) - return '' + try: + subprocess.check_call(cmd, **kwargs) + except subprocess.CalledProcessError as e: + # If the subprocess failed, it likely emitted its own distress + # message already - don't scroll that message off the screen with a + # stack trace from this program as well. Emit a terse message and + # bail out here; otherwise a later step will try doing more work and + # may hide the subprocess message. + print('Subprocess failed with return code %d.' % e.returncode) + sys.exit(e.returncode) + return '' class GclientCheckout(Checkout): + def run_gclient(self, *cmd, **kwargs): + if not spawn.find_executable('gclient'): + cmd_prefix = (sys.executable, os.path.join(SCRIPT_PATH, + 'gclient.py')) + else: + cmd_prefix = ('gclient', ) + return self.run(cmd_prefix + cmd, **kwargs) - def run_gclient(self, *cmd, **kwargs): - if not spawn.find_executable('gclient'): - cmd_prefix = (sys.executable, os.path.join(SCRIPT_PATH, 'gclient.py')) - else: - cmd_prefix = ('gclient',) - return self.run(cmd_prefix + cmd, **kwargs) - - def exists(self): - try: - gclient_root = self.run_gclient('root', return_stdout=True).strip() - return (os.path.exists(os.path.join(gclient_root, '.gclient')) or - os.path.exists(os.path.join(os.getcwd(), self.root, '.git'))) - except subprocess.CalledProcessError: - pass - return os.path.exists(os.path.join(os.getcwd(), self.root)) + def exists(self): + try: + gclient_root = self.run_gclient('root', return_stdout=True).strip() + return (os.path.exists(os.path.join(gclient_root, '.gclient')) + or os.path.exists( + os.path.join(os.getcwd(), self.root, '.git'))) + except subprocess.CalledProcessError: + pass + return os.path.exists(os.path.join(os.getcwd(), self.root)) class GitCheckout(Checkout): - - def run_git(self, *cmd, **kwargs): - print('Running: git %s' % (' '.join(pipes.quote(x) for x in cmd))) - if self.options.dry_run: - return '' - return git_common.run(*cmd, **kwargs) + def run_git(self, *cmd, **kwargs): + print('Running: git %s' % (' '.join(pipes.quote(x) for x in cmd))) + if self.options.dry_run: + return '' + return git_common.run(*cmd, **kwargs) class GclientGitCheckout(GclientCheckout, GitCheckout): + def __init__(self, options, spec, root): + super(GclientGitCheckout, self).__init__(options, spec, root) + assert 'solutions' in self.spec - def __init__(self, options, spec, root): - super(GclientGitCheckout, self).__init__(options, spec, root) - assert 'solutions' in self.spec + def _format_spec(self): + def _format_literal(lit): + if isinstance(lit, str): + return '"%s"' % lit + if isinstance(lit, list): + return '[%s]' % ', '.join(_format_literal(i) for i in lit) + return '%r' % lit - def _format_spec(self): - def _format_literal(lit): - if isinstance(lit, str): - return '"%s"' % lit - if isinstance(lit, list): - return '[%s]' % ', '.join(_format_literal(i) for i in lit) - return '%r' % lit - soln_strings = [] - for soln in self.spec['solutions']: - soln_string = '\n'.join(' "%s": %s,' % (key, _format_literal(value)) - for key, value in soln.items()) - soln_strings.append(' {\n%s\n },' % soln_string) - gclient_spec = 'solutions = [\n%s\n]\n' % '\n'.join(soln_strings) - extra_keys = ['target_os', 'target_os_only', 'cache_dir'] - gclient_spec += ''.join('%s = %s\n' % (key, _format_literal(self.spec[key])) - for key in extra_keys if key in self.spec) - return gclient_spec + soln_strings = [] + for soln in self.spec['solutions']: + soln_string = '\n'.join(' "%s": %s,' % + (key, _format_literal(value)) + for key, value in soln.items()) + soln_strings.append(' {\n%s\n },' % soln_string) + gclient_spec = 'solutions = [\n%s\n]\n' % '\n'.join(soln_strings) + extra_keys = ['target_os', 'target_os_only', 'cache_dir'] + gclient_spec += ''.join('%s = %s\n' % + (key, _format_literal(self.spec[key])) + for key in extra_keys if key in self.spec) + return gclient_spec - def init(self): - # Configure and do the gclient checkout. - self.run_gclient('config', '--spec', self._format_spec()) - sync_cmd = ['sync'] - if self.options.nohooks: - sync_cmd.append('--nohooks') - if self.options.nohistory: - sync_cmd.append('--no-history') - if self.spec.get('with_branch_heads', False): - sync_cmd.append('--with_branch_heads') - self.run_gclient(*sync_cmd) + def init(self): + # Configure and do the gclient checkout. + self.run_gclient('config', '--spec', self._format_spec()) + sync_cmd = ['sync'] + if self.options.nohooks: + sync_cmd.append('--nohooks') + if self.options.nohistory: + sync_cmd.append('--no-history') + if self.spec.get('with_branch_heads', False): + sync_cmd.append('--with_branch_heads') + self.run_gclient(*sync_cmd) - # Configure git. - wd = os.path.join(self.base, self.root) - if self.options.dry_run: - print('cd %s' % wd) - self.run_git( - 'submodule', 'foreach', - 'git config -f $toplevel/.git/config submodule.$name.ignore all', - cwd=wd) - if not self.options.nohistory: - self.run_git( - 'config', '--add', 'remote.origin.fetch', - '+refs/tags/*:refs/tags/*', cwd=wd) - self.run_git('config', 'diff.ignoreSubmodules', 'dirty', cwd=wd) + # Configure git. + wd = os.path.join(self.base, self.root) + if self.options.dry_run: + print('cd %s' % wd) + self.run_git( + 'submodule', + 'foreach', + 'git config -f $toplevel/.git/config submodule.$name.ignore all', + cwd=wd) + if not self.options.nohistory: + self.run_git('config', + '--add', + 'remote.origin.fetch', + '+refs/tags/*:refs/tags/*', + cwd=wd) + self.run_git('config', 'diff.ignoreSubmodules', 'dirty', cwd=wd) CHECKOUT_TYPE_MAP = { - 'gclient': GclientCheckout, - 'gclient_git': GclientGitCheckout, - 'git': GitCheckout, + 'gclient': GclientCheckout, + 'gclient_git': GclientGitCheckout, + 'git': GitCheckout, } def CheckoutFactory(type_name, options, spec, root): - """Factory to build Checkout class instances.""" - class_ = CHECKOUT_TYPE_MAP.get(type_name) - if not class_: - raise KeyError('unrecognized checkout type: %s' % type_name) - return class_(options, spec, root) + """Factory to build Checkout class instances.""" + class_ = CHECKOUT_TYPE_MAP.get(type_name) + if not class_: + raise KeyError('unrecognized checkout type: %s' % type_name) + return class_(options, spec, root) + def handle_args(argv): - """Gets the config name from the command line arguments.""" + """Gets the config name from the command line arguments.""" - configs_dir = os.path.join(SCRIPT_PATH, 'fetch_configs') - configs = [f[:-3] for f in os.listdir(configs_dir) if f.endswith('.py')] - configs.sort() + configs_dir = os.path.join(SCRIPT_PATH, 'fetch_configs') + configs = [f[:-3] for f in os.listdir(configs_dir) if f.endswith('.py')] + configs.sort() - parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter, - description=''' + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=''' This script can be used to download the Chromium sources. See http://www.chromium.org/developers/how-tos/get-the-code for full usage instructions.''', - epilog='Valid fetch configs:\n' + \ - '\n'.join(map(lambda s: ' ' + s, configs)) - ) + epilog='Valid fetch configs:\n' + \ + '\n'.join(map(lambda s: ' ' + s, configs)) + ) - parser.add_argument('-n', '--dry-run', action='store_true', default=False, - help='Don\'t run commands, only print them.') - parser.add_argument('--nohooks', - '--no-hooks', - action='store_true', - default=False, - help='Don\'t run hooks after checkout.') - parser.add_argument( - '--nohistory', - '--no-history', - action='store_true', - default=False, - help='Perform shallow clones, don\'t fetch the full git history.') - parser.add_argument('--force', action='store_true', default=False, - help='(dangerous) Don\'t look for existing .gclient file.') - parser.add_argument( - '-p', - '--protocol-override', - type=str, - default=None, - help='Protocol to use to fetch dependencies, defaults to https.') + parser.add_argument('-n', + '--dry-run', + action='store_true', + default=False, + help='Don\'t run commands, only print them.') + parser.add_argument('--nohooks', + '--no-hooks', + action='store_true', + default=False, + help='Don\'t run hooks after checkout.') + parser.add_argument( + '--nohistory', + '--no-history', + action='store_true', + default=False, + help='Perform shallow clones, don\'t fetch the full git history.') + parser.add_argument( + '--force', + action='store_true', + default=False, + help='(dangerous) Don\'t look for existing .gclient file.') + parser.add_argument( + '-p', + '--protocol-override', + type=str, + default=None, + help='Protocol to use to fetch dependencies, defaults to https.') - parser.add_argument('config', type=str, - help="Project to fetch, e.g. chromium.") - parser.add_argument('props', metavar='props', type=str, - nargs=argparse.REMAINDER, default=[]) + parser.add_argument('config', + type=str, + help="Project to fetch, e.g. chromium.") + parser.add_argument('props', + metavar='props', + type=str, + nargs=argparse.REMAINDER, + default=[]) - args = parser.parse_args(argv[1:]) + args = parser.parse_args(argv[1:]) - # props passed to config must be of the format --= - looks_like_arg = lambda arg: arg.startswith('--') and arg.count('=') == 1 - bad_param = [x for x in args.props if not looks_like_arg(x)] - if bad_param: - print('Error: Got bad arguments %s' % bad_param) - parser.print_help() - sys.exit(1) + # props passed to config must be of the format --= + looks_like_arg = lambda arg: arg.startswith('--') and arg.count('=') == 1 + bad_param = [x for x in args.props if not looks_like_arg(x)] + if bad_param: + print('Error: Got bad arguments %s' % bad_param) + parser.print_help() + sys.exit(1) + + return args - return args def run_config_fetch(config, props, aliased=False): - """Invoke a config's fetch method with the passed-through args + """Invoke a config's fetch method with the passed-through args and return its json output as a python object.""" - config_path = os.path.abspath( - os.path.join(SCRIPT_PATH, 'fetch_configs', config)) - if not os.path.exists(config_path + '.py'): - print("Could not find a config for %s" % config) - sys.exit(1) + config_path = os.path.abspath( + os.path.join(SCRIPT_PATH, 'fetch_configs', config)) + if not os.path.exists(config_path + '.py'): + print("Could not find a config for %s" % config) + sys.exit(1) - cmd = [sys.executable, config_path + '.py', 'fetch'] + props - result = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] + cmd = [sys.executable, config_path + '.py', 'fetch'] + props + result = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] - spec = json.loads(result.decode("utf-8")) - if 'alias' in spec: - assert not aliased - return run_config_fetch( - spec['alias']['config'], spec['alias']['props'] + props, aliased=True) - cmd = [sys.executable, config_path + '.py', 'root'] - result = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] - root = json.loads(result.decode("utf-8")) - return spec, root + spec = json.loads(result.decode("utf-8")) + if 'alias' in spec: + assert not aliased + return run_config_fetch(spec['alias']['config'], + spec['alias']['props'] + props, + aliased=True) + cmd = [sys.executable, config_path + '.py', 'root'] + result = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] + root = json.loads(result.decode("utf-8")) + return spec, root def run(options, spec, root): - """Perform a checkout with the given type and configuration. + """Perform a checkout with the given type and configuration. Args: options: Options instance. @@ -260,45 +277,48 @@ def run(options, spec, root): method (checkout type, repository url, etc.). root: The directory into which the repo expects to be checkout out. """ - assert 'type' in spec - checkout_type = spec['type'] - checkout_spec = spec['%s_spec' % checkout_type] + assert 'type' in spec + checkout_type = spec['type'] + checkout_spec = spec['%s_spec' % checkout_type] - # Use sso:// by default if the env is cog - if not options.protocol_override and \ - (any(os.getcwd().startswith(x) for x in [ - '/google/src/cloud', '/google/cog/cloud'])): - options.protocol_override = 'sso' + # Use sso:// by default if the env is cog + if not options.protocol_override and \ + (any(os.getcwd().startswith(x) for x in [ + '/google/src/cloud', '/google/cog/cloud'])): + options.protocol_override = 'sso' - # Update solutions with protocol_override field - if options.protocol_override is not None: - for solution in checkout_spec['solutions']: - solution['protocol_override'] = options.protocol_override + # Update solutions with protocol_override field + if options.protocol_override is not None: + for solution in checkout_spec['solutions']: + solution['protocol_override'] = options.protocol_override - try: - checkout = CheckoutFactory(checkout_type, options, checkout_spec, root) - except KeyError: - return 1 - if not options.force and checkout.exists(): - print('Your current directory appears to already contain, or be part of, ') - print('a checkout. "fetch" is used only to get new checkouts. Use ') - print('"gclient sync" to update existing checkouts.') - print() - print('Fetch also does not yet deal with partial checkouts, so if fetch') - print('failed, delete the checkout and start over (crbug.com/230691).') - return 1 - return checkout.init() + try: + checkout = CheckoutFactory(checkout_type, options, checkout_spec, root) + except KeyError: + return 1 + if not options.force and checkout.exists(): + print( + 'Your current directory appears to already contain, or be part of, ' + ) + print('a checkout. "fetch" is used only to get new checkouts. Use ') + print('"gclient sync" to update existing checkouts.') + print() + print( + 'Fetch also does not yet deal with partial checkouts, so if fetch') + print('failed, delete the checkout and start over (crbug.com/230691).') + return 1 + return checkout.init() def main(): - args = handle_args(sys.argv) - spec, root = run_config_fetch(args.config, args.props) - return run(args, spec, root) + args = handle_args(sys.argv) + spec, root = run_config_fetch(args.config, args.props) + return run(args, spec, root) if __name__ == '__main__': - try: - sys.exit(main()) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(1) + try: + sys.exit(main()) + except KeyboardInterrupt: + sys.stderr.write('interrupted\n') + sys.exit(1) diff --git a/fix_encoding.py b/fix_encoding.py index e97ea05203..21efb938cf 100644 --- a/fix_encoding.py +++ b/fix_encoding.py @@ -1,7 +1,6 @@ # Copyright (c) 2011 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Collection of functions and classes to fix various encoding problems on multiple platforms with python. """ @@ -15,266 +14,269 @@ import sys def complain(message): - """If any exception occurs in this file, we'll probably try to print it + """If any exception occurs in this file, we'll probably try to print it on stderr, which makes for frustrating debugging if stderr is directed to our wrapper. So be paranoid about catching errors and reporting them to sys.__stderr__, so that the user has a higher chance to see them. """ - print( - isinstance(message, str) and message or repr(message), - file=sys.__stderr__) + print(isinstance(message, str) and message or repr(message), + file=sys.__stderr__) def fix_default_encoding(): - """Forces utf8 solidly on all platforms. + """Forces utf8 solidly on all platforms. By default python execution environment is lazy and defaults to ascii encoding. http://uucode.com/blog/2007/03/23/shut-up-you-dummy-7-bit-python/ """ - if sys.getdefaultencoding() == 'utf-8': - return False + if sys.getdefaultencoding() == 'utf-8': + return False - # Regenerate setdefaultencoding. - reload(sys) - # Module 'sys' has no 'setdefaultencoding' member - # pylint: disable=no-member - sys.setdefaultencoding('utf-8') - for attr in dir(locale): - if attr[0:3] != 'LC_': - continue - aref = getattr(locale, attr) + # Regenerate setdefaultencoding. + reload(sys) + # Module 'sys' has no 'setdefaultencoding' member + # pylint: disable=no-member + sys.setdefaultencoding('utf-8') + for attr in dir(locale): + if attr[0:3] != 'LC_': + continue + aref = getattr(locale, attr) + try: + locale.setlocale(aref, '') + except locale.Error: + continue + try: + lang, _ = locale.getdefaultlocale() + except (TypeError, ValueError): + continue + if lang: + try: + locale.setlocale(aref, (lang, 'UTF-8')) + except locale.Error: + os.environ[attr] = lang + '.UTF-8' try: - locale.setlocale(aref, '') + locale.setlocale(locale.LC_ALL, '') except locale.Error: - continue - try: - lang, _ = locale.getdefaultlocale() - except (TypeError, ValueError): - continue - if lang: - try: - locale.setlocale(aref, (lang, 'UTF-8')) - except locale.Error: - os.environ[attr] = lang + '.UTF-8' - try: - locale.setlocale(locale.LC_ALL, '') - except locale.Error: - pass - return True + pass + return True ############################### # Windows specific + def fix_win_codec(): - """Works around .""" - # - try: - codecs.lookup('cp65001') - return False - except LookupError: - codecs.register( - lambda name: name == 'cp65001' and codecs.lookup('utf-8') or None) - return True + """Works around .""" + # + try: + codecs.lookup('cp65001') + return False + except LookupError: + codecs.register( + lambda name: name == 'cp65001' and codecs.lookup('utf-8') or None) + return True class WinUnicodeOutputBase(object): - """Base class to adapt sys.stdout or sys.stderr to behave correctly on + """Base class to adapt sys.stdout or sys.stderr to behave correctly on Windows. Setting encoding to utf-8 is recommended. """ - def __init__(self, fileno, name, encoding): - # Corresponding file handle. - self._fileno = fileno - self.encoding = encoding - self.name = name + def __init__(self, fileno, name, encoding): + # Corresponding file handle. + self._fileno = fileno + self.encoding = encoding + self.name = name - self.closed = False - self.softspace = False - self.mode = 'w' + self.closed = False + self.softspace = False + self.mode = 'w' - @staticmethod - def isatty(): - return False + @staticmethod + def isatty(): + return False - def close(self): - # Don't really close the handle, that would only cause problems. - self.closed = True + def close(self): + # Don't really close the handle, that would only cause problems. + self.closed = True - def fileno(self): - return self._fileno + def fileno(self): + return self._fileno - def flush(self): - raise NotImplementedError() + def flush(self): + raise NotImplementedError() - def write(self, text): - raise NotImplementedError() + def write(self, text): + raise NotImplementedError() - def writelines(self, lines): - try: - for line in lines: - self.write(line) - except Exception as e: - complain('%s.writelines: %r' % (self.name, e)) - raise + def writelines(self, lines): + try: + for line in lines: + self.write(line) + except Exception as e: + complain('%s.writelines: %r' % (self.name, e)) + raise class WinUnicodeConsoleOutput(WinUnicodeOutputBase): - """Output adapter to a Windows Console. + """Output adapter to a Windows Console. Understands how to use the win32 console API. """ - def __init__(self, console_handle, fileno, stream_name, encoding): - super(WinUnicodeConsoleOutput, self).__init__( - fileno, '' % stream_name, encoding) - # Handle to use for WriteConsoleW - self._console_handle = console_handle + def __init__(self, console_handle, fileno, stream_name, encoding): + super(WinUnicodeConsoleOutput, + self).__init__(fileno, '' % stream_name, + encoding) + # Handle to use for WriteConsoleW + self._console_handle = console_handle - # Loads the necessary function. - # These types are available on linux but not Mac. - # pylint: disable=no-name-in-module,F0401 - from ctypes import byref, GetLastError, POINTER, windll, WINFUNCTYPE - from ctypes.wintypes import BOOL, DWORD, HANDLE, LPWSTR - from ctypes.wintypes import LPVOID # pylint: disable=no-name-in-module + # Loads the necessary function. + # These types are available on linux but not Mac. + # pylint: disable=no-name-in-module,F0401 + from ctypes import byref, GetLastError, POINTER, windll, WINFUNCTYPE + from ctypes.wintypes import BOOL, DWORD, HANDLE, LPWSTR + from ctypes.wintypes import LPVOID # pylint: disable=no-name-in-module - self._DWORD = DWORD - self._byref = byref + self._DWORD = DWORD + self._byref = byref - # - self._WriteConsoleW = WINFUNCTYPE( - BOOL, HANDLE, LPWSTR, DWORD, POINTER(DWORD), LPVOID)( - ('WriteConsoleW', windll.kernel32)) - self._GetLastError = GetLastError + # + self._WriteConsoleW = WINFUNCTYPE(BOOL, HANDLE, LPWSTR, DWORD, + POINTER(DWORD), + LPVOID)(('WriteConsoleW', + windll.kernel32)) + self._GetLastError = GetLastError - def flush(self): - # No need to flush the console since it's immediate. - pass + def flush(self): + # No need to flush the console since it's immediate. + pass - def write(self, text): - try: - if isinstance(text, bytes): - # Bytestrings need to be decoded to a string before being passed to - # Windows. - text = text.decode(self.encoding, 'replace') - remaining = len(text) - while remaining > 0: - n = self._DWORD(0) - # There is a shorter-than-documented limitation on the length of the - # string passed to WriteConsoleW. See - # . - retval = self._WriteConsoleW( - self._console_handle, text, - min(remaining, 10000), - self._byref(n), None) - if retval == 0 or n.value == 0: - raise IOError( - 'WriteConsoleW returned %r, n.value = %r, last error = %r' % ( - retval, n.value, self._GetLastError())) - remaining -= n.value - if not remaining: - break - text = text[int(n.value):] - except Exception as e: - complain('%s.write: %r' % (self.name, e)) - raise + def write(self, text): + try: + if isinstance(text, bytes): + # Bytestrings need to be decoded to a string before being passed + # to Windows. + text = text.decode(self.encoding, 'replace') + remaining = len(text) + while remaining > 0: + n = self._DWORD(0) + # There is a shorter-than-documented limitation on the length of + # the string passed to WriteConsoleW. See + # . + retval = self._WriteConsoleW(self._console_handle, text, + min(remaining, 10000), + self._byref(n), None) + if retval == 0 or n.value == 0: + raise IOError('WriteConsoleW returned %r, n.value = %r, ' + 'last error = %r' % + (retval, n.value, self._GetLastError())) + remaining -= n.value + if not remaining: + break + text = text[int(n.value):] + except Exception as e: + complain('%s.write: %r' % (self.name, e)) + raise class WinUnicodeOutput(WinUnicodeOutputBase): - """Output adaptor to a file output on Windows. + """Output adaptor to a file output on Windows. If the standard FileWrite function is used, it will be encoded in the current code page. WriteConsoleW() permits writing any character. """ - def __init__(self, stream, fileno, encoding): - super(WinUnicodeOutput, self).__init__( - fileno, '' % stream.name, encoding) - # Output stream - self._stream = stream + def __init__(self, stream, fileno, encoding): + super(WinUnicodeOutput, + self).__init__(fileno, '' % stream.name, + encoding) + # Output stream + self._stream = stream - # Flush right now. - self.flush() + # Flush right now. + self.flush() - def flush(self): - try: - self._stream.flush() - except Exception as e: - complain('%s.flush: %r from %r' % (self.name, e, self._stream)) - raise + def flush(self): + try: + self._stream.flush() + except Exception as e: + complain('%s.flush: %r from %r' % (self.name, e, self._stream)) + raise - def write(self, text): - try: - if isinstance(text, bytes): - # Replace characters that cannot be printed instead of failing. - text = text.decode(self.encoding, 'replace') - # When redirecting to a file or process any \n characters will be replaced - # with \r\n. If the text to be printed already has \r\n line endings then - # \r\r\n line endings will be generated, leading to double-spacing of some - # output. Normalizing line endings to \n avoids this problem. - text = text.replace('\r\n', '\n') - self._stream.write(text) - except Exception as e: - complain('%s.write: %r' % (self.name, e)) - raise + def write(self, text): + try: + if isinstance(text, bytes): + # Replace characters that cannot be printed instead of failing. + text = text.decode(self.encoding, 'replace') + # When redirecting to a file or process any \n characters will be + # replaced with \r\n. If the text to be printed already has \r\n + # line endings then \r\r\n line endings will be generated, leading + # to double-spacing of some output. Normalizing line endings to \n + # avoids this problem. + text = text.replace('\r\n', '\n') + self._stream.write(text) + except Exception as e: + complain('%s.write: %r' % (self.name, e)) + raise def win_handle_is_a_console(handle): - """Returns True if a Windows file handle is a handle to a console.""" - # These types are available on linux but not Mac. - # pylint: disable=no-name-in-module,F0401 - from ctypes import byref, POINTER, windll, WINFUNCTYPE - from ctypes.wintypes import BOOL, DWORD, HANDLE + """Returns True if a Windows file handle is a handle to a console.""" + # These types are available on linux but not Mac. + # pylint: disable=no-name-in-module,F0401 + from ctypes import byref, POINTER, windll, WINFUNCTYPE + from ctypes.wintypes import BOOL, DWORD, HANDLE - FILE_TYPE_CHAR = 0x0002 - FILE_TYPE_REMOTE = 0x8000 - INVALID_HANDLE_VALUE = DWORD(-1).value + FILE_TYPE_CHAR = 0x0002 + FILE_TYPE_REMOTE = 0x8000 + INVALID_HANDLE_VALUE = DWORD(-1).value - # - GetConsoleMode = WINFUNCTYPE(BOOL, HANDLE, POINTER(DWORD))( - ('GetConsoleMode', windll.kernel32)) - # - GetFileType = WINFUNCTYPE(DWORD, DWORD)(('GetFileType', windll.kernel32)) + # + GetConsoleMode = WINFUNCTYPE(BOOL, HANDLE, POINTER(DWORD))( + ('GetConsoleMode', windll.kernel32)) + # + GetFileType = WINFUNCTYPE(DWORD, DWORD)(('GetFileType', windll.kernel32)) - # GetStdHandle returns INVALID_HANDLE_VALUE, NULL, or a valid handle. - if handle == INVALID_HANDLE_VALUE or handle is None: - return False - return ( - (GetFileType(handle) & ~FILE_TYPE_REMOTE) == FILE_TYPE_CHAR and - GetConsoleMode(handle, byref(DWORD()))) + # GetStdHandle returns INVALID_HANDLE_VALUE, NULL, or a valid handle. + if handle == INVALID_HANDLE_VALUE or handle is None: + return False + return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) == FILE_TYPE_CHAR + and GetConsoleMode(handle, byref(DWORD()))) def win_get_unicode_stream(stream, excepted_fileno, output_handle, encoding): - """Returns a unicode-compatible stream. + """Returns a unicode-compatible stream. This function will return a direct-Console writing object only if: - the file number is the expected console file number - the handle the expected file handle - the 'real' handle is in fact a handle to a console. """ - old_fileno = getattr(stream, 'fileno', lambda: None)() - if old_fileno == excepted_fileno: - # These types are available on linux but not Mac. - # pylint: disable=no-name-in-module,F0401 - from ctypes import windll, WINFUNCTYPE - from ctypes.wintypes import DWORD, HANDLE + old_fileno = getattr(stream, 'fileno', lambda: None)() + if old_fileno == excepted_fileno: + # These types are available on linux but not Mac. + # pylint: disable=no-name-in-module,F0401 + from ctypes import windll, WINFUNCTYPE + from ctypes.wintypes import DWORD, HANDLE - # - GetStdHandle = WINFUNCTYPE(HANDLE, DWORD)(('GetStdHandle', windll.kernel32)) + # + GetStdHandle = WINFUNCTYPE(HANDLE, + DWORD)(('GetStdHandle', windll.kernel32)) - real_output_handle = GetStdHandle(DWORD(output_handle)) - if win_handle_is_a_console(real_output_handle): - # It's a console. - return WinUnicodeConsoleOutput( - real_output_handle, old_fileno, stream.name, encoding) + real_output_handle = GetStdHandle(DWORD(output_handle)) + if win_handle_is_a_console(real_output_handle): + # It's a console. + return WinUnicodeConsoleOutput(real_output_handle, old_fileno, + stream.name, encoding) - # It's something else. Create an auto-encoding stream. - return WinUnicodeOutput(stream, old_fileno, encoding) + # It's something else. Create an auto-encoding stream. + return WinUnicodeOutput(stream, old_fileno, encoding) def fix_win_console(encoding): - """Makes Unicode console output work independently of the current code page. + """Makes Unicode console output work independently of the current code page. This also fixes . Credit to Michael Kaplan @@ -282,41 +284,41 @@ def fix_win_console(encoding): TZOmegaTZIOY . """ - if (isinstance(sys.stdout, WinUnicodeOutputBase) or - isinstance(sys.stderr, WinUnicodeOutputBase)): - return False + if (isinstance(sys.stdout, WinUnicodeOutputBase) + or isinstance(sys.stderr, WinUnicodeOutputBase)): + return False - try: - # SetConsoleCP and SetConsoleOutputCP could be used to change the code page - # but it's not really useful since the code here is using WriteConsoleW(). - # Also, changing the code page is 'permanent' to the console and needs to be - # reverted manually. - # In practice one needs to set the console font to a TTF font to be able to - # see all the characters but it failed for me in practice. In any case, it - # won't throw any exception when printing, which is the important part. - # -11 and -12 are defined in stdio.h - sys.stdout = win_get_unicode_stream(sys.stdout, 1, -11, encoding) - sys.stderr = win_get_unicode_stream(sys.stderr, 2, -12, encoding) - # TODO(maruel): Do sys.stdin with ReadConsoleW(). Albeit the limitation is - # "It doesn't appear to be possible to read Unicode characters in UTF-8 - # mode" and this appears to be a limitation of cmd.exe. - except Exception as e: - complain('exception %r while fixing up sys.stdout and sys.stderr' % e) - return True + try: + # SetConsoleCP and SetConsoleOutputCP could be used to change the code + # page but it's not really useful since the code here is using + # WriteConsoleW(). Also, changing the code page is 'permanent' to the + # console and needs to be reverted manually. In practice one needs to + # set the console font to a TTF font to be able to see all the + # characters but it failed for me in practice. In any case, it won't + # throw any exception when printing, which is the important part. -11 + # and -12 are defined in stdio.h + sys.stdout = win_get_unicode_stream(sys.stdout, 1, -11, encoding) + sys.stderr = win_get_unicode_stream(sys.stderr, 2, -12, encoding) + # TODO(maruel): Do sys.stdin with ReadConsoleW(). Albeit the limitation + # is "It doesn't appear to be possible to read Unicode characters in + # UTF-8 mode" and this appears to be a limitation of cmd.exe. + except Exception as e: + complain('exception %r while fixing up sys.stdout and sys.stderr' % e) + return True def fix_encoding(): - """Fixes various encoding problems on all platforms. + """Fixes various encoding problems on all platforms. Should be called at the very beginning of the process. """ - ret = True - if sys.platform == 'win32': - ret &= fix_win_codec() + ret = True + if sys.platform == 'win32': + ret &= fix_win_codec() - ret &= fix_default_encoding() + ret &= fix_default_encoding() - if sys.platform == 'win32': - encoding = sys.getdefaultencoding() - ret &= fix_win_console(encoding) - return ret + if sys.platform == 'win32': + encoding = sys.getdefaultencoding() + ret &= fix_win_console(encoding) + return ret diff --git a/gclient-new-workdir.py b/gclient-new-workdir.py index a63022eb67..86f0b3d764 100755 --- a/gclient-new-workdir.py +++ b/gclient-new-workdir.py @@ -18,105 +18,118 @@ import git_common def parse_options(): - if sys.platform == 'win32': - print('ERROR: This script cannot run on Windows because it uses symlinks.') - sys.exit(1) + if sys.platform == 'win32': + print( + 'ERROR: This script cannot run on Windows because it uses symlinks.' + ) + sys.exit(1) - parser = argparse.ArgumentParser(description='''\ + parser = argparse.ArgumentParser(description='''\ Clone an existing gclient directory, taking care of all sub-repositories. Works similarly to 'git new-workdir'.''') - parser.add_argument('repository', type=os.path.abspath, - help='should contain a .gclient file') - parser.add_argument('new_workdir', help='must not exist') - parser.add_argument('--reflink', action='store_true', default=None, - help='''force to use "cp --reflink" for speed and disk + parser.add_argument('repository', + type=os.path.abspath, + help='should contain a .gclient file') + parser.add_argument('new_workdir', help='must not exist') + parser.add_argument('--reflink', + action='store_true', + default=None, + help='''force to use "cp --reflink" for speed and disk space. need supported FS like btrfs or ZFS.''') - parser.add_argument('--no-reflink', action='store_false', dest='reflink', - help='''force not to use "cp --reflink" even on supported + parser.add_argument( + '--no-reflink', + action='store_false', + dest='reflink', + help='''force not to use "cp --reflink" even on supported FS like btrfs or ZFS.''') - args = parser.parse_args() + args = parser.parse_args() - if not os.path.exists(args.repository): - parser.error('Repository "%s" does not exist.' % args.repository) + if not os.path.exists(args.repository): + parser.error('Repository "%s" does not exist.' % args.repository) - gclient = os.path.join(args.repository, '.gclient') - if not os.path.exists(gclient): - parser.error('No .gclient file at "%s".' % gclient) + gclient = os.path.join(args.repository, '.gclient') + if not os.path.exists(gclient): + parser.error('No .gclient file at "%s".' % gclient) - if os.path.exists(args.new_workdir): - parser.error('New workdir "%s" already exists.' % args.new_workdir) + if os.path.exists(args.new_workdir): + parser.error('New workdir "%s" already exists.' % args.new_workdir) - return args + return args def support_cow(src, dest): - # 'cp --reflink' always succeeds when 'src' is a symlink or a directory - assert os.path.isfile(src) and not os.path.islink(src) - try: - subprocess.check_output(['cp', '-a', '--reflink', src, dest], - stderr=subprocess.STDOUT) - except subprocess.CalledProcessError: - return False - finally: - if os.path.isfile(dest): - os.remove(dest) - return True + # 'cp --reflink' always succeeds when 'src' is a symlink or a directory + assert os.path.isfile(src) and not os.path.islink(src) + try: + subprocess.check_output(['cp', '-a', '--reflink', src, dest], + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: + return False + finally: + if os.path.isfile(dest): + os.remove(dest) + return True def try_vol_snapshot(src, dest): - try: - subprocess.check_call(['btrfs', 'subvol', 'snapshot', src, dest], - stderr=subprocess.STDOUT) - except (subprocess.CalledProcessError, OSError): - return False - return True + try: + subprocess.check_call(['btrfs', 'subvol', 'snapshot', src, dest], + stderr=subprocess.STDOUT) + except (subprocess.CalledProcessError, OSError): + return False + return True def main(): - args = parse_options() + args = parse_options() - gclient = os.path.join(args.repository, '.gclient') - if os.path.islink(gclient): - gclient = os.path.realpath(gclient) - new_gclient = os.path.join(args.new_workdir, '.gclient') + gclient = os.path.join(args.repository, '.gclient') + if os.path.islink(gclient): + gclient = os.path.realpath(gclient) + new_gclient = os.path.join(args.new_workdir, '.gclient') - if try_vol_snapshot(args.repository, args.new_workdir): - args.reflink = True - else: - os.makedirs(args.new_workdir) - if args.reflink is None: - args.reflink = support_cow(gclient, new_gclient) - if args.reflink: - print('Copy-on-write support is detected.') - os.symlink(gclient, new_gclient) + if try_vol_snapshot(args.repository, args.new_workdir): + args.reflink = True + else: + os.makedirs(args.new_workdir) + if args.reflink is None: + args.reflink = support_cow(gclient, new_gclient) + if args.reflink: + print('Copy-on-write support is detected.') + os.symlink(gclient, new_gclient) - for root, dirs, _ in os.walk(args.repository): - if '.git' in dirs: - workdir = root.replace(args.repository, args.new_workdir, 1) - print('Creating: %s' % workdir) + for root, dirs, _ in os.walk(args.repository): + if '.git' in dirs: + workdir = root.replace(args.repository, args.new_workdir, 1) + print('Creating: %s' % workdir) - if args.reflink: - if not os.path.exists(workdir): - print('Copying: %s' % workdir) - subprocess.check_call(['cp', '-a', '--reflink', root, workdir]) - shutil.rmtree(os.path.join(workdir, '.git')) + if args.reflink: + if not os.path.exists(workdir): + print('Copying: %s' % workdir) + subprocess.check_call( + ['cp', '-a', '--reflink', root, workdir]) + shutil.rmtree(os.path.join(workdir, '.git')) - git_common.make_workdir(os.path.join(root, '.git'), - os.path.join(workdir, '.git')) - if args.reflink: - subprocess.check_call(['cp', '-a', '--reflink', - os.path.join(root, '.git', 'index'), - os.path.join(workdir, '.git', 'index')]) - else: - subprocess.check_call(['git', 'checkout', '-f'], cwd=workdir) + git_common.make_workdir(os.path.join(root, '.git'), + os.path.join(workdir, '.git')) + if args.reflink: + subprocess.check_call([ + 'cp', '-a', '--reflink', + os.path.join(root, '.git', 'index'), + os.path.join(workdir, '.git', 'index') + ]) + else: + subprocess.check_call(['git', 'checkout', '-f'], cwd=workdir) - if args.reflink: - print(textwrap.dedent('''\ + if args.reflink: + print( + textwrap.dedent('''\ The repo was copied with copy-on-write, and the artifacts were retained. More details on http://crbug.com/721585. Depending on your usage pattern, you might want to do "gn gen" on the output directories. More details: http://crbug.com/723856.''')) + if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/gclient.py b/gclient.py index 741f16a3ec..c305c2480e 100755 --- a/gclient.py +++ b/gclient.py @@ -2,7 +2,6 @@ # Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Meta checkout dependency manager for Git.""" # Files # .gclient : Current client configuration, written by 'config' command. @@ -111,6 +110,8 @@ import subcommand import subprocess2 from third_party.repo.progress import Progress +# TODO: Should fix these warnings. +# pylint: disable=line-too-long DEPOT_TOOLS_DIR = os.path.dirname(os.path.abspath(os.path.realpath(__file__))) @@ -127,34 +128,40 @@ NO_SYNC_EXPERIMENT = 'no-sync' class GNException(Exception): - pass + pass def ToGNString(value): - """Returns a stringified GN equivalent of the Python value.""" - if isinstance(value, str): - if value.find('\n') >= 0: - raise GNException("Trying to print a string with a newline in it.") - return '"' + \ - value.replace('\\', '\\\\').replace('"', '\\"').replace('$', '\\$') + \ - '"' + """Returns a stringified GN equivalent of the Python value.""" + if isinstance(value, str): + if value.find('\n') >= 0: + raise GNException("Trying to print a string with a newline in it.") + return '"' + \ + value.replace('\\', '\\\\').replace('"', '\\"').replace('$', '\\$') + \ + '"' - if isinstance(value, bool): - if value: - return "true" - return "false" + if isinstance(value, bool): + if value: + return "true" + return "false" - # NOTE: some type handling removed compared to chromium/src copy. + # NOTE: some type handling removed compared to chromium/src copy. - raise GNException("Unsupported type when printing to GN.") + raise GNException("Unsupported type when printing to GN.") class Hook(object): - """Descriptor of command ran before/after sync or on demand.""" - - def __init__(self, action, pattern=None, name=None, cwd=None, condition=None, - variables=None, verbose=False, cwd_base=None): - """Constructor. + """Descriptor of command ran before/after sync or on demand.""" + def __init__(self, + action, + pattern=None, + name=None, + cwd=None, + condition=None, + variables=None, + verbose=False, + cwd_base=None): + """Constructor. Arguments: action (list of str): argv of the command to run @@ -164,857 +171,887 @@ class Hook(object): condition (str): condition when to run the hook variables (dict): variables for evaluating the condition """ - self._action = gclient_utils.freeze(action) - self._pattern = pattern - self._name = name - self._cwd = cwd - self._condition = condition - self._variables = variables - self._verbose = verbose - self._cwd_base = cwd_base + self._action = gclient_utils.freeze(action) + self._pattern = pattern + self._name = name + self._cwd = cwd + self._condition = condition + self._variables = variables + self._verbose = verbose + self._cwd_base = cwd_base - @staticmethod - def from_dict(d, variables=None, verbose=False, conditions=None, - cwd_base=None): - """Creates a Hook instance from a dict like in the DEPS file.""" - # Merge any local and inherited conditions. - gclient_eval.UpdateCondition(d, 'and', conditions) - return Hook( - d['action'], - d.get('pattern'), - d.get('name'), - d.get('cwd'), - d.get('condition'), - variables=variables, - # Always print the header if not printing to a TTY. - verbose=verbose or not setup_color.IS_TTY, - cwd_base=cwd_base) + @staticmethod + def from_dict(d, + variables=None, + verbose=False, + conditions=None, + cwd_base=None): + """Creates a Hook instance from a dict like in the DEPS file.""" + # Merge any local and inherited conditions. + gclient_eval.UpdateCondition(d, 'and', conditions) + return Hook( + d['action'], + d.get('pattern'), + d.get('name'), + d.get('cwd'), + d.get('condition'), + variables=variables, + # Always print the header if not printing to a TTY. + verbose=verbose or not setup_color.IS_TTY, + cwd_base=cwd_base) - @property - def action(self): - return self._action + @property + def action(self): + return self._action - @property - def pattern(self): - return self._pattern + @property + def pattern(self): + return self._pattern - @property - def name(self): - return self._name + @property + def name(self): + return self._name - @property - def condition(self): - return self._condition + @property + def condition(self): + return self._condition - @property - def effective_cwd(self): - cwd = self._cwd_base - if self._cwd: - cwd = os.path.join(cwd, self._cwd) - return cwd + @property + def effective_cwd(self): + cwd = self._cwd_base + if self._cwd: + cwd = os.path.join(cwd, self._cwd) + return cwd - def matches(self, file_list): - """Returns true if the pattern matches any of files in the list.""" - if not self._pattern: - return True - pattern = re.compile(self._pattern) - return bool([f for f in file_list if pattern.search(f)]) + def matches(self, file_list): + """Returns true if the pattern matches any of files in the list.""" + if not self._pattern: + return True + pattern = re.compile(self._pattern) + return bool([f for f in file_list if pattern.search(f)]) - def run(self): - """Executes the hook's command (provided the condition is met).""" - if (self._condition and - not gclient_eval.EvaluateCondition(self._condition, self._variables)): - return + def run(self): + """Executes the hook's command (provided the condition is met).""" + if (self._condition and not gclient_eval.EvaluateCondition( + self._condition, self._variables)): + return - cmd = list(self._action) + cmd = list(self._action) - if cmd[0] == 'vpython3' and _detect_host_os() == 'win': - cmd[0] += '.bat' + if cmd[0] == 'vpython3' and _detect_host_os() == 'win': + cmd[0] += '.bat' - exit_code = 2 - try: - start_time = time.time() - gclient_utils.CheckCallAndFilter( - cmd, cwd=self.effective_cwd, print_stdout=True, show_header=True, - always_show_header=self._verbose) - exit_code = 0 - except (gclient_utils.Error, subprocess2.CalledProcessError) as e: - # Use a discrete exit status code of 2 to indicate that a hook action - # failed. Users of this script may wish to treat hook action failures - # differently from VC failures. - print('Error: %s' % str(e), file=sys.stderr) - sys.exit(exit_code) - finally: - elapsed_time = time.time() - start_time - metrics.collector.add_repeated('hooks', { - 'action': gclient_utils.CommandToStr(cmd), - 'name': self._name, - 'cwd': os.path.relpath( - os.path.normpath(self.effective_cwd), - self._cwd_base), - 'condition': self._condition, - 'execution_time': elapsed_time, - 'exit_code': exit_code, - }) - if elapsed_time > 10: - print("Hook '%s' took %.2f secs" % ( - gclient_utils.CommandToStr(cmd), elapsed_time)) + exit_code = 2 + try: + start_time = time.time() + gclient_utils.CheckCallAndFilter(cmd, + cwd=self.effective_cwd, + print_stdout=True, + show_header=True, + always_show_header=self._verbose) + exit_code = 0 + except (gclient_utils.Error, subprocess2.CalledProcessError) as e: + # Use a discrete exit status code of 2 to indicate that a hook + # action failed. Users of this script may wish to treat hook action + # failures differently from VC failures. + print('Error: %s' % str(e), file=sys.stderr) + sys.exit(exit_code) + finally: + elapsed_time = time.time() - start_time + metrics.collector.add_repeated( + 'hooks', { + 'action': + gclient_utils.CommandToStr(cmd), + 'name': + self._name, + 'cwd': + os.path.relpath(os.path.normpath(self.effective_cwd), + self._cwd_base), + 'condition': + self._condition, + 'execution_time': + elapsed_time, + 'exit_code': + exit_code, + }) + if elapsed_time > 10: + print("Hook '%s' took %.2f secs" % + (gclient_utils.CommandToStr(cmd), elapsed_time)) class DependencySettings(object): - """Immutable configuration settings.""" - def __init__( - self, parent, url, managed, custom_deps, custom_vars, - custom_hooks, deps_file, should_process, relative, condition): - # These are not mutable: - self._parent = parent - self._deps_file = deps_file - self._url = url - # The condition as string (or None). Useful to keep e.g. for flatten. - self._condition = condition - # 'managed' determines whether or not this dependency is synced/updated by - # gclient after gclient checks it out initially. The difference between - # 'managed' and 'should_process' is that the user specifies 'managed' via - # the --unmanaged command-line flag or a .gclient config, where - # 'should_process' is dynamically set by gclient if it goes over its - # recursion limit and controls gclient's behavior so it does not misbehave. - self._managed = managed - self._should_process = should_process - # If this is a recursed-upon sub-dependency, and the parent has - # use_relative_paths set, then this dependency should check out its own - # dependencies relative to that parent's path for this, rather than - # relative to the .gclient file. - self._relative = relative - # This is a mutable value which has the list of 'target_os' OSes listed in - # the current deps file. - self.local_target_os = None + """Immutable configuration settings.""" + def __init__(self, parent, url, managed, custom_deps, custom_vars, + custom_hooks, deps_file, should_process, relative, condition): + # These are not mutable: + self._parent = parent + self._deps_file = deps_file + self._url = url + # The condition as string (or None). Useful to keep e.g. for flatten. + self._condition = condition + # 'managed' determines whether or not this dependency is synced/updated + # by gclient after gclient checks it out initially. The difference + # between 'managed' and 'should_process' is that the user specifies + # 'managed' via the --unmanaged command-line flag or a .gclient config, + # where 'should_process' is dynamically set by gclient if it goes over + # its recursion limit and controls gclient's behavior so it does not + # misbehave. + self._managed = managed + self._should_process = should_process + # If this is a recursed-upon sub-dependency, and the parent has + # use_relative_paths set, then this dependency should check out its own + # dependencies relative to that parent's path for this, rather than + # relative to the .gclient file. + self._relative = relative + # This is a mutable value which has the list of 'target_os' OSes listed + # in the current deps file. + self.local_target_os = None - # These are only set in .gclient and not in DEPS files. - self._custom_vars = custom_vars or {} - self._custom_deps = custom_deps or {} - self._custom_hooks = custom_hooks or [] + # These are only set in .gclient and not in DEPS files. + self._custom_vars = custom_vars or {} + self._custom_deps = custom_deps or {} + self._custom_hooks = custom_hooks or [] - # Post process the url to remove trailing slashes. - if isinstance(self.url, str): - # urls are sometime incorrectly written as proto://host/path/@rev. Replace - # it to proto://host/path@rev. - self.set_url(self.url.replace('/@', '@')) - elif not isinstance(self.url, (None.__class__)): - raise gclient_utils.Error( - ('dependency url must be either string or None, ' - 'instead of %s') % self.url.__class__.__name__) + # Post process the url to remove trailing slashes. + if isinstance(self.url, str): + # urls are sometime incorrectly written as proto://host/path/@rev. + # Replace it to proto://host/path@rev. + self.set_url(self.url.replace('/@', '@')) + elif not isinstance(self.url, (None.__class__)): + raise gclient_utils.Error( + ('dependency url must be either string or None, ' + 'instead of %s') % self.url.__class__.__name__) - # Make any deps_file path platform-appropriate. - if self._deps_file: - for sep in ['/', '\\']: - self._deps_file = self._deps_file.replace(sep, os.sep) + # Make any deps_file path platform-appropriate. + if self._deps_file: + for sep in ['/', '\\']: + self._deps_file = self._deps_file.replace(sep, os.sep) - @property - def deps_file(self): - return self._deps_file + @property + def deps_file(self): + return self._deps_file - @property - def managed(self): - return self._managed + @property + def managed(self): + return self._managed - @property - def parent(self): - return self._parent + @property + def parent(self): + return self._parent - @property - def root(self): - """Returns the root node, a GClient object.""" - if not self.parent: - # This line is to signal pylint that it could be a GClient instance. - return self or GClient(None, None) - return self.parent.root + @property + def root(self): + """Returns the root node, a GClient object.""" + if not self.parent: + # This line is to signal pylint that it could be a GClient instance. + return self or GClient(None, None) + return self.parent.root - @property - def should_process(self): - """True if this dependency should be processed, i.e. checked out.""" - return self._should_process + @property + def should_process(self): + """True if this dependency should be processed, i.e. checked out.""" + return self._should_process - @property - def custom_vars(self): - return self._custom_vars.copy() + @property + def custom_vars(self): + return self._custom_vars.copy() - @property - def custom_deps(self): - return self._custom_deps.copy() + @property + def custom_deps(self): + return self._custom_deps.copy() - @property - def custom_hooks(self): - return self._custom_hooks[:] + @property + def custom_hooks(self): + return self._custom_hooks[:] - @property - def url(self): - """URL after variable expansion.""" - return self._url + @property + def url(self): + """URL after variable expansion.""" + return self._url - @property - def condition(self): - return self._condition + @property + def condition(self): + return self._condition - @property - def target_os(self): - if self.local_target_os is not None: - return tuple(set(self.local_target_os).union(self.parent.target_os)) + @property + def target_os(self): + if self.local_target_os is not None: + return tuple(set(self.local_target_os).union(self.parent.target_os)) - return self.parent.target_os + return self.parent.target_os - @property - def target_cpu(self): - return self.parent.target_cpu + @property + def target_cpu(self): + return self.parent.target_cpu - def set_url(self, url): - self._url = url + def set_url(self, url): + self._url = url - def get_custom_deps(self, name, url): - """Returns a custom deps if applicable.""" - if self.parent: - url = self.parent.get_custom_deps(name, url) - # None is a valid return value to disable a dependency. - return self.custom_deps.get(name, url) + def get_custom_deps(self, name, url): + """Returns a custom deps if applicable.""" + if self.parent: + url = self.parent.get_custom_deps(name, url) + # None is a valid return value to disable a dependency. + return self.custom_deps.get(name, url) class Dependency(gclient_utils.WorkItem, DependencySettings): - """Object that represents a dependency checkout.""" + """Object that represents a dependency checkout.""" + def __init__(self, + parent, + name, + url, + managed, + custom_deps, + custom_vars, + custom_hooks, + deps_file, + should_process, + should_recurse, + relative, + condition, + protocol='https', + git_dependencies_state=gclient_eval.DEPS, + print_outbuf=False): + gclient_utils.WorkItem.__init__(self, name) + DependencySettings.__init__(self, parent, url, managed, custom_deps, + custom_vars, custom_hooks, deps_file, + should_process, relative, condition) - def __init__(self, - parent, - name, - url, - managed, - custom_deps, - custom_vars, - custom_hooks, - deps_file, - should_process, - should_recurse, - relative, - condition, - protocol='https', - git_dependencies_state=gclient_eval.DEPS, - print_outbuf=False): - gclient_utils.WorkItem.__init__(self, name) - DependencySettings.__init__( - self, parent, url, managed, custom_deps, custom_vars, - custom_hooks, deps_file, should_process, relative, condition) + # This is in both .gclient and DEPS files: + self._deps_hooks = [] - # This is in both .gclient and DEPS files: - self._deps_hooks = [] + self._pre_deps_hooks = [] - self._pre_deps_hooks = [] + # Calculates properties: + self._dependencies = [] + self._vars = {} - # Calculates properties: - self._dependencies = [] - self._vars = {} + # A cache of the files affected by the current operation, necessary for + # hooks. + self._file_list = [] + # List of host names from which dependencies are allowed. + # Default is an empty set, meaning unspecified in DEPS file, and hence + # all hosts will be allowed. Non-empty set means allowlist of hosts. + # allowed_hosts var is scoped to its DEPS file, and so it isn't + # recursive. + self._allowed_hosts = frozenset() + self._gn_args_from = None + # Spec for .gni output to write (if any). + self._gn_args_file = None + self._gn_args = [] + # If it is not set to True, the dependency wasn't processed for its + # child dependency, i.e. its DEPS wasn't read. + self._deps_parsed = False + # This dependency has been processed, i.e. checked out + self._processed = False + # This dependency had its pre-DEPS hooks run + self._pre_deps_hooks_ran = False + # This dependency had its hook run + self._hooks_ran = False + # This is the scm used to checkout self.url. It may be used by + # dependencies to get the datetime of the revision we checked out. + self._used_scm = None + self._used_revision = None + # The actual revision we ended up getting, or None if that information + # is unavailable + self._got_revision = None + # Whether this dependency should use relative paths. + self._use_relative_paths = False - # A cache of the files affected by the current operation, necessary for - # hooks. - self._file_list = [] - # List of host names from which dependencies are allowed. - # Default is an empty set, meaning unspecified in DEPS file, and hence all - # hosts will be allowed. Non-empty set means allowlist of hosts. - # allowed_hosts var is scoped to its DEPS file, and so it isn't recursive. - self._allowed_hosts = frozenset() - self._gn_args_from = None - # Spec for .gni output to write (if any). - self._gn_args_file = None - self._gn_args = [] - # If it is not set to True, the dependency wasn't processed for its child - # dependency, i.e. its DEPS wasn't read. - self._deps_parsed = False - # This dependency has been processed, i.e. checked out - self._processed = False - # This dependency had its pre-DEPS hooks run - self._pre_deps_hooks_ran = False - # This dependency had its hook run - self._hooks_ran = False - # This is the scm used to checkout self.url. It may be used by dependencies - # to get the datetime of the revision we checked out. - self._used_scm = None - self._used_revision = None - # The actual revision we ended up getting, or None if that information is - # unavailable - self._got_revision = None - # Whether this dependency should use relative paths. - self._use_relative_paths = False + # recursedeps is a mutable value that selectively overrides the default + # 'no recursion' setting on a dep-by-dep basis. + # + # It will be a dictionary of {deps_name: depfile_namee} + self.recursedeps = {} - # recursedeps is a mutable value that selectively overrides the default - # 'no recursion' setting on a dep-by-dep basis. - # - # It will be a dictionary of {deps_name: depfile_namee} - self.recursedeps = {} + # Whether we should process this dependency's DEPS file. + self._should_recurse = should_recurse - # Whether we should process this dependency's DEPS file. - self._should_recurse = should_recurse + # Whether we should sync git/cipd dependencies and hooks from the + # DEPS file. + # This is set based on skip_sync_revisions and must be done + # after the patch refs are applied. + # If this is False, we will still run custom_hooks and process + # custom_deps, if any. + self._should_sync = True - # Whether we should sync git/cipd dependencies and hooks from the - # DEPS file. - # This is set based on skip_sync_revisions and must be done - # after the patch refs are applied. - # If this is False, we will still run custom_hooks and process - # custom_deps, if any. - self._should_sync = True + self._OverrideUrl() + # This is inherited from WorkItem. We want the URL to be a resource. + if self.url and isinstance(self.url, str): + # The url is usually given to gclient either as https://blah@123 + # or just https://blah. The @123 portion is irrelevant. + self.resources.append(self.url.split('@')[0]) - self._OverrideUrl() - # This is inherited from WorkItem. We want the URL to be a resource. - if self.url and isinstance(self.url, str): - # The url is usually given to gclient either as https://blah@123 - # or just https://blah. The @123 portion is irrelevant. - self.resources.append(self.url.split('@')[0]) + # Controls whether we want to print git's output when we first clone the + # dependency + self.print_outbuf = print_outbuf - # Controls whether we want to print git's output when we first clone the - # dependency - self.print_outbuf = print_outbuf + self.protocol = protocol + self.git_dependencies_state = git_dependencies_state - self.protocol = protocol - self.git_dependencies_state = git_dependencies_state + if not self.name and self.parent: + raise gclient_utils.Error('Dependency without name') - if not self.name and self.parent: - raise gclient_utils.Error('Dependency without name') + def _OverrideUrl(self): + """Resolves the parsed url from the parent hierarchy.""" + parsed_url = self.get_custom_deps( + self._name.replace(os.sep, posixpath.sep) \ + if self._name else self._name, self.url) + if parsed_url != self.url: + logging.info('Dependency(%s)._OverrideUrl(%s) -> %s', self._name, + self.url, parsed_url) + self.set_url(parsed_url) + return - def _OverrideUrl(self): - """Resolves the parsed url from the parent hierarchy.""" - parsed_url = self.get_custom_deps( - self._name.replace(os.sep, posixpath.sep) \ - if self._name else self._name, self.url) - if parsed_url != self.url: - logging.info('Dependency(%s)._OverrideUrl(%s) -> %s', self._name, - self.url, parsed_url) - self.set_url(parsed_url) - return + if self.url is None: + logging.info('Dependency(%s)._OverrideUrl(None) -> None', + self._name) + return - if self.url is None: - logging.info('Dependency(%s)._OverrideUrl(None) -> None', self._name) - return + if not isinstance(self.url, str): + raise gclient_utils.Error('Unknown url type') - if not isinstance(self.url, str): - raise gclient_utils.Error('Unknown url type') + # self.url is a local path + path, at, rev = self.url.partition('@') + if os.path.isdir(path): + return - # self.url is a local path - path, at, rev = self.url.partition('@') - if os.path.isdir(path): - return + # self.url is a URL + parsed_url = urllib.parse.urlparse(self.url) + if parsed_url[0] or re.match(r'^\w+\@[\w\.-]+\:[\w\/]+', parsed_url[2]): + return - # self.url is a URL - parsed_url = urllib.parse.urlparse(self.url) - if parsed_url[0] or re.match(r'^\w+\@[\w\.-]+\:[\w\/]+', parsed_url[2]): - return + # self.url is relative to the parent's URL. + if not path.startswith('/'): + raise gclient_utils.Error( + 'relative DEPS entry \'%s\' must begin with a slash' % self.url) - # self.url is relative to the parent's URL. - if not path.startswith('/'): - raise gclient_utils.Error( - 'relative DEPS entry \'%s\' must begin with a slash' % self.url) + parent_url = self.parent.url + parent_path = self.parent.url.split('@')[0] + if os.path.isdir(parent_path): + # Parent's URL is a local path. Get parent's URL dirname and append + # self.url. + parent_path = os.path.dirname(parent_path) + parsed_url = parent_path + path.replace('/', os.sep) + at + rev + else: + # Parent's URL is a URL. Get parent's URL, strip from the last '/' + # (equivalent to unix dirname) and append self.url. + parsed_url = parent_url[:parent_url.rfind('/')] + self.url - parent_url = self.parent.url - parent_path = self.parent.url.split('@')[0] - if os.path.isdir(parent_path): - # Parent's URL is a local path. Get parent's URL dirname and append - # self.url. - parent_path = os.path.dirname(parent_path) - parsed_url = parent_path + path.replace('/', os.sep) + at + rev - else: - # Parent's URL is a URL. Get parent's URL, strip from the last '/' - # (equivalent to unix dirname) and append self.url. - parsed_url = parent_url[:parent_url.rfind('/')] + self.url + logging.info('Dependency(%s)._OverrideUrl(%s) -> %s', self.name, + self.url, parsed_url) + self.set_url(parsed_url) - logging.info('Dependency(%s)._OverrideUrl(%s) -> %s', self.name, - self.url, parsed_url) - self.set_url(parsed_url) + def PinToActualRevision(self): + """Updates self.url to the revision checked out on disk.""" + if self.url is None: + return + url = None + scm = self.CreateSCM() + if scm.name == 'cipd': + revision = scm.revinfo(None, None, None) + package = self.GetExpandedPackageName() + url = '%s/p/%s/+/%s' % (scm.GetActualRemoteURL(None), package, + revision) - def PinToActualRevision(self): - """Updates self.url to the revision checked out on disk.""" - if self.url is None: - return - url = None - scm = self.CreateSCM() - if scm.name == 'cipd': - revision = scm.revinfo(None, None, None) - package = self.GetExpandedPackageName() - url = '%s/p/%s/+/%s' % (scm.GetActualRemoteURL(None), package, revision) + if os.path.isdir(scm.checkout_path): + revision = scm.revinfo(None, None, None) + url = '%s@%s' % (gclient_utils.SplitUrlRevision( + self.url)[0], revision) + self.set_url(url) - if os.path.isdir(scm.checkout_path): - revision = scm.revinfo(None, None, None) - url = '%s@%s' % (gclient_utils.SplitUrlRevision(self.url)[0], revision) - self.set_url(url) + def ToLines(self): + # () -> Sequence[str] + """Returns strings representing the deps (info, graphviz line)""" + s = [] + condition_part = ([' "condition": %r,' % + self.condition] if self.condition else []) + s.extend([ + ' # %s' % self.hierarchy(include_url=False), + ' "%s": {' % (self.name, ), + ' "url": "%s",' % (self.url, ), + ] + condition_part + [ + ' },', + '', + ]) + return s - def ToLines(self): - # () -> Sequence[str] - """Returns strings representing the deps (info, graphviz line)""" - s = [] - condition_part = ([' "condition": %r,' % self.condition] - if self.condition else []) - s.extend([ - ' # %s' % self.hierarchy(include_url=False), - ' "%s": {' % (self.name,), - ' "url": "%s",' % (self.url,), - ] + condition_part + [ - ' },', - '', - ]) - return s + @property + def requirements(self): + """Calculate the list of requirements.""" + requirements = set() + # self.parent is implicitly a requirement. This will be recursive by + # definition. + if self.parent and self.parent.name: + requirements.add(self.parent.name) - @property - def requirements(self): - """Calculate the list of requirements.""" - requirements = set() - # self.parent is implicitly a requirement. This will be recursive by - # definition. - if self.parent and self.parent.name: - requirements.add(self.parent.name) + # For a tree with at least 2 levels*, the leaf node needs to depend + # on the level higher up in an orderly way. + # This becomes messy for >2 depth as the DEPS file format is a + # dictionary, thus unsorted, while the .gclient format is a list thus + # sorted. + # + # Interestingly enough, the following condition only works in the case + # we want: self is a 2nd level node. 3rd level node wouldn't need this + # since they already have their parent as a requirement. + if self.parent and self.parent.parent and not self.parent.parent.parent: + requirements |= set(i.name for i in self.root.dependencies + if i.name) - # For a tree with at least 2 levels*, the leaf node needs to depend - # on the level higher up in an orderly way. - # This becomes messy for >2 depth as the DEPS file format is a dictionary, - # thus unsorted, while the .gclient format is a list thus sorted. - # - # Interestingly enough, the following condition only works in the case we - # want: self is a 2nd level node. 3rd level node wouldn't need this since - # they already have their parent as a requirement. - if self.parent and self.parent.parent and not self.parent.parent.parent: - requirements |= set(i.name for i in self.root.dependencies if i.name) + if self.name: + requirements |= set( + obj.name for obj in self.root.subtree(False) + if (obj is not self and obj.name + and self.name.startswith(posixpath.join(obj.name, '')))) + requirements = tuple(sorted(requirements)) + logging.info('Dependency(%s).requirements = %s' % + (self.name, requirements)) + return requirements - if self.name: - requirements |= set( - obj.name for obj in self.root.subtree(False) - if (obj is not self - and obj.name and - self.name.startswith(posixpath.join(obj.name, '')))) - requirements = tuple(sorted(requirements)) - logging.info('Dependency(%s).requirements = %s' % (self.name, requirements)) - return requirements + @property + def should_recurse(self): + return self._should_recurse - @property - def should_recurse(self): - return self._should_recurse - - def verify_validity(self): - """Verifies that this Dependency is fine to add as a child of another one. + def verify_validity(self): + """Verifies that this Dependency is fine to add as a child of another one. Returns True if this entry should be added, False if it is a duplicate of another entry. """ - logging.info('Dependency(%s).verify_validity()' % self.name) - if self.name in [s.name for s in self.parent.dependencies]: - raise gclient_utils.Error( - 'The same name "%s" appears multiple times in the deps section' % - self.name) - if not self.should_process: - # Return early, no need to set requirements. - return not any(d.name == self.name for d in self.root.subtree(True)) + logging.info('Dependency(%s).verify_validity()' % self.name) + if self.name in [s.name for s in self.parent.dependencies]: + raise gclient_utils.Error( + 'The same name "%s" appears multiple times in the deps section' + % self.name) + if not self.should_process: + # Return early, no need to set requirements. + return not any(d.name == self.name for d in self.root.subtree(True)) - # This require a full tree traversal with locks. - siblings = [d for d in self.root.subtree(False) if d.name == self.name] - for sibling in siblings: - # Allow to have only one to be None or ''. - if self.url != sibling.url and bool(self.url) == bool(sibling.url): - raise gclient_utils.Error( - ('Dependency %s specified more than once:\n' - ' %s [%s]\n' - 'vs\n' - ' %s [%s]') % ( - self.name, - sibling.hierarchy(), - sibling.url, - self.hierarchy(), - self.url)) - # In theory we could keep it as a shadow of the other one. In - # practice, simply ignore it. - logging.warning("Won't process duplicate dependency %s" % sibling) - return False - return True + # This require a full tree traversal with locks. + siblings = [d for d in self.root.subtree(False) if d.name == self.name] + for sibling in siblings: + # Allow to have only one to be None or ''. + if self.url != sibling.url and bool(self.url) == bool(sibling.url): + raise gclient_utils.Error( + ('Dependency %s specified more than once:\n' + ' %s [%s]\n' + 'vs\n' + ' %s [%s]') % (self.name, sibling.hierarchy(), + sibling.url, self.hierarchy(), self.url)) + # In theory we could keep it as a shadow of the other one. In + # practice, simply ignore it. + logging.warning("Won't process duplicate dependency %s" % sibling) + return False + return True - def _postprocess_deps(self, deps, rel_prefix): - # type: (Mapping[str, Mapping[str, str]], str) -> - # Mapping[str, Mapping[str, str]] - """Performs post-processing of deps compared to what's in the DEPS file.""" - # If we don't need to sync, only process custom_deps, if any. - if not self._should_sync: - if not self.custom_deps: - return {} + def _postprocess_deps(self, deps, rel_prefix): + # type: (Mapping[str, Mapping[str, str]], str) -> + # Mapping[str, Mapping[str, str]] + """Performs post-processing of deps compared to what's in the DEPS file.""" + # If we don't need to sync, only process custom_deps, if any. + if not self._should_sync: + if not self.custom_deps: + return {} - processed_deps = {} - for dep_name, dep_info in self.custom_deps.items(): - if dep_info and not dep_info.endswith('@unmanaged'): - if dep_name in deps: - # custom_deps that should override an existing deps gets applied - # in the Dependency itself with _OverrideUrl(). - processed_deps[dep_name] = deps[dep_name] - else: - processed_deps[dep_name] = {'url': dep_info, 'dep_type': 'git'} - else: - processed_deps = dict(deps) + processed_deps = {} + for dep_name, dep_info in self.custom_deps.items(): + if dep_info and not dep_info.endswith('@unmanaged'): + if dep_name in deps: + # custom_deps that should override an existing deps gets + # applied in the Dependency itself with _OverrideUrl(). + processed_deps[dep_name] = deps[dep_name] + else: + processed_deps[dep_name] = { + 'url': dep_info, + 'dep_type': 'git' + } + else: + processed_deps = dict(deps) - # If a line is in custom_deps, but not in the solution, we want to append - # this line to the solution. - for dep_name, dep_info in self.custom_deps.items(): - # Don't add it to the solution for the values of "None" and "unmanaged" - # in order to force these kinds of custom_deps to act as revision - # overrides (via revision_overrides). Having them function as revision - # overrides allows them to be applied to recursive dependencies. - # https://crbug.com/1031185 - if (dep_name not in processed_deps and dep_info - and not dep_info.endswith('@unmanaged')): - processed_deps[dep_name] = {'url': dep_info, 'dep_type': 'git'} + # If a line is in custom_deps, but not in the solution, we want to + # append this line to the solution. + for dep_name, dep_info in self.custom_deps.items(): + # Don't add it to the solution for the values of "None" and + # "unmanaged" in order to force these kinds of custom_deps to + # act as revision overrides (via revision_overrides). Having + # them function as revision overrides allows them to be applied + # to recursive dependencies. https://crbug.com/1031185 + if (dep_name not in processed_deps and dep_info + and not dep_info.endswith('@unmanaged')): + processed_deps[dep_name] = { + 'url': dep_info, + 'dep_type': 'git' + } - # Make child deps conditional on any parent conditions. This ensures that, - # when flattened, recursed entries have the correct restrictions, even if - # not explicitly set in the recursed DEPS file. For instance, if - # "src/ios_foo" is conditional on "checkout_ios=True", then anything - # recursively included by "src/ios_foo/DEPS" should also require - # "checkout_ios=True". - if self.condition: - for value in processed_deps.values(): - gclient_eval.UpdateCondition(value, 'and', self.condition) + # Make child deps conditional on any parent conditions. This ensures + # that, when flattened, recursed entries have the correct restrictions, + # even if not explicitly set in the recursed DEPS file. For instance, if + # "src/ios_foo" is conditional on "checkout_ios=True", then anything + # recursively included by "src/ios_foo/DEPS" should also require + # "checkout_ios=True". + if self.condition: + for value in processed_deps.values(): + gclient_eval.UpdateCondition(value, 'and', self.condition) - if not rel_prefix: - return processed_deps + if not rel_prefix: + return processed_deps - logging.warning('use_relative_paths enabled.') - rel_deps = {} - for d, url in processed_deps.items(): - # normpath is required to allow DEPS to use .. in their - # dependency local path. - # We are following the same pattern when use_relative_paths = False, - # which uses slashes. - rel_deps[os.path.normpath(os.path.join(rel_prefix, - d)).replace(os.path.sep, - '/')] = url - logging.warning('Updating deps by prepending %s.', rel_prefix) - return rel_deps - - def _deps_to_objects(self, deps, use_relative_paths): - # type: (Mapping[str, Mapping[str, str]], bool) -> Sequence[Dependency] - """Convert a deps dict to a list of Dependency objects.""" - deps_to_add = [] - cached_conditions = {} - for name, dep_value in deps.items(): - should_process = self.should_process - if dep_value is None: - continue - - condition = dep_value.get('condition') - dep_type = dep_value.get('dep_type') - - - if condition and not self._get_option('process_all_deps', False): - if condition not in cached_conditions: - cached_conditions[condition] = gclient_eval.EvaluateCondition( - condition, self.get_vars()) - should_process = should_process and cached_conditions[condition] - - # The following option is only set by the 'revinfo' command. - if self._get_option('ignore_dep_type', None) == dep_type: - continue - - if dep_type == 'cipd': - cipd_root = self.GetCipdRoot() - for package in dep_value.get('packages', []): - deps_to_add.append( - CipdDependency( - parent=self, - name=name, - dep_value=package, - cipd_root=cipd_root, - custom_vars=self.custom_vars, - should_process=should_process, - relative=use_relative_paths, - condition=condition)) - else: - url = dep_value.get('url') - deps_to_add.append( - GitDependency( - parent=self, - name=name, - # Update URL with scheme in protocol_override - url=GitDependency.updateProtocol(url, self.protocol), - managed=True, - custom_deps=None, - custom_vars=self.custom_vars, - custom_hooks=None, - deps_file=self.recursedeps.get(name, self.deps_file), - should_process=should_process, - should_recurse=name in self.recursedeps, - relative=use_relative_paths, - condition=condition, - protocol=self.protocol, - git_dependencies_state=self.git_dependencies_state)) - - # TODO(crbug.com/1341285): Understand why we need this and remove - # it if we don't. - deps_to_add.sort(key=lambda x: x.name) - return deps_to_add - - def ParseDepsFile(self): - # type: () -> None - """Parses the DEPS file for this dependency.""" - assert not self.deps_parsed - assert not self.dependencies - - deps_content = None - - # First try to locate the configured deps file. If it's missing, fallback - # to DEPS. - deps_files = [self.deps_file] - if 'DEPS' not in deps_files: - deps_files.append('DEPS') - for deps_file in deps_files: - filepath = os.path.join(self.root.root_dir, self.name, deps_file) - if os.path.isfile(filepath): - logging.info( - 'ParseDepsFile(%s): %s file found at %s', self.name, deps_file, - filepath) - break - logging.info( - 'ParseDepsFile(%s): No %s file found at %s', self.name, deps_file, - filepath) - - if os.path.isfile(filepath): - deps_content = gclient_utils.FileRead(filepath) - logging.debug('ParseDepsFile(%s) read:\n%s', self.name, deps_content) - - local_scope = {} - if deps_content: - try: - local_scope = gclient_eval.Parse( - deps_content, filepath, self.get_vars(), self.get_builtin_vars()) - except SyntaxError as e: - gclient_utils.SyntaxErrorToError(filepath, e) - - if 'git_dependencies' in local_scope: - self.git_dependencies_state = local_scope['git_dependencies'] - - if 'allowed_hosts' in local_scope: - try: - self._allowed_hosts = frozenset(local_scope.get('allowed_hosts')) - except TypeError: # raised if non-iterable - pass - if not self._allowed_hosts: - logging.warning("allowed_hosts is specified but empty %s", - self._allowed_hosts) - raise gclient_utils.Error( - 'ParseDepsFile(%s): allowed_hosts must be absent ' - 'or a non-empty iterable' % self.name) - - self._gn_args_from = local_scope.get('gclient_gn_args_from') - self._gn_args_file = local_scope.get('gclient_gn_args_file') - self._gn_args = local_scope.get('gclient_gn_args', []) - # It doesn't make sense to set all of these, since setting gn_args_from to - # another DEPS will make gclient ignore any other local gn_args* settings. - assert not (self._gn_args_from and self._gn_args_file), \ - 'Only specify one of "gclient_gn_args_from" or ' \ - '"gclient_gn_args_file + gclient_gn_args".' - - self._vars = local_scope.get('vars', {}) - if self.parent: - for key, value in self.parent.get_vars().items(): - if key in self._vars: - self._vars[key] = value - # Since we heavily post-process things, freeze ones which should - # reflect original state of DEPS. - self._vars = gclient_utils.freeze(self._vars) - - # If use_relative_paths is set in the DEPS file, regenerate - # the dictionary using paths relative to the directory containing - # the DEPS file. Also update recursedeps if use_relative_paths is - # enabled. - # If the deps file doesn't set use_relative_paths, but the parent did - # (and therefore set self.relative on this Dependency object), then we - # want to modify the deps and recursedeps by prepending the parent - # directory of this dependency. - self._use_relative_paths = local_scope.get('use_relative_paths', False) - rel_prefix = None - if self._use_relative_paths: - rel_prefix = self.name - elif self._relative: - rel_prefix = os.path.dirname(self.name) - - if 'recursion' in local_scope: - logging.warning( - '%s: Ignoring recursion = %d.', self.name, local_scope['recursion']) - - if 'recursedeps' in local_scope: - for ent in local_scope['recursedeps']: - if isinstance(ent, str): - self.recursedeps[ent] = self.deps_file - else: # (depname, depsfilename) - self.recursedeps[ent[0]] = ent[1] - logging.warning('Found recursedeps %r.', repr(self.recursedeps)) - - if rel_prefix: - logging.warning('Updating recursedeps by prepending %s.', rel_prefix) + logging.warning('use_relative_paths enabled.') rel_deps = {} - for depname, options in self.recursedeps.items(): - rel_deps[os.path.normpath(os.path.join(rel_prefix, depname)).replace( - os.path.sep, '/')] = options - self.recursedeps = rel_deps - # To get gn_args from another DEPS, that DEPS must be recursed into. - if self._gn_args_from: - assert self.recursedeps and self._gn_args_from in self.recursedeps, \ - 'The "gclient_gn_args_from" value must be in recursedeps.' + for d, url in processed_deps.items(): + # normpath is required to allow DEPS to use .. in their + # dependency local path. + # We are following the same pattern when use_relative_paths = False, + # which uses slashes. + rel_deps[os.path.normpath(os.path.join(rel_prefix, d)).replace( + os.path.sep, '/')] = url + logging.warning('Updating deps by prepending %s.', rel_prefix) + return rel_deps - # If present, save 'target_os' in the local_target_os property. - if 'target_os' in local_scope: - self.local_target_os = local_scope['target_os'] + def _deps_to_objects(self, deps, use_relative_paths): + # type: (Mapping[str, Mapping[str, str]], bool) -> Sequence[Dependency] + """Convert a deps dict to a list of Dependency objects.""" + deps_to_add = [] + cached_conditions = {} + for name, dep_value in deps.items(): + should_process = self.should_process + if dep_value is None: + continue - deps = local_scope.get('deps', {}) + condition = dep_value.get('condition') + dep_type = dep_value.get('dep_type') - # If dependencies are configured within git submodules, add them to deps. - # We don't add for SYNC since we expect submodules to be in sync. - if self.git_dependencies_state == gclient_eval.SUBMODULES: - deps.update(self.ParseGitSubmodules()) + if condition and not self._get_option('process_all_deps', False): + if condition not in cached_conditions: + cached_conditions[ + condition] = gclient_eval.EvaluateCondition( + condition, self.get_vars()) + should_process = should_process and cached_conditions[condition] - deps_to_add = self._deps_to_objects( - self._postprocess_deps(deps, rel_prefix), self._use_relative_paths) + # The following option is only set by the 'revinfo' command. + if self._get_option('ignore_dep_type', None) == dep_type: + continue - # compute which working directory should be used for hooks - if local_scope.get('use_relative_hooks', False): - print('use_relative_hooks is deprecated, please remove it from ' - '%s DEPS. (it was merged in use_relative_paths)' % self.name, - file=sys.stderr) + if dep_type == 'cipd': + cipd_root = self.GetCipdRoot() + for package in dep_value.get('packages', []): + deps_to_add.append( + CipdDependency(parent=self, + name=name, + dep_value=package, + cipd_root=cipd_root, + custom_vars=self.custom_vars, + should_process=should_process, + relative=use_relative_paths, + condition=condition)) + else: + url = dep_value.get('url') + deps_to_add.append( + GitDependency( + parent=self, + name=name, + # Update URL with scheme in protocol_override + url=GitDependency.updateProtocol(url, self.protocol), + managed=True, + custom_deps=None, + custom_vars=self.custom_vars, + custom_hooks=None, + deps_file=self.recursedeps.get(name, self.deps_file), + should_process=should_process, + should_recurse=name in self.recursedeps, + relative=use_relative_paths, + condition=condition, + protocol=self.protocol, + git_dependencies_state=self.git_dependencies_state)) - hooks_cwd = self.root.root_dir - if self._use_relative_paths: - hooks_cwd = os.path.join(hooks_cwd, self.name) - elif self._relative: - hooks_cwd = os.path.join(hooks_cwd, os.path.dirname(self.name)) - logging.warning('Using hook base working directory: %s.', hooks_cwd) + # TODO(crbug.com/1341285): Understand why we need this and remove + # it if we don't. + deps_to_add.sort(key=lambda x: x.name) + return deps_to_add - # Only add all hooks if we should sync, otherwise just add custom hooks. - # override named sets of hooks by the custom hooks - hooks_to_run = [] - if self._should_sync: - hook_names_to_suppress = [c.get('name', '') for c in self.custom_hooks] - for hook in local_scope.get('hooks', []): - if hook.get('name', '') not in hook_names_to_suppress: - hooks_to_run.append(hook) + def ParseDepsFile(self): + # type: () -> None + """Parses the DEPS file for this dependency.""" + assert not self.deps_parsed + assert not self.dependencies - # add the replacements and any additions - for hook in self.custom_hooks: - if 'action' in hook: - hooks_to_run.append(hook) + deps_content = None - if self.should_recurse and deps_to_add: - self._pre_deps_hooks = [ - Hook.from_dict(hook, variables=self.get_vars(), verbose=True, - conditions=self.condition, cwd_base=hooks_cwd) - for hook in local_scope.get('pre_deps_hooks', []) - ] + # First try to locate the configured deps file. If it's missing, + # fallback to DEPS. + deps_files = [self.deps_file] + if 'DEPS' not in deps_files: + deps_files.append('DEPS') + for deps_file in deps_files: + filepath = os.path.join(self.root.root_dir, self.name, deps_file) + if os.path.isfile(filepath): + logging.info('ParseDepsFile(%s): %s file found at %s', + self.name, deps_file, filepath) + break + logging.info('ParseDepsFile(%s): No %s file found at %s', self.name, + deps_file, filepath) - self.add_dependencies_and_close(deps_to_add, hooks_to_run, - hooks_cwd=hooks_cwd) - logging.info('ParseDepsFile(%s) done' % self.name) + if os.path.isfile(filepath): + deps_content = gclient_utils.FileRead(filepath) + logging.debug('ParseDepsFile(%s) read:\n%s', self.name, + deps_content) - def ParseGitSubmodules(self): - # type: () -> Mapping[str, str] - """ + local_scope = {} + if deps_content: + try: + local_scope = gclient_eval.Parse(deps_content, filepath, + self.get_vars(), + self.get_builtin_vars()) + except SyntaxError as e: + gclient_utils.SyntaxErrorToError(filepath, e) + + if 'git_dependencies' in local_scope: + self.git_dependencies_state = local_scope['git_dependencies'] + + if 'allowed_hosts' in local_scope: + try: + self._allowed_hosts = frozenset( + local_scope.get('allowed_hosts')) + except TypeError: # raised if non-iterable + pass + if not self._allowed_hosts: + logging.warning("allowed_hosts is specified but empty %s", + self._allowed_hosts) + raise gclient_utils.Error( + 'ParseDepsFile(%s): allowed_hosts must be absent ' + 'or a non-empty iterable' % self.name) + + self._gn_args_from = local_scope.get('gclient_gn_args_from') + self._gn_args_file = local_scope.get('gclient_gn_args_file') + self._gn_args = local_scope.get('gclient_gn_args', []) + # It doesn't make sense to set all of these, since setting gn_args_from + # to another DEPS will make gclient ignore any other local gn_args* + # settings. + assert not (self._gn_args_from and self._gn_args_file), \ + 'Only specify one of "gclient_gn_args_from" or ' \ + '"gclient_gn_args_file + gclient_gn_args".' + + self._vars = local_scope.get('vars', {}) + if self.parent: + for key, value in self.parent.get_vars().items(): + if key in self._vars: + self._vars[key] = value + # Since we heavily post-process things, freeze ones which should + # reflect original state of DEPS. + self._vars = gclient_utils.freeze(self._vars) + + # If use_relative_paths is set in the DEPS file, regenerate + # the dictionary using paths relative to the directory containing + # the DEPS file. Also update recursedeps if use_relative_paths is + # enabled. + # If the deps file doesn't set use_relative_paths, but the parent did + # (and therefore set self.relative on this Dependency object), then we + # want to modify the deps and recursedeps by prepending the parent + # directory of this dependency. + self._use_relative_paths = local_scope.get('use_relative_paths', False) + rel_prefix = None + if self._use_relative_paths: + rel_prefix = self.name + elif self._relative: + rel_prefix = os.path.dirname(self.name) + + if 'recursion' in local_scope: + logging.warning('%s: Ignoring recursion = %d.', self.name, + local_scope['recursion']) + + if 'recursedeps' in local_scope: + for ent in local_scope['recursedeps']: + if isinstance(ent, str): + self.recursedeps[ent] = self.deps_file + else: # (depname, depsfilename) + self.recursedeps[ent[0]] = ent[1] + logging.warning('Found recursedeps %r.', repr(self.recursedeps)) + + if rel_prefix: + logging.warning('Updating recursedeps by prepending %s.', + rel_prefix) + rel_deps = {} + for depname, options in self.recursedeps.items(): + rel_deps[os.path.normpath(os.path.join(rel_prefix, + depname)).replace( + os.path.sep, + '/')] = options + self.recursedeps = rel_deps + # To get gn_args from another DEPS, that DEPS must be recursed into. + if self._gn_args_from: + assert self.recursedeps and self._gn_args_from in self.recursedeps, \ + 'The "gclient_gn_args_from" value must be in recursedeps.' + + # If present, save 'target_os' in the local_target_os property. + if 'target_os' in local_scope: + self.local_target_os = local_scope['target_os'] + + deps = local_scope.get('deps', {}) + + # If dependencies are configured within git submodules, add them to + # deps. We don't add for SYNC since we expect submodules to be in sync. + if self.git_dependencies_state == gclient_eval.SUBMODULES: + deps.update(self.ParseGitSubmodules()) + + deps_to_add = self._deps_to_objects( + self._postprocess_deps(deps, rel_prefix), self._use_relative_paths) + + # compute which working directory should be used for hooks + if local_scope.get('use_relative_hooks', False): + print('use_relative_hooks is deprecated, please remove it from ' + '%s DEPS. (it was merged in use_relative_paths)' % self.name, + file=sys.stderr) + + hooks_cwd = self.root.root_dir + if self._use_relative_paths: + hooks_cwd = os.path.join(hooks_cwd, self.name) + elif self._relative: + hooks_cwd = os.path.join(hooks_cwd, os.path.dirname(self.name)) + logging.warning('Using hook base working directory: %s.', hooks_cwd) + + # Only add all hooks if we should sync, otherwise just add custom hooks. + # override named sets of hooks by the custom hooks + hooks_to_run = [] + if self._should_sync: + hook_names_to_suppress = [ + c.get('name', '') for c in self.custom_hooks + ] + for hook in local_scope.get('hooks', []): + if hook.get('name', '') not in hook_names_to_suppress: + hooks_to_run.append(hook) + + # add the replacements and any additions + for hook in self.custom_hooks: + if 'action' in hook: + hooks_to_run.append(hook) + + if self.should_recurse and deps_to_add: + self._pre_deps_hooks = [ + Hook.from_dict(hook, + variables=self.get_vars(), + verbose=True, + conditions=self.condition, + cwd_base=hooks_cwd) + for hook in local_scope.get('pre_deps_hooks', []) + ] + + self.add_dependencies_and_close(deps_to_add, + hooks_to_run, + hooks_cwd=hooks_cwd) + logging.info('ParseDepsFile(%s) done' % self.name) + + def ParseGitSubmodules(self): + # type: () -> Mapping[str, str] + """ Parses git submodules and returns a dict of path to DEPS git url entries. e.g {: @} """ - cwd = os.path.join(self.root.root_dir, self.name) - filepath = os.path.join(cwd, '.gitmodules') - if not os.path.isfile(filepath): - logging.warning('ParseGitSubmodules(): No .gitmodules found at %s', - filepath) - return {} + cwd = os.path.join(self.root.root_dir, self.name) + filepath = os.path.join(cwd, '.gitmodules') + if not os.path.isfile(filepath): + logging.warning('ParseGitSubmodules(): No .gitmodules found at %s', + filepath) + return {} - # Get .gitmodules fields - gitmodules_entries = subprocess2.check_output( - ['git', 'config', '--file', filepath, '-l']).decode('utf-8') + # Get .gitmodules fields + gitmodules_entries = subprocess2.check_output( + ['git', 'config', '--file', filepath, '-l']).decode('utf-8') - gitmodules = {} - for entry in gitmodules_entries.splitlines(): - key, value = entry.split('=', maxsplit=1) + gitmodules = {} + for entry in gitmodules_entries.splitlines(): + key, value = entry.split('=', maxsplit=1) - # git config keys consist of section.name.key, e.g., submodule.foo.path - section, submodule_key = key.split('.', maxsplit=1) + # git config keys consist of section.name.key, e.g., + # submodule.foo.path + section, submodule_key = key.split('.', maxsplit=1) - # Only parse [submodule "foo"] sections from .gitmodules. - if section.strip() != 'submodule': - continue + # Only parse [submodule "foo"] sections from .gitmodules. + if section.strip() != 'submodule': + continue - # The name of the submodule can contain '.', hence split from the back. - submodule, sub_key = submodule_key.rsplit('.', maxsplit=1) + # The name of the submodule can contain '.', hence split from the + # back. + submodule, sub_key = submodule_key.rsplit('.', maxsplit=1) - if submodule not in gitmodules: - gitmodules[submodule] = {} + if submodule not in gitmodules: + gitmodules[submodule] = {} - if sub_key in ('url', 'gclient-condition', 'path'): - gitmodules[submodule][sub_key] = value + if sub_key in ('url', 'gclient-condition', 'path'): + gitmodules[submodule][sub_key] = value - paths = [module['path'] for module in gitmodules.values()] - commit_hashes = scm_git.GIT.GetSubmoduleCommits(cwd, paths) + paths = [module['path'] for module in gitmodules.values()] + commit_hashes = scm_git.GIT.GetSubmoduleCommits(cwd, paths) - # Structure git submodules into a dict of DEPS git url entries. - submodules = {} - for module in gitmodules.values(): - if self._use_relative_paths: - path = module['path'] - else: - path = f'{self.name}/{module["path"]}' - # TODO(crbug.com/1471685): Temporary hack. In case of applied patches - # where the changes are staged but not committed, any gitlinks from - # the patch are not returned by `git ls-tree`. The path won't be found - # in commit_hashes. Use a temporary '0000000' value that will be replaced - # with w/e is found in DEPS later. - submodules[path] = { - 'dep_type': - 'git', - 'url': - '{}@{}'.format(module['url'], - commit_hashes.get(module['path'], '0000000')) - } + # Structure git submodules into a dict of DEPS git url entries. + submodules = {} + for module in gitmodules.values(): + if self._use_relative_paths: + path = module['path'] + else: + path = f'{self.name}/{module["path"]}' + # TODO(crbug.com/1471685): Temporary hack. In case of applied + # patches where the changes are staged but not committed, any + # gitlinks from the patch are not returned by `git ls-tree`. The + # path won't be found in commit_hashes. Use a temporary '0000000' + # value that will be replaced with w/e is found in DEPS later. + submodules[path] = { + 'dep_type': + 'git', + 'url': + '{}@{}'.format(module['url'], + commit_hashes.get(module['path'], '0000000')) + } - if 'gclient-condition' in module: - submodules[path]['condition'] = module['gclient-condition'] + if 'gclient-condition' in module: + submodules[path]['condition'] = module['gclient-condition'] - return submodules + return submodules - def _get_option(self, attr, default): - obj = self - while not hasattr(obj, '_options'): - obj = obj.parent - return getattr(obj._options, attr, default) + def _get_option(self, attr, default): + obj = self + while not hasattr(obj, '_options'): + obj = obj.parent + return getattr(obj._options, attr, default) - def add_dependencies_and_close(self, deps_to_add, hooks, hooks_cwd=None): - """Adds the dependencies, hooks and mark the parsing as done.""" - if hooks_cwd == None: - hooks_cwd = self.root.root_dir + def add_dependencies_and_close(self, deps_to_add, hooks, hooks_cwd=None): + """Adds the dependencies, hooks and mark the parsing as done.""" + if hooks_cwd == None: + hooks_cwd = self.root.root_dir - for dep in deps_to_add: - if dep.verify_validity(): - self.add_dependency(dep) - self._mark_as_parsed([ - Hook.from_dict( - h, variables=self.get_vars(), verbose=self.root._options.verbose, - conditions=self.condition, cwd_base=hooks_cwd) - for h in hooks - ]) + for dep in deps_to_add: + if dep.verify_validity(): + self.add_dependency(dep) + self._mark_as_parsed([ + Hook.from_dict(h, + variables=self.get_vars(), + verbose=self.root._options.verbose, + conditions=self.condition, + cwd_base=hooks_cwd) for h in hooks + ]) - def findDepsFromNotAllowedHosts(self): - """Returns a list of dependencies from not allowed hosts. + def findDepsFromNotAllowedHosts(self): + """Returns a list of dependencies from not allowed hosts. If allowed_hosts is not set, allows all hosts and returns empty list. """ - if not self._allowed_hosts: - return [] - bad_deps = [] - for dep in self._dependencies: - # Don't enforce this for custom_deps. - if dep.name in self._custom_deps: - continue - if isinstance(dep.url, str): - parsed_url = urllib.parse.urlparse(dep.url) - if parsed_url.netloc and parsed_url.netloc not in self._allowed_hosts: - bad_deps.append(dep) - return bad_deps + if not self._allowed_hosts: + return [] + bad_deps = [] + for dep in self._dependencies: + # Don't enforce this for custom_deps. + if dep.name in self._custom_deps: + continue + if isinstance(dep.url, str): + parsed_url = urllib.parse.urlparse(dep.url) + if parsed_url.netloc and parsed_url.netloc not in self._allowed_hosts: + bad_deps.append(dep) + return bad_deps - def FuzzyMatchUrl(self, candidates): - # type: (Union[Mapping[str, str], Collection[str]]) -> Optional[str] - """Attempts to find this dependency in the list of candidates. + def FuzzyMatchUrl(self, candidates): + # type: (Union[Mapping[str, str], Collection[str]]) -> Optional[str] + """Attempts to find this dependency in the list of candidates. It looks first for the URL of this dependency in the list of candidates. If it doesn't succeed, and the URL ends in '.git', it will try @@ -1033,524 +1070,543 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): - Its parsed url minus '.git': "https://example.com/src" - Its name: "src" """ - if self.url: - origin, _ = gclient_utils.SplitUrlRevision(self.url) - match = gclient_utils.FuzzyMatchRepo(origin, candidates) - if match: - return match - if self.name in candidates: - return self.name - return None - - # Arguments number differs from overridden method - # pylint: disable=arguments-differ - def run( - self, - revision_overrides, # type: Mapping[str, str] - command, # type: str - args, # type: Sequence[str] - work_queue, # type: ExecutionQueue - options, # type: optparse.Values - patch_refs, # type: Mapping[str, str] - target_branches, # type: Mapping[str, str] - skip_sync_revisions, # type: Mapping[str, str] - ): - # type: () -> None - """Runs |command| then parse the DEPS file.""" - logging.info('Dependency(%s).run()' % self.name) - assert self._file_list == [] - # When running runhooks, there's no need to consult the SCM. - # All known hooks are expected to run unconditionally regardless of working - # copy state, so skip the SCM status check. - run_scm = command not in ( - 'flatten', 'runhooks', 'recurse', 'validate', None) - file_list = [] if not options.nohooks else None - revision_override = revision_overrides.pop( - self.FuzzyMatchUrl(revision_overrides), None) - if not revision_override and not self.managed: - revision_override = 'unmanaged' - if run_scm and self.url: - # Create a shallow copy to mutate revision. - options = copy.copy(options) - options.revision = revision_override - self._used_revision = options.revision - self._used_scm = self.CreateSCM(out_cb=work_queue.out_cb) - if command != 'update' or self.GetScmName() != 'git': - self._got_revision = self._used_scm.RunCommand(command, options, args, - file_list) - else: - try: - start = time.time() - sync_status = metrics_utils.SYNC_STATUS_FAILURE - self._got_revision = self._used_scm.RunCommand(command, options, args, - file_list) - sync_status = metrics_utils.SYNC_STATUS_SUCCESS - finally: - url, revision = gclient_utils.SplitUrlRevision(self.url) - metrics.collector.add_repeated('git_deps', { - 'path': self.name, - 'url': url, - 'revision': revision, - 'execution_time': time.time() - start, - 'sync_status': sync_status, - }) - - if isinstance(self, GitDependency) and command == 'update': - patch_repo = self.url.split('@')[0] - patch_ref = patch_refs.pop(self.FuzzyMatchUrl(patch_refs), None) - target_branch = target_branches.pop( - self.FuzzyMatchUrl(target_branches), None) - if patch_ref: - latest_commit = self._used_scm.apply_patch_ref( - patch_repo, patch_ref, target_branch, options, file_list) - else: - latest_commit = self._used_scm.revinfo(None, None, None) - existing_sync_commits = json.loads( - os.environ.get(PREVIOUS_SYNC_COMMITS, '{}')) - existing_sync_commits[self.name] = latest_commit - os.environ[PREVIOUS_SYNC_COMMITS] = json.dumps(existing_sync_commits) - - if file_list: - file_list = [os.path.join(self.name, f.strip()) for f in file_list] - - # TODO(phajdan.jr): We should know exactly when the paths are absolute. - # Convert all absolute paths to relative. - for i in range(len(file_list or [])): - # It depends on the command being executed (like runhooks vs sync). - if not os.path.isabs(file_list[i]): - continue - prefix = os.path.commonprefix( - [self.root.root_dir.lower(), file_list[i].lower()]) - file_list[i] = file_list[i][len(prefix):] - # Strip any leading path separators. - while file_list[i].startswith(('\\', '/')): - file_list[i] = file_list[i][1:] - - # We must check for diffs AFTER any patch_refs have been applied. - if skip_sync_revisions: - skip_sync_rev = skip_sync_revisions.pop( - self.FuzzyMatchUrl(skip_sync_revisions), None) - self._should_sync = (skip_sync_rev is None - or self._used_scm.check_diff(skip_sync_rev, - files=['DEPS'])) - if not self._should_sync: - logging.debug('Skipping sync for %s. No DEPS changes since last ' - 'sync at %s' % (self.name, skip_sync_rev)) - else: - logging.debug('DEPS changes detected for %s since last sync at ' - '%s. Not skipping deps sync' % ( - self.name, skip_sync_rev)) - - if self.should_recurse: - self.ParseDepsFile() - - self._run_is_done(file_list or []) - - # TODO(crbug.com/1339471): If should_recurse is false, ParseDepsFile never - # gets called meaning we never fetch hooks and dependencies. So there's - # no need to check should_recurse again here. - if self.should_recurse: - if command in ('update', 'revert') and not options.noprehooks: - self.RunPreDepsHooks() - # Parse the dependencies of this dependency. - for s in self.dependencies: - if s.should_process: - work_queue.enqueue(s) - - if command == 'recurse': - # Skip file only checkout. - scm = self.GetScmName() - if not options.scm or scm in options.scm: - cwd = os.path.normpath(os.path.join(self.root.root_dir, self.name)) - # Pass in the SCM type as an env variable. Make sure we don't put - # unicode strings in the environment. - env = os.environ.copy() - if scm: - env['GCLIENT_SCM'] = str(scm) if self.url: - env['GCLIENT_URL'] = str(self.url) - env['GCLIENT_DEP_PATH'] = str(self.name) - if options.prepend_dir and scm == 'git': - print_stdout = False - def filter_fn(line): - """Git-specific path marshaling. It is optimized for git-grep.""" - - def mod_path(git_pathspec): - match = re.match('^(\\S+?:)?([^\0]+)$', git_pathspec) - modified_path = os.path.join(self.name, match.group(2)) - branch = match.group(1) or '' - return '%s%s' % (branch, modified_path) - - match = re.match('^Binary file ([^\0]+) matches$', line) + origin, _ = gclient_utils.SplitUrlRevision(self.url) + match = gclient_utils.FuzzyMatchRepo(origin, candidates) if match: - print('Binary file %s matches\n' % mod_path(match.group(1))) - return + return match + if self.name in candidates: + return self.name + return None - items = line.split('\0') - if len(items) == 2 and items[1]: - print('%s : %s' % (mod_path(items[0]), items[1])) - elif len(items) >= 2: - # Multiple null bytes or a single trailing null byte indicate - # git is likely displaying filenames only (such as with -l) - print('\n'.join(mod_path(path) for path in items if path)) + # Arguments number differs from overridden method + # pylint: disable=arguments-differ + def run( + self, + revision_overrides, # type: Mapping[str, str] + command, # type: str + args, # type: Sequence[str] + work_queue, # type: ExecutionQueue + options, # type: optparse.Values + patch_refs, # type: Mapping[str, str] + target_branches, # type: Mapping[str, str] + skip_sync_revisions, # type: Mapping[str, str] + ): + # type: () -> None + """Runs |command| then parse the DEPS file.""" + logging.info('Dependency(%s).run()' % self.name) + assert self._file_list == [] + # When running runhooks, there's no need to consult the SCM. + # All known hooks are expected to run unconditionally regardless of + # working copy state, so skip the SCM status check. + run_scm = command not in ('flatten', 'runhooks', 'recurse', 'validate', + None) + file_list = [] if not options.nohooks else None + revision_override = revision_overrides.pop( + self.FuzzyMatchUrl(revision_overrides), None) + if not revision_override and not self.managed: + revision_override = 'unmanaged' + if run_scm and self.url: + # Create a shallow copy to mutate revision. + options = copy.copy(options) + options.revision = revision_override + self._used_revision = options.revision + self._used_scm = self.CreateSCM(out_cb=work_queue.out_cb) + if command != 'update' or self.GetScmName() != 'git': + self._got_revision = self._used_scm.RunCommand( + command, options, args, file_list) else: - print(line) - else: - print_stdout = True - filter_fn = None + try: + start = time.time() + sync_status = metrics_utils.SYNC_STATUS_FAILURE + self._got_revision = self._used_scm.RunCommand( + command, options, args, file_list) + sync_status = metrics_utils.SYNC_STATUS_SUCCESS + finally: + url, revision = gclient_utils.SplitUrlRevision(self.url) + metrics.collector.add_repeated( + 'git_deps', { + 'path': self.name, + 'url': url, + 'revision': revision, + 'execution_time': time.time() - start, + 'sync_status': sync_status, + }) - if self.url is None: - print('Skipped omitted dependency %s' % cwd, file=sys.stderr) - elif os.path.isdir(cwd): - try: - gclient_utils.CheckCallAndFilter( - args, cwd=cwd, env=env, print_stdout=print_stdout, - filter_fn=filter_fn, - ) - except subprocess2.CalledProcessError: - if not options.ignore: - raise - else: - print('Skipped missing %s' % cwd, file=sys.stderr) + if isinstance(self, GitDependency) and command == 'update': + patch_repo = self.url.split('@')[0] + patch_ref = patch_refs.pop(self.FuzzyMatchUrl(patch_refs), None) + target_branch = target_branches.pop( + self.FuzzyMatchUrl(target_branches), None) + if patch_ref: + latest_commit = self._used_scm.apply_patch_ref( + patch_repo, patch_ref, target_branch, options, + file_list) + else: + latest_commit = self._used_scm.revinfo(None, None, None) + existing_sync_commits = json.loads( + os.environ.get(PREVIOUS_SYNC_COMMITS, '{}')) + existing_sync_commits[self.name] = latest_commit + os.environ[PREVIOUS_SYNC_COMMITS] = json.dumps( + existing_sync_commits) - def GetScmName(self): - raise NotImplementedError() + if file_list: + file_list = [ + os.path.join(self.name, f.strip()) for f in file_list + ] - def CreateSCM(self, out_cb=None): - raise NotImplementedError() + # TODO(phajdan.jr): We should know exactly when the paths are + # absolute. Convert all absolute paths to relative. + for i in range(len(file_list or [])): + # It depends on the command being executed (like runhooks vs + # sync). + if not os.path.isabs(file_list[i]): + continue + prefix = os.path.commonprefix( + [self.root.root_dir.lower(), file_list[i].lower()]) + file_list[i] = file_list[i][len(prefix):] + # Strip any leading path separators. + while file_list[i].startswith(('\\', '/')): + file_list[i] = file_list[i][1:] - def HasGNArgsFile(self): - return self._gn_args_file is not None + # We must check for diffs AFTER any patch_refs have been applied. + if skip_sync_revisions: + skip_sync_rev = skip_sync_revisions.pop( + self.FuzzyMatchUrl(skip_sync_revisions), None) + self._should_sync = (skip_sync_rev is None + or self._used_scm.check_diff(skip_sync_rev, + files=['DEPS'])) + if not self._should_sync: + logging.debug( + 'Skipping sync for %s. No DEPS changes since last ' + 'sync at %s' % (self.name, skip_sync_rev)) + else: + logging.debug('DEPS changes detected for %s since last sync at ' + '%s. Not skipping deps sync' % + (self.name, skip_sync_rev)) - def WriteGNArgsFile(self): - lines = ['# Generated from %r' % self.deps_file] - variables = self.get_vars() - for arg in self._gn_args: - value = variables[arg] - if isinstance(value, gclient_eval.ConstantString): - value = value.value - elif isinstance(value, str): - value = gclient_eval.EvaluateCondition(value, variables) - lines.append('%s = %s' % (arg, ToGNString(value))) + if self.should_recurse: + self.ParseDepsFile() - # When use_relative_paths is set, gn_args_file is relative to this DEPS - path_prefix = self.root.root_dir - if self._use_relative_paths: - path_prefix = os.path.join(path_prefix, self.name) + self._run_is_done(file_list or []) - with open(os.path.join(path_prefix, self._gn_args_file), 'wb') as f: - f.write('\n'.join(lines).encode('utf-8', 'replace')) + # TODO(crbug.com/1339471): If should_recurse is false, ParseDepsFile + # never gets called meaning we never fetch hooks and dependencies. So + # there's no need to check should_recurse again here. + if self.should_recurse: + if command in ('update', 'revert') and not options.noprehooks: + self.RunPreDepsHooks() + # Parse the dependencies of this dependency. + for s in self.dependencies: + if s.should_process: + work_queue.enqueue(s) - @gclient_utils.lockedmethod - def _run_is_done(self, file_list): - # Both these are kept for hooks that are run as a separate tree traversal. - self._file_list = file_list - self._processed = True + if command == 'recurse': + # Skip file only checkout. + scm = self.GetScmName() + if not options.scm or scm in options.scm: + cwd = os.path.normpath( + os.path.join(self.root.root_dir, self.name)) + # Pass in the SCM type as an env variable. Make sure we don't + # put unicode strings in the environment. + env = os.environ.copy() + if scm: + env['GCLIENT_SCM'] = str(scm) + if self.url: + env['GCLIENT_URL'] = str(self.url) + env['GCLIENT_DEP_PATH'] = str(self.name) + if options.prepend_dir and scm == 'git': + print_stdout = False - def GetHooks(self, options): - """Evaluates all hooks, and return them in a flat list. + def filter_fn(line): + """Git-specific path marshaling. It is optimized for git-grep.""" + def mod_path(git_pathspec): + match = re.match('^(\\S+?:)?([^\0]+)$', + git_pathspec) + modified_path = os.path.join( + self.name, match.group(2)) + branch = match.group(1) or '' + return '%s%s' % (branch, modified_path) + + match = re.match('^Binary file ([^\0]+) matches$', line) + if match: + print('Binary file %s matches\n' % + mod_path(match.group(1))) + return + + items = line.split('\0') + if len(items) == 2 and items[1]: + print('%s : %s' % (mod_path(items[0]), items[1])) + elif len(items) >= 2: + # Multiple null bytes or a single trailing null byte + # indicate git is likely displaying filenames only + # (such as with -l) + print('\n'.join( + mod_path(path) for path in items if path)) + else: + print(line) + else: + print_stdout = True + filter_fn = None + + if self.url is None: + print('Skipped omitted dependency %s' % cwd, + file=sys.stderr) + elif os.path.isdir(cwd): + try: + gclient_utils.CheckCallAndFilter( + args, + cwd=cwd, + env=env, + print_stdout=print_stdout, + filter_fn=filter_fn, + ) + except subprocess2.CalledProcessError: + if not options.ignore: + raise + else: + print('Skipped missing %s' % cwd, file=sys.stderr) + + def GetScmName(self): + raise NotImplementedError() + + def CreateSCM(self, out_cb=None): + raise NotImplementedError() + + def HasGNArgsFile(self): + return self._gn_args_file is not None + + def WriteGNArgsFile(self): + lines = ['# Generated from %r' % self.deps_file] + variables = self.get_vars() + for arg in self._gn_args: + value = variables[arg] + if isinstance(value, gclient_eval.ConstantString): + value = value.value + elif isinstance(value, str): + value = gclient_eval.EvaluateCondition(value, variables) + lines.append('%s = %s' % (arg, ToGNString(value))) + + # When use_relative_paths is set, gn_args_file is relative to this DEPS + path_prefix = self.root.root_dir + if self._use_relative_paths: + path_prefix = os.path.join(path_prefix, self.name) + + with open(os.path.join(path_prefix, self._gn_args_file), 'wb') as f: + f.write('\n'.join(lines).encode('utf-8', 'replace')) + + @gclient_utils.lockedmethod + def _run_is_done(self, file_list): + # Both these are kept for hooks that are run as a separate tree + # traversal. + self._file_list = file_list + self._processed = True + + def GetHooks(self, options): + """Evaluates all hooks, and return them in a flat list. RunOnDeps() must have been called before to load the DEPS. """ - result = [] - if not self.should_process or not self.should_recurse: - # Don't run the hook when it is above recursion_limit. - return result - # If "--force" was specified, run all hooks regardless of what files have - # changed. - if self.deps_hooks: - # TODO(maruel): If the user is using git, then we don't know - # what files have changed so we always run all hooks. It'd be nice to fix - # that. - result.extend(self.deps_hooks) - for s in self.dependencies: - result.extend(s.GetHooks(options)) - return result + result = [] + if not self.should_process or not self.should_recurse: + # Don't run the hook when it is above recursion_limit. + return result + # If "--force" was specified, run all hooks regardless of what files + # have changed. + if self.deps_hooks: + # TODO(maruel): If the user is using git, then we don't know + # what files have changed so we always run all hooks. It'd be nice + # to fix that. + result.extend(self.deps_hooks) + for s in self.dependencies: + result.extend(s.GetHooks(options)) + return result - def RunHooksRecursively(self, options, progress): - assert self.hooks_ran == False - self._hooks_ran = True - hooks = self.GetHooks(options) - if progress: - progress._total = len(hooks) - for hook in hooks: - if progress: - progress.update(extra=hook.name or '') - hook.run() - if progress: - progress.end() + def RunHooksRecursively(self, options, progress): + assert self.hooks_ran == False + self._hooks_ran = True + hooks = self.GetHooks(options) + if progress: + progress._total = len(hooks) + for hook in hooks: + if progress: + progress.update(extra=hook.name or '') + hook.run() + if progress: + progress.end() - def RunPreDepsHooks(self): - assert self.processed - assert self.deps_parsed - assert not self.pre_deps_hooks_ran - assert not self.hooks_ran - for s in self.dependencies: - assert not s.processed - self._pre_deps_hooks_ran = True - for hook in self.pre_deps_hooks: - hook.run() + def RunPreDepsHooks(self): + assert self.processed + assert self.deps_parsed + assert not self.pre_deps_hooks_ran + assert not self.hooks_ran + for s in self.dependencies: + assert not s.processed + self._pre_deps_hooks_ran = True + for hook in self.pre_deps_hooks: + hook.run() - def GetCipdRoot(self): - if self.root is self: - # Let's not infinitely recurse. If this is root and isn't an - # instance of GClient, do nothing. - return None - return self.root.GetCipdRoot() + def GetCipdRoot(self): + if self.root is self: + # Let's not infinitely recurse. If this is root and isn't an + # instance of GClient, do nothing. + return None + return self.root.GetCipdRoot() - def subtree(self, include_all): - """Breadth first recursion excluding root node.""" - dependencies = self.dependencies - for d in dependencies: - if d.should_process or include_all: - yield d - for d in dependencies: - for i in d.subtree(include_all): - yield i + def subtree(self, include_all): + """Breadth first recursion excluding root node.""" + dependencies = self.dependencies + for d in dependencies: + if d.should_process or include_all: + yield d + for d in dependencies: + for i in d.subtree(include_all): + yield i - @gclient_utils.lockedmethod - def add_dependency(self, new_dep): - self._dependencies.append(new_dep) + @gclient_utils.lockedmethod + def add_dependency(self, new_dep): + self._dependencies.append(new_dep) - @gclient_utils.lockedmethod - def _mark_as_parsed(self, new_hooks): - self._deps_hooks.extend(new_hooks) - self._deps_parsed = True + @gclient_utils.lockedmethod + def _mark_as_parsed(self, new_hooks): + self._deps_hooks.extend(new_hooks) + self._deps_parsed = True - @property - @gclient_utils.lockedmethod - def dependencies(self): - return tuple(self._dependencies) + @property + @gclient_utils.lockedmethod + def dependencies(self): + return tuple(self._dependencies) - @property - @gclient_utils.lockedmethod - def deps_hooks(self): - return tuple(self._deps_hooks) + @property + @gclient_utils.lockedmethod + def deps_hooks(self): + return tuple(self._deps_hooks) - @property - @gclient_utils.lockedmethod - def pre_deps_hooks(self): - return tuple(self._pre_deps_hooks) + @property + @gclient_utils.lockedmethod + def pre_deps_hooks(self): + return tuple(self._pre_deps_hooks) - @property - @gclient_utils.lockedmethod - def deps_parsed(self): - """This is purely for debugging purposes. It's not used anywhere.""" - return self._deps_parsed + @property + @gclient_utils.lockedmethod + def deps_parsed(self): + """This is purely for debugging purposes. It's not used anywhere.""" + return self._deps_parsed - @property - @gclient_utils.lockedmethod - def processed(self): - return self._processed + @property + @gclient_utils.lockedmethod + def processed(self): + return self._processed - @property - @gclient_utils.lockedmethod - def pre_deps_hooks_ran(self): - return self._pre_deps_hooks_ran + @property + @gclient_utils.lockedmethod + def pre_deps_hooks_ran(self): + return self._pre_deps_hooks_ran - @property - @gclient_utils.lockedmethod - def hooks_ran(self): - return self._hooks_ran + @property + @gclient_utils.lockedmethod + def hooks_ran(self): + return self._hooks_ran - @property - @gclient_utils.lockedmethod - def allowed_hosts(self): - return self._allowed_hosts + @property + @gclient_utils.lockedmethod + def allowed_hosts(self): + return self._allowed_hosts - @property - @gclient_utils.lockedmethod - def file_list(self): - return tuple(self._file_list) + @property + @gclient_utils.lockedmethod + def file_list(self): + return tuple(self._file_list) - @property - def used_scm(self): - """SCMWrapper instance for this dependency or None if not processed yet.""" - return self._used_scm + @property + def used_scm(self): + """SCMWrapper instance for this dependency or None if not processed yet.""" + return self._used_scm - @property - @gclient_utils.lockedmethod - def got_revision(self): - return self._got_revision + @property + @gclient_utils.lockedmethod + def got_revision(self): + return self._got_revision - @property - def file_list_and_children(self): - result = list(self.file_list) - for d in self.dependencies: - result.extend(d.file_list_and_children) - return tuple(result) + @property + def file_list_and_children(self): + result = list(self.file_list) + for d in self.dependencies: + result.extend(d.file_list_and_children) + return tuple(result) - def __str__(self): - out = [] - for i in ('name', 'url', 'custom_deps', - 'custom_vars', 'deps_hooks', 'file_list', 'should_process', - 'processed', 'hooks_ran', 'deps_parsed', 'requirements', - 'allowed_hosts'): - # First try the native property if it exists. - if hasattr(self, '_' + i): - value = getattr(self, '_' + i, False) - else: - value = getattr(self, i, False) - if value: - out.append('%s: %s' % (i, value)) + def __str__(self): + out = [] + for i in ('name', 'url', 'custom_deps', 'custom_vars', 'deps_hooks', + 'file_list', 'should_process', 'processed', 'hooks_ran', + 'deps_parsed', 'requirements', 'allowed_hosts'): + # First try the native property if it exists. + if hasattr(self, '_' + i): + value = getattr(self, '_' + i, False) + else: + value = getattr(self, i, False) + if value: + out.append('%s: %s' % (i, value)) - for d in self.dependencies: - out.extend([' ' + x for x in str(d).splitlines()]) - out.append('') - return '\n'.join(out) + for d in self.dependencies: + out.extend([' ' + x for x in str(d).splitlines()]) + out.append('') + return '\n'.join(out) - def __repr__(self): - return '%s: %s' % (self.name, self.url) + def __repr__(self): + return '%s: %s' % (self.name, self.url) - def hierarchy(self, include_url=True, graphviz=False): - """Returns a human-readable hierarchical reference to a Dependency.""" - def format_name(d): - if include_url: - return '%s(%s)' % (d.name, d.url) - return '"%s"' % d.name # quotes required for graph dot file. + def hierarchy(self, include_url=True, graphviz=False): + """Returns a human-readable hierarchical reference to a Dependency.""" + def format_name(d): + if include_url: + return '%s(%s)' % (d.name, d.url) + return '"%s"' % d.name # quotes required for graph dot file. - out = format_name(self) - i = self.parent - while i and i.name: - out = '%s -> %s' % (format_name(i), out) - if graphviz: - # for graphviz we just need each parent->child relationship listed once. + out = format_name(self) + i = self.parent + while i and i.name: + out = '%s -> %s' % (format_name(i), out) + if graphviz: + # for graphviz we just need each parent->child relationship + # listed once. + return out + i = i.parent return out - i = i.parent - return out - def hierarchy_data(self): - """Returns a machine-readable hierarchical reference to a Dependency.""" - d = self - out = [] - while d and d.name: - out.insert(0, (d.name, d.url)) - d = d.parent - return tuple(out) + def hierarchy_data(self): + """Returns a machine-readable hierarchical reference to a Dependency.""" + d = self + out = [] + while d and d.name: + out.insert(0, (d.name, d.url)) + d = d.parent + return tuple(out) - def get_builtin_vars(self): - return { - 'checkout_android': 'android' in self.target_os, - 'checkout_chromeos': 'chromeos' in self.target_os, - 'checkout_fuchsia': 'fuchsia' in self.target_os, - 'checkout_ios': 'ios' in self.target_os, - 'checkout_linux': 'unix' in self.target_os, - 'checkout_mac': 'mac' in self.target_os, - 'checkout_win': 'win' in self.target_os, - 'host_os': _detect_host_os(), + def get_builtin_vars(self): + return { + 'checkout_android': 'android' in self.target_os, + 'checkout_chromeos': 'chromeos' in self.target_os, + 'checkout_fuchsia': 'fuchsia' in self.target_os, + 'checkout_ios': 'ios' in self.target_os, + 'checkout_linux': 'unix' in self.target_os, + 'checkout_mac': 'mac' in self.target_os, + 'checkout_win': 'win' in self.target_os, + 'host_os': _detect_host_os(), + 'checkout_arm': 'arm' in self.target_cpu, + 'checkout_arm64': 'arm64' in self.target_cpu, + 'checkout_x86': 'x86' in self.target_cpu, + 'checkout_mips': 'mips' in self.target_cpu, + 'checkout_mips64': 'mips64' in self.target_cpu, + 'checkout_ppc': 'ppc' in self.target_cpu, + 'checkout_s390': 's390' in self.target_cpu, + 'checkout_x64': 'x64' in self.target_cpu, + 'host_cpu': detect_host_arch.HostArch(), + } - 'checkout_arm': 'arm' in self.target_cpu, - 'checkout_arm64': 'arm64' in self.target_cpu, - 'checkout_x86': 'x86' in self.target_cpu, - 'checkout_mips': 'mips' in self.target_cpu, - 'checkout_mips64': 'mips64' in self.target_cpu, - 'checkout_ppc': 'ppc' in self.target_cpu, - 'checkout_s390': 's390' in self.target_cpu, - 'checkout_x64': 'x64' in self.target_cpu, - 'host_cpu': detect_host_arch.HostArch(), - } - - def get_vars(self): - """Returns a dictionary of effective variable values + def get_vars(self): + """Returns a dictionary of effective variable values (DEPS file contents with applied custom_vars overrides).""" - # Variable precedence (last has highest): - # - DEPS vars - # - parents, from first to last - # - built-in - # - custom_vars overrides - result = {} - result.update(self._vars) - if self.parent: - merge_vars(result, self.parent.get_vars()) - # Provide some built-in variables. - result.update(self.get_builtin_vars()) - merge_vars(result, self.custom_vars) + # Variable precedence (last has highest): + # - DEPS vars + # - parents, from first to last + # - built-in + # - custom_vars overrides + result = {} + result.update(self._vars) + if self.parent: + merge_vars(result, self.parent.get_vars()) + # Provide some built-in variables. + result.update(self.get_builtin_vars()) + merge_vars(result, self.custom_vars) - return result + return result _PLATFORM_MAPPING = { - 'cygwin': 'win', - 'darwin': 'mac', - 'linux2': 'linux', - 'linux': 'linux', - 'win32': 'win', - 'aix6': 'aix', + 'cygwin': 'win', + 'darwin': 'mac', + 'linux2': 'linux', + 'linux': 'linux', + 'win32': 'win', + 'aix6': 'aix', } def merge_vars(result, new_vars): - for k, v in new_vars.items(): - if k in result: - if isinstance(result[k], gclient_eval.ConstantString): - if isinstance(v, gclient_eval.ConstantString): - result[k] = v + for k, v in new_vars.items(): + if k in result: + if isinstance(result[k], gclient_eval.ConstantString): + if isinstance(v, gclient_eval.ConstantString): + result[k] = v + else: + result[k].value = v + else: + result[k] = v else: - result[k].value = v - else: - result[k] = v - else: - result[k] = v + result[k] = v def _detect_host_os(): - if sys.platform in _PLATFORM_MAPPING: - return _PLATFORM_MAPPING[sys.platform] + if sys.platform in _PLATFORM_MAPPING: + return _PLATFORM_MAPPING[sys.platform] - try: - return os.uname().sysname.lower() - except AttributeError: - return sys.platform + try: + return os.uname().sysname.lower() + except AttributeError: + return sys.platform class GitDependency(Dependency): - """A Dependency object that represents a single git checkout.""" + """A Dependency object that represents a single git checkout.""" + @staticmethod + def updateProtocol(url, protocol): + """Updates given URL's protocol""" + # only works on urls, skips local paths + if not url or not protocol or not re.match('([a-z]+)://', url) or \ + re.match('file://', url): + return url - @staticmethod - def updateProtocol(url, protocol): - """Updates given URL's protocol""" - # only works on urls, skips local paths - if not url or not protocol or not re.match('([a-z]+)://', url) or \ - re.match('file://', url): - return url + return re.sub('^([a-z]+):', protocol + ':', url) - return re.sub('^([a-z]+):', protocol + ':', url) + #override + def GetScmName(self): + """Always 'git'.""" + return 'git' - #override - def GetScmName(self): - """Always 'git'.""" - return 'git' - - #override - def CreateSCM(self, out_cb=None): - """Create a Wrapper instance suitable for handling this git dependency.""" - return gclient_scm.GitWrapper( - self.url, self.root.root_dir, self.name, self.outbuf, out_cb, - print_outbuf=self.print_outbuf) + #override + def CreateSCM(self, out_cb=None): + """Create a Wrapper instance suitable for handling this git dependency.""" + return gclient_scm.GitWrapper(self.url, + self.root.root_dir, + self.name, + self.outbuf, + out_cb, + print_outbuf=self.print_outbuf) class GClient(GitDependency): - """Object that represent a gclient checkout. A tree of Dependency(), one per + """Object that represent a gclient checkout. A tree of Dependency(), one per solution or DEPS entry.""" - DEPS_OS_CHOICES = { - "aix6": "unix", - "win32": "win", - "win": "win", - "cygwin": "win", - "darwin": "mac", - "mac": "mac", - "unix": "unix", - "linux": "unix", - "linux2": "unix", - "linux3": "unix", - "android": "android", - "ios": "ios", - "fuchsia": "fuchsia", - "chromeos": "chromeos", - } + DEPS_OS_CHOICES = { + "aix6": "unix", + "win32": "win", + "win": "win", + "cygwin": "win", + "darwin": "mac", + "mac": "mac", + "unix": "unix", + "linux": "unix", + "linux2": "unix", + "linux3": "unix", + "android": "android", + "ios": "ios", + "fuchsia": "fuchsia", + "chromeos": "chromeos", + } - DEFAULT_CLIENT_FILE_TEXT = ("""\ + DEFAULT_CLIENT_FILE_TEXT = ("""\ solutions = [ { "name" : %(solution_name)r, "url" : %(solution_url)r, @@ -1563,64 +1619,62 @@ solutions = [ ] """) - DEFAULT_CLIENT_CACHE_DIR_TEXT = ("""\ + DEFAULT_CLIENT_CACHE_DIR_TEXT = ("""\ cache_dir = %(cache_dir)r """) - - DEFAULT_SNAPSHOT_FILE_TEXT = ("""\ + DEFAULT_SNAPSHOT_FILE_TEXT = ("""\ # Snapshot generated with gclient revinfo --snapshot solutions = %(solution_list)s """) - def __init__(self, root_dir, options): - # Do not change previous behavior. Only solution level and immediate DEPS - # are processed. - self._recursion_limit = 2 - super(GClient, self).__init__( - parent=None, - name=None, - url=None, - managed=True, - custom_deps=None, - custom_vars=None, - custom_hooks=None, - deps_file='unused', - should_process=True, - should_recurse=True, - relative=None, - condition=None, - print_outbuf=True) + def __init__(self, root_dir, options): + # Do not change previous behavior. Only solution level and immediate + # DEPS are processed. + self._recursion_limit = 2 + super(GClient, self).__init__(parent=None, + name=None, + url=None, + managed=True, + custom_deps=None, + custom_vars=None, + custom_hooks=None, + deps_file='unused', + should_process=True, + should_recurse=True, + relative=None, + condition=None, + print_outbuf=True) - self._options = options - if options.deps_os: - enforced_os = options.deps_os.split(',') - else: - enforced_os = [self.DEPS_OS_CHOICES.get(sys.platform, 'unix')] - if 'all' in enforced_os: - enforced_os = self.DEPS_OS_CHOICES.values() - self._enforced_os = tuple(set(enforced_os)) - self._enforced_cpu = (detect_host_arch.HostArch(), ) - self._root_dir = root_dir - self._cipd_root = None - self.config_content = None + self._options = options + if options.deps_os: + enforced_os = options.deps_os.split(',') + else: + enforced_os = [self.DEPS_OS_CHOICES.get(sys.platform, 'unix')] + if 'all' in enforced_os: + enforced_os = self.DEPS_OS_CHOICES.values() + self._enforced_os = tuple(set(enforced_os)) + self._enforced_cpu = (detect_host_arch.HostArch(), ) + self._root_dir = root_dir + self._cipd_root = None + self.config_content = None - def _CheckConfig(self): - """Verify that the config matches the state of the existing checked-out + def _CheckConfig(self): + """Verify that the config matches the state of the existing checked-out solutions.""" - for dep in self.dependencies: - if dep.managed and dep.url: - scm = dep.CreateSCM() - actual_url = scm.GetActualRemoteURL(self._options) - if actual_url and not scm.DoesRemoteURLMatch(self._options): - mirror = scm.GetCacheMirror() - if mirror: - mirror_string = '%s (exists=%s)' % (mirror.mirror_path, - mirror.exists()) - else: - mirror_string = 'not used' - raise gclient_utils.Error( - ''' + for dep in self.dependencies: + if dep.managed and dep.url: + scm = dep.CreateSCM() + actual_url = scm.GetActualRemoteURL(self._options) + if actual_url and not scm.DoesRemoteURLMatch(self._options): + mirror = scm.GetCacheMirror() + if mirror: + mirror_string = '%s (exists=%s)' % (mirror.mirror_path, + mirror.exists()) + else: + mirror_string = 'not used' + raise gclient_utils.Error( + ''' Your .gclient file seems to be broken. The requested URL is different from what is actually checked out in %(checkout_path)s. @@ -1634,292 +1688,300 @@ The local checkout in %(checkout_path)s reports: You should ensure that the URL listed in .gclient is correct and either change it or fix the checkout. ''' % { - 'checkout_path': os.path.join(self.root_dir, dep.name), - 'expected_url': dep.url, - 'expected_scm': dep.GetScmName(), - 'mirror_string': mirror_string, - 'actual_url': actual_url, - 'actual_scm': dep.GetScmName() - }) + 'checkout_path': os.path.join( + self.root_dir, dep.name), + 'expected_url': dep.url, + 'expected_scm': dep.GetScmName(), + 'mirror_string': mirror_string, + 'actual_url': actual_url, + 'actual_scm': dep.GetScmName() + }) - def SetConfig(self, content): - assert not self.dependencies - config_dict = {} - self.config_content = content - try: - exec(content, config_dict) - except SyntaxError as e: - gclient_utils.SyntaxErrorToError('.gclient', e) + def SetConfig(self, content): + assert not self.dependencies + config_dict = {} + self.config_content = content + try: + exec(content, config_dict) + except SyntaxError as e: + gclient_utils.SyntaxErrorToError('.gclient', e) - # Append any target OS that is not already being enforced to the tuple. - target_os = config_dict.get('target_os', []) - if config_dict.get('target_os_only', False): - self._enforced_os = tuple(set(target_os)) - else: - self._enforced_os = tuple(set(self._enforced_os).union(target_os)) + # Append any target OS that is not already being enforced to the tuple. + target_os = config_dict.get('target_os', []) + if config_dict.get('target_os_only', False): + self._enforced_os = tuple(set(target_os)) + else: + self._enforced_os = tuple(set(self._enforced_os).union(target_os)) - # Append any target CPU that is not already being enforced to the tuple. - target_cpu = config_dict.get('target_cpu', []) - if config_dict.get('target_cpu_only', False): - self._enforced_cpu = tuple(set(target_cpu)) - else: - self._enforced_cpu = tuple(set(self._enforced_cpu).union(target_cpu)) + # Append any target CPU that is not already being enforced to the tuple. + target_cpu = config_dict.get('target_cpu', []) + if config_dict.get('target_cpu_only', False): + self._enforced_cpu = tuple(set(target_cpu)) + else: + self._enforced_cpu = tuple( + set(self._enforced_cpu).union(target_cpu)) - cache_dir = config_dict.get('cache_dir', UNSET_CACHE_DIR) - if cache_dir is not UNSET_CACHE_DIR: - if cache_dir: - cache_dir = os.path.join(self.root_dir, cache_dir) - cache_dir = os.path.abspath(cache_dir) + cache_dir = config_dict.get('cache_dir', UNSET_CACHE_DIR) + if cache_dir is not UNSET_CACHE_DIR: + if cache_dir: + cache_dir = os.path.join(self.root_dir, cache_dir) + cache_dir = os.path.abspath(cache_dir) - git_cache.Mirror.SetCachePath(cache_dir) + git_cache.Mirror.SetCachePath(cache_dir) - if not target_os and config_dict.get('target_os_only', False): - raise gclient_utils.Error('Can\'t use target_os_only if target_os is ' - 'not specified') + if not target_os and config_dict.get('target_os_only', False): + raise gclient_utils.Error( + 'Can\'t use target_os_only if target_os is ' + 'not specified') - if not target_cpu and config_dict.get('target_cpu_only', False): - raise gclient_utils.Error('Can\'t use target_cpu_only if target_cpu is ' - 'not specified') + if not target_cpu and config_dict.get('target_cpu_only', False): + raise gclient_utils.Error( + 'Can\'t use target_cpu_only if target_cpu is ' + 'not specified') - deps_to_add = [] - for s in config_dict.get('solutions', []): - try: - deps_to_add.append( - GitDependency( - parent=self, - name=s['name'], - # Update URL with scheme in protocol_override - url=GitDependency.updateProtocol( - s['url'], s.get('protocol_override', None)), - managed=s.get('managed', True), - custom_deps=s.get('custom_deps', {}), - custom_vars=s.get('custom_vars', {}), - custom_hooks=s.get('custom_hooks', []), - deps_file=s.get('deps_file', 'DEPS'), - should_process=True, - should_recurse=True, - relative=None, - condition=None, - print_outbuf=True, - # Pass protocol_override down the tree for child deps to use. - protocol=s.get('protocol_override', None), - git_dependencies_state=self.git_dependencies_state)) - except KeyError: - raise gclient_utils.Error('Invalid .gclient file. Solution is ' - 'incomplete: %s' % s) - metrics.collector.add( - 'project_urls', - [ + deps_to_add = [] + for s in config_dict.get('solutions', []): + try: + deps_to_add.append( + GitDependency( + parent=self, + name=s['name'], + # Update URL with scheme in protocol_override + url=GitDependency.updateProtocol( + s['url'], s.get('protocol_override', None)), + managed=s.get('managed', True), + custom_deps=s.get('custom_deps', {}), + custom_vars=s.get('custom_vars', {}), + custom_hooks=s.get('custom_hooks', []), + deps_file=s.get('deps_file', 'DEPS'), + should_process=True, + should_recurse=True, + relative=None, + condition=None, + print_outbuf=True, + # Pass protocol_override down the tree for child deps to + # use. + protocol=s.get('protocol_override', None), + git_dependencies_state=self.git_dependencies_state)) + except KeyError: + raise gclient_utils.Error('Invalid .gclient file. Solution is ' + 'incomplete: %s' % s) + metrics.collector.add('project_urls', [ dep.FuzzyMatchUrl(metrics_utils.KNOWN_PROJECT_URLS) for dep in deps_to_add if dep.FuzzyMatchUrl(metrics_utils.KNOWN_PROJECT_URLS) - ] - ) + ]) - self.add_dependencies_and_close(deps_to_add, config_dict.get('hooks', [])) - logging.info('SetConfig() done') + self.add_dependencies_and_close(deps_to_add, + config_dict.get('hooks', [])) + logging.info('SetConfig() done') - def SaveConfig(self): - gclient_utils.FileWrite(os.path.join(self.root_dir, - self._options.config_filename), - self.config_content) + def SaveConfig(self): + gclient_utils.FileWrite( + os.path.join(self.root_dir, self._options.config_filename), + self.config_content) - @staticmethod - def LoadCurrentConfig(options): - # type: (optparse.Values) -> GClient - """Searches for and loads a .gclient file relative to the current working + @staticmethod + def LoadCurrentConfig(options): + # type: (optparse.Values) -> GClient + """Searches for and loads a .gclient file relative to the current working dir.""" - if options.spec: - client = GClient('.', options) - client.SetConfig(options.spec) - else: - if options.verbose: - print('Looking for %s starting from %s\n' % ( - options.config_filename, os.getcwd())) - path = gclient_paths.FindGclientRoot(os.getcwd(), options.config_filename) - if not path: - if options.verbose: - print('Couldn\'t find configuration file.') - return None - client = GClient(path, options) - client.SetConfig(gclient_utils.FileRead( - os.path.join(path, options.config_filename))) + if options.spec: + client = GClient('.', options) + client.SetConfig(options.spec) + else: + if options.verbose: + print('Looking for %s starting from %s\n' % + (options.config_filename, os.getcwd())) + path = gclient_paths.FindGclientRoot(os.getcwd(), + options.config_filename) + if not path: + if options.verbose: + print('Couldn\'t find configuration file.') + return None + client = GClient(path, options) + client.SetConfig( + gclient_utils.FileRead( + os.path.join(path, options.config_filename))) - if (options.revisions and - len(client.dependencies) > 1 and - any('@' not in r for r in options.revisions)): - print( - ('You must specify the full solution name like --revision %s@%s\n' - 'when you have multiple solutions setup in your .gclient file.\n' - 'Other solutions present are: %s.') % ( - client.dependencies[0].name, - options.revisions[0], - ', '.join(s.name for s in client.dependencies[1:])), - file=sys.stderr) + if (options.revisions and len(client.dependencies) > 1 + and any('@' not in r for r in options.revisions)): + print(( + 'You must specify the full solution name like --revision %s@%s\n' + 'when you have multiple solutions setup in your .gclient file.\n' + 'Other solutions present are: %s.') % + (client.dependencies[0].name, options.revisions[0], ', '.join( + s.name for s in client.dependencies[1:])), + file=sys.stderr) - return client + return client - def SetDefaultConfig(self, solution_name, deps_file, solution_url, - managed=True, cache_dir=UNSET_CACHE_DIR, - custom_vars=None): - text = self.DEFAULT_CLIENT_FILE_TEXT - format_dict = { - 'solution_name': solution_name, - 'solution_url': solution_url, - 'deps_file': deps_file, - 'managed': managed, - 'custom_vars': custom_vars or {}, - } + def SetDefaultConfig(self, + solution_name, + deps_file, + solution_url, + managed=True, + cache_dir=UNSET_CACHE_DIR, + custom_vars=None): + text = self.DEFAULT_CLIENT_FILE_TEXT + format_dict = { + 'solution_name': solution_name, + 'solution_url': solution_url, + 'deps_file': deps_file, + 'managed': managed, + 'custom_vars': custom_vars or {}, + } - if cache_dir is not UNSET_CACHE_DIR: - text += self.DEFAULT_CLIENT_CACHE_DIR_TEXT - format_dict['cache_dir'] = cache_dir + if cache_dir is not UNSET_CACHE_DIR: + text += self.DEFAULT_CLIENT_CACHE_DIR_TEXT + format_dict['cache_dir'] = cache_dir - self.SetConfig(text % format_dict) + self.SetConfig(text % format_dict) - def _SaveEntries(self): - """Creates a .gclient_entries file to record the list of unique checkouts. + def _SaveEntries(self): + """Creates a .gclient_entries file to record the list of unique checkouts. The .gclient_entries file lives in the same directory as .gclient. """ - # Sometimes pprint.pformat will use {', sometimes it'll use { ' ... It - # makes testing a bit too fun. - result = 'entries = {\n' - for entry in self.root.subtree(False): - result += ' %s: %s,\n' % (pprint.pformat(entry.name), - pprint.pformat(entry.url)) - result += '}\n' - file_path = os.path.join(self.root_dir, self._options.entries_filename) - logging.debug(result) - gclient_utils.FileWrite(file_path, result) + # Sometimes pprint.pformat will use {', sometimes it'll use { ' ... It + # makes testing a bit too fun. + result = 'entries = {\n' + for entry in self.root.subtree(False): + result += ' %s: %s,\n' % (pprint.pformat( + entry.name), pprint.pformat(entry.url)) + result += '}\n' + file_path = os.path.join(self.root_dir, self._options.entries_filename) + logging.debug(result) + gclient_utils.FileWrite(file_path, result) - def _ReadEntries(self): - """Read the .gclient_entries file for the given client. + def _ReadEntries(self): + """Read the .gclient_entries file for the given client. Returns: A sequence of solution names, which will be empty if there is the entries file hasn't been created yet. """ - scope = {} - filename = os.path.join(self.root_dir, self._options.entries_filename) - if not os.path.exists(filename): - return {} - try: - exec(gclient_utils.FileRead(filename), scope) - except SyntaxError as e: - gclient_utils.SyntaxErrorToError(filename, e) - return scope.get('entries', {}) + scope = {} + filename = os.path.join(self.root_dir, self._options.entries_filename) + if not os.path.exists(filename): + return {} + try: + exec(gclient_utils.FileRead(filename), scope) + except SyntaxError as e: + gclient_utils.SyntaxErrorToError(filename, e) + return scope.get('entries', {}) - def _ExtractFileJsonContents(self, default_filename): - # type: (str) -> Mapping[str,Any] - f = os.path.join(self.root_dir, default_filename) + def _ExtractFileJsonContents(self, default_filename): + # type: (str) -> Mapping[str,Any] + f = os.path.join(self.root_dir, default_filename) - if not os.path.exists(f): - logging.info('File %s does not exist.' % f) - return {} + if not os.path.exists(f): + logging.info('File %s does not exist.' % f) + return {} - with open(f, 'r') as open_f: - logging.info('Reading content from file %s' % f) - content = open_f.read().rstrip() - if content: - return json.loads(content) - return {} - - def _WriteFileContents(self, default_filename, content): - # type: (str, str) -> None - f = os.path.join(self.root_dir, default_filename) - - with open(f, 'w') as open_f: - logging.info('Writing to file %s' % f) - open_f.write(content) - - def _EnforceSkipSyncRevisions(self, patch_refs): - # type: (Mapping[str, str]) -> Mapping[str, str] - """Checks for and enforces revisions for skipping deps syncing.""" - previous_sync_commits = self._ExtractFileJsonContents( - PREVIOUS_SYNC_COMMITS_FILE) - - if not previous_sync_commits: - return {} - - # Current `self.dependencies` only contain solutions. If a patch_ref is - # not for a solution, then it is for a solution's dependency or recursed - # dependency which we cannot support while skipping sync. - if patch_refs: - unclaimed_prs = [] - candidates = [] - for dep in self.dependencies: - origin, _ = gclient_utils.SplitUrlRevision(dep.url) - candidates.extend([origin, dep.name]) - for patch_repo in patch_refs: - if not gclient_utils.FuzzyMatchRepo(patch_repo, candidates): - unclaimed_prs.append(patch_repo) - if unclaimed_prs: - print( - 'We cannot skip syncs when there are --patch-refs flags for ' - 'non-solution dependencies. To skip syncing, remove patch_refs ' - 'for: \n%s' % '\n'.join(unclaimed_prs)) + with open(f, 'r') as open_f: + logging.info('Reading content from file %s' % f) + content = open_f.read().rstrip() + if content: + return json.loads(content) return {} - # We cannot skip syncing if there are custom_vars that differ from the - # previous run's custom_vars. - previous_custom_vars = self._ExtractFileJsonContents( - PREVIOUS_CUSTOM_VARS_FILE) + def _WriteFileContents(self, default_filename, content): + # type: (str, str) -> None + f = os.path.join(self.root_dir, default_filename) - cvs_by_name = {s.name: s.custom_vars for s in self.dependencies} + with open(f, 'w') as open_f: + logging.info('Writing to file %s' % f) + open_f.write(content) - skip_sync_revisions = {} - for name, commit in previous_sync_commits.items(): - previous_vars = previous_custom_vars.get(name) - if previous_vars == cvs_by_name.get(name) or (not previous_vars and - not cvs_by_name.get(name)): - skip_sync_revisions[name] = commit - else: - print('We cannot skip syncs when custom_vars for solutions have ' - 'changed since the last sync run on this machine.\n' - '\nRemoving skip_sync_revision for:\n' - 'solution: %s, current: %r, previous: %r.' % - (name, cvs_by_name.get(name), previous_vars)) - print('no-sync experiment enabled with %r' % skip_sync_revisions) - return skip_sync_revisions + def _EnforceSkipSyncRevisions(self, patch_refs): + # type: (Mapping[str, str]) -> Mapping[str, str] + """Checks for and enforces revisions for skipping deps syncing.""" + previous_sync_commits = self._ExtractFileJsonContents( + PREVIOUS_SYNC_COMMITS_FILE) - # TODO(crbug.com/1340695): Remove handling revisions without '@'. - def _EnforceRevisions(self): - """Checks for revision overrides.""" - revision_overrides = {} - if self._options.head: - return revision_overrides - if not self._options.revisions: - return revision_overrides - solutions_names = [s.name for s in self.dependencies] - for index, revision in enumerate(self._options.revisions): - if not '@' in revision: - # Support for --revision 123 - revision = '%s@%s' % (solutions_names[index], revision) - name, rev = revision.split('@', 1) - revision_overrides[name] = rev - return revision_overrides + if not previous_sync_commits: + return {} - def _EnforcePatchRefsAndBranches(self): - # type: () -> Tuple[Mapping[str, str], Mapping[str, str]] - """Checks for patch refs.""" - patch_refs = {} - target_branches = {} - if not self._options.patch_refs: - return patch_refs, target_branches - for given_patch_ref in self._options.patch_refs: - patch_repo, _, patch_ref = given_patch_ref.partition('@') - if not patch_repo or not patch_ref or ':' not in patch_ref: - raise gclient_utils.Error( - 'Wrong revision format: %s should be of the form ' - 'patch_repo@target_branch:patch_ref.' % given_patch_ref) - target_branch, _, patch_ref = patch_ref.partition(':') - target_branches[patch_repo] = target_branch - patch_refs[patch_repo] = patch_ref - return patch_refs, target_branches + # Current `self.dependencies` only contain solutions. If a patch_ref is + # not for a solution, then it is for a solution's dependency or recursed + # dependency which we cannot support while skipping sync. + if patch_refs: + unclaimed_prs = [] + candidates = [] + for dep in self.dependencies: + origin, _ = gclient_utils.SplitUrlRevision(dep.url) + candidates.extend([origin, dep.name]) + for patch_repo in patch_refs: + if not gclient_utils.FuzzyMatchRepo(patch_repo, candidates): + unclaimed_prs.append(patch_repo) + if unclaimed_prs: + print( + 'We cannot skip syncs when there are --patch-refs flags for ' + 'non-solution dependencies. To skip syncing, remove patch_refs ' + 'for: \n%s' % '\n'.join(unclaimed_prs)) + return {} - def _RemoveUnversionedGitDirs(self): - """Remove directories that are no longer part of the checkout. + # We cannot skip syncing if there are custom_vars that differ from the + # previous run's custom_vars. + previous_custom_vars = self._ExtractFileJsonContents( + PREVIOUS_CUSTOM_VARS_FILE) + + cvs_by_name = {s.name: s.custom_vars for s in self.dependencies} + + skip_sync_revisions = {} + for name, commit in previous_sync_commits.items(): + previous_vars = previous_custom_vars.get(name) + if previous_vars == cvs_by_name.get(name) or ( + not previous_vars and not cvs_by_name.get(name)): + skip_sync_revisions[name] = commit + else: + print( + 'We cannot skip syncs when custom_vars for solutions have ' + 'changed since the last sync run on this machine.\n' + '\nRemoving skip_sync_revision for:\n' + 'solution: %s, current: %r, previous: %r.' % + (name, cvs_by_name.get(name), previous_vars)) + print('no-sync experiment enabled with %r' % skip_sync_revisions) + return skip_sync_revisions + + # TODO(crbug.com/1340695): Remove handling revisions without '@'. + def _EnforceRevisions(self): + """Checks for revision overrides.""" + revision_overrides = {} + if self._options.head: + return revision_overrides + if not self._options.revisions: + return revision_overrides + solutions_names = [s.name for s in self.dependencies] + for index, revision in enumerate(self._options.revisions): + if not '@' in revision: + # Support for --revision 123 + revision = '%s@%s' % (solutions_names[index], revision) + name, rev = revision.split('@', 1) + revision_overrides[name] = rev + return revision_overrides + + def _EnforcePatchRefsAndBranches(self): + # type: () -> Tuple[Mapping[str, str], Mapping[str, str]] + """Checks for patch refs.""" + patch_refs = {} + target_branches = {} + if not self._options.patch_refs: + return patch_refs, target_branches + for given_patch_ref in self._options.patch_refs: + patch_repo, _, patch_ref = given_patch_ref.partition('@') + if not patch_repo or not patch_ref or ':' not in patch_ref: + raise gclient_utils.Error( + 'Wrong revision format: %s should be of the form ' + 'patch_repo@target_branch:patch_ref.' % given_patch_ref) + target_branch, _, patch_ref = patch_ref.partition(':') + target_branches[patch_repo] = target_branch + patch_refs[patch_repo] = patch_ref + return patch_refs, target_branches + + def _RemoveUnversionedGitDirs(self): + """Remove directories that are no longer part of the checkout. Notify the user if there is an orphaned entry in their working copy. Only delete the directory if there are no changes in it, and @@ -1928,497 +1990,536 @@ it or fix the checkout. Returns CIPD packages that are no longer versioned. """ - entry_names_and_sync = [(i.name, i._should_sync) - for i in self.root.subtree(False) if i.url] - entries = [] - if entry_names_and_sync: - entries, _ = zip(*entry_names_and_sync) - full_entries = [os.path.join(self.root_dir, e.replace('/', os.path.sep)) - for e in entries] - no_sync_entries = [ - name for name, should_sync in entry_names_and_sync if not should_sync - ] + entry_names_and_sync = [(i.name, i._should_sync) + for i in self.root.subtree(False) if i.url] + entries = [] + if entry_names_and_sync: + entries, _ = zip(*entry_names_and_sync) + full_entries = [ + os.path.join(self.root_dir, e.replace('/', os.path.sep)) + for e in entries + ] + no_sync_entries = [ + name for name, should_sync in entry_names_and_sync + if not should_sync + ] - removed_cipd_entries = [] - for entry, prev_url in self._ReadEntries().items(): - if not prev_url: - # entry must have been overridden via .gclient custom_deps - continue - if any(entry.startswith(sln) for sln in no_sync_entries): - # Dependencies of solutions that skipped syncing would not - # show up in `entries`. - continue - if (':' in entry): - # This is a cipd package. Don't clean it up, but prepare for return - if entry not in entries: - removed_cipd_entries.append(entry) - continue - # Fix path separator on Windows. - entry_fixed = entry.replace('/', os.path.sep) - e_dir = os.path.join(self.root_dir, entry_fixed) - # Use entry and not entry_fixed there. - if (entry not in entries and - (not any(path.startswith(entry + '/') for path in entries)) and - os.path.exists(e_dir)): - # The entry has been removed from DEPS. - scm = gclient_scm.GitWrapper( - prev_url, self.root_dir, entry_fixed, self.outbuf) + removed_cipd_entries = [] + for entry, prev_url in self._ReadEntries().items(): + if not prev_url: + # entry must have been overridden via .gclient custom_deps + continue + if any(entry.startswith(sln) for sln in no_sync_entries): + # Dependencies of solutions that skipped syncing would not + # show up in `entries`. + continue + if (':' in entry): + # This is a cipd package. Don't clean it up, but prepare for + # return + if entry not in entries: + removed_cipd_entries.append(entry) + continue + # Fix path separator on Windows. + entry_fixed = entry.replace('/', os.path.sep) + e_dir = os.path.join(self.root_dir, entry_fixed) + # Use entry and not entry_fixed there. + if (entry not in entries and + (not any(path.startswith(entry + '/') for path in entries)) + and os.path.exists(e_dir)): + # The entry has been removed from DEPS. + scm = gclient_scm.GitWrapper(prev_url, self.root_dir, + entry_fixed, self.outbuf) - # Check to see if this directory is now part of a higher-up checkout. - scm_root = None - try: - scm_root = gclient_scm.scm.GIT.GetCheckoutRoot(scm.checkout_path) - except subprocess2.CalledProcessError: - pass - if not scm_root: - logging.warning('Could not find checkout root for %s. Unable to ' - 'determine whether it is part of a higher-level ' - 'checkout, so not removing.' % entry) - continue + # Check to see if this directory is now part of a higher-up + # checkout. + scm_root = None + try: + scm_root = gclient_scm.scm.GIT.GetCheckoutRoot( + scm.checkout_path) + except subprocess2.CalledProcessError: + pass + if not scm_root: + logging.warning( + 'Could not find checkout root for %s. Unable to ' + 'determine whether it is part of a higher-level ' + 'checkout, so not removing.' % entry) + continue - # This is to handle the case of third_party/WebKit migrating from - # being a DEPS entry to being part of the main project. - # If the subproject is a Git project, we need to remove its .git - # folder. Otherwise git operations on that folder will have different - # effects depending on the current working directory. - if os.path.abspath(scm_root) == os.path.abspath(e_dir): - e_par_dir = os.path.join(e_dir, os.pardir) - if gclient_scm.scm.GIT.IsInsideWorkTree(e_par_dir): - par_scm_root = gclient_scm.scm.GIT.GetCheckoutRoot(e_par_dir) - # rel_e_dir : relative path of entry w.r.t. its parent repo. - rel_e_dir = os.path.relpath(e_dir, par_scm_root) - if gclient_scm.scm.GIT.IsDirectoryVersioned( - par_scm_root, rel_e_dir): - save_dir = scm.GetGitBackupDirPath() - # Remove any eventual stale backup dir for the same project. - if os.path.exists(save_dir): - gclient_utils.rmtree(save_dir) - os.rename(os.path.join(e_dir, '.git'), save_dir) - # When switching between the two states (entry/ is a subproject - # -> entry/ is part of the outer project), it is very likely - # that some files are changed in the checkout, unless we are - # jumping *exactly* across the commit which changed just DEPS. - # In such case we want to cleanup any eventual stale files - # (coming from the old subproject) in order to end up with a - # clean checkout. - gclient_scm.scm.GIT.CleanupDir(par_scm_root, rel_e_dir) - assert not os.path.exists(os.path.join(e_dir, '.git')) - print('\nWARNING: \'%s\' has been moved from DEPS to a higher ' - 'level checkout. The git folder containing all the local' - ' branches has been saved to %s.\n' - 'If you don\'t care about its state you can safely ' - 'remove that folder to free up space.' % (entry, save_dir)) - continue + # This is to handle the case of third_party/WebKit migrating + # from being a DEPS entry to being part of the main project. If + # the subproject is a Git project, we need to remove its .git + # folder. Otherwise git operations on that folder will have + # different effects depending on the current working directory. + if os.path.abspath(scm_root) == os.path.abspath(e_dir): + e_par_dir = os.path.join(e_dir, os.pardir) + if gclient_scm.scm.GIT.IsInsideWorkTree(e_par_dir): + par_scm_root = gclient_scm.scm.GIT.GetCheckoutRoot( + e_par_dir) + # rel_e_dir : relative path of entry w.r.t. its parent + # repo. + rel_e_dir = os.path.relpath(e_dir, par_scm_root) + if gclient_scm.scm.GIT.IsDirectoryVersioned( + par_scm_root, rel_e_dir): + save_dir = scm.GetGitBackupDirPath() + # Remove any eventual stale backup dir for the same + # project. + if os.path.exists(save_dir): + gclient_utils.rmtree(save_dir) + os.rename(os.path.join(e_dir, '.git'), save_dir) + # When switching between the two states (entry/ is a + # subproject -> entry/ is part of the outer + # project), it is very likely that some files are + # changed in the checkout, unless we are jumping + # *exactly* across the commit which changed just + # DEPS. In such case we want to cleanup any eventual + # stale files (coming from the old subproject) in + # order to end up with a clean checkout. + gclient_scm.scm.GIT.CleanupDir( + par_scm_root, rel_e_dir) + assert not os.path.exists( + os.path.join(e_dir, '.git')) + print( + '\nWARNING: \'%s\' has been moved from DEPS to a higher ' + 'level checkout. The git folder containing all the local' + ' branches has been saved to %s.\n' + 'If you don\'t care about its state you can safely ' + 'remove that folder to free up space.' % + (entry, save_dir)) + continue - if scm_root in full_entries: - logging.info('%s is part of a higher level checkout, not removing', - scm.GetCheckoutRoot()) - continue + if scm_root in full_entries: + logging.info( + '%s is part of a higher level checkout, not removing', + scm.GetCheckoutRoot()) + continue - file_list = [] - scm.status(self._options, [], file_list) - modified_files = file_list != [] - if (not self._options.delete_unversioned_trees or - (modified_files and not self._options.force)): - # There are modified files in this entry. Keep warning until - # removed. - self.add_dependency( - GitDependency( - parent=self, - name=entry, - # Update URL with scheme in protocol_override - url=GitDependency.updateProtocol(prev_url, self.protocol), - managed=False, - custom_deps={}, - custom_vars={}, - custom_hooks=[], - deps_file=None, - should_process=True, - should_recurse=False, - relative=None, - condition=None, - protocol=self.protocol, - git_dependencies_state=self.git_dependencies_state)) - if modified_files and self._options.delete_unversioned_trees: - print('\nWARNING: \'%s\' is no longer part of this client.\n' - 'Despite running \'gclient sync -D\' no action was taken ' - 'as there are modifications.\nIt is recommended you revert ' - 'all changes or run \'gclient sync -D --force\' next ' - 'time.' % entry_fixed) - else: - print('\nWARNING: \'%s\' is no longer part of this client.\n' - 'It is recommended that you manually remove it or use ' - '\'gclient sync -D\' next time.' % entry_fixed) - else: - # Delete the entry - print('\n________ deleting \'%s\' in \'%s\'' % ( - entry_fixed, self.root_dir)) - gclient_utils.rmtree(e_dir) - # record the current list of entries for next time - self._SaveEntries() - return removed_cipd_entries + file_list = [] + scm.status(self._options, [], file_list) + modified_files = file_list != [] + if (not self._options.delete_unversioned_trees + or (modified_files and not self._options.force)): + # There are modified files in this entry. Keep warning until + # removed. + self.add_dependency( + GitDependency( + parent=self, + name=entry, + # Update URL with scheme in protocol_override + url=GitDependency.updateProtocol( + prev_url, self.protocol), + managed=False, + custom_deps={}, + custom_vars={}, + custom_hooks=[], + deps_file=None, + should_process=True, + should_recurse=False, + relative=None, + condition=None, + protocol=self.protocol, + git_dependencies_state=self.git_dependencies_state)) + if modified_files and self._options.delete_unversioned_trees: + print( + '\nWARNING: \'%s\' is no longer part of this client.\n' + 'Despite running \'gclient sync -D\' no action was taken ' + 'as there are modifications.\nIt is recommended you revert ' + 'all changes or run \'gclient sync -D --force\' next ' + 'time.' % entry_fixed) + else: + print( + '\nWARNING: \'%s\' is no longer part of this client.\n' + 'It is recommended that you manually remove it or use ' + '\'gclient sync -D\' next time.' % entry_fixed) + else: + # Delete the entry + print('\n________ deleting \'%s\' in \'%s\'' % + (entry_fixed, self.root_dir)) + gclient_utils.rmtree(e_dir) + # record the current list of entries for next time + self._SaveEntries() + return removed_cipd_entries - def RunOnDeps(self, command, args, ignore_requirements=False, progress=True): - """Runs a command on each dependency in a client and its dependencies. + def RunOnDeps(self, + command, + args, + ignore_requirements=False, + progress=True): + """Runs a command on each dependency in a client and its dependencies. Args: command: The command to use (e.g., 'status' or 'diff') args: list of str - extra arguments to add to the command line. """ - if not self.dependencies: - raise gclient_utils.Error('No solution specified') + if not self.dependencies: + raise gclient_utils.Error('No solution specified') - revision_overrides = {} - patch_refs = {} - target_branches = {} - skip_sync_revisions = {} - # It's unnecessary to check for revision overrides for 'recurse'. - # Save a few seconds by not calling _EnforceRevisions() in that case. - if command not in ('diff', 'recurse', 'runhooks', 'status', 'revert', - 'validate'): - self._CheckConfig() - revision_overrides = self._EnforceRevisions() + revision_overrides = {} + patch_refs = {} + target_branches = {} + skip_sync_revisions = {} + # It's unnecessary to check for revision overrides for 'recurse'. + # Save a few seconds by not calling _EnforceRevisions() in that case. + if command not in ('diff', 'recurse', 'runhooks', 'status', 'revert', + 'validate'): + self._CheckConfig() + revision_overrides = self._EnforceRevisions() - if command == 'update': - patch_refs, target_branches = self._EnforcePatchRefsAndBranches() - if NO_SYNC_EXPERIMENT in self._options.experiments: - skip_sync_revisions = self._EnforceSkipSyncRevisions(patch_refs) + if command == 'update': + patch_refs, target_branches = self._EnforcePatchRefsAndBranches() + if NO_SYNC_EXPERIMENT in self._options.experiments: + skip_sync_revisions = self._EnforceSkipSyncRevisions(patch_refs) - # Store solutions' custom_vars on memory to compare in the next run. - # All dependencies added later are inherited from the current - # self.dependencies. - custom_vars = { - dep.name: dep.custom_vars - for dep in self.dependencies if dep.custom_vars - } - if custom_vars: - self._WriteFileContents(PREVIOUS_CUSTOM_VARS_FILE, - json.dumps(custom_vars)) - - # Disable progress for non-tty stdout. - should_show_progress = ( - setup_color.IS_TTY and not self._options.verbose and progress) - pm = None - if should_show_progress: - if command in ('update', 'revert'): - pm = Progress('Syncing projects', 1) - elif command in ('recurse', 'validate'): - pm = Progress(' '.join(args), 1) - work_queue = gclient_utils.ExecutionQueue( - self._options.jobs, pm, ignore_requirements=ignore_requirements, - verbose=self._options.verbose) - for s in self.dependencies: - if s.should_process: - work_queue.enqueue(s) - work_queue.flush(revision_overrides, - command, - args, - options=self._options, - patch_refs=patch_refs, - target_branches=target_branches, - skip_sync_revisions=skip_sync_revisions) - - if revision_overrides: - print('Please fix your script, having invalid --revision flags will soon ' - 'be considered an error.', file=sys.stderr) - - if patch_refs: - raise gclient_utils.Error( - 'The following --patch-ref flags were not used. Please fix it:\n%s' % - ('\n'.join( - patch_repo + '@' + patch_ref - for patch_repo, patch_ref in patch_refs.items()))) - - # TODO(crbug.com/1475405): Warn users if the project uses submodules and - # they have fsmonitor enabled. - if command == 'update': - # Check if any of the root dependency have submodules. - is_submoduled = any( - map( - lambda d: d.git_dependencies_state in - (gclient_eval.SUBMODULES, gclient_eval.SYNC), self.dependencies)) - if is_submoduled: - git_common.warn_submodule() - - # Once all the dependencies have been processed, it's now safe to write - # out the gn_args_file and run the hooks. - removed_cipd_entries = [] - if command == 'update': - for dependency in self.dependencies: - gn_args_dep = dependency - if gn_args_dep._gn_args_from: - deps_map = {dep.name: dep for dep in gn_args_dep.dependencies} - gn_args_dep = deps_map.get(gn_args_dep._gn_args_from) - if gn_args_dep and gn_args_dep.HasGNArgsFile(): - gn_args_dep.WriteGNArgsFile() - - removed_cipd_entries = self._RemoveUnversionedGitDirs() - - # Sync CIPD dependencies once removed deps are deleted. In case a git - # dependency was moved to CIPD, we want to remove the old git directory - # first and then sync the CIPD dep. - if self._cipd_root: - self._cipd_root.run(command) - # It's possible that CIPD removed some entries that are now part of git - # worktree. Try to checkout those directories - if removed_cipd_entries: - for cipd_entry in removed_cipd_entries: - cwd = os.path.join(self._root_dir, cipd_entry.split(':')[0]) - cwd, tail = os.path.split(cwd) - if cwd: - try: - gclient_scm.scm.GIT.Capture(['checkout', tail], cwd=cwd) - except subprocess2.CalledProcessError: - pass - - if not self._options.nohooks: - if should_show_progress: - pm = Progress('Running hooks', 1) - self.RunHooksRecursively(self._options, pm) - - self._WriteFileContents(PREVIOUS_SYNC_COMMITS_FILE, - os.environ.get(PREVIOUS_SYNC_COMMITS, '{}')) - - return 0 - - def PrintRevInfo(self): - if not self.dependencies: - raise gclient_utils.Error('No solution specified') - # Load all the settings. - work_queue = gclient_utils.ExecutionQueue( - self._options.jobs, None, False, verbose=self._options.verbose) - for s in self.dependencies: - if s.should_process: - work_queue.enqueue(s) - work_queue.flush({}, - None, [], - options=self._options, - patch_refs=None, - target_branches=None, - skip_sync_revisions=None) - - def ShouldPrintRevision(dep): - return (not self._options.filter - or dep.FuzzyMatchUrl(self._options.filter)) - - if self._options.snapshot: - json_output = [] - # First level at .gclient - for d in self.dependencies: - entries = {} - def GrabDeps(dep): - """Recursively grab dependencies.""" - for rec_d in dep.dependencies: - rec_d.PinToActualRevision() - if ShouldPrintRevision(rec_d): - entries[rec_d.name] = rec_d.url - GrabDeps(rec_d) - - GrabDeps(d) - json_output.append({ - 'name': d.name, - 'solution_url': d.url, - 'deps_file': d.deps_file, - 'managed': d.managed, - 'custom_deps': entries, - }) - if self._options.output_json == '-': - print(json.dumps(json_output, indent=2, separators=(',', ': '))) - elif self._options.output_json: - with open(self._options.output_json, 'w') as f: - json.dump(json_output, f) - else: - # Print the snapshot configuration file - print(self.DEFAULT_SNAPSHOT_FILE_TEXT % { - 'solution_list': pprint.pformat(json_output, indent=2), - }) - else: - entries = {} - for d in self.root.subtree(False): - if self._options.actual: - d.PinToActualRevision() - if ShouldPrintRevision(d): - entries[d.name] = d.url - if self._options.output_json: - json_output = { - name: { - 'url': rev.split('@')[0] if rev else None, - 'rev': rev.split('@')[1] if rev and '@' in rev else None, - } - for name, rev in entries.items() + # Store solutions' custom_vars on memory to compare in the next run. + # All dependencies added later are inherited from the current + # self.dependencies. + custom_vars = { + dep.name: dep.custom_vars + for dep in self.dependencies if dep.custom_vars } - if self._options.output_json == '-': - print(json.dumps(json_output, indent=2, separators=(',', ': '))) + if custom_vars: + self._WriteFileContents(PREVIOUS_CUSTOM_VARS_FILE, + json.dumps(custom_vars)) + + # Disable progress for non-tty stdout. + should_show_progress = (setup_color.IS_TTY and not self._options.verbose + and progress) + pm = None + if should_show_progress: + if command in ('update', 'revert'): + pm = Progress('Syncing projects', 1) + elif command in ('recurse', 'validate'): + pm = Progress(' '.join(args), 1) + work_queue = gclient_utils.ExecutionQueue( + self._options.jobs, + pm, + ignore_requirements=ignore_requirements, + verbose=self._options.verbose) + for s in self.dependencies: + if s.should_process: + work_queue.enqueue(s) + work_queue.flush(revision_overrides, + command, + args, + options=self._options, + patch_refs=patch_refs, + target_branches=target_branches, + skip_sync_revisions=skip_sync_revisions) + + if revision_overrides: + print( + 'Please fix your script, having invalid --revision flags will soon ' + 'be considered an error.', + file=sys.stderr) + + if patch_refs: + raise gclient_utils.Error( + 'The following --patch-ref flags were not used. Please fix it:\n%s' + % ('\n'.join(patch_repo + '@' + patch_ref + for patch_repo, patch_ref in patch_refs.items()))) + + # TODO(crbug.com/1475405): Warn users if the project uses submodules and + # they have fsmonitor enabled. + if command == 'update': + # Check if any of the root dependency have submodules. + is_submoduled = any( + map( + lambda d: d.git_dependencies_state in + (gclient_eval.SUBMODULES, gclient_eval.SYNC), + self.dependencies)) + if is_submoduled: + git_common.warn_submodule() + + # Once all the dependencies have been processed, it's now safe to write + # out the gn_args_file and run the hooks. + removed_cipd_entries = [] + if command == 'update': + for dependency in self.dependencies: + gn_args_dep = dependency + if gn_args_dep._gn_args_from: + deps_map = { + dep.name: dep + for dep in gn_args_dep.dependencies + } + gn_args_dep = deps_map.get(gn_args_dep._gn_args_from) + if gn_args_dep and gn_args_dep.HasGNArgsFile(): + gn_args_dep.WriteGNArgsFile() + + removed_cipd_entries = self._RemoveUnversionedGitDirs() + + # Sync CIPD dependencies once removed deps are deleted. In case a git + # dependency was moved to CIPD, we want to remove the old git directory + # first and then sync the CIPD dep. + if self._cipd_root: + self._cipd_root.run(command) + # It's possible that CIPD removed some entries that are now part of + # git worktree. Try to checkout those directories + if removed_cipd_entries: + for cipd_entry in removed_cipd_entries: + cwd = os.path.join(self._root_dir, cipd_entry.split(':')[0]) + cwd, tail = os.path.split(cwd) + if cwd: + try: + gclient_scm.scm.GIT.Capture(['checkout', tail], + cwd=cwd) + except subprocess2.CalledProcessError: + pass + + if not self._options.nohooks: + if should_show_progress: + pm = Progress('Running hooks', 1) + self.RunHooksRecursively(self._options, pm) + + self._WriteFileContents(PREVIOUS_SYNC_COMMITS_FILE, + os.environ.get(PREVIOUS_SYNC_COMMITS, '{}')) + + return 0 + + def PrintRevInfo(self): + if not self.dependencies: + raise gclient_utils.Error('No solution specified') + # Load all the settings. + work_queue = gclient_utils.ExecutionQueue(self._options.jobs, + None, + False, + verbose=self._options.verbose) + for s in self.dependencies: + if s.should_process: + work_queue.enqueue(s) + work_queue.flush({}, + None, [], + options=self._options, + patch_refs=None, + target_branches=None, + skip_sync_revisions=None) + + def ShouldPrintRevision(dep): + return (not self._options.filter + or dep.FuzzyMatchUrl(self._options.filter)) + + if self._options.snapshot: + json_output = [] + # First level at .gclient + for d in self.dependencies: + entries = {} + + def GrabDeps(dep): + """Recursively grab dependencies.""" + for rec_d in dep.dependencies: + rec_d.PinToActualRevision() + if ShouldPrintRevision(rec_d): + entries[rec_d.name] = rec_d.url + GrabDeps(rec_d) + + GrabDeps(d) + json_output.append({ + 'name': d.name, + 'solution_url': d.url, + 'deps_file': d.deps_file, + 'managed': d.managed, + 'custom_deps': entries, + }) + if self._options.output_json == '-': + print(json.dumps(json_output, indent=2, separators=(',', ': '))) + elif self._options.output_json: + with open(self._options.output_json, 'w') as f: + json.dump(json_output, f) + else: + # Print the snapshot configuration file + print(self.DEFAULT_SNAPSHOT_FILE_TEXT % { + 'solution_list': pprint.pformat(json_output, indent=2), + }) else: - with open(self._options.output_json, 'w') as f: - json.dump(json_output, f) - else: - keys = sorted(entries.keys()) - for x in keys: - print('%s: %s' % (x, entries[x])) - logging.info(str(self)) + entries = {} + for d in self.root.subtree(False): + if self._options.actual: + d.PinToActualRevision() + if ShouldPrintRevision(d): + entries[d.name] = d.url + if self._options.output_json: + json_output = { + name: { + 'url': rev.split('@')[0] if rev else None, + 'rev': + rev.split('@')[1] if rev and '@' in rev else None, + } + for name, rev in entries.items() + } + if self._options.output_json == '-': + print( + json.dumps(json_output, + indent=2, + separators=(',', ': '))) + else: + with open(self._options.output_json, 'w') as f: + json.dump(json_output, f) + else: + keys = sorted(entries.keys()) + for x in keys: + print('%s: %s' % (x, entries[x])) + logging.info(str(self)) - def ParseDepsFile(self): - """No DEPS to parse for a .gclient file.""" - raise gclient_utils.Error('Internal error') + def ParseDepsFile(self): + """No DEPS to parse for a .gclient file.""" + raise gclient_utils.Error('Internal error') - def PrintLocationAndContents(self): - # Print out the .gclient file. This is longer than if we just printed the - # client dict, but more legible, and it might contain helpful comments. - print('Loaded .gclient config in %s:\n%s' % ( - self.root_dir, self.config_content)) + def PrintLocationAndContents(self): + # Print out the .gclient file. This is longer than if we just printed + # the client dict, but more legible, and it might contain helpful + # comments. + print('Loaded .gclient config in %s:\n%s' % + (self.root_dir, self.config_content)) - def GetCipdRoot(self): - if not self._cipd_root: - self._cipd_root = gclient_scm.CipdRoot( - self.root_dir, - # TODO(jbudorick): Support other service URLs as necessary. - # Service URLs should be constant over the scope of a cipd - # root, so a var per DEPS file specifying the service URL - # should suffice. - 'https://chrome-infra-packages.appspot.com') - return self._cipd_root + def GetCipdRoot(self): + if not self._cipd_root: + self._cipd_root = gclient_scm.CipdRoot( + self.root_dir, + # TODO(jbudorick): Support other service URLs as necessary. + # Service URLs should be constant over the scope of a cipd + # root, so a var per DEPS file specifying the service URL + # should suffice. + 'https://chrome-infra-packages.appspot.com') + return self._cipd_root - @property - def root_dir(self): - """Root directory of gclient checkout.""" - return self._root_dir + @property + def root_dir(self): + """Root directory of gclient checkout.""" + return self._root_dir - @property - def enforced_os(self): - """What deps_os entries that are to be parsed.""" - return self._enforced_os + @property + def enforced_os(self): + """What deps_os entries that are to be parsed.""" + return self._enforced_os - @property - def target_os(self): - return self._enforced_os + @property + def target_os(self): + return self._enforced_os - @property - def target_cpu(self): - return self._enforced_cpu + @property + def target_cpu(self): + return self._enforced_cpu class CipdDependency(Dependency): - """A Dependency object that represents a single CIPD package.""" + """A Dependency object that represents a single CIPD package.""" + def __init__(self, parent, name, dep_value, cipd_root, custom_vars, + should_process, relative, condition): + package = dep_value['package'] + version = dep_value['version'] + url = urllib.parse.urljoin(cipd_root.service_url, + '%s@%s' % (package, version)) + super(CipdDependency, self).__init__(parent=parent, + name=name + ':' + package, + url=url, + managed=None, + custom_deps=None, + custom_vars=custom_vars, + custom_hooks=None, + deps_file=None, + should_process=should_process, + should_recurse=False, + relative=relative, + condition=condition) + self._cipd_package = None + self._cipd_root = cipd_root + # CIPD wants /-separated paths, even on Windows. + native_subdir_path = os.path.relpath( + os.path.join(self.root.root_dir, name), cipd_root.root_dir) + self._cipd_subdir = posixpath.join(*native_subdir_path.split(os.sep)) + self._package_name = package + self._package_version = version - def __init__( - self, parent, name, dep_value, cipd_root, - custom_vars, should_process, relative, condition): - package = dep_value['package'] - version = dep_value['version'] - url = urllib.parse.urljoin(cipd_root.service_url, - '%s@%s' % (package, version)) - super(CipdDependency, self).__init__( - parent=parent, - name=name + ':' + package, - url=url, - managed=None, - custom_deps=None, - custom_vars=custom_vars, - custom_hooks=None, - deps_file=None, - should_process=should_process, - should_recurse=False, - relative=relative, - condition=condition) - self._cipd_package = None - self._cipd_root = cipd_root - # CIPD wants /-separated paths, even on Windows. - native_subdir_path = os.path.relpath( - os.path.join(self.root.root_dir, name), cipd_root.root_dir) - self._cipd_subdir = posixpath.join(*native_subdir_path.split(os.sep)) - self._package_name = package - self._package_version = version + #override + def run(self, revision_overrides, command, args, work_queue, options, + patch_refs, target_branches, skip_sync_revisions): + """Runs |command| then parse the DEPS file.""" + logging.info('CipdDependency(%s).run()' % self.name) + if not self.should_process: + return + self._CreatePackageIfNecessary() + super(CipdDependency, + self).run(revision_overrides, command, args, work_queue, options, + patch_refs, target_branches, skip_sync_revisions) - #override - def run(self, revision_overrides, command, args, work_queue, options, - patch_refs, target_branches, skip_sync_revisions): - """Runs |command| then parse the DEPS file.""" - logging.info('CipdDependency(%s).run()' % self.name) - if not self.should_process: - return - self._CreatePackageIfNecessary() - super(CipdDependency, - self).run(revision_overrides, command, args, work_queue, options, - patch_refs, target_branches, skip_sync_revisions) + def _CreatePackageIfNecessary(self): + # We lazily create the CIPD package to make sure that only packages + # that we want (as opposed to all packages defined in all DEPS files + # we parse) get added to the root and subsequently ensured. + if not self._cipd_package: + self._cipd_package = self._cipd_root.add_package( + self._cipd_subdir, self._package_name, self._package_version) - def _CreatePackageIfNecessary(self): - # We lazily create the CIPD package to make sure that only packages - # that we want (as opposed to all packages defined in all DEPS files - # we parse) get added to the root and subsequently ensured. - if not self._cipd_package: - self._cipd_package = self._cipd_root.add_package( - self._cipd_subdir, self._package_name, self._package_version) + def ParseDepsFile(self): + """CIPD dependencies are not currently allowed to have nested deps.""" + self.add_dependencies_and_close([], []) - def ParseDepsFile(self): - """CIPD dependencies are not currently allowed to have nested deps.""" - self.add_dependencies_and_close([], []) + #override + def verify_validity(self): + """CIPD dependencies allow duplicate name for packages in same directory.""" + logging.info('Dependency(%s).verify_validity()' % self.name) + return True - #override - def verify_validity(self): - """CIPD dependencies allow duplicate name for packages in same directory.""" - logging.info('Dependency(%s).verify_validity()' % self.name) - return True + #override + def GetScmName(self): + """Always 'cipd'.""" + return 'cipd' - #override - def GetScmName(self): - """Always 'cipd'.""" - return 'cipd' + def GetExpandedPackageName(self): + """Return the CIPD package name with the variables evaluated.""" + package = self._cipd_root.expand_package_name(self._package_name) + if package: + return package + return self._package_name - def GetExpandedPackageName(self): - """Return the CIPD package name with the variables evaluated.""" - package = self._cipd_root.expand_package_name(self._package_name) - if package: - return package - return self._package_name + #override + def CreateSCM(self, out_cb=None): + """Create a Wrapper instance suitable for handling this CIPD dependency.""" + self._CreatePackageIfNecessary() + return gclient_scm.CipdWrapper(self.url, + self.root.root_dir, + self.name, + self.outbuf, + out_cb, + root=self._cipd_root, + package=self._cipd_package) - #override - def CreateSCM(self, out_cb=None): - """Create a Wrapper instance suitable for handling this CIPD dependency.""" - self._CreatePackageIfNecessary() - return gclient_scm.CipdWrapper( - self.url, self.root.root_dir, self.name, self.outbuf, out_cb, - root=self._cipd_root, package=self._cipd_package) + def hierarchy(self, include_url=False, graphviz=False): + if graphviz: + return '' # graphviz lines not implemented for cipd deps. + return self.parent.hierarchy(include_url) + ' -> ' + self._cipd_subdir - def hierarchy(self, include_url=False, graphviz=False): - if graphviz: - return '' # graphviz lines not implemented for cipd deps. - return self.parent.hierarchy(include_url) + ' -> ' + self._cipd_subdir + def ToLines(self): + # () -> Sequence[str] + """Return a list of lines representing this in a DEPS file.""" + def escape_cipd_var(package): + return package.replace('{', '{{').replace('}', '}}') - def ToLines(self): - # () -> Sequence[str] - """Return a list of lines representing this in a DEPS file.""" - def escape_cipd_var(package): - return package.replace('{', '{{').replace('}', '}}') + s = [] + self._CreatePackageIfNecessary() + if self._cipd_package.authority_for_subdir: + condition_part = ([' "condition": %r,' % + self.condition] if self.condition else []) + s.extend([ + ' # %s' % self.hierarchy(include_url=False), + ' "%s": {' % (self.name.split(':')[0], ), + ' "packages": [', + ]) + for p in sorted(self._cipd_root.packages(self._cipd_subdir), + key=lambda x: x.name): + s.extend([ + ' {', + ' "package": "%s",' % escape_cipd_var(p.name), + ' "version": "%s",' % p.version, + ' },', + ]) - s = [] - self._CreatePackageIfNecessary() - if self._cipd_package.authority_for_subdir: - condition_part = ([' "condition": %r,' % self.condition] - if self.condition else []) - s.extend([ - ' # %s' % self.hierarchy(include_url=False), - ' "%s": {' % (self.name.split(':')[0],), - ' "packages": [', - ]) - for p in sorted( - self._cipd_root.packages(self._cipd_subdir), - key=lambda x: x.name): - s.extend([ - ' {', - ' "package": "%s",' % escape_cipd_var(p.name), - ' "version": "%s",' % p.version, - ' },', - ]) - - s.extend([ - ' ],', - ' "dep_type": "cipd",', - ] + condition_part + [ - ' },', - '', - ]) - return s + s.extend([ + ' ],', + ' "dep_type": "cipd",', + ] + condition_part + [ + ' },', + '', + ]) + return s #### gclient commands. @@ -2427,7 +2528,7 @@ class CipdDependency(Dependency): @subcommand.usage('[command] [args ...]') @metrics.collector.collect_metrics('gclient recurse') def CMDrecurse(parser, args): - """Operates [command args ...] on all the dependencies. + """Operates [command args ...] on all the dependencies. Change directory to each dependency's directory, and call [command args ...] there. Sets GCLIENT_DEP_PATH environment variable as the @@ -2439,526 +2540,545 @@ def CMDrecurse(parser, args): * `gclient recurse --no-progress -j1 sh -c "pwd"` print the absolute path of each dependency. """ - # Stop parsing at the first non-arg so that these go through to the command - parser.disable_interspersed_args() - parser.add_option('-s', '--scm', action='append', default=[], - help='Choose scm types to operate upon.') - parser.add_option('-i', '--ignore', action='store_true', - help='Ignore non-zero return codes from subcommands.') - parser.add_option('--prepend-dir', action='store_true', - help='Prepend relative dir for use with git --null.') - parser.add_option('--no-progress', action='store_true', - help='Disable progress bar that shows sub-command updates') - options, args = parser.parse_args(args) - if not args: - print('Need to supply a command!', file=sys.stderr) - return 1 - root_and_entries = gclient_utils.GetGClientRootAndEntries() - if not root_and_entries: - print( - 'You need to run gclient sync at least once to use \'recurse\'.\n' - 'This is because .gclient_entries needs to exist and be up to date.', - file=sys.stderr) - return 1 + # Stop parsing at the first non-arg so that these go through to the command + parser.disable_interspersed_args() + parser.add_option('-s', + '--scm', + action='append', + default=[], + help='Choose scm types to operate upon.') + parser.add_option('-i', + '--ignore', + action='store_true', + help='Ignore non-zero return codes from subcommands.') + parser.add_option( + '--prepend-dir', + action='store_true', + help='Prepend relative dir for use with git --null.') + parser.add_option( + '--no-progress', + action='store_true', + help='Disable progress bar that shows sub-command updates') + options, args = parser.parse_args(args) + if not args: + print('Need to supply a command!', file=sys.stderr) + return 1 + root_and_entries = gclient_utils.GetGClientRootAndEntries() + if not root_and_entries: + print( + 'You need to run gclient sync at least once to use \'recurse\'.\n' + 'This is because .gclient_entries needs to exist and be up to date.', + file=sys.stderr) + return 1 - # Normalize options.scm to a set() - scm_set = set() - for scm in options.scm: - scm_set.update(scm.split(',')) - options.scm = scm_set + # Normalize options.scm to a set() + scm_set = set() + for scm in options.scm: + scm_set.update(scm.split(',')) + options.scm = scm_set - options.nohooks = True - client = GClient.LoadCurrentConfig(options) - if not client: - raise gclient_utils.Error('client not configured; see \'gclient config\'') - return client.RunOnDeps('recurse', args, ignore_requirements=True, - progress=not options.no_progress) + options.nohooks = True + client = GClient.LoadCurrentConfig(options) + if not client: + raise gclient_utils.Error( + 'client not configured; see \'gclient config\'') + return client.RunOnDeps('recurse', + args, + ignore_requirements=True, + progress=not options.no_progress) @subcommand.usage('[args ...]') @metrics.collector.collect_metrics('gclient fetch') def CMDfetch(parser, args): - """Fetches upstream commits for all modules. + """Fetches upstream commits for all modules. Completely git-specific. Simply runs 'git fetch [args ...]' for each module. """ - (options, args) = parser.parse_args(args) - return CMDrecurse(OptionParser(), [ - '--jobs=%d' % options.jobs, '--scm=git', 'git', 'fetch'] + args) + (options, args) = parser.parse_args(args) + return CMDrecurse( + OptionParser(), + ['--jobs=%d' % options.jobs, '--scm=git', 'git', 'fetch'] + args) class Flattener(object): - """Flattens a gclient solution.""" - - def __init__(self, client, pin_all_deps=False): - """Constructor. + """Flattens a gclient solution.""" + def __init__(self, client, pin_all_deps=False): + """Constructor. Arguments: client (GClient): client to flatten pin_all_deps (bool): whether to pin all deps, even if they're not pinned in DEPS """ - self._client = client + self._client = client - self._deps_string = None - self._deps_graph_lines = None - self._deps_files = set() + self._deps_string = None + self._deps_graph_lines = None + self._deps_files = set() - self._allowed_hosts = set() - self._deps = {} - self._hooks = [] - self._pre_deps_hooks = [] - self._vars = {} + self._allowed_hosts = set() + self._deps = {} + self._hooks = [] + self._pre_deps_hooks = [] + self._vars = {} - self._flatten(pin_all_deps=pin_all_deps) + self._flatten(pin_all_deps=pin_all_deps) - @property - def deps_string(self): - assert self._deps_string is not None - return self._deps_string + @property + def deps_string(self): + assert self._deps_string is not None + return self._deps_string - @property - def deps_graph_lines(self): - assert self._deps_graph_lines is not None - return self._deps_graph_lines + @property + def deps_graph_lines(self): + assert self._deps_graph_lines is not None + return self._deps_graph_lines - @property - def deps_files(self): - return self._deps_files + @property + def deps_files(self): + return self._deps_files - def _pin_dep(self, dep): - """Pins a dependency to specific full revision sha. + def _pin_dep(self, dep): + """Pins a dependency to specific full revision sha. Arguments: dep (Dependency): dependency to process """ - if dep.url is None: - return + if dep.url is None: + return - # Make sure the revision is always fully specified (a hash), - # as opposed to refs or tags which might change. Similarly, - # shortened shas might become ambiguous; make sure to always - # use full one for pinning. - revision = gclient_utils.SplitUrlRevision(dep.url)[1] - if not revision or not gclient_utils.IsFullGitSha(revision): - dep.PinToActualRevision() + # Make sure the revision is always fully specified (a hash), + # as opposed to refs or tags which might change. Similarly, + # shortened shas might become ambiguous; make sure to always + # use full one for pinning. + revision = gclient_utils.SplitUrlRevision(dep.url)[1] + if not revision or not gclient_utils.IsFullGitSha(revision): + dep.PinToActualRevision() - def _flatten(self, pin_all_deps=False): - """Runs the flattener. Saves resulting DEPS string. + def _flatten(self, pin_all_deps=False): + """Runs the flattener. Saves resulting DEPS string. Arguments: pin_all_deps (bool): whether to pin all deps, even if they're not pinned in DEPS """ - for solution in self._client.dependencies: - self._add_dep(solution) - self._flatten_dep(solution) + for solution in self._client.dependencies: + self._add_dep(solution) + self._flatten_dep(solution) - if pin_all_deps: - for dep in self._deps.values(): - self._pin_dep(dep) + if pin_all_deps: + for dep in self._deps.values(): + self._pin_dep(dep) - def add_deps_file(dep): - # Only include DEPS files referenced by recursedeps. - if not dep.should_recurse: - return - deps_file = dep.deps_file - deps_path = os.path.join(self._client.root_dir, dep.name, deps_file) - if not os.path.exists(deps_path): - # gclient has a fallback that if deps_file doesn't exist, it'll try - # DEPS. Do the same here. - deps_file = 'DEPS' - deps_path = os.path.join(self._client.root_dir, dep.name, deps_file) - if not os.path.exists(deps_path): - return - assert dep.url - self._deps_files.add((dep.url, deps_file, dep.hierarchy_data())) - for dep in self._deps.values(): - add_deps_file(dep) + def add_deps_file(dep): + # Only include DEPS files referenced by recursedeps. + if not dep.should_recurse: + return + deps_file = dep.deps_file + deps_path = os.path.join(self._client.root_dir, dep.name, deps_file) + if not os.path.exists(deps_path): + # gclient has a fallback that if deps_file doesn't exist, it'll + # try DEPS. Do the same here. + deps_file = 'DEPS' + deps_path = os.path.join(self._client.root_dir, dep.name, + deps_file) + if not os.path.exists(deps_path): + return + assert dep.url + self._deps_files.add((dep.url, deps_file, dep.hierarchy_data())) - gn_args_dep = self._deps.get(self._client.dependencies[0]._gn_args_from, - self._client.dependencies[0]) + for dep in self._deps.values(): + add_deps_file(dep) - self._deps_graph_lines = _DepsToDotGraphLines(self._deps) - self._deps_string = '\n'.join( - _GNSettingsToLines(gn_args_dep._gn_args_file, gn_args_dep._gn_args) + - _AllowedHostsToLines(self._allowed_hosts) + _DepsToLines(self._deps) + - _HooksToLines('hooks', self._hooks) + - _HooksToLines('pre_deps_hooks', self._pre_deps_hooks) + - _VarsToLines(self._vars) + [ - '# %s, %s' % (url, deps_file) - for url, deps_file, _ in sorted(self._deps_files) - ] + ['']) # Ensure newline at end of file. + gn_args_dep = self._deps.get(self._client.dependencies[0]._gn_args_from, + self._client.dependencies[0]) - def _add_dep(self, dep): - """Helper to add a dependency to flattened DEPS. + self._deps_graph_lines = _DepsToDotGraphLines(self._deps) + self._deps_string = '\n'.join( + _GNSettingsToLines(gn_args_dep._gn_args_file, gn_args_dep._gn_args) + + _AllowedHostsToLines(self._allowed_hosts) + + _DepsToLines(self._deps) + _HooksToLines('hooks', self._hooks) + + _HooksToLines('pre_deps_hooks', self._pre_deps_hooks) + + _VarsToLines(self._vars) + [ + '# %s, %s' % (url, deps_file) + for url, deps_file, _ in sorted(self._deps_files) + ] + ['']) # Ensure newline at end of file. + + def _add_dep(self, dep): + """Helper to add a dependency to flattened DEPS. Arguments: dep (Dependency): dependency to add """ - assert dep.name not in self._deps or self._deps.get(dep.name) == dep, ( - dep.name, self._deps.get(dep.name)) - if dep.url: - self._deps[dep.name] = dep + assert dep.name not in self._deps or self._deps.get( + dep.name) == dep, (dep.name, self._deps.get(dep.name)) + if dep.url: + self._deps[dep.name] = dep - def _flatten_dep(self, dep): - """Visits a dependency in order to flatten it (see CMDflatten). + def _flatten_dep(self, dep): + """Visits a dependency in order to flatten it (see CMDflatten). Arguments: dep (Dependency): dependency to process """ - logging.debug('_flatten_dep(%s)', dep.name) + logging.debug('_flatten_dep(%s)', dep.name) - assert dep.deps_parsed, ( - "Attempted to flatten %s but it has not been processed." % dep.name) + assert dep.deps_parsed, ( + "Attempted to flatten %s but it has not been processed." % dep.name) - self._allowed_hosts.update(dep.allowed_hosts) + self._allowed_hosts.update(dep.allowed_hosts) - # Only include vars explicitly listed in the DEPS files or gclient solution, - # not automatic, local overrides (i.e. not all of dep.get_vars()). - hierarchy = dep.hierarchy(include_url=False) - for key, value in dep._vars.items(): - # Make sure there are no conflicting variables. It is fine however - # to use same variable name, as long as the value is consistent. - assert key not in self._vars or self._vars[key][1] == value, ( - "dep:%s key:%s value:%s != %s" % ( - dep.name, key, value, self._vars[key][1])) - self._vars[key] = (hierarchy, value) - # Override explicit custom variables. - for key, value in dep.custom_vars.items(): - # Do custom_vars that don't correspond to DEPS vars ever make sense? DEPS - # conditionals shouldn't be using vars that aren't also defined in the - # DEPS (presubmit actually disallows this), so any new custom_var must be - # unused in the DEPS, so no need to add it to the flattened output either. - if key not in self._vars: - continue - # Don't "override" existing vars if it's actually the same value. - if self._vars[key][1] == value: - continue - # Anything else is overriding a default value from the DEPS. - self._vars[key] = (hierarchy + ' [custom_var override]', value) + # Only include vars explicitly listed in the DEPS files or gclient + # solution, not automatic, local overrides (i.e. not all of + # dep.get_vars()). + hierarchy = dep.hierarchy(include_url=False) + for key, value in dep._vars.items(): + # Make sure there are no conflicting variables. It is fine however + # to use same variable name, as long as the value is consistent. + assert key not in self._vars or self._vars[key][1] == value, ( + "dep:%s key:%s value:%s != %s" % + (dep.name, key, value, self._vars[key][1])) + self._vars[key] = (hierarchy, value) + # Override explicit custom variables. + for key, value in dep.custom_vars.items(): + # Do custom_vars that don't correspond to DEPS vars ever make sense? + # DEPS conditionals shouldn't be using vars that aren't also defined + # in the DEPS (presubmit actually disallows this), so any new + # custom_var must be unused in the DEPS, so no need to add it to the + # flattened output either. + if key not in self._vars: + continue + # Don't "override" existing vars if it's actually the same value. + if self._vars[key][1] == value: + continue + # Anything else is overriding a default value from the DEPS. + self._vars[key] = (hierarchy + ' [custom_var override]', value) - self._pre_deps_hooks.extend([(dep, hook) for hook in dep.pre_deps_hooks]) - self._hooks.extend([(dep, hook) for hook in dep.deps_hooks]) + self._pre_deps_hooks.extend([(dep, hook) + for hook in dep.pre_deps_hooks]) + self._hooks.extend([(dep, hook) for hook in dep.deps_hooks]) - for sub_dep in dep.dependencies: - self._add_dep(sub_dep) + for sub_dep in dep.dependencies: + self._add_dep(sub_dep) - for d in dep.dependencies: - if d.should_recurse: - self._flatten_dep(d) + for d in dep.dependencies: + if d.should_recurse: + self._flatten_dep(d) @metrics.collector.collect_metrics('gclient gitmodules') def CMDgitmodules(parser, args): - """Adds or updates Git Submodules based on the contents of the DEPS file. + """Adds or updates Git Submodules based on the contents of the DEPS file. This command should be run in the root director of the repo. It will create or update the .gitmodules file and include `gclient-condition` values. Commits in gitlinks will also be updated. """ - parser.add_option('--output-gitmodules', - help='name of the .gitmodules file to write to', - default='.gitmodules') - parser.add_option( - '--deps-file', - help= - 'name of the deps file to parse for git dependency paths and commits.', - default='DEPS') - parser.add_option( - '--skip-dep', - action="append", - help='skip adding gitmodules for the git dependency at the given path', - default=[]) - options, args = parser.parse_args(args) + parser.add_option('--output-gitmodules', + help='name of the .gitmodules file to write to', + default='.gitmodules') + parser.add_option( + '--deps-file', + help= + 'name of the deps file to parse for git dependency paths and commits.', + default='DEPS') + parser.add_option( + '--skip-dep', + action="append", + help='skip adding gitmodules for the git dependency at the given path', + default=[]) + options, args = parser.parse_args(args) - deps_dir = os.path.dirname(os.path.abspath(options.deps_file)) - gclient_path = gclient_paths.FindGclientRoot(deps_dir) - if not gclient_path: - logging.error( - '.gclient not found\n' - 'Make sure you are running this script from a gclient workspace.') - sys.exit(1) + deps_dir = os.path.dirname(os.path.abspath(options.deps_file)) + gclient_path = gclient_paths.FindGclientRoot(deps_dir) + if not gclient_path: + logging.error( + '.gclient not found\n' + 'Make sure you are running this script from a gclient workspace.') + sys.exit(1) - deps_content = gclient_utils.FileRead(options.deps_file) - ls = gclient_eval.Parse(deps_content, options.deps_file, None, None) + deps_content = gclient_utils.FileRead(options.deps_file) + ls = gclient_eval.Parse(deps_content, options.deps_file, None, None) - prefix_length = 0 - if not 'use_relative_paths' in ls or ls['use_relative_paths'] != True: - delta_path = os.path.relpath(deps_dir, os.path.abspath(gclient_path)) - if delta_path: - prefix_length = len(delta_path.replace(os.path.sep, '/')) + 1 + prefix_length = 0 + if not 'use_relative_paths' in ls or ls['use_relative_paths'] != True: + delta_path = os.path.relpath(deps_dir, os.path.abspath(gclient_path)) + if delta_path: + prefix_length = len(delta_path.replace(os.path.sep, '/')) + 1 - cache_info = [] - with open(options.output_gitmodules, 'w', newline='') as f: - for path, dep in ls.get('deps').items(): - if path in options.skip_dep: - continue - if dep.get('dep_type') == 'cipd': - continue - try: - url, commit = dep['url'].split('@', maxsplit=1) - except ValueError: - logging.error('error on %s; %s, not adding it', path, dep["url"]) - continue - if prefix_length: - path = path[prefix_length:] + cache_info = [] + with open(options.output_gitmodules, 'w', newline='') as f: + for path, dep in ls.get('deps').items(): + if path in options.skip_dep: + continue + if dep.get('dep_type') == 'cipd': + continue + try: + url, commit = dep['url'].split('@', maxsplit=1) + except ValueError: + logging.error('error on %s; %s, not adding it', path, + dep["url"]) + continue + if prefix_length: + path = path[prefix_length:] - cache_info += ['--cacheinfo', f'160000,{commit},{path}'] - f.write(f'[submodule "{path}"]\n\tpath = {path}\n\turl = {url}\n') - if 'condition' in dep: - f.write(f'\tgclient-condition = {dep["condition"]}\n') - # Windows has limit how long, so let's chunk those calls. - if len(cache_info) >= 100: + cache_info += ['--cacheinfo', f'160000,{commit},{path}'] + f.write(f'[submodule "{path}"]\n\tpath = {path}\n\turl = {url}\n') + if 'condition' in dep: + f.write(f'\tgclient-condition = {dep["condition"]}\n') + # Windows has limit how long, so let's chunk those calls. + if len(cache_info) >= 100: + subprocess2.call(['git', 'update-index', '--add'] + cache_info) + cache_info = [] + + if cache_info: subprocess2.call(['git', 'update-index', '--add'] + cache_info) - cache_info = [] - - if cache_info: - subprocess2.call(['git', 'update-index', '--add'] + cache_info) - subprocess2.call(['git', 'add', '.gitmodules']) - print('.gitmodules and gitlinks updated. Please check git diff and ' - 'commit changes.') + subprocess2.call(['git', 'add', '.gitmodules']) + print('.gitmodules and gitlinks updated. Please check git diff and ' + 'commit changes.') @metrics.collector.collect_metrics('gclient flatten') def CMDflatten(parser, args): - """Flattens the solutions into a single DEPS file.""" - parser.add_option('--output-deps', help='Path to the output DEPS file') - parser.add_option( - '--output-deps-files', - help=('Path to the output metadata about DEPS files referenced by ' - 'recursedeps.')) - parser.add_option( - '--pin-all-deps', action='store_true', - help=('Pin all deps, even if not pinned in DEPS. CAVEAT: only does so ' - 'for checked out deps, NOT deps_os.')) - parser.add_option('--deps-graph-file', - help='Provide a path for the output graph file') - options, args = parser.parse_args(args) + """Flattens the solutions into a single DEPS file.""" + parser.add_option('--output-deps', help='Path to the output DEPS file') + parser.add_option( + '--output-deps-files', + help=('Path to the output metadata about DEPS files referenced by ' + 'recursedeps.')) + parser.add_option( + '--pin-all-deps', + action='store_true', + help=('Pin all deps, even if not pinned in DEPS. CAVEAT: only does so ' + 'for checked out deps, NOT deps_os.')) + parser.add_option('--deps-graph-file', + help='Provide a path for the output graph file') + options, args = parser.parse_args(args) - options.nohooks = True - options.process_all_deps = True - client = GClient.LoadCurrentConfig(options) - if not client: - raise gclient_utils.Error('client not configured; see \'gclient config\'') + options.nohooks = True + options.process_all_deps = True + client = GClient.LoadCurrentConfig(options) + if not client: + raise gclient_utils.Error( + 'client not configured; see \'gclient config\'') - # Only print progress if we're writing to a file. Otherwise, progress updates - # could obscure intended output. - code = client.RunOnDeps('flatten', args, progress=options.output_deps) - if code != 0: - return code + # Only print progress if we're writing to a file. Otherwise, progress + # updates could obscure intended output. + code = client.RunOnDeps('flatten', args, progress=options.output_deps) + if code != 0: + return code - flattener = Flattener(client, pin_all_deps=options.pin_all_deps) + flattener = Flattener(client, pin_all_deps=options.pin_all_deps) - if options.output_deps: - with open(options.output_deps, 'w') as f: - f.write(flattener.deps_string) - else: - print(flattener.deps_string) + if options.output_deps: + with open(options.output_deps, 'w') as f: + f.write(flattener.deps_string) + else: + print(flattener.deps_string) - if options.deps_graph_file: - with open(options.deps_graph_file, 'w') as f: - f.write('\n'.join(flattener.deps_graph_lines)) + if options.deps_graph_file: + with open(options.deps_graph_file, 'w') as f: + f.write('\n'.join(flattener.deps_graph_lines)) - deps_files = [{'url': d[0], 'deps_file': d[1], 'hierarchy': d[2]} - for d in sorted(flattener.deps_files)] - if options.output_deps_files: - with open(options.output_deps_files, 'w') as f: - json.dump(deps_files, f) + deps_files = [{ + 'url': d[0], + 'deps_file': d[1], + 'hierarchy': d[2] + } for d in sorted(flattener.deps_files)] + if options.output_deps_files: + with open(options.output_deps_files, 'w') as f: + json.dump(deps_files, f) - return 0 + return 0 def _GNSettingsToLines(gn_args_file, gn_args): - s = [] - if gn_args_file: - s.extend([ - 'gclient_gn_args_file = "%s"' % gn_args_file, - 'gclient_gn_args = %r' % gn_args, - ]) - return s + s = [] + if gn_args_file: + s.extend([ + 'gclient_gn_args_file = "%s"' % gn_args_file, + 'gclient_gn_args = %r' % gn_args, + ]) + return s def _AllowedHostsToLines(allowed_hosts): - """Converts |allowed_hosts| set to list of lines for output.""" - if not allowed_hosts: - return [] - s = ['allowed_hosts = ['] - for h in sorted(allowed_hosts): - s.append(' "%s",' % h) - s.extend([']', '']) - return s + """Converts |allowed_hosts| set to list of lines for output.""" + if not allowed_hosts: + return [] + s = ['allowed_hosts = ['] + for h in sorted(allowed_hosts): + s.append(' "%s",' % h) + s.extend([']', '']) + return s def _DepsToLines(deps): - # type: (Mapping[str, Dependency]) -> Sequence[str] - """Converts |deps| dict to list of lines for output.""" - if not deps: - return [] - s = ['deps = {'] - for _, dep in sorted(deps.items()): - s.extend(dep.ToLines()) - s.extend(['}', '']) - return s + # type: (Mapping[str, Dependency]) -> Sequence[str] + """Converts |deps| dict to list of lines for output.""" + if not deps: + return [] + s = ['deps = {'] + for _, dep in sorted(deps.items()): + s.extend(dep.ToLines()) + s.extend(['}', '']) + return s def _DepsToDotGraphLines(deps): - # type: (Mapping[str, Dependency]) -> Sequence[str] - """Converts |deps| dict to list of lines for dot graphs""" - if not deps: - return [] - graph_lines = ["digraph {\n\trankdir=\"LR\";"] - for _, dep in sorted(deps.items()): - line = dep.hierarchy(include_url=False, graphviz=True) - if line: - graph_lines.append("\t%s" % line) - graph_lines.append("}") - return graph_lines + # type: (Mapping[str, Dependency]) -> Sequence[str] + """Converts |deps| dict to list of lines for dot graphs""" + if not deps: + return [] + graph_lines = ["digraph {\n\trankdir=\"LR\";"] + for _, dep in sorted(deps.items()): + line = dep.hierarchy(include_url=False, graphviz=True) + if line: + graph_lines.append("\t%s" % line) + graph_lines.append("}") + return graph_lines def _DepsOsToLines(deps_os): - """Converts |deps_os| dict to list of lines for output.""" - if not deps_os: - return [] - s = ['deps_os = {'] - for dep_os, os_deps in sorted(deps_os.items()): - s.append(' "%s": {' % dep_os) - for name, dep in sorted(os_deps.items()): - condition_part = ([' "condition": %r,' % dep.condition] - if dep.condition else []) - s.extend([ - ' # %s' % dep.hierarchy(include_url=False), - ' "%s": {' % (name,), - ' "url": "%s",' % (dep.url,), - ] + condition_part + [ - ' },', - '', - ]) - s.extend([' },', '']) - s.extend(['}', '']) - return s + """Converts |deps_os| dict to list of lines for output.""" + if not deps_os: + return [] + s = ['deps_os = {'] + for dep_os, os_deps in sorted(deps_os.items()): + s.append(' "%s": {' % dep_os) + for name, dep in sorted(os_deps.items()): + condition_part = ([' "condition": %r,' % + dep.condition] if dep.condition else []) + s.extend([ + ' # %s' % dep.hierarchy(include_url=False), + ' "%s": {' % (name, ), + ' "url": "%s",' % (dep.url, ), + ] + condition_part + [ + ' },', + '', + ]) + s.extend([' },', '']) + s.extend(['}', '']) + return s def _HooksToLines(name, hooks): - """Converts |hooks| list to list of lines for output.""" - if not hooks: - return [] - s = ['%s = [' % name] - for dep, hook in hooks: - s.extend([ - ' # %s' % dep.hierarchy(include_url=False), - ' {', - ]) - if hook.name is not None: - s.append(' "name": "%s",' % hook.name) - if hook.pattern is not None: - s.append(' "pattern": "%s",' % hook.pattern) - if hook.condition is not None: - s.append(' "condition": %r,' % hook.condition) - # Flattened hooks need to be written relative to the root gclient dir - cwd = os.path.relpath(os.path.normpath(hook.effective_cwd)) - s.extend( - [' "cwd": "%s",' % cwd] + - [' "action": ['] + - [' "%s",' % arg for arg in hook.action] + - [' ]', ' },', ''] - ) - s.extend([']', '']) - return s + """Converts |hooks| list to list of lines for output.""" + if not hooks: + return [] + s = ['%s = [' % name] + for dep, hook in hooks: + s.extend([ + ' # %s' % dep.hierarchy(include_url=False), + ' {', + ]) + if hook.name is not None: + s.append(' "name": "%s",' % hook.name) + if hook.pattern is not None: + s.append(' "pattern": "%s",' % hook.pattern) + if hook.condition is not None: + s.append(' "condition": %r,' % hook.condition) + # Flattened hooks need to be written relative to the root gclient dir + cwd = os.path.relpath(os.path.normpath(hook.effective_cwd)) + s.extend([' "cwd": "%s",' % cwd] + [' "action": ['] + + [' "%s",' % arg + for arg in hook.action] + [' ]', ' },', '']) + s.extend([']', '']) + return s def _HooksOsToLines(hooks_os): - """Converts |hooks| list to list of lines for output.""" - if not hooks_os: - return [] - s = ['hooks_os = {'] - for hook_os, os_hooks in hooks_os.items(): - s.append(' "%s": [' % hook_os) - for dep, hook in os_hooks: - s.extend([ - ' # %s' % dep.hierarchy(include_url=False), - ' {', - ]) - if hook.name is not None: - s.append(' "name": "%s",' % hook.name) - if hook.pattern is not None: - s.append(' "pattern": "%s",' % hook.pattern) - if hook.condition is not None: - s.append(' "condition": %r,' % hook.condition) - # Flattened hooks need to be written relative to the root gclient dir - cwd = os.path.relpath(os.path.normpath(hook.effective_cwd)) - s.extend( - [' "cwd": "%s",' % cwd] + - [' "action": ['] + - [' "%s",' % arg for arg in hook.action] + - [' ]', ' },', ''] - ) - s.extend([' ],', '']) - s.extend(['}', '']) - return s + """Converts |hooks| list to list of lines for output.""" + if not hooks_os: + return [] + s = ['hooks_os = {'] + for hook_os, os_hooks in hooks_os.items(): + s.append(' "%s": [' % hook_os) + for dep, hook in os_hooks: + s.extend([ + ' # %s' % dep.hierarchy(include_url=False), + ' {', + ]) + if hook.name is not None: + s.append(' "name": "%s",' % hook.name) + if hook.pattern is not None: + s.append(' "pattern": "%s",' % hook.pattern) + if hook.condition is not None: + s.append(' "condition": %r,' % hook.condition) + # Flattened hooks need to be written relative to the root gclient + # dir + cwd = os.path.relpath(os.path.normpath(hook.effective_cwd)) + s.extend([' "cwd": "%s",' % cwd] + [' "action": ['] + + [' "%s",' % arg + for arg in hook.action] + [' ]', ' },', '']) + s.extend([' ],', '']) + s.extend(['}', '']) + return s def _VarsToLines(variables): - """Converts |variables| dict to list of lines for output.""" - if not variables: - return [] - s = ['vars = {'] - for key, tup in sorted(variables.items()): - hierarchy, value = tup - s.extend([ - ' # %s' % hierarchy, - ' "%s": %r,' % (key, value), - '', - ]) - s.extend(['}', '']) - return s + """Converts |variables| dict to list of lines for output.""" + if not variables: + return [] + s = ['vars = {'] + for key, tup in sorted(variables.items()): + hierarchy, value = tup + s.extend([ + ' # %s' % hierarchy, + ' "%s": %r,' % (key, value), + '', + ]) + s.extend(['}', '']) + return s @metrics.collector.collect_metrics('gclient grep') def CMDgrep(parser, args): - """Greps through git repos managed by gclient. + """Greps through git repos managed by gclient. Runs 'git grep [args...]' for each module. """ - # We can't use optparse because it will try to parse arguments sent - # to git grep and throw an error. :-( - if not args or re.match('(-h|--help)$', args[0]): - print( - 'Usage: gclient grep [-j ] git-grep-args...\n\n' - 'Example: "gclient grep -j10 -A2 RefCountedBase" runs\n"git grep ' - '-A2 RefCountedBase" on each of gclient\'s git\nrepos with up to ' - '10 jobs.\n\nBonus: page output by appending "|& less -FRSX" to the' - ' end of your query.', - file=sys.stderr) - return 1 + # We can't use optparse because it will try to parse arguments sent + # to git grep and throw an error. :-( + if not args or re.match('(-h|--help)$', args[0]): + print( + 'Usage: gclient grep [-j ] git-grep-args...\n\n' + 'Example: "gclient grep -j10 -A2 RefCountedBase" runs\n"git grep ' + '-A2 RefCountedBase" on each of gclient\'s git\nrepos with up to ' + '10 jobs.\n\nBonus: page output by appending "|& less -FRSX" to the' + ' end of your query.', + file=sys.stderr) + return 1 - jobs_arg = ['--jobs=1'] - if re.match(r'(-j|--jobs=)\d+$', args[0]): - jobs_arg, args = args[:1], args[1:] - elif re.match(r'(-j|--jobs)$', args[0]): - jobs_arg, args = args[:2], args[2:] + jobs_arg = ['--jobs=1'] + if re.match(r'(-j|--jobs=)\d+$', args[0]): + jobs_arg, args = args[:1], args[1:] + elif re.match(r'(-j|--jobs)$', args[0]): + jobs_arg, args = args[:2], args[2:] - return CMDrecurse( - parser, - jobs_arg + ['--ignore', '--prepend-dir', '--no-progress', '--scm=git', - 'git', 'grep', '--null', '--color=Always'] + args) + return CMDrecurse( + parser, jobs_arg + [ + '--ignore', '--prepend-dir', '--no-progress', '--scm=git', 'git', + 'grep', '--null', '--color=Always' + ] + args) @metrics.collector.collect_metrics('gclient root') def CMDroot(parser, args): - """Outputs the solution root (or current dir if there isn't one).""" - (options, args) = parser.parse_args(args) - client = GClient.LoadCurrentConfig(options) - if client: - print(os.path.abspath(client.root_dir)) - else: - print(os.path.abspath('.')) + """Outputs the solution root (or current dir if there isn't one).""" + (options, args) = parser.parse_args(args) + client = GClient.LoadCurrentConfig(options) + if client: + print(os.path.abspath(client.root_dir)) + else: + print(os.path.abspath('.')) @subcommand.usage('[url]') @metrics.collector.collect_metrics('gclient config') def CMDconfig(parser, args): - """Creates a .gclient file in the current directory. + """Creates a .gclient file in the current directory. This specifies the configuration for further commands. After update/sync, top-level DEPS files in each module are read to determine dependent @@ -2966,76 +3086,89 @@ def CMDconfig(parser, args): provided, then configuration is read from a specified Subversion server URL. """ - # We do a little dance with the --gclientfile option. 'gclient config' is the - # only command where it's acceptable to have both '--gclientfile' and '--spec' - # arguments. So, we temporarily stash any --gclientfile parameter into - # options.output_config_file until after the (gclientfile xor spec) error - # check. - parser.remove_option('--gclientfile') - parser.add_option('--gclientfile', dest='output_config_file', - help='Specify an alternate .gclient file') - parser.add_option('--name', - help='overrides the default name for the solution') - parser.add_option('--deps-file', default='DEPS', - help='overrides the default name for the DEPS file for the ' - 'main solutions and all sub-dependencies') - parser.add_option('--unmanaged', action='store_true', default=False, - help='overrides the default behavior to make it possible ' - 'to have the main solution untouched by gclient ' - '(gclient will check out unmanaged dependencies but ' - 'will never sync them)') - parser.add_option('--cache-dir', default=UNSET_CACHE_DIR, - help='Cache all git repos into this dir and do shared ' - 'clones from the cache, instead of cloning directly ' - 'from the remote. Pass "None" to disable cache, even ' - 'if globally enabled due to $GIT_CACHE_PATH.') - parser.add_option('--custom-var', action='append', dest='custom_vars', - default=[], - help='overrides variables; key=value syntax') - parser.set_defaults(config_filename=None) - (options, args) = parser.parse_args(args) - if options.output_config_file: - setattr(options, 'config_filename', getattr(options, 'output_config_file')) - if ((options.spec and args) or len(args) > 2 or - (not options.spec and not args)): - parser.error('Inconsistent arguments. Use either --spec or one or 2 args') + # We do a little dance with the --gclientfile option. 'gclient config' is + # the only command where it's acceptable to have both '--gclientfile' and + # '--spec' arguments. So, we temporarily stash any --gclientfile parameter + # into options.output_config_file until after the (gclientfile xor spec) + # error check. + parser.remove_option('--gclientfile') + parser.add_option('--gclientfile', + dest='output_config_file', + help='Specify an alternate .gclient file') + parser.add_option('--name', + help='overrides the default name for the solution') + parser.add_option( + '--deps-file', + default='DEPS', + help='overrides the default name for the DEPS file for the ' + 'main solutions and all sub-dependencies') + parser.add_option('--unmanaged', + action='store_true', + default=False, + help='overrides the default behavior to make it possible ' + 'to have the main solution untouched by gclient ' + '(gclient will check out unmanaged dependencies but ' + 'will never sync them)') + parser.add_option('--cache-dir', + default=UNSET_CACHE_DIR, + help='Cache all git repos into this dir and do shared ' + 'clones from the cache, instead of cloning directly ' + 'from the remote. Pass "None" to disable cache, even ' + 'if globally enabled due to $GIT_CACHE_PATH.') + parser.add_option('--custom-var', + action='append', + dest='custom_vars', + default=[], + help='overrides variables; key=value syntax') + parser.set_defaults(config_filename=None) + (options, args) = parser.parse_args(args) + if options.output_config_file: + setattr(options, 'config_filename', + getattr(options, 'output_config_file')) + if ((options.spec and args) or len(args) > 2 + or (not options.spec and not args)): + parser.error( + 'Inconsistent arguments. Use either --spec or one or 2 args') - if (options.cache_dir is not UNSET_CACHE_DIR - and options.cache_dir.lower() == 'none'): - options.cache_dir = None + if (options.cache_dir is not UNSET_CACHE_DIR + and options.cache_dir.lower() == 'none'): + options.cache_dir = None - custom_vars = {} - for arg in options.custom_vars: - kv = arg.split('=', 1) - if len(kv) != 2: - parser.error('Invalid --custom-var argument: %r' % arg) - custom_vars[kv[0]] = gclient_eval.EvaluateCondition(kv[1], {}) + custom_vars = {} + for arg in options.custom_vars: + kv = arg.split('=', 1) + if len(kv) != 2: + parser.error('Invalid --custom-var argument: %r' % arg) + custom_vars[kv[0]] = gclient_eval.EvaluateCondition(kv[1], {}) - client = GClient('.', options) - if options.spec: - client.SetConfig(options.spec) - else: - base_url = args[0].rstrip('/') - if not options.name: - name = base_url.split('/')[-1] - if name.endswith('.git'): - name = name[:-4] + client = GClient('.', options) + if options.spec: + client.SetConfig(options.spec) else: - # specify an alternate relpath for the given URL. - name = options.name - if not os.path.abspath(os.path.join(os.getcwd(), name)).startswith( - os.getcwd()): - parser.error('Do not pass a relative path for --name.') - if any(x in ('..', '.', '/', '\\') for x in name.split(os.sep)): - parser.error('Do not include relative path components in --name.') + base_url = args[0].rstrip('/') + if not options.name: + name = base_url.split('/')[-1] + if name.endswith('.git'): + name = name[:-4] + else: + # specify an alternate relpath for the given URL. + name = options.name + if not os.path.abspath(os.path.join(os.getcwd(), name)).startswith( + os.getcwd()): + parser.error('Do not pass a relative path for --name.') + if any(x in ('..', '.', '/', '\\') for x in name.split(os.sep)): + parser.error( + 'Do not include relative path components in --name.') - deps_file = options.deps_file - client.SetDefaultConfig(name, deps_file, base_url, - managed=not options.unmanaged, - cache_dir=options.cache_dir, - custom_vars=custom_vars) - client.SaveConfig() - return 0 + deps_file = options.deps_file + client.SetDefaultConfig(name, + deps_file, + base_url, + managed=not options.unmanaged, + cache_dir=options.cache_dir, + custom_vars=custom_vars) + client.SaveConfig() + return 0 @subcommand.epilog("""Example: @@ -3044,43 +3177,49 @@ def CMDconfig(parser, args): """) @metrics.collector.collect_metrics('gclient pack') def CMDpack(parser, args): - """Generates a patch which can be applied at the root of the tree. + """Generates a patch which can be applied at the root of the tree. Internally, runs 'git diff' on each checked out module and dependencies, and performs minimal postprocessing of the output. The resulting patch is printed to stdout and can be applied to a freshly checked out tree via 'patch -p0 < patchfile'. """ - parser.add_option('--deps', dest='deps_os', metavar='OS_LIST', - help='override deps for the specified (comma-separated) ' - 'platform(s); \'all\' will process all deps_os ' - 'references') - parser.remove_option('--jobs') - (options, args) = parser.parse_args(args) - # Force jobs to 1 so the stdout is not annotated with the thread ids - options.jobs = 1 - client = GClient.LoadCurrentConfig(options) - if not client: - raise gclient_utils.Error('client not configured; see \'gclient config\'') - if options.verbose: - client.PrintLocationAndContents() - return client.RunOnDeps('pack', args) + parser.add_option('--deps', + dest='deps_os', + metavar='OS_LIST', + help='override deps for the specified (comma-separated) ' + 'platform(s); \'all\' will process all deps_os ' + 'references') + parser.remove_option('--jobs') + (options, args) = parser.parse_args(args) + # Force jobs to 1 so the stdout is not annotated with the thread ids + options.jobs = 1 + client = GClient.LoadCurrentConfig(options) + if not client: + raise gclient_utils.Error( + 'client not configured; see \'gclient config\'') + if options.verbose: + client.PrintLocationAndContents() + return client.RunOnDeps('pack', args) @metrics.collector.collect_metrics('gclient status') def CMDstatus(parser, args): - """Shows modification status for every dependencies.""" - parser.add_option('--deps', dest='deps_os', metavar='OS_LIST', - help='override deps for the specified (comma-separated) ' - 'platform(s); \'all\' will process all deps_os ' - 'references') - (options, args) = parser.parse_args(args) - client = GClient.LoadCurrentConfig(options) - if not client: - raise gclient_utils.Error('client not configured; see \'gclient config\'') - if options.verbose: - client.PrintLocationAndContents() - return client.RunOnDeps('status', args) + """Shows modification status for every dependencies.""" + parser.add_option('--deps', + dest='deps_os', + metavar='OS_LIST', + help='override deps for the specified (comma-separated) ' + 'platform(s); \'all\' will process all deps_os ' + 'references') + (options, args) = parser.parse_args(args) + client = GClient.LoadCurrentConfig(options) + if not client: + raise gclient_utils.Error( + 'client not configured; see \'gclient config\'') + if options.verbose: + client.PrintLocationAndContents() + return client.RunOnDeps('status', args) @subcommand.epilog("""Examples: @@ -3110,147 +3249,195 @@ os_deps, etc.) """) @metrics.collector.collect_metrics('gclient sync') def CMDsync(parser, args): - """Checkout/update all modules.""" - parser.add_option('-f', '--force', action='store_true', - help='force update even for unchanged modules') - parser.add_option('-n', '--nohooks', action='store_true', - help='don\'t run hooks after the update is complete') - parser.add_option('-p', '--noprehooks', action='store_true', - help='don\'t run pre-DEPS hooks', default=False) - parser.add_option('-r', '--revision', action='append', - dest='revisions', metavar='REV', default=[], - help='Enforces git ref/hash for the solutions with the ' - 'format src@rev. The src@ part is optional and can be ' - 'skipped. You can also specify URLs instead of paths ' - 'and gclient will find the solution corresponding to ' - 'the given URL. If a path is also specified, the URL ' - 'takes precedence. -r can be used multiple times when ' - '.gclient has multiple solutions configured, and will ' - 'work even if the src@ part is skipped. Revision ' - 'numbers (e.g. 31000 or r31000) are not supported.') - parser.add_option('--patch-ref', action='append', - dest='patch_refs', metavar='GERRIT_REF', default=[], - help='Patches the given reference with the format ' - 'dep@target-ref:patch-ref. ' - 'For |dep|, you can specify URLs as well as paths, ' - 'with URLs taking preference. ' - '|patch-ref| will be applied to |dep|, rebased on top ' - 'of what |dep| was synced to, and a soft reset will ' - 'be done. Use --no-rebase-patch-ref and ' - '--no-reset-patch-ref to disable this behavior. ' - '|target-ref| is the target branch against which a ' - 'patch was created, it is used to determine which ' - 'commits from the |patch-ref| actually constitute a ' - 'patch.') - parser.add_option('-t', '--download-topics', action='store_true', - help='Downloads and patches locally changes from all open ' - 'Gerrit CLs that have the same topic as the changes ' - 'in the specified patch_refs. Only works if atleast ' - 'one --patch-ref is specified.') - parser.add_option('--with_branch_heads', action='store_true', - help='Clone git "branch_heads" refspecs in addition to ' - 'the default refspecs. This adds about 1/2GB to a ' - 'full checkout. (git only)') - parser.add_option('--with_tags', action='store_true', - help='Clone git tags in addition to the default refspecs.') - parser.add_option('-H', '--head', action='store_true', - help='DEPRECATED: only made sense with safesync urls.') - parser.add_option('-D', '--delete_unversioned_trees', action='store_true', - help='Deletes from the working copy any dependencies that ' - 'have been removed since the last sync, as long as ' - 'there are no local modifications. When used with ' - '--force, such dependencies are removed even if they ' - 'have local modifications. When used with --reset, ' - 'all untracked directories are removed from the ' - 'working copy, excluding those which are explicitly ' - 'ignored in the repository.') - parser.add_option('-R', '--reset', action='store_true', - help='resets any local changes before updating (git only)') - parser.add_option('-M', '--merge', action='store_true', - help='merge upstream changes instead of trying to ' - 'fast-forward or rebase') - parser.add_option('-A', '--auto_rebase', action='store_true', - help='Automatically rebase repositories against local ' - 'checkout during update (git only).') - parser.add_option('--deps', dest='deps_os', metavar='OS_LIST', - help='override deps for the specified (comma-separated) ' - 'platform(s); \'all\' will process all deps_os ' - 'references') - parser.add_option('--process-all-deps', action='store_true', - help='Check out all deps, even for different OS-es, ' - 'or with conditions evaluating to false') - parser.add_option('--upstream', action='store_true', - help='Make repo state match upstream branch.') - parser.add_option('--output-json', - help='Output a json document to this path containing ' - 'summary information about the sync.') - parser.add_option('--no-history', action='store_true', - help='GIT ONLY - Reduces the size/time of the checkout at ' - 'the cost of no history. Requires Git 1.9+') - parser.add_option('--shallow', action='store_true', - help='GIT ONLY - Do a shallow clone into the cache dir. ' - 'Requires Git 1.9+') - parser.add_option('--no_bootstrap', '--no-bootstrap', - action='store_true', - help='Don\'t bootstrap from Google Storage.') - parser.add_option('--ignore_locks', - action='store_true', - help='No longer used.') - parser.add_option('--break_repo_locks', - action='store_true', - help='No longer used.') - parser.add_option('--lock_timeout', type='int', default=5000, - help='GIT ONLY - Deadline (in seconds) to wait for git ' - 'cache lock to become available. Default is %default.') - parser.add_option('--no-rebase-patch-ref', action='store_false', - dest='rebase_patch_ref', default=True, - help='Bypass rebase of the patch ref after checkout.') - parser.add_option('--no-reset-patch-ref', action='store_false', - dest='reset_patch_ref', default=True, - help='Bypass calling reset after patching the ref.') - parser.add_option('--experiment', - action='append', - dest='experiments', - default=[], - help='Which experiments should be enabled.') - (options, args) = parser.parse_args(args) - client = GClient.LoadCurrentConfig(options) + """Checkout/update all modules.""" + parser.add_option('-f', + '--force', + action='store_true', + help='force update even for unchanged modules') + parser.add_option('-n', + '--nohooks', + action='store_true', + help='don\'t run hooks after the update is complete') + parser.add_option('-p', + '--noprehooks', + action='store_true', + help='don\'t run pre-DEPS hooks', + default=False) + parser.add_option('-r', + '--revision', + action='append', + dest='revisions', + metavar='REV', + default=[], + help='Enforces git ref/hash for the solutions with the ' + 'format src@rev. The src@ part is optional and can be ' + 'skipped. You can also specify URLs instead of paths ' + 'and gclient will find the solution corresponding to ' + 'the given URL. If a path is also specified, the URL ' + 'takes precedence. -r can be used multiple times when ' + '.gclient has multiple solutions configured, and will ' + 'work even if the src@ part is skipped. Revision ' + 'numbers (e.g. 31000 or r31000) are not supported.') + parser.add_option('--patch-ref', + action='append', + dest='patch_refs', + metavar='GERRIT_REF', + default=[], + help='Patches the given reference with the format ' + 'dep@target-ref:patch-ref. ' + 'For |dep|, you can specify URLs as well as paths, ' + 'with URLs taking preference. ' + '|patch-ref| will be applied to |dep|, rebased on top ' + 'of what |dep| was synced to, and a soft reset will ' + 'be done. Use --no-rebase-patch-ref and ' + '--no-reset-patch-ref to disable this behavior. ' + '|target-ref| is the target branch against which a ' + 'patch was created, it is used to determine which ' + 'commits from the |patch-ref| actually constitute a ' + 'patch.') + parser.add_option( + '-t', + '--download-topics', + action='store_true', + help='Downloads and patches locally changes from all open ' + 'Gerrit CLs that have the same topic as the changes ' + 'in the specified patch_refs. Only works if atleast ' + 'one --patch-ref is specified.') + parser.add_option('--with_branch_heads', + action='store_true', + help='Clone git "branch_heads" refspecs in addition to ' + 'the default refspecs. This adds about 1/2GB to a ' + 'full checkout. (git only)') + parser.add_option( + '--with_tags', + action='store_true', + help='Clone git tags in addition to the default refspecs.') + parser.add_option('-H', + '--head', + action='store_true', + help='DEPRECATED: only made sense with safesync urls.') + parser.add_option( + '-D', + '--delete_unversioned_trees', + action='store_true', + help='Deletes from the working copy any dependencies that ' + 'have been removed since the last sync, as long as ' + 'there are no local modifications. When used with ' + '--force, such dependencies are removed even if they ' + 'have local modifications. When used with --reset, ' + 'all untracked directories are removed from the ' + 'working copy, excluding those which are explicitly ' + 'ignored in the repository.') + parser.add_option( + '-R', + '--reset', + action='store_true', + help='resets any local changes before updating (git only)') + parser.add_option('-M', + '--merge', + action='store_true', + help='merge upstream changes instead of trying to ' + 'fast-forward or rebase') + parser.add_option('-A', + '--auto_rebase', + action='store_true', + help='Automatically rebase repositories against local ' + 'checkout during update (git only).') + parser.add_option('--deps', + dest='deps_os', + metavar='OS_LIST', + help='override deps for the specified (comma-separated) ' + 'platform(s); \'all\' will process all deps_os ' + 'references') + parser.add_option('--process-all-deps', + action='store_true', + help='Check out all deps, even for different OS-es, ' + 'or with conditions evaluating to false') + parser.add_option('--upstream', + action='store_true', + help='Make repo state match upstream branch.') + parser.add_option('--output-json', + help='Output a json document to this path containing ' + 'summary information about the sync.') + parser.add_option( + '--no-history', + action='store_true', + help='GIT ONLY - Reduces the size/time of the checkout at ' + 'the cost of no history. Requires Git 1.9+') + parser.add_option('--shallow', + action='store_true', + help='GIT ONLY - Do a shallow clone into the cache dir. ' + 'Requires Git 1.9+') + parser.add_option('--no_bootstrap', + '--no-bootstrap', + action='store_true', + help='Don\'t bootstrap from Google Storage.') + parser.add_option('--ignore_locks', + action='store_true', + help='No longer used.') + parser.add_option('--break_repo_locks', + action='store_true', + help='No longer used.') + parser.add_option('--lock_timeout', + type='int', + default=5000, + help='GIT ONLY - Deadline (in seconds) to wait for git ' + 'cache lock to become available. Default is %default.') + parser.add_option('--no-rebase-patch-ref', + action='store_false', + dest='rebase_patch_ref', + default=True, + help='Bypass rebase of the patch ref after checkout.') + parser.add_option('--no-reset-patch-ref', + action='store_false', + dest='reset_patch_ref', + default=True, + help='Bypass calling reset after patching the ref.') + parser.add_option('--experiment', + action='append', + dest='experiments', + default=[], + help='Which experiments should be enabled.') + (options, args) = parser.parse_args(args) + client = GClient.LoadCurrentConfig(options) - if not client: - raise gclient_utils.Error('client not configured; see \'gclient config\'') + if not client: + raise gclient_utils.Error( + 'client not configured; see \'gclient config\'') - if options.download_topics and not options.rebase_patch_ref: - raise gclient_utils.Error( - 'Warning: You cannot download topics and not rebase each patch ref') + if options.download_topics and not options.rebase_patch_ref: + raise gclient_utils.Error( + 'Warning: You cannot download topics and not rebase each patch ref') - if options.ignore_locks: - print('Warning: ignore_locks is no longer used. Please remove its usage.') + if options.ignore_locks: + print( + 'Warning: ignore_locks is no longer used. Please remove its usage.') - if options.break_repo_locks: - print('Warning: break_repo_locks is no longer used. Please remove its ' - 'usage.') + if options.break_repo_locks: + print('Warning: break_repo_locks is no longer used. Please remove its ' + 'usage.') - if options.revisions and options.head: - # TODO(maruel): Make it a parser.error if it doesn't break any builder. - print('Warning: you cannot use both --head and --revision') + if options.revisions and options.head: + # TODO(maruel): Make it a parser.error if it doesn't break any builder. + print('Warning: you cannot use both --head and --revision') - if options.verbose: - client.PrintLocationAndContents() - ret = client.RunOnDeps('update', args) - if options.output_json: - slns = {} - for d in client.subtree(True): - normed = d.name.replace('\\', '/').rstrip('/') + '/' - slns[normed] = { - 'revision': d.got_revision, - 'scm': d.used_scm.name if d.used_scm else None, - 'url': str(d.url) if d.url else None, - 'was_processed': d.should_process, - 'was_synced': d._should_sync, - } - with open(options.output_json, 'w') as f: - json.dump({'solutions': slns}, f) - return ret + if options.verbose: + client.PrintLocationAndContents() + ret = client.RunOnDeps('update', args) + if options.output_json: + slns = {} + for d in client.subtree(True): + normed = d.name.replace('\\', '/').rstrip('/') + '/' + slns[normed] = { + 'revision': d.got_revision, + 'scm': d.used_scm.name if d.used_scm else None, + 'url': str(d.url) if d.url else None, + 'was_processed': d.should_process, + 'was_synced': d._should_sync, + } + with open(options.output_json, 'w') as f: + json.dump({'solutions': slns}, f) + return ret CMDupdate = CMDsync @@ -3258,515 +3445,590 @@ CMDupdate = CMDsync @metrics.collector.collect_metrics('gclient validate') def CMDvalidate(parser, args): - """Validates the .gclient and DEPS syntax.""" - options, args = parser.parse_args(args) - client = GClient.LoadCurrentConfig(options) - if not client: - raise gclient_utils.Error('client not configured; see \'gclient config\'') - rv = client.RunOnDeps('validate', args) - if rv == 0: - print('validate: SUCCESS') - else: - print('validate: FAILURE') - return rv + """Validates the .gclient and DEPS syntax.""" + options, args = parser.parse_args(args) + client = GClient.LoadCurrentConfig(options) + if not client: + raise gclient_utils.Error( + 'client not configured; see \'gclient config\'') + rv = client.RunOnDeps('validate', args) + if rv == 0: + print('validate: SUCCESS') + else: + print('validate: FAILURE') + return rv @metrics.collector.collect_metrics('gclient diff') def CMDdiff(parser, args): - """Displays local diff for every dependencies.""" - parser.add_option('--deps', dest='deps_os', metavar='OS_LIST', - help='override deps for the specified (comma-separated) ' - 'platform(s); \'all\' will process all deps_os ' - 'references') - (options, args) = parser.parse_args(args) - client = GClient.LoadCurrentConfig(options) - if not client: - raise gclient_utils.Error('client not configured; see \'gclient config\'') - if options.verbose: - client.PrintLocationAndContents() - return client.RunOnDeps('diff', args) + """Displays local diff for every dependencies.""" + parser.add_option('--deps', + dest='deps_os', + metavar='OS_LIST', + help='override deps for the specified (comma-separated) ' + 'platform(s); \'all\' will process all deps_os ' + 'references') + (options, args) = parser.parse_args(args) + client = GClient.LoadCurrentConfig(options) + if not client: + raise gclient_utils.Error( + 'client not configured; see \'gclient config\'') + if options.verbose: + client.PrintLocationAndContents() + return client.RunOnDeps('diff', args) @metrics.collector.collect_metrics('gclient revert') def CMDrevert(parser, args): - """Reverts all modifications in every dependencies. + """Reverts all modifications in every dependencies. That's the nuclear option to get back to a 'clean' state. It removes anything that shows up in git status.""" - parser.add_option('--deps', dest='deps_os', metavar='OS_LIST', - help='override deps for the specified (comma-separated) ' - 'platform(s); \'all\' will process all deps_os ' - 'references') - parser.add_option('-n', '--nohooks', action='store_true', - help='don\'t run hooks after the revert is complete') - parser.add_option('-p', '--noprehooks', action='store_true', - help='don\'t run pre-DEPS hooks', default=False) - parser.add_option('--upstream', action='store_true', - help='Make repo state match upstream branch.') - parser.add_option('--break_repo_locks', - action='store_true', - help='No longer used.') - (options, args) = parser.parse_args(args) - if options.break_repo_locks: - print('Warning: break_repo_locks is no longer used. Please remove its ' + - 'usage.') + parser.add_option('--deps', + dest='deps_os', + metavar='OS_LIST', + help='override deps for the specified (comma-separated) ' + 'platform(s); \'all\' will process all deps_os ' + 'references') + parser.add_option('-n', + '--nohooks', + action='store_true', + help='don\'t run hooks after the revert is complete') + parser.add_option('-p', + '--noprehooks', + action='store_true', + help='don\'t run pre-DEPS hooks', + default=False) + parser.add_option('--upstream', + action='store_true', + help='Make repo state match upstream branch.') + parser.add_option('--break_repo_locks', + action='store_true', + help='No longer used.') + (options, args) = parser.parse_args(args) + if options.break_repo_locks: + print( + 'Warning: break_repo_locks is no longer used. Please remove its ' + + 'usage.') - # --force is implied. - options.force = True - options.reset = False - options.delete_unversioned_trees = False - options.merge = False - client = GClient.LoadCurrentConfig(options) - if not client: - raise gclient_utils.Error('client not configured; see \'gclient config\'') - return client.RunOnDeps('revert', args) + # --force is implied. + options.force = True + options.reset = False + options.delete_unversioned_trees = False + options.merge = False + client = GClient.LoadCurrentConfig(options) + if not client: + raise gclient_utils.Error( + 'client not configured; see \'gclient config\'') + return client.RunOnDeps('revert', args) @metrics.collector.collect_metrics('gclient runhooks') def CMDrunhooks(parser, args): - """Runs hooks for files that have been modified in the local working copy.""" - parser.add_option('--deps', dest='deps_os', metavar='OS_LIST', - help='override deps for the specified (comma-separated) ' - 'platform(s); \'all\' will process all deps_os ' - 'references') - parser.add_option('-f', '--force', action='store_true', default=True, - help='Deprecated. No effect.') - (options, args) = parser.parse_args(args) - client = GClient.LoadCurrentConfig(options) - if not client: - raise gclient_utils.Error('client not configured; see \'gclient config\'') - if options.verbose: - client.PrintLocationAndContents() - options.force = True - options.nohooks = False - return client.RunOnDeps('runhooks', args) + """Runs hooks for files that have been modified in the local working copy.""" + parser.add_option('--deps', + dest='deps_os', + metavar='OS_LIST', + help='override deps for the specified (comma-separated) ' + 'platform(s); \'all\' will process all deps_os ' + 'references') + parser.add_option('-f', + '--force', + action='store_true', + default=True, + help='Deprecated. No effect.') + (options, args) = parser.parse_args(args) + client = GClient.LoadCurrentConfig(options) + if not client: + raise gclient_utils.Error( + 'client not configured; see \'gclient config\'') + if options.verbose: + client.PrintLocationAndContents() + options.force = True + options.nohooks = False + return client.RunOnDeps('runhooks', args) @metrics.collector.collect_metrics('gclient revinfo') def CMDrevinfo(parser, args): - """Outputs revision info mapping for the client and its dependencies. + """Outputs revision info mapping for the client and its dependencies. This allows the capture of an overall 'revision' for the source tree that can be used to reproduce the same tree in the future. It is only useful for 'unpinned dependencies', i.e. DEPS/deps references without a git hash. A git branch name isn't 'pinned' since the actual commit can change. """ - parser.add_option('--deps', dest='deps_os', metavar='OS_LIST', - help='override deps for the specified (comma-separated) ' - 'platform(s); \'all\' will process all deps_os ' - 'references') - parser.add_option('-a', '--actual', action='store_true', - help='gets the actual checked out revisions instead of the ' - 'ones specified in the DEPS and .gclient files') - parser.add_option('-s', '--snapshot', action='store_true', - help='creates a snapshot .gclient file of the current ' - 'version of all repositories to reproduce the tree, ' - 'implies -a') - parser.add_option('--filter', action='append', dest='filter', - help='Display revision information only for the specified ' - 'dependencies (filtered by URL or path).') - parser.add_option('--output-json', - help='Output a json document to this path containing ' - 'information about the revisions.') - parser.add_option('--ignore-dep-type', choices=['git', 'cipd'], - help='Specify to skip processing of a certain type of dep.') - (options, args) = parser.parse_args(args) - client = GClient.LoadCurrentConfig(options) - if not client: - raise gclient_utils.Error('client not configured; see \'gclient config\'') - client.PrintRevInfo() - return 0 + parser.add_option('--deps', + dest='deps_os', + metavar='OS_LIST', + help='override deps for the specified (comma-separated) ' + 'platform(s); \'all\' will process all deps_os ' + 'references') + parser.add_option( + '-a', + '--actual', + action='store_true', + help='gets the actual checked out revisions instead of the ' + 'ones specified in the DEPS and .gclient files') + parser.add_option('-s', + '--snapshot', + action='store_true', + help='creates a snapshot .gclient file of the current ' + 'version of all repositories to reproduce the tree, ' + 'implies -a') + parser.add_option( + '--filter', + action='append', + dest='filter', + help='Display revision information only for the specified ' + 'dependencies (filtered by URL or path).') + parser.add_option('--output-json', + help='Output a json document to this path containing ' + 'information about the revisions.') + parser.add_option( + '--ignore-dep-type', + choices=['git', 'cipd'], + help='Specify to skip processing of a certain type of dep.') + (options, args) = parser.parse_args(args) + client = GClient.LoadCurrentConfig(options) + if not client: + raise gclient_utils.Error( + 'client not configured; see \'gclient config\'') + client.PrintRevInfo() + return 0 @metrics.collector.collect_metrics('gclient getdep') def CMDgetdep(parser, args): - """Gets revision information and variable values from a DEPS file. + """Gets revision information and variable values from a DEPS file. If key doesn't exist or is incorrectly declared, this script exits with exit code 2.""" - parser.add_option('--var', action='append', - dest='vars', metavar='VAR', default=[], - help='Gets the value of a given variable.') - parser.add_option('-r', '--revision', action='append', - dest='getdep_revisions', metavar='DEP', default=[], - help='Gets the revision/version for the given dependency. ' - 'If it is a git dependency, dep must be a path. If it ' - 'is a CIPD dependency, dep must be of the form ' - 'path:package.') - parser.add_option('--deps-file', default='DEPS', - # TODO(ehmaldonado): Try to find the DEPS file pointed by - # .gclient first. - help='The DEPS file to be edited. Defaults to the DEPS ' - 'file in the current directory.') - (options, args) = parser.parse_args(args) + parser.add_option('--var', + action='append', + dest='vars', + metavar='VAR', + default=[], + help='Gets the value of a given variable.') + parser.add_option( + '-r', + '--revision', + action='append', + dest='getdep_revisions', + metavar='DEP', + default=[], + help='Gets the revision/version for the given dependency. ' + 'If it is a git dependency, dep must be a path. If it ' + 'is a CIPD dependency, dep must be of the form ' + 'path:package.') + parser.add_option( + '--deps-file', + default='DEPS', + # TODO(ehmaldonado): Try to find the DEPS file pointed by + # .gclient first. + help='The DEPS file to be edited. Defaults to the DEPS ' + 'file in the current directory.') + (options, args) = parser.parse_args(args) - if not os.path.isfile(options.deps_file): - raise gclient_utils.Error( - 'DEPS file %s does not exist.' % options.deps_file) - with open(options.deps_file) as f: - contents = f.read() - client = GClient.LoadCurrentConfig(options) - if client is not None: - builtin_vars = client.get_builtin_vars() - else: - logging.warning( - 'Couldn\'t find a valid gclient config. Will attempt to parse the DEPS ' - 'file without support for built-in variables.') - builtin_vars = None - local_scope = gclient_eval.Exec(contents, options.deps_file, - builtin_vars=builtin_vars) - - for var in options.vars: - print(gclient_eval.GetVar(local_scope, var)) - - commits = {} - if local_scope.get('git_dependencies' - ) == gclient_eval.SUBMODULES and options.getdep_revisions: - commits.update( - scm_git.GIT.GetSubmoduleCommits( - os.getcwd(), - [path for path in options.getdep_revisions if ':' not in path])) - - for name in options.getdep_revisions: - if ':' in name: - name, _, package = name.partition(':') - if not name or not package: - parser.error( - 'Wrong CIPD format: %s:%s should be of the form path:pkg.' - % (name, package)) - print(gclient_eval.GetCIPD(local_scope, name, package)) - elif commits: - print(commits[name]) + if not os.path.isfile(options.deps_file): + raise gclient_utils.Error('DEPS file %s does not exist.' % + options.deps_file) + with open(options.deps_file) as f: + contents = f.read() + client = GClient.LoadCurrentConfig(options) + if client is not None: + builtin_vars = client.get_builtin_vars() else: - try: - print(gclient_eval.GetRevision(local_scope, name)) - except KeyError as e: - print(repr(e), file=sys.stderr) - sys.exit(2) + logging.warning( + 'Couldn\'t find a valid gclient config. Will attempt to parse the DEPS ' + 'file without support for built-in variables.') + builtin_vars = None + local_scope = gclient_eval.Exec(contents, + options.deps_file, + builtin_vars=builtin_vars) + + for var in options.vars: + print(gclient_eval.GetVar(local_scope, var)) + + commits = {} + if local_scope.get( + 'git_dependencies' + ) == gclient_eval.SUBMODULES and options.getdep_revisions: + commits.update( + scm_git.GIT.GetSubmoduleCommits( + os.getcwd(), + [path for path in options.getdep_revisions if ':' not in path])) + + for name in options.getdep_revisions: + if ':' in name: + name, _, package = name.partition(':') + if not name or not package: + parser.error( + 'Wrong CIPD format: %s:%s should be of the form path:pkg.' % + (name, package)) + print(gclient_eval.GetCIPD(local_scope, name, package)) + elif commits: + print(commits[name]) + else: + try: + print(gclient_eval.GetRevision(local_scope, name)) + except KeyError as e: + print(repr(e), file=sys.stderr) + sys.exit(2) @metrics.collector.collect_metrics('gclient setdep') def CMDsetdep(parser, args): - """Modifies dependency revisions and variable values in a DEPS file""" - parser.add_option('--var', action='append', - dest='vars', metavar='VAR=VAL', default=[], - help='Sets a variable to the given value with the format ' - 'name=value.') - parser.add_option('-r', '--revision', action='append', - dest='setdep_revisions', metavar='DEP@REV', default=[], - help='Sets the revision/version for the dependency with ' - 'the format dep@rev. If it is a git dependency, dep ' - 'must be a path and rev must be a git hash or ' - 'reference (e.g. src/dep@deadbeef). If it is a CIPD ' - 'dependency, dep must be of the form path:package and ' - 'rev must be the package version ' - '(e.g. src/pkg:chromium/pkg@2.1-cr0).') - parser.add_option('--deps-file', default='DEPS', - # TODO(ehmaldonado): Try to find the DEPS file pointed by - # .gclient first. - help='The DEPS file to be edited. Defaults to the DEPS ' - 'file in the current directory.') - (options, args) = parser.parse_args(args) - if args: - parser.error('Unused arguments: "%s"' % '" "'.join(args)) - if not options.setdep_revisions and not options.vars: - parser.error( - 'You must specify at least one variable or revision to modify.') - - if not os.path.isfile(options.deps_file): - raise gclient_utils.Error( - 'DEPS file %s does not exist.' % options.deps_file) - with open(options.deps_file) as f: - contents = f.read() - - client = GClient.LoadCurrentConfig(options) - if client is not None: - builtin_vars = client.get_builtin_vars() - else: - logging.warning( - 'Couldn\'t find a valid gclient config. Will attempt to parse the DEPS ' - 'file without support for built-in variables.') - builtin_vars = None - - local_scope = gclient_eval.Exec(contents, options.deps_file, - builtin_vars=builtin_vars) - - # Create a set of all git submodules. - cwd = os.path.dirname(options.deps_file) or os.getcwd() - git_modules = None - if 'git_dependencies' in local_scope and local_scope['git_dependencies'] in ( - gclient_eval.SUBMODULES, gclient_eval.SYNC): - try: - submodule_status = subprocess2.check_output( - ['git', 'submodule', 'status'], cwd=cwd).decode('utf-8') - git_modules = {l.split()[1] for l in submodule_status.splitlines()} - except subprocess2.CalledProcessError as e: - print('Warning: gitlinks won\'t be updated: ', e) - - for var in options.vars: - name, _, value = var.partition('=') - if not name or not value: - parser.error( - 'Wrong var format: %s should be of the form name=value.' % var) - if name in local_scope['vars']: - gclient_eval.SetVar(local_scope, name, value) - else: - gclient_eval.AddVar(local_scope, name, value) - - for revision in options.setdep_revisions: - name, _, value = revision.partition('@') - if not name or not value: - parser.error( - 'Wrong dep format: %s should be of the form dep@rev.' % revision) - if ':' in name: - name, _, package = name.partition(':') - if not name or not package: + """Modifies dependency revisions and variable values in a DEPS file""" + parser.add_option('--var', + action='append', + dest='vars', + metavar='VAR=VAL', + default=[], + help='Sets a variable to the given value with the format ' + 'name=value.') + parser.add_option('-r', + '--revision', + action='append', + dest='setdep_revisions', + metavar='DEP@REV', + default=[], + help='Sets the revision/version for the dependency with ' + 'the format dep@rev. If it is a git dependency, dep ' + 'must be a path and rev must be a git hash or ' + 'reference (e.g. src/dep@deadbeef). If it is a CIPD ' + 'dependency, dep must be of the form path:package and ' + 'rev must be the package version ' + '(e.g. src/pkg:chromium/pkg@2.1-cr0).') + parser.add_option( + '--deps-file', + default='DEPS', + # TODO(ehmaldonado): Try to find the DEPS file pointed by + # .gclient first. + help='The DEPS file to be edited. Defaults to the DEPS ' + 'file in the current directory.') + (options, args) = parser.parse_args(args) + if args: + parser.error('Unused arguments: "%s"' % '" "'.join(args)) + if not options.setdep_revisions and not options.vars: parser.error( - 'Wrong CIPD format: %s:%s should be of the form path:pkg@version.' - % (name, package)) - gclient_eval.SetCIPD(local_scope, name, package, value) + 'You must specify at least one variable or revision to modify.') + + if not os.path.isfile(options.deps_file): + raise gclient_utils.Error('DEPS file %s does not exist.' % + options.deps_file) + with open(options.deps_file) as f: + contents = f.read() + + client = GClient.LoadCurrentConfig(options) + if client is not None: + builtin_vars = client.get_builtin_vars() else: - # Update DEPS only when `git_dependencies` == DEPS or SYNC. - # git_dependencies is defaulted to DEPS when not set. - if 'git_dependencies' not in local_scope or local_scope[ - 'git_dependencies'] in (gclient_eval.DEPS, gclient_eval.SYNC): - gclient_eval.SetRevision(local_scope, name, value) + logging.warning( + 'Couldn\'t find a valid gclient config. Will attempt to parse the DEPS ' + 'file without support for built-in variables.') + builtin_vars = None - # Update git submodules when `git_dependencies` == SYNC or SUBMODULES. - if git_modules and 'git_dependencies' in local_scope and local_scope[ - 'git_dependencies'] in (gclient_eval.SUBMODULES, gclient_eval.SYNC): - git_module_name = name - if not 'use_relative_paths' in local_scope or \ - local_scope['use_relative_paths'] != True: - deps_dir = os.path.dirname(os.path.abspath(options.deps_file)) - gclient_path = gclient_paths.FindGclientRoot(deps_dir) - delta_path = None - if gclient_path: - delta_path = os.path.relpath(deps_dir, - os.path.abspath(gclient_path)) - if delta_path: - prefix_length = len(delta_path.replace(os.path.sep, '/')) + 1 - git_module_name = name[prefix_length:] - # gclient setdep should update the revision, i.e., the gitlink only - # when the submodule entry is already present within .gitmodules. - if git_module_name not in git_modules: - raise KeyError( - f'Could not find any dependency called "{git_module_name}" in ' - f'.gitmodules.') + local_scope = gclient_eval.Exec(contents, + options.deps_file, + builtin_vars=builtin_vars) - # Update the gitlink for the submodule. - subprocess2.call([ - 'git', 'update-index', '--add', '--cacheinfo', - f'160000,{value},{git_module_name}' - ], - cwd=cwd) + # Create a set of all git submodules. + cwd = os.path.dirname(options.deps_file) or os.getcwd() + git_modules = None + if 'git_dependencies' in local_scope and local_scope[ + 'git_dependencies'] in (gclient_eval.SUBMODULES, gclient_eval.SYNC): + try: + submodule_status = subprocess2.check_output( + ['git', 'submodule', 'status'], cwd=cwd).decode('utf-8') + git_modules = {l.split()[1] for l in submodule_status.splitlines()} + except subprocess2.CalledProcessError as e: + print('Warning: gitlinks won\'t be updated: ', e) - with open(options.deps_file, 'wb') as f: - f.write(gclient_eval.RenderDEPSFile(local_scope).encode('utf-8')) + for var in options.vars: + name, _, value = var.partition('=') + if not name or not value: + parser.error( + 'Wrong var format: %s should be of the form name=value.' % var) + if name in local_scope['vars']: + gclient_eval.SetVar(local_scope, name, value) + else: + gclient_eval.AddVar(local_scope, name, value) - if git_modules: - subprocess2.call(['git', 'add', options.deps_file], cwd=cwd) - print('Changes have been staged. See changes with `git status`.\n' - 'Use `git commit -m "Manual roll"` to commit your changes. \n' - 'Run gclient sync to update your local dependency checkout.') + for revision in options.setdep_revisions: + name, _, value = revision.partition('@') + if not name or not value: + parser.error('Wrong dep format: %s should be of the form dep@rev.' % + revision) + if ':' in name: + name, _, package = name.partition(':') + if not name or not package: + parser.error( + 'Wrong CIPD format: %s:%s should be of the form path:pkg@version.' + % (name, package)) + gclient_eval.SetCIPD(local_scope, name, package, value) + else: + # Update DEPS only when `git_dependencies` == DEPS or SYNC. + # git_dependencies is defaulted to DEPS when not set. + if 'git_dependencies' not in local_scope or local_scope[ + 'git_dependencies'] in (gclient_eval.DEPS, + gclient_eval.SYNC): + gclient_eval.SetRevision(local_scope, name, value) + + # Update git submodules when `git_dependencies` == SYNC or + # SUBMODULES. + if git_modules and 'git_dependencies' in local_scope and local_scope[ + 'git_dependencies'] in (gclient_eval.SUBMODULES, + gclient_eval.SYNC): + git_module_name = name + if not 'use_relative_paths' in local_scope or \ + local_scope['use_relative_paths'] != True: + deps_dir = os.path.dirname( + os.path.abspath(options.deps_file)) + gclient_path = gclient_paths.FindGclientRoot(deps_dir) + delta_path = None + if gclient_path: + delta_path = os.path.relpath( + deps_dir, os.path.abspath(gclient_path)) + if delta_path: + prefix_length = len(delta_path.replace( + os.path.sep, '/')) + 1 + git_module_name = name[prefix_length:] + # gclient setdep should update the revision, i.e., the gitlink + # only when the submodule entry is already present within + # .gitmodules. + if git_module_name not in git_modules: + raise KeyError( + f'Could not find any dependency called "{git_module_name}" in ' + f'.gitmodules.') + + # Update the gitlink for the submodule. + subprocess2.call([ + 'git', 'update-index', '--add', '--cacheinfo', + f'160000,{value},{git_module_name}' + ], + cwd=cwd) + + with open(options.deps_file, 'wb') as f: + f.write(gclient_eval.RenderDEPSFile(local_scope).encode('utf-8')) + + if git_modules: + subprocess2.call(['git', 'add', options.deps_file], cwd=cwd) + print('Changes have been staged. See changes with `git status`.\n' + 'Use `git commit -m "Manual roll"` to commit your changes. \n' + 'Run gclient sync to update your local dependency checkout.') @metrics.collector.collect_metrics('gclient verify') def CMDverify(parser, args): - """Verifies the DEPS file deps are only from allowed_hosts.""" - (options, args) = parser.parse_args(args) - client = GClient.LoadCurrentConfig(options) - if not client: - raise gclient_utils.Error('client not configured; see \'gclient config\'') - client.RunOnDeps(None, []) - # Look at each first-level dependency of this gclient only. - for dep in client.dependencies: - bad_deps = dep.findDepsFromNotAllowedHosts() - if not bad_deps: - continue - print("There are deps from not allowed hosts in file %s" % dep.deps_file) - for bad_dep in bad_deps: - print("\t%s at %s" % (bad_dep.name, bad_dep.url)) - print("allowed_hosts:", ', '.join(dep.allowed_hosts)) - sys.stdout.flush() - raise gclient_utils.Error( - 'dependencies from disallowed hosts; check your DEPS file.') - return 0 + """Verifies the DEPS file deps are only from allowed_hosts.""" + (options, args) = parser.parse_args(args) + client = GClient.LoadCurrentConfig(options) + if not client: + raise gclient_utils.Error( + 'client not configured; see \'gclient config\'') + client.RunOnDeps(None, []) + # Look at each first-level dependency of this gclient only. + for dep in client.dependencies: + bad_deps = dep.findDepsFromNotAllowedHosts() + if not bad_deps: + continue + print("There are deps from not allowed hosts in file %s" % + dep.deps_file) + for bad_dep in bad_deps: + print("\t%s at %s" % (bad_dep.name, bad_dep.url)) + print("allowed_hosts:", ', '.join(dep.allowed_hosts)) + sys.stdout.flush() + raise gclient_utils.Error( + 'dependencies from disallowed hosts; check your DEPS file.') + return 0 @subcommand.epilog("""For more information on what metrics are we collecting and why, please read metrics.README.md or visit https://bit.ly/2ufRS4p""") @metrics.collector.collect_metrics('gclient metrics') def CMDmetrics(parser, args): - """Reports, and optionally modifies, the status of metric collection.""" - parser.add_option('--opt-in', action='store_true', dest='enable_metrics', - help='Opt-in to metrics collection.', - default=None) - parser.add_option('--opt-out', action='store_false', dest='enable_metrics', - help='Opt-out of metrics collection.') - options, args = parser.parse_args(args) - if args: - parser.error('Unused arguments: "%s"' % '" "'.join(args)) - if not metrics.collector.config.is_googler: - print("You're not a Googler. Metrics collection is disabled for you.") + """Reports, and optionally modifies, the status of metric collection.""" + parser.add_option('--opt-in', + action='store_true', + dest='enable_metrics', + help='Opt-in to metrics collection.', + default=None) + parser.add_option('--opt-out', + action='store_false', + dest='enable_metrics', + help='Opt-out of metrics collection.') + options, args = parser.parse_args(args) + if args: + parser.error('Unused arguments: "%s"' % '" "'.join(args)) + if not metrics.collector.config.is_googler: + print("You're not a Googler. Metrics collection is disabled for you.") + return 0 + + if options.enable_metrics is not None: + metrics.collector.config.opted_in = options.enable_metrics + + if metrics.collector.config.opted_in is None: + print("You haven't opted in or out of metrics collection.") + elif metrics.collector.config.opted_in: + print("You have opted in. Thanks!") + else: + print("You have opted out. Please consider opting in.") return 0 - if options.enable_metrics is not None: - metrics.collector.config.opted_in = options.enable_metrics - - if metrics.collector.config.opted_in is None: - print("You haven't opted in or out of metrics collection.") - elif metrics.collector.config.opted_in: - print("You have opted in. Thanks!") - else: - print("You have opted out. Please consider opting in.") - return 0 - class OptionParser(optparse.OptionParser): - gclientfile_default = os.environ.get('GCLIENT_FILE', '.gclient') + gclientfile_default = os.environ.get('GCLIENT_FILE', '.gclient') - def __init__(self, **kwargs): - optparse.OptionParser.__init__( - self, version='%prog ' + __version__, **kwargs) + def __init__(self, **kwargs): + optparse.OptionParser.__init__(self, + version='%prog ' + __version__, + **kwargs) - # Some arm boards have issues with parallel sync. - if platform.machine().startswith('arm'): - jobs = 1 - else: - jobs = max(8, gclient_utils.NumLocalCpus()) + # Some arm boards have issues with parallel sync. + if platform.machine().startswith('arm'): + jobs = 1 + else: + jobs = max(8, gclient_utils.NumLocalCpus()) - self.add_option( - '-j', '--jobs', default=jobs, type='int', - help='Specify how many SCM commands can run in parallel; defaults to ' - '%default on this machine') - self.add_option( - '-v', '--verbose', action='count', default=0, - help='Produces additional output for diagnostics. Can be used up to ' - 'three times for more logging info.') - self.add_option( - '--gclientfile', dest='config_filename', - help='Specify an alternate %s file' % self.gclientfile_default) - self.add_option( - '--spec', - help='create a gclient file containing the provided string. Due to ' + self.add_option( + '-j', + '--jobs', + default=jobs, + type='int', + help='Specify how many SCM commands can run in parallel; defaults to ' + '%default on this machine') + self.add_option( + '-v', + '--verbose', + action='count', + default=0, + help='Produces additional output for diagnostics. Can be used up to ' + 'three times for more logging info.') + self.add_option('--gclientfile', + dest='config_filename', + help='Specify an alternate %s file' % + self.gclientfile_default) + self.add_option( + '--spec', + help='create a gclient file containing the provided string. Due to ' 'Cygwin/Python brokenness, it can\'t contain any newlines.') - self.add_option( - '--no-nag-max', default=False, action='store_true', - help='Ignored for backwards compatibility.') + self.add_option('--no-nag-max', + default=False, + action='store_true', + help='Ignored for backwards compatibility.') - def parse_args(self, args=None, _values=None): - """Integrates standard options processing.""" - # Create an optparse.Values object that will store only the actual passed - # options, without the defaults. - actual_options = optparse.Values() - _, args = optparse.OptionParser.parse_args(self, args, actual_options) - # Create an optparse.Values object with the default options. - options = optparse.Values(self.get_default_values().__dict__) - # Update it with the options passed by the user. - options._update_careful(actual_options.__dict__) - # Store the options passed by the user in an _actual_options attribute. - # We store only the keys, and not the values, since the values can contain - # arbitrary information, which might be PII. - metrics.collector.add('arguments', list(actual_options.__dict__)) + def parse_args(self, args=None, _values=None): + """Integrates standard options processing.""" + # Create an optparse.Values object that will store only the actual + # passed options, without the defaults. + actual_options = optparse.Values() + _, args = optparse.OptionParser.parse_args(self, args, actual_options) + # Create an optparse.Values object with the default options. + options = optparse.Values(self.get_default_values().__dict__) + # Update it with the options passed by the user. + options._update_careful(actual_options.__dict__) + # Store the options passed by the user in an _actual_options attribute. + # We store only the keys, and not the values, since the values can + # contain arbitrary information, which might be PII. + metrics.collector.add('arguments', list(actual_options.__dict__)) - levels = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG] - logging.basicConfig( - level=levels[min(options.verbose, len(levels) - 1)], - format='%(module)s(%(lineno)d) %(funcName)s:%(message)s') - if options.config_filename and options.spec: - self.error('Cannot specify both --gclientfile and --spec') - if (options.config_filename and - options.config_filename != os.path.basename(options.config_filename)): - self.error('--gclientfile target must be a filename, not a path') - if not options.config_filename: - options.config_filename = self.gclientfile_default - options.entries_filename = options.config_filename + '_entries' - if options.jobs < 1: - self.error('--jobs must be 1 or higher') + levels = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG] + logging.basicConfig( + level=levels[min(options.verbose, + len(levels) - 1)], + format='%(module)s(%(lineno)d) %(funcName)s:%(message)s') + if options.config_filename and options.spec: + self.error('Cannot specify both --gclientfile and --spec') + if (options.config_filename and options.config_filename != + os.path.basename(options.config_filename)): + self.error('--gclientfile target must be a filename, not a path') + if not options.config_filename: + options.config_filename = self.gclientfile_default + options.entries_filename = options.config_filename + '_entries' + if options.jobs < 1: + self.error('--jobs must be 1 or higher') - # These hacks need to die. - if not hasattr(options, 'revisions'): - # GClient.RunOnDeps expects it even if not applicable. - options.revisions = [] - if not hasattr(options, 'experiments'): - options.experiments = [] - if not hasattr(options, 'head'): - options.head = None - if not hasattr(options, 'nohooks'): - options.nohooks = True - if not hasattr(options, 'noprehooks'): - options.noprehooks = True - if not hasattr(options, 'deps_os'): - options.deps_os = None - if not hasattr(options, 'force'): - options.force = None - return (options, args) + # These hacks need to die. + if not hasattr(options, 'revisions'): + # GClient.RunOnDeps expects it even if not applicable. + options.revisions = [] + if not hasattr(options, 'experiments'): + options.experiments = [] + if not hasattr(options, 'head'): + options.head = None + if not hasattr(options, 'nohooks'): + options.nohooks = True + if not hasattr(options, 'noprehooks'): + options.noprehooks = True + if not hasattr(options, 'deps_os'): + options.deps_os = None + if not hasattr(options, 'force'): + options.force = None + return (options, args) def disable_buffering(): - # Make stdout auto-flush so buildbot doesn't kill us during lengthy - # operations. Python as a strong tendency to buffer sys.stdout. - sys.stdout = gclient_utils.MakeFileAutoFlush(sys.stdout) - # Make stdout annotated with the thread ids. - sys.stdout = gclient_utils.MakeFileAnnotated(sys.stdout) + # Make stdout auto-flush so buildbot doesn't kill us during lengthy + # operations. Python as a strong tendency to buffer sys.stdout. + sys.stdout = gclient_utils.MakeFileAutoFlush(sys.stdout) + # Make stdout annotated with the thread ids. + sys.stdout = gclient_utils.MakeFileAnnotated(sys.stdout) def path_contains_tilde(): - for element in os.environ['PATH'].split(os.pathsep): - if element.startswith('~') and os.path.abspath( - os.path.realpath(os.path.expanduser(element))) == DEPOT_TOOLS_DIR: - return True - return False + for element in os.environ['PATH'].split(os.pathsep): + if element.startswith('~') and os.path.abspath( + os.path.realpath( + os.path.expanduser(element))) == DEPOT_TOOLS_DIR: + return True + return False def can_run_gclient_and_helpers(): - if sys.hexversion < 0x02060000: - print( - '\nYour python version %s is unsupported, please upgrade.\n' % - sys.version.split(' ', 1)[0], - file=sys.stderr) - return False - if not sys.executable: - print( - '\nPython cannot find the location of it\'s own executable.\n', - file=sys.stderr) - return False - if path_contains_tilde(): - print( - '\nYour PATH contains a literal "~", which works in some shells ' + - 'but will break when python tries to run subprocesses. ' + - 'Replace the "~" with $HOME.\n' + - 'See https://crbug.com/952865.\n', - file=sys.stderr) - return False - return True + if sys.hexversion < 0x02060000: + print('\nYour python version %s is unsupported, please upgrade.\n' % + sys.version.split(' ', 1)[0], + file=sys.stderr) + return False + if not sys.executable: + print('\nPython cannot find the location of it\'s own executable.\n', + file=sys.stderr) + return False + if path_contains_tilde(): + print( + '\nYour PATH contains a literal "~", which works in some shells ' + + 'but will break when python tries to run subprocesses. ' + + 'Replace the "~" with $HOME.\n' + 'See https://crbug.com/952865.\n', + file=sys.stderr) + return False + return True def main(argv): - """Doesn't parse the arguments here, just find the right subcommand to + """Doesn't parse the arguments here, just find the right subcommand to execute.""" - if not can_run_gclient_and_helpers(): - return 2 - fix_encoding.fix_encoding() - disable_buffering() - setup_color.init() - dispatcher = subcommand.CommandDispatcher(__name__) - try: - return dispatcher.execute(OptionParser(), argv) - except KeyboardInterrupt: - gclient_utils.GClientChildren.KillAllRemainingChildren() - raise - except (gclient_utils.Error, subprocess2.CalledProcessError) as e: - print('Error: %s' % str(e), file=sys.stderr) - return 1 - finally: - gclient_utils.PrintWarnings() - return 0 + if not can_run_gclient_and_helpers(): + return 2 + fix_encoding.fix_encoding() + disable_buffering() + setup_color.init() + dispatcher = subcommand.CommandDispatcher(__name__) + try: + return dispatcher.execute(OptionParser(), argv) + except KeyboardInterrupt: + gclient_utils.GClientChildren.KillAllRemainingChildren() + raise + except (gclient_utils.Error, subprocess2.CalledProcessError) as e: + print('Error: %s' % str(e), file=sys.stderr) + return 1 + finally: + gclient_utils.PrintWarnings() + return 0 if '__main__' == __name__: - with metrics.collector.print_notice_and_exit(): - sys.exit(main(sys.argv[1:])) + with metrics.collector.print_notice_and_exit(): + sys.exit(main(sys.argv[1:])) # vim: ts=2:sw=2:tw=80:et: diff --git a/gclient_eval.py b/gclient_eval.py index 89e7351e30..bc3427cc5e 100644 --- a/gclient_eval.py +++ b/gclient_eval.py @@ -12,6 +12,8 @@ import tokenize import gclient_utils from third_party import schema +# TODO: Should fix these warnings. +# pylint: disable=line-too-long # git_dependencies migration states. Used within the DEPS file to indicate # the current migration state. @@ -21,81 +23,82 @@ SUBMODULES = 'SUBMODULES' class ConstantString(object): - def __init__(self, value): - self.value = value + def __init__(self, value): + self.value = value - def __format__(self, format_spec): - del format_spec - return self.value + def __format__(self, format_spec): + del format_spec + return self.value - def __repr__(self): - return "Str('" + self.value + "')" + def __repr__(self): + return "Str('" + self.value + "')" - def __eq__(self, other): - if isinstance(other, ConstantString): - return self.value == other.value + def __eq__(self, other): + if isinstance(other, ConstantString): + return self.value == other.value - return self.value == other + return self.value == other - def __hash__(self): - return self.value.__hash__() + def __hash__(self): + return self.value.__hash__() class _NodeDict(collections.abc.MutableMapping): - """Dict-like type that also stores information on AST nodes and tokens.""" - def __init__(self, data=None, tokens=None): - self.data = collections.OrderedDict(data or []) - self.tokens = tokens + """Dict-like type that also stores information on AST nodes and tokens.""" + def __init__(self, data=None, tokens=None): + self.data = collections.OrderedDict(data or []) + self.tokens = tokens - def __str__(self): - return str({k: v[0] for k, v in self.data.items()}) + def __str__(self): + return str({k: v[0] for k, v in self.data.items()}) - def __repr__(self): - return self.__str__() + def __repr__(self): + return self.__str__() - def __getitem__(self, key): - return self.data[key][0] + def __getitem__(self, key): + return self.data[key][0] - def __setitem__(self, key, value): - self.data[key] = (value, None) + def __setitem__(self, key, value): + self.data[key] = (value, None) - def __delitem__(self, key): - del self.data[key] + def __delitem__(self, key): + del self.data[key] - def __iter__(self): - return iter(self.data) + def __iter__(self): + return iter(self.data) - def __len__(self): - return len(self.data) + def __len__(self): + return len(self.data) - def MoveTokens(self, origin, delta): - if self.tokens: - new_tokens = {} - for pos, token in self.tokens.items(): - if pos[0] >= origin: - pos = (pos[0] + delta, pos[1]) - token = token[:2] + (pos,) + token[3:] - new_tokens[pos] = token + def MoveTokens(self, origin, delta): + if self.tokens: + new_tokens = {} + for pos, token in self.tokens.items(): + if pos[0] >= origin: + pos = (pos[0] + delta, pos[1]) + token = token[:2] + (pos, ) + token[3:] + new_tokens[pos] = token - for value, node in self.data.values(): - if node.lineno >= origin: - node.lineno += delta - if isinstance(value, _NodeDict): - value.MoveTokens(origin, delta) + for value, node in self.data.values(): + if node.lineno >= origin: + node.lineno += delta + if isinstance(value, _NodeDict): + value.MoveTokens(origin, delta) - def GetNode(self, key): - return self.data[key][1] + def GetNode(self, key): + return self.data[key][1] - def SetNode(self, key, value, node): - self.data[key] = (value, node) + def SetNode(self, key, value, node): + self.data[key] = (value, node) def _NodeDictSchema(dict_schema): - """Validate dict_schema after converting _NodeDict to a regular dict.""" - def validate(d): - schema.Schema(dict_schema).validate(dict(d)) - return True - return validate + """Validate dict_schema after converting _NodeDict to a regular dict.""" + def validate(d): + schema.Schema(dict_schema).validate(dict(d)) + return True + + return validate # See https://github.com/keleshev/schema for docs how to configure schema. @@ -269,245 +272,252 @@ _GCLIENT_SCHEMA = schema.Schema( def _gclient_eval(node_or_string, filename='', vars_dict=None): - """Safely evaluates a single expression. Returns the result.""" - _allowed_names = {'None': None, 'True': True, 'False': False} - if isinstance(node_or_string, ConstantString): - return node_or_string.value - if isinstance(node_or_string, str): - node_or_string = ast.parse(node_or_string, filename=filename, mode='eval') - if isinstance(node_or_string, ast.Expression): - node_or_string = node_or_string.body - def _convert(node): - if isinstance(node, ast.Str): - if vars_dict is None: - return node.s - try: - return node.s.format(**vars_dict) - except KeyError as e: - raise KeyError( - '%s was used as a variable, but was not declared in the vars dict ' - '(file %r, line %s)' % ( - e.args[0], filename, getattr(node, 'lineno', ''))) - elif isinstance(node, ast.Num): - return node.n - elif isinstance(node, ast.Tuple): - return tuple(map(_convert, node.elts)) - elif isinstance(node, ast.List): - return list(map(_convert, node.elts)) - elif isinstance(node, ast.Dict): - node_dict = _NodeDict() - for key_node, value_node in zip(node.keys, node.values): - key = _convert(key_node) - if key in node_dict: - raise ValueError( - 'duplicate key in dictionary: %s (file %r, line %s)' % ( - key, filename, getattr(key_node, 'lineno', ''))) - node_dict.SetNode(key, _convert(value_node), value_node) - return node_dict - elif isinstance(node, ast.Name): - if node.id not in _allowed_names: - raise ValueError( - 'invalid name %r (file %r, line %s)' % ( - node.id, filename, getattr(node, 'lineno', ''))) - return _allowed_names[node.id] - elif not sys.version_info[:2] < (3, 4) and isinstance( - node, ast.NameConstant): # Since Python 3.4 - return node.value - elif isinstance(node, ast.Call): - if (not isinstance(node.func, ast.Name) or - (node.func.id not in ('Str', 'Var'))): - raise ValueError( - 'Str and Var are the only allowed functions (file %r, line %s)' % ( - filename, getattr(node, 'lineno', ''))) - if node.keywords or getattr(node, 'starargs', None) or getattr( - node, 'kwargs', None) or len(node.args) != 1: - raise ValueError( - '%s takes exactly one argument (file %r, line %s)' % ( - node.func.id, filename, getattr(node, 'lineno', ''))) + """Safely evaluates a single expression. Returns the result.""" + _allowed_names = {'None': None, 'True': True, 'False': False} + if isinstance(node_or_string, ConstantString): + return node_or_string.value + if isinstance(node_or_string, str): + node_or_string = ast.parse(node_or_string, + filename=filename, + mode='eval') + if isinstance(node_or_string, ast.Expression): + node_or_string = node_or_string.body - if node.func.id == 'Str': - if isinstance(node.args[0], ast.Str): - return ConstantString(node.args[0].s) - raise ValueError('Passed a non-string to Str() (file %r, line%s)' % ( - filename, getattr(node, 'lineno', ''))) + def _convert(node): + if isinstance(node, ast.Str): + if vars_dict is None: + return node.s + try: + return node.s.format(**vars_dict) + except KeyError as e: + raise KeyError( + '%s was used as a variable, but was not declared in the vars dict ' + '(file %r, line %s)' % + (e.args[0], filename, getattr(node, 'lineno', ''))) + elif isinstance(node, ast.Num): + return node.n + elif isinstance(node, ast.Tuple): + return tuple(map(_convert, node.elts)) + elif isinstance(node, ast.List): + return list(map(_convert, node.elts)) + elif isinstance(node, ast.Dict): + node_dict = _NodeDict() + for key_node, value_node in zip(node.keys, node.values): + key = _convert(key_node) + if key in node_dict: + raise ValueError( + 'duplicate key in dictionary: %s (file %r, line %s)' % + (key, filename, getattr(key_node, 'lineno', + ''))) + node_dict.SetNode(key, _convert(value_node), value_node) + return node_dict + elif isinstance(node, ast.Name): + if node.id not in _allowed_names: + raise ValueError( + 'invalid name %r (file %r, line %s)' % + (node.id, filename, getattr(node, 'lineno', ''))) + return _allowed_names[node.id] + elif not sys.version_info[:2] < (3, 4) and isinstance( + node, ast.NameConstant): # Since Python 3.4 + return node.value + elif isinstance(node, ast.Call): + if (not isinstance(node.func, ast.Name) + or (node.func.id not in ('Str', 'Var'))): + raise ValueError( + 'Str and Var are the only allowed functions (file %r, line %s)' + % (filename, getattr(node, 'lineno', ''))) + if node.keywords or getattr(node, 'starargs', None) or getattr( + node, 'kwargs', None) or len(node.args) != 1: + raise ValueError( + '%s takes exactly one argument (file %r, line %s)' % + (node.func.id, filename, getattr(node, 'lineno', + ''))) - arg = _convert(node.args[0]) - if not isinstance(arg, str): - raise ValueError( - 'Var\'s argument must be a variable name (file %r, line %s)' % ( - filename, getattr(node, 'lineno', ''))) - if vars_dict is None: - return '{' + arg + '}' - if arg not in vars_dict: - raise KeyError( - '%s was used as a variable, but was not declared in the vars dict ' - '(file %r, line %s)' % ( - arg, filename, getattr(node, 'lineno', ''))) - val = vars_dict[arg] - if isinstance(val, ConstantString): - val = val.value - return val - elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): - return _convert(node.left) + _convert(node.right) - elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.Mod): - return _convert(node.left) % _convert(node.right) - else: - raise ValueError( - 'unexpected AST node: %s %s (file %r, line %s)' % ( - node, ast.dump(node), filename, - getattr(node, 'lineno', ''))) - return _convert(node_or_string) + if node.func.id == 'Str': + if isinstance(node.args[0], ast.Str): + return ConstantString(node.args[0].s) + raise ValueError( + 'Passed a non-string to Str() (file %r, line%s)' % + (filename, getattr(node, 'lineno', ''))) + + arg = _convert(node.args[0]) + if not isinstance(arg, str): + raise ValueError( + 'Var\'s argument must be a variable name (file %r, line %s)' + % (filename, getattr(node, 'lineno', ''))) + if vars_dict is None: + return '{' + arg + '}' + if arg not in vars_dict: + raise KeyError( + '%s was used as a variable, but was not declared in the vars dict ' + '(file %r, line %s)' % + (arg, filename, getattr(node, 'lineno', ''))) + val = vars_dict[arg] + if isinstance(val, ConstantString): + val = val.value + return val + elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): + return _convert(node.left) + _convert(node.right) + elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.Mod): + return _convert(node.left) % _convert(node.right) + else: + raise ValueError('unexpected AST node: %s %s (file %r, line %s)' % + (node, ast.dump(node), filename, + getattr(node, 'lineno', ''))) + + return _convert(node_or_string) def Exec(content, filename='', vars_override=None, builtin_vars=None): - """Safely execs a set of assignments.""" - def _validate_statement(node, local_scope): - if not isinstance(node, ast.Assign): - raise ValueError( - 'unexpected AST node: %s %s (file %r, line %s)' % ( - node, ast.dump(node), filename, - getattr(node, 'lineno', ''))) + """Safely execs a set of assignments.""" + def _validate_statement(node, local_scope): + if not isinstance(node, ast.Assign): + raise ValueError('unexpected AST node: %s %s (file %r, line %s)' % + (node, ast.dump(node), filename, + getattr(node, 'lineno', ''))) - if len(node.targets) != 1: - raise ValueError( - 'invalid assignment: use exactly one target (file %r, line %s)' % ( - filename, getattr(node, 'lineno', ''))) + if len(node.targets) != 1: + raise ValueError( + 'invalid assignment: use exactly one target (file %r, line %s)' + % (filename, getattr(node, 'lineno', ''))) - target = node.targets[0] - if not isinstance(target, ast.Name): - raise ValueError( - 'invalid assignment: target should be a name (file %r, line %s)' % ( - filename, getattr(node, 'lineno', ''))) - if target.id in local_scope: - raise ValueError( - 'invalid assignment: overrides var %r (file %r, line %s)' % ( - target.id, filename, getattr(node, 'lineno', ''))) + target = node.targets[0] + if not isinstance(target, ast.Name): + raise ValueError( + 'invalid assignment: target should be a name (file %r, line %s)' + % (filename, getattr(node, 'lineno', ''))) + if target.id in local_scope: + raise ValueError( + 'invalid assignment: overrides var %r (file %r, line %s)' % + (target.id, filename, getattr(node, 'lineno', ''))) - node_or_string = ast.parse(content, filename=filename, mode='exec') - if isinstance(node_or_string, ast.Expression): - node_or_string = node_or_string.body + node_or_string = ast.parse(content, filename=filename, mode='exec') + if isinstance(node_or_string, ast.Expression): + node_or_string = node_or_string.body - if not isinstance(node_or_string, ast.Module): - raise ValueError( - 'unexpected AST node: %s %s (file %r, line %s)' % ( - node_or_string, - ast.dump(node_or_string), - filename, - getattr(node_or_string, 'lineno', ''))) + if not isinstance(node_or_string, ast.Module): + raise ValueError('unexpected AST node: %s %s (file %r, line %s)' % + (node_or_string, ast.dump(node_or_string), filename, + getattr(node_or_string, 'lineno', ''))) - statements = {} - for statement in node_or_string.body: - _validate_statement(statement, statements) - statements[statement.targets[0].id] = statement.value + statements = {} + for statement in node_or_string.body: + _validate_statement(statement, statements) + statements[statement.targets[0].id] = statement.value - # The tokenized representation needs to end with a newline token, otherwise - # untokenization will trigger an assert later on. - # In Python 2.7 on Windows we need to ensure the input ends with a newline - # for a newline token to be generated. - # In other cases a newline token is always generated during tokenization so - # this has no effect. - # TODO: Remove this workaround after migrating to Python 3. - content += '\n' - tokens = { - token[2]: list(token) for token in tokenize.generate_tokens( - StringIO(content).readline) - } + # The tokenized representation needs to end with a newline token, otherwise + # untokenization will trigger an assert later on. + # In Python 2.7 on Windows we need to ensure the input ends with a newline + # for a newline token to be generated. + # In other cases a newline token is always generated during tokenization so + # this has no effect. + # TODO: Remove this workaround after migrating to Python 3. + content += '\n' + tokens = { + token[2]: list(token) + for token in tokenize.generate_tokens(StringIO(content).readline) + } - local_scope = _NodeDict({}, tokens) + local_scope = _NodeDict({}, tokens) - # Process vars first, so we can expand variables in the rest of the DEPS file. - vars_dict = {} - if 'vars' in statements: - vars_statement = statements['vars'] - value = _gclient_eval(vars_statement, filename) - local_scope.SetNode('vars', value, vars_statement) - # Update the parsed vars with the overrides, but only if they are already - # present (overrides do not introduce new variables). - vars_dict.update(value) + # Process vars first, so we can expand variables in the rest of the DEPS + # file. + vars_dict = {} + if 'vars' in statements: + vars_statement = statements['vars'] + value = _gclient_eval(vars_statement, filename) + local_scope.SetNode('vars', value, vars_statement) + # Update the parsed vars with the overrides, but only if they are + # already present (overrides do not introduce new variables). + vars_dict.update(value) - if builtin_vars: - vars_dict.update(builtin_vars) + if builtin_vars: + vars_dict.update(builtin_vars) - if vars_override: - vars_dict.update({k: v for k, v in vars_override.items() if k in vars_dict}) + if vars_override: + vars_dict.update( + {k: v + for k, v in vars_override.items() if k in vars_dict}) - for name, node in statements.items(): - value = _gclient_eval(node, filename, vars_dict) - local_scope.SetNode(name, value, node) + for name, node in statements.items(): + value = _gclient_eval(node, filename, vars_dict) + local_scope.SetNode(name, value, node) - try: - return _GCLIENT_SCHEMA.validate(local_scope) - except schema.SchemaError as e: - raise gclient_utils.Error(str(e)) + try: + return _GCLIENT_SCHEMA.validate(local_scope) + except schema.SchemaError as e: + raise gclient_utils.Error(str(e)) def _StandardizeDeps(deps_dict, vars_dict): - """"Standardizes the deps_dict. + """"Standardizes the deps_dict. For each dependency: - Expands the variable in the dependency name. - Ensures the dependency is a dictionary. - Set's the 'dep_type' to be 'git' by default. """ - new_deps_dict = {} - for dep_name, dep_info in deps_dict.items(): - dep_name = dep_name.format(**vars_dict) - if not isinstance(dep_info, collections.abc.Mapping): - dep_info = {'url': dep_info} - dep_info.setdefault('dep_type', 'git') - new_deps_dict[dep_name] = dep_info - return new_deps_dict + new_deps_dict = {} + for dep_name, dep_info in deps_dict.items(): + dep_name = dep_name.format(**vars_dict) + if not isinstance(dep_info, collections.abc.Mapping): + dep_info = {'url': dep_info} + dep_info.setdefault('dep_type', 'git') + new_deps_dict[dep_name] = dep_info + return new_deps_dict def _MergeDepsOs(deps_dict, os_deps_dict, os_name): - """Merges the deps in os_deps_dict into conditional dependencies in deps_dict. + """Merges the deps in os_deps_dict into conditional dependencies in deps_dict. The dependencies in os_deps_dict are transformed into conditional dependencies using |'checkout_' + os_name|. If the dependency is already present, the URL and revision must coincide. """ - for dep_name, dep_info in os_deps_dict.items(): - # Make this condition very visible, so it's not a silent failure. - # It's unclear how to support None override in deps_os. - if dep_info['url'] is None: - logging.error('Ignoring %r:%r in %r deps_os', dep_name, dep_info, os_name) - continue + for dep_name, dep_info in os_deps_dict.items(): + # Make this condition very visible, so it's not a silent failure. + # It's unclear how to support None override in deps_os. + if dep_info['url'] is None: + logging.error('Ignoring %r:%r in %r deps_os', dep_name, dep_info, + os_name) + continue - os_condition = 'checkout_' + (os_name if os_name != 'unix' else 'linux') - UpdateCondition(dep_info, 'and', os_condition) + os_condition = 'checkout_' + (os_name if os_name != 'unix' else 'linux') + UpdateCondition(dep_info, 'and', os_condition) - if dep_name in deps_dict: - if deps_dict[dep_name]['url'] != dep_info['url']: - raise gclient_utils.Error( - 'Value from deps_os (%r; %r: %r) conflicts with existing deps ' - 'entry (%r).' % ( - os_name, dep_name, dep_info, deps_dict[dep_name])) + if dep_name in deps_dict: + if deps_dict[dep_name]['url'] != dep_info['url']: + raise gclient_utils.Error( + 'Value from deps_os (%r; %r: %r) conflicts with existing deps ' + 'entry (%r).' % + (os_name, dep_name, dep_info, deps_dict[dep_name])) - UpdateCondition(dep_info, 'or', deps_dict[dep_name].get('condition')) + UpdateCondition(dep_info, 'or', + deps_dict[dep_name].get('condition')) - deps_dict[dep_name] = dep_info + deps_dict[dep_name] = dep_info def UpdateCondition(info_dict, op, new_condition): - """Updates info_dict's condition with |new_condition|. + """Updates info_dict's condition with |new_condition|. An absent value is treated as implicitly True. """ - curr_condition = info_dict.get('condition') - # Easy case: Both are present. - if curr_condition and new_condition: - info_dict['condition'] = '(%s) %s (%s)' % ( - curr_condition, op, new_condition) - # If |op| == 'and', and at least one condition is present, then use it. - elif op == 'and' and (curr_condition or new_condition): - info_dict['condition'] = curr_condition or new_condition - # Otherwise, no condition should be set - elif curr_condition: - del info_dict['condition'] + curr_condition = info_dict.get('condition') + # Easy case: Both are present. + if curr_condition and new_condition: + info_dict['condition'] = '(%s) %s (%s)' % (curr_condition, op, + new_condition) + # If |op| == 'and', and at least one condition is present, then use it. + elif op == 'and' and (curr_condition or new_condition): + info_dict['condition'] = curr_condition or new_condition + # Otherwise, no condition should be set + elif curr_condition: + del info_dict['condition'] def Parse(content, filename, vars_override=None, builtin_vars=None): - """Parses DEPS strings. + """Parses DEPS strings. Executes the Python-like string stored in content, resulting in a Python dictionary specified by the schema above. Supports syntax validation and @@ -526,408 +536,397 @@ def Parse(content, filename, vars_override=None, builtin_vars=None): A Python dict with the parsed contents of the DEPS file, as specified by the schema above. """ - result = Exec(content, filename, vars_override, builtin_vars) + result = Exec(content, filename, vars_override, builtin_vars) - vars_dict = result.get('vars', {}) - if 'deps' in result: - result['deps'] = _StandardizeDeps(result['deps'], vars_dict) + vars_dict = result.get('vars', {}) + if 'deps' in result: + result['deps'] = _StandardizeDeps(result['deps'], vars_dict) - if 'deps_os' in result: - deps = result.setdefault('deps', {}) - for os_name, os_deps in result['deps_os'].items(): - os_deps = _StandardizeDeps(os_deps, vars_dict) - _MergeDepsOs(deps, os_deps, os_name) - del result['deps_os'] + if 'deps_os' in result: + deps = result.setdefault('deps', {}) + for os_name, os_deps in result['deps_os'].items(): + os_deps = _StandardizeDeps(os_deps, vars_dict) + _MergeDepsOs(deps, os_deps, os_name) + del result['deps_os'] - if 'hooks_os' in result: - hooks = result.setdefault('hooks', []) - for os_name, os_hooks in result['hooks_os'].items(): - for hook in os_hooks: - UpdateCondition(hook, 'and', 'checkout_' + os_name) - hooks.extend(os_hooks) - del result['hooks_os'] + if 'hooks_os' in result: + hooks = result.setdefault('hooks', []) + for os_name, os_hooks in result['hooks_os'].items(): + for hook in os_hooks: + UpdateCondition(hook, 'and', 'checkout_' + os_name) + hooks.extend(os_hooks) + del result['hooks_os'] - return result + return result def EvaluateCondition(condition, variables, referenced_variables=None): - """Safely evaluates a boolean condition. Returns the result.""" - if not referenced_variables: - referenced_variables = set() - _allowed_names = {'None': None, 'True': True, 'False': False} - main_node = ast.parse(condition, mode='eval') - if isinstance(main_node, ast.Expression): - main_node = main_node.body - def _convert(node, allow_tuple=False): - if isinstance(node, ast.Str): - return node.s + """Safely evaluates a boolean condition. Returns the result.""" + if not referenced_variables: + referenced_variables = set() + _allowed_names = {'None': None, 'True': True, 'False': False} + main_node = ast.parse(condition, mode='eval') + if isinstance(main_node, ast.Expression): + main_node = main_node.body - if isinstance(node, ast.Tuple) and allow_tuple: - return tuple(map(_convert, node.elts)) + def _convert(node, allow_tuple=False): + if isinstance(node, ast.Str): + return node.s - if isinstance(node, ast.Name): - if node.id in referenced_variables: - raise ValueError( - 'invalid cyclic reference to %r (inside %r)' % ( - node.id, condition)) + if isinstance(node, ast.Tuple) and allow_tuple: + return tuple(map(_convert, node.elts)) - if node.id in _allowed_names: - return _allowed_names[node.id] + if isinstance(node, ast.Name): + if node.id in referenced_variables: + raise ValueError('invalid cyclic reference to %r (inside %r)' % + (node.id, condition)) - if node.id in variables: - value = variables[node.id] + if node.id in _allowed_names: + return _allowed_names[node.id] - # Allow using "native" types, without wrapping everything in strings. - # Note that schema constraints still apply to variables. - if not isinstance(value, str): - return value + if node.id in variables: + value = variables[node.id] - # Recursively evaluate the variable reference. - return EvaluateCondition( - variables[node.id], - variables, - referenced_variables.union([node.id])) + # Allow using "native" types, without wrapping everything in + # strings. Note that schema constraints still apply to + # variables. + if not isinstance(value, str): + return value - # Implicitly convert unrecognized names to strings. - # If we want to change this, we'll need to explicitly distinguish - # between arguments for GN to be passed verbatim, and ones to - # be evaluated. - return node.id + # Recursively evaluate the variable reference. + return EvaluateCondition(variables[node.id], variables, + referenced_variables.union([node.id])) - if not sys.version_info[:2] < (3, 4) and isinstance( - node, ast.NameConstant): # Since Python 3.4 - return node.value + # Implicitly convert unrecognized names to strings. + # If we want to change this, we'll need to explicitly distinguish + # between arguments for GN to be passed verbatim, and ones to + # be evaluated. + return node.id - if isinstance(node, ast.BoolOp) and isinstance(node.op, ast.Or): - bool_values = [] - for value in node.values: - bool_values.append(_convert(value)) - if not isinstance(bool_values[-1], bool): - raise ValueError( - 'invalid "or" operand %r (inside %r)' % ( - bool_values[-1], condition)) - return any(bool_values) + if not sys.version_info[:2] < (3, 4) and isinstance( + node, ast.NameConstant): # Since Python 3.4 + return node.value - if isinstance(node, ast.BoolOp) and isinstance(node.op, ast.And): - bool_values = [] - for value in node.values: - bool_values.append(_convert(value)) - if not isinstance(bool_values[-1], bool): - raise ValueError( - 'invalid "and" operand %r (inside %r)' % ( - bool_values[-1], condition)) - return all(bool_values) + if isinstance(node, ast.BoolOp) and isinstance(node.op, ast.Or): + bool_values = [] + for value in node.values: + bool_values.append(_convert(value)) + if not isinstance(bool_values[-1], bool): + raise ValueError('invalid "or" operand %r (inside %r)' % + (bool_values[-1], condition)) + return any(bool_values) - if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): - value = _convert(node.operand) - if not isinstance(value, bool): - raise ValueError( - 'invalid "not" operand %r (inside %r)' % (value, condition)) - return not value + if isinstance(node, ast.BoolOp) and isinstance(node.op, ast.And): + bool_values = [] + for value in node.values: + bool_values.append(_convert(value)) + if not isinstance(bool_values[-1], bool): + raise ValueError('invalid "and" operand %r (inside %r)' % + (bool_values[-1], condition)) + return all(bool_values) - if isinstance(node, ast.Compare): - if len(node.ops) != 1: - raise ValueError( - 'invalid compare: exactly 1 operator required (inside %r)' % ( - condition)) - if len(node.comparators) != 1: - raise ValueError( - 'invalid compare: exactly 1 comparator required (inside %r)' % ( - condition)) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): + value = _convert(node.operand) + if not isinstance(value, bool): + raise ValueError('invalid "not" operand %r (inside %r)' % + (value, condition)) + return not value - left = _convert(node.left) - right = _convert( - node.comparators[0], allow_tuple=isinstance(node.ops[0], ast.In)) + if isinstance(node, ast.Compare): + if len(node.ops) != 1: + raise ValueError( + 'invalid compare: exactly 1 operator required (inside %r)' % + (condition)) + if len(node.comparators) != 1: + raise ValueError( + 'invalid compare: exactly 1 comparator required (inside %r)' + % (condition)) - if isinstance(node.ops[0], ast.Eq): - return left == right - if isinstance(node.ops[0], ast.NotEq): - return left != right - if isinstance(node.ops[0], ast.In): - return left in right + left = _convert(node.left) + right = _convert(node.comparators[0], + allow_tuple=isinstance(node.ops[0], ast.In)) - raise ValueError( - 'unexpected operator: %s %s (inside %r)' % ( - node.ops[0], ast.dump(node), condition)) + if isinstance(node.ops[0], ast.Eq): + return left == right + if isinstance(node.ops[0], ast.NotEq): + return left != right + if isinstance(node.ops[0], ast.In): + return left in right - raise ValueError( - 'unexpected AST node: %s %s (inside %r)' % ( - node, ast.dump(node), condition)) - return _convert(main_node) + raise ValueError('unexpected operator: %s %s (inside %r)' % + (node.ops[0], ast.dump(node), condition)) + + raise ValueError('unexpected AST node: %s %s (inside %r)' % + (node, ast.dump(node), condition)) + + return _convert(main_node) def RenderDEPSFile(gclient_dict): - contents = sorted(gclient_dict.tokens.values(), key=lambda token: token[2]) - # The last token is a newline, which we ensure in Exec() for compatibility. - # However tests pass in inputs not ending with a newline and expect the same - # back, so for backwards compatibility need to remove that newline character. - # TODO: Fix tests to expect the newline - return tokenize.untokenize(contents)[:-1] + contents = sorted(gclient_dict.tokens.values(), key=lambda token: token[2]) + # The last token is a newline, which we ensure in Exec() for compatibility. + # However tests pass in inputs not ending with a newline and expect the same + # back, so for backwards compatibility need to remove that newline + # character. TODO: Fix tests to expect the newline + return tokenize.untokenize(contents)[:-1] def _UpdateAstString(tokens, node, value): - if isinstance(node, ast.Call): - node = node.args[0] - position = node.lineno, node.col_offset - quote_char = '' - if isinstance(node, ast.Str): - quote_char = tokens[position][1][0] - value = value.encode('unicode_escape').decode('utf-8') - tokens[position][1] = quote_char + value + quote_char - node.s = value + if isinstance(node, ast.Call): + node = node.args[0] + position = node.lineno, node.col_offset + quote_char = '' + if isinstance(node, ast.Str): + quote_char = tokens[position][1][0] + value = value.encode('unicode_escape').decode('utf-8') + tokens[position][1] = quote_char + value + quote_char + node.s = value def _ShiftLinesInTokens(tokens, delta, start): - new_tokens = {} - for token in tokens.values(): - if token[2][0] >= start: - token[2] = token[2][0] + delta, token[2][1] - token[3] = token[3][0] + delta, token[3][1] - new_tokens[token[2]] = token - return new_tokens + new_tokens = {} + for token in tokens.values(): + if token[2][0] >= start: + token[2] = token[2][0] + delta, token[2][1] + token[3] = token[3][0] + delta, token[3][1] + new_tokens[token[2]] = token + return new_tokens def AddVar(gclient_dict, var_name, value): - if not isinstance(gclient_dict, _NodeDict) or gclient_dict.tokens is None: - raise ValueError( - "Can't use SetVar for the given gclient dict. It contains no " - "formatting information.") + if not isinstance(gclient_dict, _NodeDict) or gclient_dict.tokens is None: + raise ValueError( + "Can't use SetVar for the given gclient dict. It contains no " + "formatting information.") - if 'vars' not in gclient_dict: - raise KeyError("vars dict is not defined.") + if 'vars' not in gclient_dict: + raise KeyError("vars dict is not defined.") - if var_name in gclient_dict['vars']: - raise ValueError( - "%s has already been declared in the vars dict. Consider using SetVar " - "instead." % var_name) + if var_name in gclient_dict['vars']: + raise ValueError( + "%s has already been declared in the vars dict. Consider using SetVar " + "instead." % var_name) - if not gclient_dict['vars']: - raise ValueError('vars dict is empty. This is not yet supported.') + if not gclient_dict['vars']: + raise ValueError('vars dict is empty. This is not yet supported.') - # We will attempt to add the var right after 'vars = {'. - node = gclient_dict.GetNode('vars') - if node is None: - raise ValueError( - "The vars dict has no formatting information." % var_name) - line = node.lineno + 1 + # We will attempt to add the var right after 'vars = {'. + node = gclient_dict.GetNode('vars') + if node is None: + raise ValueError("The vars dict has no formatting information." % + var_name) + line = node.lineno + 1 - # We will try to match the new var's indentation to the next variable. - col = node.keys[0].col_offset + # We will try to match the new var's indentation to the next variable. + col = node.keys[0].col_offset - # We use a minimal Python dictionary, so that ast can parse it. - var_content = '{\n%s"%s": "%s",\n}\n' % (' ' * col, var_name, value) - var_ast = ast.parse(var_content).body[0].value + # We use a minimal Python dictionary, so that ast can parse it. + var_content = '{\n%s"%s": "%s",\n}\n' % (' ' * col, var_name, value) + var_ast = ast.parse(var_content).body[0].value - # Set the ast nodes for the key and value. - vars_node = gclient_dict.GetNode('vars') + # Set the ast nodes for the key and value. + vars_node = gclient_dict.GetNode('vars') - var_name_node = var_ast.keys[0] - var_name_node.lineno += line - 2 - vars_node.keys.insert(0, var_name_node) + var_name_node = var_ast.keys[0] + var_name_node.lineno += line - 2 + vars_node.keys.insert(0, var_name_node) - value_node = var_ast.values[0] - value_node.lineno += line - 2 - vars_node.values.insert(0, value_node) + value_node = var_ast.values[0] + value_node.lineno += line - 2 + vars_node.values.insert(0, value_node) - # Update the tokens. - var_tokens = list(tokenize.generate_tokens(StringIO(var_content).readline)) - var_tokens = { - token[2]: list(token) - # Ignore the tokens corresponding to braces and new lines. - for token in var_tokens[2:-3] - } + # Update the tokens. + var_tokens = list(tokenize.generate_tokens(StringIO(var_content).readline)) + var_tokens = { + token[2]: list(token) + # Ignore the tokens corresponding to braces and new lines. + for token in var_tokens[2:-3] + } - gclient_dict.tokens = _ShiftLinesInTokens(gclient_dict.tokens, 1, line) - gclient_dict.tokens.update(_ShiftLinesInTokens(var_tokens, line - 2, 0)) + gclient_dict.tokens = _ShiftLinesInTokens(gclient_dict.tokens, 1, line) + gclient_dict.tokens.update(_ShiftLinesInTokens(var_tokens, line - 2, 0)) def SetVar(gclient_dict, var_name, value): - if not isinstance(gclient_dict, _NodeDict) or gclient_dict.tokens is None: - raise ValueError( - "Can't use SetVar for the given gclient dict. It contains no " - "formatting information.") - tokens = gclient_dict.tokens + if not isinstance(gclient_dict, _NodeDict) or gclient_dict.tokens is None: + raise ValueError( + "Can't use SetVar for the given gclient dict. It contains no " + "formatting information.") + tokens = gclient_dict.tokens - if 'vars' not in gclient_dict: - raise KeyError("vars dict is not defined.") + if 'vars' not in gclient_dict: + raise KeyError("vars dict is not defined.") - if var_name not in gclient_dict['vars']: - raise ValueError( - "%s has not been declared in the vars dict. Consider using AddVar " - "instead." % var_name) + if var_name not in gclient_dict['vars']: + raise ValueError( + "%s has not been declared in the vars dict. Consider using AddVar " + "instead." % var_name) - node = gclient_dict['vars'].GetNode(var_name) - if node is None: - raise ValueError( - "The vars entry for %s has no formatting information." % var_name) + node = gclient_dict['vars'].GetNode(var_name) + if node is None: + raise ValueError( + "The vars entry for %s has no formatting information." % var_name) - _UpdateAstString(tokens, node, value) - gclient_dict['vars'].SetNode(var_name, value, node) + _UpdateAstString(tokens, node, value) + gclient_dict['vars'].SetNode(var_name, value, node) def _GetVarName(node): - if isinstance(node, ast.Call): - return node.args[0].s + if isinstance(node, ast.Call): + return node.args[0].s - if node.s.endswith('}'): - last_brace = node.s.rfind('{') - return node.s[last_brace+1:-1] - return None + if node.s.endswith('}'): + last_brace = node.s.rfind('{') + return node.s[last_brace + 1:-1] + return None def SetCIPD(gclient_dict, dep_name, package_name, new_version): - if not isinstance(gclient_dict, _NodeDict) or gclient_dict.tokens is None: - raise ValueError( - "Can't use SetCIPD for the given gclient dict. It contains no " - "formatting information.") - tokens = gclient_dict.tokens - - if 'deps' not in gclient_dict or dep_name not in gclient_dict['deps']: - raise KeyError( - "Could not find any dependency called %s." % dep_name) - - # Find the package with the given name - packages = [ - package - for package in gclient_dict['deps'][dep_name]['packages'] - if package['package'] == package_name - ] - if len(packages) != 1: - raise ValueError( - "There must be exactly one package with the given name (%s), " - "%s were found." % (package_name, len(packages))) - - # TODO(ehmaldonado): Support Var in package's version. - node = packages[0].GetNode('version') - if node is None: - raise ValueError( - "The deps entry for %s:%s has no formatting information." % - (dep_name, package_name)) - - if not isinstance(node, ast.Call) and not isinstance(node, ast.Str): - raise ValueError( - "Unsupported dependency revision format. Please file a bug to the " - "Infra>SDK component in crbug.com") - - var_name = _GetVarName(node) - if var_name is not None: - SetVar(gclient_dict, var_name, new_version) - else: - _UpdateAstString(tokens, node, new_version) - packages[0].SetNode('version', new_version, node) - - -def SetRevision(gclient_dict, dep_name, new_revision): - def _UpdateRevision(dep_dict, dep_key, new_revision): - dep_node = dep_dict.GetNode(dep_key) - if dep_node is None: - raise ValueError( - "The deps entry for %s has no formatting information." % dep_name) - - node = dep_node - if isinstance(node, ast.BinOp): - node = node.right - - if isinstance(node, ast.Str): - token = _gclient_eval(tokens[node.lineno, node.col_offset][1]) - if token != node.s: + if not isinstance(gclient_dict, _NodeDict) or gclient_dict.tokens is None: raise ValueError( - 'Can\'t update value for %s. Multiline strings and implicitly ' - 'concatenated strings are not supported.\n' - 'Consider reformatting the DEPS file.' % dep_key) + "Can't use SetCIPD for the given gclient dict. It contains no " + "formatting information.") + tokens = gclient_dict.tokens + if 'deps' not in gclient_dict or dep_name not in gclient_dict['deps']: + raise KeyError("Could not find any dependency called %s." % dep_name) + + # Find the package with the given name + packages = [ + package for package in gclient_dict['deps'][dep_name]['packages'] + if package['package'] == package_name + ] + if len(packages) != 1: + raise ValueError( + "There must be exactly one package with the given name (%s), " + "%s were found." % (package_name, len(packages))) + + # TODO(ehmaldonado): Support Var in package's version. + node = packages[0].GetNode('version') + if node is None: + raise ValueError( + "The deps entry for %s:%s has no formatting information." % + (dep_name, package_name)) if not isinstance(node, ast.Call) and not isinstance(node, ast.Str): - raise ValueError( - "Unsupported dependency revision format. Please file a bug to the " - "Infra>SDK component in crbug.com") + raise ValueError( + "Unsupported dependency revision format. Please file a bug to the " + "Infra>SDK component in crbug.com") var_name = _GetVarName(node) if var_name is not None: - SetVar(gclient_dict, var_name, new_revision) + SetVar(gclient_dict, var_name, new_version) else: - if '@' in node.s: - # '@' is part of the last string, which we want to modify. Discard - # whatever was after the '@' and put the new revision in its place. - new_revision = node.s.split('@')[0] + '@' + new_revision - elif '@' not in dep_dict[dep_key]: - # '@' is not part of the URL at all. This mean the dependency is - # unpinned and we should pin it. - new_revision = node.s + '@' + new_revision - _UpdateAstString(tokens, node, new_revision) - dep_dict.SetNode(dep_key, new_revision, node) + _UpdateAstString(tokens, node, new_version) + packages[0].SetNode('version', new_version, node) - if not isinstance(gclient_dict, _NodeDict) or gclient_dict.tokens is None: - raise ValueError( - "Can't use SetRevision for the given gclient dict. It contains no " - "formatting information.") - tokens = gclient_dict.tokens - if 'deps' not in gclient_dict or dep_name not in gclient_dict['deps']: - raise KeyError( - "Could not find any dependency called %s." % dep_name) +def SetRevision(gclient_dict, dep_name, new_revision): + def _UpdateRevision(dep_dict, dep_key, new_revision): + dep_node = dep_dict.GetNode(dep_key) + if dep_node is None: + raise ValueError( + "The deps entry for %s has no formatting information." % + dep_name) - if isinstance(gclient_dict['deps'][dep_name], _NodeDict): - _UpdateRevision(gclient_dict['deps'][dep_name], 'url', new_revision) - else: - _UpdateRevision(gclient_dict['deps'], dep_name, new_revision) + node = dep_node + if isinstance(node, ast.BinOp): + node = node.right + + if isinstance(node, ast.Str): + token = _gclient_eval(tokens[node.lineno, node.col_offset][1]) + if token != node.s: + raise ValueError( + 'Can\'t update value for %s. Multiline strings and implicitly ' + 'concatenated strings are not supported.\n' + 'Consider reformatting the DEPS file.' % dep_key) + + if not isinstance(node, ast.Call) and not isinstance(node, ast.Str): + raise ValueError( + "Unsupported dependency revision format. Please file a bug to the " + "Infra>SDK component in crbug.com") + + var_name = _GetVarName(node) + if var_name is not None: + SetVar(gclient_dict, var_name, new_revision) + else: + if '@' in node.s: + # '@' is part of the last string, which we want to modify. + # Discard whatever was after the '@' and put the new revision in + # its place. + new_revision = node.s.split('@')[0] + '@' + new_revision + elif '@' not in dep_dict[dep_key]: + # '@' is not part of the URL at all. This mean the dependency is + # unpinned and we should pin it. + new_revision = node.s + '@' + new_revision + _UpdateAstString(tokens, node, new_revision) + dep_dict.SetNode(dep_key, new_revision, node) + + if not isinstance(gclient_dict, _NodeDict) or gclient_dict.tokens is None: + raise ValueError( + "Can't use SetRevision for the given gclient dict. It contains no " + "formatting information.") + tokens = gclient_dict.tokens + + if 'deps' not in gclient_dict or dep_name not in gclient_dict['deps']: + raise KeyError("Could not find any dependency called %s." % dep_name) + + if isinstance(gclient_dict['deps'][dep_name], _NodeDict): + _UpdateRevision(gclient_dict['deps'][dep_name], 'url', new_revision) + else: + _UpdateRevision(gclient_dict['deps'], dep_name, new_revision) def GetVar(gclient_dict, var_name): - if 'vars' not in gclient_dict or var_name not in gclient_dict['vars']: - raise KeyError( - "Could not find any variable called %s." % var_name) + if 'vars' not in gclient_dict or var_name not in gclient_dict['vars']: + raise KeyError("Could not find any variable called %s." % var_name) - val = gclient_dict['vars'][var_name] - if isinstance(val, ConstantString): - return val.value - return val + val = gclient_dict['vars'][var_name] + if isinstance(val, ConstantString): + return val.value + return val def GetCIPD(gclient_dict, dep_name, package_name): - if 'deps' not in gclient_dict or dep_name not in gclient_dict['deps']: - raise KeyError( - "Could not find any dependency called %s." % dep_name) + if 'deps' not in gclient_dict or dep_name not in gclient_dict['deps']: + raise KeyError("Could not find any dependency called %s." % dep_name) - # Find the package with the given name - packages = [ - package - for package in gclient_dict['deps'][dep_name]['packages'] - if package['package'] == package_name - ] - if len(packages) != 1: - raise ValueError( - "There must be exactly one package with the given name (%s), " - "%s were found." % (package_name, len(packages))) + # Find the package with the given name + packages = [ + package for package in gclient_dict['deps'][dep_name]['packages'] + if package['package'] == package_name + ] + if len(packages) != 1: + raise ValueError( + "There must be exactly one package with the given name (%s), " + "%s were found." % (package_name, len(packages))) - return packages[0]['version'] + return packages[0]['version'] def GetRevision(gclient_dict, dep_name): - if 'deps' not in gclient_dict or dep_name not in gclient_dict['deps']: - suggestions = [] - if 'deps' in gclient_dict: - for key in gclient_dict['deps']: - if dep_name in key: - suggestions.append(key) - if suggestions: - raise KeyError( - "Could not find any dependency called %s. Did you mean %s" % - (dep_name, ' or '.join(suggestions))) - raise KeyError( - "Could not find any dependency called %s." % dep_name) + if 'deps' not in gclient_dict or dep_name not in gclient_dict['deps']: + suggestions = [] + if 'deps' in gclient_dict: + for key in gclient_dict['deps']: + if dep_name in key: + suggestions.append(key) + if suggestions: + raise KeyError( + "Could not find any dependency called %s. Did you mean %s" % + (dep_name, ' or '.join(suggestions))) + raise KeyError("Could not find any dependency called %s." % dep_name) - dep = gclient_dict['deps'][dep_name] - if dep is None: - return None + dep = gclient_dict['deps'][dep_name] + if dep is None: + return None - if isinstance(dep, str): - _, _, revision = dep.partition('@') - return revision or None + if isinstance(dep, str): + _, _, revision = dep.partition('@') + return revision or None - if isinstance(dep, collections.abc.Mapping) and 'url' in dep: - _, _, revision = dep['url'].partition('@') - return revision or None + if isinstance(dep, collections.abc.Mapping) and 'url' in dep: + _, _, revision = dep['url'].partition('@') + return revision or None - raise ValueError( - '%s is not a valid git dependency.' % dep_name) + raise ValueError('%s is not a valid git dependency.' % dep_name) diff --git a/gclient_paths.py b/gclient_paths.py index 10754eb653..1cfa213f12 100644 --- a/gclient_paths.py +++ b/gclient_paths.py @@ -15,136 +15,139 @@ import sys import gclient_utils import subprocess2 +# TODO: Should fix these warnings. +# pylint: disable=line-too-long + def FindGclientRoot(from_dir, filename='.gclient'): - """Tries to find the gclient root.""" - real_from_dir = os.path.abspath(from_dir) - path = real_from_dir - while not os.path.exists(os.path.join(path, filename)): - split_path = os.path.split(path) - if not split_path[1]: - return None - path = split_path[0] + """Tries to find the gclient root.""" + real_from_dir = os.path.abspath(from_dir) + path = real_from_dir + while not os.path.exists(os.path.join(path, filename)): + split_path = os.path.split(path) + if not split_path[1]: + return None + path = split_path[0] - logging.info('Found gclient root at ' + path) + logging.info('Found gclient root at ' + path) - if path == real_from_dir: - return path + if path == real_from_dir: + return path - # If we did not find the file in the current directory, make sure we are in a - # sub directory that is controlled by this configuration. - entries_filename = os.path.join(path, filename + '_entries') - if not os.path.exists(entries_filename): - # If .gclient_entries does not exist, a previous call to gclient sync - # might have failed. In that case, we cannot verify that the .gclient - # is the one we want to use. In order to not to cause too much trouble, - # just issue a warning and return the path anyway. - print( - "%s missing, %s file in parent directory %s might not be the file " - "you want to use." % (entries_filename, filename, path), - file=sys.stderr) - return path + # If we did not find the file in the current directory, make sure we are in + # a sub directory that is controlled by this configuration. + entries_filename = os.path.join(path, filename + '_entries') + if not os.path.exists(entries_filename): + # If .gclient_entries does not exist, a previous call to gclient sync + # might have failed. In that case, we cannot verify that the .gclient + # is the one we want to use. In order to not to cause too much trouble, + # just issue a warning and return the path anyway. + print( + "%s missing, %s file in parent directory %s might not be the file " + "you want to use." % (entries_filename, filename, path), + file=sys.stderr) + return path - entries_content = gclient_utils.FileRead(entries_filename) - scope = {} - try: - exec(entries_content, scope) - except (SyntaxError, Exception) as e: - gclient_utils.SyntaxErrorToError(filename, e) + entries_content = gclient_utils.FileRead(entries_filename) + scope = {} + try: + exec(entries_content, scope) + except (SyntaxError, Exception) as e: + gclient_utils.SyntaxErrorToError(filename, e) - all_directories = scope['entries'].keys() - path_to_check = os.path.relpath(real_from_dir, path) - while path_to_check: - if path_to_check in all_directories: - return path - path_to_check = os.path.dirname(path_to_check) + all_directories = scope['entries'].keys() + path_to_check = os.path.relpath(real_from_dir, path) + while path_to_check: + if path_to_check in all_directories: + return path + path_to_check = os.path.dirname(path_to_check) - return None + return None def GetPrimarySolutionPath(): - """Returns the full path to the primary solution. (gclient_root + src)""" + """Returns the full path to the primary solution. (gclient_root + src)""" - gclient_root = FindGclientRoot(os.getcwd()) - if gclient_root: - # Some projects' top directory is not named 'src'. - source_dir_name = GetGClientPrimarySolutionName(gclient_root) or 'src' - return os.path.join(gclient_root, source_dir_name) + gclient_root = FindGclientRoot(os.getcwd()) + if gclient_root: + # Some projects' top directory is not named 'src'. + source_dir_name = GetGClientPrimarySolutionName(gclient_root) or 'src' + return os.path.join(gclient_root, source_dir_name) - # Some projects might not use .gclient. Try to see whether we're in a git - # checkout that contains a 'buildtools' subdir. - top_dir = os.getcwd() - try: - top_dir = subprocess2.check_output(['git', 'rev-parse', '--show-toplevel'], - stderr=subprocess2.DEVNULL) - top_dir = top_dir.decode('utf-8', 'replace') - top_dir = os.path.normpath(top_dir.strip()) - except subprocess2.CalledProcessError: - pass + # Some projects might not use .gclient. Try to see whether we're in a git + # checkout that contains a 'buildtools' subdir. + top_dir = os.getcwd() + try: + top_dir = subprocess2.check_output( + ['git', 'rev-parse', '--show-toplevel'], stderr=subprocess2.DEVNULL) + top_dir = top_dir.decode('utf-8', 'replace') + top_dir = os.path.normpath(top_dir.strip()) + except subprocess2.CalledProcessError: + pass - if os.path.exists(os.path.join(top_dir, 'buildtools')): - return top_dir - return None + if os.path.exists(os.path.join(top_dir, 'buildtools')): + return top_dir + return None def GetBuildtoolsPath(): - """Returns the full path to the buildtools directory. + """Returns the full path to the buildtools directory. This is based on the root of the checkout containing the current directory.""" - # Overriding the build tools path by environment is highly unsupported and may - # break without warning. Do not rely on this for anything important. - override = os.environ.get('CHROMIUM_BUILDTOOLS_PATH') - if override is not None: - return override + # Overriding the build tools path by environment is highly unsupported and + # may break without warning. Do not rely on this for anything important. + override = os.environ.get('CHROMIUM_BUILDTOOLS_PATH') + if override is not None: + return override + + primary_solution = GetPrimarySolutionPath() + if not primary_solution: + return None + + buildtools_path = os.path.join(primary_solution, 'buildtools') + if os.path.exists(buildtools_path): + return buildtools_path + + # buildtools may be in the gclient root. + gclient_root = FindGclientRoot(os.getcwd()) + buildtools_path = os.path.join(gclient_root, 'buildtools') + if os.path.exists(buildtools_path): + return buildtools_path - primary_solution = GetPrimarySolutionPath() - if not primary_solution: return None - buildtools_path = os.path.join(primary_solution, 'buildtools') - if os.path.exists(buildtools_path): - return buildtools_path - - # buildtools may be in the gclient root. - gclient_root = FindGclientRoot(os.getcwd()) - buildtools_path = os.path.join(gclient_root, 'buildtools') - if os.path.exists(buildtools_path): - return buildtools_path - - return None - def GetBuildtoolsPlatformBinaryPath(): - """Returns the full path to the binary directory for the current platform.""" - buildtools_path = GetBuildtoolsPath() - if not buildtools_path: - return None + """Returns the full path to the binary directory for the current platform.""" + buildtools_path = GetBuildtoolsPath() + if not buildtools_path: + return None - if sys.platform.startswith(('cygwin', 'win')): - subdir = 'win' - elif sys.platform == 'darwin': - subdir = 'mac' - elif sys.platform.startswith('linux'): - subdir = 'linux64' - else: - raise gclient_utils.Error('Unknown platform: ' + sys.platform) - return os.path.join(buildtools_path, subdir) + if sys.platform.startswith(('cygwin', 'win')): + subdir = 'win' + elif sys.platform == 'darwin': + subdir = 'mac' + elif sys.platform.startswith('linux'): + subdir = 'linux64' + else: + raise gclient_utils.Error('Unknown platform: ' + sys.platform) + return os.path.join(buildtools_path, subdir) def GetExeSuffix(): - """Returns '' or '.exe' depending on how executables work on this platform.""" - if sys.platform.startswith(('cygwin', 'win')): - return '.exe' - return '' + """Returns '' or '.exe' depending on how executables work on this platform.""" + if sys.platform.startswith(('cygwin', 'win')): + return '.exe' + return '' def GetGClientPrimarySolutionName(gclient_root_dir_path): - """Returns the name of the primary solution in the .gclient file specified.""" - gclient_config_file = os.path.join(gclient_root_dir_path, '.gclient') - gclient_config_contents = gclient_utils.FileRead(gclient_config_file) - env = {} - exec(gclient_config_contents, env) - solutions = env.get('solutions', []) - if solutions: - return solutions[0].get('name') - return None + """Returns the name of the primary solution in the .gclient file specified.""" + gclient_config_file = os.path.join(gclient_root_dir_path, '.gclient') + gclient_config_contents = gclient_utils.FileRead(gclient_config_file) + env = {} + exec(gclient_config_contents, env) + solutions = env.get('solutions', []) + if solutions: + return solutions[0].get('name') + return None diff --git a/gclient_scm.py b/gclient_scm.py index 5294ceafa7..d9556ac0d6 100644 --- a/gclient_scm.py +++ b/gclient_scm.py @@ -1,7 +1,6 @@ # Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Gclient-specific SCM-specific operations.""" import collections @@ -25,322 +24,342 @@ import git_cache import scm import subprocess2 +# TODO: Should fix these warnings. +# pylint: disable=line-too-long + class NoUsableRevError(gclient_utils.Error): - """Raised if requested revision isn't found in checkout.""" + """Raised if requested revision isn't found in checkout.""" class DiffFiltererWrapper(object): - """Simple base class which tracks which file is being diffed and + """Simple base class which tracks which file is being diffed and replaces instances of its file name in the original and working copy lines of the git diff output.""" - index_string = None - original_prefix = "--- " - working_prefix = "+++ " + index_string = None + original_prefix = "--- " + working_prefix = "+++ " - def __init__(self, relpath, print_func): - # Note that we always use '/' as the path separator to be - # consistent with cygwin-style output on Windows - self._relpath = relpath.replace("\\", "/") - self._current_file = None - self._print_func = print_func + def __init__(self, relpath, print_func): + # Note that we always use '/' as the path separator to be + # consistent with cygwin-style output on Windows + self._relpath = relpath.replace("\\", "/") + self._current_file = None + self._print_func = print_func - def SetCurrentFile(self, current_file): - self._current_file = current_file + def SetCurrentFile(self, current_file): + self._current_file = current_file - @property - def _replacement_file(self): - return posixpath.join(self._relpath, self._current_file) + @property + def _replacement_file(self): + return posixpath.join(self._relpath, self._current_file) - def _Replace(self, line): - return line.replace(self._current_file, self._replacement_file) + def _Replace(self, line): + return line.replace(self._current_file, self._replacement_file) - def Filter(self, line): - if (line.startswith(self.index_string)): - self.SetCurrentFile(line[len(self.index_string):]) - line = self._Replace(line) - else: - if (line.startswith(self.original_prefix) or - line.startswith(self.working_prefix)): - line = self._Replace(line) - self._print_func(line) + def Filter(self, line): + if (line.startswith(self.index_string)): + self.SetCurrentFile(line[len(self.index_string):]) + line = self._Replace(line) + else: + if (line.startswith(self.original_prefix) + or line.startswith(self.working_prefix)): + line = self._Replace(line) + self._print_func(line) class GitDiffFilterer(DiffFiltererWrapper): - index_string = "diff --git " + index_string = "diff --git " - def SetCurrentFile(self, current_file): - # Get filename by parsing "a/ b/" - self._current_file = current_file[:(len(current_file)/2)][2:] + def SetCurrentFile(self, current_file): + # Get filename by parsing "a/ b/" + self._current_file = current_file[:(len(current_file) / 2)][2:] - def _Replace(self, line): - return re.sub("[a|b]/" + self._current_file, self._replacement_file, line) + def _Replace(self, line): + return re.sub("[a|b]/" + self._current_file, self._replacement_file, + line) # SCMWrapper base class + class SCMWrapper(object): - """Add necessary glue between all the supported SCM. + """Add necessary glue between all the supported SCM. This is the abstraction layer to bind to different SCM. """ - def __init__(self, url=None, root_dir=None, relpath=None, out_fh=None, - out_cb=None, print_outbuf=False): - self.url = url - self._root_dir = root_dir - if self._root_dir: - self._root_dir = self._root_dir.replace('/', os.sep) - self.relpath = relpath - if self.relpath: - self.relpath = self.relpath.replace('/', os.sep) - if self.relpath and self._root_dir: - self.checkout_path = os.path.join(self._root_dir, self.relpath) - if out_fh is None: - out_fh = sys.stdout - self.out_fh = out_fh - self.out_cb = out_cb - self.print_outbuf = print_outbuf + def __init__(self, + url=None, + root_dir=None, + relpath=None, + out_fh=None, + out_cb=None, + print_outbuf=False): + self.url = url + self._root_dir = root_dir + if self._root_dir: + self._root_dir = self._root_dir.replace('/', os.sep) + self.relpath = relpath + if self.relpath: + self.relpath = self.relpath.replace('/', os.sep) + if self.relpath and self._root_dir: + self.checkout_path = os.path.join(self._root_dir, self.relpath) + if out_fh is None: + out_fh = sys.stdout + self.out_fh = out_fh + self.out_cb = out_cb + self.print_outbuf = print_outbuf - def Print(self, *args, **kwargs): - kwargs.setdefault('file', self.out_fh) - if kwargs.pop('timestamp', True): - self.out_fh.write('[%s] ' % gclient_utils.Elapsed()) - print(*args, **kwargs) + def Print(self, *args, **kwargs): + kwargs.setdefault('file', self.out_fh) + if kwargs.pop('timestamp', True): + self.out_fh.write('[%s] ' % gclient_utils.Elapsed()) + print(*args, **kwargs) - def RunCommand(self, command, options, args, file_list=None): - commands = ['update', 'updatesingle', 'revert', - 'revinfo', 'status', 'diff', 'pack', 'runhooks'] + def RunCommand(self, command, options, args, file_list=None): + commands = [ + 'update', 'updatesingle', 'revert', 'revinfo', 'status', 'diff', + 'pack', 'runhooks' + ] - if not command in commands: - raise gclient_utils.Error('Unknown command %s' % command) + if not command in commands: + raise gclient_utils.Error('Unknown command %s' % command) - if not command in dir(self): - raise gclient_utils.Error('Command %s not implemented in %s wrapper' % ( - command, self.__class__.__name__)) + if not command in dir(self): + raise gclient_utils.Error( + 'Command %s not implemented in %s wrapper' % + (command, self.__class__.__name__)) - return getattr(self, command)(options, args, file_list) + return getattr(self, command)(options, args, file_list) - @staticmethod - def _get_first_remote_url(checkout_path): - log = scm.GIT.Capture( - ['config', '--local', '--get-regexp', r'remote.*.url'], - cwd=checkout_path) - # Get the second token of the first line of the log. - return log.splitlines()[0].split(' ', 1)[1] + @staticmethod + def _get_first_remote_url(checkout_path): + log = scm.GIT.Capture( + ['config', '--local', '--get-regexp', r'remote.*.url'], + cwd=checkout_path) + # Get the second token of the first line of the log. + return log.splitlines()[0].split(' ', 1)[1] - def GetCacheMirror(self): - if getattr(self, 'cache_dir', None): - url, _ = gclient_utils.SplitUrlRevision(self.url) - return git_cache.Mirror(url) - return None + def GetCacheMirror(self): + if getattr(self, 'cache_dir', None): + url, _ = gclient_utils.SplitUrlRevision(self.url) + return git_cache.Mirror(url) + return None - def GetActualRemoteURL(self, options): - """Attempt to determine the remote URL for this SCMWrapper.""" - # Git - if os.path.exists(os.path.join(self.checkout_path, '.git')): - actual_remote_url = self._get_first_remote_url(self.checkout_path) + def GetActualRemoteURL(self, options): + """Attempt to determine the remote URL for this SCMWrapper.""" + # Git + if os.path.exists(os.path.join(self.checkout_path, '.git')): + actual_remote_url = self._get_first_remote_url(self.checkout_path) - mirror = self.GetCacheMirror() - # If the cache is used, obtain the actual remote URL from there. - if (mirror and mirror.exists() and - mirror.mirror_path.replace('\\', '/') == - actual_remote_url.replace('\\', '/')): - actual_remote_url = self._get_first_remote_url(mirror.mirror_path) - return actual_remote_url - return None + mirror = self.GetCacheMirror() + # If the cache is used, obtain the actual remote URL from there. + if (mirror and mirror.exists() and mirror.mirror_path.replace( + '\\', '/') == actual_remote_url.replace('\\', '/')): + actual_remote_url = self._get_first_remote_url( + mirror.mirror_path) + return actual_remote_url + return None - def DoesRemoteURLMatch(self, options): - """Determine whether the remote URL of this checkout is the expected URL.""" - if not os.path.exists(self.checkout_path): - # A checkout which doesn't exist can't be broken. - return True + def DoesRemoteURLMatch(self, options): + """Determine whether the remote URL of this checkout is the expected URL.""" + if not os.path.exists(self.checkout_path): + # A checkout which doesn't exist can't be broken. + return True - actual_remote_url = self.GetActualRemoteURL(options) - if actual_remote_url: - return (gclient_utils.SplitUrlRevision(actual_remote_url)[0].rstrip('/') - == gclient_utils.SplitUrlRevision(self.url)[0].rstrip('/')) + actual_remote_url = self.GetActualRemoteURL(options) + if actual_remote_url: + return (gclient_utils.SplitUrlRevision(actual_remote_url)[0].rstrip( + '/') == gclient_utils.SplitUrlRevision(self.url)[0].rstrip('/')) - # This may occur if the self.checkout_path exists but does not contain a - # valid git checkout. - return False + # This may occur if the self.checkout_path exists but does not contain a + # valid git checkout. + return False - def _DeleteOrMove(self, force): - """Delete the checkout directory or move it out of the way. + def _DeleteOrMove(self, force): + """Delete the checkout directory or move it out of the way. Args: force: bool; if True, delete the directory. Otherwise, just move it. """ - if force and os.environ.get('CHROME_HEADLESS') == '1': - self.Print('_____ Conflicting directory found in %s. Removing.' - % self.checkout_path) - gclient_utils.AddWarning('Conflicting directory %s deleted.' - % self.checkout_path) - gclient_utils.rmtree(self.checkout_path) - else: - bad_scm_dir = os.path.join(self._root_dir, '_bad_scm', - os.path.dirname(self.relpath)) + if force and os.environ.get('CHROME_HEADLESS') == '1': + self.Print('_____ Conflicting directory found in %s. Removing.' % + self.checkout_path) + gclient_utils.AddWarning('Conflicting directory %s deleted.' % + self.checkout_path) + gclient_utils.rmtree(self.checkout_path) + else: + bad_scm_dir = os.path.join(self._root_dir, '_bad_scm', + os.path.dirname(self.relpath)) - try: - os.makedirs(bad_scm_dir) - except OSError as e: - if e.errno != errno.EEXIST: - raise + try: + os.makedirs(bad_scm_dir) + except OSError as e: + if e.errno != errno.EEXIST: + raise - dest_path = tempfile.mkdtemp( - prefix=os.path.basename(self.relpath), - dir=bad_scm_dir) - self.Print('_____ Conflicting directory found in %s. Moving to %s.' - % (self.checkout_path, dest_path)) - gclient_utils.AddWarning('Conflicting directory %s moved to %s.' - % (self.checkout_path, dest_path)) - shutil.move(self.checkout_path, dest_path) + dest_path = tempfile.mkdtemp(prefix=os.path.basename(self.relpath), + dir=bad_scm_dir) + self.Print( + '_____ Conflicting directory found in %s. Moving to %s.' % + (self.checkout_path, dest_path)) + gclient_utils.AddWarning('Conflicting directory %s moved to %s.' % + (self.checkout_path, dest_path)) + shutil.move(self.checkout_path, dest_path) class GitWrapper(SCMWrapper): - """Wrapper for Git""" - name = 'git' - remote = 'origin' + """Wrapper for Git""" + name = 'git' + remote = 'origin' - _is_env_cog = None + _is_env_cog = None - @staticmethod - def _IsCog(): - """Returns true if the env is cog""" - if not GitWrapper._is_env_cog: - GitWrapper._is_env_cog = any(os.getcwd().startswith(x) for x in [ - '/google/cog/cloud', '/google/src/cloud']) + @staticmethod + def _IsCog(): + """Returns true if the env is cog""" + if not GitWrapper._is_env_cog: + GitWrapper._is_env_cog = any( + os.getcwd().startswith(x) + for x in ['/google/cog/cloud', '/google/src/cloud']) - return GitWrapper._is_env_cog + return GitWrapper._is_env_cog - @property - def cache_dir(self): - try: - return git_cache.Mirror.GetCachePath() - except RuntimeError: - return None + @property + def cache_dir(self): + try: + return git_cache.Mirror.GetCachePath() + except RuntimeError: + return None - def __init__(self, url=None, *args, **kwargs): - """Removes 'git+' fake prefix from git URL.""" - if url and (url.startswith('git+http://') or - url.startswith('git+https://')): - url = url[4:] - SCMWrapper.__init__(self, url, *args, **kwargs) - filter_kwargs = { 'time_throttle': 1, 'out_fh': self.out_fh } - if self.out_cb: - filter_kwargs['predicate'] = self.out_cb - self.filter = gclient_utils.GitFilter(**filter_kwargs) - self._running_under_rosetta = None + def __init__(self, url=None, *args, **kwargs): + """Removes 'git+' fake prefix from git URL.""" + if url and (url.startswith('git+http://') + or url.startswith('git+https://')): + url = url[4:] + SCMWrapper.__init__(self, url, *args, **kwargs) + filter_kwargs = {'time_throttle': 1, 'out_fh': self.out_fh} + if self.out_cb: + filter_kwargs['predicate'] = self.out_cb + self.filter = gclient_utils.GitFilter(**filter_kwargs) + self._running_under_rosetta = None - def GetCheckoutRoot(self): - return scm.GIT.GetCheckoutRoot(self.checkout_path) + def GetCheckoutRoot(self): + return scm.GIT.GetCheckoutRoot(self.checkout_path) - def GetRevisionDate(self, _revision): - """Returns the given revision's date in ISO-8601 format (which contains the + def GetRevisionDate(self, _revision): + """Returns the given revision's date in ISO-8601 format (which contains the time zone).""" - # TODO(floitsch): get the time-stamp of the given revision and not just the - # time-stamp of the currently checked out revision. - return self._Capture(['log', '-n', '1', '--format=%ai']) + # TODO(floitsch): get the time-stamp of the given revision and not just + # the time-stamp of the currently checked out revision. + return self._Capture(['log', '-n', '1', '--format=%ai']) - def _GetDiffFilenames(self, base): - """Returns the names of files modified since base.""" - return self._Capture( - # Filter to remove base if it is None. - list(filter(bool, ['-c', 'core.quotePath=false', 'diff', '--name-only', - base]) - )).split() + def _GetDiffFilenames(self, base): + """Returns the names of files modified since base.""" + return self._Capture( + # Filter to remove base if it is None. + list( + filter( + bool, + ['-c', 'core.quotePath=false', 'diff', '--name-only', base]) + )).split() - def diff(self, options, _args, _file_list): - _, revision = gclient_utils.SplitUrlRevision(self.url) - if not revision: - revision = 'refs/remotes/%s/main' % self.remote - self._Run(['-c', 'core.quotePath=false', 'diff', revision], options) + def diff(self, options, _args, _file_list): + _, revision = gclient_utils.SplitUrlRevision(self.url) + if not revision: + revision = 'refs/remotes/%s/main' % self.remote + self._Run(['-c', 'core.quotePath=false', 'diff', revision], options) - def pack(self, _options, _args, _file_list): - """Generates a patch file which can be applied to the root of the + def pack(self, _options, _args, _file_list): + """Generates a patch file which can be applied to the root of the repository. The patch file is generated from a diff of the merge base of HEAD and its upstream branch. """ - try: - merge_base = [self._Capture(['merge-base', 'HEAD', self.remote])] - except subprocess2.CalledProcessError: - merge_base = [] - gclient_utils.CheckCallAndFilter( - ['git', 'diff'] + merge_base, - cwd=self.checkout_path, - filter_fn=GitDiffFilterer(self.relpath, print_func=self.Print).Filter) - - def _Scrub(self, target, options): - """Scrubs out all changes in the local repo, back to the state of target.""" - quiet = [] - if not options.verbose: - quiet = ['--quiet'] - self._Run(['reset', '--hard', target] + quiet, options) - if options.force and options.delete_unversioned_trees: - # where `target` is a commit that contains both upper and lower case - # versions of the same file on a case insensitive filesystem, we are - # actually in a broken state here. The index will have both 'a' and 'A', - # but only one of them will exist on the disk. To progress, we delete - # everything that status thinks is modified. - output = self._Capture([ - '-c', 'core.quotePath=false', 'status', '--porcelain'], strip=False) - for line in output.splitlines(): - # --porcelain (v1) looks like: - # XY filename try: - filename = line[3:] - self.Print('_____ Deleting residual after reset: %r.' % filename) - gclient_utils.rm_file_or_tree( - os.path.join(self.checkout_path, filename)) - except OSError: - pass + merge_base = [self._Capture(['merge-base', 'HEAD', self.remote])] + except subprocess2.CalledProcessError: + merge_base = [] + gclient_utils.CheckCallAndFilter(['git', 'diff'] + merge_base, + cwd=self.checkout_path, + filter_fn=GitDiffFilterer( + self.relpath, + print_func=self.Print).Filter) - def _FetchAndReset(self, revision, file_list, options): - """Equivalent to git fetch; git reset.""" - self._SetFetchConfig(options) + def _Scrub(self, target, options): + """Scrubs out all changes in the local repo, back to the state of target.""" + quiet = [] + if not options.verbose: + quiet = ['--quiet'] + self._Run(['reset', '--hard', target] + quiet, options) + if options.force and options.delete_unversioned_trees: + # where `target` is a commit that contains both upper and lower case + # versions of the same file on a case insensitive filesystem, we are + # actually in a broken state here. The index will have both 'a' and + # 'A', but only one of them will exist on the disk. To progress, we + # delete everything that status thinks is modified. + output = self._Capture( + ['-c', 'core.quotePath=false', 'status', '--porcelain'], + strip=False) + for line in output.splitlines(): + # --porcelain (v1) looks like: + # XY filename + try: + filename = line[3:] + self.Print('_____ Deleting residual after reset: %r.' % + filename) + gclient_utils.rm_file_or_tree( + os.path.join(self.checkout_path, filename)) + except OSError: + pass - self._Fetch(options, prune=True, quiet=options.verbose) - self._Scrub(revision, options) - if file_list is not None: - files = self._Capture( - ['-c', 'core.quotePath=false', 'ls-files']).splitlines() - file_list.extend( - [os.path.join(self.checkout_path, f) for f in files]) + def _FetchAndReset(self, revision, file_list, options): + """Equivalent to git fetch; git reset.""" + self._SetFetchConfig(options) - def _DisableHooks(self): - hook_dir = os.path.join(self.checkout_path, '.git', 'hooks') - if not os.path.isdir(hook_dir): - return - for f in os.listdir(hook_dir): - if not f.endswith('.sample') and not f.endswith('.disabled'): - disabled_hook_path = os.path.join(hook_dir, f + '.disabled') - if os.path.exists(disabled_hook_path): - os.remove(disabled_hook_path) - os.rename(os.path.join(hook_dir, f), disabled_hook_path) + self._Fetch(options, prune=True, quiet=options.verbose) + self._Scrub(revision, options) + if file_list is not None: + files = self._Capture(['-c', 'core.quotePath=false', + 'ls-files']).splitlines() + file_list.extend( + [os.path.join(self.checkout_path, f) for f in files]) - def _maybe_break_locks(self, options): - """This removes all .lock files from this repo's .git directory, if the + def _DisableHooks(self): + hook_dir = os.path.join(self.checkout_path, '.git', 'hooks') + if not os.path.isdir(hook_dir): + return + for f in os.listdir(hook_dir): + if not f.endswith('.sample') and not f.endswith('.disabled'): + disabled_hook_path = os.path.join(hook_dir, f + '.disabled') + if os.path.exists(disabled_hook_path): + os.remove(disabled_hook_path) + os.rename(os.path.join(hook_dir, f), disabled_hook_path) + + def _maybe_break_locks(self, options): + """This removes all .lock files from this repo's .git directory, if the user passed the --break_repo_locks command line flag. In particular, this will cleanup index.lock files, as well as ref lock files. """ - if options.break_repo_locks: - git_dir = os.path.join(self.checkout_path, '.git') - for path, _, filenames in os.walk(git_dir): - for filename in filenames: - if filename.endswith('.lock'): - to_break = os.path.join(path, filename) - self.Print('breaking lock: %s' % (to_break,)) - try: - os.remove(to_break) - except OSError as ex: - self.Print('FAILED to break lock: %s: %s' % (to_break, ex)) - raise + if options.break_repo_locks: + git_dir = os.path.join(self.checkout_path, '.git') + for path, _, filenames in os.walk(git_dir): + for filename in filenames: + if filename.endswith('.lock'): + to_break = os.path.join(path, filename) + self.Print('breaking lock: %s' % (to_break, )) + try: + os.remove(to_break) + except OSError as ex: + self.Print('FAILED to break lock: %s: %s' % + (to_break, ex)) + raise - def _download_topics(self, patch_rev, googlesource_url): - """This method returns new patch_revs to process that have the same topic. + def _download_topics(self, patch_rev, googlesource_url): + """This method returns new patch_revs to process that have the same topic. It does the following: 1. Finds the topic of the Gerrit change specified in the patch_rev. @@ -349,50 +368,50 @@ class GitWrapper(SCMWrapper): to process. 4. Returns the new patch_revs to process. """ - patch_revs_to_process = [] - # Parse the patch_rev to extract the CL and patchset. - patch_rev_tokens = patch_rev.split('/') - change = patch_rev_tokens[-2] - # Parse the googlesource_url. - tokens = re.search( - '//(.+).googlesource.com/(.+?)(?:\.git)?$', googlesource_url) - if not tokens or len(tokens.groups()) != 2: - # googlesource_url is not in the expected format. - return patch_revs_to_process + patch_revs_to_process = [] + # Parse the patch_rev to extract the CL and patchset. + patch_rev_tokens = patch_rev.split('/') + change = patch_rev_tokens[-2] + # Parse the googlesource_url. + tokens = re.search('//(.+).googlesource.com/(.+?)(?:\.git)?$', + googlesource_url) + if not tokens or len(tokens.groups()) != 2: + # googlesource_url is not in the expected format. + return patch_revs_to_process - # parse the gerrit host and repo out of googlesource_url. - host, repo = tokens.groups()[:2] - gerrit_host_url = '%s-review.googlesource.com' % host + # parse the gerrit host and repo out of googlesource_url. + host, repo = tokens.groups()[:2] + gerrit_host_url = '%s-review.googlesource.com' % host - # 1. Find the topic of the Gerrit change specified in the patch_rev. - change_object = gerrit_util.GetChange(gerrit_host_url, change) - topic = change_object.get('topic') - if not topic: - # This change has no topic set. - return patch_revs_to_process + # 1. Find the topic of the Gerrit change specified in the patch_rev. + change_object = gerrit_util.GetChange(gerrit_host_url, change) + topic = change_object.get('topic') + if not topic: + # This change has no topic set. + return patch_revs_to_process - # 2. Find all changes with that topic. - changes_with_same_topic = gerrit_util.QueryChanges( - gerrit_host_url, - [('topic', topic), ('status', 'open'), ('repo', repo)], - o_params=['ALL_REVISIONS']) - for c in changes_with_same_topic: - if str(c['_number']) == change: - # This change is already in the patch_rev. - continue - self.Print('Found CL %d with the topic name %s' % ( - c['_number'], topic)) - # 3. Append patch_rev of the changes with the same topic to the - # patch_revs to process. - curr_rev = c['current_revision'] - new_patch_rev = c['revisions'][curr_rev]['ref'] - patch_revs_to_process.append(new_patch_rev) + # 2. Find all changes with that topic. + changes_with_same_topic = gerrit_util.QueryChanges( + gerrit_host_url, [('topic', topic), ('status', 'open'), + ('repo', repo)], + o_params=['ALL_REVISIONS']) + for c in changes_with_same_topic: + if str(c['_number']) == change: + # This change is already in the patch_rev. + continue + self.Print('Found CL %d with the topic name %s' % + (c['_number'], topic)) + # 3. Append patch_rev of the changes with the same topic to the + # patch_revs to process. + curr_rev = c['current_revision'] + new_patch_rev = c['revisions'][curr_rev]['ref'] + patch_revs_to_process.append(new_patch_rev) - # 4. Return the new patch_revs to process. - return patch_revs_to_process + # 4. Return the new patch_revs to process. + return patch_revs_to_process - def _ref_to_remote_ref(self, target_rev): - """Helper function for scm.GIT.RefToRemoteRef with error checking. + def _ref_to_remote_ref(self, target_rev): + """Helper function for scm.GIT.RefToRemoteRef with error checking. Joins the results of scm.GIT.RefToRemoteRef into a string, but raises a comprehensible error if RefToRemoteRef fails. @@ -400,17 +419,17 @@ class GitWrapper(SCMWrapper): Args: target_rev: a ref somewhere under refs/. """ - tmp_ref = scm.GIT.RefToRemoteRef(target_rev, self.remote) - if not tmp_ref: - raise gclient_utils.Error( - 'Failed to turn target revision %r in repo %r into remote ref' % - (target_rev, self.checkout_path)) - return ''.join(tmp_ref) + tmp_ref = scm.GIT.RefToRemoteRef(target_rev, self.remote) + if not tmp_ref: + raise gclient_utils.Error( + 'Failed to turn target revision %r in repo %r into remote ref' % + (target_rev, self.checkout_path)) + return ''.join(tmp_ref) - def apply_patch_ref(self, patch_repo, patch_rev, target_rev, options, - file_list): - # type: (str, str, str, optparse.Values, Collection[str]) -> str - """Apply a patch on top of the revision we're synced at. + def apply_patch_ref(self, patch_repo, patch_rev, target_rev, options, + file_list): + # type: (str, str, str, optparse.Values, Collection[str]) -> str + """Apply a patch on top of the revision we're synced at. The patch ref is given by |patch_repo|@|patch_rev|. |target_rev| is usually the branch that the |patch_rev| was uploaded against @@ -445,1007 +464,1075 @@ class GitWrapper(SCMWrapper): file_list: A list where modified files will be appended. """ - # Abort any cherry-picks in progress. - try: - self._Capture(['cherry-pick', '--abort']) - except subprocess2.CalledProcessError: - pass - - base_rev = self.revinfo(None, None, None) - - if not target_rev: - raise gclient_utils.Error('A target revision for the patch must be given') - - if target_rev.startswith(('refs/heads/', 'refs/branch-heads')): - # If |target_rev| is in refs/heads/** or refs/branch-heads/**, try first - # to find the corresponding remote ref for it, since |target_rev| might - # point to a local ref which is not up to date with the corresponding - # remote ref. - remote_ref = self._ref_to_remote_ref(target_rev) - self.Print('Trying the corresponding remote ref for %r: %r\n' % ( - target_rev, remote_ref)) - if scm.GIT.IsValidRevision(self.checkout_path, remote_ref): - # refs/remotes may need to be updated to cleanly cherry-pick changes. - # See https://crbug.com/1255178. - self._Capture(['fetch', '--no-tags', self.remote, target_rev]) - target_rev = remote_ref - elif not scm.GIT.IsValidRevision(self.checkout_path, target_rev): - # Fetch |target_rev| if it's not already available. - url, _ = gclient_utils.SplitUrlRevision(self.url) - mirror = self._GetMirror(url, options, target_rev, target_rev) - if mirror: - rev_type = 'branch' if target_rev.startswith('refs/') else 'hash' - self._UpdateMirrorIfNotContains(mirror, options, rev_type, target_rev) - self._Fetch(options, refspec=target_rev) - - patch_revs_to_process = [patch_rev] - - if hasattr(options, 'download_topics') and options.download_topics: - patch_revs_to_process_from_topics = self._download_topics( - patch_rev, self.url) - patch_revs_to_process.extend(patch_revs_to_process_from_topics) - - self._Capture(['reset', '--hard']) - for pr in patch_revs_to_process: - self.Print('===Applying patch===') - self.Print('Revision to patch is %r @ %r.' % (patch_repo, pr)) - self.Print('Current dir is %r' % self.checkout_path) - self._Capture(['fetch', '--no-tags', patch_repo, pr]) - pr = self._Capture(['rev-parse', 'FETCH_HEAD']) - - if not options.rebase_patch_ref: - self._Capture(['checkout', pr]) - # Adjust base_rev to be the first parent of our checked out patch ref; - # This will allow us to correctly extend `file_list`, and will show the - # correct file-list to programs which do `git diff --cached` expecting - # to see the patch diff. - base_rev = self._Capture(['rev-parse', pr+'~']) - else: - self.Print('Will cherrypick %r .. %r on top of %r.' % ( - target_rev, pr, base_rev)) + # Abort any cherry-picks in progress. try: - if scm.GIT.IsAncestor(pr, target_rev, cwd=self.checkout_path): - if len(patch_revs_to_process) > 1: - # If there are multiple patch_revs_to_process then we do not want - # want to invalidate a previous patch so throw an error. - raise gclient_utils.Error( - 'patch_rev %s is an ancestor of target_rev %s. This ' - 'situation is unsupported when we need to apply multiple ' - 'patch_revs: %s' % (pr, target_rev, patch_revs_to_process)) - # If |patch_rev| is an ancestor of |target_rev|, check it out. - self._Capture(['checkout', pr]) - else: - # If a change was uploaded on top of another change, which has - # already landed, one of the commits in the cherry-pick range will - # be redundant, since it has already landed and its changes - # incorporated in the tree. - # We pass '--keep-redundant-commits' to ignore those changes. - self._Capture(['cherry-pick', target_rev + '..' + pr, - '--keep-redundant-commits']) - - except subprocess2.CalledProcessError as e: - self.Print('Failed to apply patch.') - self.Print('Revision to patch was %r @ %r.' % (patch_repo, pr)) - self.Print('Tried to cherrypick %r .. %r on top of %r.' % ( - target_rev, pr, base_rev)) - self.Print('Current dir is %r' % self.checkout_path) - self.Print('git returned non-zero exit status %s:\n%s' % ( - e.returncode, e.stderr.decode('utf-8'))) - # Print the current status so that developers know what changes caused - # the patch failure, since git cherry-pick doesn't show that - # information. - self.Print(self._Capture(['status'])) - try: self._Capture(['cherry-pick', '--abort']) - except subprocess2.CalledProcessError: + except subprocess2.CalledProcessError: pass - raise - if file_list is not None: - file_list.extend(self._GetDiffFilenames(base_rev)) + base_rev = self.revinfo(None, None, None) - latest_commit = self.revinfo(None, None, None) - if options.reset_patch_ref: - self._Capture(['reset', '--soft', base_rev]) - return latest_commit + if not target_rev: + raise gclient_utils.Error( + 'A target revision for the patch must be given') - def check_diff(self, previous_commit, files=None): - # type: (str, Optional[List[str]]) -> bool - """Check if a diff exists between the current commit and `previous_commit`. + if target_rev.startswith(('refs/heads/', 'refs/branch-heads')): + # If |target_rev| is in refs/heads/** or refs/branch-heads/**, try + # first to find the corresponding remote ref for it, since + # |target_rev| might point to a local ref which is not up to date + # with the corresponding remote ref. + remote_ref = self._ref_to_remote_ref(target_rev) + self.Print('Trying the corresponding remote ref for %r: %r\n' % + (target_rev, remote_ref)) + if scm.GIT.IsValidRevision(self.checkout_path, remote_ref): + # refs/remotes may need to be updated to cleanly cherry-pick + # changes. See https://crbug.com/1255178. + self._Capture(['fetch', '--no-tags', self.remote, target_rev]) + target_rev = remote_ref + elif not scm.GIT.IsValidRevision(self.checkout_path, target_rev): + # Fetch |target_rev| if it's not already available. + url, _ = gclient_utils.SplitUrlRevision(self.url) + mirror = self._GetMirror(url, options, target_rev, target_rev) + if mirror: + rev_type = 'branch' if target_rev.startswith( + 'refs/') else 'hash' + self._UpdateMirrorIfNotContains(mirror, options, rev_type, + target_rev) + self._Fetch(options, refspec=target_rev) + + patch_revs_to_process = [patch_rev] + + if hasattr(options, 'download_topics') and options.download_topics: + patch_revs_to_process_from_topics = self._download_topics( + patch_rev, self.url) + patch_revs_to_process.extend(patch_revs_to_process_from_topics) + + self._Capture(['reset', '--hard']) + for pr in patch_revs_to_process: + self.Print('===Applying patch===') + self.Print('Revision to patch is %r @ %r.' % (patch_repo, pr)) + self.Print('Current dir is %r' % self.checkout_path) + self._Capture(['fetch', '--no-tags', patch_repo, pr]) + pr = self._Capture(['rev-parse', 'FETCH_HEAD']) + + if not options.rebase_patch_ref: + self._Capture(['checkout', pr]) + # Adjust base_rev to be the first parent of our checked out + # patch ref; This will allow us to correctly extend `file_list`, + # and will show the correct file-list to programs which do `git + # diff --cached` expecting to see the patch diff. + base_rev = self._Capture(['rev-parse', pr + '~']) + else: + self.Print('Will cherrypick %r .. %r on top of %r.' % + (target_rev, pr, base_rev)) + try: + if scm.GIT.IsAncestor(pr, + target_rev, + cwd=self.checkout_path): + if len(patch_revs_to_process) > 1: + # If there are multiple patch_revs_to_process then + # we do not want want to invalidate a previous patch + # so throw an error. + raise gclient_utils.Error( + 'patch_rev %s is an ancestor of target_rev %s. This ' + 'situation is unsupported when we need to apply multiple ' + 'patch_revs: %s' % + (pr, target_rev, patch_revs_to_process)) + # If |patch_rev| is an ancestor of |target_rev|, check + # it out. + self._Capture(['checkout', pr]) + else: + # If a change was uploaded on top of another change, + # which has already landed, one of the commits in the + # cherry-pick range will be redundant, since it has + # already landed and its changes incorporated in the + # tree. We pass '--keep-redundant-commits' to ignore + # those changes. + self._Capture([ + 'cherry-pick', target_rev + '..' + pr, + '--keep-redundant-commits' + ]) + + except subprocess2.CalledProcessError as e: + self.Print('Failed to apply patch.') + self.Print('Revision to patch was %r @ %r.' % + (patch_repo, pr)) + self.Print('Tried to cherrypick %r .. %r on top of %r.' % + (target_rev, pr, base_rev)) + self.Print('Current dir is %r' % self.checkout_path) + self.Print('git returned non-zero exit status %s:\n%s' % + (e.returncode, e.stderr.decode('utf-8'))) + # Print the current status so that developers know what + # changes caused the patch failure, since git cherry-pick + # doesn't show that information. + self.Print(self._Capture(['status'])) + try: + self._Capture(['cherry-pick', '--abort']) + except subprocess2.CalledProcessError: + pass + raise + + if file_list is not None: + file_list.extend(self._GetDiffFilenames(base_rev)) + + latest_commit = self.revinfo(None, None, None) + if options.reset_patch_ref: + self._Capture(['reset', '--soft', base_rev]) + return latest_commit + + def check_diff(self, previous_commit, files=None): + # type: (str, Optional[List[str]]) -> bool + """Check if a diff exists between the current commit and `previous_commit`. Returns True if there were diffs or if an error was encountered. """ - cmd = ['diff', previous_commit, '--quiet'] - if files: - cmd += ['--'] + files - try: - self._Capture(cmd) - return False - except subprocess2.CalledProcessError as e: - # git diff --quiet exits with 1 if there were diffs. - if e.returncode != 1: - self.Print('git returned non-zero exit status %s:\n%s' % - (e.returncode, e.stderr.decode('utf-8'))) - return True + cmd = ['diff', previous_commit, '--quiet'] + if files: + cmd += ['--'] + files + try: + self._Capture(cmd) + return False + except subprocess2.CalledProcessError as e: + # git diff --quiet exits with 1 if there were diffs. + if e.returncode != 1: + self.Print('git returned non-zero exit status %s:\n%s' % + (e.returncode, e.stderr.decode('utf-8'))) + return True - def set_config(f): - def wrapper(*args): - return_val = f(*args) - if os.path.exists(os.path.join(args[0].checkout_path, '.git')): - # If diff.ignoreSubmodules is not already set, set it to `all`. - config = subprocess2.capture( - ['git', 'config', '-l'], - cwd=args[0].checkout_path).decode('utf-8').strip().splitlines() - if 'diff.ignoresubmodules=dirty' not in config: - subprocess2.capture( - ['git', 'config', 'diff.ignoreSubmodules', 'dirty'], - cwd=args[0].checkout_path) - if 'fetch.recursesubmodules=off' not in config: - subprocess2.capture( - ['git', 'config', 'fetch.recurseSubmodules', 'off'], - cwd=args[0].checkout_path) - return return_val + def set_config(f): + def wrapper(*args): + return_val = f(*args) + if os.path.exists(os.path.join(args[0].checkout_path, '.git')): + # If diff.ignoreSubmodules is not already set, set it to `all`. + config = subprocess2.capture(['git', 'config', '-l'], + cwd=args[0].checkout_path).decode( + 'utf-8').strip().splitlines() + if 'diff.ignoresubmodules=dirty' not in config: + subprocess2.capture( + ['git', 'config', 'diff.ignoreSubmodules', 'dirty'], + cwd=args[0].checkout_path) + if 'fetch.recursesubmodules=off' not in config: + subprocess2.capture( + ['git', 'config', 'fetch.recurseSubmodules', 'off'], + cwd=args[0].checkout_path) + return return_val - return wrapper + return wrapper - @set_config - def update(self, options, args, file_list): - """Runs git to update or transparently checkout the working copy. + @set_config + def update(self, options, args, file_list): + """Runs git to update or transparently checkout the working copy. All updated files will be appended to file_list. Raises: Error: if can't get URL for relative path. """ - if args: - raise gclient_utils.Error("Unsupported argument(s): %s" % ",".join(args)) + if args: + raise gclient_utils.Error("Unsupported argument(s): %s" % + ",".join(args)) - self._CheckMinVersion("1.6.6") + self._CheckMinVersion("1.6.6") - url, deps_revision = gclient_utils.SplitUrlRevision(self.url) - revision = deps_revision - managed = True - if options.revision: - # Override the revision number. - revision = str(options.revision) - if revision == 'unmanaged': - # Check again for a revision in case an initial ref was specified - # in the url, for example bla.git@refs/heads/custombranch - revision = deps_revision - managed = False - if not revision: - # If a dependency is not pinned, track the default remote branch. - revision = scm.GIT.GetRemoteHeadRef(self.checkout_path, self.url, - self.remote) - if revision.startswith('origin/'): - revision = 'refs/remotes/' + revision + url, deps_revision = gclient_utils.SplitUrlRevision(self.url) + revision = deps_revision + managed = True + if options.revision: + # Override the revision number. + revision = str(options.revision) + if revision == 'unmanaged': + # Check again for a revision in case an initial ref was specified + # in the url, for example bla.git@refs/heads/custombranch + revision = deps_revision + managed = False + if not revision: + # If a dependency is not pinned, track the default remote branch. + revision = scm.GIT.GetRemoteHeadRef(self.checkout_path, self.url, + self.remote) + if revision.startswith('origin/'): + revision = 'refs/remotes/' + revision - if managed and platform.system() == 'Windows': - self._DisableHooks() + if managed and platform.system() == 'Windows': + self._DisableHooks() - printed_path = False - verbose = [] - if options.verbose: - self.Print('_____ %s at %s' % (self.relpath, revision), timestamp=False) - verbose = ['--verbose'] - printed_path = True + printed_path = False + verbose = [] + if options.verbose: + self.Print('_____ %s at %s' % (self.relpath, revision), + timestamp=False) + verbose = ['--verbose'] + printed_path = True - revision_ref = revision - if ':' in revision: - revision_ref, _, revision = revision.partition(':') + revision_ref = revision + if ':' in revision: + revision_ref, _, revision = revision.partition(':') - if revision_ref.startswith('refs/branch-heads'): - options.with_branch_heads = True + if revision_ref.startswith('refs/branch-heads'): + options.with_branch_heads = True - mirror = self._GetMirror(url, options, revision, revision_ref) - if mirror: - url = mirror.mirror_path + mirror = self._GetMirror(url, options, revision, revision_ref) + if mirror: + url = mirror.mirror_path - remote_ref = scm.GIT.RefToRemoteRef(revision, self.remote) - if remote_ref: - # Rewrite remote refs to their local equivalents. - revision = ''.join(remote_ref) - rev_type = "branch" - elif revision.startswith('refs/'): - # Local branch? We probably don't want to support, since DEPS should - # always specify branches as they are in the upstream repo. - rev_type = "branch" - else: - # hash is also a tag, only make a distinction at checkout - rev_type = "hash" - - # If we are going to introduce a new project, there is a possibility that - # we are syncing back to a state where the project was originally a - # sub-project rolled by DEPS (realistic case: crossing the Blink merge point - # syncing backwards, when Blink was a DEPS entry and not part of src.git). - # In such case, we might have a backup of the former .git folder, which can - # be used to avoid re-fetching the entire repo again (useful for bisects). - backup_dir = self.GetGitBackupDirPath() - target_dir = os.path.join(self.checkout_path, '.git') - if os.path.exists(backup_dir) and not os.path.exists(target_dir): - gclient_utils.safe_makedirs(self.checkout_path) - os.rename(backup_dir, target_dir) - # Reset to a clean state - self._Scrub('HEAD', options) - - if (not os.path.exists(self.checkout_path) or - (os.path.isdir(self.checkout_path) and - not os.path.exists(os.path.join(self.checkout_path, '.git')))): - if mirror: - self._UpdateMirrorIfNotContains(mirror, options, rev_type, revision) - try: - self._Clone(revision, url, options) - except subprocess2.CalledProcessError as e: - logging.warning('Clone failed due to: %s', e) - self._DeleteOrMove(options.force) - self._Clone(revision, url, options) - if file_list is not None: - files = self._Capture( - ['-c', 'core.quotePath=false', 'ls-files']).splitlines() - file_list.extend( - [os.path.join(self.checkout_path, f) for f in files]) - if mirror: - self._Capture( - ['remote', 'set-url', '--push', 'origin', mirror.url]) - if not verbose: - # Make the output a little prettier. It's nice to have some whitespace - # between projects when cloning. - self.Print('') - return self._Capture(['rev-parse', '--verify', 'HEAD']) - - if mirror: - self._Capture( - ['remote', 'set-url', '--push', 'origin', mirror.url]) - - if not managed: - self._SetFetchConfig(options) - self.Print('________ unmanaged solution; skipping %s' % self.relpath) - return self._Capture(['rev-parse', '--verify', 'HEAD']) - - self._maybe_break_locks(options) - - if mirror: - self._UpdateMirrorIfNotContains(mirror, options, rev_type, revision) - - # See if the url has changed (the unittests use git://foo for the url, let - # that through). - current_url = self._Capture(['config', 'remote.%s.url' % self.remote]) - return_early = False - # TODO(maruel): Delete url != 'git://foo' since it's just to make the - # unit test pass. (and update the comment above) - # Skip url auto-correction if remote.origin.gclient-auto-fix-url is set. - # This allows devs to use experimental repos which have a different url - # but whose branch(s) are the same as official repos. - if (current_url.rstrip('/') != url.rstrip('/') and url != 'git://foo' and - subprocess2.capture( - ['git', 'config', 'remote.%s.gclient-auto-fix-url' % self.remote], - cwd=self.checkout_path).strip() != 'False'): - self.Print('_____ switching %s from %s to new upstream %s' % ( - self.relpath, current_url, url)) - if not (options.force or options.reset): - # Make sure it's clean - self._CheckClean(revision) - # Switch over to the new upstream - self._Run(['remote', 'set-url', self.remote, url], options) - if mirror: - if git_cache.Mirror.CacheDirToUrl( - current_url.rstrip('/')) == git_cache.Mirror.CacheDirToUrl( - url.rstrip('/')): - # Reset alternates when the cache dir is updated. - with open( - os.path.join(self.checkout_path, '.git', 'objects', 'info', - 'alternates'), 'w') as fh: - fh.write(os.path.join(url, 'objects')) + remote_ref = scm.GIT.RefToRemoteRef(revision, self.remote) + if remote_ref: + # Rewrite remote refs to their local equivalents. + revision = ''.join(remote_ref) + rev_type = "branch" + elif revision.startswith('refs/'): + # Local branch? We probably don't want to support, since DEPS should + # always specify branches as they are in the upstream repo. + rev_type = "branch" else: - # Because we use Git alternatives, our existing repository is not - # self-contained. It's possible that new git alternative doesn't have - # all necessary objects that the current repository needs. Instead of - # blindly hoping that new alternative contains all necessary objects, - # keep the old alternative and just append a new one on top of it. - with open( - os.path.join(self.checkout_path, '.git', 'objects', 'info', - 'alternates'), 'a') as fh: - fh.write("\n" + os.path.join(url, 'objects')) - self._EnsureValidHeadObjectOrCheckout(revision, options, url) - self._FetchAndReset(revision, file_list, options) + # hash is also a tag, only make a distinction at checkout + rev_type = "hash" - return_early = True - else: - self._EnsureValidHeadObjectOrCheckout(revision, options, url) + # If we are going to introduce a new project, there is a possibility + # that we are syncing back to a state where the project was originally a + # sub-project rolled by DEPS (realistic case: crossing the Blink merge + # point syncing backwards, when Blink was a DEPS entry and not part of + # src.git). In such case, we might have a backup of the former .git + # folder, which can be used to avoid re-fetching the entire repo again + # (useful for bisects). + backup_dir = self.GetGitBackupDirPath() + target_dir = os.path.join(self.checkout_path, '.git') + if os.path.exists(backup_dir) and not os.path.exists(target_dir): + gclient_utils.safe_makedirs(self.checkout_path) + os.rename(backup_dir, target_dir) + # Reset to a clean state + self._Scrub('HEAD', options) - if return_early: - return self._Capture(['rev-parse', '--verify', 'HEAD']) + if (not os.path.exists(self.checkout_path) or + (os.path.isdir(self.checkout_path) + and not os.path.exists(os.path.join(self.checkout_path, '.git')))): + if mirror: + self._UpdateMirrorIfNotContains(mirror, options, rev_type, + revision) + try: + self._Clone(revision, url, options) + except subprocess2.CalledProcessError as e: + logging.warning('Clone failed due to: %s', e) + self._DeleteOrMove(options.force) + self._Clone(revision, url, options) + if file_list is not None: + files = self._Capture( + ['-c', 'core.quotePath=false', 'ls-files']).splitlines() + file_list.extend( + [os.path.join(self.checkout_path, f) for f in files]) + if mirror: + self._Capture( + ['remote', 'set-url', '--push', 'origin', mirror.url]) + if not verbose: + # Make the output a little prettier. It's nice to have some + # whitespace between projects when cloning. + self.Print('') + return self._Capture(['rev-parse', '--verify', 'HEAD']) - cur_branch = self._GetCurrentBranch() + if mirror: + self._Capture(['remote', 'set-url', '--push', 'origin', mirror.url]) - # Cases: - # 0) HEAD is detached. Probably from our initial clone. - # - make sure HEAD is contained by a named ref, then update. - # Cases 1-4. HEAD is a branch. - # 1) current branch is not tracking a remote branch - # - try to rebase onto the new hash or branch - # 2) current branch is tracking a remote branch with local committed - # changes, but the DEPS file switched to point to a hash - # - rebase those changes on top of the hash - # 3) current branch is tracking a remote branch w/or w/out changes, and - # no DEPS switch - # - see if we can FF, if not, prompt the user for rebase, merge, or stop - # 4) current branch is tracking a remote branch, but DEPS switches to a - # different remote branch, and - # a) current branch has no local changes, and --force: - # - checkout new branch - # b) current branch has local changes, and --force and --reset: - # - checkout new branch - # c) otherwise exit + if not managed: + self._SetFetchConfig(options) + self.Print('________ unmanaged solution; skipping %s' % + self.relpath) + return self._Capture(['rev-parse', '--verify', 'HEAD']) - # GetUpstreamBranch returns something like 'refs/remotes/origin/main' for - # a tracking branch - # or 'main' if not a tracking branch (it's based on a specific rev/hash) - # or it returns None if it couldn't find an upstream - if cur_branch is None: - upstream_branch = None - current_type = "detached" - logging.debug("Detached HEAD") - else: - upstream_branch = scm.GIT.GetUpstreamBranch(self.checkout_path) - if not upstream_branch or not upstream_branch.startswith('refs/remotes'): - current_type = "hash" - logging.debug("Current branch is not tracking an upstream (remote)" - " branch.") - elif upstream_branch.startswith('refs/remotes'): - current_type = "branch" - else: - raise gclient_utils.Error('Invalid Upstream: %s' % upstream_branch) + self._maybe_break_locks(options) - self._SetFetchConfig(options) + if mirror: + self._UpdateMirrorIfNotContains(mirror, options, rev_type, revision) - # Fetch upstream if we don't already have |revision|. - if not scm.GIT.IsValidRevision(self.checkout_path, revision, sha_only=True): - self._Fetch(options, prune=options.force) + # See if the url has changed (the unittests use git://foo for the url, + # let that through). + current_url = self._Capture(['config', 'remote.%s.url' % self.remote]) + return_early = False + # TODO(maruel): Delete url != 'git://foo' since it's just to make the + # unit test pass. (and update the comment above) + # Skip url auto-correction if remote.origin.gclient-auto-fix-url is set. + # This allows devs to use experimental repos which have a different url + # but whose branch(s) are the same as official repos. + if (current_url.rstrip('/') != url.rstrip('/') and url != 'git://foo' + and + subprocess2.capture([ + 'git', 'config', + 'remote.%s.gclient-auto-fix-url' % self.remote + ], + cwd=self.checkout_path).strip() != 'False'): + self.Print('_____ switching %s from %s to new upstream %s' % + (self.relpath, current_url, url)) + if not (options.force or options.reset): + # Make sure it's clean + self._CheckClean(revision) + # Switch over to the new upstream + self._Run(['remote', 'set-url', self.remote, url], options) + if mirror: + if git_cache.Mirror.CacheDirToUrl(current_url.rstrip( + '/')) == git_cache.Mirror.CacheDirToUrl( + url.rstrip('/')): + # Reset alternates when the cache dir is updated. + with open( + os.path.join(self.checkout_path, '.git', 'objects', + 'info', 'alternates'), 'w') as fh: + fh.write(os.path.join(url, 'objects')) + else: + # Because we use Git alternatives, our existing repository + # is not self-contained. It's possible that new git + # alternative doesn't have all necessary objects that the + # current repository needs. Instead of blindly hoping that + # new alternative contains all necessary objects, keep the + # old alternative and just append a new one on top of it. + with open( + os.path.join(self.checkout_path, '.git', 'objects', + 'info', 'alternates'), 'a') as fh: + fh.write("\n" + os.path.join(url, 'objects')) + self._EnsureValidHeadObjectOrCheckout(revision, options, url) + self._FetchAndReset(revision, file_list, options) + + return_early = True + else: + self._EnsureValidHeadObjectOrCheckout(revision, options, url) + + if return_early: + return self._Capture(['rev-parse', '--verify', 'HEAD']) + + cur_branch = self._GetCurrentBranch() + + # Cases: + # 0) HEAD is detached. Probably from our initial clone. + # - make sure HEAD is contained by a named ref, then update. + # Cases 1-4. HEAD is a branch. + # 1) current branch is not tracking a remote branch + # - try to rebase onto the new hash or branch + # 2) current branch is tracking a remote branch with local committed + # changes, but the DEPS file switched to point to a hash + # - rebase those changes on top of the hash + # 3) current branch is tracking a remote branch w/or w/out changes, and + # no DEPS switch + # - see if we can FF, if not, prompt the user for rebase, merge, or stop + # 4) current branch is tracking a remote branch, but DEPS switches to a + # different remote branch, and a) current branch has no local changes, + # and --force: - checkout new branch b) current branch has local + # changes, and --force and --reset: - checkout new branch c) otherwise + # exit + + # GetUpstreamBranch returns something like 'refs/remotes/origin/main' + # for a tracking branch or 'main' if not a tracking branch (it's based + # on a specific rev/hash) or it returns None if it couldn't find an + # upstream + if cur_branch is None: + upstream_branch = None + current_type = "detached" + logging.debug("Detached HEAD") + else: + upstream_branch = scm.GIT.GetUpstreamBranch(self.checkout_path) + if not upstream_branch or not upstream_branch.startswith( + 'refs/remotes'): + current_type = "hash" + logging.debug( + "Current branch is not tracking an upstream (remote)" + " branch.") + elif upstream_branch.startswith('refs/remotes'): + current_type = "branch" + else: + raise gclient_utils.Error('Invalid Upstream: %s' % + upstream_branch) + + self._SetFetchConfig(options) + + # Fetch upstream if we don't already have |revision|. + if not scm.GIT.IsValidRevision( + self.checkout_path, revision, sha_only=True): + self._Fetch(options, prune=options.force) + + if not scm.GIT.IsValidRevision( + self.checkout_path, revision, sha_only=True): + # Update the remotes first so we have all the refs. + remote_output = scm.GIT.Capture(['remote'] + verbose + + ['update'], + cwd=self.checkout_path) + if verbose: + self.Print(remote_output) + + revision = self._AutoFetchRef(options, revision) + + # This is a big hammer, debatable if it should even be here... + if options.force or options.reset: + target = 'HEAD' + if options.upstream and upstream_branch: + target = upstream_branch + self._Scrub(target, options) + + if current_type == 'detached': + # case 0 + # We just did a Scrub, this is as clean as it's going to get. In + # particular if HEAD is a commit that contains two versions of the + # same file on a case-insensitive filesystem (e.g. 'a' and 'A'), + # there's no way to actually "Clean" the checkout; that commit is + # uncheckoutable on this system. The best we can do is carry forward + # to the checkout step. + if not (options.force or options.reset): + self._CheckClean(revision) + self._CheckDetachedHead(revision, options) + if self._Capture(['rev-list', '-n', '1', 'HEAD']) == revision: + self.Print('Up-to-date; skipping checkout.') + else: + # 'git checkout' may need to overwrite existing untracked files. + # Allow it only when nuclear options are enabled. + self._Checkout( + options, + revision, + force=(options.force and options.delete_unversioned_trees), + quiet=True, + ) + if not printed_path: + self.Print('_____ %s at %s' % (self.relpath, revision), + timestamp=False) + elif current_type == 'hash': + # case 1 + # Can't find a merge-base since we don't know our upstream. That + # makes this command VERY likely to produce a rebase failure. For + # now we assume origin is our upstream since that's what the old + # behavior was. + upstream_branch = self.remote + if options.revision or deps_revision: + upstream_branch = revision + self._AttemptRebase(upstream_branch, + file_list, + options, + printed_path=printed_path, + merge=options.merge) + printed_path = True + elif rev_type == 'hash': + # case 2 + self._AttemptRebase(upstream_branch, + file_list, + options, + newbase=revision, + printed_path=printed_path, + merge=options.merge) + printed_path = True + elif remote_ref and ''.join(remote_ref) != upstream_branch: + # case 4 + new_base = ''.join(remote_ref) + if not printed_path: + self.Print('_____ %s at %s' % (self.relpath, revision), + timestamp=False) + switch_error = ( + "Could not switch upstream branch from %s to %s\n" % + (upstream_branch, new_base) + + "Please use --force or merge or rebase manually:\n" + + "cd %s; git rebase %s\n" % (self.checkout_path, new_base) + + "OR git checkout -b %s" % new_base) + force_switch = False + if options.force: + try: + self._CheckClean(revision) + # case 4a + force_switch = True + except gclient_utils.Error as e: + if options.reset: + # case 4b + force_switch = True + else: + switch_error = '%s\n%s' % (e.message, switch_error) + if force_switch: + self.Print("Switching upstream branch from %s to %s" % + (upstream_branch, new_base)) + switch_branch = 'gclient_' + remote_ref[1] + self._Capture(['branch', '-f', switch_branch, new_base]) + self._Checkout(options, switch_branch, force=True, quiet=True) + else: + # case 4c + raise gclient_utils.Error(switch_error) + else: + # case 3 - the default case + rebase_files = self._GetDiffFilenames(upstream_branch) + if verbose: + self.Print('Trying fast-forward merge to branch : %s' % + upstream_branch) + try: + merge_args = ['merge'] + if options.merge: + merge_args.append('--ff') + else: + merge_args.append('--ff-only') + merge_args.append(upstream_branch) + merge_output = self._Capture(merge_args) + except subprocess2.CalledProcessError as e: + rebase_files = [] + if re.search(b'fatal: Not possible to fast-forward, aborting.', + e.stderr): + if not printed_path: + self.Print('_____ %s at %s' % (self.relpath, revision), + timestamp=False) + printed_path = True + while True: + if not options.auto_rebase: + try: + action = self._AskForData( + 'Cannot %s, attempt to rebase? ' + '(y)es / (q)uit / (s)kip : ' % + ('merge' if options.merge else + 'fast-forward merge'), options) + except ValueError: + raise gclient_utils.Error('Invalid Character') + if options.auto_rebase or re.match( + r'yes|y', action, re.I): + self._AttemptRebase(upstream_branch, + rebase_files, + options, + printed_path=printed_path, + merge=False) + printed_path = True + break + + if re.match(r'quit|q', action, re.I): + raise gclient_utils.Error( + "Can't fast-forward, please merge or " + "rebase manually.\n" + "cd %s && git " % self.checkout_path + + "rebase %s" % upstream_branch) + + if re.match(r'skip|s', action, re.I): + self.Print('Skipping %s' % self.relpath) + return + + self.Print('Input not recognized') + elif re.match( + b"error: Your local changes to '.*' would be " + b"overwritten by merge. Aborting.\nPlease, commit your " + b"changes or stash them before you can merge.\n", + e.stderr): + if not printed_path: + self.Print('_____ %s at %s' % (self.relpath, revision), + timestamp=False) + printed_path = True + raise gclient_utils.Error(e.stderr.decode('utf-8')) + else: + # Some other problem happened with the merge + logging.error("Error during fast-forward merge in %s!" % + self.relpath) + self.Print(e.stderr.decode('utf-8')) + raise + else: + # Fast-forward merge was successful + if not re.match('Already up-to-date.', merge_output) or verbose: + if not printed_path: + self.Print('_____ %s at %s' % (self.relpath, revision), + timestamp=False) + printed_path = True + self.Print(merge_output.strip()) + if not verbose: + # Make the output a little prettier. It's nice to have + # some whitespace between projects when syncing. + self.Print('') + + if file_list is not None: + file_list.extend( + [os.path.join(self.checkout_path, f) for f in rebase_files]) + + # If the rebase generated a conflict, abort and ask user to fix + if self._IsRebasing(): + raise gclient_utils.Error( + '\n____ %s at %s\n' + '\nConflict while rebasing this branch.\n' + 'Fix the conflict and run gclient again.\n' + 'See man git-rebase for details.\n' % (self.relpath, revision)) - if not scm.GIT.IsValidRevision(self.checkout_path, revision, - sha_only=True): - # Update the remotes first so we have all the refs. - remote_output = scm.GIT.Capture(['remote'] + verbose + ['update'], - cwd=self.checkout_path) if verbose: - self.Print(remote_output) - - revision = self._AutoFetchRef(options, revision) - - # This is a big hammer, debatable if it should even be here... - if options.force or options.reset: - target = 'HEAD' - if options.upstream and upstream_branch: - target = upstream_branch - self._Scrub(target, options) - - if current_type == 'detached': - # case 0 - # We just did a Scrub, this is as clean as it's going to get. In - # particular if HEAD is a commit that contains two versions of the same - # file on a case-insensitive filesystem (e.g. 'a' and 'A'), there's no way - # to actually "Clean" the checkout; that commit is uncheckoutable on this - # system. The best we can do is carry forward to the checkout step. - if not (options.force or options.reset): - self._CheckClean(revision) - self._CheckDetachedHead(revision, options) - if self._Capture(['rev-list', '-n', '1', 'HEAD']) == revision: - self.Print('Up-to-date; skipping checkout.') - else: - # 'git checkout' may need to overwrite existing untracked files. Allow - # it only when nuclear options are enabled. - self._Checkout( - options, - revision, - force=(options.force and options.delete_unversioned_trees), - quiet=True, - ) - if not printed_path: - self.Print('_____ %s at %s' % (self.relpath, revision), timestamp=False) - elif current_type == 'hash': - # case 1 - # Can't find a merge-base since we don't know our upstream. That makes - # this command VERY likely to produce a rebase failure. For now we - # assume origin is our upstream since that's what the old behavior was. - upstream_branch = self.remote - if options.revision or deps_revision: - upstream_branch = revision - self._AttemptRebase(upstream_branch, file_list, options, - printed_path=printed_path, merge=options.merge) - printed_path = True - elif rev_type == 'hash': - # case 2 - self._AttemptRebase(upstream_branch, file_list, options, - newbase=revision, printed_path=printed_path, - merge=options.merge) - printed_path = True - elif remote_ref and ''.join(remote_ref) != upstream_branch: - # case 4 - new_base = ''.join(remote_ref) - if not printed_path: - self.Print('_____ %s at %s' % (self.relpath, revision), timestamp=False) - switch_error = ("Could not switch upstream branch from %s to %s\n" - % (upstream_branch, new_base) + - "Please use --force or merge or rebase manually:\n" + - "cd %s; git rebase %s\n" % (self.checkout_path, new_base) + - "OR git checkout -b %s" % new_base) - force_switch = False - if options.force: - try: - self._CheckClean(revision) - # case 4a - force_switch = True - except gclient_utils.Error as e: - if options.reset: - # case 4b - force_switch = True - else: - switch_error = '%s\n%s' % (e.message, switch_error) - if force_switch: - self.Print("Switching upstream branch from %s to %s" % - (upstream_branch, new_base)) - switch_branch = 'gclient_' + remote_ref[1] - self._Capture(['branch', '-f', switch_branch, new_base]) - self._Checkout(options, switch_branch, force=True, quiet=True) - else: - # case 4c - raise gclient_utils.Error(switch_error) - else: - # case 3 - the default case - rebase_files = self._GetDiffFilenames(upstream_branch) - if verbose: - self.Print('Trying fast-forward merge to branch : %s' % upstream_branch) - try: - merge_args = ['merge'] - if options.merge: - merge_args.append('--ff') - else: - merge_args.append('--ff-only') - merge_args.append(upstream_branch) - merge_output = self._Capture(merge_args) - except subprocess2.CalledProcessError as e: - rebase_files = [] - if re.search(b'fatal: Not possible to fast-forward, aborting.', - e.stderr): - if not printed_path: - self.Print('_____ %s at %s' % (self.relpath, revision), + self.Print('Checked out revision %s' % + self.revinfo(options, (), None), timestamp=False) - printed_path = True - while True: - if not options.auto_rebase: - try: - action = self._AskForData( - 'Cannot %s, attempt to rebase? ' - '(y)es / (q)uit / (s)kip : ' % - ('merge' if options.merge else 'fast-forward merge'), - options) - except ValueError: - raise gclient_utils.Error('Invalid Character') - if options.auto_rebase or re.match(r'yes|y', action, re.I): - self._AttemptRebase(upstream_branch, rebase_files, options, - printed_path=printed_path, merge=False) - printed_path = True - break - if re.match(r'quit|q', action, re.I): - raise gclient_utils.Error("Can't fast-forward, please merge or " - "rebase manually.\n" - "cd %s && git " % self.checkout_path - + "rebase %s" % upstream_branch) + # If --reset and --delete_unversioned_trees are specified, remove any + # untracked directories. + if options.reset and options.delete_unversioned_trees: + # GIT.CaptureStatus() uses 'dit diff' to compare to a specific SHA1 + # (the merge-base by default), so doesn't include untracked files. + # So we use 'git ls-files --directory --others --exclude-standard' + # here directly. + paths = scm.GIT.Capture([ + '-c', 'core.quotePath=false', 'ls-files', '--directory', + '--others', '--exclude-standard' + ], self.checkout_path) + for path in (p for p in paths.splitlines() if p.endswith('/')): + full_path = os.path.join(self.checkout_path, path) + if not os.path.islink(full_path): + self.Print('_____ removing unversioned directory %s' % path) + gclient_utils.rmtree(full_path) - if re.match(r'skip|s', action, re.I): - self.Print('Skipping %s' % self.relpath) - return + return self._Capture(['rev-parse', '--verify', 'HEAD']) - self.Print('Input not recognized') - elif re.match(b"error: Your local changes to '.*' would be " - b"overwritten by merge. Aborting.\nPlease, commit your " - b"changes or stash them before you can merge.\n", - e.stderr): - if not printed_path: - self.Print('_____ %s at %s' % (self.relpath, revision), - timestamp=False) - printed_path = True - raise gclient_utils.Error(e.stderr.decode('utf-8')) - else: - # Some other problem happened with the merge - logging.error("Error during fast-forward merge in %s!" % self.relpath) - self.Print(e.stderr.decode('utf-8')) - raise - else: - # Fast-forward merge was successful - if not re.match('Already up-to-date.', merge_output) or verbose: - if not printed_path: - self.Print('_____ %s at %s' % (self.relpath, revision), - timestamp=False) - printed_path = True - self.Print(merge_output.strip()) - if not verbose: - # Make the output a little prettier. It's nice to have some - # whitespace between projects when syncing. - self.Print('') - - if file_list is not None: - file_list.extend( - [os.path.join(self.checkout_path, f) for f in rebase_files]) - - # If the rebase generated a conflict, abort and ask user to fix - if self._IsRebasing(): - raise gclient_utils.Error('\n____ %s at %s\n' - '\nConflict while rebasing this branch.\n' - 'Fix the conflict and run gclient again.\n' - 'See man git-rebase for details.\n' - % (self.relpath, revision)) - - if verbose: - self.Print('Checked out revision %s' % self.revinfo(options, (), None), - timestamp=False) - - # If --reset and --delete_unversioned_trees are specified, remove any - # untracked directories. - if options.reset and options.delete_unversioned_trees: - # GIT.CaptureStatus() uses 'dit diff' to compare to a specific SHA1 (the - # merge-base by default), so doesn't include untracked files. So we use - # 'git ls-files --directory --others --exclude-standard' here directly. - paths = scm.GIT.Capture( - ['-c', 'core.quotePath=false', 'ls-files', - '--directory', '--others', '--exclude-standard'], - self.checkout_path) - for path in (p for p in paths.splitlines() if p.endswith('/')): - full_path = os.path.join(self.checkout_path, path) - if not os.path.islink(full_path): - self.Print('_____ removing unversioned directory %s' % path) - gclient_utils.rmtree(full_path) - - return self._Capture(['rev-parse', '--verify', 'HEAD']) - - def revert(self, options, _args, file_list): - """Reverts local modifications. + def revert(self, options, _args, file_list): + """Reverts local modifications. All reverted files will be appended to file_list. """ - if not os.path.isdir(self.checkout_path): - # revert won't work if the directory doesn't exist. It needs to - # checkout instead. - self.Print('_____ %s is missing, syncing instead' % self.relpath) - # Don't reuse the args. - return self.update(options, [], file_list) + if not os.path.isdir(self.checkout_path): + # revert won't work if the directory doesn't exist. It needs to + # checkout instead. + self.Print('_____ %s is missing, syncing instead' % self.relpath) + # Don't reuse the args. + return self.update(options, [], file_list) - default_rev = "refs/heads/main" - if options.upstream: - if self._GetCurrentBranch(): - upstream_branch = scm.GIT.GetUpstreamBranch(self.checkout_path) - default_rev = upstream_branch or default_rev - _, deps_revision = gclient_utils.SplitUrlRevision(self.url) - if not deps_revision: - deps_revision = default_rev - if deps_revision.startswith('refs/heads/'): - deps_revision = deps_revision.replace('refs/heads/', self.remote + '/') - try: - deps_revision = self.GetUsableRev(deps_revision, options) - except NoUsableRevError as e: - # If the DEPS entry's url and hash changed, try to update the origin. - # See also http://crbug.com/520067. - logging.warning( - "Couldn't find usable revision, will retrying to update instead: %s", - e.message) - return self.update(options, [], file_list) + default_rev = "refs/heads/main" + if options.upstream: + if self._GetCurrentBranch(): + upstream_branch = scm.GIT.GetUpstreamBranch(self.checkout_path) + default_rev = upstream_branch or default_rev + _, deps_revision = gclient_utils.SplitUrlRevision(self.url) + if not deps_revision: + deps_revision = default_rev + if deps_revision.startswith('refs/heads/'): + deps_revision = deps_revision.replace('refs/heads/', + self.remote + '/') + try: + deps_revision = self.GetUsableRev(deps_revision, options) + except NoUsableRevError as e: + # If the DEPS entry's url and hash changed, try to update the + # origin. See also http://crbug.com/520067. + logging.warning( + "Couldn't find usable revision, will retrying to update instead: %s", + e.message) + return self.update(options, [], file_list) - if file_list is not None: - files = self._GetDiffFilenames(deps_revision) + if file_list is not None: + files = self._GetDiffFilenames(deps_revision) - self._Scrub(deps_revision, options) - self._Run(['clean', '-f', '-d'], options) + self._Scrub(deps_revision, options) + self._Run(['clean', '-f', '-d'], options) - if file_list is not None: - file_list.extend([os.path.join(self.checkout_path, f) for f in files]) + if file_list is not None: + file_list.extend( + [os.path.join(self.checkout_path, f) for f in files]) - def revinfo(self, _options, _args, _file_list): - """Returns revision""" - return self._Capture(['rev-parse', 'HEAD']) + def revinfo(self, _options, _args, _file_list): + """Returns revision""" + return self._Capture(['rev-parse', 'HEAD']) - def runhooks(self, options, args, file_list): - self.status(options, args, file_list) + def runhooks(self, options, args, file_list): + self.status(options, args, file_list) - def status(self, options, _args, file_list): - """Display status information.""" - if not os.path.isdir(self.checkout_path): - self.Print('________ couldn\'t run status in %s:\n' - 'The directory does not exist.' % self.checkout_path) - else: - merge_base = [] - if self.url: - _, base_rev = gclient_utils.SplitUrlRevision(self.url) - if base_rev: - if base_rev.startswith('refs/'): - base_rev = self._ref_to_remote_ref(base_rev) - merge_base = [base_rev] - self._Run( - ['-c', 'core.quotePath=false', 'diff', '--name-status'] + merge_base, - options, always_show_header=options.verbose) - if file_list is not None: - files = self._GetDiffFilenames(merge_base[0] if merge_base else None) - file_list.extend([os.path.join(self.checkout_path, f) for f in files]) + def status(self, options, _args, file_list): + """Display status information.""" + if not os.path.isdir(self.checkout_path): + self.Print('________ couldn\'t run status in %s:\n' + 'The directory does not exist.' % self.checkout_path) + else: + merge_base = [] + if self.url: + _, base_rev = gclient_utils.SplitUrlRevision(self.url) + if base_rev: + if base_rev.startswith('refs/'): + base_rev = self._ref_to_remote_ref(base_rev) + merge_base = [base_rev] + self._Run(['-c', 'core.quotePath=false', 'diff', '--name-status'] + + merge_base, + options, + always_show_header=options.verbose) + if file_list is not None: + files = self._GetDiffFilenames( + merge_base[0] if merge_base else None) + file_list.extend( + [os.path.join(self.checkout_path, f) for f in files]) - def GetUsableRev(self, rev, options): - """Finds a useful revision for this repository.""" - sha1 = None - if not os.path.isdir(self.checkout_path): - raise NoUsableRevError( - 'This is not a git repo, so we cannot get a usable rev.') + def GetUsableRev(self, rev, options): + """Finds a useful revision for this repository.""" + sha1 = None + if not os.path.isdir(self.checkout_path): + raise NoUsableRevError( + 'This is not a git repo, so we cannot get a usable rev.') - if scm.GIT.IsValidRevision(cwd=self.checkout_path, rev=rev): - sha1 = rev - else: - # May exist in origin, but we don't have it yet, so fetch and look - # again. - self._Fetch(options) - if scm.GIT.IsValidRevision(cwd=self.checkout_path, rev=rev): - sha1 = rev + if scm.GIT.IsValidRevision(cwd=self.checkout_path, rev=rev): + sha1 = rev + else: + # May exist in origin, but we don't have it yet, so fetch and look + # again. + self._Fetch(options) + if scm.GIT.IsValidRevision(cwd=self.checkout_path, rev=rev): + sha1 = rev - if not sha1: - raise NoUsableRevError( - 'Hash %s does not appear to be a valid hash in this repo.' % rev) + if not sha1: + raise NoUsableRevError( + 'Hash %s does not appear to be a valid hash in this repo.' % + rev) - return sha1 + return sha1 - def GetGitBackupDirPath(self): - """Returns the path where the .git folder for the current project can be + def GetGitBackupDirPath(self): + """Returns the path where the .git folder for the current project can be staged/restored. Use case: subproject moved from DEPS <-> outer project.""" - return os.path.join(self._root_dir, - 'old_' + self.relpath.replace(os.sep, '_')) + '.git' + return os.path.join(self._root_dir, + 'old_' + self.relpath.replace(os.sep, '_')) + '.git' - def _GetMirror(self, url, options, revision=None, revision_ref=None): - """Get a git_cache.Mirror object for the argument url.""" - if not self.cache_dir: - return None - mirror_kwargs = { - 'print_func': self.filter, - 'refs': [], - 'commits': [], - } - if hasattr(options, 'with_branch_heads') and options.with_branch_heads: - mirror_kwargs['refs'].append('refs/branch-heads/*') - elif revision_ref and revision_ref.startswith('refs/branch-heads/'): - mirror_kwargs['refs'].append(revision_ref) - if hasattr(options, 'with_tags') and options.with_tags: - mirror_kwargs['refs'].append('refs/tags/*') - elif revision_ref and revision_ref.startswith('refs/tags/'): - mirror_kwargs['refs'].append(revision_ref) - if revision and not revision.startswith('refs/'): - mirror_kwargs['commits'].append(revision) - return git_cache.Mirror(url, **mirror_kwargs) + def _GetMirror(self, url, options, revision=None, revision_ref=None): + """Get a git_cache.Mirror object for the argument url.""" + if not self.cache_dir: + return None + mirror_kwargs = { + 'print_func': self.filter, + 'refs': [], + 'commits': [], + } + if hasattr(options, 'with_branch_heads') and options.with_branch_heads: + mirror_kwargs['refs'].append('refs/branch-heads/*') + elif revision_ref and revision_ref.startswith('refs/branch-heads/'): + mirror_kwargs['refs'].append(revision_ref) + if hasattr(options, 'with_tags') and options.with_tags: + mirror_kwargs['refs'].append('refs/tags/*') + elif revision_ref and revision_ref.startswith('refs/tags/'): + mirror_kwargs['refs'].append(revision_ref) + if revision and not revision.startswith('refs/'): + mirror_kwargs['commits'].append(revision) + return git_cache.Mirror(url, **mirror_kwargs) - def _UpdateMirrorIfNotContains(self, mirror, options, rev_type, revision): - """Update a git mirror by fetching the latest commits from the remote, + def _UpdateMirrorIfNotContains(self, mirror, options, rev_type, revision): + """Update a git mirror by fetching the latest commits from the remote, unless mirror already contains revision whose type is sha1 hash. """ - if rev_type == 'hash' and mirror.contains_revision(revision): - if options.verbose: - self.Print('skipping mirror update, it has rev=%s already' % revision, - timestamp=False) - return + if rev_type == 'hash' and mirror.contains_revision(revision): + if options.verbose: + self.Print('skipping mirror update, it has rev=%s already' % + revision, + timestamp=False) + return - if getattr(options, 'shallow', False): - depth = 10000 - else: - depth = None - mirror.populate(verbose=options.verbose, - bootstrap=not getattr(options, 'no_bootstrap', False), - depth=depth, - lock_timeout=getattr(options, 'lock_timeout', 0)) + if getattr(options, 'shallow', False): + depth = 10000 + else: + depth = None + mirror.populate(verbose=options.verbose, + bootstrap=not getattr(options, 'no_bootstrap', False), + depth=depth, + lock_timeout=getattr(options, 'lock_timeout', 0)) - def _Clone(self, revision, url, options): - """Clone a git repository from the given URL. + def _Clone(self, revision, url, options): + """Clone a git repository from the given URL. Once we've cloned the repo, we checkout a working branch if the specified revision is a branch head. If it is a tag or a specific commit, then we leave HEAD detached as it makes future updates simpler -- in this case the user should first create a new branch or switch to an existing branch before making changes in the repo.""" - in_cog_workspace = self._IsCog() + in_cog_workspace = self._IsCog() - if self.print_outbuf: - print_stdout = True - filter_fn = None - else: - print_stdout = False - filter_fn = self.filter + if self.print_outbuf: + print_stdout = True + filter_fn = None + else: + print_stdout = False + filter_fn = self.filter - if not options.verbose: - # git clone doesn't seem to insert a newline properly before printing - # to stdout - self.Print('') + if not options.verbose: + # git clone doesn't seem to insert a newline properly before + # printing to stdout + self.Print('') - # If the parent directory does not exist, Git clone on Windows will not - # create it, so we need to do it manually. - parent_dir = os.path.dirname(self.checkout_path) - gclient_utils.safe_makedirs(parent_dir) + # If the parent directory does not exist, Git clone on Windows will not + # create it, so we need to do it manually. + parent_dir = os.path.dirname(self.checkout_path) + gclient_utils.safe_makedirs(parent_dir) - if in_cog_workspace: - clone_cmd = ['citc', 'clone-repo', url, self.checkout_path] - clone_cmd.append( - gclient_utils.ExtractRefName(self.remote, revision) or revision) - try: - self._Run(clone_cmd, - options, - cwd=self._root_dir, - retry=True, - print_stdout=print_stdout, - filter_fn=filter_fn) - except: - traceback.print_exc(file=self.out_fh) - raise - self._SetFetchConfig(options) - elif hasattr(options, 'no_history') and options.no_history: - self._Run(['init', self.checkout_path], options, cwd=self._root_dir) - self._Run(['remote', 'add', 'origin', url], options) - revision = self._AutoFetchRef(options, revision, depth=1) - remote_ref = scm.GIT.RefToRemoteRef(revision, self.remote) - self._Checkout(options, ''.join(remote_ref or revision), quiet=True) - else: - cfg = gclient_utils.DefaultIndexPackConfig(url) - clone_cmd = cfg + ['clone', '--no-checkout', '--progress'] - if self.cache_dir: - clone_cmd.append('--shared') - if options.verbose: - clone_cmd.append('--verbose') - clone_cmd.append(url) - tmp_dir = tempfile.mkdtemp(prefix='_gclient_%s_' % - os.path.basename(self.checkout_path), - dir=parent_dir) - clone_cmd.append(tmp_dir) + if in_cog_workspace: + clone_cmd = ['citc', 'clone-repo', url, self.checkout_path] + clone_cmd.append( + gclient_utils.ExtractRefName(self.remote, revision) or revision) + try: + self._Run(clone_cmd, + options, + cwd=self._root_dir, + retry=True, + print_stdout=print_stdout, + filter_fn=filter_fn) + except: + traceback.print_exc(file=self.out_fh) + raise + self._SetFetchConfig(options) + elif hasattr(options, 'no_history') and options.no_history: + self._Run(['init', self.checkout_path], options, cwd=self._root_dir) + self._Run(['remote', 'add', 'origin', url], options) + revision = self._AutoFetchRef(options, revision, depth=1) + remote_ref = scm.GIT.RefToRemoteRef(revision, self.remote) + self._Checkout(options, ''.join(remote_ref or revision), quiet=True) + else: + cfg = gclient_utils.DefaultIndexPackConfig(url) + clone_cmd = cfg + ['clone', '--no-checkout', '--progress'] + if self.cache_dir: + clone_cmd.append('--shared') + if options.verbose: + clone_cmd.append('--verbose') + clone_cmd.append(url) + tmp_dir = tempfile.mkdtemp(prefix='_gclient_%s_' % + os.path.basename(self.checkout_path), + dir=parent_dir) + clone_cmd.append(tmp_dir) - try: - self._Run(clone_cmd, - options, - cwd=self._root_dir, - retry=True, - print_stdout=print_stdout, - filter_fn=filter_fn) - logging.debug('Cloned into temporary dir, moving to checkout_path') - gclient_utils.safe_makedirs(self.checkout_path) - gclient_utils.safe_rename(os.path.join(tmp_dir, '.git'), - os.path.join(self.checkout_path, '.git')) - except: - traceback.print_exc(file=self.out_fh) - raise - finally: - if os.listdir(tmp_dir): - self.Print('_____ removing non-empty tmp dir %s' % tmp_dir) - gclient_utils.rmtree(tmp_dir) + try: + self._Run(clone_cmd, + options, + cwd=self._root_dir, + retry=True, + print_stdout=print_stdout, + filter_fn=filter_fn) + logging.debug( + 'Cloned into temporary dir, moving to checkout_path') + gclient_utils.safe_makedirs(self.checkout_path) + gclient_utils.safe_rename( + os.path.join(tmp_dir, '.git'), + os.path.join(self.checkout_path, '.git')) + except: + traceback.print_exc(file=self.out_fh) + raise + finally: + if os.listdir(tmp_dir): + self.Print('_____ removing non-empty tmp dir %s' % tmp_dir) + gclient_utils.rmtree(tmp_dir) - self._SetFetchConfig(options) - self._Fetch(options, prune=options.force) - revision = self._AutoFetchRef(options, revision) - remote_ref = scm.GIT.RefToRemoteRef(revision, self.remote) - self._Checkout(options, ''.join(remote_ref or revision), quiet=True) + self._SetFetchConfig(options) + self._Fetch(options, prune=options.force) + revision = self._AutoFetchRef(options, revision) + remote_ref = scm.GIT.RefToRemoteRef(revision, self.remote) + self._Checkout(options, ''.join(remote_ref or revision), quiet=True) - if self._GetCurrentBranch() is None: - # Squelch git's very verbose detached HEAD warning and use our own - self.Print( - ('Checked out %s to a detached HEAD. Before making any commits\n' - 'in this repo, you should use \'git checkout \' to switch \n' - 'to an existing branch or use \'git checkout %s -b \' to\n' - 'create a new branch for your work.') % (revision, self.remote)) + if self._GetCurrentBranch() is None: + # Squelch git's very verbose detached HEAD warning and use our own + self.Print(( + 'Checked out %s to a detached HEAD. Before making any commits\n' + 'in this repo, you should use \'git checkout \' to switch \n' + 'to an existing branch or use \'git checkout %s -b \' to\n' + 'create a new branch for your work.') % (revision, self.remote)) - def _AskForData(self, prompt, options): - if options.jobs > 1: - self.Print(prompt) - raise gclient_utils.Error("Background task requires input. Rerun " - "gclient with --jobs=1 so that\n" - "interaction is possible.") - return gclient_utils.AskForData(prompt) + def _AskForData(self, prompt, options): + if options.jobs > 1: + self.Print(prompt) + raise gclient_utils.Error("Background task requires input. Rerun " + "gclient with --jobs=1 so that\n" + "interaction is possible.") + return gclient_utils.AskForData(prompt) + def _AttemptRebase(self, + upstream, + files, + options, + newbase=None, + branch=None, + printed_path=False, + merge=False): + """Attempt to rebase onto either upstream or, if specified, newbase.""" + if files is not None: + files.extend(self._GetDiffFilenames(upstream)) + revision = upstream + if newbase: + revision = newbase + action = 'merge' if merge else 'rebase' + if not printed_path: + self.Print('_____ %s : Attempting %s onto %s...' % + (self.relpath, action, revision)) + printed_path = True + else: + self.Print('Attempting %s onto %s...' % (action, revision)) - def _AttemptRebase(self, upstream, files, options, newbase=None, - branch=None, printed_path=False, merge=False): - """Attempt to rebase onto either upstream or, if specified, newbase.""" - if files is not None: - files.extend(self._GetDiffFilenames(upstream)) - revision = upstream - if newbase: - revision = newbase - action = 'merge' if merge else 'rebase' - if not printed_path: - self.Print('_____ %s : Attempting %s onto %s...' % ( - self.relpath, action, revision)) - printed_path = True - else: - self.Print('Attempting %s onto %s...' % (action, revision)) + if merge: + merge_output = self._Capture(['merge', revision]) + if options.verbose: + self.Print(merge_output) + return - if merge: - merge_output = self._Capture(['merge', revision]) - if options.verbose: - self.Print(merge_output) - return + # Build the rebase command here using the args + # git rebase [options] [--onto ] [] + rebase_cmd = ['rebase'] + if options.verbose: + rebase_cmd.append('--verbose') + if newbase: + rebase_cmd.extend(['--onto', newbase]) + rebase_cmd.append(upstream) + if branch: + rebase_cmd.append(branch) - # Build the rebase command here using the args - # git rebase [options] [--onto ] [] - rebase_cmd = ['rebase'] - if options.verbose: - rebase_cmd.append('--verbose') - if newbase: - rebase_cmd.extend(['--onto', newbase]) - rebase_cmd.append(upstream) - if branch: - rebase_cmd.append(branch) - - try: - rebase_output = scm.GIT.Capture(rebase_cmd, cwd=self.checkout_path) - except subprocess2.CalledProcessError as e: - if (re.match(br'cannot rebase: you have unstaged changes', e.stderr) or - re.match(br'cannot rebase: your index contains uncommitted changes', - e.stderr)): - while True: - rebase_action = self._AskForData( - 'Cannot rebase because of unstaged changes.\n' - '\'git reset --hard HEAD\' ?\n' - 'WARNING: destroys any uncommitted work in your current branch!' - ' (y)es / (q)uit / (s)how : ', options) - if re.match(r'yes|y', rebase_action, re.I): - self._Scrub('HEAD', options) - # Should this be recursive? + try: rebase_output = scm.GIT.Capture(rebase_cmd, cwd=self.checkout_path) - break + except subprocess2.CalledProcessError as e: + if (re.match( + br'cannot rebase: you have unstaged changes', e.stderr + ) or re.match( + br'cannot rebase: your index contains uncommitted changes', + e.stderr)): + while True: + rebase_action = self._AskForData( + 'Cannot rebase because of unstaged changes.\n' + '\'git reset --hard HEAD\' ?\n' + 'WARNING: destroys any uncommitted work in your current branch!' + ' (y)es / (q)uit / (s)how : ', options) + if re.match(r'yes|y', rebase_action, re.I): + self._Scrub('HEAD', options) + # Should this be recursive? + rebase_output = scm.GIT.Capture(rebase_cmd, + cwd=self.checkout_path) + break - if re.match(r'quit|q', rebase_action, re.I): - raise gclient_utils.Error("Please merge or rebase manually\n" - "cd %s && git " % self.checkout_path - + "%s" % ' '.join(rebase_cmd)) + if re.match(r'quit|q', rebase_action, re.I): + raise gclient_utils.Error( + "Please merge or rebase manually\n" + "cd %s && git " % self.checkout_path + + "%s" % ' '.join(rebase_cmd)) - if re.match(r'show|s', rebase_action, re.I): - self.Print('%s' % e.stderr.decode('utf-8').strip()) - continue + if re.match(r'show|s', rebase_action, re.I): + self.Print('%s' % e.stderr.decode('utf-8').strip()) + continue - gclient_utils.Error("Input not recognized") - continue - elif re.search(br'^CONFLICT', e.stdout, re.M): - raise gclient_utils.Error("Conflict while rebasing this branch.\n" - "Fix the conflict and run gclient again.\n" - "See 'man git-rebase' for details.\n") - else: - self.Print(e.stdout.decode('utf-8').strip()) - self.Print('Rebase produced error output:\n%s' % - e.stderr.decode('utf-8').strip()) - raise gclient_utils.Error("Unrecognized error, please merge or rebase " - "manually.\ncd %s && git " % - self.checkout_path - + "%s" % ' '.join(rebase_cmd)) + gclient_utils.Error("Input not recognized") + continue + elif re.search(br'^CONFLICT', e.stdout, re.M): + raise gclient_utils.Error( + "Conflict while rebasing this branch.\n" + "Fix the conflict and run gclient again.\n" + "See 'man git-rebase' for details.\n") + else: + self.Print(e.stdout.decode('utf-8').strip()) + self.Print('Rebase produced error output:\n%s' % + e.stderr.decode('utf-8').strip()) + raise gclient_utils.Error( + "Unrecognized error, please merge or rebase " + "manually.\ncd %s && git " % self.checkout_path + + "%s" % ' '.join(rebase_cmd)) - self.Print(rebase_output.strip()) - if not options.verbose: - # Make the output a little prettier. It's nice to have some - # whitespace between projects when syncing. - self.Print('') + self.Print(rebase_output.strip()) + if not options.verbose: + # Make the output a little prettier. It's nice to have some + # whitespace between projects when syncing. + self.Print('') - @staticmethod - def _CheckMinVersion(min_version): - (ok, current_version) = scm.GIT.AssertVersion(min_version) - if not ok: - raise gclient_utils.Error('git version %s < minimum required %s' % - (current_version, min_version)) + @staticmethod + def _CheckMinVersion(min_version): + (ok, current_version) = scm.GIT.AssertVersion(min_version) + if not ok: + raise gclient_utils.Error('git version %s < minimum required %s' % + (current_version, min_version)) - def _EnsureValidHeadObjectOrCheckout(self, revision, options, url): - # Special case handling if all 3 conditions are met: - # * the mirros have recently changed, but deps destination remains same, - # * the git histories of mirrors are conflicting. - # * git cache is used - # This manifests itself in current checkout having invalid HEAD commit on - # most git operations. Since git cache is used, just deleted the .git - # folder, and re-create it by cloning. - try: - self._Capture(['rev-list', '-n', '1', 'HEAD']) - except subprocess2.CalledProcessError as e: - if (b'fatal: bad object HEAD' in e.stderr - and self.cache_dir and self.cache_dir in url): - self.Print(( - 'Likely due to DEPS change with git cache_dir, ' - 'the current commit points to no longer existing object.\n' - '%s' % e) - ) - self._DeleteOrMove(options.force) - self._Clone(revision, url, options) - else: - raise + def _EnsureValidHeadObjectOrCheckout(self, revision, options, url): + # Special case handling if all 3 conditions are met: + # * the mirros have recently changed, but deps destination remains same, + # * the git histories of mirrors are conflicting. * git cache is used + # This manifests itself in current checkout having invalid HEAD commit + # on most git operations. Since git cache is used, just deleted the .git + # folder, and re-create it by cloning. + try: + self._Capture(['rev-list', '-n', '1', 'HEAD']) + except subprocess2.CalledProcessError as e: + if (b'fatal: bad object HEAD' in e.stderr and self.cache_dir + and self.cache_dir in url): + self.Print( + ('Likely due to DEPS change with git cache_dir, ' + 'the current commit points to no longer existing object.\n' + '%s' % e)) + self._DeleteOrMove(options.force) + self._Clone(revision, url, options) + else: + raise - def _IsRebasing(self): - # Check for any of REBASE-i/REBASE-m/REBASE/AM. Unfortunately git doesn't - # have a plumbing command to determine whether a rebase is in progress, so - # for now emualate (more-or-less) git-rebase.sh / git-completion.bash - g = os.path.join(self.checkout_path, '.git') - return ( - os.path.isdir(os.path.join(g, "rebase-merge")) or - os.path.isdir(os.path.join(g, "rebase-apply"))) + def _IsRebasing(self): + # Check for any of REBASE-i/REBASE-m/REBASE/AM. Unfortunately git + # doesn't have a plumbing command to determine whether a rebase is in + # progress, so for now emualate (more-or-less) git-rebase.sh / + # git-completion.bash + g = os.path.join(self.checkout_path, '.git') + return (os.path.isdir(os.path.join(g, "rebase-merge")) + or os.path.isdir(os.path.join(g, "rebase-apply"))) - def _CheckClean(self, revision): - lockfile = os.path.join(self.checkout_path, ".git", "index.lock") - if os.path.exists(lockfile): - raise gclient_utils.Error( - '\n____ %s at %s\n' - '\tYour repo is locked, possibly due to a concurrent git process.\n' - '\tIf no git executable is running, then clean up %r and try again.\n' - % (self.relpath, revision, lockfile)) + def _CheckClean(self, revision): + lockfile = os.path.join(self.checkout_path, ".git", "index.lock") + if os.path.exists(lockfile): + raise gclient_utils.Error( + '\n____ %s at %s\n' + '\tYour repo is locked, possibly due to a concurrent git process.\n' + '\tIf no git executable is running, then clean up %r and try again.\n' + % (self.relpath, revision, lockfile)) - # Make sure the tree is clean; see git-rebase.sh for reference - try: - scm.GIT.Capture(['update-index', '--ignore-submodules', '--refresh'], - cwd=self.checkout_path) - except subprocess2.CalledProcessError: - raise gclient_utils.Error('\n____ %s at %s\n' - '\tYou have unstaged changes.\n' - '\tcd into %s, run git status to see changes,\n' - '\tand commit, stash, or reset.\n' % - (self.relpath, revision, self.relpath)) - try: - scm.GIT.Capture(['diff-index', '--cached', '--name-status', '-r', - '--ignore-submodules', 'HEAD', '--'], - cwd=self.checkout_path) - except subprocess2.CalledProcessError: - raise gclient_utils.Error('\n____ %s at %s\n' - '\tYour index contains uncommitted changes\n' - '\tcd into %s, run git status to see changes,\n' - '\tand commit, stash, or reset.\n' % - (self.relpath, revision, self.relpath)) + # Make sure the tree is clean; see git-rebase.sh for reference + try: + scm.GIT.Capture( + ['update-index', '--ignore-submodules', '--refresh'], + cwd=self.checkout_path) + except subprocess2.CalledProcessError: + raise gclient_utils.Error( + '\n____ %s at %s\n' + '\tYou have unstaged changes.\n' + '\tcd into %s, run git status to see changes,\n' + '\tand commit, stash, or reset.\n' % + (self.relpath, revision, self.relpath)) + try: + scm.GIT.Capture([ + 'diff-index', '--cached', '--name-status', '-r', + '--ignore-submodules', 'HEAD', '--' + ], + cwd=self.checkout_path) + except subprocess2.CalledProcessError: + raise gclient_utils.Error( + '\n____ %s at %s\n' + '\tYour index contains uncommitted changes\n' + '\tcd into %s, run git status to see changes,\n' + '\tand commit, stash, or reset.\n' % + (self.relpath, revision, self.relpath)) - def _CheckDetachedHead(self, revision, _options): - # HEAD is detached. Make sure it is safe to move away from (i.e., it is - # reference by a commit). If not, error out -- most likely a rebase is - # in progress, try to detect so we can give a better error. - try: - scm.GIT.Capture(['name-rev', '--no-undefined', 'HEAD'], - cwd=self.checkout_path) - except subprocess2.CalledProcessError: - # Commit is not contained by any rev. See if the user is rebasing: - if self._IsRebasing(): - # Punt to the user - raise gclient_utils.Error('\n____ %s at %s\n' - '\tAlready in a conflict, i.e. (no branch).\n' - '\tFix the conflict and run gclient again.\n' - '\tOr to abort run:\n\t\tgit-rebase --abort\n' - '\tSee man git-rebase for details.\n' - % (self.relpath, revision)) - # Let's just save off the commit so we can proceed. - name = ('saved-by-gclient-' + - self._Capture(['rev-parse', '--short', 'HEAD'])) - self._Capture(['branch', '-f', name]) - self.Print('_____ found an unreferenced commit and saved it as \'%s\'' % - name) + def _CheckDetachedHead(self, revision, _options): + # HEAD is detached. Make sure it is safe to move away from (i.e., it is + # reference by a commit). If not, error out -- most likely a rebase is + # in progress, try to detect so we can give a better error. + try: + scm.GIT.Capture(['name-rev', '--no-undefined', 'HEAD'], + cwd=self.checkout_path) + except subprocess2.CalledProcessError: + # Commit is not contained by any rev. See if the user is rebasing: + if self._IsRebasing(): + # Punt to the user + raise gclient_utils.Error( + '\n____ %s at %s\n' + '\tAlready in a conflict, i.e. (no branch).\n' + '\tFix the conflict and run gclient again.\n' + '\tOr to abort run:\n\t\tgit-rebase --abort\n' + '\tSee man git-rebase for details.\n' % + (self.relpath, revision)) + # Let's just save off the commit so we can proceed. + name = ('saved-by-gclient-' + + self._Capture(['rev-parse', '--short', 'HEAD'])) + self._Capture(['branch', '-f', name]) + self.Print( + '_____ found an unreferenced commit and saved it as \'%s\'' % + name) - def _GetCurrentBranch(self): - # Returns name of current branch or None for detached HEAD - branch = self._Capture(['rev-parse', '--abbrev-ref=strict', 'HEAD']) - if branch == 'HEAD': - return None - return branch + def _GetCurrentBranch(self): + # Returns name of current branch or None for detached HEAD + branch = self._Capture(['rev-parse', '--abbrev-ref=strict', 'HEAD']) + if branch == 'HEAD': + return None + return branch - def _Capture(self, args, **kwargs): - set_git_dir = 'cwd' not in kwargs - kwargs.setdefault('cwd', self.checkout_path) - kwargs.setdefault('stderr', subprocess2.PIPE) - strip = kwargs.pop('strip', True) - env = scm.GIT.ApplyEnvVars(kwargs) - # If an explicit cwd isn't set, then default to the .git/ subdir so we get - # stricter behavior. This can be useful in cases of slight corruption -- - # we don't accidentally go corrupting parent git checks too. See - # https://crbug.com/1000825 for an example. - if set_git_dir: - git_dir = os.path.abspath(os.path.join(self.checkout_path, '.git')) - # Depending on how the .gclient file was defined, self.checkout_path - # might be set to a unicode string, not a regular string; on Windows - # Python2, we can't set env vars to be unicode strings, so we - # forcibly cast the value to a string before setting it. - env.setdefault('GIT_DIR', str(git_dir)) - ret = subprocess2.check_output( - ['git'] + args, env=env, **kwargs).decode('utf-8') - if strip: - ret = ret.strip() - self.Print('Finished running: %s %s' % ('git', ' '.join(args))) - return ret + def _Capture(self, args, **kwargs): + set_git_dir = 'cwd' not in kwargs + kwargs.setdefault('cwd', self.checkout_path) + kwargs.setdefault('stderr', subprocess2.PIPE) + strip = kwargs.pop('strip', True) + env = scm.GIT.ApplyEnvVars(kwargs) + # If an explicit cwd isn't set, then default to the .git/ subdir so we + # get stricter behavior. This can be useful in cases of slight + # corruption -- we don't accidentally go corrupting parent git checks + # too. See https://crbug.com/1000825 for an example. + if set_git_dir: + git_dir = os.path.abspath(os.path.join(self.checkout_path, '.git')) + # Depending on how the .gclient file was defined, self.checkout_path + # might be set to a unicode string, not a regular string; on Windows + # Python2, we can't set env vars to be unicode strings, so we + # forcibly cast the value to a string before setting it. + env.setdefault('GIT_DIR', str(git_dir)) + ret = subprocess2.check_output(['git'] + args, env=env, + **kwargs).decode('utf-8') + if strip: + ret = ret.strip() + self.Print('Finished running: %s %s' % ('git', ' '.join(args))) + return ret - def _Checkout(self, options, ref, force=False, quiet=None): - """Performs a 'git-checkout' operation. + def _Checkout(self, options, ref, force=False, quiet=None): + """Performs a 'git-checkout' operation. Args: options: The configured option set @@ -1454,112 +1541,120 @@ class GitWrapper(SCMWrapper): 'None', the behavior is inferred from 'options.verbose'. Returns: (str) The output of the checkout operation """ - if quiet is None: - quiet = (not options.verbose) - checkout_args = ['checkout'] - if force: - checkout_args.append('--force') - if quiet: - checkout_args.append('--quiet') - checkout_args.append(ref) - return self._Capture(checkout_args) + if quiet is None: + quiet = (not options.verbose) + checkout_args = ['checkout'] + if force: + checkout_args.append('--force') + if quiet: + checkout_args.append('--quiet') + checkout_args.append(ref) + return self._Capture(checkout_args) - def _Fetch(self, - options, - remote=None, - prune=False, - quiet=False, - refspec=None, - depth=None): - cfg = gclient_utils.DefaultIndexPackConfig(self.url) - # When updating, the ref is modified to be a remote ref . - # (e.g. refs/heads/NAME becomes refs/remotes/REMOTE/NAME). - # Try to reverse that mapping. - original_ref = scm.GIT.RemoteRefToRef(refspec, self.remote) - if original_ref: - refspec = original_ref + ':' + refspec - # When a mirror is configured, it only fetches - # refs/{heads,branch-heads,tags}/*. - # If asked to fetch other refs, we must fetch those directly from the - # repository, and not from the mirror. - if not original_ref.startswith( - ('refs/heads/', 'refs/branch-heads/', 'refs/tags/')): - remote, _ = gclient_utils.SplitUrlRevision(self.url) - fetch_cmd = cfg + [ - 'fetch', - remote or self.remote, - ] - if refspec: - fetch_cmd.append(refspec) + def _Fetch(self, + options, + remote=None, + prune=False, + quiet=False, + refspec=None, + depth=None): + cfg = gclient_utils.DefaultIndexPackConfig(self.url) + # When updating, the ref is modified to be a remote ref . + # (e.g. refs/heads/NAME becomes refs/remotes/REMOTE/NAME). + # Try to reverse that mapping. + original_ref = scm.GIT.RemoteRefToRef(refspec, self.remote) + if original_ref: + refspec = original_ref + ':' + refspec + # When a mirror is configured, it only fetches + # refs/{heads,branch-heads,tags}/*. + # If asked to fetch other refs, we must fetch those directly from + # the repository, and not from the mirror. + if not original_ref.startswith( + ('refs/heads/', 'refs/branch-heads/', 'refs/tags/')): + remote, _ = gclient_utils.SplitUrlRevision(self.url) + fetch_cmd = cfg + [ + 'fetch', + remote or self.remote, + ] + if refspec: + fetch_cmd.append(refspec) - if prune: - fetch_cmd.append('--prune') - if options.verbose: - fetch_cmd.append('--verbose') - if not hasattr(options, 'with_tags') or not options.with_tags: - fetch_cmd.append('--no-tags') - elif quiet: - fetch_cmd.append('--quiet') - if depth: - fetch_cmd.append('--depth=' + str(depth)) - self._Run(fetch_cmd, options, show_header=options.verbose, retry=True) + if prune: + fetch_cmd.append('--prune') + if options.verbose: + fetch_cmd.append('--verbose') + if not hasattr(options, 'with_tags') or not options.with_tags: + fetch_cmd.append('--no-tags') + elif quiet: + fetch_cmd.append('--quiet') + if depth: + fetch_cmd.append('--depth=' + str(depth)) + self._Run(fetch_cmd, options, show_header=options.verbose, retry=True) - def _SetFetchConfig(self, options): - """Adds, and optionally fetches, "branch-heads" and "tags" refspecs + def _SetFetchConfig(self, options): + """Adds, and optionally fetches, "branch-heads" and "tags" refspecs if requested.""" - if options.force or options.reset: - try: - self._Run(['config', '--unset-all', 'remote.%s.fetch' % self.remote], - options) - self._Run(['config', 'remote.%s.fetch' % self.remote, - '+refs/heads/*:refs/remotes/%s/*' % self.remote], options) - except subprocess2.CalledProcessError as e: - # If exit code was 5, it means we attempted to unset a config that - # didn't exist. Ignore it. - if e.returncode != 5: - raise - if hasattr(options, 'with_branch_heads') and options.with_branch_heads: - config_cmd = ['config', 'remote.%s.fetch' % self.remote, - '+refs/branch-heads/*:refs/remotes/branch-heads/*', - '^\\+refs/branch-heads/\\*:.*$'] - self._Run(config_cmd, options) - if hasattr(options, 'with_tags') and options.with_tags: - config_cmd = ['config', 'remote.%s.fetch' % self.remote, - '+refs/tags/*:refs/tags/*', - '^\\+refs/tags/\\*:.*$'] - self._Run(config_cmd, options) + if options.force or options.reset: + try: + self._Run( + ['config', '--unset-all', + 'remote.%s.fetch' % self.remote], options) + self._Run([ + 'config', + 'remote.%s.fetch' % self.remote, + '+refs/heads/*:refs/remotes/%s/*' % self.remote + ], options) + except subprocess2.CalledProcessError as e: + # If exit code was 5, it means we attempted to unset a config + # that didn't exist. Ignore it. + if e.returncode != 5: + raise + if hasattr(options, 'with_branch_heads') and options.with_branch_heads: + config_cmd = [ + 'config', + 'remote.%s.fetch' % self.remote, + '+refs/branch-heads/*:refs/remotes/branch-heads/*', + '^\\+refs/branch-heads/\\*:.*$' + ] + self._Run(config_cmd, options) + if hasattr(options, 'with_tags') and options.with_tags: + config_cmd = [ + 'config', + 'remote.%s.fetch' % self.remote, '+refs/tags/*:refs/tags/*', + '^\\+refs/tags/\\*:.*$' + ] + self._Run(config_cmd, options) - def _AutoFetchRef(self, options, revision, depth=None): - """Attempts to fetch |revision| if not available in local repo. + def _AutoFetchRef(self, options, revision, depth=None): + """Attempts to fetch |revision| if not available in local repo. Returns possibly updated revision.""" - if not scm.GIT.IsValidRevision(self.checkout_path, revision): - self._Fetch(options, refspec=revision, depth=depth) - revision = self._Capture(['rev-parse', 'FETCH_HEAD']) - return revision + if not scm.GIT.IsValidRevision(self.checkout_path, revision): + self._Fetch(options, refspec=revision, depth=depth) + revision = self._Capture(['rev-parse', 'FETCH_HEAD']) + return revision - def _Run(self, args, options, **kwargs): - # Disable 'unused options' warning | pylint: disable=unused-argument - kwargs.setdefault('cwd', self.checkout_path) - kwargs.setdefault('filter_fn', self.filter) - kwargs.setdefault('show_header', True) - env = scm.GIT.ApplyEnvVars(kwargs) + def _Run(self, args, options, **kwargs): + # Disable 'unused options' warning | pylint: disable=unused-argument + kwargs.setdefault('cwd', self.checkout_path) + kwargs.setdefault('filter_fn', self.filter) + kwargs.setdefault('show_header', True) + env = scm.GIT.ApplyEnvVars(kwargs) - cmd = ['git'] + args - gclient_utils.CheckCallAndFilter(cmd, env=env, **kwargs) + cmd = ['git'] + args + gclient_utils.CheckCallAndFilter(cmd, env=env, **kwargs) class CipdPackage(object): - """A representation of a single CIPD package.""" + """A representation of a single CIPD package.""" + def __init__(self, name, version, authority_for_subdir): + self._authority_for_subdir = authority_for_subdir + self._name = name + self._version = version - def __init__(self, name, version, authority_for_subdir): - self._authority_for_subdir = authority_for_subdir - self._name = name - self._version = version - - @property - def authority_for_subdir(self): - """Whether this package has authority to act on behalf of its subdir. + @property + def authority_for_subdir(self): + """Whether this package has authority to act on behalf of its subdir. Some operations should only be performed once per subdirectory. A package that has authority for its subdirectory is the only package that should @@ -1568,29 +1663,29 @@ class CipdPackage(object): Returns: bool; whether this package has subdir authority. """ - return self._authority_for_subdir + return self._authority_for_subdir - @property - def name(self): - return self._name + @property + def name(self): + return self._name - @property - def version(self): - return self._version + @property + def version(self): + return self._version class CipdRoot(object): - """A representation of a single CIPD root.""" - def __init__(self, root_dir, service_url): - self._all_packages = set() - self._mutator_lock = threading.Lock() - self._packages_by_subdir = collections.defaultdict(list) - self._root_dir = root_dir - self._service_url = service_url - self._resolved_packages = None + """A representation of a single CIPD root.""" + def __init__(self, root_dir, service_url): + self._all_packages = set() + self._mutator_lock = threading.Lock() + self._packages_by_subdir = collections.defaultdict(list) + self._root_dir = root_dir + self._service_url = service_url + self._resolved_packages = None - def add_package(self, subdir, package, version): - """Adds a package to this CIPD root. + def add_package(self, subdir, package, version): + """Adds a package to this CIPD root. As far as clients are concerned, this grants both root and subdir authority to packages arbitrarily. (The implementation grants root authority to the @@ -1605,39 +1700,38 @@ class CipdRoot(object): Returns: CipdPackage; the package that was created and added to this root. """ - with self._mutator_lock: - cipd_package = CipdPackage( - package, version, - not self._packages_by_subdir[subdir]) - self._all_packages.add(cipd_package) - self._packages_by_subdir[subdir].append(cipd_package) - return cipd_package + with self._mutator_lock: + cipd_package = CipdPackage(package, version, + not self._packages_by_subdir[subdir]) + self._all_packages.add(cipd_package) + self._packages_by_subdir[subdir].append(cipd_package) + return cipd_package - def packages(self, subdir): - """Get the list of configured packages for the given subdir.""" - return list(self._packages_by_subdir[subdir]) + def packages(self, subdir): + """Get the list of configured packages for the given subdir.""" + return list(self._packages_by_subdir[subdir]) - def resolved_packages(self): - if not self._resolved_packages: - self._resolved_packages = self.ensure_file_resolve() - return self._resolved_packages + def resolved_packages(self): + if not self._resolved_packages: + self._resolved_packages = self.ensure_file_resolve() + return self._resolved_packages - def clobber(self): - """Remove the .cipd directory. + def clobber(self): + """Remove the .cipd directory. This is useful for forcing ensure to redownload and reinitialize all packages. """ - with self._mutator_lock: - cipd_cache_dir = os.path.join(self.root_dir, '.cipd') - try: - gclient_utils.rmtree(os.path.join(cipd_cache_dir)) - except OSError: - if os.path.exists(cipd_cache_dir): - raise + with self._mutator_lock: + cipd_cache_dir = os.path.join(self.root_dir, '.cipd') + try: + gclient_utils.rmtree(os.path.join(cipd_cache_dir)) + except OSError: + if os.path.exists(cipd_cache_dir): + raise - def expand_package_name(self, package_name_string, **kwargs): - """Run `cipd expand-package-name`. + def expand_package_name(self, package_name_string, **kwargs): + """Run `cipd expand-package-name`. CIPD package names can be declared with placeholder variables such as '${platform}', this cmd will return the package name @@ -1645,199 +1739,214 @@ class CipdRoot(object): the command is executing on. """ - kwargs.setdefault('stderr', subprocess2.PIPE) - cmd = ['cipd', 'expand-package-name', package_name_string] - ret = subprocess2.check_output(cmd, **kwargs).decode('utf-8') - return ret.strip() + kwargs.setdefault('stderr', subprocess2.PIPE) + cmd = ['cipd', 'expand-package-name', package_name_string] + ret = subprocess2.check_output(cmd, **kwargs).decode('utf-8') + return ret.strip() - @contextlib.contextmanager - def _create_ensure_file(self): - try: - contents = '$ParanoidMode CheckPresence\n' - # TODO(crbug/1329641): Remove once cipd packages have been updated - # to always be created in copy mode. - contents += '$OverrideInstallMode copy\n\n' - for subdir, packages in sorted(self._packages_by_subdir.items()): - contents += '@Subdir %s\n' % subdir - for package in sorted(packages, key=lambda p: p.name): - contents += '%s %s\n' % (package.name, package.version) - contents += '\n' - ensure_file = None - with tempfile.NamedTemporaryFile( - suffix='.ensure', delete=False, mode='wb') as ensure_file: - ensure_file.write(contents.encode('utf-8', 'replace')) - yield ensure_file.name - finally: - if ensure_file is not None and os.path.exists(ensure_file.name): - os.remove(ensure_file.name) + @contextlib.contextmanager + def _create_ensure_file(self): + try: + contents = '$ParanoidMode CheckPresence\n' + # TODO(crbug/1329641): Remove once cipd packages have been updated + # to always be created in copy mode. + contents += '$OverrideInstallMode copy\n\n' + for subdir, packages in sorted(self._packages_by_subdir.items()): + contents += '@Subdir %s\n' % subdir + for package in sorted(packages, key=lambda p: p.name): + contents += '%s %s\n' % (package.name, package.version) + contents += '\n' + ensure_file = None + with tempfile.NamedTemporaryFile(suffix='.ensure', + delete=False, + mode='wb') as ensure_file: + ensure_file.write(contents.encode('utf-8', 'replace')) + yield ensure_file.name + finally: + if ensure_file is not None and os.path.exists(ensure_file.name): + os.remove(ensure_file.name) - def ensure(self): - """Run `cipd ensure`.""" - with self._mutator_lock: - with self._create_ensure_file() as ensure_file: - cmd = [ - 'cipd', 'ensure', - '-log-level', 'error', - '-root', self.root_dir, - '-ensure-file', ensure_file, - ] - gclient_utils.CheckCallAndFilter( - cmd, print_stdout=True, show_header=True) + def ensure(self): + """Run `cipd ensure`.""" + with self._mutator_lock: + with self._create_ensure_file() as ensure_file: + cmd = [ + 'cipd', + 'ensure', + '-log-level', + 'error', + '-root', + self.root_dir, + '-ensure-file', + ensure_file, + ] + gclient_utils.CheckCallAndFilter(cmd, + print_stdout=True, + show_header=True) - @contextlib.contextmanager - def _create_ensure_file_for_resolve(self): - try: - contents = '$ResolvedVersions %s\n' % os.devnull - for subdir, packages in sorted(self._packages_by_subdir.items()): - contents += '@Subdir %s\n' % subdir - for package in sorted(packages, key=lambda p: p.name): - contents += '%s %s\n' % (package.name, package.version) - contents += '\n' - ensure_file = None - with tempfile.NamedTemporaryFile(suffix='.ensure', - delete=False, - mode='wb') as ensure_file: - ensure_file.write(contents.encode('utf-8', 'replace')) - yield ensure_file.name - finally: - if ensure_file is not None and os.path.exists(ensure_file.name): - os.remove(ensure_file.name) + @contextlib.contextmanager + def _create_ensure_file_for_resolve(self): + try: + contents = '$ResolvedVersions %s\n' % os.devnull + for subdir, packages in sorted(self._packages_by_subdir.items()): + contents += '@Subdir %s\n' % subdir + for package in sorted(packages, key=lambda p: p.name): + contents += '%s %s\n' % (package.name, package.version) + contents += '\n' + ensure_file = None + with tempfile.NamedTemporaryFile(suffix='.ensure', + delete=False, + mode='wb') as ensure_file: + ensure_file.write(contents.encode('utf-8', 'replace')) + yield ensure_file.name + finally: + if ensure_file is not None and os.path.exists(ensure_file.name): + os.remove(ensure_file.name) - def _create_resolved_file(self): - return tempfile.NamedTemporaryFile(suffix='.resolved', - delete=False, - mode='wb') + def _create_resolved_file(self): + return tempfile.NamedTemporaryFile(suffix='.resolved', + delete=False, + mode='wb') - def ensure_file_resolve(self): - """Run `cipd ensure-file-resolve`.""" - with self._mutator_lock: - with self._create_resolved_file() as output_file: - with self._create_ensure_file_for_resolve() as ensure_file: - cmd = [ - 'cipd', - 'ensure-file-resolve', - '-log-level', - 'error', - '-ensure-file', - ensure_file, - '-json-output', - output_file.name, - ] - gclient_utils.CheckCallAndFilter(cmd, - print_stdout=False, - show_header=False) - with open(output_file.name) as f: - output_json = json.load(f) - return output_json.get('result', {}) + def ensure_file_resolve(self): + """Run `cipd ensure-file-resolve`.""" + with self._mutator_lock: + with self._create_resolved_file() as output_file: + with self._create_ensure_file_for_resolve() as ensure_file: + cmd = [ + 'cipd', + 'ensure-file-resolve', + '-log-level', + 'error', + '-ensure-file', + ensure_file, + '-json-output', + output_file.name, + ] + gclient_utils.CheckCallAndFilter(cmd, + print_stdout=False, + show_header=False) + with open(output_file.name) as f: + output_json = json.load(f) + return output_json.get('result', {}) - def run(self, command): - if command == 'update': - self.ensure() - elif command == 'revert': - self.clobber() - self.ensure() + def run(self, command): + if command == 'update': + self.ensure() + elif command == 'revert': + self.clobber() + self.ensure() - def created_package(self, package): - """Checks whether this root created the given package. + def created_package(self, package): + """Checks whether this root created the given package. Args: package: CipdPackage; the package to check. Returns: bool; whether this root created the given package. """ - return package in self._all_packages + return package in self._all_packages - @property - def root_dir(self): - return self._root_dir + @property + def root_dir(self): + return self._root_dir - @property - def service_url(self): - return self._service_url + @property + def service_url(self): + return self._service_url class CipdWrapper(SCMWrapper): - """Wrapper for CIPD. + """Wrapper for CIPD. Currently only supports chrome-infra-packages.appspot.com. """ - name = 'cipd' + name = 'cipd' - def __init__(self, url=None, root_dir=None, relpath=None, out_fh=None, - out_cb=None, root=None, package=None): - super(CipdWrapper, self).__init__( - url=url, root_dir=root_dir, relpath=relpath, out_fh=out_fh, - out_cb=out_cb) - assert root.created_package(package) - self._package = package - self._root = root + def __init__(self, + url=None, + root_dir=None, + relpath=None, + out_fh=None, + out_cb=None, + root=None, + package=None): + super(CipdWrapper, self).__init__(url=url, + root_dir=root_dir, + relpath=relpath, + out_fh=out_fh, + out_cb=out_cb) + assert root.created_package(package) + self._package = package + self._root = root - #override - def GetCacheMirror(self): - return None + #override + def GetCacheMirror(self): + return None - #override - def GetActualRemoteURL(self, options): - return self._root.service_url + #override + def GetActualRemoteURL(self, options): + return self._root.service_url - #override - def DoesRemoteURLMatch(self, options): - del options - return True + #override + def DoesRemoteURLMatch(self, options): + del options + return True - def revert(self, options, args, file_list): - """Does nothing. + def revert(self, options, args, file_list): + """Does nothing. CIPD packages should be reverted at the root by running `CipdRoot.run('revert')`. """ - def diff(self, options, args, file_list): - """CIPD has no notion of diffing.""" + def diff(self, options, args, file_list): + """CIPD has no notion of diffing.""" - def pack(self, options, args, file_list): - """CIPD has no notion of diffing.""" + def pack(self, options, args, file_list): + """CIPD has no notion of diffing.""" - def revinfo(self, options, args, file_list): - """Grab the instance ID.""" - try: - tmpdir = tempfile.mkdtemp() - # Attempt to get instance_id from the root resolved cache. - # Resolved cache will not match on any CIPD packages with - # variables such as ${platform}, they will fall back to - # the slower method below. - resolved = self._root.resolved_packages() - if resolved: - # CIPD uses POSIX separators across all platforms, so - # replace any Windows separators. - path_split = self.relpath.replace(os.sep, "/").split(":") - if len(path_split) > 1: - src_path, package = path_split - if src_path in resolved: - for resolved_package in resolved[src_path]: - if package == resolved_package.get('pin', {}).get('package'): - return resolved_package.get('pin', {}).get('instance_id') + def revinfo(self, options, args, file_list): + """Grab the instance ID.""" + try: + tmpdir = tempfile.mkdtemp() + # Attempt to get instance_id from the root resolved cache. + # Resolved cache will not match on any CIPD packages with + # variables such as ${platform}, they will fall back to + # the slower method below. + resolved = self._root.resolved_packages() + if resolved: + # CIPD uses POSIX separators across all platforms, so + # replace any Windows separators. + path_split = self.relpath.replace(os.sep, "/").split(":") + if len(path_split) > 1: + src_path, package = path_split + if src_path in resolved: + for resolved_package in resolved[src_path]: + if package == resolved_package.get( + 'pin', {}).get('package'): + return resolved_package.get( + 'pin', {}).get('instance_id') - describe_json_path = os.path.join(tmpdir, 'describe.json') - cmd = [ - 'cipd', 'describe', - self._package.name, - '-log-level', 'error', - '-version', self._package.version, - '-json-output', describe_json_path - ] - gclient_utils.CheckCallAndFilter(cmd) - with open(describe_json_path) as f: - describe_json = json.load(f) - return describe_json.get('result', {}).get('pin', {}).get('instance_id') - finally: - gclient_utils.rmtree(tmpdir) + describe_json_path = os.path.join(tmpdir, 'describe.json') + cmd = [ + 'cipd', 'describe', self._package.name, '-log-level', 'error', + '-version', self._package.version, '-json-output', + describe_json_path + ] + gclient_utils.CheckCallAndFilter(cmd) + with open(describe_json_path) as f: + describe_json = json.load(f) + return describe_json.get('result', {}).get('pin', + {}).get('instance_id') + finally: + gclient_utils.rmtree(tmpdir) - def status(self, options, args, file_list): - pass + def status(self, options, args, file_list): + pass - def update(self, options, args, file_list): - """Does nothing. + def update(self, options, args, file_list): + """Does nothing. CIPD packages should be updated at the root by running `CipdRoot.run('update')`. diff --git a/gclient_utils.py b/gclient_utils.py index 575d021dbb..e017f99691 100644 --- a/gclient_utils.py +++ b/gclient_utils.py @@ -1,7 +1,6 @@ # Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Generic utils.""" import codecs @@ -28,7 +27,6 @@ import urllib.parse import subprocess2 - # Git wrapper retries on a transient error, and some callees do retries too, # such as GitWrapper.update (doing clone). One retry attempt should be # sufficient to help with any transient errors at this level. @@ -36,57 +34,56 @@ RETRY_MAX = 1 RETRY_INITIAL_SLEEP = 2 # in seconds START = datetime.datetime.now() - _WARNINGS = [] - # These repos are known to cause OOM errors on 32-bit platforms, due the the # very large objects they contain. It is not safe to use threaded index-pack # when cloning/fetching them. THREADED_INDEX_PACK_BLOCKLIST = [ - 'https://chromium.googlesource.com/chromium/reference_builds/chrome_win.git' + 'https://chromium.googlesource.com/chromium/reference_builds/chrome_win.git' ] + def reraise(typ, value, tb=None): - """To support rethrowing exceptions with tracebacks.""" - if value is None: - value = typ() - if value.__traceback__ is not tb: - raise value.with_traceback(tb) - raise value + """To support rethrowing exceptions with tracebacks.""" + if value is None: + value = typ() + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value class Error(Exception): - """gclient exception class.""" - def __init__(self, msg, *args, **kwargs): - index = getattr(threading.currentThread(), 'index', 0) - if index: - msg = '\n'.join('%d> %s' % (index, l) for l in msg.splitlines()) - super(Error, self).__init__(msg, *args, **kwargs) + """gclient exception class.""" + def __init__(self, msg, *args, **kwargs): + index = getattr(threading.currentThread(), 'index', 0) + if index: + msg = '\n'.join('%d> %s' % (index, l) for l in msg.splitlines()) + super(Error, self).__init__(msg, *args, **kwargs) def Elapsed(until=None): - if until is None: - until = datetime.datetime.now() - return str(until - START).partition('.')[0] + if until is None: + until = datetime.datetime.now() + return str(until - START).partition('.')[0] def PrintWarnings(): - """Prints any accumulated warnings.""" - if _WARNINGS: - print('\n\nWarnings:', file=sys.stderr) - for warning in _WARNINGS: - print(warning, file=sys.stderr) + """Prints any accumulated warnings.""" + if _WARNINGS: + print('\n\nWarnings:', file=sys.stderr) + for warning in _WARNINGS: + print(warning, file=sys.stderr) def AddWarning(msg): - """Adds the given warning message to the list of accumulated warnings.""" - _WARNINGS.append(msg) + """Adds the given warning message to the list of accumulated warnings.""" + _WARNINGS.append(msg) def FuzzyMatchRepo(repo, candidates): - # type: (str, Union[Collection[str], Mapping[str, Any]]) -> Optional[str] - """Attempts to find a representation of repo in the candidates. + # type: (str, Union[Collection[str], Mapping[str, Any]]) -> Optional[str] + """Attempts to find a representation of repo in the candidates. Args: repo: a string representation of a repo in the form of a url or the @@ -96,134 +93,135 @@ def FuzzyMatchRepo(repo, candidates): Returns: The matching string, if any, which may be in a different form from `repo`. """ - if repo in candidates: - return repo - if repo.endswith('.git') and repo[:-len('.git')] in candidates: - return repo[:-len('.git')] - if repo + '.git' in candidates: - return repo + '.git' - return None + if repo in candidates: + return repo + if repo.endswith('.git') and repo[:-len('.git')] in candidates: + return repo[:-len('.git')] + if repo + '.git' in candidates: + return repo + '.git' + return None def SplitUrlRevision(url): - """Splits url and returns a two-tuple: url, rev""" - if url.startswith('ssh:'): - # Make sure ssh://user-name@example.com/~/test.git@stable works - regex = r'(ssh://(?:[-.\w]+@)?[-\w:\.]+/[-~\w\./]+)(?:@(.+))?' - components = re.search(regex, url).groups() - else: - components = url.rsplit('@', 1) - if re.match(r'^\w+\@', url) and '@' not in components[0]: - components = [url] + """Splits url and returns a two-tuple: url, rev""" + if url.startswith('ssh:'): + # Make sure ssh://user-name@example.com/~/test.git@stable works + regex = r'(ssh://(?:[-.\w]+@)?[-\w:\.]+/[-~\w\./]+)(?:@(.+))?' + components = re.search(regex, url).groups() + else: + components = url.rsplit('@', 1) + if re.match(r'^\w+\@', url) and '@' not in components[0]: + components = [url] - if len(components) == 1: - components += [None] - return tuple(components) + if len(components) == 1: + components += [None] + return tuple(components) def ExtractRefName(remote, full_refs_str): - """Returns the ref name if full_refs_str is a valid ref.""" - result = re.compile(r'^refs(\/.+)?\/((%s)|(heads)|(tags))\/(?P.+)' % - remote).match(full_refs_str) - if result: - return result.group('ref_name') - return None + """Returns the ref name if full_refs_str is a valid ref.""" + result = re.compile( + r'^refs(\/.+)?\/((%s)|(heads)|(tags))\/(?P.+)' % + remote).match(full_refs_str) + if result: + return result.group('ref_name') + return None def IsGitSha(revision): - """Returns true if the given string is a valid hex-encoded sha""" - return re.match('^[a-fA-F0-9]{6,40}$', revision) is not None + """Returns true if the given string is a valid hex-encoded sha""" + return re.match('^[a-fA-F0-9]{6,40}$', revision) is not None def IsFullGitSha(revision): - """Returns true if the given string is a valid hex-encoded full sha""" - return re.match('^[a-fA-F0-9]{40}$', revision) is not None + """Returns true if the given string is a valid hex-encoded full sha""" + return re.match('^[a-fA-F0-9]{40}$', revision) is not None def IsDateRevision(revision): - """Returns true if the given revision is of the form "{ ... }".""" - return bool(revision and re.match(r'^\{.+\}$', str(revision))) + """Returns true if the given revision is of the form "{ ... }".""" + return bool(revision and re.match(r'^\{.+\}$', str(revision))) def MakeDateRevision(date): - """Returns a revision representing the latest revision before the given + """Returns a revision representing the latest revision before the given date.""" - return "{" + date + "}" + return "{" + date + "}" def SyntaxErrorToError(filename, e): - """Raises a gclient_utils.Error exception with the human readable message""" - try: - # Try to construct a human readable error message - if filename: - error_message = 'There is a syntax error in %s\n' % filename + """Raises a gclient_utils.Error exception with the human readable message""" + try: + # Try to construct a human readable error message + if filename: + error_message = 'There is a syntax error in %s\n' % filename + else: + error_message = 'There is a syntax error\n' + error_message += 'Line #%s, character %s: "%s"' % ( + e.lineno, e.offset, re.sub(r'[\r\n]*$', '', e.text)) + except: + # Something went wrong, re-raise the original exception + raise e else: - error_message = 'There is a syntax error\n' - error_message += 'Line #%s, character %s: "%s"' % ( - e.lineno, e.offset, re.sub(r'[\r\n]*$', '', e.text)) - except: - # Something went wrong, re-raise the original exception - raise e - else: - raise Error(error_message) + raise Error(error_message) class PrintableObject(object): - def __str__(self): - output = '' - for i in dir(self): - if i.startswith('__'): - continue - output += '%s = %s\n' % (i, str(getattr(self, i, ''))) - return output + def __str__(self): + output = '' + for i in dir(self): + if i.startswith('__'): + continue + output += '%s = %s\n' % (i, str(getattr(self, i, ''))) + return output def AskForData(message): - # Try to load the readline module, so that "elaborate line editing" features - # such as backspace work for `raw_input` / `input`. - try: - import readline - except ImportError: - # The readline module does not exist in all Python distributions, e.g. on - # Windows. Fall back to simple input handling. - pass + # Try to load the readline module, so that "elaborate line editing" features + # such as backspace work for `raw_input` / `input`. + try: + import readline + except ImportError: + # The readline module does not exist in all Python distributions, e.g. + # on Windows. Fall back to simple input handling. + pass - # Use this so that it can be mocked in tests. - try: - return input(message) - except KeyboardInterrupt: - # Hide the exception. - sys.exit(1) + # Use this so that it can be mocked in tests. + try: + return input(message) + except KeyboardInterrupt: + # Hide the exception. + sys.exit(1) def FileRead(filename, mode='rbU'): - # mode is ignored now; we always return unicode strings. - with open(filename, mode='rb') as f: - s = f.read() - try: - return s.decode('utf-8', 'replace') - except (UnicodeDecodeError, AttributeError): - return s + # mode is ignored now; we always return unicode strings. + with open(filename, mode='rb') as f: + s = f.read() + try: + return s.decode('utf-8', 'replace') + except (UnicodeDecodeError, AttributeError): + return s def FileWrite(filename, content, mode='w', encoding='utf-8'): - with codecs.open(filename, mode=mode, encoding=encoding) as f: - f.write(content) + with codecs.open(filename, mode=mode, encoding=encoding) as f: + f.write(content) @contextlib.contextmanager def temporary_directory(**kwargs): - tdir = tempfile.mkdtemp(**kwargs) - try: - yield tdir - finally: - if tdir: - rmtree(tdir) + tdir = tempfile.mkdtemp(**kwargs) + try: + yield tdir + finally: + if tdir: + rmtree(tdir) @contextlib.contextmanager def temporary_file(): - """Creates a temporary file. + """Creates a temporary file. On Windows, a file must be closed before it can be opened again. This function allows to write something like: @@ -242,46 +240,47 @@ def temporary_file(): finally: os.remove(tmp.name) """ - handle, name = tempfile.mkstemp() - os.close(handle) - try: - yield name - finally: - os.remove(name) + handle, name = tempfile.mkstemp() + os.close(handle) + try: + yield name + finally: + os.remove(name) def safe_rename(old, new): - """Renames a file reliably. + """Renames a file reliably. Sometimes os.rename does not work because a dying git process keeps a handle on it for a few seconds. An exception is then thrown, which make the program give up what it was doing and remove what was deleted. The only solution is to catch the exception and try again until it works. """ - # roughly 10s - retries = 100 - for i in range(retries): - try: - os.rename(old, new) - break - except OSError: - if i == (retries - 1): - # Give up. - raise - # retry - logging.debug("Renaming failed from %s to %s. Retrying ..." % (old, new)) - time.sleep(0.1) + # roughly 10s + retries = 100 + for i in range(retries): + try: + os.rename(old, new) + break + except OSError: + if i == (retries - 1): + # Give up. + raise + # retry + logging.debug("Renaming failed from %s to %s. Retrying ..." % + (old, new)) + time.sleep(0.1) def rm_file_or_tree(path): - if os.path.isfile(path) or os.path.islink(path): - os.remove(path) - else: - rmtree(path) + if os.path.isfile(path) or os.path.islink(path): + os.remove(path) + else: + rmtree(path) def rmtree(path): - """shutil.rmtree() on steroids. + """shutil.rmtree() on steroids. Recursively removes a directory, even if it's marked read-only. @@ -305,201 +304,205 @@ def rmtree(path): In the ordinary case, this is not a problem: for our purposes, the user will never lack write permission on *path's parent. """ - if not os.path.exists(path): - return - - if os.path.islink(path) or not os.path.isdir(path): - raise Error('Called rmtree(%s) in non-directory' % path) - - if sys.platform == 'win32': - # Give up and use cmd.exe's rd command. - path = os.path.normcase(path) - for _ in range(3): - exitcode = subprocess.call(['cmd.exe', '/c', 'rd', '/q', '/s', path]) - if exitcode == 0: + if not os.path.exists(path): return - print('rd exited with code %d' % exitcode, file=sys.stderr) - time.sleep(3) - raise Exception('Failed to remove path %s' % path) + if os.path.islink(path) or not os.path.isdir(path): + raise Error('Called rmtree(%s) in non-directory' % path) - # On POSIX systems, we need the x-bit set on the directory to access it, - # the r-bit to see its contents, and the w-bit to remove files from it. - # The actual modes of the files within the directory is irrelevant. - os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + if sys.platform == 'win32': + # Give up and use cmd.exe's rd command. + path = os.path.normcase(path) + for _ in range(3): + exitcode = subprocess.call( + ['cmd.exe', '/c', 'rd', '/q', '/s', path]) + if exitcode == 0: + return - def remove(func, subpath): - func(subpath) + print('rd exited with code %d' % exitcode, file=sys.stderr) + time.sleep(3) + raise Exception('Failed to remove path %s' % path) - for fn in os.listdir(path): - # If fullpath is a symbolic link that points to a directory, isdir will - # be True, but we don't want to descend into that as a directory, we just - # want to remove the link. Check islink and treat links as ordinary files - # would be treated regardless of what they reference. - fullpath = os.path.join(path, fn) - if os.path.islink(fullpath) or not os.path.isdir(fullpath): - remove(os.remove, fullpath) - else: - # Recurse. - rmtree(fullpath) + # On POSIX systems, we need the x-bit set on the directory to access it, + # the r-bit to see its contents, and the w-bit to remove files from it. + # The actual modes of the files within the directory is irrelevant. + os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) - remove(os.rmdir, path) + def remove(func, subpath): + func(subpath) + + for fn in os.listdir(path): + # If fullpath is a symbolic link that points to a directory, isdir will + # be True, but we don't want to descend into that as a directory, we + # just want to remove the link. Check islink and treat links as + # ordinary files would be treated regardless of what they reference. + fullpath = os.path.join(path, fn) + if os.path.islink(fullpath) or not os.path.isdir(fullpath): + remove(os.remove, fullpath) + else: + # Recurse. + rmtree(fullpath) + + remove(os.rmdir, path) def safe_makedirs(tree): - """Creates the directory in a safe manner. + """Creates the directory in a safe manner. Because multiple threads can create these directories concurrently, trap the exception and pass on. """ - count = 0 - while not os.path.exists(tree): - count += 1 - try: - os.makedirs(tree) - except OSError as e: - # 17 POSIX, 183 Windows - if e.errno not in (17, 183): - raise - if count > 40: - # Give up. - raise + count = 0 + while not os.path.exists(tree): + count += 1 + try: + os.makedirs(tree) + except OSError as e: + # 17 POSIX, 183 Windows + if e.errno not in (17, 183): + raise + if count > 40: + # Give up. + raise def CommandToStr(args): - """Converts an arg list into a shell escaped string.""" - return ' '.join(pipes.quote(arg) for arg in args) + """Converts an arg list into a shell escaped string.""" + return ' '.join(pipes.quote(arg) for arg in args) class Wrapper(object): - """Wraps an object, acting as a transparent proxy for all properties by + """Wraps an object, acting as a transparent proxy for all properties by default. """ - def __init__(self, wrapped): - self._wrapped = wrapped + def __init__(self, wrapped): + self._wrapped = wrapped - def __getattr__(self, name): - return getattr(self._wrapped, name) + def __getattr__(self, name): + return getattr(self._wrapped, name) class AutoFlush(Wrapper): - """Creates a file object clone to automatically flush after N seconds.""" - def __init__(self, wrapped, delay): - super(AutoFlush, self).__init__(wrapped) - if not hasattr(self, 'lock'): - self.lock = threading.Lock() - self.__last_flushed_at = time.time() - self.delay = delay - - @property - def autoflush(self): - return self - - def write(self, out, *args, **kwargs): - self._wrapped.write(out, *args, **kwargs) - should_flush = False - self.lock.acquire() - try: - if self.delay and (time.time() - self.__last_flushed_at) > self.delay: - should_flush = True + """Creates a file object clone to automatically flush after N seconds.""" + def __init__(self, wrapped, delay): + super(AutoFlush, self).__init__(wrapped) + if not hasattr(self, 'lock'): + self.lock = threading.Lock() self.__last_flushed_at = time.time() - finally: - self.lock.release() - if should_flush: - self.flush() + self.delay = delay + + @property + def autoflush(self): + return self + + def write(self, out, *args, **kwargs): + self._wrapped.write(out, *args, **kwargs) + should_flush = False + self.lock.acquire() + try: + if self.delay and (time.time() - + self.__last_flushed_at) > self.delay: + should_flush = True + self.__last_flushed_at = time.time() + finally: + self.lock.release() + if should_flush: + self.flush() class Annotated(Wrapper): - """Creates a file object clone to automatically prepends every line in worker - threads with a NN> prefix. - """ - def __init__(self, wrapped, include_zero=False): - super(Annotated, self).__init__(wrapped) - if not hasattr(self, 'lock'): - self.lock = threading.Lock() - self.__output_buffers = {} - self.__include_zero = include_zero - self._wrapped_write = getattr(self._wrapped, 'buffer', self._wrapped).write + """Creates a file object clone to automatically prepends every line in + worker threads with a NN> prefix. + """ + def __init__(self, wrapped, include_zero=False): + super(Annotated, self).__init__(wrapped) + if not hasattr(self, 'lock'): + self.lock = threading.Lock() + self.__output_buffers = {} + self.__include_zero = include_zero + self._wrapped_write = getattr(self._wrapped, 'buffer', + self._wrapped).write - @property - def annotated(self): - return self + @property + def annotated(self): + return self - def write(self, out): - # Store as bytes to ensure Unicode characters get output correctly. - if not isinstance(out, bytes): - out = out.encode('utf-8') + def write(self, out): + # Store as bytes to ensure Unicode characters get output correctly. + if not isinstance(out, bytes): + out = out.encode('utf-8') - index = getattr(threading.currentThread(), 'index', 0) - if not index and not self.__include_zero: - # Unindexed threads aren't buffered. - return self._wrapped_write(out) + index = getattr(threading.currentThread(), 'index', 0) + if not index and not self.__include_zero: + # Unindexed threads aren't buffered. + return self._wrapped_write(out) - self.lock.acquire() - try: - # Use a dummy array to hold the string so the code can be lockless. - # Strings are immutable, requiring to keep a lock for the whole dictionary - # otherwise. Using an array is faster than using a dummy object. - if not index in self.__output_buffers: - obj = self.__output_buffers[index] = [b''] - else: - obj = self.__output_buffers[index] - finally: - self.lock.release() + self.lock.acquire() + try: + # Use a dummy array to hold the string so the code can be lockless. + # Strings are immutable, requiring to keep a lock for the whole + # dictionary otherwise. Using an array is faster than using a dummy + # object. + if not index in self.__output_buffers: + obj = self.__output_buffers[index] = [b''] + else: + obj = self.__output_buffers[index] + finally: + self.lock.release() - # Continue lockless. - obj[0] += out - while True: - cr_loc = obj[0].find(b'\r') - lf_loc = obj[0].find(b'\n') - if cr_loc == lf_loc == -1: - break + # Continue lockless. + obj[0] += out + while True: + cr_loc = obj[0].find(b'\r') + lf_loc = obj[0].find(b'\n') + if cr_loc == lf_loc == -1: + break - if cr_loc == -1 or (0 <= lf_loc < cr_loc): - line, remaining = obj[0].split(b'\n', 1) - if line: - self._wrapped_write(b'%d>%s\n' % (index, line)) - elif lf_loc == -1 or (0 <= cr_loc < lf_loc): - line, remaining = obj[0].split(b'\r', 1) - if line: - self._wrapped_write(b'%d>%s\r' % (index, line)) - obj[0] = remaining + if cr_loc == -1 or (0 <= lf_loc < cr_loc): + line, remaining = obj[0].split(b'\n', 1) + if line: + self._wrapped_write(b'%d>%s\n' % (index, line)) + elif lf_loc == -1 or (0 <= cr_loc < lf_loc): + line, remaining = obj[0].split(b'\r', 1) + if line: + self._wrapped_write(b'%d>%s\r' % (index, line)) + obj[0] = remaining - def flush(self): - """Flush buffered output.""" - orphans = [] - self.lock.acquire() - try: - # Detect threads no longer existing. - indexes = (getattr(t, 'index', None) for t in threading.enumerate()) - indexes = filter(None, indexes) - for index in self.__output_buffers: - if not index in indexes: - orphans.append((index, self.__output_buffers[index][0])) - for orphan in orphans: - del self.__output_buffers[orphan[0]] - finally: - self.lock.release() + def flush(self): + """Flush buffered output.""" + orphans = [] + self.lock.acquire() + try: + # Detect threads no longer existing. + indexes = (getattr(t, 'index', None) for t in threading.enumerate()) + indexes = filter(None, indexes) + for index in self.__output_buffers: + if not index in indexes: + orphans.append((index, self.__output_buffers[index][0])) + for orphan in orphans: + del self.__output_buffers[orphan[0]] + finally: + self.lock.release() - # Don't keep the lock while writing. Will append \n when it shouldn't. - for orphan in orphans: - if orphan[1]: - self._wrapped_write(b'%d>%s\n' % (orphan[0], orphan[1])) - return self._wrapped.flush() + # Don't keep the lock while writing. Will append \n when it shouldn't. + for orphan in orphans: + if orphan[1]: + self._wrapped_write(b'%d>%s\n' % (orphan[0], orphan[1])) + return self._wrapped.flush() def MakeFileAutoFlush(fileobj, delay=10): - autoflush = getattr(fileobj, 'autoflush', None) - if autoflush: - autoflush.delay = delay - return fileobj - return AutoFlush(fileobj, delay) + autoflush = getattr(fileobj, 'autoflush', None) + if autoflush: + autoflush.delay = delay + return fileobj + return AutoFlush(fileobj, delay) def MakeFileAnnotated(fileobj, include_zero=False): - if getattr(fileobj, 'annotated', None): - return fileobj - return Annotated(fileobj, include_zero) + if getattr(fileobj, 'annotated', None): + return fileobj + return Annotated(fileobj, include_zero) GCLIENT_CHILDREN = [] @@ -507,55 +510,62 @@ GCLIENT_CHILDREN_LOCK = threading.Lock() class GClientChildren(object): - @staticmethod - def add(popen_obj): - with GCLIENT_CHILDREN_LOCK: - GCLIENT_CHILDREN.append(popen_obj) + @staticmethod + def add(popen_obj): + with GCLIENT_CHILDREN_LOCK: + GCLIENT_CHILDREN.append(popen_obj) - @staticmethod - def remove(popen_obj): - with GCLIENT_CHILDREN_LOCK: - GCLIENT_CHILDREN.remove(popen_obj) + @staticmethod + def remove(popen_obj): + with GCLIENT_CHILDREN_LOCK: + GCLIENT_CHILDREN.remove(popen_obj) - @staticmethod - def _attemptToKillChildren(): - global GCLIENT_CHILDREN - with GCLIENT_CHILDREN_LOCK: - zombies = [c for c in GCLIENT_CHILDREN if c.poll() is None] + @staticmethod + def _attemptToKillChildren(): + global GCLIENT_CHILDREN + with GCLIENT_CHILDREN_LOCK: + zombies = [c for c in GCLIENT_CHILDREN if c.poll() is None] - for zombie in zombies: - try: - zombie.kill() - except OSError: - pass + for zombie in zombies: + try: + zombie.kill() + except OSError: + pass - with GCLIENT_CHILDREN_LOCK: - GCLIENT_CHILDREN = [k for k in GCLIENT_CHILDREN if k.poll() is not None] + with GCLIENT_CHILDREN_LOCK: + GCLIENT_CHILDREN = [ + k for k in GCLIENT_CHILDREN if k.poll() is not None + ] - @staticmethod - def _areZombies(): - with GCLIENT_CHILDREN_LOCK: - return bool(GCLIENT_CHILDREN) + @staticmethod + def _areZombies(): + with GCLIENT_CHILDREN_LOCK: + return bool(GCLIENT_CHILDREN) - @staticmethod - def KillAllRemainingChildren(): - GClientChildren._attemptToKillChildren() + @staticmethod + def KillAllRemainingChildren(): + GClientChildren._attemptToKillChildren() - if GClientChildren._areZombies(): - time.sleep(0.5) - GClientChildren._attemptToKillChildren() + if GClientChildren._areZombies(): + time.sleep(0.5) + GClientChildren._attemptToKillChildren() - with GCLIENT_CHILDREN_LOCK: - if GCLIENT_CHILDREN: - print('Could not kill the following subprocesses:', file=sys.stderr) - for zombie in GCLIENT_CHILDREN: - print(' ', zombie.pid, file=sys.stderr) + with GCLIENT_CHILDREN_LOCK: + if GCLIENT_CHILDREN: + print('Could not kill the following subprocesses:', + file=sys.stderr) + for zombie in GCLIENT_CHILDREN: + print(' ', zombie.pid, file=sys.stderr) -def CheckCallAndFilter(args, print_stdout=False, filter_fn=None, - show_header=False, always_show_header=False, retry=False, +def CheckCallAndFilter(args, + print_stdout=False, + filter_fn=None, + show_header=False, + always_show_header=False, + retry=False, **kwargs): - """Runs a command and calls back a filter function if needed. + """Runs a command and calls back a filter function if needed. Accepts all subprocess2.Popen() parameters plus: print_stdout: If True, the command's stdout is forwarded to stdout. @@ -571,145 +581,148 @@ def CheckCallAndFilter(args, print_stdout=False, filter_fn=None, Returns the output of the command as a binary string. """ - def show_header_if_necessary(needs_header, attempt): - """Show the header at most once.""" - if not needs_header[0]: - return + def show_header_if_necessary(needs_header, attempt): + """Show the header at most once.""" + if not needs_header[0]: + return - needs_header[0] = False - # Automatically generated header. We only prepend a newline if - # always_show_header is false, since it usually indicates there's an - # external progress display, and it's better not to clobber it in that case. - header = '' if always_show_header else '\n' - header += '________ running \'%s\' in \'%s\'' % ( - ' '.join(args), kwargs.get('cwd', '.')) - if attempt: - header += ' attempt %s / %s' % (attempt + 1, RETRY_MAX + 1) - header += '\n' + needs_header[0] = False + # Automatically generated header. We only prepend a newline if + # always_show_header is false, since it usually indicates there's an + # external progress display, and it's better not to clobber it in that + # case. + header = '' if always_show_header else '\n' + header += '________ running \'%s\' in \'%s\'' % (' '.join(args), + kwargs.get('cwd', '.')) + if attempt: + header += ' attempt %s / %s' % (attempt + 1, RETRY_MAX + 1) + header += '\n' + if print_stdout: + stdout_write = getattr(sys.stdout, 'buffer', sys.stdout).write + stdout_write(header.encode()) + if filter_fn: + filter_fn(header) + + def filter_line(command_output, line_start): + """Extract the last line from command output and filter it.""" + if not filter_fn or line_start is None: + return + command_output.seek(line_start) + filter_fn(command_output.read().decode('utf-8')) + + # Initialize stdout writer if needed. On Python 3, sys.stdout does not + # accept byte inputs and sys.stdout.buffer must be used instead. if print_stdout: - stdout_write = getattr(sys.stdout, 'buffer', sys.stdout).write - stdout_write(header.encode()) - if filter_fn: - filter_fn(header) - - def filter_line(command_output, line_start): - """Extract the last line from command output and filter it.""" - if not filter_fn or line_start is None: - return - command_output.seek(line_start) - filter_fn(command_output.read().decode('utf-8')) - - # Initialize stdout writer if needed. On Python 3, sys.stdout does not accept - # byte inputs and sys.stdout.buffer must be used instead. - if print_stdout: - sys.stdout.flush() - stdout_write = getattr(sys.stdout, 'buffer', sys.stdout).write - else: - stdout_write = lambda _: None - - sleep_interval = RETRY_INITIAL_SLEEP - run_cwd = kwargs.get('cwd', os.getcwd()) - - # Store the output of the command regardless of the value of print_stdout or - # filter_fn. - command_output = io.BytesIO() - for attempt in range(RETRY_MAX + 1): - # If our stdout is a terminal, then pass in a psuedo-tty pipe to our - # subprocess when filtering its output. This makes the subproc believe - # it was launched from a terminal, which will preserve ANSI color codes. - os_type = GetOperatingSystem() - if sys.stdout.isatty() and os_type != 'win' and os_type != 'aix': - pipe_reader, pipe_writer = os.openpty() + sys.stdout.flush() + stdout_write = getattr(sys.stdout, 'buffer', sys.stdout).write else: - pipe_reader, pipe_writer = os.pipe() + stdout_write = lambda _: None - kid = subprocess2.Popen( - args, bufsize=0, stdout=pipe_writer, stderr=subprocess2.STDOUT, - **kwargs) - # Close the write end of the pipe once we hand it off to the child proc. - os.close(pipe_writer) + sleep_interval = RETRY_INITIAL_SLEEP + run_cwd = kwargs.get('cwd', os.getcwd()) - GClientChildren.add(kid) - - # Passed as a list for "by ref" semantics. - needs_header = [show_header] - if always_show_header: - show_header_if_necessary(needs_header, attempt) - - # Also, we need to forward stdout to prevent weird re-ordering of output. - # This has to be done on a per byte basis to make sure it is not buffered: - # normally buffering is done for each line, but if the process requests - # input, no end-of-line character is output after the prompt and it would - # not show up. - try: - line_start = None - while True: - try: - in_byte = os.read(pipe_reader, 1) - except (IOError, OSError) as e: - if e.errno == errno.EIO: - # An errno.EIO means EOF? - in_byte = None - else: - raise e - is_newline = in_byte in (b'\n', b'\r') - if not in_byte: - break - - show_header_if_necessary(needs_header, attempt) - - if is_newline: - filter_line(command_output, line_start) - line_start = None - elif line_start is None: - line_start = command_output.tell() - - stdout_write(in_byte) - command_output.write(in_byte) - - # Flush the rest of buffered output. - sys.stdout.flush() - if line_start is not None: - filter_line(command_output, line_start) - - os.close(pipe_reader) - rv = kid.wait() - - # Don't put this in a 'finally,' since the child may still run if we get - # an exception. - GClientChildren.remove(kid) - - except KeyboardInterrupt: - print('Failed while running "%s"' % ' '.join(args), file=sys.stderr) - raise - - if rv == 0: - return command_output.getvalue() - - if not retry: - break - - print("WARNING: subprocess '%s' in %s failed; will retry after a short " - 'nap...' % (' '.join('"%s"' % x for x in args), run_cwd)) + # Store the output of the command regardless of the value of print_stdout or + # filter_fn. command_output = io.BytesIO() - time.sleep(sleep_interval) - sleep_interval *= 2 + for attempt in range(RETRY_MAX + 1): + # If our stdout is a terminal, then pass in a psuedo-tty pipe to our + # subprocess when filtering its output. This makes the subproc believe + # it was launched from a terminal, which will preserve ANSI color codes. + os_type = GetOperatingSystem() + if sys.stdout.isatty() and os_type != 'win' and os_type != 'aix': + pipe_reader, pipe_writer = os.openpty() + else: + pipe_reader, pipe_writer = os.pipe() - raise subprocess2.CalledProcessError( - rv, args, kwargs.get('cwd', None), command_output.getvalue(), None) + kid = subprocess2.Popen(args, + bufsize=0, + stdout=pipe_writer, + stderr=subprocess2.STDOUT, + **kwargs) + # Close the write end of the pipe once we hand it off to the child proc. + os.close(pipe_writer) + + GClientChildren.add(kid) + + # Passed as a list for "by ref" semantics. + needs_header = [show_header] + if always_show_header: + show_header_if_necessary(needs_header, attempt) + + # Also, we need to forward stdout to prevent weird re-ordering of + # output. This has to be done on a per byte basis to make sure it is not + # buffered: normally buffering is done for each line, but if the process + # requests input, no end-of-line character is output after the prompt + # and it would not show up. + try: + line_start = None + while True: + try: + in_byte = os.read(pipe_reader, 1) + except (IOError, OSError) as e: + if e.errno == errno.EIO: + # An errno.EIO means EOF? + in_byte = None + else: + raise e + is_newline = in_byte in (b'\n', b'\r') + if not in_byte: + break + + show_header_if_necessary(needs_header, attempt) + + if is_newline: + filter_line(command_output, line_start) + line_start = None + elif line_start is None: + line_start = command_output.tell() + + stdout_write(in_byte) + command_output.write(in_byte) + + # Flush the rest of buffered output. + sys.stdout.flush() + if line_start is not None: + filter_line(command_output, line_start) + + os.close(pipe_reader) + rv = kid.wait() + + # Don't put this in a 'finally,' since the child may still run if we + # get an exception. + GClientChildren.remove(kid) + + except KeyboardInterrupt: + print('Failed while running "%s"' % ' '.join(args), file=sys.stderr) + raise + + if rv == 0: + return command_output.getvalue() + + if not retry: + break + + print("WARNING: subprocess '%s' in %s failed; will retry after a short " + 'nap...' % (' '.join('"%s"' % x for x in args), run_cwd)) + command_output = io.BytesIO() + time.sleep(sleep_interval) + sleep_interval *= 2 + + raise subprocess2.CalledProcessError(rv, args, kwargs.get('cwd', None), + command_output.getvalue(), None) class GitFilter(object): - """A filter_fn implementation for quieting down git output messages. + """A filter_fn implementation for quieting down git output messages. Allows a custom function to skip certain lines (predicate), and will throttle the output of percentage completed lines to only output every X seconds. """ - PERCENT_RE = re.compile('(.*) ([0-9]{1,3})% .*') + PERCENT_RE = re.compile('(.*) ([0-9]{1,3})% .*') - def __init__(self, time_throttle=0, predicate=None, out_fh=None): - """ + def __init__(self, time_throttle=0, predicate=None, out_fh=None): + """ Args: time_throttle (int): GitFilter will throttle 'noisy' output (such as the XX% complete messages) to only be printed at least |time_throttle| @@ -718,127 +731,128 @@ class GitFilter(object): The line will be skipped if predicate(line) returns False. out_fh: File handle to write output to. """ - self.first_line = True - self.last_time = 0 - self.time_throttle = time_throttle - self.predicate = predicate - self.out_fh = out_fh or sys.stdout - self.progress_prefix = None + self.first_line = True + self.last_time = 0 + self.time_throttle = time_throttle + self.predicate = predicate + self.out_fh = out_fh or sys.stdout + self.progress_prefix = None - def __call__(self, line): - # git uses an escape sequence to clear the line; elide it. - esc = line.find(chr(0o33)) - if esc > -1: - line = line[:esc] - if self.predicate and not self.predicate(line): - return - now = time.time() - match = self.PERCENT_RE.match(line) - if match: - if match.group(1) != self.progress_prefix: - self.progress_prefix = match.group(1) - elif now - self.last_time < self.time_throttle: - return - self.last_time = now - if not self.first_line: - self.out_fh.write('[%s] ' % Elapsed()) - self.first_line = False - print(line, file=self.out_fh) + def __call__(self, line): + # git uses an escape sequence to clear the line; elide it. + esc = line.find(chr(0o33)) + if esc > -1: + line = line[:esc] + if self.predicate and not self.predicate(line): + return + now = time.time() + match = self.PERCENT_RE.match(line) + if match: + if match.group(1) != self.progress_prefix: + self.progress_prefix = match.group(1) + elif now - self.last_time < self.time_throttle: + return + self.last_time = now + if not self.first_line: + self.out_fh.write('[%s] ' % Elapsed()) + self.first_line = False + print(line, file=self.out_fh) def FindFileUpwards(filename, path=None): - """Search upwards from the a directory (default: current) to find a file. + """Search upwards from the a directory (default: current) to find a file. Returns nearest upper-level directory with the passed in file. """ - if not path: - path = os.getcwd() - path = os.path.realpath(path) - while True: - file_path = os.path.join(path, filename) - if os.path.exists(file_path): - return path - (new_path, _) = os.path.split(path) - if new_path == path: - return None - path = new_path + if not path: + path = os.getcwd() + path = os.path.realpath(path) + while True: + file_path = os.path.join(path, filename) + if os.path.exists(file_path): + return path + (new_path, _) = os.path.split(path) + if new_path == path: + return None + path = new_path def GetOperatingSystem(): - """Returns 'mac', 'win', 'linux', or the name of the current platform.""" - if sys.platform.startswith(('cygwin', 'win')): - return 'win' + """Returns 'mac', 'win', 'linux', or the name of the current platform.""" + if sys.platform.startswith(('cygwin', 'win')): + return 'win' - if sys.platform.startswith('linux'): - return 'linux' + if sys.platform.startswith('linux'): + return 'linux' - if sys.platform == 'darwin': - return 'mac' + if sys.platform == 'darwin': + return 'mac' - if sys.platform.startswith('aix'): - return 'aix' + if sys.platform.startswith('aix'): + return 'aix' - try: - return os.uname().sysname.lower() - except AttributeError: - return sys.platform + try: + return os.uname().sysname.lower() + except AttributeError: + return sys.platform def GetGClientRootAndEntries(path=None): - """Returns the gclient root and the dict of entries.""" - config_file = '.gclient_entries' - root = FindFileUpwards(config_file, path) - if not root: - print("Can't find %s" % config_file) - return None - config_path = os.path.join(root, config_file) - env = {} - with open(config_path) as config: - exec(config.read(), env) - config_dir = os.path.dirname(config_path) - return config_dir, env['entries'] + """Returns the gclient root and the dict of entries.""" + config_file = '.gclient_entries' + root = FindFileUpwards(config_file, path) + if not root: + print("Can't find %s" % config_file) + return None + config_path = os.path.join(root, config_file) + env = {} + with open(config_path) as config: + exec(config.read(), env) + config_dir = os.path.dirname(config_path) + return config_dir, env['entries'] def lockedmethod(method): - """Method decorator that holds self.lock for the duration of the call.""" - def inner(self, *args, **kwargs): - try: - try: - self.lock.acquire() - except KeyboardInterrupt: - print('Was deadlocked', file=sys.stderr) - raise - return method(self, *args, **kwargs) - finally: - self.lock.release() - return inner + """Method decorator that holds self.lock for the duration of the call.""" + def inner(self, *args, **kwargs): + try: + try: + self.lock.acquire() + except KeyboardInterrupt: + print('Was deadlocked', file=sys.stderr) + raise + return method(self, *args, **kwargs) + finally: + self.lock.release() + + return inner class WorkItem(object): - """One work item.""" - # On cygwin, creating a lock throwing randomly when nearing ~100 locks. - # As a workaround, use a single lock. Yep you read it right. Single lock for - # all the 100 objects. - lock = threading.Lock() + """One work item.""" + # On cygwin, creating a lock throwing randomly when nearing ~100 locks. + # As a workaround, use a single lock. Yep you read it right. Single lock for + # all the 100 objects. + lock = threading.Lock() - def __init__(self, name): - # A unique string representing this work item. - self._name = name - self.outbuf = io.StringIO() - self.start = self.finish = None - self.resources = [] # List of resources this work item requires. + def __init__(self, name): + # A unique string representing this work item. + self._name = name + self.outbuf = io.StringIO() + self.start = self.finish = None + self.resources = [] # List of resources this work item requires. - def run(self, work_queue): - """work_queue is passed as keyword argument so it should be + def run(self, work_queue): + """work_queue is passed as keyword argument so it should be the last parameters of the function when you override it.""" - @property - def name(self): - return self._name + @property + def name(self): + return self._name class ExecutionQueue(object): - """Runs a set of WorkItem that have interdependencies and were WorkItem are + """Runs a set of WorkItem that have interdependencies and were WorkItem are added as they are processed. This class manages that all the required dependencies are run @@ -846,259 +860,271 @@ class ExecutionQueue(object): Methods of this class are thread safe. """ - def __init__(self, jobs, progress, ignore_requirements, verbose=False): - """jobs specifies the number of concurrent tasks to allow. progress is a + def __init__(self, jobs, progress, ignore_requirements, verbose=False): + """jobs specifies the number of concurrent tasks to allow. progress is a Progress instance.""" - # Set when a thread is done or a new item is enqueued. - self.ready_cond = threading.Condition() - # Maximum number of concurrent tasks. - self.jobs = jobs - # List of WorkItem, for gclient, these are Dependency instances. - self.queued = [] - # List of strings representing each Dependency.name that was run. - self.ran = [] - # List of items currently running. - self.running = [] - # Exceptions thrown if any. - self.exceptions = queue.Queue() - # Progress status - self.progress = progress - if self.progress: - self.progress.update(0) + # Set when a thread is done or a new item is enqueued. + self.ready_cond = threading.Condition() + # Maximum number of concurrent tasks. + self.jobs = jobs + # List of WorkItem, for gclient, these are Dependency instances. + self.queued = [] + # List of strings representing each Dependency.name that was run. + self.ran = [] + # List of items currently running. + self.running = [] + # Exceptions thrown if any. + self.exceptions = queue.Queue() + # Progress status + self.progress = progress + if self.progress: + self.progress.update(0) - self.ignore_requirements = ignore_requirements - self.verbose = verbose - self.last_join = None - self.last_subproc_output = None + self.ignore_requirements = ignore_requirements + self.verbose = verbose + self.last_join = None + self.last_subproc_output = None - def enqueue(self, d): - """Enqueue one Dependency to be executed later once its requirements are + def enqueue(self, d): + """Enqueue one Dependency to be executed later once its requirements are satisfied. """ - assert isinstance(d, WorkItem) - self.ready_cond.acquire() - try: - self.queued.append(d) - total = len(self.queued) + len(self.ran) + len(self.running) - if self.jobs == 1: - total += 1 - logging.debug('enqueued(%s)' % d.name) - if self.progress: - self.progress._total = total - self.progress.update(0) - self.ready_cond.notifyAll() - finally: - self.ready_cond.release() + assert isinstance(d, WorkItem) + self.ready_cond.acquire() + try: + self.queued.append(d) + total = len(self.queued) + len(self.ran) + len(self.running) + if self.jobs == 1: + total += 1 + logging.debug('enqueued(%s)' % d.name) + if self.progress: + self.progress._total = total + self.progress.update(0) + self.ready_cond.notifyAll() + finally: + self.ready_cond.release() - def out_cb(self, _): - self.last_subproc_output = datetime.datetime.now() - return True + def out_cb(self, _): + self.last_subproc_output = datetime.datetime.now() + return True - @staticmethod - def format_task_output(task, comment=''): - if comment: - comment = ' (%s)' % comment - if task.start and task.finish: - elapsed = ' (Elapsed: %s)' % ( - str(task.finish - task.start).partition('.')[0]) - else: - elapsed = '' - return """ + @staticmethod + def format_task_output(task, comment=''): + if comment: + comment = ' (%s)' % comment + if task.start and task.finish: + elapsed = ' (Elapsed: %s)' % (str(task.finish - + task.start).partition('.')[0]) + else: + elapsed = '' + return """ %s%s%s ---------------------------------------- %s -----------------------------------------""" % ( - task.name, comment, elapsed, task.outbuf.getvalue().strip()) +----------------------------------------""" % (task.name, comment, elapsed, + task.outbuf.getvalue().strip()) - def _is_conflict(self, job): - """Checks to see if a job will conflict with another running job.""" - for running_job in self.running: - for used_resource in running_job.item.resources: - logging.debug('Checking resource %s' % used_resource) - if used_resource in job.resources: - return True - return False + def _is_conflict(self, job): + """Checks to see if a job will conflict with another running job.""" + for running_job in self.running: + for used_resource in running_job.item.resources: + logging.debug('Checking resource %s' % used_resource) + if used_resource in job.resources: + return True + return False - def flush(self, *args, **kwargs): - """Runs all enqueued items until all are executed.""" - kwargs['work_queue'] = self - self.last_subproc_output = self.last_join = datetime.datetime.now() - self.ready_cond.acquire() - try: - while True: - # Check for task to run first, then wait. - while True: - if not self.exceptions.empty(): - # Systematically flush the queue when an exception logged. - self.queued = [] - self._flush_terminated_threads() - if (not self.queued and not self.running or - self.jobs == len(self.running)): - logging.debug('No more worker threads or can\'t queue anything.') - break - - # Check for new tasks to start. - for i in range(len(self.queued)): - # Verify its requirements. - if (self.ignore_requirements or - not (set(self.queued[i].requirements) - set(self.ran))): - if not self._is_conflict(self.queued[i]): - # Start one work item: all its requirements are satisfied. - self._run_one_task(self.queued.pop(i), args, kwargs) - break - else: - # Couldn't find an item that could run. Break out the outher loop. - break - - if not self.queued and not self.running: - # We're done. - break - # We need to poll here otherwise Ctrl-C isn't processed. + def flush(self, *args, **kwargs): + """Runs all enqueued items until all are executed.""" + kwargs['work_queue'] = self + self.last_subproc_output = self.last_join = datetime.datetime.now() + self.ready_cond.acquire() try: - self.ready_cond.wait(10) - # If we haven't printed to terminal for a while, but we have received - # spew from a suprocess, let the user know we're still progressing. - now = datetime.datetime.now() - if (now - self.last_join > datetime.timedelta(seconds=60) and - self.last_subproc_output > self.last_join): - if self.progress: - print('') - sys.stdout.flush() - elapsed = Elapsed() - print('[%s] Still working on:' % elapsed) - sys.stdout.flush() - for task in self.running: - print('[%s] %s' % (elapsed, task.item.name)) - sys.stdout.flush() - except KeyboardInterrupt: - # Help debugging by printing some information: - print( - ('\nAllowed parallel jobs: %d\n# queued: %d\nRan: %s\n' - 'Running: %d') % (self.jobs, len(self.queued), ', '.join( - self.ran), len(self.running)), - file=sys.stderr) - for i in self.queued: - print( - '%s (not started): %s' % (i.name, ', '.join(i.requirements)), - file=sys.stderr) - for i in self.running: - print( - self.format_task_output(i.item, 'interrupted'), file=sys.stderr) - raise - # Something happened: self.enqueue() or a thread terminated. Loop again. - finally: - self.ready_cond.release() + while True: + # Check for task to run first, then wait. + while True: + if not self.exceptions.empty(): + # Systematically flush the queue when an exception + # logged. + self.queued = [] + self._flush_terminated_threads() + if (not self.queued and not self.running + or self.jobs == len(self.running)): + logging.debug( + 'No more worker threads or can\'t queue anything.') + break - assert not self.running, 'Now guaranteed to be single-threaded' - if not self.exceptions.empty(): - if self.progress: - print('') - # To get back the stack location correctly, the raise a, b, c form must be - # used, passing a tuple as the first argument doesn't work. - e, task = self.exceptions.get() - print(self.format_task_output(task.item, 'ERROR'), file=sys.stderr) - reraise(e[0], e[1], e[2]) - elif self.progress: - self.progress.end() + # Check for new tasks to start. + for i in range(len(self.queued)): + # Verify its requirements. + if (self.ignore_requirements + or not (set(self.queued[i].requirements) - + set(self.ran))): + if not self._is_conflict(self.queued[i]): + # Start one work item: all its requirements are + # satisfied. + self._run_one_task(self.queued.pop(i), args, + kwargs) + break + else: + # Couldn't find an item that could run. Break out the + # outher loop. + break - def _flush_terminated_threads(self): - """Flush threads that have terminated.""" - running = self.running - self.running = [] - for t in running: - if t.is_alive(): - self.running.append(t) - else: - t.join() - self.last_join = datetime.datetime.now() - sys.stdout.flush() - if self.verbose: - print(self.format_task_output(t.item)) - if self.progress: - self.progress.update(1, t.item.name) - if t.item.name in self.ran: - raise Error( - 'gclient is confused, "%s" is already in "%s"' % ( - t.item.name, ', '.join(self.ran))) - if not t.item.name in self.ran: - self.ran.append(t.item.name) - - def _run_one_task(self, task_item, args, kwargs): - if self.jobs > 1: - # Start the thread. - index = len(self.ran) + len(self.running) + 1 - new_thread = self._Worker(task_item, index, args, kwargs) - self.running.append(new_thread) - new_thread.start() - else: - # Run the 'thread' inside the main thread. Don't try to catch any - # exception. - try: - task_item.start = datetime.datetime.now() - print('[%s] Started.' % Elapsed(task_item.start), file=task_item.outbuf) - task_item.run(*args, **kwargs) - task_item.finish = datetime.datetime.now() - print( - '[%s] Finished.' % Elapsed(task_item.finish), file=task_item.outbuf) - self.ran.append(task_item.name) - if self.verbose: - if self.progress: - print('') - print(self.format_task_output(task_item)) - if self.progress: - self.progress.update(1, ', '.join(t.item.name for t in self.running)) - except KeyboardInterrupt: - print( - self.format_task_output(task_item, 'interrupted'), file=sys.stderr) - raise - except Exception: - print(self.format_task_output(task_item, 'ERROR'), file=sys.stderr) - raise - - - class _Worker(threading.Thread): - """One thread to execute one WorkItem.""" - def __init__(self, item, index, args, kwargs): - threading.Thread.__init__(self, name=item.name or 'Worker') - logging.info('_Worker(%s) reqs:%s' % (item.name, item.requirements)) - self.item = item - self.index = index - self.args = args - self.kwargs = kwargs - self.daemon = True - - def run(self): - """Runs in its own thread.""" - logging.debug('_Worker.run(%s)' % self.item.name) - work_queue = self.kwargs['work_queue'] - try: - self.item.start = datetime.datetime.now() - print('[%s] Started.' % Elapsed(self.item.start), file=self.item.outbuf) - self.item.run(*self.args, **self.kwargs) - self.item.finish = datetime.datetime.now() - print( - '[%s] Finished.' % Elapsed(self.item.finish), file=self.item.outbuf) - except KeyboardInterrupt: - logging.info('Caught KeyboardInterrupt in thread %s', self.item.name) - logging.info(str(sys.exc_info())) - work_queue.exceptions.put((sys.exc_info(), self)) - raise - except Exception: - # Catch exception location. - logging.info('Caught exception in thread %s', self.item.name) - logging.info(str(sys.exc_info())) - work_queue.exceptions.put((sys.exc_info(), self)) - finally: - logging.info('_Worker.run(%s) done', self.item.name) - work_queue.ready_cond.acquire() - try: - work_queue.ready_cond.notifyAll() + if not self.queued and not self.running: + # We're done. + break + # We need to poll here otherwise Ctrl-C isn't processed. + try: + self.ready_cond.wait(10) + # If we haven't printed to terminal for a while, but we have + # received spew from a suprocess, let the user know we're + # still progressing. + now = datetime.datetime.now() + if (now - self.last_join > datetime.timedelta(seconds=60) + and self.last_subproc_output > self.last_join): + if self.progress: + print('') + sys.stdout.flush() + elapsed = Elapsed() + print('[%s] Still working on:' % elapsed) + sys.stdout.flush() + for task in self.running: + print('[%s] %s' % (elapsed, task.item.name)) + sys.stdout.flush() + except KeyboardInterrupt: + # Help debugging by printing some information: + print( + ('\nAllowed parallel jobs: %d\n# queued: %d\nRan: %s\n' + 'Running: %d') % + (self.jobs, len(self.queued), ', '.join( + self.ran), len(self.running)), + file=sys.stderr) + for i in self.queued: + print('%s (not started): %s' % + (i.name, ', '.join(i.requirements)), + file=sys.stderr) + for i in self.running: + print(self.format_task_output(i.item, 'interrupted'), + file=sys.stderr) + raise + # Something happened: self.enqueue() or a thread terminated. + # Loop again. finally: - work_queue.ready_cond.release() + self.ready_cond.release() + + assert not self.running, 'Now guaranteed to be single-threaded' + if not self.exceptions.empty(): + if self.progress: + print('') + # To get back the stack location correctly, the raise a, b, c form + # must be used, passing a tuple as the first argument doesn't work. + e, task = self.exceptions.get() + print(self.format_task_output(task.item, 'ERROR'), file=sys.stderr) + reraise(e[0], e[1], e[2]) + elif self.progress: + self.progress.end() + + def _flush_terminated_threads(self): + """Flush threads that have terminated.""" + running = self.running + self.running = [] + for t in running: + if t.is_alive(): + self.running.append(t) + else: + t.join() + self.last_join = datetime.datetime.now() + sys.stdout.flush() + if self.verbose: + print(self.format_task_output(t.item)) + if self.progress: + self.progress.update(1, t.item.name) + if t.item.name in self.ran: + raise Error('gclient is confused, "%s" is already in "%s"' % + (t.item.name, ', '.join(self.ran))) + if not t.item.name in self.ran: + self.ran.append(t.item.name) + + def _run_one_task(self, task_item, args, kwargs): + if self.jobs > 1: + # Start the thread. + index = len(self.ran) + len(self.running) + 1 + new_thread = self._Worker(task_item, index, args, kwargs) + self.running.append(new_thread) + new_thread.start() + else: + # Run the 'thread' inside the main thread. Don't try to catch any + # exception. + try: + task_item.start = datetime.datetime.now() + print('[%s] Started.' % Elapsed(task_item.start), + file=task_item.outbuf) + task_item.run(*args, **kwargs) + task_item.finish = datetime.datetime.now() + print('[%s] Finished.' % Elapsed(task_item.finish), + file=task_item.outbuf) + self.ran.append(task_item.name) + if self.verbose: + if self.progress: + print('') + print(self.format_task_output(task_item)) + if self.progress: + self.progress.update( + 1, ', '.join(t.item.name for t in self.running)) + except KeyboardInterrupt: + print(self.format_task_output(task_item, 'interrupted'), + file=sys.stderr) + raise + except Exception: + print(self.format_task_output(task_item, 'ERROR'), + file=sys.stderr) + raise + + class _Worker(threading.Thread): + """One thread to execute one WorkItem.""" + def __init__(self, item, index, args, kwargs): + threading.Thread.__init__(self, name=item.name or 'Worker') + logging.info('_Worker(%s) reqs:%s' % (item.name, item.requirements)) + self.item = item + self.index = index + self.args = args + self.kwargs = kwargs + self.daemon = True + + def run(self): + """Runs in its own thread.""" + logging.debug('_Worker.run(%s)' % self.item.name) + work_queue = self.kwargs['work_queue'] + try: + self.item.start = datetime.datetime.now() + print('[%s] Started.' % Elapsed(self.item.start), + file=self.item.outbuf) + self.item.run(*self.args, **self.kwargs) + self.item.finish = datetime.datetime.now() + print('[%s] Finished.' % Elapsed(self.item.finish), + file=self.item.outbuf) + except KeyboardInterrupt: + logging.info('Caught KeyboardInterrupt in thread %s', + self.item.name) + logging.info(str(sys.exc_info())) + work_queue.exceptions.put((sys.exc_info(), self)) + raise + except Exception: + # Catch exception location. + logging.info('Caught exception in thread %s', self.item.name) + logging.info(str(sys.exc_info())) + work_queue.exceptions.put((sys.exc_info(), self)) + finally: + logging.info('_Worker.run(%s) done', self.item.name) + work_queue.ready_cond.acquire() + try: + work_queue.ready_cond.notifyAll() + finally: + work_queue.ready_cond.release() def GetEditor(git_editor=None): - """Returns the most plausible editor to use. + """Returns the most plausible editor to use. In order of preference: - GIT_EDITOR environment variable @@ -1110,180 +1136,185 @@ def GetEditor(git_editor=None): In the case of git-cl, this matches git's behaviour, except that it does not include dumb terminal detection. """ - editor = os.environ.get('GIT_EDITOR') or git_editor - if not editor: - editor = os.environ.get('VISUAL') - if not editor: - editor = os.environ.get('EDITOR') - if not editor: - if sys.platform.startswith('win'): - editor = 'notepad' - else: - editor = 'vi' - return editor + editor = os.environ.get('GIT_EDITOR') or git_editor + if not editor: + editor = os.environ.get('VISUAL') + if not editor: + editor = os.environ.get('EDITOR') + if not editor: + if sys.platform.startswith('win'): + editor = 'notepad' + else: + editor = 'vi' + return editor def RunEditor(content, git, git_editor=None): - """Opens up the default editor in the system to get the CL description.""" - editor = GetEditor(git_editor=git_editor) - if not editor: - return None - # Make sure CRLF is handled properly by requiring none. - if '\r' in content: - print( - '!! Please remove \\r from your change description !!', file=sys.stderr) + """Opens up the default editor in the system to get the CL description.""" + editor = GetEditor(git_editor=git_editor) + if not editor: + return None + # Make sure CRLF is handled properly by requiring none. + if '\r' in content: + print('!! Please remove \\r from your change description !!', + file=sys.stderr) - file_handle, filename = tempfile.mkstemp(text=True, prefix='cl_description.') - fileobj = os.fdopen(file_handle, 'wb') - # Still remove \r if present. - content = re.sub('\r?\n', '\n', content) - # Some editors complain when the file doesn't end in \n. - if not content.endswith('\n'): - content += '\n' + file_handle, filename = tempfile.mkstemp(text=True, + prefix='cl_description.') + fileobj = os.fdopen(file_handle, 'wb') + # Still remove \r if present. + content = re.sub('\r?\n', '\n', content) + # Some editors complain when the file doesn't end in \n. + if not content.endswith('\n'): + content += '\n' - if 'vim' in editor or editor == 'vi': - # If the user is using vim and has 'modelines' enabled, this will change the - # filetype from a generic auto-detected 'conf' to 'gitcommit', which is used - # to activate proper column wrapping, spell checking, syntax highlighting - # for git footers, etc. - # - # Because of the implementation of GetEditor above, we also check for the - # exact string 'vi' here, to help users get a sane default when they have vi - # symlink'd to vim (or something like vim). - fileobj.write('# vim: ft=gitcommit\n'.encode('utf-8')) + if 'vim' in editor or editor == 'vi': + # If the user is using vim and has 'modelines' enabled, this will change + # the filetype from a generic auto-detected 'conf' to 'gitcommit', which + # is used to activate proper column wrapping, spell checking, syntax + # highlighting for git footers, etc. + # + # Because of the implementation of GetEditor above, we also check for + # the exact string 'vi' here, to help users get a sane default when they + # have vi symlink'd to vim (or something like vim). + fileobj.write('# vim: ft=gitcommit\n'.encode('utf-8')) - fileobj.write(content.encode('utf-8')) - fileobj.close() + fileobj.write(content.encode('utf-8')) + fileobj.close() - try: - cmd = '%s %s' % (editor, filename) - if sys.platform == 'win32' and os.environ.get('TERM') == 'msys': - # Msysgit requires the usage of 'env' to be present. - cmd = 'env ' + cmd try: - # shell=True to allow the shell to handle all forms of quotes in - # $EDITOR. - subprocess2.check_call(cmd, shell=True) - except subprocess2.CalledProcessError: - return None - return FileRead(filename) - finally: - os.remove(filename) + cmd = '%s %s' % (editor, filename) + if sys.platform == 'win32' and os.environ.get('TERM') == 'msys': + # Msysgit requires the usage of 'env' to be present. + cmd = 'env ' + cmd + try: + # shell=True to allow the shell to handle all forms of quotes in + # $EDITOR. + subprocess2.check_call(cmd, shell=True) + except subprocess2.CalledProcessError: + return None + return FileRead(filename) + finally: + os.remove(filename) def UpgradeToHttps(url): - """Upgrades random urls to https://. + """Upgrades random urls to https://. Do not touch unknown urls like ssh:// or git://. Do not touch http:// urls with a port number, Fixes invalid GAE url. """ - if not url: - return url - if not re.match(r'[a-z\-]+\://.*', url): - # Make sure it is a valid uri. Otherwise, urlparse() will consider it a - # relative url and will use http:///foo. Note that it defaults to http:// - # for compatibility with naked url like "localhost:8080". - url = 'http://%s' % url - parsed = list(urllib.parse.urlparse(url)) - # Do not automatically upgrade http to https if a port number is provided. - if parsed[0] == 'http' and not re.match(r'^.+?\:\d+$', parsed[1]): - parsed[0] = 'https' - return urllib.parse.urlunparse(parsed) + if not url: + return url + if not re.match(r'[a-z\-]+\://.*', url): + # Make sure it is a valid uri. Otherwise, urlparse() will consider it a + # relative url and will use http:///foo. Note that it defaults to + # http:// for compatibility with naked url like "localhost:8080". + url = 'http://%s' % url + parsed = list(urllib.parse.urlparse(url)) + # Do not automatically upgrade http to https if a port number is provided. + if parsed[0] == 'http' and not re.match(r'^.+?\:\d+$', parsed[1]): + parsed[0] = 'https' + return urllib.parse.urlunparse(parsed) def ParseCodereviewSettingsContent(content): - """Process a codereview.settings file properly.""" - lines = (l for l in content.splitlines() if not l.strip().startswith("#")) - try: - keyvals = dict([x.strip() for x in l.split(':', 1)] for l in lines if l) - except ValueError: - raise Error( - 'Failed to process settings, please fix. Content:\n\n%s' % content) - def fix_url(key): - if keyvals.get(key): - keyvals[key] = UpgradeToHttps(keyvals[key]) - fix_url('CODE_REVIEW_SERVER') - fix_url('VIEW_VC') - return keyvals + """Process a codereview.settings file properly.""" + lines = (l for l in content.splitlines() if not l.strip().startswith("#")) + try: + keyvals = dict([x.strip() for x in l.split(':', 1)] for l in lines if l) + except ValueError: + raise Error('Failed to process settings, please fix. Content:\n\n%s' % + content) + + def fix_url(key): + if keyvals.get(key): + keyvals[key] = UpgradeToHttps(keyvals[key]) + + fix_url('CODE_REVIEW_SERVER') + fix_url('VIEW_VC') + return keyvals def NumLocalCpus(): - """Returns the number of processors. + """Returns the number of processors. multiprocessing.cpu_count() is permitted to raise NotImplementedError, and is known to do this on some Windows systems and OSX 10.6. If we can't get the CPU count, we will fall back to '1'. """ - # Surround the entire thing in try/except; no failure here should stop gclient - # from working. - try: - # Use multiprocessing to get CPU count. This may raise - # NotImplementedError. + # Surround the entire thing in try/except; no failure here should stop + # gclient from working. try: - import multiprocessing - return multiprocessing.cpu_count() - except NotImplementedError: # pylint: disable=bare-except - # (UNIX) Query 'os.sysconf'. - # pylint: disable=no-member - if hasattr(os, 'sysconf') and 'SC_NPROCESSORS_ONLN' in os.sysconf_names: - return int(os.sysconf('SC_NPROCESSORS_ONLN')) + # Use multiprocessing to get CPU count. This may raise + # NotImplementedError. + try: + import multiprocessing + return multiprocessing.cpu_count() + except NotImplementedError: # pylint: disable=bare-except + # (UNIX) Query 'os.sysconf'. + # pylint: disable=no-member + if hasattr(os, + 'sysconf') and 'SC_NPROCESSORS_ONLN' in os.sysconf_names: + return int(os.sysconf('SC_NPROCESSORS_ONLN')) - # (Windows) Query 'NUMBER_OF_PROCESSORS' environment variable. - if 'NUMBER_OF_PROCESSORS' in os.environ: - return int(os.environ['NUMBER_OF_PROCESSORS']) - except Exception as e: - logging.exception("Exception raised while probing CPU count: %s", e) + # (Windows) Query 'NUMBER_OF_PROCESSORS' environment variable. + if 'NUMBER_OF_PROCESSORS' in os.environ: + return int(os.environ['NUMBER_OF_PROCESSORS']) + except Exception as e: + logging.exception("Exception raised while probing CPU count: %s", e) - logging.debug('Failed to get CPU count. Defaulting to 1.') - return 1 + logging.debug('Failed to get CPU count. Defaulting to 1.') + return 1 def DefaultDeltaBaseCacheLimit(): - """Return a reasonable default for the git config core.deltaBaseCacheLimit. + """Return a reasonable default for the git config core.deltaBaseCacheLimit. The primary constraint is the address space of virtual memory. The cache size limit is per-thread, and 32-bit systems can hit OOM errors if this parameter is set too high. """ - if platform.architecture()[0].startswith('64'): - return '2g' + if platform.architecture()[0].startswith('64'): + return '2g' - return '512m' + return '512m' def DefaultIndexPackConfig(url=''): - """Return reasonable default values for configuring git-index-pack. + """Return reasonable default values for configuring git-index-pack. Experiments suggest that higher values for pack.threads don't improve performance.""" - cache_limit = DefaultDeltaBaseCacheLimit() - result = ['-c', 'core.deltaBaseCacheLimit=%s' % cache_limit] - if url in THREADED_INDEX_PACK_BLOCKLIST: - result.extend(['-c', 'pack.threads=1']) - return result + cache_limit = DefaultDeltaBaseCacheLimit() + result = ['-c', 'core.deltaBaseCacheLimit=%s' % cache_limit] + if url in THREADED_INDEX_PACK_BLOCKLIST: + result.extend(['-c', 'pack.threads=1']) + return result def FindExecutable(executable): - """This mimics the "which" utility.""" - path_folders = os.environ.get('PATH').split(os.pathsep) + """This mimics the "which" utility.""" + path_folders = os.environ.get('PATH').split(os.pathsep) - for path_folder in path_folders: - target = os.path.join(path_folder, executable) - # Just in case we have some ~/blah paths. - target = os.path.abspath(os.path.expanduser(target)) - if os.path.isfile(target) and os.access(target, os.X_OK): - return target - if sys.platform.startswith('win'): - for suffix in ('.bat', '.cmd', '.exe'): - alt_target = target + suffix - if os.path.isfile(alt_target) and os.access(alt_target, os.X_OK): - return alt_target - return None + for path_folder in path_folders: + target = os.path.join(path_folder, executable) + # Just in case we have some ~/blah paths. + target = os.path.abspath(os.path.expanduser(target)) + if os.path.isfile(target) and os.access(target, os.X_OK): + return target + if sys.platform.startswith('win'): + for suffix in ('.bat', '.cmd', '.exe'): + alt_target = target + suffix + if os.path.isfile(alt_target) and os.access( + alt_target, os.X_OK): + return alt_target + return None def freeze(obj): - """Takes a generic object ``obj``, and returns an immutable version of it. + """Takes a generic object ``obj``, and returns an immutable version of it. Supported types: * dict / OrderedDict -> FrozenDict @@ -1294,55 +1325,56 @@ def freeze(obj): Will raise TypeError if you pass an object which is not hashable. """ - if isinstance(obj, collections.abc.Mapping): - return FrozenDict((freeze(k), freeze(v)) for k, v in obj.items()) + if isinstance(obj, collections.abc.Mapping): + return FrozenDict((freeze(k), freeze(v)) for k, v in obj.items()) - if isinstance(obj, (list, tuple)): - return tuple(freeze(i) for i in obj) + if isinstance(obj, (list, tuple)): + return tuple(freeze(i) for i in obj) - if isinstance(obj, set): - return frozenset(freeze(i) for i in obj) + if isinstance(obj, set): + return frozenset(freeze(i) for i in obj) - hash(obj) - return obj + hash(obj) + return obj class FrozenDict(collections.abc.Mapping): - """An immutable OrderedDict. + """An immutable OrderedDict. Modified From: http://stackoverflow.com/a/2704866 """ - def __init__(self, *args, **kwargs): - self._d = collections.OrderedDict(*args, **kwargs) + def __init__(self, *args, **kwargs): + self._d = collections.OrderedDict(*args, **kwargs) - # Calculate the hash immediately so that we know all the items are - # hashable too. - self._hash = functools.reduce( - operator.xor, (hash(i) for i in enumerate(self._d.items())), 0) + # Calculate the hash immediately so that we know all the items are + # hashable too. + self._hash = functools.reduce(operator.xor, + (hash(i) + for i in enumerate(self._d.items())), 0) - def __eq__(self, other): - if not isinstance(other, collections.abc.Mapping): - return NotImplemented - if self is other: - return True - if len(self) != len(other): - return False - for k, v in self.items(): - if k not in other or other[k] != v: - return False - return True + def __eq__(self, other): + if not isinstance(other, collections.abc.Mapping): + return NotImplemented + if self is other: + return True + if len(self) != len(other): + return False + for k, v in self.items(): + if k not in other or other[k] != v: + return False + return True - def __iter__(self): - return iter(self._d) + def __iter__(self): + return iter(self._d) - def __len__(self): - return len(self._d) + def __len__(self): + return len(self._d) - def __getitem__(self, key): - return self._d[key] + def __getitem__(self, key): + return self._d[key] - def __hash__(self): - return self._hash + def __hash__(self): + return self._hash - def __repr__(self): - return 'FrozenDict(%r)' % (self._d.items(),) + def __repr__(self): + return 'FrozenDict(%r)' % (self._d.items(), ) diff --git a/gerrit_client.py b/gerrit_client.py index 8ea4df4d1b..4feb146e96 100755 --- a/gerrit_client.py +++ b/gerrit_client.py @@ -2,7 +2,6 @@ # Copyright 2017 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Simple client for the Gerrit REST API. Example usage: @@ -24,376 +23,397 @@ __version__ = '0.1' def write_result(result, opt): - if opt.json_file: - with open(opt.json_file, 'w') as json_file: - json_file.write(json.dumps(result)) + if opt.json_file: + with open(opt.json_file, 'w') as json_file: + json_file.write(json.dumps(result)) @subcommand.usage('[args ...]') def CMDmovechanges(parser, args): - """Move changes to a different destination branch.""" - parser.add_option('-p', '--param', dest='params', action='append', - help='repeatable query parameter, format: -p key=value') - parser.add_option('--destination_branch', dest='destination_branch', - help='where to move changes to') + """Move changes to a different destination branch.""" + parser.add_option('-p', + '--param', + dest='params', + action='append', + help='repeatable query parameter, format: -p key=value') + parser.add_option('--destination_branch', + dest='destination_branch', + help='where to move changes to') - (opt, args) = parser.parse_args(args) - assert opt.destination_branch, "--destination_branch not defined" - for p in opt.params: - assert '=' in p, '--param is key=value, not "%s"' % p - host = urllib.parse.urlparse(opt.host).netloc + (opt, args) = parser.parse_args(args) + assert opt.destination_branch, "--destination_branch not defined" + for p in opt.params: + assert '=' in p, '--param is key=value, not "%s"' % p + host = urllib.parse.urlparse(opt.host).netloc - limit = 100 - while True: - result = gerrit_util.QueryChanges( - host, - list(tuple(p.split('=', 1)) for p in opt.params), - limit=limit, - ) - for change in result: - gerrit_util.MoveChange(host, change['id'], opt.destination_branch) + limit = 100 + while True: + result = gerrit_util.QueryChanges( + host, + list(tuple(p.split('=', 1)) for p in opt.params), + limit=limit, + ) + for change in result: + gerrit_util.MoveChange(host, change['id'], opt.destination_branch) - if len(result) < limit: - break - logging.info("Done") + if len(result) < limit: + break + logging.info("Done") @subcommand.usage('[args ...]') def CMDbranchinfo(parser, args): - """Get information on a gerrit branch.""" - parser.add_option('--branch', dest='branch', help='branch name') + """Get information on a gerrit branch.""" + parser.add_option('--branch', dest='branch', help='branch name') - (opt, args) = parser.parse_args(args) - host = urllib.parse.urlparse(opt.host).netloc - project = urllib.parse.quote_plus(opt.project) - branch = urllib.parse.quote_plus(opt.branch) - result = gerrit_util.GetGerritBranch(host, project, branch) - logging.info(result) - write_result(result, opt) + (opt, args) = parser.parse_args(args) + host = urllib.parse.urlparse(opt.host).netloc + project = urllib.parse.quote_plus(opt.project) + branch = urllib.parse.quote_plus(opt.branch) + result = gerrit_util.GetGerritBranch(host, project, branch) + logging.info(result) + write_result(result, opt) @subcommand.usage('[args ...]') def CMDrawapi(parser, args): - """Call an arbitrary Gerrit REST API endpoint.""" - parser.add_option('--path', dest='path', help='HTTP path of the API endpoint') - parser.add_option('--method', dest='method', - help='HTTP method for the API (default: GET)') - parser.add_option('--body', dest='body', help='API JSON body contents') - parser.add_option('--accept_status', - dest='accept_status', - help='Comma-delimited list of status codes for success.') + """Call an arbitrary Gerrit REST API endpoint.""" + parser.add_option('--path', + dest='path', + help='HTTP path of the API endpoint') + parser.add_option('--method', + dest='method', + help='HTTP method for the API (default: GET)') + parser.add_option('--body', dest='body', help='API JSON body contents') + parser.add_option('--accept_status', + dest='accept_status', + help='Comma-delimited list of status codes for success.') - (opt, args) = parser.parse_args(args) - assert opt.path, "--path not defined" + (opt, args) = parser.parse_args(args) + assert opt.path, "--path not defined" - host = urllib.parse.urlparse(opt.host).netloc - kwargs = {} - if opt.method: - kwargs['reqtype'] = opt.method.upper() - if opt.body: - kwargs['body'] = json.loads(opt.body) - if opt.accept_status: - kwargs['accept_statuses'] = [int(x) for x in opt.accept_status.split(',')] - result = gerrit_util.CallGerritApi(host, opt.path, **kwargs) - logging.info(result) - write_result(result, opt) + host = urllib.parse.urlparse(opt.host).netloc + kwargs = {} + if opt.method: + kwargs['reqtype'] = opt.method.upper() + if opt.body: + kwargs['body'] = json.loads(opt.body) + if opt.accept_status: + kwargs['accept_statuses'] = [ + int(x) for x in opt.accept_status.split(',') + ] + result = gerrit_util.CallGerritApi(host, opt.path, **kwargs) + logging.info(result) + write_result(result, opt) @subcommand.usage('[args ...]') def CMDbranch(parser, args): - """Create a branch in a gerrit project.""" - parser.add_option('--branch', dest='branch', help='branch name') - parser.add_option('--commit', dest='commit', help='commit hash') - parser.add_option('--allow-existent-branch', - action='store_true', - help=('Accept that the branch alread exists as long as the' - ' branch head points the given commit')) + """Create a branch in a gerrit project.""" + parser.add_option('--branch', dest='branch', help='branch name') + parser.add_option('--commit', dest='commit', help='commit hash') + parser.add_option( + '--allow-existent-branch', + action='store_true', + help=('Accept that the branch alread exists as long as the' + ' branch head points the given commit')) - (opt, args) = parser.parse_args(args) - assert opt.project, "--project not defined" - assert opt.branch, "--branch not defined" - assert opt.commit, "--commit not defined" + (opt, args) = parser.parse_args(args) + assert opt.project, "--project not defined" + assert opt.branch, "--branch not defined" + assert opt.commit, "--commit not defined" - project = urllib.parse.quote_plus(opt.project) - host = urllib.parse.urlparse(opt.host).netloc - branch = urllib.parse.quote_plus(opt.branch) - result = gerrit_util.GetGerritBranch(host, project, branch) - if result: - if not opt.allow_existent_branch: - raise gerrit_util.GerritError(200, 'Branch already exists') - if result.get('revision') != opt.commit: - raise gerrit_util.GerritError( - 200, ('Branch already exists but ' - 'the branch head is not at the given commit')) - else: - try: - result = gerrit_util.CreateGerritBranch(host, project, branch, opt.commit) - except gerrit_util.GerritError as e: - result = gerrit_util.GetGerritBranch(host, project, branch) - if not result: - raise e - # If reached here, we hit a real conflict error, because the - # branch just created is pointing a different commit. - if result.get('revision') != opt.commit: - raise gerrit_util.GerritError( - 200, ('Conflict: branch was created but ' - 'the branch head is not at the given commit')) - logging.info(result) - write_result(result, opt) + project = urllib.parse.quote_plus(opt.project) + host = urllib.parse.urlparse(opt.host).netloc + branch = urllib.parse.quote_plus(opt.branch) + result = gerrit_util.GetGerritBranch(host, project, branch) + if result: + if not opt.allow_existent_branch: + raise gerrit_util.GerritError(200, 'Branch already exists') + if result.get('revision') != opt.commit: + raise gerrit_util.GerritError( + 200, ('Branch already exists but ' + 'the branch head is not at the given commit')) + else: + try: + result = gerrit_util.CreateGerritBranch(host, project, branch, + opt.commit) + except gerrit_util.GerritError as e: + result = gerrit_util.GetGerritBranch(host, project, branch) + if not result: + raise e + # If reached here, we hit a real conflict error, because the + # branch just created is pointing a different commit. + if result.get('revision') != opt.commit: + raise gerrit_util.GerritError( + 200, ('Conflict: branch was created but ' + 'the branch head is not at the given commit')) + logging.info(result) + write_result(result, opt) @subcommand.usage('[args ...]') def CMDtag(parser, args): - """Create a tag in a gerrit project.""" - parser.add_option('--tag', dest='tag', help='tag name') - parser.add_option('--commit', dest='commit', help='commit hash') + """Create a tag in a gerrit project.""" + parser.add_option('--tag', dest='tag', help='tag name') + parser.add_option('--commit', dest='commit', help='commit hash') - (opt, args) = parser.parse_args(args) - assert opt.project, "--project not defined" - assert opt.tag, "--tag not defined" - assert opt.commit, "--commit not defined" + (opt, args) = parser.parse_args(args) + assert opt.project, "--project not defined" + assert opt.tag, "--tag not defined" + assert opt.commit, "--commit not defined" - project = urllib.parse.quote_plus(opt.project) - host = urllib.parse.urlparse(opt.host).netloc - tag = urllib.parse.quote_plus(opt.tag) - result = gerrit_util.CreateGerritTag(host, project, tag, opt.commit) - logging.info(result) - write_result(result, opt) + project = urllib.parse.quote_plus(opt.project) + host = urllib.parse.urlparse(opt.host).netloc + tag = urllib.parse.quote_plus(opt.tag) + result = gerrit_util.CreateGerritTag(host, project, tag, opt.commit) + logging.info(result) + write_result(result, opt) @subcommand.usage('[args ...]') def CMDhead(parser, args): - """Update which branch the project HEAD points to.""" - parser.add_option('--branch', dest='branch', help='branch name') + """Update which branch the project HEAD points to.""" + parser.add_option('--branch', dest='branch', help='branch name') - (opt, args) = parser.parse_args(args) - assert opt.project, "--project not defined" - assert opt.branch, "--branch not defined" + (opt, args) = parser.parse_args(args) + assert opt.project, "--project not defined" + assert opt.branch, "--branch not defined" - project = urllib.parse.quote_plus(opt.project) - host = urllib.parse.urlparse(opt.host).netloc - branch = urllib.parse.quote_plus(opt.branch) - result = gerrit_util.UpdateHead(host, project, branch) - logging.info(result) - write_result(result, opt) + project = urllib.parse.quote_plus(opt.project) + host = urllib.parse.urlparse(opt.host).netloc + branch = urllib.parse.quote_plus(opt.branch) + result = gerrit_util.UpdateHead(host, project, branch) + logging.info(result) + write_result(result, opt) @subcommand.usage('[args ...]') def CMDheadinfo(parser, args): - """Retrieves the current HEAD of the project.""" + """Retrieves the current HEAD of the project.""" - (opt, args) = parser.parse_args(args) - assert opt.project, "--project not defined" + (opt, args) = parser.parse_args(args) + assert opt.project, "--project not defined" - project = urllib.parse.quote_plus(opt.project) - host = urllib.parse.urlparse(opt.host).netloc - result = gerrit_util.GetHead(host, project) - logging.info(result) - write_result(result, opt) + project = urllib.parse.quote_plus(opt.project) + host = urllib.parse.urlparse(opt.host).netloc + result = gerrit_util.GetHead(host, project) + logging.info(result) + write_result(result, opt) @subcommand.usage('[args ...]') def CMDchanges(parser, args): - """Queries gerrit for matching changes.""" - parser.add_option('-p', - '--param', - dest='params', - action='append', - default=[], - help='repeatable query parameter, format: -p key=value') - parser.add_option('--query', help='raw gerrit search query string') - parser.add_option('-o', '--o-param', dest='o_params', action='append', - help='gerrit output parameters, e.g. ALL_REVISIONS') - parser.add_option('--limit', dest='limit', type=int, - help='maximum number of results to return') - parser.add_option('--start', dest='start', type=int, - help='how many changes to skip ' - '(starting with the most recent)') + """Queries gerrit for matching changes.""" + parser.add_option('-p', + '--param', + dest='params', + action='append', + default=[], + help='repeatable query parameter, format: -p key=value') + parser.add_option('--query', help='raw gerrit search query string') + parser.add_option('-o', + '--o-param', + dest='o_params', + action='append', + help='gerrit output parameters, e.g. ALL_REVISIONS') + parser.add_option('--limit', + dest='limit', + type=int, + help='maximum number of results to return') + parser.add_option('--start', + dest='start', + type=int, + help='how many changes to skip ' + '(starting with the most recent)') - (opt, args) = parser.parse_args(args) - assert opt.params or opt.query, '--param or --query required' - for p in opt.params: - assert '=' in p, '--param is key=value, not "%s"' % p + (opt, args) = parser.parse_args(args) + assert opt.params or opt.query, '--param or --query required' + for p in opt.params: + assert '=' in p, '--param is key=value, not "%s"' % p - result = gerrit_util.QueryChanges( - urllib.parse.urlparse(opt.host).netloc, - list(tuple(p.split('=', 1)) for p in opt.params), - first_param=opt.query, - start=opt.start, # Default: None - limit=opt.limit, # Default: None - o_params=opt.o_params, # Default: None - ) - logging.info('Change query returned %d changes.', len(result)) - write_result(result, opt) + result = gerrit_util.QueryChanges( + urllib.parse.urlparse(opt.host).netloc, + list(tuple(p.split('=', 1)) for p in opt.params), + first_param=opt.query, + start=opt.start, # Default: None + limit=opt.limit, # Default: None + o_params=opt.o_params, # Default: None + ) + logging.info('Change query returned %d changes.', len(result)) + write_result(result, opt) @subcommand.usage('[args ...]') def CMDrelatedchanges(parser, args): - """Gets related changes for a given change and revision.""" - parser.add_option('-c', '--change', type=str, help='change id') - parser.add_option('-r', '--revision', type=str, help='revision id') + """Gets related changes for a given change and revision.""" + parser.add_option('-c', '--change', type=str, help='change id') + parser.add_option('-r', '--revision', type=str, help='revision id') - (opt, args) = parser.parse_args(args) + (opt, args) = parser.parse_args(args) - result = gerrit_util.GetRelatedChanges( - urllib.parse.urlparse(opt.host).netloc, - change=opt.change, - revision=opt.revision, - ) - logging.info(result) - write_result(result, opt) + result = gerrit_util.GetRelatedChanges( + urllib.parse.urlparse(opt.host).netloc, + change=opt.change, + revision=opt.revision, + ) + logging.info(result) + write_result(result, opt) @subcommand.usage('[args ...]') def CMDcreatechange(parser, args): - """Create a new change in gerrit.""" - parser.add_option('-s', '--subject', help='subject for change') - parser.add_option('-b', - '--branch', - default='main', - help='target branch for change') - parser.add_option( - '-p', - '--param', - dest='params', - action='append', - help='repeatable field value parameter, format: -p key=value') + """Create a new change in gerrit.""" + parser.add_option('-s', '--subject', help='subject for change') + parser.add_option('-b', + '--branch', + default='main', + help='target branch for change') + parser.add_option( + '-p', + '--param', + dest='params', + action='append', + help='repeatable field value parameter, format: -p key=value') - parser.add_option('--cc', - dest='cc_list', - action='append', - help='CC address to notify, format: --cc foo@example.com') + parser.add_option('--cc', + dest='cc_list', + action='append', + help='CC address to notify, format: --cc foo@example.com') - (opt, args) = parser.parse_args(args) - for p in opt.params: - assert '=' in p, '--param is key=value, not "%s"' % p + (opt, args) = parser.parse_args(args) + for p in opt.params: + assert '=' in p, '--param is key=value, not "%s"' % p - params = list(tuple(p.split('=', 1)) for p in opt.params) + params = list(tuple(p.split('=', 1)) for p in opt.params) - if opt.cc_list: - params.append(('notify_details', {'CC': {'accounts': opt.cc_list}})) + if opt.cc_list: + params.append(('notify_details', {'CC': {'accounts': opt.cc_list}})) - result = gerrit_util.CreateChange( - urllib.parse.urlparse(opt.host).netloc, - opt.project, - branch=opt.branch, - subject=opt.subject, - params=params, - ) - logging.info(result) - write_result(result, opt) + result = gerrit_util.CreateChange( + urllib.parse.urlparse(opt.host).netloc, + opt.project, + branch=opt.branch, + subject=opt.subject, + params=params, + ) + logging.info(result) + write_result(result, opt) @subcommand.usage('[args ...]') def CMDchangeedit(parser, args): - """Puts content of a file into a change edit.""" - parser.add_option('-c', '--change', type=int, help='change number') - parser.add_option('--path', help='path for file') - parser.add_option('--file', help='file to place at |path|') + """Puts content of a file into a change edit.""" + parser.add_option('-c', '--change', type=int, help='change number') + parser.add_option('--path', help='path for file') + parser.add_option('--file', help='file to place at |path|') - (opt, args) = parser.parse_args(args) + (opt, args) = parser.parse_args(args) - with open(opt.file) as f: - data = f.read() - result = gerrit_util.ChangeEdit( - urllib.parse.urlparse(opt.host).netloc, opt.change, opt.path, data) - logging.info(result) - write_result(result, opt) + with open(opt.file) as f: + data = f.read() + result = gerrit_util.ChangeEdit( + urllib.parse.urlparse(opt.host).netloc, opt.change, opt.path, data) + logging.info(result) + write_result(result, opt) @subcommand.usage('[args ...]') def CMDpublishchangeedit(parser, args): - """Publish a Gerrit change edit.""" - parser.add_option('-c', '--change', type=int, help='change number') - parser.add_option('--notify', help='whether to notify') + """Publish a Gerrit change edit.""" + parser.add_option('-c', '--change', type=int, help='change number') + parser.add_option('--notify', help='whether to notify') - (opt, args) = parser.parse_args(args) + (opt, args) = parser.parse_args(args) - result = gerrit_util.PublishChangeEdit( - urllib.parse.urlparse(opt.host).netloc, opt.change, opt.notify) - logging.info(result) - write_result(result, opt) + result = gerrit_util.PublishChangeEdit( + urllib.parse.urlparse(opt.host).netloc, opt.change, opt.notify) + logging.info(result) + write_result(result, opt) @subcommand.usage('[args ...]') def CMDsubmitchange(parser, args): - """Submit a Gerrit change.""" - parser.add_option('-c', '--change', type=int, help='change number') - (opt, args) = parser.parse_args(args) - result = gerrit_util.SubmitChange( - urllib.parse.urlparse(opt.host).netloc, opt.change) - logging.info(result) - write_result(result, opt) + """Submit a Gerrit change.""" + parser.add_option('-c', '--change', type=int, help='change number') + (opt, args) = parser.parse_args(args) + result = gerrit_util.SubmitChange( + urllib.parse.urlparse(opt.host).netloc, opt.change) + logging.info(result) + write_result(result, opt) @subcommand.usage('[args ...]') def CMDchangesubmittedtogether(parser, args): - """Get all changes submitted with the given one.""" - parser.add_option('-c', '--change', type=int, help='change number') - (opt, args) = parser.parse_args(args) - result = gerrit_util.GetChangesSubmittedTogether( - urllib.parse.urlparse(opt.host).netloc, opt.change) - logging.info(result) - write_result(result, opt) + """Get all changes submitted with the given one.""" + parser.add_option('-c', '--change', type=int, help='change number') + (opt, args) = parser.parse_args(args) + result = gerrit_util.GetChangesSubmittedTogether( + urllib.parse.urlparse(opt.host).netloc, opt.change) + logging.info(result) + write_result(result, opt) @subcommand.usage('[args ...]') def CMDgetcommitincludedin(parser, args): - """Retrieves the branches and tags for a given commit.""" - parser.add_option('--commit', dest='commit', help='commit hash') - (opt, args) = parser.parse_args(args) - result = gerrit_util.GetCommitIncludedIn( - urllib.parse.urlparse(opt.host).netloc, opt.project, opt.commit) - logging.info(result) - write_result(result, opt) + """Retrieves the branches and tags for a given commit.""" + parser.add_option('--commit', dest='commit', help='commit hash') + (opt, args) = parser.parse_args(args) + result = gerrit_util.GetCommitIncludedIn( + urllib.parse.urlparse(opt.host).netloc, opt.project, opt.commit) + logging.info(result) + write_result(result, opt) @subcommand.usage('[args ...]') def CMDsetbotcommit(parser, args): - """Sets bot-commit+1 to a bot generated change.""" - parser.add_option('-c', '--change', type=int, help='change number') - (opt, args) = parser.parse_args(args) - result = gerrit_util.SetReview(urllib.parse.urlparse(opt.host).netloc, - opt.change, - labels={'Bot-Commit': 1}, - ready=True) - logging.info(result) - write_result(result, opt) + """Sets bot-commit+1 to a bot generated change.""" + parser.add_option('-c', '--change', type=int, help='change number') + (opt, args) = parser.parse_args(args) + result = gerrit_util.SetReview(urllib.parse.urlparse(opt.host).netloc, + opt.change, + labels={'Bot-Commit': 1}, + ready=True) + logging.info(result) + write_result(result, opt) @subcommand.usage('[args ...]') def CMDsetlabel(parser, args): - """Sets a label to a specific value on a given change.""" - parser.add_option('-c', '--change', type=int, help='change number') - parser.add_option('-l', - '--label', - nargs=2, - metavar=('label_name', 'label_value')) - (opt, args) = parser.parse_args(args) - result = gerrit_util.SetReview(urllib.parse.urlparse(opt.host).netloc, - opt.change, - labels={opt.label[0]: opt.label[1]}) - logging.info(result) - write_result(result, opt) + """Sets a label to a specific value on a given change.""" + parser.add_option('-c', '--change', type=int, help='change number') + parser.add_option('-l', + '--label', + nargs=2, + metavar=('label_name', 'label_value')) + (opt, args) = parser.parse_args(args) + result = gerrit_util.SetReview(urllib.parse.urlparse(opt.host).netloc, + opt.change, + labels={opt.label[0]: opt.label[1]}) + logging.info(result) + write_result(result, opt) @subcommand.usage('') def CMDabandon(parser, args): - """Abandons a Gerrit change.""" - parser.add_option('-c', '--change', type=int, help='change number') - parser.add_option('-m', '--message', default='', help='reason for abandoning') + """Abandons a Gerrit change.""" + parser.add_option('-c', '--change', type=int, help='change number') + parser.add_option('-m', + '--message', + default='', + help='reason for abandoning') - (opt, args) = parser.parse_args(args) - assert opt.change, "-c not defined" - result = gerrit_util.AbandonChange( - urllib.parse.urlparse(opt.host).netloc, opt.change, opt.message) - logging.info(result) - write_result(result, opt) + (opt, args) = parser.parse_args(args) + assert opt.change, "-c not defined" + result = gerrit_util.AbandonChange( + urllib.parse.urlparse(opt.host).netloc, opt.change, opt.message) + logging.info(result) + write_result(result, opt) @subcommand.usage('') def CMDmass_abandon(parser, args): - """Mass abandon changes + """Mass abandon changes Abandons CLs that match search criteria provided by user. Before any change is actually abandoned, user is presented with a list of CLs that will be affected @@ -406,98 +426,106 @@ def CMDmass_abandon(parser, args): gerrit_client.py mass-abandon --host https://HOST -p 'message=testing' gerrit_client.py mass-abandon --host https://HOST -p 'is=wip' -p 'age=1y' """ - parser.add_option('-p', - '--param', - dest='params', - action='append', - default=[], - help='repeatable query parameter, format: -p key=value') - parser.add_option('-m', '--message', default='', help='reason for abandoning') - parser.add_option('-f', - '--force', - action='store_true', - help='Don\'t prompt for confirmation') + parser.add_option('-p', + '--param', + dest='params', + action='append', + default=[], + help='repeatable query parameter, format: -p key=value') + parser.add_option('-m', + '--message', + default='', + help='reason for abandoning') + parser.add_option('-f', + '--force', + action='store_true', + help='Don\'t prompt for confirmation') - opt, args = parser.parse_args(args) + opt, args = parser.parse_args(args) - for p in opt.params: - assert '=' in p, '--param is key=value, not "%s"' % p - search_query = list(tuple(p.split('=', 1)) for p in opt.params) - if not any(t for t in search_query if t[0] == 'owner'): - # owner should always be present when abandoning changes - search_query.append(('owner', 'me')) - search_query.append(('status', 'open')) - logging.info("Searching for: %s" % search_query) + for p in opt.params: + assert '=' in p, '--param is key=value, not "%s"' % p + search_query = list(tuple(p.split('=', 1)) for p in opt.params) + if not any(t for t in search_query if t[0] == 'owner'): + # owner should always be present when abandoning changes + search_query.append(('owner', 'me')) + search_query.append(('status', 'open')) + logging.info("Searching for: %s" % search_query) - host = urllib.parse.urlparse(opt.host).netloc + host = urllib.parse.urlparse(opt.host).netloc - result = gerrit_util.QueryChanges( - host, - search_query, - # abandon at most 100 changes as not all Gerrit instances support - # unlimited results. - limit=100, - ) - if len(result) == 0: - logging.warning("Nothing to abandon") - return + result = gerrit_util.QueryChanges( + host, + search_query, + # abandon at most 100 changes as not all Gerrit instances support + # unlimited results. + limit=100, + ) + if len(result) == 0: + logging.warning("Nothing to abandon") + return - logging.warning("%s CLs match search query: " % len(result)) - for change in result: - logging.warning("[ID: %d] %s" % (change['_number'], change['subject'])) + logging.warning("%s CLs match search query: " % len(result)) + for change in result: + logging.warning("[ID: %d] %s" % (change['_number'], change['subject'])) - if not opt.force: - q = input( - 'Do you want to move forward with abandoning? [y to confirm] ').strip() - if q not in ['y', 'Y']: - logging.warning("Aborting...") - return + if not opt.force: + q = input('Do you want to move forward with abandoning? [y to confirm] ' + ).strip() + if q not in ['y', 'Y']: + logging.warning("Aborting...") + return - for change in result: - logging.warning("Abandoning: %s" % change['subject']) - gerrit_util.AbandonChange(host, change['id'], opt.message) + for change in result: + logging.warning("Abandoning: %s" % change['subject']) + gerrit_util.AbandonChange(host, change['id'], opt.message) - logging.warning("Done") + logging.warning("Done") class OptionParser(optparse.OptionParser): - """Creates the option parse and add --verbose support.""" - def __init__(self, *args, **kwargs): - optparse.OptionParser.__init__(self, *args, version=__version__, **kwargs) - self.add_option( - '--verbose', action='count', default=0, - help='Use 2 times for more debugging info') - self.add_option('--host', dest='host', help='Url of host.') - self.add_option('--project', dest='project', help='project name') - self.add_option( - '--json_file', dest='json_file', help='output json filepath') + """Creates the option parse and add --verbose support.""" + def __init__(self, *args, **kwargs): + optparse.OptionParser.__init__(self, + *args, + version=__version__, + **kwargs) + self.add_option('--verbose', + action='count', + default=0, + help='Use 2 times for more debugging info') + self.add_option('--host', dest='host', help='Url of host.') + self.add_option('--project', dest='project', help='project name') + self.add_option('--json_file', + dest='json_file', + help='output json filepath') - def parse_args(self, args=None, values=None): - options, args = optparse.OptionParser.parse_args(self, args, values) - # Host is always required - assert options.host, "--host not defined." - levels = [logging.WARNING, logging.INFO, logging.DEBUG] - logging.basicConfig(level=levels[min(options.verbose, len(levels) - 1)]) - return options, args + def parse_args(self, args=None, values=None): + options, args = optparse.OptionParser.parse_args(self, args, values) + # Host is always required + assert options.host, "--host not defined." + levels = [logging.WARNING, logging.INFO, logging.DEBUG] + logging.basicConfig(level=levels[min(options.verbose, len(levels) - 1)]) + return options, args def main(argv): - if sys.hexversion < 0x02060000: - print('\nYour python version %s is unsupported, please upgrade.\n' - % (sys.version.split(' ', 1)[0],), - file=sys.stderr) - return 2 - dispatcher = subcommand.CommandDispatcher(__name__) - return dispatcher.execute(OptionParser(), argv) + if sys.hexversion < 0x02060000: + print('\nYour python version %s is unsupported, please upgrade.\n' % + (sys.version.split(' ', 1)[0], ), + file=sys.stderr) + return 2 + dispatcher = subcommand.CommandDispatcher(__name__) + return dispatcher.execute(OptionParser(), argv) if __name__ == '__main__': - # These affect sys.stdout so do it outside of main() to simplify mocks in - # unit testing. - fix_encoding.fix_encoding() - setup_color.init() - try: - sys.exit(main(sys.argv[1:])) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(1) + # These affect sys.stdout so do it outside of main() to simplify mocks in + # unit testing. + fix_encoding.fix_encoding() + setup_color.init() + try: + sys.exit(main(sys.argv[1:])) + except KeyboardInterrupt: + sys.stderr.write('interrupted\n') + sys.exit(1) diff --git a/gerrit_util.py b/gerrit_util.py index 1aeb94c51a..a01732207c 100644 --- a/gerrit_util.py +++ b/gerrit_util.py @@ -1,7 +1,6 @@ # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """ Utilities for requesting information for a Gerrit server via HTTPS. @@ -35,6 +34,9 @@ from six.moves import urllib import http.cookiejar from io import StringIO +# TODO: Should fix these warnings. +# pylint: disable=line-too-long + LOGGER = logging.getLogger() # With a starting sleep time of 12.0 seconds, x <= [1.8-2.2]x backoff, and six # total tries, the sleep time between the first and last tries will be ~6 min @@ -50,228 +52,232 @@ GERRIT_PROTOCOL = 'https' def time_sleep(seconds): - # Use this so that it can be mocked in tests without interfering with python - # system machinery. - return time.sleep(seconds) + # Use this so that it can be mocked in tests without interfering with python + # system machinery. + return time.sleep(seconds) def time_time(): - # Use this so that it can be mocked in tests without interfering with python - # system machinery. - return time.time() + # Use this so that it can be mocked in tests without interfering with python + # system machinery. + return time.time() def log_retry_and_sleep(seconds, attempt): - LOGGER.info('Will retry in %d seconds (%d more times)...', seconds, - TRY_LIMIT - attempt - 1) - time_sleep(seconds) - return seconds * random.uniform(MIN_BACKOFF, MAX_BACKOFF) + LOGGER.info('Will retry in %d seconds (%d more times)...', seconds, + TRY_LIMIT - attempt - 1) + time_sleep(seconds) + return seconds * random.uniform(MIN_BACKOFF, MAX_BACKOFF) class GerritError(Exception): - """Exception class for errors commuicating with the gerrit-on-borg service.""" - def __init__(self, http_status, message, *args, **kwargs): - super(GerritError, self).__init__(*args, **kwargs) - self.http_status = http_status - self.message = '(%d) %s' % (self.http_status, message) + """Exception class for errors commuicating with the gerrit-on-borg service.""" + def __init__(self, http_status, message, *args, **kwargs): + super(GerritError, self).__init__(*args, **kwargs) + self.http_status = http_status + self.message = '(%d) %s' % (self.http_status, message) - def __str__(self): - return self.message + def __str__(self): + return self.message def _QueryString(params, first_param=None): - """Encodes query parameters in the key:val[+key:val...] format specified here: + """Encodes query parameters in the key:val[+key:val...] format specified here: https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes """ - q = [urllib.parse.quote(first_param)] if first_param else [] - q.extend(['%s:%s' % (key, val.replace(" ", "+")) for key, val in params]) - return '+'.join(q) + q = [urllib.parse.quote(first_param)] if first_param else [] + q.extend(['%s:%s' % (key, val.replace(" ", "+")) for key, val in params]) + return '+'.join(q) class Authenticator(object): - """Base authenticator class for authenticator implementations to subclass.""" + """Base authenticator class for authenticator implementations to subclass.""" + def get_auth_header(self, host): + raise NotImplementedError() - def get_auth_header(self, host): - raise NotImplementedError() - - @staticmethod - def get(): - """Returns: (Authenticator) The identified Authenticator to use. + @staticmethod + def get(): + """Returns: (Authenticator) The identified Authenticator to use. Probes the local system and its environment and identifies the Authenticator instance to use. """ - # LUCI Context takes priority since it's normally present only on bots, - # which then must use it. - if LuciContextAuthenticator.is_luci(): - return LuciContextAuthenticator() - # TODO(crbug.com/1059384): Automatically detect when running on cloudtop, - # and use CookiesAuthenticator instead. - if GceAuthenticator.is_gce(): - return GceAuthenticator() - return CookiesAuthenticator() + # LUCI Context takes priority since it's normally present only on bots, + # which then must use it. + if LuciContextAuthenticator.is_luci(): + return LuciContextAuthenticator() + # TODO(crbug.com/1059384): Automatically detect when running on + # cloudtop, and use CookiesAuthenticator instead. + if GceAuthenticator.is_gce(): + return GceAuthenticator() + return CookiesAuthenticator() class CookiesAuthenticator(Authenticator): - """Authenticator implementation that uses ".netrc" or ".gitcookies" for token. + """Authenticator implementation that uses ".netrc" or ".gitcookies" for token. Expected case for developer workstations. """ - _EMPTY = object() + _EMPTY = object() - def __init__(self): - # Credentials will be loaded lazily on first use. This ensures Authenticator - # get() can always construct an authenticator, even if something is broken. - # This allows 'creds-check' to proceed to actually checking creds later, - # rigorously (instead of blowing up with a cryptic error if they are wrong). - self._netrc = self._EMPTY - self._gitcookies = self._EMPTY + def __init__(self): + # Credentials will be loaded lazily on first use. This ensures + # Authenticator get() can always construct an authenticator, even if + # something is broken. This allows 'creds-check' to proceed to actually + # checking creds later, rigorously (instead of blowing up with a cryptic + # error if they are wrong). + self._netrc = self._EMPTY + self._gitcookies = self._EMPTY - @property - def netrc(self): - if self._netrc is self._EMPTY: - self._netrc = self._get_netrc() - return self._netrc + @property + def netrc(self): + if self._netrc is self._EMPTY: + self._netrc = self._get_netrc() + return self._netrc - @property - def gitcookies(self): - if self._gitcookies is self._EMPTY: - self._gitcookies = self._get_gitcookies() - return self._gitcookies + @property + def gitcookies(self): + if self._gitcookies is self._EMPTY: + self._gitcookies = self._get_gitcookies() + return self._gitcookies - @classmethod - def get_new_password_url(cls, host): - assert not host.startswith('http') - # Assume *.googlesource.com pattern. - parts = host.split('.') + @classmethod + def get_new_password_url(cls, host): + assert not host.startswith('http') + # Assume *.googlesource.com pattern. + parts = host.split('.') - # remove -review suffix if present. - if parts[0].endswith('-review'): - parts[0] = parts[0][:-len('-review')] + # remove -review suffix if present. + if parts[0].endswith('-review'): + parts[0] = parts[0][:-len('-review')] - return 'https://%s/new-password' % ('.'.join(parts)) + return 'https://%s/new-password' % ('.'.join(parts)) - @classmethod - def get_new_password_message(cls, host): - if host is None: - return ('Git host for Gerrit upload is unknown. Check your remote ' - 'and the branch your branch is tracking. This tool assumes ' - 'that you are using a git server at *.googlesource.com.') - url = cls.get_new_password_url(host) - return 'You can (re)generate your credentials by visiting %s' % url + @classmethod + def get_new_password_message(cls, host): + if host is None: + return ('Git host for Gerrit upload is unknown. Check your remote ' + 'and the branch your branch is tracking. This tool assumes ' + 'that you are using a git server at *.googlesource.com.') + url = cls.get_new_password_url(host) + return 'You can (re)generate your credentials by visiting %s' % url - @classmethod - def get_netrc_path(cls): - path = '_netrc' if sys.platform.startswith('win') else '.netrc' - return os.path.expanduser(os.path.join('~', path)) + @classmethod + def get_netrc_path(cls): + path = '_netrc' if sys.platform.startswith('win') else '.netrc' + return os.path.expanduser(os.path.join('~', path)) - @classmethod - def _get_netrc(cls): - # Buffer the '.netrc' path. Use an empty file if it doesn't exist. - path = cls.get_netrc_path() - if not os.path.exists(path): - return netrc.netrc(os.devnull) + @classmethod + def _get_netrc(cls): + # Buffer the '.netrc' path. Use an empty file if it doesn't exist. + path = cls.get_netrc_path() + if not os.path.exists(path): + return netrc.netrc(os.devnull) - st = os.stat(path) - if st.st_mode & (stat.S_IRWXG | stat.S_IRWXO): - print( - 'WARNING: netrc file %s cannot be used because its file ' - 'permissions are insecure. netrc file permissions should be ' - '600.' % path, file=sys.stderr) - with open(path) as fd: - content = fd.read() + st = os.stat(path) + if st.st_mode & (stat.S_IRWXG | stat.S_IRWXO): + print('WARNING: netrc file %s cannot be used because its file ' + 'permissions are insecure. netrc file permissions should be ' + '600.' % path, + file=sys.stderr) + with open(path) as fd: + content = fd.read() - # Load the '.netrc' file. We strip comments from it because processing them - # can trigger a bug in Windows. See crbug.com/664664. - content = '\n'.join(l for l in content.splitlines() - if l.strip() and not l.strip().startswith('#')) - with tempdir() as tdir: - netrc_path = os.path.join(tdir, 'netrc') - with open(netrc_path, 'w') as fd: - fd.write(content) - os.chmod(netrc_path, (stat.S_IRUSR | stat.S_IWUSR)) - return cls._get_netrc_from_path(netrc_path) + # Load the '.netrc' file. We strip comments from it because processing + # them can trigger a bug in Windows. See crbug.com/664664. + content = '\n'.join(l for l in content.splitlines() + if l.strip() and not l.strip().startswith('#')) + with tempdir() as tdir: + netrc_path = os.path.join(tdir, 'netrc') + with open(netrc_path, 'w') as fd: + fd.write(content) + os.chmod(netrc_path, (stat.S_IRUSR | stat.S_IWUSR)) + return cls._get_netrc_from_path(netrc_path) - @classmethod - def _get_netrc_from_path(cls, path): - try: - return netrc.netrc(path) - except IOError: - print('WARNING: Could not read netrc file %s' % path, file=sys.stderr) - return netrc.netrc(os.devnull) - except netrc.NetrcParseError as e: - print('ERROR: Cannot use netrc file %s due to a parsing error: %s' % - (path, e), file=sys.stderr) - return netrc.netrc(os.devnull) + @classmethod + def _get_netrc_from_path(cls, path): + try: + return netrc.netrc(path) + except IOError: + print('WARNING: Could not read netrc file %s' % path, + file=sys.stderr) + return netrc.netrc(os.devnull) + except netrc.NetrcParseError as e: + print('ERROR: Cannot use netrc file %s due to a parsing error: %s' % + (path, e), + file=sys.stderr) + return netrc.netrc(os.devnull) - @classmethod - def get_gitcookies_path(cls): - if os.getenv('GIT_COOKIES_PATH'): - return os.getenv('GIT_COOKIES_PATH') - try: - path = subprocess2.check_output( - ['git', 'config', '--path', 'http.cookiefile']) - return path.decode('utf-8', 'ignore').strip() - except subprocess2.CalledProcessError: - return os.path.expanduser(os.path.join('~', '.gitcookies')) + @classmethod + def get_gitcookies_path(cls): + if os.getenv('GIT_COOKIES_PATH'): + return os.getenv('GIT_COOKIES_PATH') + try: + path = subprocess2.check_output( + ['git', 'config', '--path', 'http.cookiefile']) + return path.decode('utf-8', 'ignore').strip() + except subprocess2.CalledProcessError: + return os.path.expanduser(os.path.join('~', '.gitcookies')) - @classmethod - def _get_gitcookies(cls): - gitcookies = {} - path = cls.get_gitcookies_path() - if not os.path.exists(path): - return gitcookies + @classmethod + def _get_gitcookies(cls): + gitcookies = {} + path = cls.get_gitcookies_path() + if not os.path.exists(path): + return gitcookies - try: - f = gclient_utils.FileRead(path, 'rb').splitlines() - except IOError: - return gitcookies + try: + f = gclient_utils.FileRead(path, 'rb').splitlines() + except IOError: + return gitcookies - for line in f: - try: - fields = line.strip().split('\t') - if line.strip().startswith('#') or len(fields) != 7: - continue - domain, xpath, key, value = fields[0], fields[2], fields[5], fields[6] - if xpath == '/' and key == 'o': - if value.startswith('git-'): - login, secret_token = value.split('=', 1) - gitcookies[domain] = (login, secret_token) - else: - gitcookies[domain] = ('', value) - except (IndexError, ValueError, TypeError) as exc: - LOGGER.warning(exc) - return gitcookies + for line in f: + try: + fields = line.strip().split('\t') + if line.strip().startswith('#') or len(fields) != 7: + continue + domain, xpath, key, value = fields[0], fields[2], fields[ + 5], fields[6] + if xpath == '/' and key == 'o': + if value.startswith('git-'): + login, secret_token = value.split('=', 1) + gitcookies[domain] = (login, secret_token) + else: + gitcookies[domain] = ('', value) + except (IndexError, ValueError, TypeError) as exc: + LOGGER.warning(exc) + return gitcookies - def _get_auth_for_host(self, host): - for domain, creds in self.gitcookies.items(): - if http.cookiejar.domain_match(host, domain): - return (creds[0], None, creds[1]) - return self.netrc.authenticators(host) + def _get_auth_for_host(self, host): + for domain, creds in self.gitcookies.items(): + if http.cookiejar.domain_match(host, domain): + return (creds[0], None, creds[1]) + return self.netrc.authenticators(host) - def get_auth_header(self, host): - a = self._get_auth_for_host(host) - if a: - if a[0]: - secret = base64.b64encode(('%s:%s' % (a[0], a[2])).encode('utf-8')) - return 'Basic %s' % secret.decode('utf-8') + def get_auth_header(self, host): + a = self._get_auth_for_host(host) + if a: + if a[0]: + secret = base64.b64encode( + ('%s:%s' % (a[0], a[2])).encode('utf-8')) + return 'Basic %s' % secret.decode('utf-8') - return 'Bearer %s' % a[2] - return None + return 'Bearer %s' % a[2] + return None - def get_auth_email(self, host): - """Best effort parsing of email to be used for auth for the given host.""" - a = self._get_auth_for_host(host) - if not a: - return None - login = a[0] - # login typically looks like 'git-xxx.example.com' - if not login.startswith('git-') or '.' not in login: - return None - username, domain = login[len('git-'):].split('.', 1) - return '%s@%s' % (username, domain) + def get_auth_email(self, host): + """Best effort parsing of email to be used for auth for the given host.""" + a = self._get_auth_for_host(host) + if not a: + return None + login = a[0] + # login typically looks like 'git-xxx.example.com' + if not login.startswith('git-') or '.' not in login: + return None + username, domain = login[len('git-'):].split('.', 1) + return '%s@%s' % (username, domain) # Backwards compatibility just in case somebody imports this outside of @@ -280,92 +286,93 @@ NetrcAuthenticator = CookiesAuthenticator class GceAuthenticator(Authenticator): - """Authenticator implementation that uses GCE metadata service for token. + """Authenticator implementation that uses GCE metadata service for token. """ - _INFO_URL = 'http://metadata.google.internal' - _ACQUIRE_URL = ('%s/computeMetadata/v1/instance/' - 'service-accounts/default/token' % _INFO_URL) - _ACQUIRE_HEADERS = {"Metadata-Flavor": "Google"} + _INFO_URL = 'http://metadata.google.internal' + _ACQUIRE_URL = ('%s/computeMetadata/v1/instance/' + 'service-accounts/default/token' % _INFO_URL) + _ACQUIRE_HEADERS = {"Metadata-Flavor": "Google"} - _cache_is_gce = None - _token_cache = None - _token_expiration = None + _cache_is_gce = None + _token_cache = None + _token_expiration = None - @classmethod - def is_gce(cls): - if os.getenv('SKIP_GCE_AUTH_FOR_GIT'): - return False - if cls._cache_is_gce is None: - cls._cache_is_gce = cls._test_is_gce() - return cls._cache_is_gce + @classmethod + def is_gce(cls): + if os.getenv('SKIP_GCE_AUTH_FOR_GIT'): + return False + if cls._cache_is_gce is None: + cls._cache_is_gce = cls._test_is_gce() + return cls._cache_is_gce - @classmethod - def _test_is_gce(cls): - # Based on https://cloud.google.com/compute/docs/metadata#runninggce - resp, _ = cls._get(cls._INFO_URL) - if resp is None: - return False - return resp.get('metadata-flavor') == 'Google' + @classmethod + def _test_is_gce(cls): + # Based on https://cloud.google.com/compute/docs/metadata#runninggce + resp, _ = cls._get(cls._INFO_URL) + if resp is None: + return False + return resp.get('metadata-flavor') == 'Google' - @staticmethod - def _get(url, **kwargs): - next_delay_sec = 1.0 - for i in range(TRY_LIMIT): - p = urllib.parse.urlparse(url) - if p.scheme not in ('http', 'https'): - raise RuntimeError( - "Don't know how to work with protocol '%s'" % protocol) - try: - resp, contents = httplib2.Http().request(url, 'GET', **kwargs) - except (socket.error, httplib2.HttpLib2Error, - httplib2.socks.ProxyError) as e: - LOGGER.debug('GET [%s] raised %s', url, e) + @staticmethod + def _get(url, **kwargs): + next_delay_sec = 1.0 + for i in range(TRY_LIMIT): + p = urllib.parse.urlparse(url) + if p.scheme not in ('http', 'https'): + raise RuntimeError("Don't know how to work with protocol '%s'" % + protocol) + try: + resp, contents = httplib2.Http().request(url, 'GET', **kwargs) + except (socket.error, httplib2.HttpLib2Error, + httplib2.socks.ProxyError) as e: + LOGGER.debug('GET [%s] raised %s', url, e) + return None, None + LOGGER.debug('GET [%s] #%d/%d (%d)', url, i + 1, TRY_LIMIT, + resp.status) + if resp.status < 500: + return (resp, contents) + + # Retry server error status codes. + LOGGER.warn('Encountered server error') + if TRY_LIMIT - i > 1: + next_delay_sec = log_retry_and_sleep(next_delay_sec, i) return None, None - LOGGER.debug('GET [%s] #%d/%d (%d)', url, i+1, TRY_LIMIT, resp.status) - if resp.status < 500: - return (resp, contents) - # Retry server error status codes. - LOGGER.warn('Encountered server error') - if TRY_LIMIT - i > 1: - next_delay_sec = log_retry_and_sleep(next_delay_sec, i) - return None, None + @classmethod + def _get_token_dict(cls): + # If cached token is valid for at least 25 seconds, return it. + if cls._token_cache and time_time() + 25 < cls._token_expiration: + return cls._token_cache - @classmethod - def _get_token_dict(cls): - # If cached token is valid for at least 25 seconds, return it. - if cls._token_cache and time_time() + 25 < cls._token_expiration: - return cls._token_cache + resp, contents = cls._get(cls._ACQUIRE_URL, + headers=cls._ACQUIRE_HEADERS) + if resp is None or resp.status != 200: + return None + cls._token_cache = json.loads(contents) + cls._token_expiration = cls._token_cache['expires_in'] + time_time() + return cls._token_cache - resp, contents = cls._get(cls._ACQUIRE_URL, headers=cls._ACQUIRE_HEADERS) - if resp is None or resp.status != 200: - return None - cls._token_cache = json.loads(contents) - cls._token_expiration = cls._token_cache['expires_in'] + time_time() - return cls._token_cache - - def get_auth_header(self, _host): - token_dict = self._get_token_dict() - if not token_dict: - return None - return '%(token_type)s %(access_token)s' % token_dict + def get_auth_header(self, _host): + token_dict = self._get_token_dict() + if not token_dict: + return None + return '%(token_type)s %(access_token)s' % token_dict class LuciContextAuthenticator(Authenticator): - """Authenticator implementation that uses LUCI_CONTEXT ambient local auth. + """Authenticator implementation that uses LUCI_CONTEXT ambient local auth. """ + @staticmethod + def is_luci(): + return auth.has_luci_context_local_auth() - @staticmethod - def is_luci(): - return auth.has_luci_context_local_auth() + def __init__(self): + self._authenticator = auth.Authenticator(' '.join( + [auth.OAUTH_SCOPE_EMAIL, auth.OAUTH_SCOPE_GERRIT])) - def __init__(self): - self._authenticator = auth.Authenticator( - ' '.join([auth.OAUTH_SCOPE_EMAIL, auth.OAUTH_SCOPE_GERRIT])) - - def get_auth_header(self, _host): - return 'Bearer %s' % self._authenticator.get_access_token().token + def get_auth_header(self, _host): + return 'Bearer %s' % self._authenticator.get_access_token().token def CreateHttpConn(host, @@ -374,54 +381,54 @@ def CreateHttpConn(host, headers=None, body=None, timeout=300): - """Opens an HTTPS connection to a Gerrit service, and sends a request.""" - headers = headers or {} - bare_host = host.partition(':')[0] + """Opens an HTTPS connection to a Gerrit service, and sends a request.""" + headers = headers or {} + bare_host = host.partition(':')[0] - a = Authenticator.get() - # TODO(crbug.com/1059384): Automatically detect when running on cloudtop. - if isinstance(a, GceAuthenticator): - print('If you\'re on a cloudtop instance, export ' - 'SKIP_GCE_AUTH_FOR_GIT=1 in your env.') + a = Authenticator.get() + # TODO(crbug.com/1059384): Automatically detect when running on cloudtop. + if isinstance(a, GceAuthenticator): + print('If you\'re on a cloudtop instance, export ' + 'SKIP_GCE_AUTH_FOR_GIT=1 in your env.') - a = a.get_auth_header(bare_host) - if a: - headers.setdefault('Authorization', a) - else: - LOGGER.debug('No authorization found for %s.' % bare_host) + a = a.get_auth_header(bare_host) + if a: + headers.setdefault('Authorization', a) + else: + LOGGER.debug('No authorization found for %s.' % bare_host) - url = path - if not url.startswith('/'): - url = '/' + url - if 'Authorization' in headers and not url.startswith('/a/'): - url = '/a%s' % url + url = path + if not url.startswith('/'): + url = '/' + url + if 'Authorization' in headers and not url.startswith('/a/'): + url = '/a%s' % url - if body: - body = json.dumps(body, sort_keys=True) - headers.setdefault('Content-Type', 'application/json') - if LOGGER.isEnabledFor(logging.DEBUG): - LOGGER.debug('%s %s://%s%s' % (reqtype, GERRIT_PROTOCOL, host, url)) - for key, val in headers.items(): - if key == 'Authorization': - val = 'HIDDEN' - LOGGER.debug('%s: %s' % (key, val)) if body: - LOGGER.debug(body) - conn = httplib2.Http(timeout=timeout) - # HACK: httplib2.Http has no such attribute; we store req_host here for later - # use in ReadHttpResponse. - conn.req_host = host - conn.req_params = { - 'uri': urllib.parse.urljoin('%s://%s' % (GERRIT_PROTOCOL, host), url), - 'method': reqtype, - 'headers': headers, - 'body': body, - } - return conn + body = json.dumps(body, sort_keys=True) + headers.setdefault('Content-Type', 'application/json') + if LOGGER.isEnabledFor(logging.DEBUG): + LOGGER.debug('%s %s://%s%s' % (reqtype, GERRIT_PROTOCOL, host, url)) + for key, val in headers.items(): + if key == 'Authorization': + val = 'HIDDEN' + LOGGER.debug('%s: %s' % (key, val)) + if body: + LOGGER.debug(body) + conn = httplib2.Http(timeout=timeout) + # HACK: httplib2.Http has no such attribute; we store req_host here for + # later use in ReadHttpResponse. + conn.req_host = host + conn.req_params = { + 'uri': urllib.parse.urljoin('%s://%s' % (GERRIT_PROTOCOL, host), url), + 'method': reqtype, + 'headers': headers, + 'body': body, + } + return conn def ReadHttpResponse(conn, accept_statuses=frozenset([200])): - """Reads an HTTP response from a connection into a string buffer. + """Reads an HTTP response from a connection into a string buffer. Args: conn: An Http object created by CreateHttpConn above. @@ -429,101 +436,105 @@ def ReadHttpResponse(conn, accept_statuses=frozenset([200])): Common additions include 204, 400, and 404. Returns: A string buffer containing the connection's reply. """ - sleep_time = SLEEP_TIME - for idx in range(TRY_LIMIT): - before_response = time.time() - try: - response, contents = conn.request(**conn.req_params) - except socket.timeout: - if idx < TRY_LIMIT - 1: - sleep_time = log_retry_and_sleep(sleep_time, idx) - continue - raise - contents = contents.decode('utf-8', 'replace') + sleep_time = SLEEP_TIME + for idx in range(TRY_LIMIT): + before_response = time.time() + try: + response, contents = conn.request(**conn.req_params) + except socket.timeout: + if idx < TRY_LIMIT - 1: + sleep_time = log_retry_and_sleep(sleep_time, idx) + continue + raise + contents = contents.decode('utf-8', 'replace') - response_time = time.time() - before_response - metrics.collector.add_repeated( - 'http_requests', - metrics_utils.extract_http_metrics( - conn.req_params['uri'], conn.req_params['method'], response.status, - response_time)) + response_time = time.time() - before_response + metrics.collector.add_repeated( + 'http_requests', + metrics_utils.extract_http_metrics(conn.req_params['uri'], + conn.req_params['method'], + response.status, response_time)) - # If response.status is an accepted status, - # or response.status < 500 then the result is final; break retry loop. - # If the response is 404/409 it might be because of replication lag, - # so keep trying anyway. If it is 429, it is generally ok to retry after - # a backoff. - if (response.status in accept_statuses - or response.status < 500 and response.status not in [404, 409, 429]): - LOGGER.debug('got response %d for %s %s', response.status, - conn.req_params['method'], conn.req_params['uri']) - # If 404 was in accept_statuses, then it's expected that the file might - # not exist, so don't return the gitiles error page because that's not - # the "content" that was actually requested. - if response.status == 404: - contents = '' - break + # If response.status is an accepted status, + # or response.status < 500 then the result is final; break retry loop. + # If the response is 404/409 it might be because of replication lag, + # so keep trying anyway. If it is 429, it is generally ok to retry after + # a backoff. + if (response.status in accept_statuses or response.status < 500 + and response.status not in [404, 409, 429]): + LOGGER.debug('got response %d for %s %s', response.status, + conn.req_params['method'], conn.req_params['uri']) + # If 404 was in accept_statuses, then it's expected that the file + # might not exist, so don't return the gitiles error page because + # that's not the "content" that was actually requested. + if response.status == 404: + contents = '' + break - # A status >=500 is assumed to be a possible transient error; retry. - http_version = 'HTTP/%s' % ('1.1' if response.version == 11 else '1.0') - LOGGER.warn('A transient error occurred while querying %s:\n' - '%s %s %s\n' - '%s %d %s\n' - '%s', - conn.req_host, conn.req_params['method'], - conn.req_params['uri'], - http_version, http_version, response.status, response.reason, - contents) + # A status >=500 is assumed to be a possible transient error; retry. + http_version = 'HTTP/%s' % ('1.1' if response.version == 11 else '1.0') + LOGGER.warn( + 'A transient error occurred while querying %s:\n' + '%s %s %s\n' + '%s %d %s\n' + '%s', conn.req_host, conn.req_params['method'], + conn.req_params['uri'], http_version, http_version, response.status, + response.reason, contents) - if idx < TRY_LIMIT - 1: - sleep_time = log_retry_and_sleep(sleep_time, idx) - # end of retries loop + if idx < TRY_LIMIT - 1: + sleep_time = log_retry_and_sleep(sleep_time, idx) + # end of retries loop - if response.status in accept_statuses: - return StringIO(contents) + if response.status in accept_statuses: + return StringIO(contents) - if response.status in (302, 401, 403): - www_authenticate = response.get('www-authenticate') - if not www_authenticate: - print('Your Gerrit credentials might be misconfigured.') - else: - auth_match = re.search('realm="([^"]+)"', www_authenticate, re.I) - host = auth_match.group(1) if auth_match else conn.req_host - print('Authentication failed. Please make sure your .gitcookies ' - 'file has credentials for %s.' % host) - print('Try:\n git cl creds-check') + if response.status in (302, 401, 403): + www_authenticate = response.get('www-authenticate') + if not www_authenticate: + print('Your Gerrit credentials might be misconfigured.') + else: + auth_match = re.search('realm="([^"]+)"', www_authenticate, re.I) + host = auth_match.group(1) if auth_match else conn.req_host + print('Authentication failed. Please make sure your .gitcookies ' + 'file has credentials for %s.' % host) + print('Try:\n git cl creds-check') - reason = '%s: %s' % (response.reason, contents) - raise GerritError(response.status, reason) + reason = '%s: %s' % (response.reason, contents) + raise GerritError(response.status, reason) def ReadHttpJsonResponse(conn, accept_statuses=frozenset([200])): - """Parses an https response as json.""" - fh = ReadHttpResponse(conn, accept_statuses) - # The first line of the response should always be: )]}' - s = fh.readline() - if s and s.rstrip() != ")]}'": - raise GerritError(200, 'Unexpected json output: %s' % s) - s = fh.read() - if not s: - return None - return json.loads(s) + """Parses an https response as json.""" + fh = ReadHttpResponse(conn, accept_statuses) + # The first line of the response should always be: )]}' + s = fh.readline() + if s and s.rstrip() != ")]}'": + raise GerritError(200, 'Unexpected json output: %s' % s) + s = fh.read() + if not s: + return None + return json.loads(s) def CallGerritApi(host, path, **kwargs): - """Helper for calling a Gerrit API that returns a JSON response.""" - conn_kwargs = {} - conn_kwargs.update( - (k, kwargs[k]) for k in ['reqtype', 'headers', 'body'] if k in kwargs) - conn = CreateHttpConn(host, path, **conn_kwargs) - read_kwargs = {} - read_kwargs.update((k, kwargs[k]) for k in ['accept_statuses'] if k in kwargs) - return ReadHttpJsonResponse(conn, **read_kwargs) + """Helper for calling a Gerrit API that returns a JSON response.""" + conn_kwargs = {} + conn_kwargs.update( + (k, kwargs[k]) for k in ['reqtype', 'headers', 'body'] if k in kwargs) + conn = CreateHttpConn(host, path, **conn_kwargs) + read_kwargs = {} + read_kwargs.update( + (k, kwargs[k]) for k in ['accept_statuses'] if k in kwargs) + return ReadHttpJsonResponse(conn, **read_kwargs) -def QueryChanges(host, params, first_param=None, limit=None, o_params=None, +def QueryChanges(host, + params, + first_param=None, + limit=None, + o_params=None, start=None): - """ + """ Queries a gerrit-on-borg server for changes matching query terms. Args: @@ -539,22 +550,26 @@ def QueryChanges(host, params, first_param=None, limit=None, o_params=None, Returns: A list of json-decoded query results. """ - # Note that no attempt is made to escape special characters; YMMV. - if not params and not first_param: - raise RuntimeError('QueryChanges requires search parameters') - path = 'changes/?q=%s' % _QueryString(params, first_param) - if start: - path = '%s&start=%s' % (path, start) - if limit: - path = '%s&n=%d' % (path, limit) - if o_params: - path = '%s&%s' % (path, '&'.join(['o=%s' % p for p in o_params])) - return ReadHttpJsonResponse(CreateHttpConn(host, path, timeout=30.0)) + # Note that no attempt is made to escape special characters; YMMV. + if not params and not first_param: + raise RuntimeError('QueryChanges requires search parameters') + path = 'changes/?q=%s' % _QueryString(params, first_param) + if start: + path = '%s&start=%s' % (path, start) + if limit: + path = '%s&n=%d' % (path, limit) + if o_params: + path = '%s&%s' % (path, '&'.join(['o=%s' % p for p in o_params])) + return ReadHttpJsonResponse(CreateHttpConn(host, path, timeout=30.0)) -def GenerateAllChanges(host, params, first_param=None, limit=500, - o_params=None, start=None): - """Queries a gerrit-on-borg server for all the changes matching the query +def GenerateAllChanges(host, + params, + first_param=None, + limit=500, + o_params=None, + start=None): + """Queries a gerrit-on-borg server for all the changes matching the query terms. WARNING: this is unreliable if a change matching the query is modified while @@ -573,313 +588,317 @@ def GenerateAllChanges(host, params, first_param=None, limit=500, Returns: A generator object to the list of returned changes. """ - already_returned = set() + already_returned = set() - def at_most_once(cls): - for cl in cls: - if cl['_number'] not in already_returned: - already_returned.add(cl['_number']) - yield cl + def at_most_once(cls): + for cl in cls: + if cl['_number'] not in already_returned: + already_returned.add(cl['_number']) + yield cl - start = start or 0 - cur_start = start - more_changes = True + start = start or 0 + cur_start = start + more_changes = True - while more_changes: - # This will fetch changes[start..start+limit] sorted by most recently - # updated. Since the rank of any change in this list can be changed any time - # (say user posting comment), subsequent calls may overalp like this: - # > initial order ABCDEFGH - # query[0..3] => ABC - # > E gets updated. New order: EABCDFGH - # query[3..6] => CDF # C is a dup - # query[6..9] => GH # E is missed. - page = QueryChanges(host, params, first_param, limit, o_params, - cur_start) - for cl in at_most_once(page): - yield cl + while more_changes: + # This will fetch changes[start..start+limit] sorted by most recently + # updated. Since the rank of any change in this list can be changed any + # time (say user posting comment), subsequent calls may overalp like + # this: > initial order ABCDEFGH query[0..3] => ABC > E gets updated. + # New order: EABCDFGH query[3..6] => CDF # C is a dup query[6..9] => + # GH # E is missed. + page = QueryChanges(host, params, first_param, limit, o_params, + cur_start) + for cl in at_most_once(page): + yield cl - more_changes = [cl for cl in page if '_more_changes' in cl] - if len(more_changes) > 1: - raise GerritError( - 200, - 'Received %d changes with a _more_changes attribute set but should ' - 'receive at most one.' % len(more_changes)) - if more_changes: - cur_start += len(page) + more_changes = [cl for cl in page if '_more_changes' in cl] + if len(more_changes) > 1: + raise GerritError( + 200, + 'Received %d changes with a _more_changes attribute set but should ' + 'receive at most one.' % len(more_changes)) + if more_changes: + cur_start += len(page) - # If we paged through, query again the first page which in most circumstances - # will fetch all changes that were modified while this function was run. - if start != cur_start: - page = QueryChanges(host, params, first_param, limit, o_params, start) - for cl in at_most_once(page): - yield cl + # If we paged through, query again the first page which in most + # circumstances will fetch all changes that were modified while this + # function was run. + if start != cur_start: + page = QueryChanges(host, params, first_param, limit, o_params, start) + for cl in at_most_once(page): + yield cl -def MultiQueryChanges(host, params, change_list, limit=None, o_params=None, +def MultiQueryChanges(host, + params, + change_list, + limit=None, + o_params=None, start=None): - """Initiate a query composed of multiple sets of query parameters.""" - if not change_list: - raise RuntimeError( - "MultiQueryChanges requires a list of change numbers/id's") - q = ['q=%s' % '+OR+'.join([urllib.parse.quote(str(x)) for x in change_list])] - if params: - q.append(_QueryString(params)) - if limit: - q.append('n=%d' % limit) - if start: - q.append('S=%s' % start) - if o_params: - q.extend(['o=%s' % p for p in o_params]) - path = 'changes/?%s' % '&'.join(q) - try: - result = ReadHttpJsonResponse(CreateHttpConn(host, path)) - except GerritError as e: - msg = '%s:\n%s' % (e.message, path) - raise GerritError(e.http_status, msg) - return result + """Initiate a query composed of multiple sets of query parameters.""" + if not change_list: + raise RuntimeError( + "MultiQueryChanges requires a list of change numbers/id's") + q = [ + 'q=%s' % '+OR+'.join([urllib.parse.quote(str(x)) for x in change_list]) + ] + if params: + q.append(_QueryString(params)) + if limit: + q.append('n=%d' % limit) + if start: + q.append('S=%s' % start) + if o_params: + q.extend(['o=%s' % p for p in o_params]) + path = 'changes/?%s' % '&'.join(q) + try: + result = ReadHttpJsonResponse(CreateHttpConn(host, path)) + except GerritError as e: + msg = '%s:\n%s' % (e.message, path) + raise GerritError(e.http_status, msg) + return result def GetGerritFetchUrl(host): - """Given a Gerrit host name returns URL of a Gerrit instance to fetch from.""" - return '%s://%s/' % (GERRIT_PROTOCOL, host) + """Given a Gerrit host name returns URL of a Gerrit instance to fetch from.""" + return '%s://%s/' % (GERRIT_PROTOCOL, host) def GetCodeReviewTbrScore(host, project): - """Given a Gerrit host name and project, return the Code-Review score for TBR. + """Given a Gerrit host name and project, return the Code-Review score for TBR. """ - conn = CreateHttpConn( - host, '/projects/%s' % urllib.parse.quote(project, '')) - project = ReadHttpJsonResponse(conn) - if ('labels' not in project - or 'Code-Review' not in project['labels'] - or 'values' not in project['labels']['Code-Review']): - return 1 - return max([int(x) for x in project['labels']['Code-Review']['values']]) + conn = CreateHttpConn(host, + '/projects/%s' % urllib.parse.quote(project, '')) + project = ReadHttpJsonResponse(conn) + if ('labels' not in project or 'Code-Review' not in project['labels'] + or 'values' not in project['labels']['Code-Review']): + return 1 + return max([int(x) for x in project['labels']['Code-Review']['values']]) def GetChangePageUrl(host, change_number): - """Given a Gerrit host name and change number, returns change page URL.""" - return '%s://%s/#/c/%d/' % (GERRIT_PROTOCOL, host, change_number) + """Given a Gerrit host name and change number, returns change page URL.""" + return '%s://%s/#/c/%d/' % (GERRIT_PROTOCOL, host, change_number) def GetChangeUrl(host, change): - """Given a Gerrit host name and change ID, returns a URL for the change.""" - return '%s://%s/a/changes/%s' % (GERRIT_PROTOCOL, host, change) + """Given a Gerrit host name and change ID, returns a URL for the change.""" + return '%s://%s/a/changes/%s' % (GERRIT_PROTOCOL, host, change) def GetChange(host, change): - """Queries a Gerrit server for information about a single change.""" - path = 'changes/%s' % change - return ReadHttpJsonResponse(CreateHttpConn(host, path)) + """Queries a Gerrit server for information about a single change.""" + path = 'changes/%s' % change + return ReadHttpJsonResponse(CreateHttpConn(host, path)) def GetChangeDetail(host, change, o_params=None): - """Queries a Gerrit server for extended information about a single change.""" - path = 'changes/%s/detail' % change - if o_params: - path += '?%s' % '&'.join(['o=%s' % p for p in o_params]) - return ReadHttpJsonResponse(CreateHttpConn(host, path)) + """Queries a Gerrit server for extended information about a single change.""" + path = 'changes/%s/detail' % change + if o_params: + path += '?%s' % '&'.join(['o=%s' % p for p in o_params]) + return ReadHttpJsonResponse(CreateHttpConn(host, path)) def GetChangeCommit(host, change, revision='current'): - """Query a Gerrit server for a revision associated with a change.""" - path = 'changes/%s/revisions/%s/commit?links' % (change, revision) - return ReadHttpJsonResponse(CreateHttpConn(host, path)) + """Query a Gerrit server for a revision associated with a change.""" + path = 'changes/%s/revisions/%s/commit?links' % (change, revision) + return ReadHttpJsonResponse(CreateHttpConn(host, path)) def GetChangeCurrentRevision(host, change): - """Get information about the latest revision for a given change.""" - return QueryChanges(host, [], change, o_params=('CURRENT_REVISION',)) + """Get information about the latest revision for a given change.""" + return QueryChanges(host, [], change, o_params=('CURRENT_REVISION', )) def GetChangeRevisions(host, change): - """Gets information about all revisions associated with a change.""" - return QueryChanges(host, [], change, o_params=('ALL_REVISIONS',)) + """Gets information about all revisions associated with a change.""" + return QueryChanges(host, [], change, o_params=('ALL_REVISIONS', )) def GetChangeReview(host, change, revision=None): - """Gets the current review information for a change.""" - if not revision: - jmsg = GetChangeRevisions(host, change) - if not jmsg: - return None + """Gets the current review information for a change.""" + if not revision: + jmsg = GetChangeRevisions(host, change) + if not jmsg: + return None - if len(jmsg) > 1: - raise GerritError(200, 'Multiple changes found for ChangeId %s.' % change) - revision = jmsg[0]['current_revision'] - path = 'changes/%s/revisions/%s/review' - return ReadHttpJsonResponse(CreateHttpConn(host, path)) + if len(jmsg) > 1: + raise GerritError( + 200, 'Multiple changes found for ChangeId %s.' % change) + revision = jmsg[0]['current_revision'] + path = 'changes/%s/revisions/%s/review' + return ReadHttpJsonResponse(CreateHttpConn(host, path)) def GetChangeComments(host, change): - """Get the line- and file-level comments on a change.""" - path = 'changes/%s/comments' % change - return ReadHttpJsonResponse(CreateHttpConn(host, path)) + """Get the line- and file-level comments on a change.""" + path = 'changes/%s/comments' % change + return ReadHttpJsonResponse(CreateHttpConn(host, path)) def GetChangeRobotComments(host, change): - """Gets the line- and file-level robot comments on a change.""" - path = 'changes/%s/robotcomments' % change - return ReadHttpJsonResponse(CreateHttpConn(host, path)) + """Gets the line- and file-level robot comments on a change.""" + path = 'changes/%s/robotcomments' % change + return ReadHttpJsonResponse(CreateHttpConn(host, path)) def GetRelatedChanges(host, change, revision='current'): - """Gets the related changes for a given change and revision.""" - path = 'changes/%s/revisions/%s/related' % (change, revision) - return ReadHttpJsonResponse(CreateHttpConn(host, path)) + """Gets the related changes for a given change and revision.""" + path = 'changes/%s/revisions/%s/related' % (change, revision) + return ReadHttpJsonResponse(CreateHttpConn(host, path)) def AbandonChange(host, change, msg=''): - """Abandons a Gerrit change.""" - path = 'changes/%s/abandon' % change - body = {'message': msg} if msg else {} - conn = CreateHttpConn(host, path, reqtype='POST', body=body) - return ReadHttpJsonResponse(conn) + """Abandons a Gerrit change.""" + path = 'changes/%s/abandon' % change + body = {'message': msg} if msg else {} + conn = CreateHttpConn(host, path, reqtype='POST', body=body) + return ReadHttpJsonResponse(conn) def MoveChange(host, change, destination_branch): - """Move a Gerrit change to different destination branch.""" - path = 'changes/%s/move' % change - body = {'destination_branch': destination_branch, - 'keep_all_votes': True} - conn = CreateHttpConn(host, path, reqtype='POST', body=body) - return ReadHttpJsonResponse(conn) - + """Move a Gerrit change to different destination branch.""" + path = 'changes/%s/move' % change + body = {'destination_branch': destination_branch, 'keep_all_votes': True} + conn = CreateHttpConn(host, path, reqtype='POST', body=body) + return ReadHttpJsonResponse(conn) def RestoreChange(host, change, msg=''): - """Restores a previously abandoned change.""" - path = 'changes/%s/restore' % change - body = {'message': msg} if msg else {} - conn = CreateHttpConn(host, path, reqtype='POST', body=body) - return ReadHttpJsonResponse(conn) + """Restores a previously abandoned change.""" + path = 'changes/%s/restore' % change + body = {'message': msg} if msg else {} + conn = CreateHttpConn(host, path, reqtype='POST', body=body) + return ReadHttpJsonResponse(conn) def SubmitChange(host, change): - """Submits a Gerrit change via Gerrit.""" - path = 'changes/%s/submit' % change - conn = CreateHttpConn(host, path, reqtype='POST') - return ReadHttpJsonResponse(conn) + """Submits a Gerrit change via Gerrit.""" + path = 'changes/%s/submit' % change + conn = CreateHttpConn(host, path, reqtype='POST') + return ReadHttpJsonResponse(conn) def GetChangesSubmittedTogether(host, change): - """Get all changes submitted with the given one.""" - path = 'changes/%s/submitted_together?o=NON_VISIBLE_CHANGES' % change - conn = CreateHttpConn(host, path, reqtype='GET') - return ReadHttpJsonResponse(conn) + """Get all changes submitted with the given one.""" + path = 'changes/%s/submitted_together?o=NON_VISIBLE_CHANGES' % change + conn = CreateHttpConn(host, path, reqtype='GET') + return ReadHttpJsonResponse(conn) def PublishChangeEdit(host, change, notify=True): - """Publish a Gerrit change edit.""" - path = 'changes/%s/edit:publish' % change - body = {'notify': 'ALL' if notify else 'NONE'} - conn = CreateHttpConn(host, path, reqtype='POST', body=body) - return ReadHttpJsonResponse(conn, accept_statuses=(204, )) + """Publish a Gerrit change edit.""" + path = 'changes/%s/edit:publish' % change + body = {'notify': 'ALL' if notify else 'NONE'} + conn = CreateHttpConn(host, path, reqtype='POST', body=body) + return ReadHttpJsonResponse(conn, accept_statuses=(204, )) def ChangeEdit(host, change, path, data): - """Puts content of a file into a change edit.""" - path = 'changes/%s/edit/%s' % (change, urllib.parse.quote(path, '')) - body = { - 'binary_content': - 'data:text/plain;base64,%s' % - base64.b64encode(data.encode('utf-8')).decode('utf-8') - } - conn = CreateHttpConn(host, path, reqtype='PUT', body=body) - return ReadHttpJsonResponse(conn, accept_statuses=(204, 409)) + """Puts content of a file into a change edit.""" + path = 'changes/%s/edit/%s' % (change, urllib.parse.quote(path, '')) + body = { + 'binary_content': + 'data:text/plain;base64,%s' % + base64.b64encode(data.encode('utf-8')).decode('utf-8') + } + conn = CreateHttpConn(host, path, reqtype='PUT', body=body) + return ReadHttpJsonResponse(conn, accept_statuses=(204, 409)) def SetChangeEditMessage(host, change, message): - """Sets the commit message of a change edit.""" - path = 'changes/%s/edit:message' % change - body = {'message': message} - conn = CreateHttpConn(host, path, reqtype='PUT', body=body) - return ReadHttpJsonResponse(conn, accept_statuses=(204, 409)) + """Sets the commit message of a change edit.""" + path = 'changes/%s/edit:message' % change + body = {'message': message} + conn = CreateHttpConn(host, path, reqtype='PUT', body=body) + return ReadHttpJsonResponse(conn, accept_statuses=(204, 409)) def HasPendingChangeEdit(host, change): - conn = CreateHttpConn(host, 'changes/%s/edit' % change) - try: - ReadHttpResponse(conn) - except GerritError as e: - # 204 No Content means no pending change. - if e.http_status == 204: - return False - raise - return True + conn = CreateHttpConn(host, 'changes/%s/edit' % change) + try: + ReadHttpResponse(conn) + except GerritError as e: + # 204 No Content means no pending change. + if e.http_status == 204: + return False + raise + return True def DeletePendingChangeEdit(host, change): - conn = CreateHttpConn(host, 'changes/%s/edit' % change, reqtype='DELETE') - # On success, Gerrit returns status 204; if the edit was already deleted it - # returns 404. Anything else is an error. - ReadHttpResponse(conn, accept_statuses=[204, 404]) + conn = CreateHttpConn(host, 'changes/%s/edit' % change, reqtype='DELETE') + # On success, Gerrit returns status 204; if the edit was already deleted it + # returns 404. Anything else is an error. + ReadHttpResponse(conn, accept_statuses=[204, 404]) def CherryPick(host, change, destination, revision='current'): - """Create a cherry-pick commit from the given change, onto the given + """Create a cherry-pick commit from the given change, onto the given destination. """ - path = 'changes/%s/revisions/%s/cherrypick' % (change, revision) - body = {'destination': destination} - conn = CreateHttpConn(host, path, reqtype='POST', body=body) - return ReadHttpJsonResponse(conn) + path = 'changes/%s/revisions/%s/cherrypick' % (change, revision) + body = {'destination': destination} + conn = CreateHttpConn(host, path, reqtype='POST', body=body) + return ReadHttpJsonResponse(conn) def GetFileContents(host, change, path): - """Get the contents of a file with the given path in the given revision. + """Get the contents of a file with the given path in the given revision. Returns: A bytes object with the file's contents. """ - path = 'changes/%s/revisions/current/files/%s/content' % ( - change, urllib.parse.quote(path, '')) - conn = CreateHttpConn(host, path, reqtype='GET') - return base64.b64decode(ReadHttpResponse(conn).read()) + path = 'changes/%s/revisions/current/files/%s/content' % ( + change, urllib.parse.quote(path, '')) + conn = CreateHttpConn(host, path, reqtype='GET') + return base64.b64decode(ReadHttpResponse(conn).read()) def SetCommitMessage(host, change, description, notify='ALL'): - """Updates a commit message.""" - assert notify in ('ALL', 'NONE') - path = 'changes/%s/message' % change - body = {'message': description, 'notify': notify} - conn = CreateHttpConn(host, path, reqtype='PUT', body=body) - try: - ReadHttpResponse(conn, accept_statuses=[200, 204]) - except GerritError as e: - raise GerritError( - e.http_status, - 'Received unexpected http status while editing message ' - 'in change %s' % change) + """Updates a commit message.""" + assert notify in ('ALL', 'NONE') + path = 'changes/%s/message' % change + body = {'message': description, 'notify': notify} + conn = CreateHttpConn(host, path, reqtype='PUT', body=body) + try: + ReadHttpResponse(conn, accept_statuses=[200, 204]) + except GerritError as e: + raise GerritError( + e.http_status, + 'Received unexpected http status while editing message ' + 'in change %s' % change) def GetCommitIncludedIn(host, project, commit): - """Retrieves the branches and tags for a given commit. + """Retrieves the branches and tags for a given commit. https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#get-included-in Returns: A JSON object with keys of 'branches' and 'tags'. """ - path = 'projects/%s/commits/%s/in' % (urllib.parse.quote(project, ''), commit) - conn = CreateHttpConn(host, path, reqtype='GET') - return ReadHttpJsonResponse(conn, accept_statuses=[200]) + path = 'projects/%s/commits/%s/in' % (urllib.parse.quote(project, + ''), commit) + conn = CreateHttpConn(host, path, reqtype='GET') + return ReadHttpJsonResponse(conn, accept_statuses=[200]) def IsCodeOwnersEnabledOnHost(host): - """Check if the code-owners plugin is enabled for the host.""" - path = 'config/server/capabilities' - capabilities = ReadHttpJsonResponse(CreateHttpConn(host, path)) - return 'code-owners-checkCodeOwner' in capabilities + """Check if the code-owners plugin is enabled for the host.""" + path = 'config/server/capabilities' + capabilities = ReadHttpJsonResponse(CreateHttpConn(host, path)) + return 'code-owners-checkCodeOwner' in capabilities def IsCodeOwnersEnabledOnRepo(host, repo): - """Check if the code-owners plugin is enabled for the repo.""" - repo = PercentEncodeForGitRef(repo) - path = '/projects/%s/code_owners.project_config' % repo - config = ReadHttpJsonResponse(CreateHttpConn(host, path)) - return not config['status'].get('disabled', False) + """Check if the code-owners plugin is enabled for the repo.""" + repo = PercentEncodeForGitRef(repo) + path = '/projects/%s/code_owners.project_config' % repo + config = ReadHttpJsonResponse(CreateHttpConn(host, path)) + return not config['status'].get('disabled', False) def GetOwnersForFile(host, @@ -890,154 +909,171 @@ def GetOwnersForFile(host, resolve_all_users=True, highest_score_only=False, seed=None, - o_params=('DETAILS',)): - """Gets information about owners attached to a file.""" - path = 'projects/%s/branches/%s/code_owners/%s' % ( - urllib.parse.quote(project, ''), - urllib.parse.quote(branch, ''), - urllib.parse.quote(path, '')) - q = ['resolve-all-users=%s' % json.dumps(resolve_all_users)] - if highest_score_only: - q.append('highest-score-only=%s' % json.dumps(highest_score_only)) - if seed: - q.append('seed=%d' % seed) - if limit: - q.append('n=%d' % limit) - if o_params: - q.extend(['o=%s' % p for p in o_params]) - if q: - path = '%s?%s' % (path, '&'.join(q)) - return ReadHttpJsonResponse(CreateHttpConn(host, path)) + o_params=('DETAILS', )): + """Gets information about owners attached to a file.""" + path = 'projects/%s/branches/%s/code_owners/%s' % (urllib.parse.quote( + project, ''), urllib.parse.quote(branch, + ''), urllib.parse.quote(path, '')) + q = ['resolve-all-users=%s' % json.dumps(resolve_all_users)] + if highest_score_only: + q.append('highest-score-only=%s' % json.dumps(highest_score_only)) + if seed: + q.append('seed=%d' % seed) + if limit: + q.append('n=%d' % limit) + if o_params: + q.extend(['o=%s' % p for p in o_params]) + if q: + path = '%s?%s' % (path, '&'.join(q)) + return ReadHttpJsonResponse(CreateHttpConn(host, path)) def GetReviewers(host, change): - """Gets information about all reviewers attached to a change.""" - path = 'changes/%s/reviewers' % change - return ReadHttpJsonResponse(CreateHttpConn(host, path)) + """Gets information about all reviewers attached to a change.""" + path = 'changes/%s/reviewers' % change + return ReadHttpJsonResponse(CreateHttpConn(host, path)) def GetReview(host, change, revision): - """Gets review information about a specific revision of a change.""" - path = 'changes/%s/revisions/%s/review' % (change, revision) - return ReadHttpJsonResponse(CreateHttpConn(host, path)) + """Gets review information about a specific revision of a change.""" + path = 'changes/%s/revisions/%s/review' % (change, revision) + return ReadHttpJsonResponse(CreateHttpConn(host, path)) -def AddReviewers(host, change, reviewers=None, ccs=None, notify=True, +def AddReviewers(host, + change, + reviewers=None, + ccs=None, + notify=True, accept_statuses=frozenset([200, 400, 422])): - """Add reviewers to a change.""" - if not reviewers and not ccs: - return None - if not change: - return None - reviewers = frozenset(reviewers or []) - ccs = frozenset(ccs or []) - path = 'changes/%s/revisions/current/review' % change + """Add reviewers to a change.""" + if not reviewers and not ccs: + return None + if not change: + return None + reviewers = frozenset(reviewers or []) + ccs = frozenset(ccs or []) + path = 'changes/%s/revisions/current/review' % change - body = { - 'drafts': 'KEEP', - 'reviewers': [], - 'notify': 'ALL' if notify else 'NONE', - } - for r in sorted(reviewers | ccs): - state = 'REVIEWER' if r in reviewers else 'CC' - body['reviewers'].append({ - 'reviewer': r, - 'state': state, - 'notify': 'NONE', # We handled `notify` argument above. - }) + body = { + 'drafts': 'KEEP', + 'reviewers': [], + 'notify': 'ALL' if notify else 'NONE', + } + for r in sorted(reviewers | ccs): + state = 'REVIEWER' if r in reviewers else 'CC' + body['reviewers'].append({ + 'reviewer': r, + 'state': state, + 'notify': 'NONE', # We handled `notify` argument above. + }) - conn = CreateHttpConn(host, path, reqtype='POST', body=body) - # Gerrit will return 400 if one or more of the requested reviewers are - # unprocessable. We read the response object to see which were rejected, - # warn about them, and retry with the remainder. - resp = ReadHttpJsonResponse(conn, accept_statuses=accept_statuses) + conn = CreateHttpConn(host, path, reqtype='POST', body=body) + # Gerrit will return 400 if one or more of the requested reviewers are + # unprocessable. We read the response object to see which were rejected, + # warn about them, and retry with the remainder. + resp = ReadHttpJsonResponse(conn, accept_statuses=accept_statuses) - errored = set() - for result in resp.get('reviewers', {}).values(): - r = result.get('input') - state = 'REVIEWER' if r in reviewers else 'CC' - if result.get('error'): - errored.add(r) - LOGGER.warn('Note: "%s" not added as a %s' % (r, state.lower())) - if errored: - # Try again, adding only those that didn't fail, and only accepting 200. - AddReviewers(host, change, reviewers=(reviewers-errored), - ccs=(ccs-errored), notify=notify, accept_statuses=[200]) + errored = set() + for result in resp.get('reviewers', {}).values(): + r = result.get('input') + state = 'REVIEWER' if r in reviewers else 'CC' + if result.get('error'): + errored.add(r) + LOGGER.warn('Note: "%s" not added as a %s' % (r, state.lower())) + if errored: + # Try again, adding only those that didn't fail, and only accepting 200. + AddReviewers(host, + change, + reviewers=(reviewers - errored), + ccs=(ccs - errored), + notify=notify, + accept_statuses=[200]) def SetReview(host, change, msg=None, labels=None, notify=None, ready=None): - """Sets labels and/or adds a message to a code review.""" - if not msg and not labels: - return - path = 'changes/%s/revisions/current/review' % change - body = {'drafts': 'KEEP'} - if msg: - body['message'] = msg - if labels: - body['labels'] = labels - if notify is not None: - body['notify'] = 'ALL' if notify else 'NONE' - if ready: - body['ready'] = True - conn = CreateHttpConn(host, path, reqtype='POST', body=body) - response = ReadHttpJsonResponse(conn) - if labels: - for key, val in labels.items(): - if ('labels' not in response or key not in response['labels'] or - int(response['labels'][key] != int(val))): - raise GerritError(200, 'Unable to set "%s" label on change %s.' % ( - key, change)) - return response + """Sets labels and/or adds a message to a code review.""" + if not msg and not labels: + return + path = 'changes/%s/revisions/current/review' % change + body = {'drafts': 'KEEP'} + if msg: + body['message'] = msg + if labels: + body['labels'] = labels + if notify is not None: + body['notify'] = 'ALL' if notify else 'NONE' + if ready: + body['ready'] = True + conn = CreateHttpConn(host, path, reqtype='POST', body=body) + response = ReadHttpJsonResponse(conn) + if labels: + for key, val in labels.items(): + if ('labels' not in response or key not in response['labels'] + or int(response['labels'][key] != int(val))): + raise GerritError( + 200, + 'Unable to set "%s" label on change %s.' % (key, change)) + return response -def ResetReviewLabels(host, change, label, value='0', message=None, + +def ResetReviewLabels(host, + change, + label, + value='0', + message=None, notify=None): - """Resets the value of a given label for all reviewers on a change.""" - # This is tricky, because we want to work on the "current revision", but - # there's always the risk that "current revision" will change in between - # API calls. So, we check "current revision" at the beginning and end; if - # it has changed, raise an exception. - jmsg = GetChangeCurrentRevision(host, change) - if not jmsg: - raise GerritError( - 200, 'Could not get review information for change "%s"' % change) - value = str(value) - revision = jmsg[0]['current_revision'] - path = 'changes/%s/revisions/%s/review' % (change, revision) - message = message or ( - '%s label set to %s programmatically.' % (label, value)) - jmsg = GetReview(host, change, revision) - if not jmsg: - raise GerritError(200, 'Could not get review information for revision %s ' - 'of change %s' % (revision, change)) - for review in jmsg.get('labels', {}).get(label, {}).get('all', []): - if str(review.get('value', value)) != value: - body = { - 'drafts': 'KEEP', - 'message': message, - 'labels': {label: value}, - 'on_behalf_of': review['_account_id'], - } - if notify: - body['notify'] = notify - conn = CreateHttpConn( - host, path, reqtype='POST', body=body) - response = ReadHttpJsonResponse(conn) - if str(response['labels'][label]) != value: - username = review.get('email', jmsg.get('name', '')) - raise GerritError(200, 'Unable to set %s label for user "%s"' - ' on change %s.' % (label, username, change)) - jmsg = GetChangeCurrentRevision(host, change) - if not jmsg: - raise GerritError( - 200, 'Could not get review information for change "%s"' % change) + """Resets the value of a given label for all reviewers on a change.""" + # This is tricky, because we want to work on the "current revision", but + # there's always the risk that "current revision" will change in between + # API calls. So, we check "current revision" at the beginning and end; if + # it has changed, raise an exception. + jmsg = GetChangeCurrentRevision(host, change) + if not jmsg: + raise GerritError( + 200, 'Could not get review information for change "%s"' % change) + value = str(value) + revision = jmsg[0]['current_revision'] + path = 'changes/%s/revisions/%s/review' % (change, revision) + message = message or ('%s label set to %s programmatically.' % + (label, value)) + jmsg = GetReview(host, change, revision) + if not jmsg: + raise GerritError( + 200, 'Could not get review information for revision %s ' + 'of change %s' % (revision, change)) + for review in jmsg.get('labels', {}).get(label, {}).get('all', []): + if str(review.get('value', value)) != value: + body = { + 'drafts': 'KEEP', + 'message': message, + 'labels': { + label: value + }, + 'on_behalf_of': review['_account_id'], + } + if notify: + body['notify'] = notify + conn = CreateHttpConn(host, path, reqtype='POST', body=body) + response = ReadHttpJsonResponse(conn) + if str(response['labels'][label]) != value: + username = review.get('email', jmsg.get('name', '')) + raise GerritError( + 200, 'Unable to set %s label for user "%s"' + ' on change %s.' % (label, username, change)) + jmsg = GetChangeCurrentRevision(host, change) + if not jmsg: + raise GerritError( + 200, 'Could not get review information for change "%s"' % change) - if jmsg[0]['current_revision'] != revision: - raise GerritError(200, 'While resetting labels on change "%s", ' - 'a new patchset was uploaded.' % change) + if jmsg[0]['current_revision'] != revision: + raise GerritError( + 200, 'While resetting labels on change "%s", ' + 'a new patchset was uploaded.' % change) def CreateChange(host, project, branch='main', subject='', params=()): - """ + """ Creates a new change. Args: @@ -1048,86 +1084,86 @@ def CreateChange(host, project, branch='main', subject='', params=()): Returns: ChangeInfo for the new change. """ - path = 'changes/' - body = {'project': project, 'branch': branch, 'subject': subject} - body.update(dict(params)) - for key in 'project', 'branch', 'subject': - if not body[key]: - raise GerritError(200, '%s is required' % key.title()) + path = 'changes/' + body = {'project': project, 'branch': branch, 'subject': subject} + body.update(dict(params)) + for key in 'project', 'branch', 'subject': + if not body[key]: + raise GerritError(200, '%s is required' % key.title()) - conn = CreateHttpConn(host, path, reqtype='POST', body=body) - return ReadHttpJsonResponse(conn, accept_statuses=[201]) + conn = CreateHttpConn(host, path, reqtype='POST', body=body) + return ReadHttpJsonResponse(conn, accept_statuses=[201]) def CreateGerritBranch(host, project, branch, commit): - """Creates a new branch from given project and commit + """Creates a new branch from given project and commit https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#create-branch Returns: A JSON object with 'ref' key. """ - path = 'projects/%s/branches/%s' % (project, branch) - body = {'revision': commit} - conn = CreateHttpConn(host, path, reqtype='PUT', body=body) - response = ReadHttpJsonResponse(conn, accept_statuses=[201, 409]) - if response: - return response - raise GerritError(200, 'Unable to create gerrit branch') + path = 'projects/%s/branches/%s' % (project, branch) + body = {'revision': commit} + conn = CreateHttpConn(host, path, reqtype='PUT', body=body) + response = ReadHttpJsonResponse(conn, accept_statuses=[201, 409]) + if response: + return response + raise GerritError(200, 'Unable to create gerrit branch') def CreateGerritTag(host, project, tag, commit): - """Creates a new tag at the given commit. + """Creates a new tag at the given commit. https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#create-tag Returns: A JSON object with 'ref' key. """ - path = 'projects/%s/tags/%s' % (project, tag) - body = {'revision': commit} - conn = CreateHttpConn(host, path, reqtype='PUT', body=body) - response = ReadHttpJsonResponse(conn, accept_statuses=[201]) - if response: - return response - raise GerritError(200, 'Unable to create gerrit tag') + path = 'projects/%s/tags/%s' % (project, tag) + body = {'revision': commit} + conn = CreateHttpConn(host, path, reqtype='PUT', body=body) + response = ReadHttpJsonResponse(conn, accept_statuses=[201]) + if response: + return response + raise GerritError(200, 'Unable to create gerrit tag') def GetHead(host, project): - """Retrieves current HEAD of Gerrit project + """Retrieves current HEAD of Gerrit project https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#get-head Returns: A JSON object with 'ref' key. """ - path = 'projects/%s/HEAD' % (project) - conn = CreateHttpConn(host, path, reqtype='GET') - response = ReadHttpJsonResponse(conn, accept_statuses=[200]) - if response: - return response - raise GerritError(200, 'Unable to update gerrit HEAD') + path = 'projects/%s/HEAD' % (project) + conn = CreateHttpConn(host, path, reqtype='GET') + response = ReadHttpJsonResponse(conn, accept_statuses=[200]) + if response: + return response + raise GerritError(200, 'Unable to update gerrit HEAD') def UpdateHead(host, project, branch): - """Updates Gerrit HEAD to point to branch + """Updates Gerrit HEAD to point to branch https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#set-head Returns: A JSON object with 'ref' key. """ - path = 'projects/%s/HEAD' % (project) - body = {'ref': branch} - conn = CreateHttpConn(host, path, reqtype='PUT', body=body) - response = ReadHttpJsonResponse(conn, accept_statuses=[200]) - if response: - return response - raise GerritError(200, 'Unable to update gerrit HEAD') + path = 'projects/%s/HEAD' % (project) + body = {'ref': branch} + conn = CreateHttpConn(host, path, reqtype='PUT', body=body) + response = ReadHttpJsonResponse(conn, accept_statuses=[200]) + if response: + return response + raise GerritError(200, 'Unable to update gerrit HEAD') def GetGerritBranch(host, project, branch): - """Gets a branch info from given project and branch name. + """Gets a branch info from given project and branch name. See: https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#get-branch @@ -1135,19 +1171,19 @@ def GetGerritBranch(host, project, branch): Returns: A JSON object with 'revision' key if the branch exists, otherwise None. """ - path = 'projects/%s/branches/%s' % (project, branch) - conn = CreateHttpConn(host, path, reqtype='GET') - return ReadHttpJsonResponse(conn, accept_statuses=[200, 404]) + path = 'projects/%s/branches/%s' % (project, branch) + conn = CreateHttpConn(host, path, reqtype='GET') + return ReadHttpJsonResponse(conn, accept_statuses=[200, 404]) def GetProjectHead(host, project): - conn = CreateHttpConn(host, - '/projects/%s/HEAD' % urllib.parse.quote(project, '')) - return ReadHttpJsonResponse(conn, accept_statuses=[200]) + conn = CreateHttpConn(host, + '/projects/%s/HEAD' % urllib.parse.quote(project, '')) + return ReadHttpJsonResponse(conn, accept_statuses=[200]) def GetAccountDetails(host, account_id='self'): - """Returns details of the account. + """Returns details of the account. If account_id is not given, uses magic value 'self' which corresponds to whichever account user is authenticating as. @@ -1157,37 +1193,38 @@ def GetAccountDetails(host, account_id='self'): Returns None if account is not found (i.e., Gerrit returned 404). """ - conn = CreateHttpConn(host, '/accounts/%s' % account_id) - return ReadHttpJsonResponse(conn, accept_statuses=[200, 404]) + conn = CreateHttpConn(host, '/accounts/%s' % account_id) + return ReadHttpJsonResponse(conn, accept_statuses=[200, 404]) def ValidAccounts(host, accounts, max_threads=10): - """Returns a mapping from valid account to its details. + """Returns a mapping from valid account to its details. Invalid accounts, either not existing or without unique match, are not present as returned dictionary keys. """ - assert not isinstance(accounts, str), type(accounts) - accounts = list(set(accounts)) - if not accounts: - return {} + assert not isinstance(accounts, str), type(accounts) + accounts = list(set(accounts)) + if not accounts: + return {} - def get_one(account): - try: - return account, GetAccountDetails(host, account) - except GerritError: - return None, None + def get_one(account): + try: + return account, GetAccountDetails(host, account) + except GerritError: + return None, None - valid = {} - with contextlib.closing(ThreadPool(min(max_threads, len(accounts)))) as pool: - for account, details in pool.map(get_one, accounts): - if account and details: - valid[account] = details - return valid + valid = {} + with contextlib.closing(ThreadPool(min(max_threads, + len(accounts)))) as pool: + for account, details in pool.map(get_one, accounts): + if account and details: + valid[account] = details + return valid def PercentEncodeForGitRef(original): - """Applies percent-encoding for strings sent to Gerrit via git ref metadata. + """Applies percent-encoding for strings sent to Gerrit via git ref metadata. The encoding used is based on but stricter than URL encoding (Section 2.1 of RFC 3986). The only non-escaped characters are alphanumerics, and 'SPACE' @@ -1197,31 +1234,32 @@ def PercentEncodeForGitRef(original): https://gerrit-review.googlesource.com/Documentation/user-upload.html#message """ - safe = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 ' - encoded = ''.join(c if c in safe else '%%%02X' % ord(c) for c in original) + safe = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 ' + encoded = ''.join(c if c in safe else '%%%02X' % ord(c) for c in original) - # Spaces are not allowed in git refs; gerrit will interpret either '_' or - # '+' (or '%20') as space. Use '_' since that has been supported the longest. - return encoded.replace(' ', '_') + # Spaces are not allowed in git refs; gerrit will interpret either '_' or + # '+' (or '%20') as space. Use '_' since that has been supported the + # longest. + return encoded.replace(' ', '_') @contextlib.contextmanager def tempdir(): - tdir = None - try: - tdir = tempfile.mkdtemp(suffix='gerrit_util') - yield tdir - finally: - if tdir: - gclient_utils.rmtree(tdir) + tdir = None + try: + tdir = tempfile.mkdtemp(suffix='gerrit_util') + yield tdir + finally: + if tdir: + gclient_utils.rmtree(tdir) def ChangeIdentifier(project, change_number): - """Returns change identifier "project~number" suitable for |change| arg of + """Returns change identifier "project~number" suitable for |change| arg of this module API. Such format is allows for more efficient Gerrit routing of HTTP requests, comparing to specifying just change_number. """ - assert int(change_number) - return '%s~%s' % (urllib.parse.quote(project, ''), change_number) + assert int(change_number) + return '%s~%s' % (urllib.parse.quote(project, ''), change_number) diff --git a/git_cache.py b/git_cache.py index 816f415ac5..a15b3f939d 100755 --- a/git_cache.py +++ b/git_cache.py @@ -2,7 +2,6 @@ # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """A git command for managing a local cache of git repositories.""" import contextlib @@ -35,19 +34,25 @@ GIT_CACHE_CORRUPT_MESSAGE = 'WARNING: The Git cache is corrupt.' GSUTIL_CP_SEMAPHORE = threading.Semaphore(2) try: - # pylint: disable=undefined-variable - WinErr = WindowsError + # pylint: disable=undefined-variable + WinErr = WindowsError except NameError: - class WinErr(Exception): - pass + + class WinErr(Exception): + pass + class ClobberNeeded(Exception): - pass + pass -def exponential_backoff_retry(fn, excs=(Exception,), name=None, count=10, - sleep_time=0.25, printerr=None): - """Executes |fn| up to |count| times, backing off exponentially. +def exponential_backoff_retry(fn, + excs=(Exception, ), + name=None, + count=10, + sleep_time=0.25, + printerr=None): + """Executes |fn| up to |count| times, backing off exponentially. Args: fn (callable): The function to execute. If this raises a handled @@ -66,818 +71,862 @@ def exponential_backoff_retry(fn, excs=(Exception,), name=None, count=10, Returns: The return value of the successful fn. """ - printerr = printerr or logging.warning - for i in range(count): - try: - return fn() - except excs as e: - if (i+1) >= count: - raise + printerr = printerr or logging.warning + for i in range(count): + try: + return fn() + except excs as e: + if (i + 1) >= count: + raise - printerr('Retrying %s in %.2f second(s) (%d / %d attempts): %s' % ( - (name or 'operation'), sleep_time, (i+1), count, e)) - time.sleep(sleep_time) - sleep_time *= 2 + printerr('Retrying %s in %.2f second(s) (%d / %d attempts): %s' % + ((name or 'operation'), sleep_time, (i + 1), count, e)) + time.sleep(sleep_time) + sleep_time *= 2 class Mirror(object): - git_exe = 'git.bat' if sys.platform.startswith('win') else 'git' - gsutil_exe = os.path.join( - os.path.dirname(os.path.abspath(__file__)), 'gsutil.py') - cachepath_lock = threading.Lock() + git_exe = 'git.bat' if sys.platform.startswith('win') else 'git' + gsutil_exe = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'gsutil.py') + cachepath_lock = threading.Lock() - UNSET_CACHEPATH = object() + UNSET_CACHEPATH = object() - # Used for tests - _GIT_CONFIG_LOCATION = [] + # Used for tests + _GIT_CONFIG_LOCATION = [] - @staticmethod - def parse_fetch_spec(spec): - """Parses and canonicalizes a fetch spec. + @staticmethod + def parse_fetch_spec(spec): + """Parses and canonicalizes a fetch spec. Returns (fetchspec, value_regex), where value_regex can be used with 'git config --replace-all'. """ - parts = spec.split(':', 1) - src = parts[0].lstrip('+').rstrip('/') - if not src.startswith('refs/'): - src = 'refs/heads/%s' % src - dest = parts[1].rstrip('/') if len(parts) > 1 else src - regex = r'\+%s:.*' % src.replace('*', r'\*') - return ('+%s:%s' % (src, dest), regex) + parts = spec.split(':', 1) + src = parts[0].lstrip('+').rstrip('/') + if not src.startswith('refs/'): + src = 'refs/heads/%s' % src + dest = parts[1].rstrip('/') if len(parts) > 1 else src + regex = r'\+%s:.*' % src.replace('*', r'\*') + return ('+%s:%s' % (src, dest), regex) - def __init__(self, url, refs=None, commits=None, print_func=None): - self.url = url - self.fetch_specs = {self.parse_fetch_spec(ref) for ref in (refs or [])} - self.fetch_commits = set(commits or []) - self.basedir = self.UrlToCacheDir(url) - self.mirror_path = os.path.join(self.GetCachePath(), self.basedir) - if print_func: - self.print = self.print_without_file - self.print_func = print_func - else: - self.print = print + def __init__(self, url, refs=None, commits=None, print_func=None): + self.url = url + self.fetch_specs = {self.parse_fetch_spec(ref) for ref in (refs or [])} + self.fetch_commits = set(commits or []) + self.basedir = self.UrlToCacheDir(url) + self.mirror_path = os.path.join(self.GetCachePath(), self.basedir) + if print_func: + self.print = self.print_without_file + self.print_func = print_func + else: + self.print = print - def print_without_file(self, message, **_kwargs): - self.print_func(message) + def print_without_file(self, message, **_kwargs): + self.print_func(message) - @contextlib.contextmanager - def print_duration_of(self, what): - start = time.time() - try: - yield - finally: - self.print('%s took %.1f minutes' % (what, (time.time() - start) / 60.0)) - - @property - def bootstrap_bucket(self): - b = os.getenv('OVERRIDE_BOOTSTRAP_BUCKET') - if b: - return b - u = urllib.parse.urlparse(self.url) - if u.netloc == 'chromium.googlesource.com': - return 'chromium-git-cache' - # Not recognized. - return None - - @property - def _gs_path(self): - return 'gs://%s/v2/%s' % (self.bootstrap_bucket, self.basedir) - - @classmethod - def FromPath(cls, path): - return cls(cls.CacheDirToUrl(path)) - - @staticmethod - def UrlToCacheDir(url): - """Convert a git url to a normalized form for the cache dir path.""" - if os.path.isdir(url): - # Ignore the drive letter in Windows - url = os.path.splitdrive(url)[1] - return url.replace('-', '--').replace(os.sep, '-') - - parsed = urllib.parse.urlparse(url) - norm_url = parsed.netloc + parsed.path - if norm_url.endswith('.git'): - norm_url = norm_url[:-len('.git')] - - # Use the same dir for authenticated URLs and unauthenticated URLs. - norm_url = norm_url.replace('googlesource.com/a/', 'googlesource.com/') - - return norm_url.replace('-', '--').replace('/', '-').lower() - - @staticmethod - def CacheDirToUrl(path): - """Convert a cache dir path to its corresponding url.""" - netpath = re.sub(r'\b-\b', '/', os.path.basename(path)).replace('--', '-') - return 'https://%s' % netpath - - @classmethod - def SetCachePath(cls, cachepath): - with cls.cachepath_lock: - setattr(cls, 'cachepath', cachepath) - - @classmethod - def GetCachePath(cls): - with cls.cachepath_lock: - if not hasattr(cls, 'cachepath'): + @contextlib.contextmanager + def print_duration_of(self, what): + start = time.time() try: - cachepath = subprocess.check_output( - [cls.git_exe, 'config'] + - cls._GIT_CONFIG_LOCATION + - ['cache.cachepath']).decode('utf-8', 'ignore').strip() + yield + finally: + self.print('%s took %.1f minutes' % (what, + (time.time() - start) / 60.0)) + + @property + def bootstrap_bucket(self): + b = os.getenv('OVERRIDE_BOOTSTRAP_BUCKET') + if b: + return b + u = urllib.parse.urlparse(self.url) + if u.netloc == 'chromium.googlesource.com': + return 'chromium-git-cache' + # Not recognized. + return None + + @property + def _gs_path(self): + return 'gs://%s/v2/%s' % (self.bootstrap_bucket, self.basedir) + + @classmethod + def FromPath(cls, path): + return cls(cls.CacheDirToUrl(path)) + + @staticmethod + def UrlToCacheDir(url): + """Convert a git url to a normalized form for the cache dir path.""" + if os.path.isdir(url): + # Ignore the drive letter in Windows + url = os.path.splitdrive(url)[1] + return url.replace('-', '--').replace(os.sep, '-') + + parsed = urllib.parse.urlparse(url) + norm_url = parsed.netloc + parsed.path + if norm_url.endswith('.git'): + norm_url = norm_url[:-len('.git')] + + # Use the same dir for authenticated URLs and unauthenticated URLs. + norm_url = norm_url.replace('googlesource.com/a/', 'googlesource.com/') + + return norm_url.replace('-', '--').replace('/', '-').lower() + + @staticmethod + def CacheDirToUrl(path): + """Convert a cache dir path to its corresponding url.""" + netpath = re.sub(r'\b-\b', '/', + os.path.basename(path)).replace('--', '-') + return 'https://%s' % netpath + + @classmethod + def SetCachePath(cls, cachepath): + with cls.cachepath_lock: + setattr(cls, 'cachepath', cachepath) + + @classmethod + def GetCachePath(cls): + with cls.cachepath_lock: + if not hasattr(cls, 'cachepath'): + try: + cachepath = subprocess.check_output( + [cls.git_exe, 'config'] + cls._GIT_CONFIG_LOCATION + + ['cache.cachepath']).decode('utf-8', 'ignore').strip() + except subprocess.CalledProcessError: + cachepath = os.environ.get('GIT_CACHE_PATH', + cls.UNSET_CACHEPATH) + setattr(cls, 'cachepath', cachepath) + + ret = getattr(cls, 'cachepath') + if ret is cls.UNSET_CACHEPATH: + raise RuntimeError('No cache.cachepath git configuration or ' + '$GIT_CACHE_PATH is set.') + return ret + + @staticmethod + def _GetMostRecentCacheDirectory(ls_out_set): + ready_file_pattern = re.compile(r'.*/(\d+).ready$') + ready_dirs = [] + + for name in ls_out_set: + m = ready_file_pattern.match(name) + # Given /.ready, + # we are interested in / directory + if m and (name[:-len('.ready')] + '/') in ls_out_set: + ready_dirs.append((int(m.group(1)), name[:-len('.ready')])) + + if not ready_dirs: + return None + + return max(ready_dirs)[1] + + def Rename(self, src, dst): + # This is somehow racy on Windows. + # Catching OSError because WindowsError isn't portable and + # pylint complains. + exponential_backoff_retry(lambda: os.rename(src, dst), + excs=(OSError, ), + name='rename [%s] => [%s]' % (src, dst), + printerr=self.print) + + def RunGit(self, cmd, print_stdout=True, **kwargs): + """Run git in a subprocess.""" + cwd = kwargs.setdefault('cwd', self.mirror_path) + if "--git-dir" not in cmd: + cmd = ['--git-dir', os.path.abspath(cwd)] + cmd + + kwargs.setdefault('print_stdout', False) + if print_stdout: + kwargs.setdefault('filter_fn', self.print) + env = kwargs.get('env') or kwargs.setdefault('env', os.environ.copy()) + env.setdefault('GIT_ASKPASS', 'true') + env.setdefault('SSH_ASKPASS', 'true') + self.print('running "git %s" in "%s"' % (' '.join(cmd), cwd)) + gclient_utils.CheckCallAndFilter([self.git_exe] + cmd, **kwargs) + + def config(self, reset_fetch_config=False): + if reset_fetch_config: + try: + self.RunGit(['config', '--unset-all', 'remote.origin.fetch']) + except subprocess.CalledProcessError as e: + # If exit code was 5, it means we attempted to unset a config + # that didn't exist. Ignore it. + if e.returncode != 5: + raise + + # Don't run git-gc in a daemon. Bad things can happen if it gets + # killed. + try: + self.RunGit(['config', 'gc.autodetach', '0']) except subprocess.CalledProcessError: - cachepath = os.environ.get('GIT_CACHE_PATH', cls.UNSET_CACHEPATH) - setattr(cls, 'cachepath', cachepath) + # Hard error, need to clobber. + raise ClobberNeeded() - ret = getattr(cls, 'cachepath') - if ret is cls.UNSET_CACHEPATH: - raise RuntimeError('No cache.cachepath git configuration or ' - '$GIT_CACHE_PATH is set.') - return ret + # Don't combine pack files into one big pack file. It's really slow for + # repositories, and there's no way to track progress and make sure it's + # not stuck. + if self.supported_project(): + self.RunGit(['config', 'gc.autopacklimit', '0']) - @staticmethod - def _GetMostRecentCacheDirectory(ls_out_set): - ready_file_pattern = re.compile(r'.*/(\d+).ready$') - ready_dirs = [] + # Allocate more RAM for cache-ing delta chains, for better performance + # of "Resolving deltas". + self.RunGit([ + 'config', 'core.deltaBaseCacheLimit', + gclient_utils.DefaultDeltaBaseCacheLimit() + ]) - for name in ls_out_set: - m = ready_file_pattern.match(name) - # Given /.ready, - # we are interested in / directory - if m and (name[:-len('.ready')] + '/') in ls_out_set: - ready_dirs.append((int(m.group(1)), name[:-len('.ready')])) + self.RunGit(['config', 'remote.origin.url', self.url]) + self.RunGit([ + 'config', '--replace-all', 'remote.origin.fetch', + '+refs/heads/*:refs/heads/*', r'\+refs/heads/\*:.*' + ]) + for spec, value_regex in self.fetch_specs: + self.RunGit([ + 'config', '--replace-all', 'remote.origin.fetch', spec, + value_regex + ]) - if not ready_dirs: - return None - - return max(ready_dirs)[1] - - def Rename(self, src, dst): - # This is somehow racy on Windows. - # Catching OSError because WindowsError isn't portable and - # pylint complains. - exponential_backoff_retry( - lambda: os.rename(src, dst), - excs=(OSError,), - name='rename [%s] => [%s]' % (src, dst), - printerr=self.print) - - def RunGit(self, cmd, print_stdout=True, **kwargs): - """Run git in a subprocess.""" - cwd = kwargs.setdefault('cwd', self.mirror_path) - if "--git-dir" not in cmd: - cmd = ['--git-dir', os.path.abspath(cwd)] + cmd - - kwargs.setdefault('print_stdout', False) - if print_stdout: - kwargs.setdefault('filter_fn', self.print) - env = kwargs.get('env') or kwargs.setdefault('env', os.environ.copy()) - env.setdefault('GIT_ASKPASS', 'true') - env.setdefault('SSH_ASKPASS', 'true') - self.print('running "git %s" in "%s"' % (' '.join(cmd), cwd)) - gclient_utils.CheckCallAndFilter([self.git_exe] + cmd, **kwargs) - - def config(self, reset_fetch_config=False): - if reset_fetch_config: - try: - self.RunGit(['config', '--unset-all', 'remote.origin.fetch']) - except subprocess.CalledProcessError as e: - # If exit code was 5, it means we attempted to unset a config that - # didn't exist. Ignore it. - if e.returncode != 5: - raise - - # Don't run git-gc in a daemon. Bad things can happen if it gets killed. - try: - self.RunGit(['config', 'gc.autodetach', '0']) - except subprocess.CalledProcessError: - # Hard error, need to clobber. - raise ClobberNeeded() - - # Don't combine pack files into one big pack file. It's really slow for - # repositories, and there's no way to track progress and make sure it's - # not stuck. - if self.supported_project(): - self.RunGit(['config', 'gc.autopacklimit', '0']) - - # Allocate more RAM for cache-ing delta chains, for better performance - # of "Resolving deltas". - self.RunGit([ - 'config', 'core.deltaBaseCacheLimit', - gclient_utils.DefaultDeltaBaseCacheLimit() - ]) - - self.RunGit(['config', 'remote.origin.url', self.url]) - self.RunGit([ - 'config', '--replace-all', 'remote.origin.fetch', - '+refs/heads/*:refs/heads/*', r'\+refs/heads/\*:.*' - ]) - for spec, value_regex in self.fetch_specs: - self.RunGit( - ['config', '--replace-all', 'remote.origin.fetch', spec, value_regex]) - - def bootstrap_repo(self, directory): - """Bootstrap the repo from Google Storage if possible. + def bootstrap_repo(self, directory): + """Bootstrap the repo from Google Storage if possible. More apt-ly named bootstrap_repo_from_cloud_if_possible_else_do_nothing(). """ - if not self.bootstrap_bucket: - return False + if not self.bootstrap_bucket: + return False - gsutil = Gsutil(self.gsutil_exe, boto_path=None) + gsutil = Gsutil(self.gsutil_exe, boto_path=None) - # Get the most recent version of the directory. - # This is determined from the most recent version of a .ready file. - # The .ready file is only uploaded when an entire directory has been - # uploaded to GS. - _, ls_out, ls_err = gsutil.check_call('ls', self._gs_path) - ls_out_set = set(ls_out.strip().splitlines()) - latest_dir = self._GetMostRecentCacheDirectory(ls_out_set) + # Get the most recent version of the directory. + # This is determined from the most recent version of a .ready file. + # The .ready file is only uploaded when an entire directory has been + # uploaded to GS. + _, ls_out, ls_err = gsutil.check_call('ls', self._gs_path) + ls_out_set = set(ls_out.strip().splitlines()) + latest_dir = self._GetMostRecentCacheDirectory(ls_out_set) - if not latest_dir: - self.print('No bootstrap file for %s found in %s, stderr:\n %s' % - (self.mirror_path, self.bootstrap_bucket, - ' '.join((ls_err or '').splitlines(True)))) - return False + if not latest_dir: + self.print('No bootstrap file for %s found in %s, stderr:\n %s' % + (self.mirror_path, self.bootstrap_bucket, ' '.join( + (ls_err or '').splitlines(True)))) + return False - try: - # create new temporary directory locally - tempdir = tempfile.mkdtemp(prefix='_cache_tmp', dir=self.GetCachePath()) - self.RunGit(['init', '-b', 'main', '--bare'], cwd=tempdir) - self.print('Downloading files in %s/* into %s.' % - (latest_dir, tempdir)) - with self.print_duration_of('download'): - with GSUTIL_CP_SEMAPHORE: - code = gsutil.call('-m', 'cp', '-r', latest_dir + "/*", - tempdir) - if code: - return False - # A quick validation that all references are valid. - self.RunGit(['for-each-ref'], print_stdout=False, cwd=tempdir) - except Exception as e: - self.print('Encountered error: %s' % str(e), file=sys.stderr) - gclient_utils.rmtree(tempdir) - return False - # delete the old directory - if os.path.exists(directory): - gclient_utils.rmtree(directory) - self.Rename(tempdir, directory) - return True + try: + # create new temporary directory locally + tempdir = tempfile.mkdtemp(prefix='_cache_tmp', + dir=self.GetCachePath()) + self.RunGit(['init', '-b', 'main', '--bare'], cwd=tempdir) + self.print('Downloading files in %s/* into %s.' % + (latest_dir, tempdir)) + with self.print_duration_of('download'): + with GSUTIL_CP_SEMAPHORE: + code = gsutil.call('-m', 'cp', '-r', latest_dir + "/*", + tempdir) + if code: + return False + # A quick validation that all references are valid. + self.RunGit(['for-each-ref'], print_stdout=False, cwd=tempdir) + except Exception as e: + self.print('Encountered error: %s' % str(e), file=sys.stderr) + gclient_utils.rmtree(tempdir) + return False + # delete the old directory + if os.path.exists(directory): + gclient_utils.rmtree(directory) + self.Rename(tempdir, directory) + return True - def contains_revision(self, revision): - if not self.exists(): - return False + def contains_revision(self, revision): + if not self.exists(): + return False - if sys.platform.startswith('win'): - # Windows .bat scripts use ^ as escape sequence, which means we have to - # escape it with itself for every .bat invocation. - needle = '%s^^^^{commit}' % revision - else: - needle = '%s^{commit}' % revision - try: - # cat-file exits with 0 on success, that is git object of given hash was - # found. - self.RunGit(['cat-file', '-e', needle]) - return True - except subprocess.CalledProcessError: - self.print('Commit with hash "%s" not found' % revision, file=sys.stderr) - return False + if sys.platform.startswith('win'): + # Windows .bat scripts use ^ as escape sequence, which means we have + # to escape it with itself for every .bat invocation. + needle = '%s^^^^{commit}' % revision + else: + needle = '%s^{commit}' % revision + try: + # cat-file exits with 0 on success, that is git object of given hash + # was found. + self.RunGit(['cat-file', '-e', needle]) + return True + except subprocess.CalledProcessError: + self.print('Commit with hash "%s" not found' % revision, + file=sys.stderr) + return False - def exists(self): - return os.path.isfile(os.path.join(self.mirror_path, 'config')) + def exists(self): + return os.path.isfile(os.path.join(self.mirror_path, 'config')) - def supported_project(self): - """Returns true if this repo is known to have a bootstrap zip file.""" - u = urllib.parse.urlparse(self.url) - return u.netloc in [ - 'chromium.googlesource.com', - 'chrome-internal.googlesource.com'] + def supported_project(self): + """Returns true if this repo is known to have a bootstrap zip file.""" + u = urllib.parse.urlparse(self.url) + return u.netloc in [ + 'chromium.googlesource.com', 'chrome-internal.googlesource.com' + ] - def _preserve_fetchspec(self): - """Read and preserve remote.origin.fetch from an existing mirror. + def _preserve_fetchspec(self): + """Read and preserve remote.origin.fetch from an existing mirror. This modifies self.fetch_specs. """ - if not self.exists(): - return - try: - config_fetchspecs = subprocess.check_output( - [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'], - cwd=self.mirror_path).decode('utf-8', 'ignore') - for fetchspec in config_fetchspecs.splitlines(): - self.fetch_specs.add(self.parse_fetch_spec(fetchspec)) - except subprocess.CalledProcessError: - logging.warning( - 'Tried and failed to preserve remote.origin.fetch from the ' - 'existing cache directory. You may need to manually edit ' - '%s and "git cache fetch" again.' % - os.path.join(self.mirror_path, 'config')) + if not self.exists(): + return + try: + config_fetchspecs = subprocess.check_output( + [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'], + cwd=self.mirror_path).decode('utf-8', 'ignore') + for fetchspec in config_fetchspecs.splitlines(): + self.fetch_specs.add(self.parse_fetch_spec(fetchspec)) + except subprocess.CalledProcessError: + logging.warning( + 'Tried and failed to preserve remote.origin.fetch from the ' + 'existing cache directory. You may need to manually edit ' + '%s and "git cache fetch" again.' % + os.path.join(self.mirror_path, 'config')) - def _ensure_bootstrapped( - self, depth, bootstrap, reset_fetch_config, force=False): - pack_dir = os.path.join(self.mirror_path, 'objects', 'pack') - pack_files = [] - if os.path.isdir(pack_dir): - pack_files = [f for f in os.listdir(pack_dir) if f.endswith('.pack')] - self.print('%s has %d .pack files, re-bootstrapping if >%d or ==0' % - (self.mirror_path, len(pack_files), GC_AUTOPACKLIMIT)) + def _ensure_bootstrapped(self, + depth, + bootstrap, + reset_fetch_config, + force=False): + pack_dir = os.path.join(self.mirror_path, 'objects', 'pack') + pack_files = [] + if os.path.isdir(pack_dir): + pack_files = [ + f for f in os.listdir(pack_dir) if f.endswith('.pack') + ] + self.print('%s has %d .pack files, re-bootstrapping if >%d or ==0' % + (self.mirror_path, len(pack_files), GC_AUTOPACKLIMIT)) - # master->main branch migration left the cache in some builders to have its - # HEAD still pointing to refs/heads/master. This causes bot_update to fail. - # If in this state, delete the cache and force bootstrap. - try: - with open(os.path.join(self.mirror_path, 'HEAD')) as f: - head_ref = f.read() - except FileNotFoundError: - head_ref = '' + # master->main branch migration left the cache in some builders to have + # its HEAD still pointing to refs/heads/master. This causes bot_update + # to fail. If in this state, delete the cache and force bootstrap. + try: + with open(os.path.join(self.mirror_path, 'HEAD')) as f: + head_ref = f.read() + except FileNotFoundError: + head_ref = '' - # Check only when HEAD points to master. - if 'master' in head_ref: - # Some repos could still have master so verify if the ref exists first. - show_ref_master_cmd = subprocess.run( - [Mirror.git_exe, 'show-ref', '--verify', 'refs/heads/master'], - cwd=self.mirror_path) + # Check only when HEAD points to master. + if 'master' in head_ref: + # Some repos could still have master so verify if the ref exists + # first. + show_ref_master_cmd = subprocess.run( + [Mirror.git_exe, 'show-ref', '--verify', 'refs/heads/master'], + cwd=self.mirror_path) - if show_ref_master_cmd.returncode != 0: - # Remove mirror - gclient_utils.rmtree(self.mirror_path) + if show_ref_master_cmd.returncode != 0: + # Remove mirror + gclient_utils.rmtree(self.mirror_path) - # force bootstrap - force = True + # force bootstrap + force = True - should_bootstrap = (force or - not self.exists() or - len(pack_files) > GC_AUTOPACKLIMIT or - len(pack_files) == 0) + should_bootstrap = (force or not self.exists() + or len(pack_files) > GC_AUTOPACKLIMIT + or len(pack_files) == 0) - if not should_bootstrap: - if depth and os.path.exists(os.path.join(self.mirror_path, 'shallow')): - logging.warning( - 'Shallow fetch requested, but repo cache already exists.') - return + if not should_bootstrap: + if depth and os.path.exists( + os.path.join(self.mirror_path, 'shallow')): + logging.warning( + 'Shallow fetch requested, but repo cache already exists.') + return - if not self.exists(): - if os.path.exists(self.mirror_path): - # If the mirror path exists but self.exists() returns false, we're - # in an unexpected state. Nuke the previous mirror directory and - # start fresh. - gclient_utils.rmtree(self.mirror_path) - os.mkdir(self.mirror_path) - elif not reset_fetch_config: - # Re-bootstrapping an existing mirror; preserve existing fetch spec. - self._preserve_fetchspec() + if not self.exists(): + if os.path.exists(self.mirror_path): + # If the mirror path exists but self.exists() returns false, + # we're in an unexpected state. Nuke the previous mirror + # directory and start fresh. + gclient_utils.rmtree(self.mirror_path) + os.mkdir(self.mirror_path) + elif not reset_fetch_config: + # Re-bootstrapping an existing mirror; preserve existing fetch spec. + self._preserve_fetchspec() - bootstrapped = (not depth and bootstrap and - self.bootstrap_repo(self.mirror_path)) + bootstrapped = (not depth and bootstrap + and self.bootstrap_repo(self.mirror_path)) - if not bootstrapped: - if not self.exists() or not self.supported_project(): - # Bootstrap failed due to: - # 1. No previous cache. - # 2. Project doesn't have a bootstrap folder. - # Start with a bare git dir. - self.RunGit(['init', '--bare']) - # Set appropriate symbolic-ref - remote_info = exponential_backoff_retry(lambda: subprocess.check_output( + if not bootstrapped: + if not self.exists() or not self.supported_project(): + # Bootstrap failed due to: + # 1. No previous cache. + # 2. Project doesn't have a bootstrap folder. + # Start with a bare git dir. + self.RunGit(['init', '--bare']) + # Set appropriate symbolic-ref + remote_info = exponential_backoff_retry( + lambda: subprocess.check_output( + [ + self.git_exe, '--git-dir', + os.path.abspath(self.mirror_path), 'remote', 'show', + self.url + ], + cwd=self.mirror_path).decode('utf-8', 'ignore').strip()) + default_branch_regexp = re.compile(r'HEAD branch: (.*)$') + m = default_branch_regexp.search(remote_info, re.MULTILINE) + if m: + self.RunGit( + ['symbolic-ref', 'HEAD', 'refs/heads/' + m.groups()[0]]) + else: + # Bootstrap failed, previous cache exists; warn and continue. + logging.warning( + 'Git cache has a lot of pack files (%d). Tried to ' + 're-bootstrap but failed. Continuing with non-optimized ' + 'repository.' % len(pack_files)) + + def _fetch(self, + verbose, + depth, + no_fetch_tags, + reset_fetch_config, + prune=True): + self.config(reset_fetch_config) + + fetch_cmd = ['fetch'] + if verbose: + fetch_cmd.extend(['-v', '--progress']) + if depth: + fetch_cmd.extend(['--depth', str(depth)]) + if no_fetch_tags: + fetch_cmd.append('--no-tags') + if prune: + fetch_cmd.append('--prune') + fetch_cmd.append('origin') + + fetch_specs = subprocess.check_output( [ self.git_exe, '--git-dir', - os.path.abspath(self.mirror_path), 'remote', 'show', self.url + os.path.abspath(self.mirror_path), 'config', '--get-all', + 'remote.origin.fetch' ], - cwd=self.mirror_path).decode('utf-8', 'ignore').strip()) - default_branch_regexp = re.compile(r'HEAD branch: (.*)$') - m = default_branch_regexp.search(remote_info, re.MULTILINE) - if m: - self.RunGit(['symbolic-ref', 'HEAD', 'refs/heads/' + m.groups()[0]]) - else: - # Bootstrap failed, previous cache exists; warn and continue. - logging.warning( - 'Git cache has a lot of pack files (%d). Tried to re-bootstrap ' - 'but failed. Continuing with non-optimized repository.' % - len(pack_files)) + cwd=self.mirror_path).decode('utf-8', + 'ignore').strip().splitlines() + for spec in fetch_specs: + try: + self.print('Fetching %s' % spec) + with self.print_duration_of('fetch %s' % spec): + self.RunGit(fetch_cmd + [spec], retry=True) + except subprocess.CalledProcessError: + if spec == '+refs/heads/*:refs/heads/*': + raise ClobberNeeded() # Corrupted cache. + logging.warning('Fetch of %s failed' % spec) + for commit in self.fetch_commits: + self.print('Fetching %s' % commit) + try: + with self.print_duration_of('fetch %s' % commit): + self.RunGit(['fetch', 'origin', commit], retry=True) + except subprocess.CalledProcessError: + logging.warning('Fetch of %s failed' % commit) - def _fetch(self, - verbose, - depth, - no_fetch_tags, - reset_fetch_config, - prune=True): - self.config(reset_fetch_config) + def populate(self, + depth=None, + no_fetch_tags=False, + shallow=False, + bootstrap=False, + verbose=False, + lock_timeout=0, + reset_fetch_config=False): + assert self.GetCachePath() + if shallow and not depth: + depth = 10000 + gclient_utils.safe_makedirs(self.GetCachePath()) - fetch_cmd = ['fetch'] - if verbose: - fetch_cmd.extend(['-v', '--progress']) - if depth: - fetch_cmd.extend(['--depth', str(depth)]) - if no_fetch_tags: - fetch_cmd.append('--no-tags') - if prune: - fetch_cmd.append('--prune') - fetch_cmd.append('origin') + with lockfile.lock(self.mirror_path, lock_timeout): + try: + self._ensure_bootstrapped(depth, bootstrap, reset_fetch_config) + self._fetch(verbose, depth, no_fetch_tags, reset_fetch_config) + except ClobberNeeded: + # This is a major failure, we need to clean and force a + # bootstrap. + gclient_utils.rmtree(self.mirror_path) + self.print(GIT_CACHE_CORRUPT_MESSAGE) + self._ensure_bootstrapped(depth, + bootstrap, + reset_fetch_config, + force=True) + self._fetch(verbose, depth, no_fetch_tags, reset_fetch_config) - fetch_specs = subprocess.check_output([ - self.git_exe, '--git-dir', - os.path.abspath(self.mirror_path), 'config', '--get-all', - 'remote.origin.fetch' - ], - cwd=self.mirror_path).decode( - 'utf-8', - 'ignore').strip().splitlines() - for spec in fetch_specs: - try: - self.print('Fetching %s' % spec) - with self.print_duration_of('fetch %s' % spec): - self.RunGit(fetch_cmd + [spec], retry=True) - except subprocess.CalledProcessError: - if spec == '+refs/heads/*:refs/heads/*': - raise ClobberNeeded() # Corrupted cache. - logging.warning('Fetch of %s failed' % spec) - for commit in self.fetch_commits: - self.print('Fetching %s' % commit) - try: - with self.print_duration_of('fetch %s' % commit): - self.RunGit(['fetch', 'origin', commit], retry=True) - except subprocess.CalledProcessError: - logging.warning('Fetch of %s failed' % commit) + def update_bootstrap(self, prune=False, gc_aggressive=False): + # NOTE: There have been cases where repos were being recursively + # uploaded to google storage. E.g. + # `-//-/` in GS and + # -/-/ on the bot. Check for recursed + # files on the bot here and remove them if found before we upload to GS. + # See crbug.com/1370443; keep this check until root cause is found. + recursed_dir = os.path.join(self.mirror_path, + self.mirror_path.split(os.path.sep)[-1]) + if os.path.exists(recursed_dir): + self.print('Deleting unexpected directory: %s' % recursed_dir) + gclient_utils.rmtree(recursed_dir) - def populate(self, - depth=None, - no_fetch_tags=False, - shallow=False, - bootstrap=False, - verbose=False, - lock_timeout=0, - reset_fetch_config=False): - assert self.GetCachePath() - if shallow and not depth: - depth = 10000 - gclient_utils.safe_makedirs(self.GetCachePath()) + # The folder is + gen_number = subprocess.check_output([self.git_exe, 'number'], + cwd=self.mirror_path).decode( + 'utf-8', 'ignore').strip() + gsutil = Gsutil(path=self.gsutil_exe, boto_path=None) - with lockfile.lock(self.mirror_path, lock_timeout): - try: - self._ensure_bootstrapped(depth, bootstrap, reset_fetch_config) - self._fetch(verbose, depth, no_fetch_tags, reset_fetch_config) - except ClobberNeeded: - # This is a major failure, we need to clean and force a bootstrap. - gclient_utils.rmtree(self.mirror_path) - self.print(GIT_CACHE_CORRUPT_MESSAGE) - self._ensure_bootstrapped(depth, - bootstrap, - reset_fetch_config, - force=True) - self._fetch(verbose, depth, no_fetch_tags, reset_fetch_config) + dest_prefix = '%s/%s' % (self._gs_path, gen_number) - def update_bootstrap(self, prune=False, gc_aggressive=False): - # NOTE: There have been cases where repos were being recursively uploaded - # to google storage. - # E.g. `-//-/` in GS and - # -/-/ on the bot. - # Check for recursed files on the bot here and remove them if found - # before we upload to GS. - # See crbug.com/1370443; keep this check until root cause is found. - recursed_dir = os.path.join(self.mirror_path, - self.mirror_path.split(os.path.sep)[-1]) - if os.path.exists(recursed_dir): - self.print('Deleting unexpected directory: %s' % recursed_dir) - gclient_utils.rmtree(recursed_dir) + # ls_out lists contents in the format: gs://blah/blah/123... + self.print('running "gsutil ls %s":' % self._gs_path) + ls_code, ls_out, ls_error = gsutil.check_call_with_retries( + 'ls', self._gs_path) + if ls_code != 0: + self.print(ls_error) + else: + self.print(ls_out) - # The folder is - gen_number = subprocess.check_output([self.git_exe, 'number'], - cwd=self.mirror_path).decode( - 'utf-8', 'ignore').strip() - gsutil = Gsutil(path=self.gsutil_exe, boto_path=None) + # Check to see if folder already exists in gs + ls_out_set = set(ls_out.strip().splitlines()) + if (dest_prefix + '/' in ls_out_set + and dest_prefix + '.ready' in ls_out_set): + print('Cache %s already exists.' % dest_prefix) + return - dest_prefix = '%s/%s' % (self._gs_path, gen_number) + # Reduce the number of individual files to download & write on disk. + self.RunGit(['pack-refs', '--all']) - # ls_out lists contents in the format: gs://blah/blah/123... - self.print('running "gsutil ls %s":' % self._gs_path) - ls_code, ls_out, ls_error = gsutil.check_call_with_retries( - 'ls', self._gs_path) - if ls_code != 0: - self.print(ls_error) - else: - self.print(ls_out) + # Run Garbage Collect to compress packfile. + gc_args = ['gc', '--prune=all'] + if gc_aggressive: + # The default "gc --aggressive" is often too aggressive for some + # machines, since it attempts to create as many threads as there are + # CPU cores, while not limiting per-thread memory usage, which puts + # too much pressure on RAM on high-core machines, causing them to + # thrash. Using lower-level commands gives more control over those + # settings. - # Check to see if folder already exists in gs - ls_out_set = set(ls_out.strip().splitlines()) - if (dest_prefix + '/' in ls_out_set and - dest_prefix + '.ready' in ls_out_set): - print('Cache %s already exists.' % dest_prefix) - return + # This might not be strictly necessary, but it's fast and is + # normally run by 'gc --aggressive', so it shouldn't hurt. + self.RunGit(['reflog', 'expire', '--all']) - # Reduce the number of individual files to download & write on disk. - self.RunGit(['pack-refs', '--all']) + # These are the default repack settings for 'gc --aggressive'. + gc_args = [ + 'repack', '-d', '-l', '-f', '--depth=50', '--window=250', '-A', + '--unpack-unreachable=all' + ] + # A 1G memory limit seems to provide comparable pack results as the + # default, even for our largest repos, while preventing runaway + # memory (at least on current Chromium builders which have about 4G + # RAM per core). + gc_args.append('--window-memory=1g') + # NOTE: It might also be possible to avoid thrashing with a larger + # window (e.g. "--window-memory=2g") by limiting the number of + # threads created (e.g. "--threads=[cores/2]"). Some limited testing + # didn't show much difference in outcomes on our current repos, but + # it might be worth trying if the repos grow much larger and the + # packs don't seem to be getting compressed enough. + self.RunGit(gc_args) - # Run Garbage Collect to compress packfile. - gc_args = ['gc', '--prune=all'] - if gc_aggressive: - # The default "gc --aggressive" is often too aggressive for some machines, - # since it attempts to create as many threads as there are CPU cores, - # while not limiting per-thread memory usage, which puts too much pressure - # on RAM on high-core machines, causing them to thrash. Using lower-level - # commands gives more control over those settings. + self.print('running "gsutil -m rsync -r -d %s %s"' % + (self.mirror_path, dest_prefix)) + gsutil.call('-m', 'rsync', '-r', '-d', self.mirror_path, dest_prefix) - # This might not be strictly necessary, but it's fast and is normally run - # by 'gc --aggressive', so it shouldn't hurt. - self.RunGit(['reflog', 'expire', '--all']) + # Create .ready file and upload + _, ready_file_name = tempfile.mkstemp(suffix='.ready') + try: + self.print('running "gsutil cp %s %s.ready"' % + (ready_file_name, dest_prefix)) + gsutil.call('cp', ready_file_name, '%s.ready' % (dest_prefix)) + finally: + os.remove(ready_file_name) - # These are the default repack settings for 'gc --aggressive'. - gc_args = ['repack', '-d', '-l', '-f', '--depth=50', '--window=250', '-A', - '--unpack-unreachable=all'] - # A 1G memory limit seems to provide comparable pack results as the - # default, even for our largest repos, while preventing runaway memory (at - # least on current Chromium builders which have about 4G RAM per core). - gc_args.append('--window-memory=1g') - # NOTE: It might also be possible to avoid thrashing with a larger window - # (e.g. "--window-memory=2g") by limiting the number of threads created - # (e.g. "--threads=[cores/2]"). Some limited testing didn't show much - # difference in outcomes on our current repos, but it might be worth - # trying if the repos grow much larger and the packs don't seem to be - # getting compressed enough. - self.RunGit(gc_args) + # remove all other directory/.ready files in the same gs_path + # except for the directory/.ready file previously created + # which can be used for bootstrapping while the current one is + # being uploaded + if not prune: + return + prev_dest_prefix = self._GetMostRecentCacheDirectory(ls_out_set) + if not prev_dest_prefix: + return + for path in ls_out_set: + if path in (prev_dest_prefix + '/', prev_dest_prefix + '.ready'): + continue + if path.endswith('.ready'): + gsutil.call('rm', path) + continue + gsutil.call('-m', 'rm', '-r', path) - self.print('running "gsutil -m rsync -r -d %s %s"' % - (self.mirror_path, dest_prefix)) - gsutil.call('-m', 'rsync', '-r', '-d', self.mirror_path, dest_prefix) - - # Create .ready file and upload - _, ready_file_name = tempfile.mkstemp(suffix='.ready') - try: - self.print('running "gsutil cp %s %s.ready"' % - (ready_file_name, dest_prefix)) - gsutil.call('cp', ready_file_name, '%s.ready' % (dest_prefix)) - finally: - os.remove(ready_file_name) - - # remove all other directory/.ready files in the same gs_path - # except for the directory/.ready file previously created - # which can be used for bootstrapping while the current one is - # being uploaded - if not prune: - return - prev_dest_prefix = self._GetMostRecentCacheDirectory(ls_out_set) - if not prev_dest_prefix: - return - for path in ls_out_set: - if path in (prev_dest_prefix + '/', prev_dest_prefix + '.ready'): - continue - if path.endswith('.ready'): - gsutil.call('rm', path) - continue - gsutil.call('-m', 'rm', '-r', path) - - - @staticmethod - def DeleteTmpPackFiles(path): - pack_dir = os.path.join(path, 'objects', 'pack') - if not os.path.isdir(pack_dir): - return - pack_files = [f for f in os.listdir(pack_dir) if - f.startswith('.tmp-') or f.startswith('tmp_pack_')] - for f in pack_files: - f = os.path.join(pack_dir, f) - try: - os.remove(f) - logging.warning('Deleted stale temporary pack file %s' % f) - except OSError: - logging.warning('Unable to delete temporary pack file %s' % f) + @staticmethod + def DeleteTmpPackFiles(path): + pack_dir = os.path.join(path, 'objects', 'pack') + if not os.path.isdir(pack_dir): + return + pack_files = [ + f for f in os.listdir(pack_dir) + if f.startswith('.tmp-') or f.startswith('tmp_pack_') + ] + for f in pack_files: + f = os.path.join(pack_dir, f) + try: + os.remove(f) + logging.warning('Deleted stale temporary pack file %s' % f) + except OSError: + logging.warning('Unable to delete temporary pack file %s' % f) @subcommand.usage('[url of repo to check for caching]') @metrics.collector.collect_metrics('git cache exists') def CMDexists(parser, args): - """Check to see if there already is a cache of the given repo.""" - _, args = parser.parse_args(args) - if not len(args) == 1: - parser.error('git cache exists only takes exactly one repo url.') - url = args[0] - mirror = Mirror(url) - if mirror.exists(): - print(mirror.mirror_path) - return 0 - return 1 + """Check to see if there already is a cache of the given repo.""" + _, args = parser.parse_args(args) + if not len(args) == 1: + parser.error('git cache exists only takes exactly one repo url.') + url = args[0] + mirror = Mirror(url) + if mirror.exists(): + print(mirror.mirror_path) + return 0 + return 1 @subcommand.usage('[url of repo to create a bootstrap zip file]') @metrics.collector.collect_metrics('git cache update-bootstrap') def CMDupdate_bootstrap(parser, args): - """Create and uploads a bootstrap tarball.""" - # Lets just assert we can't do this on Windows. - if sys.platform.startswith('win'): - print('Sorry, update bootstrap will not work on Windows.', file=sys.stderr) - return 1 + """Create and uploads a bootstrap tarball.""" + # Lets just assert we can't do this on Windows. + if sys.platform.startswith('win'): + print('Sorry, update bootstrap will not work on Windows.', + file=sys.stderr) + return 1 - parser.add_option('--skip-populate', action='store_true', - help='Skips "populate" step if mirror already exists.') - parser.add_option('--gc-aggressive', action='store_true', - help='Run aggressive repacking of the repo.') - parser.add_option('--prune', action='store_true', - help='Prune all other cached bundles of the same repo.') + parser.add_option('--skip-populate', + action='store_true', + help='Skips "populate" step if mirror already exists.') + parser.add_option('--gc-aggressive', + action='store_true', + help='Run aggressive repacking of the repo.') + parser.add_option('--prune', + action='store_true', + help='Prune all other cached bundles of the same repo.') - populate_args = args[:] - options, args = parser.parse_args(args) - url = args[0] - mirror = Mirror(url) - if not options.skip_populate or not mirror.exists(): - CMDpopulate(parser, populate_args) - else: - print('Skipped populate step.') + populate_args = args[:] + options, args = parser.parse_args(args) + url = args[0] + mirror = Mirror(url) + if not options.skip_populate or not mirror.exists(): + CMDpopulate(parser, populate_args) + else: + print('Skipped populate step.') - # Get the repo directory. - _, args2 = parser.parse_args(args) - url = args2[0] - mirror = Mirror(url) - mirror.update_bootstrap(options.prune, options.gc_aggressive) - return 0 + # Get the repo directory. + _, args2 = parser.parse_args(args) + url = args2[0] + mirror = Mirror(url) + mirror.update_bootstrap(options.prune, options.gc_aggressive) + return 0 @subcommand.usage('[url of repo to add to or update in cache]') @metrics.collector.collect_metrics('git cache populate') def CMDpopulate(parser, args): - """Ensure that the cache has all up-to-date objects for the given repo.""" - parser.add_option('--depth', type='int', - help='Only cache DEPTH commits of history') - parser.add_option( - '--no-fetch-tags', - action='store_true', - help=('Don\'t fetch tags from the server. This can speed up ' - 'fetch considerably when there are many tags.')) - parser.add_option('--shallow', '-s', action='store_true', - help='Only cache 10000 commits of history') - parser.add_option('--ref', action='append', - help='Specify additional refs to be fetched') - parser.add_option('--commit', action='append', - help='Specify additional commits to be fetched') - parser.add_option('--no_bootstrap', '--no-bootstrap', - action='store_true', - help='Don\'t bootstrap from Google Storage') - parser.add_option('--ignore_locks', - '--ignore-locks', - action='store_true', - help='NOOP. This flag will be removed in the future.') - parser.add_option('--break-locks', - action='store_true', - help='Break any existing lock instead of just ignoring it') - parser.add_option('--reset-fetch-config', action='store_true', default=False, - help='Reset the fetch config before populating the cache.') + """Ensure that the cache has all up-to-date objects for the given repo.""" + parser.add_option('--depth', + type='int', + help='Only cache DEPTH commits of history') + parser.add_option( + '--no-fetch-tags', + action='store_true', + help=('Don\'t fetch tags from the server. This can speed up ' + 'fetch considerably when there are many tags.')) + parser.add_option('--shallow', + '-s', + action='store_true', + help='Only cache 10000 commits of history') + parser.add_option('--ref', + action='append', + help='Specify additional refs to be fetched') + parser.add_option('--commit', + action='append', + help='Specify additional commits to be fetched') + parser.add_option('--no_bootstrap', + '--no-bootstrap', + action='store_true', + help='Don\'t bootstrap from Google Storage') + parser.add_option('--ignore_locks', + '--ignore-locks', + action='store_true', + help='NOOP. This flag will be removed in the future.') + parser.add_option( + '--break-locks', + action='store_true', + help='Break any existing lock instead of just ignoring it') + parser.add_option( + '--reset-fetch-config', + action='store_true', + default=False, + help='Reset the fetch config before populating the cache.') - options, args = parser.parse_args(args) - if not len(args) == 1: - parser.error('git cache populate only takes exactly one repo url.') - if options.ignore_locks: - print('ignore_locks is no longer used. Please remove its usage.') - if options.break_locks: - print('break_locks is no longer used. Please remove its usage.') - url = args[0] + options, args = parser.parse_args(args) + if not len(args) == 1: + parser.error('git cache populate only takes exactly one repo url.') + if options.ignore_locks: + print('ignore_locks is no longer used. Please remove its usage.') + if options.break_locks: + print('break_locks is no longer used. Please remove its usage.') + url = args[0] - mirror = Mirror(url, refs=options.ref, commits=options.commit) - kwargs = { - 'no_fetch_tags': options.no_fetch_tags, - 'verbose': options.verbose, - 'shallow': options.shallow, - 'bootstrap': not options.no_bootstrap, - 'lock_timeout': options.timeout, - 'reset_fetch_config': options.reset_fetch_config, - } - if options.depth: - kwargs['depth'] = options.depth - mirror.populate(**kwargs) + mirror = Mirror(url, refs=options.ref, commits=options.commit) + kwargs = { + 'no_fetch_tags': options.no_fetch_tags, + 'verbose': options.verbose, + 'shallow': options.shallow, + 'bootstrap': not options.no_bootstrap, + 'lock_timeout': options.timeout, + 'reset_fetch_config': options.reset_fetch_config, + } + if options.depth: + kwargs['depth'] = options.depth + mirror.populate(**kwargs) @subcommand.usage('Fetch new commits into cache and current checkout') @metrics.collector.collect_metrics('git cache fetch') def CMDfetch(parser, args): - """Update mirror, and fetch in cwd.""" - parser.add_option('--all', action='store_true', help='Fetch all remotes') - parser.add_option('--no_bootstrap', '--no-bootstrap', - action='store_true', - help='Don\'t (re)bootstrap from Google Storage') - parser.add_option( - '--no-fetch-tags', - action='store_true', - help=('Don\'t fetch tags from the server. This can speed up ' - 'fetch considerably when there are many tags.')) - options, args = parser.parse_args(args) + """Update mirror, and fetch in cwd.""" + parser.add_option('--all', action='store_true', help='Fetch all remotes') + parser.add_option('--no_bootstrap', + '--no-bootstrap', + action='store_true', + help='Don\'t (re)bootstrap from Google Storage') + parser.add_option( + '--no-fetch-tags', + action='store_true', + help=('Don\'t fetch tags from the server. This can speed up ' + 'fetch considerably when there are many tags.')) + options, args = parser.parse_args(args) - # Figure out which remotes to fetch. This mimics the behavior of regular - # 'git fetch'. Note that in the case of "stacked" or "pipelined" branches, - # this will NOT try to traverse up the branching structure to find the - # ultimate remote to update. - remotes = [] - if options.all: - assert not args, 'fatal: fetch --all does not take a repository argument' - remotes = subprocess.check_output([Mirror.git_exe, 'remote']) - remotes = remotes.decode('utf-8', 'ignore').splitlines() - elif args: - remotes = args - else: - current_branch = subprocess.check_output( - [Mirror.git_exe, 'rev-parse', '--abbrev-ref', 'HEAD']) - current_branch = current_branch.decode('utf-8', 'ignore').strip() - if current_branch != 'HEAD': - upstream = subprocess.check_output( - [Mirror.git_exe, 'config', 'branch.%s.remote' % current_branch]) - upstream = upstream.decode('utf-8', 'ignore').strip() - if upstream and upstream != '.': - remotes = [upstream] - if not remotes: - remotes = ['origin'] + # Figure out which remotes to fetch. This mimics the behavior of regular + # 'git fetch'. Note that in the case of "stacked" or "pipelined" branches, + # this will NOT try to traverse up the branching structure to find the + # ultimate remote to update. + remotes = [] + if options.all: + assert not args, 'fatal: fetch --all does not take repository argument' + remotes = subprocess.check_output([Mirror.git_exe, 'remote']) + remotes = remotes.decode('utf-8', 'ignore').splitlines() + elif args: + remotes = args + else: + current_branch = subprocess.check_output( + [Mirror.git_exe, 'rev-parse', '--abbrev-ref', 'HEAD']) + current_branch = current_branch.decode('utf-8', 'ignore').strip() + if current_branch != 'HEAD': + upstream = subprocess.check_output( + [Mirror.git_exe, 'config', + 'branch.%s.remote' % current_branch]) + upstream = upstream.decode('utf-8', 'ignore').strip() + if upstream and upstream != '.': + remotes = [upstream] + if not remotes: + remotes = ['origin'] - cachepath = Mirror.GetCachePath() - git_dir = os.path.abspath(subprocess.check_output( - [Mirror.git_exe, 'rev-parse', '--git-dir']).decode('utf-8', 'ignore')) - git_dir = os.path.abspath(git_dir) - if git_dir.startswith(cachepath): - mirror = Mirror.FromPath(git_dir) - mirror.populate( - bootstrap=not options.no_bootstrap, - no_fetch_tags=options.no_fetch_tags, - lock_timeout=options.timeout) + cachepath = Mirror.GetCachePath() + git_dir = os.path.abspath( + subprocess.check_output([Mirror.git_exe, 'rev-parse', + '--git-dir']).decode('utf-8', 'ignore')) + git_dir = os.path.abspath(git_dir) + if git_dir.startswith(cachepath): + mirror = Mirror.FromPath(git_dir) + mirror.populate(bootstrap=not options.no_bootstrap, + no_fetch_tags=options.no_fetch_tags, + lock_timeout=options.timeout) + return 0 + for remote in remotes: + remote_url = subprocess.check_output( + [Mirror.git_exe, 'config', + 'remote.%s.url' % remote]) + remote_url = remote_url.decode('utf-8', 'ignore').strip() + if remote_url.startswith(cachepath): + mirror = Mirror.FromPath(remote_url) + mirror.print = lambda *args: None + print('Updating git cache...') + mirror.populate(bootstrap=not options.no_bootstrap, + no_fetch_tags=options.no_fetch_tags, + lock_timeout=options.timeout) + subprocess.check_call([Mirror.git_exe, 'fetch', remote]) return 0 - for remote in remotes: - remote_url = subprocess.check_output( - [Mirror.git_exe, 'config', 'remote.%s.url' % remote]) - remote_url = remote_url.decode('utf-8', 'ignore').strip() - if remote_url.startswith(cachepath): - mirror = Mirror.FromPath(remote_url) - mirror.print = lambda *args: None - print('Updating git cache...') - mirror.populate( - bootstrap=not options.no_bootstrap, - no_fetch_tags=options.no_fetch_tags, - lock_timeout=options.timeout) - subprocess.check_call([Mirror.git_exe, 'fetch', remote]) - return 0 @subcommand.usage('do not use - it is a noop.') @metrics.collector.collect_metrics('git cache unlock') def CMDunlock(parser, args): - """This command does nothing.""" - print('This command does nothing and will be removed in the future.') + """This command does nothing.""" + print('This command does nothing and will be removed in the future.') class OptionParser(optparse.OptionParser): - """Wrapper class for OptionParser to handle global options.""" + """Wrapper class for OptionParser to handle global options.""" + def __init__(self, *args, **kwargs): + optparse.OptionParser.__init__(self, *args, prog='git cache', **kwargs) + self.add_option( + '-c', + '--cache-dir', + help=('Path to the directory containing the caches. Normally ' + 'deduced from git config cache.cachepath or ' + '$GIT_CACHE_PATH.')) + self.add_option( + '-v', + '--verbose', + action='count', + default=1, + help='Increase verbosity (can be passed multiple times)') + self.add_option('-q', + '--quiet', + action='store_true', + help='Suppress all extraneous output') + self.add_option('--timeout', + type='int', + default=0, + help='Timeout for acquiring cache lock, in seconds') - def __init__(self, *args, **kwargs): - optparse.OptionParser.__init__(self, *args, prog='git cache', **kwargs) - self.add_option('-c', '--cache-dir', - help=( - 'Path to the directory containing the caches. Normally ' - 'deduced from git config cache.cachepath or ' - '$GIT_CACHE_PATH.')) - self.add_option('-v', '--verbose', action='count', default=1, - help='Increase verbosity (can be passed multiple times)') - self.add_option('-q', '--quiet', action='store_true', - help='Suppress all extraneous output') - self.add_option('--timeout', type='int', default=0, - help='Timeout for acquiring cache lock, in seconds') + def parse_args(self, args=None, values=None): + # Create an optparse.Values object that will store only the actual + # passed options, without the defaults. + actual_options = optparse.Values() + _, args = optparse.OptionParser.parse_args(self, args, actual_options) + # Create an optparse.Values object with the default options. + options = optparse.Values(self.get_default_values().__dict__) + # Update it with the options passed by the user. + options._update_careful(actual_options.__dict__) + # Store the options passed by the user in an _actual_options attribute. + # We store only the keys, and not the values, since the values can + # contain arbitrary information, which might be PII. + metrics.collector.add('arguments', list(actual_options.__dict__.keys())) - def parse_args(self, args=None, values=None): - # Create an optparse.Values object that will store only the actual passed - # options, without the defaults. - actual_options = optparse.Values() - _, args = optparse.OptionParser.parse_args(self, args, actual_options) - # Create an optparse.Values object with the default options. - options = optparse.Values(self.get_default_values().__dict__) - # Update it with the options passed by the user. - options._update_careful(actual_options.__dict__) - # Store the options passed by the user in an _actual_options attribute. - # We store only the keys, and not the values, since the values can contain - # arbitrary information, which might be PII. - metrics.collector.add('arguments', list(actual_options.__dict__.keys())) + if options.quiet: + options.verbose = 0 - if options.quiet: - options.verbose = 0 + levels = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG] + logging.basicConfig(level=levels[min(options.verbose, len(levels) - 1)]) - levels = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG] - logging.basicConfig(level=levels[min(options.verbose, len(levels) - 1)]) + try: + global_cache_dir = Mirror.GetCachePath() + except RuntimeError: + global_cache_dir = None + if options.cache_dir: + if global_cache_dir and (os.path.abspath(options.cache_dir) != + os.path.abspath(global_cache_dir)): + logging.warning( + 'Overriding globally-configured cache directory.') + Mirror.SetCachePath(options.cache_dir) - try: - global_cache_dir = Mirror.GetCachePath() - except RuntimeError: - global_cache_dir = None - if options.cache_dir: - if global_cache_dir and ( - os.path.abspath(options.cache_dir) != - os.path.abspath(global_cache_dir)): - logging.warning('Overriding globally-configured cache directory.') - Mirror.SetCachePath(options.cache_dir) - - return options, args + return options, args def main(argv): - dispatcher = subcommand.CommandDispatcher(__name__) - return dispatcher.execute(OptionParser(), argv) + dispatcher = subcommand.CommandDispatcher(__name__) + return dispatcher.execute(OptionParser(), argv) if __name__ == '__main__': - try: - with metrics.collector.print_notice_and_exit(): - sys.exit(main(sys.argv[1:])) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(1) + try: + with metrics.collector.print_notice_and_exit(): + sys.exit(main(sys.argv[1:])) + except KeyboardInterrupt: + sys.stderr.write('interrupted\n') + sys.exit(1) diff --git a/git_cl.py b/git_cl.py index 98108daede..5ba2f4cb57 100755 --- a/git_cl.py +++ b/git_cl.py @@ -4,7 +4,6 @@ # found in the LICENSE file. # Copyright (C) 2008 Evan Martin - """A git-command for integrating reviews on Gerrit.""" import base64 @@ -62,9 +61,11 @@ import watchlists from six.moves import urllib - __version__ = '2.0' +# TODO: Should fix these warnings. +# pylint: disable=line-too-long + # Traces for git push will be stored in a traces directory inside the # depot_tools checkout. DEPOT_TOOLS = os.path.dirname(os.path.abspath(__file__)) @@ -85,23 +86,22 @@ MAX_TRACES = 3 * 10 # Message to be displayed to the user to inform where to find the traces for a # git-cl upload execution. TRACES_MESSAGE = ( -'\n' -'The traces of this git-cl execution have been recorded at:\n' -' %(trace_name)s-traces.zip\n' -'Copies of your gitcookies file and git config have been recorded at:\n' -' %(trace_name)s-git-info.zip\n') + '\n' + 'The traces of this git-cl execution have been recorded at:\n' + ' %(trace_name)s-traces.zip\n' + 'Copies of your gitcookies file and git config have been recorded at:\n' + ' %(trace_name)s-git-info.zip\n') # Format of the message to be stored as part of the traces to give developers a # better context when they go through traces. -TRACES_README_FORMAT = ( -'Date: %(now)s\n' -'\n' -'Change: https://%(gerrit_host)s/q/%(change_id)s\n' -'Title: %(title)s\n' -'\n' -'%(description)s\n' -'\n' -'Execution time: %(execution_time)s\n' -'Exit code: %(exit_code)s\n') + TRACES_MESSAGE +TRACES_README_FORMAT = ('Date: %(now)s\n' + '\n' + 'Change: https://%(gerrit_host)s/q/%(change_id)s\n' + 'Title: %(title)s\n' + '\n' + '%(description)s\n' + '\n' + 'Execution time: %(execution_time)s\n' + 'Exit code: %(exit_code)s\n') + TRACES_MESSAGE POSTUPSTREAM_HOOK = '.git/hooks/post-cl-land' DESCRIPTION_BACKUP_FILE = '.git_cl_description_backup' @@ -156,330 +156,336 @@ _KNOWN_GERRIT_TO_SHORT_URLS = { assert len(_KNOWN_GERRIT_TO_SHORT_URLS) == len( set(_KNOWN_GERRIT_TO_SHORT_URLS.values())), 'must have unique values' - # Maximum number of branches in a stack that can be traversed and uploaded # at once. Picked arbitrarily. _MAX_STACKED_BRANCHES_UPLOAD = 20 - # Environment variable to indicate if user is participating in the stcked # changes dogfood. DOGFOOD_STACKED_CHANGES_VAR = 'DOGFOOD_STACKED_CHANGES' class GitPushError(Exception): - pass + pass def DieWithError(message, change_desc=None) -> NoReturn: - if change_desc: - SaveDescriptionBackup(change_desc) - print('\n ** Content of CL description **\n' + - '='*72 + '\n' + - change_desc.description + '\n' + - '='*72 + '\n') + if change_desc: + SaveDescriptionBackup(change_desc) + print('\n ** Content of CL description **\n' + '=' * 72 + '\n' + + change_desc.description + '\n' + '=' * 72 + '\n') - print(message, file=sys.stderr) - sys.exit(1) + print(message, file=sys.stderr) + sys.exit(1) def SaveDescriptionBackup(change_desc): - backup_path = os.path.join(DEPOT_TOOLS, DESCRIPTION_BACKUP_FILE) - print('\nsaving CL description to %s\n' % backup_path) - with open(backup_path, 'wb') as backup_file: - backup_file.write(change_desc.description.encode('utf-8')) + backup_path = os.path.join(DEPOT_TOOLS, DESCRIPTION_BACKUP_FILE) + print('\nsaving CL description to %s\n' % backup_path) + with open(backup_path, 'wb') as backup_file: + backup_file.write(change_desc.description.encode('utf-8')) def GetNoGitPagerEnv(): - env = os.environ.copy() - # 'cat' is a magical git string that disables pagers on all platforms. - env['GIT_PAGER'] = 'cat' - return env + env = os.environ.copy() + # 'cat' is a magical git string that disables pagers on all platforms. + env['GIT_PAGER'] = 'cat' + return env def RunCommand(args, error_ok=False, error_message=None, shell=False, **kwargs): - try: - stdout = subprocess2.check_output(args, shell=shell, **kwargs) - return stdout.decode('utf-8', 'replace') - except subprocess2.CalledProcessError as e: - logging.debug('Failed running %s', args) - if not error_ok: - message = error_message or e.stdout.decode('utf-8', 'replace') or '' - DieWithError('Command "%s" failed.\n%s' % (' '.join(args), message)) - out = e.stdout.decode('utf-8', 'replace') - if e.stderr: - out += e.stderr.decode('utf-8', 'replace') - return out + try: + stdout = subprocess2.check_output(args, shell=shell, **kwargs) + return stdout.decode('utf-8', 'replace') + except subprocess2.CalledProcessError as e: + logging.debug('Failed running %s', args) + if not error_ok: + message = error_message or e.stdout.decode('utf-8', 'replace') or '' + DieWithError('Command "%s" failed.\n%s' % (' '.join(args), message)) + out = e.stdout.decode('utf-8', 'replace') + if e.stderr: + out += e.stderr.decode('utf-8', 'replace') + return out def RunGit(args, **kwargs): - """Returns stdout.""" - return RunCommand(['git'] + args, **kwargs) + """Returns stdout.""" + return RunCommand(['git'] + args, **kwargs) def RunGitWithCode(args, suppress_stderr=False): - """Returns return code and stdout.""" - if suppress_stderr: - stderr = subprocess2.DEVNULL - else: - stderr = sys.stderr - try: - (out, _), code = subprocess2.communicate(['git'] + args, - env=GetNoGitPagerEnv(), - stdout=subprocess2.PIPE, - stderr=stderr) - return code, out.decode('utf-8', 'replace') - except subprocess2.CalledProcessError as e: - logging.debug('Failed running %s', ['git'] + args) - return e.returncode, e.stdout.decode('utf-8', 'replace') + """Returns return code and stdout.""" + if suppress_stderr: + stderr = subprocess2.DEVNULL + else: + stderr = sys.stderr + try: + (out, _), code = subprocess2.communicate(['git'] + args, + env=GetNoGitPagerEnv(), + stdout=subprocess2.PIPE, + stderr=stderr) + return code, out.decode('utf-8', 'replace') + except subprocess2.CalledProcessError as e: + logging.debug('Failed running %s', ['git'] + args) + return e.returncode, e.stdout.decode('utf-8', 'replace') def RunGitSilent(args): - """Returns stdout, suppresses stderr and ignores the return code.""" - return RunGitWithCode(args, suppress_stderr=True)[1] + """Returns stdout, suppresses stderr and ignores the return code.""" + return RunGitWithCode(args, suppress_stderr=True)[1] def time_sleep(seconds): - # Use this so that it can be mocked in tests without interfering with python - # system machinery. - return time.sleep(seconds) + # Use this so that it can be mocked in tests without interfering with python + # system machinery. + return time.sleep(seconds) def time_time(): - # Use this so that it can be mocked in tests without interfering with python - # system machinery. - return time.time() + # Use this so that it can be mocked in tests without interfering with python + # system machinery. + return time.time() def datetime_now(): - # Use this so that it can be mocked in tests without interfering with python - # system machinery. - return datetime.datetime.now() + # Use this so that it can be mocked in tests without interfering with python + # system machinery. + return datetime.datetime.now() def confirm_or_exit(prefix='', action='confirm'): - """Asks user to press enter to continue or press Ctrl+C to abort.""" - if not prefix or prefix.endswith('\n'): - mid = 'Press' - elif prefix.endswith('.') or prefix.endswith('?'): - mid = ' Press' - elif prefix.endswith(' '): - mid = 'press' - else: - mid = ' press' - gclient_utils.AskForData( - '%s%s Enter to %s, or Ctrl+C to abort' % (prefix, mid, action)) + """Asks user to press enter to continue or press Ctrl+C to abort.""" + if not prefix or prefix.endswith('\n'): + mid = 'Press' + elif prefix.endswith('.') or prefix.endswith('?'): + mid = ' Press' + elif prefix.endswith(' '): + mid = 'press' + else: + mid = ' press' + gclient_utils.AskForData('%s%s Enter to %s, or Ctrl+C to abort' % + (prefix, mid, action)) def ask_for_explicit_yes(prompt): - """Returns whether user typed 'y' or 'yes' to confirm the given prompt.""" - result = gclient_utils.AskForData(prompt + ' [Yes/No]: ').lower() - while True: - if 'yes'.startswith(result): - return True - if 'no'.startswith(result): - return False - result = gclient_utils.AskForData('Please, type yes or no: ').lower() + """Returns whether user typed 'y' or 'yes' to confirm the given prompt.""" + result = gclient_utils.AskForData(prompt + ' [Yes/No]: ').lower() + while True: + if 'yes'.startswith(result): + return True + if 'no'.startswith(result): + return False + result = gclient_utils.AskForData('Please, type yes or no: ').lower() def _get_properties_from_options(options): - prop_list = getattr(options, 'properties', []) - properties = dict(x.split('=', 1) for x in prop_list) - for key, val in properties.items(): - try: - properties[key] = json.loads(val) - except ValueError: - pass # If a value couldn't be evaluated, treat it as a string. - return properties + prop_list = getattr(options, 'properties', []) + properties = dict(x.split('=', 1) for x in prop_list) + for key, val in properties.items(): + try: + properties[key] = json.loads(val) + except ValueError: + pass # If a value couldn't be evaluated, treat it as a string. + return properties def _call_buildbucket(http, buildbucket_host, method, request): - """Calls a buildbucket v2 method and returns the parsed json response.""" - headers = { - 'Accept': 'application/json', - 'Content-Type': 'application/json', - } - request = json.dumps(request) - url = 'https://%s/prpc/buildbucket.v2.Builds/%s' % (buildbucket_host, method) + """Calls a buildbucket v2 method and returns the parsed json response.""" + headers = { + 'Accept': 'application/json', + 'Content-Type': 'application/json', + } + request = json.dumps(request) + url = 'https://%s/prpc/buildbucket.v2.Builds/%s' % (buildbucket_host, + method) - logging.info('POST %s with %s' % (url, request)) + logging.info('POST %s with %s' % (url, request)) - attempts = 1 - time_to_sleep = 1 - while True: - response, content = http.request(url, 'POST', body=request, headers=headers) - if response.status == 200: - return json.loads(content[4:]) - if attempts >= MAX_ATTEMPTS or 400 <= response.status < 500: - msg = '%s error when calling POST %s with %s: %s' % ( - response.status, url, request, content) - raise BuildbucketResponseException(msg) - logging.debug( - '%s error when calling POST %s with %s. ' - 'Sleeping for %d seconds and retrying...' % ( - response.status, url, request, time_to_sleep)) - time.sleep(time_to_sleep) - time_to_sleep *= 2 - attempts += 1 + attempts = 1 + time_to_sleep = 1 + while True: + response, content = http.request(url, + 'POST', + body=request, + headers=headers) + if response.status == 200: + return json.loads(content[4:]) + if attempts >= MAX_ATTEMPTS or 400 <= response.status < 500: + msg = '%s error when calling POST %s with %s: %s' % ( + response.status, url, request, content) + raise BuildbucketResponseException(msg) + logging.debug('%s error when calling POST %s with %s. ' + 'Sleeping for %d seconds and retrying...' % + (response.status, url, request, time_to_sleep)) + time.sleep(time_to_sleep) + time_to_sleep *= 2 + attempts += 1 - assert False, 'unreachable' + assert False, 'unreachable' def _parse_bucket(raw_bucket): - legacy = True - project = bucket = None - if '/' in raw_bucket: - legacy = False - project, bucket = raw_bucket.split('/', 1) - # Assume luci... - elif raw_bucket.startswith('luci.'): - project, bucket = raw_bucket[len('luci.'):].split('.', 1) - # Otherwise, assume prefix is also the project name. - elif '.' in raw_bucket: - project = raw_bucket.split('.')[0] - bucket = raw_bucket - # Legacy buckets. - if legacy and project and bucket: - print('WARNING Please use %s/%s to specify the bucket.' % (project, bucket)) - return project, bucket + legacy = True + project = bucket = None + if '/' in raw_bucket: + legacy = False + project, bucket = raw_bucket.split('/', 1) + # Assume luci... + elif raw_bucket.startswith('luci.'): + project, bucket = raw_bucket[len('luci.'):].split('.', 1) + # Otherwise, assume prefix is also the project name. + elif '.' in raw_bucket: + project = raw_bucket.split('.')[0] + bucket = raw_bucket + # Legacy buckets. + if legacy and project and bucket: + print('WARNING Please use %s/%s to specify the bucket.' % + (project, bucket)) + return project, bucket def _canonical_git_googlesource_host(host): - """Normalizes Gerrit hosts (with '-review') to Git host.""" - assert host.endswith(_GOOGLESOURCE) - # Prefix doesn't include '.' at the end. - prefix = host[:-(1 + len(_GOOGLESOURCE))] - if prefix.endswith('-review'): - prefix = prefix[:-len('-review')] - return prefix + '.' + _GOOGLESOURCE + """Normalizes Gerrit hosts (with '-review') to Git host.""" + assert host.endswith(_GOOGLESOURCE) + # Prefix doesn't include '.' at the end. + prefix = host[:-(1 + len(_GOOGLESOURCE))] + if prefix.endswith('-review'): + prefix = prefix[:-len('-review')] + return prefix + '.' + _GOOGLESOURCE def _canonical_gerrit_googlesource_host(host): - git_host = _canonical_git_googlesource_host(host) - prefix = git_host.split('.', 1)[0] - return prefix + '-review.' + _GOOGLESOURCE + git_host = _canonical_git_googlesource_host(host) + prefix = git_host.split('.', 1)[0] + return prefix + '-review.' + _GOOGLESOURCE def _get_counterpart_host(host): - assert host.endswith(_GOOGLESOURCE) - git = _canonical_git_googlesource_host(host) - gerrit = _canonical_gerrit_googlesource_host(git) - return git if gerrit == host else gerrit + assert host.endswith(_GOOGLESOURCE) + git = _canonical_git_googlesource_host(host) + gerrit = _canonical_gerrit_googlesource_host(git) + return git if gerrit == host else gerrit def _trigger_tryjobs(changelist, jobs, options, patchset): - """Sends a request to Buildbucket to trigger tryjobs for a changelist. + """Sends a request to Buildbucket to trigger tryjobs for a changelist. Args: changelist: Changelist that the tryjobs are associated with. jobs: A list of (project, bucket, builder). options: Command-line options. """ - print('Scheduling jobs on:') - for project, bucket, builder in jobs: - print(' %s/%s: %s' % (project, bucket, builder)) - print('To see results here, run: git cl try-results') - print('To see results in browser, run: git cl web') + print('Scheduling jobs on:') + for project, bucket, builder in jobs: + print(' %s/%s: %s' % (project, bucket, builder)) + print('To see results here, run: git cl try-results') + print('To see results in browser, run: git cl web') - requests = _make_tryjob_schedule_requests(changelist, jobs, options, patchset) - if not requests: - return + requests = _make_tryjob_schedule_requests(changelist, jobs, options, + patchset) + if not requests: + return - http = auth.Authenticator().authorize(httplib2.Http()) - http.force_exception_to_status_code = True + http = auth.Authenticator().authorize(httplib2.Http()) + http.force_exception_to_status_code = True - batch_request = {'requests': requests} - batch_response = _call_buildbucket(http, DEFAULT_BUILDBUCKET_HOST, 'Batch', - batch_request) + batch_request = {'requests': requests} + batch_response = _call_buildbucket(http, DEFAULT_BUILDBUCKET_HOST, 'Batch', + batch_request) - errors = [ - ' ' + response['error']['message'] - for response in batch_response.get('responses', []) - if 'error' in response - ] - if errors: - raise BuildbucketResponseException( - 'Failed to schedule builds for some bots:\n%s' % '\n'.join(errors)) + errors = [ + ' ' + response['error']['message'] + for response in batch_response.get('responses', []) + if 'error' in response + ] + if errors: + raise BuildbucketResponseException( + 'Failed to schedule builds for some bots:\n%s' % '\n'.join(errors)) def _make_tryjob_schedule_requests(changelist, jobs, options, patchset): - """Constructs requests for Buildbucket to trigger tryjobs.""" - gerrit_changes = [changelist.GetGerritChange(patchset)] - shared_properties = { - 'category': options.ensure_value('category', 'git_cl_try') - } - if options.ensure_value('clobber', False): - shared_properties['clobber'] = True - shared_properties.update(_get_properties_from_options(options) or {}) + """Constructs requests for Buildbucket to trigger tryjobs.""" + gerrit_changes = [changelist.GetGerritChange(patchset)] + shared_properties = { + 'category': options.ensure_value('category', 'git_cl_try') + } + if options.ensure_value('clobber', False): + shared_properties['clobber'] = True + shared_properties.update(_get_properties_from_options(options) or {}) - shared_tags = [{'key': 'user_agent', 'value': 'git_cl_try'}] - if options.ensure_value('retry_failed', False): - shared_tags.append({'key': 'retry_failed', - 'value': '1'}) + shared_tags = [{'key': 'user_agent', 'value': 'git_cl_try'}] + if options.ensure_value('retry_failed', False): + shared_tags.append({'key': 'retry_failed', 'value': '1'}) - requests = [] - for (project, bucket, builder) in jobs: - properties = shared_properties.copy() - if 'presubmit' in builder.lower(): - properties['dry_run'] = 'true' + requests = [] + for (project, bucket, builder) in jobs: + properties = shared_properties.copy() + if 'presubmit' in builder.lower(): + properties['dry_run'] = 'true' - requests.append({ - 'scheduleBuild': { - 'requestId': str(uuid.uuid4()), - 'builder': { - 'project': getattr(options, 'project', None) or project, - 'bucket': bucket, - 'builder': builder, - }, - 'gerritChanges': gerrit_changes, - 'properties': properties, - 'tags': [ - {'key': 'builder', 'value': builder}, - ] + shared_tags, - } - }) + requests.append({ + 'scheduleBuild': { + 'requestId': str(uuid.uuid4()), + 'builder': { + 'project': getattr(options, 'project', None) or project, + 'bucket': bucket, + 'builder': builder, + }, + 'gerritChanges': gerrit_changes, + 'properties': properties, + 'tags': [ + { + 'key': 'builder', + 'value': builder + }, + ] + shared_tags, + } + }) - if options.ensure_value('revision', None): - remote, remote_branch = changelist.GetRemoteBranch() - requests[-1]['scheduleBuild']['gitilesCommit'] = { - 'host': _canonical_git_googlesource_host(gerrit_changes[0]['host']), - 'project': gerrit_changes[0]['project'], - 'id': options.revision, - 'ref': GetTargetRef(remote, remote_branch, None) - } + if options.ensure_value('revision', None): + remote, remote_branch = changelist.GetRemoteBranch() + requests[-1]['scheduleBuild']['gitilesCommit'] = { + 'host': + _canonical_git_googlesource_host(gerrit_changes[0]['host']), + 'project': gerrit_changes[0]['project'], + 'id': options.revision, + 'ref': GetTargetRef(remote, remote_branch, None) + } - return requests + return requests def _fetch_tryjobs(changelist, buildbucket_host, patchset=None): - """Fetches tryjobs from buildbucket. + """Fetches tryjobs from buildbucket. Returns list of buildbucket.v2.Build with the try jobs for the changelist. """ - fields = ['id', 'builder', 'status', 'createTime', 'tags'] - request = { - 'predicate': { - 'gerritChanges': [changelist.GetGerritChange(patchset)], - }, - 'fields': ','.join('builds.*.' + field for field in fields), - } + fields = ['id', 'builder', 'status', 'createTime', 'tags'] + request = { + 'predicate': { + 'gerritChanges': [changelist.GetGerritChange(patchset)], + }, + 'fields': ','.join('builds.*.' + field for field in fields), + } - authenticator = auth.Authenticator() - if authenticator.has_cached_credentials(): - http = authenticator.authorize(httplib2.Http()) - else: - print('Warning: Some results might be missing because %s' % - # Get the message on how to login. - (str(auth.LoginRequiredError()),)) - http = httplib2.Http() - http.force_exception_to_status_code = True + authenticator = auth.Authenticator() + if authenticator.has_cached_credentials(): + http = authenticator.authorize(httplib2.Http()) + else: + print('Warning: Some results might be missing because %s' % + # Get the message on how to login. + ( + str(auth.LoginRequiredError()), )) + http = httplib2.Http() + http.force_exception_to_status_code = True - response = _call_buildbucket(http, buildbucket_host, 'SearchBuilds', request) - return response.get('builds', []) + response = _call_buildbucket(http, buildbucket_host, 'SearchBuilds', + request) + return response.get('builds', []) def _fetch_latest_builds(changelist, buildbucket_host, latest_patchset=None): - """Fetches builds from the latest patchset that has builds (within + """Fetches builds from the latest patchset that has builds (within the last few patchsets). Args: @@ -491,27 +497,27 @@ def _fetch_latest_builds(changelist, buildbucket_host, latest_patchset=None): A tuple (builds, patchset) where builds is a list of buildbucket.v2.Build, and patchset is the patchset number where those builds came from. """ - assert buildbucket_host - assert changelist.GetIssue(), 'CL must be uploaded first' - assert changelist.GetCodereviewServer(), 'CL must be uploaded first' - if latest_patchset is None: - assert changelist.GetMostRecentPatchset() - ps = changelist.GetMostRecentPatchset() - else: - assert latest_patchset > 0, latest_patchset - ps = latest_patchset + assert buildbucket_host + assert changelist.GetIssue(), 'CL must be uploaded first' + assert changelist.GetCodereviewServer(), 'CL must be uploaded first' + if latest_patchset is None: + assert changelist.GetMostRecentPatchset() + ps = changelist.GetMostRecentPatchset() + else: + assert latest_patchset > 0, latest_patchset + ps = latest_patchset - min_ps = max(1, ps - 5) - while ps >= min_ps: - builds = _fetch_tryjobs(changelist, buildbucket_host, patchset=ps) - if len(builds): - return builds, ps - ps -= 1 - return [], 0 + min_ps = max(1, ps - 5) + while ps >= min_ps: + builds = _fetch_tryjobs(changelist, buildbucket_host, patchset=ps) + if len(builds): + return builds, ps + ps -= 1 + return [], 0 def _filter_failed_for_retry(all_builds): - """Returns a list of buckets/builders that are worth retrying. + """Returns a list of buckets/builders that are worth retrying. Args: all_builds (list): Builds, in the format returned by _fetch_tryjobs, @@ -522,179 +528,191 @@ def _filter_failed_for_retry(all_builds): A dict {(proj, bucket): [builders]}. This is the same format accepted by _trigger_tryjobs. """ - grouped = {} - for build in all_builds: - builder = build['builder'] - key = (builder['project'], builder['bucket'], builder['builder']) - grouped.setdefault(key, []).append(build) + grouped = {} + for build in all_builds: + builder = build['builder'] + key = (builder['project'], builder['bucket'], builder['builder']) + grouped.setdefault(key, []).append(build) - jobs = [] - for (project, bucket, builder), builds in grouped.items(): - if 'triggered' in builder: - print('WARNING: Not scheduling %s. Triggered bots require an initial job ' - 'from a parent. Please schedule a manual job for the parent ' - 'instead.') - continue - if any(b['status'] in ('STARTED', 'SCHEDULED') for b in builds): - # Don't retry if any are running. - continue - # If builder had several builds, retry only if the last one failed. - # This is a bit different from CQ, which would re-use *any* SUCCESS-full - # build, but in case of retrying failed jobs retrying a flaky one makes - # sense. - builds = sorted(builds, key=lambda b: b['createTime']) - if builds[-1]['status'] not in ('FAILURE', 'INFRA_FAILURE'): - continue - # Don't retry experimental build previously triggered by CQ. - if any(t['key'] == 'cq_experimental' and t['value'] == 'true' - for t in builds[-1]['tags']): - continue - jobs.append((project, bucket, builder)) + jobs = [] + for (project, bucket, builder), builds in grouped.items(): + if 'triggered' in builder: + print( + 'WARNING: Not scheduling %s. Triggered bots require an initial job ' + 'from a parent. Please schedule a manual job for the parent ' + 'instead.') + continue + if any(b['status'] in ('STARTED', 'SCHEDULED') for b in builds): + # Don't retry if any are running. + continue + # If builder had several builds, retry only if the last one failed. + # This is a bit different from CQ, which would re-use *any* SUCCESS-full + # build, but in case of retrying failed jobs retrying a flaky one makes + # sense. + builds = sorted(builds, key=lambda b: b['createTime']) + if builds[-1]['status'] not in ('FAILURE', 'INFRA_FAILURE'): + continue + # Don't retry experimental build previously triggered by CQ. + if any(t['key'] == 'cq_experimental' and t['value'] == 'true' + for t in builds[-1]['tags']): + continue + jobs.append((project, bucket, builder)) - # Sort the jobs to make testing easier. - return sorted(jobs) + # Sort the jobs to make testing easier. + return sorted(jobs) def _print_tryjobs(options, builds): - """Prints nicely result of _fetch_tryjobs.""" - if not builds: - print('No tryjobs scheduled.') - return - - longest_builder = max(len(b['builder']['builder']) for b in builds) - name_fmt = '{builder:<%d}' % longest_builder - if options.print_master: - longest_bucket = max(len(b['builder']['bucket']) for b in builds) - name_fmt = ('{bucket:>%d} ' % longest_bucket) + name_fmt - - builds_by_status = {} - for b in builds: - builds_by_status.setdefault(b['status'], []).append({ - 'id': b['id'], - 'name': name_fmt.format( - builder=b['builder']['builder'], bucket=b['builder']['bucket']), - }) - - sort_key = lambda b: (b['name'], b['id']) - - def print_builds(title, builds, fmt=None, color=None): - """Pop matching builds from `builds` dict and print them.""" + """Prints nicely result of _fetch_tryjobs.""" if not builds: - return + print('No tryjobs scheduled.') + return - fmt = fmt or '{name} https://ci.chromium.org/b/{id}' - if not options.color or color is None: - colorize = lambda x: x - else: - colorize = lambda x: '%s%s%s' % (color, x, Fore.RESET) + longest_builder = max(len(b['builder']['builder']) for b in builds) + name_fmt = '{builder:<%d}' % longest_builder + if options.print_master: + longest_bucket = max(len(b['builder']['bucket']) for b in builds) + name_fmt = ('{bucket:>%d} ' % longest_bucket) + name_fmt - print(colorize(title)) - for b in sorted(builds, key=sort_key): - print(' ', colorize(fmt.format(**b))) + builds_by_status = {} + for b in builds: + builds_by_status.setdefault(b['status'], []).append({ + 'id': + b['id'], + 'name': + name_fmt.format(builder=b['builder']['builder'], + bucket=b['builder']['bucket']), + }) - total = len(builds) - print_builds( - 'Successes:', builds_by_status.pop('SUCCESS', []), color=Fore.GREEN) - print_builds( - 'Infra Failures:', builds_by_status.pop('INFRA_FAILURE', []), - color=Fore.MAGENTA) - print_builds('Failures:', builds_by_status.pop('FAILURE', []), color=Fore.RED) - print_builds('Canceled:', builds_by_status.pop('CANCELED', []), fmt='{name}', - color=Fore.MAGENTA) - print_builds('Started:', builds_by_status.pop('STARTED', []), - color=Fore.YELLOW) - print_builds( - 'Scheduled:', builds_by_status.pop('SCHEDULED', []), fmt='{name} id={id}') - # The last section is just in case buildbucket API changes OR there is a bug. - print_builds( - 'Other:', sum(builds_by_status.values(), []), fmt='{name} id={id}') - print('Total: %d tryjobs' % total) + sort_key = lambda b: (b['name'], b['id']) + + def print_builds(title, builds, fmt=None, color=None): + """Pop matching builds from `builds` dict and print them.""" + if not builds: + return + + fmt = fmt or '{name} https://ci.chromium.org/b/{id}' + if not options.color or color is None: + colorize = lambda x: x + else: + colorize = lambda x: '%s%s%s' % (color, x, Fore.RESET) + + print(colorize(title)) + for b in sorted(builds, key=sort_key): + print(' ', colorize(fmt.format(**b))) + + total = len(builds) + print_builds('Successes:', + builds_by_status.pop('SUCCESS', []), + color=Fore.GREEN) + print_builds('Infra Failures:', + builds_by_status.pop('INFRA_FAILURE', []), + color=Fore.MAGENTA) + print_builds('Failures:', + builds_by_status.pop('FAILURE', []), + color=Fore.RED) + print_builds('Canceled:', + builds_by_status.pop('CANCELED', []), + fmt='{name}', + color=Fore.MAGENTA) + print_builds('Started:', + builds_by_status.pop('STARTED', []), + color=Fore.YELLOW) + print_builds('Scheduled:', + builds_by_status.pop('SCHEDULED', []), + fmt='{name} id={id}') + # The last section is just in case buildbucket API changes OR there is a + # bug. + print_builds('Other:', + sum(builds_by_status.values(), []), + fmt='{name} id={id}') + print('Total: %d tryjobs' % total) def _ComputeDiffLineRanges(files, upstream_commit): - """Gets the changed line ranges for each file since upstream_commit. + """Gets the changed line ranges for each file since upstream_commit. Parses a git diff on provided files and returns a dict that maps a file name to an ordered list of range tuples in the form (start_line, count). Ranges are in the same format as a git diff. """ - # If files is empty then diff_output will be a full diff. - if len(files) == 0: - return {} + # If files is empty then diff_output will be a full diff. + if len(files) == 0: + return {} - # Take the git diff and find the line ranges where there are changes. - diff_cmd = BuildGitDiffCmd('-U0', upstream_commit, files, allow_prefix=True) - diff_output = RunGit(diff_cmd) + # Take the git diff and find the line ranges where there are changes. + diff_cmd = BuildGitDiffCmd('-U0', upstream_commit, files, allow_prefix=True) + diff_output = RunGit(diff_cmd) - pattern = r'(?:^diff --git a/(?:.*) b/(.*))|(?:^@@.*\+(.*) @@)' - # 2 capture groups - # 0 == fname of diff file - # 1 == 'diff_start,diff_count' or 'diff_start' - # will match each of - # diff --git a/foo.foo b/foo.py - # @@ -12,2 +14,3 @@ - # @@ -12,2 +17 @@ - # running re.findall on the above string with pattern will give - # [('foo.py', ''), ('', '14,3'), ('', '17')] + pattern = r'(?:^diff --git a/(?:.*) b/(.*))|(?:^@@.*\+(.*) @@)' + # 2 capture groups + # 0 == fname of diff file + # 1 == 'diff_start,diff_count' or 'diff_start' + # will match each of + # diff --git a/foo.foo b/foo.py + # @@ -12,2 +14,3 @@ + # @@ -12,2 +17 @@ + # running re.findall on the above string with pattern will give + # [('foo.py', ''), ('', '14,3'), ('', '17')] - curr_file = None - line_diffs = {} - for match in re.findall(pattern, diff_output, flags=re.MULTILINE): - if match[0] != '': - # Will match the second filename in diff --git a/a.py b/b.py. - curr_file = match[0] - line_diffs[curr_file] = [] - else: - # Matches +14,3 - if ',' in match[1]: - diff_start, diff_count = match[1].split(',') - else: - # Single line changes are of the form +12 instead of +12,1. - diff_start = match[1] - diff_count = 1 + curr_file = None + line_diffs = {} + for match in re.findall(pattern, diff_output, flags=re.MULTILINE): + if match[0] != '': + # Will match the second filename in diff --git a/a.py b/b.py. + curr_file = match[0] + line_diffs[curr_file] = [] + else: + # Matches +14,3 + if ',' in match[1]: + diff_start, diff_count = match[1].split(',') + else: + # Single line changes are of the form +12 instead of +12,1. + diff_start = match[1] + diff_count = 1 - diff_start = int(diff_start) - diff_count = int(diff_count) + diff_start = int(diff_start) + diff_count = int(diff_count) - # If diff_count == 0 this is a removal we can ignore. - line_diffs[curr_file].append((diff_start, diff_count)) + # If diff_count == 0 this is a removal we can ignore. + line_diffs[curr_file].append((diff_start, diff_count)) - return line_diffs + return line_diffs def _FindYapfConfigFile(fpath, yapf_config_cache, top_dir=None): - """Checks if a yapf file is in any parent directory of fpath until top_dir. + """Checks if a yapf file is in any parent directory of fpath until top_dir. Recursively checks parent directories to find yapf file and if no yapf file is found returns None. Uses yapf_config_cache as a cache for previously found configs. """ - fpath = os.path.abspath(fpath) - # Return result if we've already computed it. - if fpath in yapf_config_cache: - return yapf_config_cache[fpath] + fpath = os.path.abspath(fpath) + # Return result if we've already computed it. + if fpath in yapf_config_cache: + return yapf_config_cache[fpath] - parent_dir = os.path.dirname(fpath) - if os.path.isfile(fpath): - ret = _FindYapfConfigFile(parent_dir, yapf_config_cache, top_dir) - else: - # Otherwise fpath is a directory - yapf_file = os.path.join(fpath, YAPF_CONFIG_FILENAME) - if os.path.isfile(yapf_file): - ret = yapf_file - elif fpath in (top_dir, parent_dir): - # If we're at the top level directory, or if we're at root - # there is no provided style. - ret = None + parent_dir = os.path.dirname(fpath) + if os.path.isfile(fpath): + ret = _FindYapfConfigFile(parent_dir, yapf_config_cache, top_dir) else: - # Otherwise recurse on the current directory. - ret = _FindYapfConfigFile(parent_dir, yapf_config_cache, top_dir) - yapf_config_cache[fpath] = ret - return ret + # Otherwise fpath is a directory + yapf_file = os.path.join(fpath, YAPF_CONFIG_FILENAME) + if os.path.isfile(yapf_file): + ret = yapf_file + elif fpath in (top_dir, parent_dir): + # If we're at the top level directory, or if we're at root + # there is no provided style. + ret = None + else: + # Otherwise recurse on the current directory. + ret = _FindYapfConfigFile(parent_dir, yapf_config_cache, top_dir) + yapf_config_cache[fpath] = ret + return ret def _GetYapfIgnorePatterns(top_dir): - """Returns all patterns in the .yapfignore file. + """Returns all patterns in the .yapfignore file. yapf is supposed to handle the ignoring of files listed in .yapfignore itself, but this functionality appears to break when explicitly passing files to @@ -709,22 +727,22 @@ def _GetYapfIgnorePatterns(top_dir): Returns: A set of all fnmatch patterns to be ignored. """ - yapfignore_file = os.path.join(top_dir, '.yapfignore') - ignore_patterns = set() - if not os.path.exists(yapfignore_file): - return ignore_patterns + yapfignore_file = os.path.join(top_dir, '.yapfignore') + ignore_patterns = set() + if not os.path.exists(yapfignore_file): + return ignore_patterns - for line in gclient_utils.FileRead(yapfignore_file).split('\n'): - stripped_line = line.strip() - # Comments and blank lines should be ignored. - if stripped_line.startswith('#') or stripped_line == '': - continue - ignore_patterns.add(stripped_line) - return ignore_patterns + for line in gclient_utils.FileRead(yapfignore_file).split('\n'): + stripped_line = line.strip() + # Comments and blank lines should be ignored. + if stripped_line.startswith('#') or stripped_line == '': + continue + ignore_patterns.add(stripped_line) + return ignore_patterns def _FilterYapfIgnoredFiles(filepaths, patterns): - """Filters out any filepaths that match any of the given patterns. + """Filters out any filepaths that match any of the given patterns. Args: filepaths: An iterable of strings containing filepaths to filter. @@ -734,288 +752,295 @@ def _FilterYapfIgnoredFiles(filepaths, patterns): A list of strings containing all the elements of |filepaths| that did not match any of the patterns in |patterns|. """ - # Not inlined so that tests can use the same implementation. - return [f for f in filepaths - if not any(fnmatch.fnmatch(f, p) for p in patterns)] + # Not inlined so that tests can use the same implementation. + return [ + f for f in filepaths + if not any(fnmatch.fnmatch(f, p) for p in patterns) + ] def _GetCommitCountSummary(begin_commit: str, end_commit: str = "HEAD") -> Optional[str]: - """Generate a summary of the number of commits in (begin_commit, end_commit). + """Generate a summary of the number of commits in (begin_commit, end_commit). Returns a string containing the summary, or None if the range is empty. """ - count = int( - RunGitSilent(['rev-list', '--count', f'{begin_commit}..{end_commit}'])) + count = int( + RunGitSilent(['rev-list', '--count', f'{begin_commit}..{end_commit}'])) - if not count: - return None + if not count: + return None - return f'{count} commit{"s"[:count!=1]}' + return f'{count} commit{"s"[:count!=1]}' def print_stats(args): - """Prints statistics about the change to the user.""" - # --no-ext-diff is broken in some versions of Git, so try to work around - # this by overriding the environment (but there is still a problem if the - # git config key "diff.external" is used). - env = GetNoGitPagerEnv() - if 'GIT_EXTERNAL_DIFF' in env: - del env['GIT_EXTERNAL_DIFF'] + """Prints statistics about the change to the user.""" + # --no-ext-diff is broken in some versions of Git, so try to work around + # this by overriding the environment (but there is still a problem if the + # git config key "diff.external" is used). + env = GetNoGitPagerEnv() + if 'GIT_EXTERNAL_DIFF' in env: + del env['GIT_EXTERNAL_DIFF'] - return subprocess2.call( - ['git', 'diff', '--no-ext-diff', '--stat', '-l100000', '-C50'] + args, - env=env) + return subprocess2.call( + ['git', 'diff', '--no-ext-diff', '--stat', '-l100000', '-C50'] + args, + env=env) class BuildbucketResponseException(Exception): - pass + pass class Settings(object): - def __init__(self): - self.cc = None - self.root = None - self.tree_status_url = None - self.viewvc_url = None - self.updated = False - self.is_gerrit = None - self.squash_gerrit_uploads = None - self.gerrit_skip_ensure_authenticated = None - self.git_editor = None - self.format_full_by_default = None - self.is_status_commit_order_by_date = None + def __init__(self): + self.cc = None + self.root = None + self.tree_status_url = None + self.viewvc_url = None + self.updated = False + self.is_gerrit = None + self.squash_gerrit_uploads = None + self.gerrit_skip_ensure_authenticated = None + self.git_editor = None + self.format_full_by_default = None + self.is_status_commit_order_by_date = None - def _LazyUpdateIfNeeded(self): - """Updates the settings from a codereview.settings file, if available.""" - if self.updated: - return + def _LazyUpdateIfNeeded(self): + """Updates the settings from a codereview.settings file, if available.""" + if self.updated: + return - # The only value that actually changes the behavior is - # autoupdate = "false". Everything else means "true". - autoupdate = ( - scm.GIT.GetConfig(self.GetRoot(), 'rietveld.autoupdate', '').lower()) + # The only value that actually changes the behavior is + # autoupdate = "false". Everything else means "true". + autoupdate = (scm.GIT.GetConfig(self.GetRoot(), 'rietveld.autoupdate', + '').lower()) - cr_settings_file = FindCodereviewSettingsFile() - if autoupdate != 'false' and cr_settings_file: - LoadCodereviewSettingsFromFile(cr_settings_file) - cr_settings_file.close() + cr_settings_file = FindCodereviewSettingsFile() + if autoupdate != 'false' and cr_settings_file: + LoadCodereviewSettingsFromFile(cr_settings_file) + cr_settings_file.close() - self.updated = True + self.updated = True - @staticmethod - def GetRelativeRoot(): - return scm.GIT.GetCheckoutRoot('.') + @staticmethod + def GetRelativeRoot(): + return scm.GIT.GetCheckoutRoot('.') - def GetRoot(self): - if self.root is None: - self.root = os.path.abspath(self.GetRelativeRoot()) - return self.root + def GetRoot(self): + if self.root is None: + self.root = os.path.abspath(self.GetRelativeRoot()) + return self.root - def GetTreeStatusUrl(self, error_ok=False): - if not self.tree_status_url: - self.tree_status_url = self._GetConfig('rietveld.tree-status-url') - if self.tree_status_url is None and not error_ok: - DieWithError( - 'You must configure your tree status URL by running ' - '"git cl config".') - return self.tree_status_url + def GetTreeStatusUrl(self, error_ok=False): + if not self.tree_status_url: + self.tree_status_url = self._GetConfig('rietveld.tree-status-url') + if self.tree_status_url is None and not error_ok: + DieWithError( + 'You must configure your tree status URL by running ' + '"git cl config".') + return self.tree_status_url - def GetViewVCUrl(self): - if not self.viewvc_url: - self.viewvc_url = self._GetConfig('rietveld.viewvc-url') - return self.viewvc_url + def GetViewVCUrl(self): + if not self.viewvc_url: + self.viewvc_url = self._GetConfig('rietveld.viewvc-url') + return self.viewvc_url - def GetBugPrefix(self): - return self._GetConfig('rietveld.bug-prefix') + def GetBugPrefix(self): + return self._GetConfig('rietveld.bug-prefix') - def GetRunPostUploadHook(self): - run_post_upload_hook = self._GetConfig( - 'rietveld.run-post-upload-hook') - return run_post_upload_hook == "True" + def GetRunPostUploadHook(self): + run_post_upload_hook = self._GetConfig('rietveld.run-post-upload-hook') + return run_post_upload_hook == "True" - def GetDefaultCCList(self): - return self._GetConfig('rietveld.cc') + def GetDefaultCCList(self): + return self._GetConfig('rietveld.cc') - def GetSquashGerritUploads(self): - """Returns True if uploads to Gerrit should be squashed by default.""" - if self.squash_gerrit_uploads is None: - self.squash_gerrit_uploads = self.GetSquashGerritUploadsOverride() - if self.squash_gerrit_uploads is None: - # Default is squash now (http://crbug.com/611892#c23). - self.squash_gerrit_uploads = self._GetConfig( - 'gerrit.squash-uploads').lower() != 'false' - return self.squash_gerrit_uploads + def GetSquashGerritUploads(self): + """Returns True if uploads to Gerrit should be squashed by default.""" + if self.squash_gerrit_uploads is None: + self.squash_gerrit_uploads = self.GetSquashGerritUploadsOverride() + if self.squash_gerrit_uploads is None: + # Default is squash now (http://crbug.com/611892#c23). + self.squash_gerrit_uploads = self._GetConfig( + 'gerrit.squash-uploads').lower() != 'false' + return self.squash_gerrit_uploads - def GetSquashGerritUploadsOverride(self): - """Return True or False if codereview.settings should be overridden. + def GetSquashGerritUploadsOverride(self): + """Return True or False if codereview.settings should be overridden. Returns None if no override has been defined. """ - # See also http://crbug.com/611892#c23 - result = self._GetConfig('gerrit.override-squash-uploads').lower() - if result == 'true': - return True - if result == 'false': - return False - return None + # See also http://crbug.com/611892#c23 + result = self._GetConfig('gerrit.override-squash-uploads').lower() + if result == 'true': + return True + if result == 'false': + return False + return None - def GetIsGerrit(self): - """Return True if gerrit.host is set.""" - if self.is_gerrit is None: - self.is_gerrit = bool(self._GetConfig('gerrit.host', False)) - return self.is_gerrit + def GetIsGerrit(self): + """Return True if gerrit.host is set.""" + if self.is_gerrit is None: + self.is_gerrit = bool(self._GetConfig('gerrit.host', False)) + return self.is_gerrit - def GetGerritSkipEnsureAuthenticated(self): - """Return True if EnsureAuthenticated should not be done for Gerrit + def GetGerritSkipEnsureAuthenticated(self): + """Return True if EnsureAuthenticated should not be done for Gerrit uploads.""" - if self.gerrit_skip_ensure_authenticated is None: - self.gerrit_skip_ensure_authenticated = self._GetConfig( - 'gerrit.skip-ensure-authenticated').lower() == 'true' - return self.gerrit_skip_ensure_authenticated + if self.gerrit_skip_ensure_authenticated is None: + self.gerrit_skip_ensure_authenticated = self._GetConfig( + 'gerrit.skip-ensure-authenticated').lower() == 'true' + return self.gerrit_skip_ensure_authenticated - def GetGitEditor(self): - """Returns the editor specified in the git config, or None if none is.""" - if self.git_editor is None: - # Git requires single quotes for paths with spaces. We need to replace - # them with double quotes for Windows to treat such paths as a single - # path. - self.git_editor = self._GetConfig('core.editor').replace('\'', '"') - return self.git_editor or None + def GetGitEditor(self): + """Returns the editor specified in the git config, or None if none is.""" + if self.git_editor is None: + # Git requires single quotes for paths with spaces. We need to + # replace them with double quotes for Windows to treat such paths as + # a single path. + self.git_editor = self._GetConfig('core.editor').replace('\'', '"') + return self.git_editor or None - def GetLintRegex(self): - return self._GetConfig('rietveld.cpplint-regex', DEFAULT_LINT_REGEX) + def GetLintRegex(self): + return self._GetConfig('rietveld.cpplint-regex', DEFAULT_LINT_REGEX) - def GetLintIgnoreRegex(self): - return self._GetConfig( - 'rietveld.cpplint-ignore-regex', DEFAULT_LINT_IGNORE_REGEX) + def GetLintIgnoreRegex(self): + return self._GetConfig('rietveld.cpplint-ignore-regex', + DEFAULT_LINT_IGNORE_REGEX) - def GetFormatFullByDefault(self): - if self.format_full_by_default is None: - self._LazyUpdateIfNeeded() - result = ( - RunGit(['config', '--bool', 'rietveld.format-full-by-default'], - error_ok=True).strip()) - self.format_full_by_default = (result == 'true') - return self.format_full_by_default + def GetFormatFullByDefault(self): + if self.format_full_by_default is None: + self._LazyUpdateIfNeeded() + result = (RunGit( + ['config', '--bool', 'rietveld.format-full-by-default'], + error_ok=True).strip()) + self.format_full_by_default = (result == 'true') + return self.format_full_by_default - def IsStatusCommitOrderByDate(self): - if self.is_status_commit_order_by_date is None: - result = (RunGit(['config', '--bool', 'cl.date-order'], - error_ok=True).strip()) - self.is_status_commit_order_by_date = (result == 'true') - return self.is_status_commit_order_by_date + def IsStatusCommitOrderByDate(self): + if self.is_status_commit_order_by_date is None: + result = (RunGit(['config', '--bool', 'cl.date-order'], + error_ok=True).strip()) + self.is_status_commit_order_by_date = (result == 'true') + return self.is_status_commit_order_by_date - def _GetConfig(self, key, default=''): - self._LazyUpdateIfNeeded() - return scm.GIT.GetConfig(self.GetRoot(), key, default) + def _GetConfig(self, key, default=''): + self._LazyUpdateIfNeeded() + return scm.GIT.GetConfig(self.GetRoot(), key, default) class _CQState(object): - """Enum for states of CL with respect to CQ.""" - NONE = 'none' - DRY_RUN = 'dry_run' - COMMIT = 'commit' + """Enum for states of CL with respect to CQ.""" + NONE = 'none' + DRY_RUN = 'dry_run' + COMMIT = 'commit' - ALL_STATES = [NONE, DRY_RUN, COMMIT] + ALL_STATES = [NONE, DRY_RUN, COMMIT] class _ParsedIssueNumberArgument(object): - def __init__(self, issue=None, patchset=None, hostname=None): - self.issue = issue - self.patchset = patchset - self.hostname = hostname + def __init__(self, issue=None, patchset=None, hostname=None): + self.issue = issue + self.patchset = patchset + self.hostname = hostname - @property - def valid(self): - return self.issue is not None + @property + def valid(self): + return self.issue is not None def ParseIssueNumberArgument(arg): - """Parses the issue argument and returns _ParsedIssueNumberArgument.""" - fail_result = _ParsedIssueNumberArgument() + """Parses the issue argument and returns _ParsedIssueNumberArgument.""" + fail_result = _ParsedIssueNumberArgument() - if isinstance(arg, int): - return _ParsedIssueNumberArgument(issue=arg) - if not isinstance(arg, str): - return fail_result + if isinstance(arg, int): + return _ParsedIssueNumberArgument(issue=arg) + if not isinstance(arg, str): + return fail_result - if arg.isdigit(): - return _ParsedIssueNumberArgument(issue=int(arg)) + if arg.isdigit(): + return _ParsedIssueNumberArgument(issue=int(arg)) - url = gclient_utils.UpgradeToHttps(arg) - if not url.startswith('http'): - return fail_result - for gerrit_url, short_url in _KNOWN_GERRIT_TO_SHORT_URLS.items(): - if url.startswith(short_url): - url = gerrit_url + url[len(short_url):] - break + url = gclient_utils.UpgradeToHttps(arg) + if not url.startswith('http'): + return fail_result + for gerrit_url, short_url in _KNOWN_GERRIT_TO_SHORT_URLS.items(): + if url.startswith(short_url): + url = gerrit_url + url[len(short_url):] + break - try: - parsed_url = urllib.parse.urlparse(url) - except ValueError: - return fail_result + try: + parsed_url = urllib.parse.urlparse(url) + except ValueError: + return fail_result - # If "https://" was automatically added, fail if `arg` looks unlikely to be a - # URL. - if not arg.startswith('http') and '.' not in parsed_url.netloc: - return fail_result + # If "https://" was automatically added, fail if `arg` looks unlikely to be + # a URL. + if not arg.startswith('http') and '.' not in parsed_url.netloc: + return fail_result - # Gerrit's new UI is https://domain/c/project/+/[/[patchset]] - # But old GWT UI is https://domain/#/c/project/+/[/[patchset]] - # Short urls like https://domain/ can be used, but don't allow - # specifying the patchset (you'd 404), but we allow that here. - if parsed_url.path == '/': - part = parsed_url.fragment - else: - part = parsed_url.path + # Gerrit's new UI is https://domain/c/project/+/[/[patchset]] + # But old GWT UI is https://domain/#/c/project/+/[/[patchset]] + # Short urls like https://domain/ can be used, but don't allow + # specifying the patchset (you'd 404), but we allow that here. + if parsed_url.path == '/': + part = parsed_url.fragment + else: + part = parsed_url.path - match = re.match( - r'(/c(/.*/\+)?)?/(?P\d+)(/(?P\d+)?/?)?$', part) - if not match: - return fail_result + match = re.match(r'(/c(/.*/\+)?)?/(?P\d+)(/(?P\d+)?/?)?$', + part) + if not match: + return fail_result - issue = int(match.group('issue')) - patchset = match.group('patchset') - return _ParsedIssueNumberArgument( - issue=issue, - patchset=int(patchset) if patchset else None, - hostname=parsed_url.netloc) + issue = int(match.group('issue')) + patchset = match.group('patchset') + return _ParsedIssueNumberArgument( + issue=issue, + patchset=int(patchset) if patchset else None, + hostname=parsed_url.netloc) def _create_description_from_log(args): - """Pulls out the commit log to use as a base for the CL description.""" - log_args = [] - if len(args) == 1 and args[0] == None: - # Handle the case where None is passed as the branch. - return '' - if len(args) == 1 and not args[0].endswith('.'): - log_args = [args[0] + '..'] - elif len(args) == 1 and args[0].endswith('...'): - log_args = [args[0][:-1]] - elif len(args) == 2: - log_args = [args[0] + '..' + args[1]] - else: - log_args = args[:] # Hope for the best! - return RunGit(['log', '--pretty=format:%B%n'] + log_args) + """Pulls out the commit log to use as a base for the CL description.""" + log_args = [] + if len(args) == 1 and args[0] == None: + # Handle the case where None is passed as the branch. + return '' + if len(args) == 1 and not args[0].endswith('.'): + log_args = [args[0] + '..'] + elif len(args) == 1 and args[0].endswith('...'): + log_args = [args[0][:-1]] + elif len(args) == 2: + log_args = [args[0] + '..' + args[1]] + else: + log_args = args[:] # Hope for the best! + return RunGit(['log', '--pretty=format:%B%n'] + log_args) class GerritChangeNotExists(Exception): - def __init__(self, issue, url): - self.issue = issue - self.url = url - super(GerritChangeNotExists, self).__init__() + def __init__(self, issue, url): + self.issue = issue + self.url = url + super(GerritChangeNotExists, self).__init__() - def __str__(self): - return 'change %s at %s does not exist or you have no access to it' % ( - self.issue, self.url) + def __str__(self): + return 'change %s at %s does not exist or you have no access to it' % ( + self.issue, self.url) _CommentSummary = collections.namedtuple( - '_CommentSummary', ['date', 'message', 'sender', 'autogenerated', - # TODO(tandrii): these two aren't known in Gerrit. - 'approval', 'disapproval']) - + '_CommentSummary', + [ + 'date', + 'message', + 'sender', + 'autogenerated', + # TODO(tandrii): these two aren't known in Gerrit. + 'approval', + 'disapproval' + ]) # TODO(b/265929888): Change `parent` to `pushed_commit_base`. _NewUpload = collections.namedtuple('NewUpload', [ @@ -1025,855 +1050,888 @@ _NewUpload = collections.namedtuple('NewUpload', [ class ChangeDescription(object): - """Contains a parsed form of the change description.""" - R_LINE = r'^[ \t]*(TBR|R)[ \t]*=[ \t]*(.*?)[ \t]*$' - CC_LINE = r'^[ \t]*(CC)[ \t]*=[ \t]*(.*?)[ \t]*$' - BUG_LINE = r'^[ \t]*(?:(BUG)[ \t]*=|Bug:)[ \t]*(.*?)[ \t]*$' - FIXED_LINE = r'^[ \t]*Fixed[ \t]*:[ \t]*(.*?)[ \t]*$' - CHERRY_PICK_LINE = r'^\(cherry picked from commit [a-fA-F0-9]{40}\)$' - STRIP_HASH_TAG_PREFIX = r'^(\s*(revert|reland)( "|:)?\s*)*' - BRACKET_HASH_TAG = r'\s*\[([^\[\]]+)\]' - COLON_SEPARATED_HASH_TAG = r'^([a-zA-Z0-9_\- ]+):($|[^:])' - BAD_HASH_TAG_CHUNK = r'[^a-zA-Z0-9]+' + """Contains a parsed form of the change description.""" + R_LINE = r'^[ \t]*(TBR|R)[ \t]*=[ \t]*(.*?)[ \t]*$' + CC_LINE = r'^[ \t]*(CC)[ \t]*=[ \t]*(.*?)[ \t]*$' + BUG_LINE = r'^[ \t]*(?:(BUG)[ \t]*=|Bug:)[ \t]*(.*?)[ \t]*$' + FIXED_LINE = r'^[ \t]*Fixed[ \t]*:[ \t]*(.*?)[ \t]*$' + CHERRY_PICK_LINE = r'^\(cherry picked from commit [a-fA-F0-9]{40}\)$' + STRIP_HASH_TAG_PREFIX = r'^(\s*(revert|reland)( "|:)?\s*)*' + BRACKET_HASH_TAG = r'\s*\[([^\[\]]+)\]' + COLON_SEPARATED_HASH_TAG = r'^([a-zA-Z0-9_\- ]+):($|[^:])' + BAD_HASH_TAG_CHUNK = r'[^a-zA-Z0-9]+' - def __init__(self, description, bug=None, fixed=None): - self._description_lines = (description or '').strip().splitlines() - if bug: - regexp = re.compile(self.BUG_LINE) - prefix = settings.GetBugPrefix() - if not any((regexp.match(line) for line in self._description_lines)): - values = list(_get_bug_line_values(prefix, bug)) - self.append_footer('Bug: %s' % ', '.join(values)) - if fixed: - regexp = re.compile(self.FIXED_LINE) - prefix = settings.GetBugPrefix() - if not any((regexp.match(line) for line in self._description_lines)): - values = list(_get_bug_line_values(prefix, fixed)) - self.append_footer('Fixed: %s' % ', '.join(values)) + def __init__(self, description, bug=None, fixed=None): + self._description_lines = (description or '').strip().splitlines() + if bug: + regexp = re.compile(self.BUG_LINE) + prefix = settings.GetBugPrefix() + if not any( + (regexp.match(line) for line in self._description_lines)): + values = list(_get_bug_line_values(prefix, bug)) + self.append_footer('Bug: %s' % ', '.join(values)) + if fixed: + regexp = re.compile(self.FIXED_LINE) + prefix = settings.GetBugPrefix() + if not any( + (regexp.match(line) for line in self._description_lines)): + values = list(_get_bug_line_values(prefix, fixed)) + self.append_footer('Fixed: %s' % ', '.join(values)) - @property # www.logilab.org/ticket/89786 - def description(self): # pylint: disable=method-hidden - return '\n'.join(self._description_lines) + @property # www.logilab.org/ticket/89786 + def description(self): # pylint: disable=method-hidden + return '\n'.join(self._description_lines) - def set_description(self, desc): - if isinstance(desc, str): - lines = desc.splitlines() - else: - lines = [line.rstrip() for line in desc] - while lines and not lines[0]: - lines.pop(0) - while lines and not lines[-1]: - lines.pop(-1) - self._description_lines = lines + def set_description(self, desc): + if isinstance(desc, str): + lines = desc.splitlines() + else: + lines = [line.rstrip() for line in desc] + while lines and not lines[0]: + lines.pop(0) + while lines and not lines[-1]: + lines.pop(-1) + self._description_lines = lines - def ensure_change_id(self, change_id): - description = self.description - footer_change_ids = git_footers.get_footer_change_id(description) - # Make sure that the Change-Id in the description matches the given one. - if footer_change_ids != [change_id]: - if footer_change_ids: - # Remove any existing Change-Id footers since they don't match the - # expected change_id footer. - description = git_footers.remove_footer(description, 'Change-Id') - print('WARNING: Change-Id has been set to %s. Use `git cl issue 0` ' - 'if you want to set a new one.') - # Add the expected Change-Id footer. - description = git_footers.add_footer_change_id(description, change_id) - self.set_description(description) + def ensure_change_id(self, change_id): + description = self.description + footer_change_ids = git_footers.get_footer_change_id(description) + # Make sure that the Change-Id in the description matches the given one. + if footer_change_ids != [change_id]: + if footer_change_ids: + # Remove any existing Change-Id footers since they don't match + # the expected change_id footer. + description = git_footers.remove_footer(description, + 'Change-Id') + print( + 'WARNING: Change-Id has been set to %s. Use `git cl issue 0` ' + 'if you want to set a new one.') + # Add the expected Change-Id footer. + description = git_footers.add_footer_change_id( + description, change_id) + self.set_description(description) - def update_reviewers(self, reviewers): - """Rewrites the R= line(s) as a single line each. + def update_reviewers(self, reviewers): + """Rewrites the R= line(s) as a single line each. Args: reviewers (list(str)) - list of additional emails to use for reviewers. """ - if not reviewers: - return + if not reviewers: + return - reviewers = set(reviewers) + reviewers = set(reviewers) - # Get the set of R= lines and remove them from the description. - regexp = re.compile(self.R_LINE) - matches = [regexp.match(line) for line in self._description_lines] - new_desc = [ - l for i, l in enumerate(self._description_lines) if not matches[i] - ] - self.set_description(new_desc) + # Get the set of R= lines and remove them from the description. + regexp = re.compile(self.R_LINE) + matches = [regexp.match(line) for line in self._description_lines] + new_desc = [ + l for i, l in enumerate(self._description_lines) if not matches[i] + ] + self.set_description(new_desc) - # Construct new unified R= lines. + # Construct new unified R= lines. - # First, update reviewers with names from the R= lines (if any). - for match in matches: - if not match: - continue - reviewers.update(cleanup_list([match.group(2).strip()])) + # First, update reviewers with names from the R= lines (if any). + for match in matches: + if not match: + continue + reviewers.update(cleanup_list([match.group(2).strip()])) - new_r_line = 'R=' + ', '.join(sorted(reviewers)) + new_r_line = 'R=' + ', '.join(sorted(reviewers)) - # Put the new lines in the description where the old first R= line was. - line_loc = next((i for i, match in enumerate(matches) if match), -1) - if 0 <= line_loc < len(self._description_lines): - self._description_lines.insert(line_loc, new_r_line) - else: - self.append_footer(new_r_line) + # Put the new lines in the description where the old first R= line was. + line_loc = next((i for i, match in enumerate(matches) if match), -1) + if 0 <= line_loc < len(self._description_lines): + self._description_lines.insert(line_loc, new_r_line) + else: + self.append_footer(new_r_line) - def set_preserve_tryjobs(self): - """Ensures description footer contains 'Cq-Do-Not-Cancel-Tryjobs: true'.""" - footers = git_footers.parse_footers(self.description) - for v in footers.get('Cq-Do-Not-Cancel-Tryjobs', []): - if v.lower() == 'true': - return - self.append_footer('Cq-Do-Not-Cancel-Tryjobs: true') + def set_preserve_tryjobs(self): + """Ensures description footer contains 'Cq-Do-Not-Cancel-Tryjobs: true'.""" + footers = git_footers.parse_footers(self.description) + for v in footers.get('Cq-Do-Not-Cancel-Tryjobs', []): + if v.lower() == 'true': + return + self.append_footer('Cq-Do-Not-Cancel-Tryjobs: true') - def prompt(self): - """Asks the user to update the description.""" - self.set_description([ - '# Enter a description of the change.', - '# This will be displayed on the codereview site.', - '# The first line will also be used as the subject of the review.', - '#--------------------This line is 72 characters long' - '--------------------', - ] + self._description_lines) - bug_regexp = re.compile(self.BUG_LINE) - fixed_regexp = re.compile(self.FIXED_LINE) - prefix = settings.GetBugPrefix() - has_issue = lambda l: bug_regexp.match(l) or fixed_regexp.match(l) + def prompt(self): + """Asks the user to update the description.""" + self.set_description([ + '# Enter a description of the change.', + '# This will be displayed on the codereview site.', + '# The first line will also be used as the subject of the review.', + '#--------------------This line is 72 characters long' + '--------------------', + ] + self._description_lines) + bug_regexp = re.compile(self.BUG_LINE) + fixed_regexp = re.compile(self.FIXED_LINE) + prefix = settings.GetBugPrefix() + has_issue = lambda l: bug_regexp.match(l) or fixed_regexp.match(l) - if not any((has_issue(line) for line in self._description_lines)): - self.append_footer('Bug: %s' % prefix) + if not any((has_issue(line) for line in self._description_lines)): + self.append_footer('Bug: %s' % prefix) - print('Waiting for editor...') - content = gclient_utils.RunEditor(self.description, - True, - git_editor=settings.GetGitEditor()) - if not content: - DieWithError('Running editor failed') - lines = content.splitlines() + print('Waiting for editor...') + content = gclient_utils.RunEditor(self.description, + True, + git_editor=settings.GetGitEditor()) + if not content: + DieWithError('Running editor failed') + lines = content.splitlines() - # Strip off comments and default inserted "Bug:" line. - clean_lines = [ - line.rstrip() for line in lines - if not (line.startswith('#') or line.rstrip() == "Bug:" - or line.rstrip() == "Bug: " + prefix) - ] - if not clean_lines: - DieWithError('No CL description, aborting') - self.set_description(clean_lines) + # Strip off comments and default inserted "Bug:" line. + clean_lines = [ + line.rstrip() for line in lines + if not (line.startswith('#') or line.rstrip() == "Bug:" + or line.rstrip() == "Bug: " + prefix) + ] + if not clean_lines: + DieWithError('No CL description, aborting') + self.set_description(clean_lines) - def append_footer(self, line): - """Adds a footer line to the description. + def append_footer(self, line): + """Adds a footer line to the description. Differentiates legacy "KEY=xxx" footers (used to be called tags) and Gerrit's footers in the form of "Footer-Key: footer any value" and ensures that Gerrit footers are always at the end. """ - parsed_footer_line = git_footers.parse_footer(line) - if parsed_footer_line: - # Line is a gerrit footer in the form: Footer-Key: any value. - # Thus, must be appended observing Gerrit footer rules. - self.set_description( - git_footers.add_footer(self.description, - key=parsed_footer_line[0], - value=parsed_footer_line[1])) - return + parsed_footer_line = git_footers.parse_footer(line) + if parsed_footer_line: + # Line is a gerrit footer in the form: Footer-Key: any value. + # Thus, must be appended observing Gerrit footer rules. + self.set_description( + git_footers.add_footer(self.description, + key=parsed_footer_line[0], + value=parsed_footer_line[1])) + return - if not self._description_lines: - self._description_lines.append(line) - return + if not self._description_lines: + self._description_lines.append(line) + return - top_lines, gerrit_footers, _ = git_footers.split_footers(self.description) - if gerrit_footers: - # git_footers.split_footers ensures that there is an empty line before - # actual (gerrit) footers, if any. We have to keep it that way. - assert top_lines and top_lines[-1] == '' - top_lines, separator = top_lines[:-1], top_lines[-1:] - else: - separator = [] # No need for separator if there are no gerrit_footers. + top_lines, gerrit_footers, _ = git_footers.split_footers( + self.description) + if gerrit_footers: + # git_footers.split_footers ensures that there is an empty line + # before actual (gerrit) footers, if any. We have to keep it that + # way. + assert top_lines and top_lines[-1] == '' + top_lines, separator = top_lines[:-1], top_lines[-1:] + else: + separator = [ + ] # No need for separator if there are no gerrit_footers. - prev_line = top_lines[-1] if top_lines else '' - if (not presubmit_support.Change.TAG_LINE_RE.match(prev_line) - or not presubmit_support.Change.TAG_LINE_RE.match(line)): - top_lines.append('') - top_lines.append(line) - self._description_lines = top_lines + separator + gerrit_footers + prev_line = top_lines[-1] if top_lines else '' + if (not presubmit_support.Change.TAG_LINE_RE.match(prev_line) + or not presubmit_support.Change.TAG_LINE_RE.match(line)): + top_lines.append('') + top_lines.append(line) + self._description_lines = top_lines + separator + gerrit_footers - def get_reviewers(self, tbr_only=False): - """Retrieves the list of reviewers.""" - matches = [re.match(self.R_LINE, line) for line in self._description_lines] - reviewers = [ - match.group(2).strip() for match in matches - if match and (not tbr_only or match.group(1).upper() == 'TBR') - ] - return cleanup_list(reviewers) + def get_reviewers(self, tbr_only=False): + """Retrieves the list of reviewers.""" + matches = [ + re.match(self.R_LINE, line) for line in self._description_lines + ] + reviewers = [ + match.group(2).strip() for match in matches + if match and (not tbr_only or match.group(1).upper() == 'TBR') + ] + return cleanup_list(reviewers) - def get_cced(self): - """Retrieves the list of reviewers.""" - matches = [re.match(self.CC_LINE, line) for line in self._description_lines] - cced = [match.group(2).strip() for match in matches if match] - return cleanup_list(cced) + def get_cced(self): + """Retrieves the list of reviewers.""" + matches = [ + re.match(self.CC_LINE, line) for line in self._description_lines + ] + cced = [match.group(2).strip() for match in matches if match] + return cleanup_list(cced) - def get_hash_tags(self): - """Extracts and sanitizes a list of Gerrit hashtags.""" - subject = (self._description_lines or ('', ))[0] - subject = re.sub(self.STRIP_HASH_TAG_PREFIX, - '', - subject, - flags=re.IGNORECASE) + def get_hash_tags(self): + """Extracts and sanitizes a list of Gerrit hashtags.""" + subject = (self._description_lines or ('', ))[0] + subject = re.sub(self.STRIP_HASH_TAG_PREFIX, + '', + subject, + flags=re.IGNORECASE) - tags = [] - start = 0 - bracket_exp = re.compile(self.BRACKET_HASH_TAG) - while True: - m = bracket_exp.match(subject, start) - if not m: - break - tags.append(self.sanitize_hash_tag(m.group(1))) - start = m.end() + tags = [] + start = 0 + bracket_exp = re.compile(self.BRACKET_HASH_TAG) + while True: + m = bracket_exp.match(subject, start) + if not m: + break + tags.append(self.sanitize_hash_tag(m.group(1))) + start = m.end() - if not tags: - # Try "Tag: " prefix. - m = re.match(self.COLON_SEPARATED_HASH_TAG, subject) - if m: - tags.append(self.sanitize_hash_tag(m.group(1))) - return tags + if not tags: + # Try "Tag: " prefix. + m = re.match(self.COLON_SEPARATED_HASH_TAG, subject) + if m: + tags.append(self.sanitize_hash_tag(m.group(1))) + return tags - @classmethod - def sanitize_hash_tag(cls, tag): - """Returns a sanitized Gerrit hash tag. + @classmethod + def sanitize_hash_tag(cls, tag): + """Returns a sanitized Gerrit hash tag. A sanitized hashtag can be used as a git push refspec parameter value. """ - return re.sub(cls.BAD_HASH_TAG_CHUNK, '-', tag).strip('-').lower() + return re.sub(cls.BAD_HASH_TAG_CHUNK, '-', tag).strip('-').lower() class Changelist(object): - """Changelist works with one changelist in local branch. + """Changelist works with one changelist in local branch. Notes: * Not safe for concurrent multi-{thread,process} use. * Caches values from current branch. Therefore, re-use after branch change with great care. """ - - def __init__(self, - branchref=None, - issue=None, - codereview_host=None, - commit_date=None): - """Create a new ChangeList instance. + def __init__(self, + branchref=None, + issue=None, + codereview_host=None, + commit_date=None): + """Create a new ChangeList instance. **kwargs will be passed directly to Gerrit implementation. """ - # Poke settings so we get the "configure your server" message if necessary. - global settings - if not settings: - # Happens when git_cl.py is used as a utility library. - settings = Settings() + # Poke settings so we get the "configure your server" message if + # necessary. + global settings + if not settings: + # Happens when git_cl.py is used as a utility library. + settings = Settings() - self.branchref = branchref - if self.branchref: - assert branchref.startswith('refs/heads/') - self.branch = scm.GIT.ShortBranchName(self.branchref) - else: - self.branch = None - self.commit_date = commit_date - self.upstream_branch = None - self.lookedup_issue = False - self.issue = issue or None - self.description = None - self.lookedup_patchset = False - self.patchset = None - self.cc = None - self.more_cc = [] - self._remote = None - self._cached_remote_url = (False, None) # (is_cached, value) + self.branchref = branchref + if self.branchref: + assert branchref.startswith('refs/heads/') + self.branch = scm.GIT.ShortBranchName(self.branchref) + else: + self.branch = None + self.commit_date = commit_date + self.upstream_branch = None + self.lookedup_issue = False + self.issue = issue or None + self.description = None + self.lookedup_patchset = False + self.patchset = None + self.cc = None + self.more_cc = [] + self._remote = None + self._cached_remote_url = (False, None) # (is_cached, value) - # Lazily cached values. - self._gerrit_host = None # e.g. chromium-review.googlesource.com - self._gerrit_server = None # e.g. https://chromium-review.googlesource.com - self._owners_client = None - # Map from change number (issue) to its detail cache. - self._detail_cache = {} + # Lazily cached values. + self._gerrit_host = None # e.g. chromium-review.googlesource.com + self._gerrit_server = None # e.g. https://chromium-review.googlesource.com + self._owners_client = None + # Map from change number (issue) to its detail cache. + self._detail_cache = {} - if codereview_host is not None: - assert not codereview_host.startswith('https://'), codereview_host - self._gerrit_host = codereview_host - self._gerrit_server = 'https://%s' % codereview_host + if codereview_host is not None: + assert not codereview_host.startswith('https://'), codereview_host + self._gerrit_host = codereview_host + self._gerrit_server = 'https://%s' % codereview_host - @property - def owners_client(self): - if self._owners_client is None: - remote, remote_branch = self.GetRemoteBranch() - branch = GetTargetRef(remote, remote_branch, None) - self._owners_client = owners_client.GetCodeOwnersClient( - host=self.GetGerritHost(), - project=self.GetGerritProject(), - branch=branch) - return self._owners_client + @property + def owners_client(self): + if self._owners_client is None: + remote, remote_branch = self.GetRemoteBranch() + branch = GetTargetRef(remote, remote_branch, None) + self._owners_client = owners_client.GetCodeOwnersClient( + host=self.GetGerritHost(), + project=self.GetGerritProject(), + branch=branch) + return self._owners_client - def GetCCList(self): - """Returns the users cc'd on this CL. + def GetCCList(self): + """Returns the users cc'd on this CL. The return value is a string suitable for passing to git cl with the --cc flag. """ - if self.cc is None: - base_cc = settings.GetDefaultCCList() - more_cc = ','.join(self.more_cc) - self.cc = ','.join(filter(None, (base_cc, more_cc))) or '' - return self.cc + if self.cc is None: + base_cc = settings.GetDefaultCCList() + more_cc = ','.join(self.more_cc) + self.cc = ','.join(filter(None, (base_cc, more_cc))) or '' + return self.cc - def ExtendCC(self, more_cc): - """Extends the list of users to cc on this CL based on the changed files.""" - self.more_cc.extend(more_cc) + def ExtendCC(self, more_cc): + """Extends the list of users to cc on this CL based on the changed files.""" + self.more_cc.extend(more_cc) - def GetCommitDate(self): - """Returns the commit date as provided in the constructor""" - return self.commit_date + def GetCommitDate(self): + """Returns the commit date as provided in the constructor""" + return self.commit_date - def GetBranch(self): - """Returns the short branch name, e.g. 'main'.""" - if not self.branch: - branchref = scm.GIT.GetBranchRef(settings.GetRoot()) - if not branchref: - return None - self.branchref = branchref - self.branch = scm.GIT.ShortBranchName(self.branchref) - return self.branch + def GetBranch(self): + """Returns the short branch name, e.g. 'main'.""" + if not self.branch: + branchref = scm.GIT.GetBranchRef(settings.GetRoot()) + if not branchref: + return None + self.branchref = branchref + self.branch = scm.GIT.ShortBranchName(self.branchref) + return self.branch - def GetBranchRef(self): - """Returns the full branch name, e.g. 'refs/heads/main'.""" - self.GetBranch() # Poke the lazy loader. - return self.branchref + def GetBranchRef(self): + """Returns the full branch name, e.g. 'refs/heads/main'.""" + self.GetBranch() # Poke the lazy loader. + return self.branchref - def _GitGetBranchConfigValue(self, key, default=None): - return scm.GIT.GetBranchConfig( - settings.GetRoot(), self.GetBranch(), key, default) + def _GitGetBranchConfigValue(self, key, default=None): + return scm.GIT.GetBranchConfig(settings.GetRoot(), self.GetBranch(), + key, default) - def _GitSetBranchConfigValue(self, key, value): - action = 'set %s to %r' % (key, value) - if not value: - action = 'unset %s' % key - assert self.GetBranch(), 'a branch is needed to ' + action - return scm.GIT.SetBranchConfig( - settings.GetRoot(), self.GetBranch(), key, value) + def _GitSetBranchConfigValue(self, key, value): + action = 'set %s to %r' % (key, value) + if not value: + action = 'unset %s' % key + assert self.GetBranch(), 'a branch is needed to ' + action + return scm.GIT.SetBranchConfig(settings.GetRoot(), self.GetBranch(), + key, value) - @staticmethod - def FetchUpstreamTuple(branch): - """Returns a tuple containing remote and remote ref, + @staticmethod + def FetchUpstreamTuple(branch): + """Returns a tuple containing remote and remote ref, e.g. 'origin', 'refs/heads/main' """ - remote, upstream_branch = scm.GIT.FetchUpstreamTuple( - settings.GetRoot(), branch) - if not remote or not upstream_branch: - DieWithError( - 'Unable to determine default branch to diff against.\n' - 'Verify this branch is set up to track another \n' - '(via the --track argument to "git checkout -b ..."). \n' - 'or pass complete "git diff"-style arguments if supported, like\n' - ' git cl upload origin/main\n') + remote, upstream_branch = scm.GIT.FetchUpstreamTuple( + settings.GetRoot(), branch) + if not remote or not upstream_branch: + DieWithError( + 'Unable to determine default branch to diff against.\n' + 'Verify this branch is set up to track another \n' + '(via the --track argument to "git checkout -b ..."). \n' + 'or pass complete "git diff"-style arguments if supported, like\n' + ' git cl upload origin/main\n') - return remote, upstream_branch + return remote, upstream_branch - def GetCommonAncestorWithUpstream(self): - upstream_branch = self.GetUpstreamBranch() - if not scm.GIT.IsValidRevision(settings.GetRoot(), upstream_branch): - DieWithError('The upstream for the current branch (%s) does not exist ' - 'anymore.\nPlease fix it and try again.' % self.GetBranch()) - return git_common.get_or_create_merge_base(self.GetBranch(), - upstream_branch) + def GetCommonAncestorWithUpstream(self): + upstream_branch = self.GetUpstreamBranch() + if not scm.GIT.IsValidRevision(settings.GetRoot(), upstream_branch): + DieWithError( + 'The upstream for the current branch (%s) does not exist ' + 'anymore.\nPlease fix it and try again.' % self.GetBranch()) + return git_common.get_or_create_merge_base(self.GetBranch(), + upstream_branch) - def GetUpstreamBranch(self): - if self.upstream_branch is None: - remote, upstream_branch = self.FetchUpstreamTuple(self.GetBranch()) - if remote != '.': - upstream_branch = upstream_branch.replace('refs/heads/', - 'refs/remotes/%s/' % remote) - upstream_branch = upstream_branch.replace('refs/branch-heads/', - 'refs/remotes/branch-heads/') - self.upstream_branch = upstream_branch - return self.upstream_branch + def GetUpstreamBranch(self): + if self.upstream_branch is None: + remote, upstream_branch = self.FetchUpstreamTuple(self.GetBranch()) + if remote != '.': + upstream_branch = upstream_branch.replace( + 'refs/heads/', 'refs/remotes/%s/' % remote) + upstream_branch = upstream_branch.replace( + 'refs/branch-heads/', 'refs/remotes/branch-heads/') + self.upstream_branch = upstream_branch + return self.upstream_branch - def GetRemoteBranch(self): - if not self._remote: - remote, branch = None, self.GetBranch() - seen_branches = set() - while branch not in seen_branches: - seen_branches.add(branch) - remote, branch = self.FetchUpstreamTuple(branch) - branch = scm.GIT.ShortBranchName(branch) - if remote != '.' or branch.startswith('refs/remotes'): - break - else: - remotes = RunGit(['remote'], error_ok=True).split() - if len(remotes) == 1: - remote, = remotes - elif 'origin' in remotes: - remote = 'origin' - logging.warning('Could not determine which remote this change is ' - 'associated with, so defaulting to "%s".' % - self._remote) - else: - logging.warning('Could not determine which remote this change is ' - 'associated with.') - branch = 'HEAD' - if branch.startswith('refs/remotes'): - self._remote = (remote, branch) - elif branch.startswith('refs/branch-heads/'): - self._remote = (remote, branch.replace('refs/', 'refs/remotes/')) - else: - self._remote = (remote, 'refs/remotes/%s/%s' % (remote, branch)) - return self._remote + def GetRemoteBranch(self): + if not self._remote: + remote, branch = None, self.GetBranch() + seen_branches = set() + while branch not in seen_branches: + seen_branches.add(branch) + remote, branch = self.FetchUpstreamTuple(branch) + branch = scm.GIT.ShortBranchName(branch) + if remote != '.' or branch.startswith('refs/remotes'): + break + else: + remotes = RunGit(['remote'], error_ok=True).split() + if len(remotes) == 1: + remote, = remotes + elif 'origin' in remotes: + remote = 'origin' + logging.warning( + 'Could not determine which remote this change is ' + 'associated with, so defaulting to "%s".' % + self._remote) + else: + logging.warning( + 'Could not determine which remote this change is ' + 'associated with.') + branch = 'HEAD' + if branch.startswith('refs/remotes'): + self._remote = (remote, branch) + elif branch.startswith('refs/branch-heads/'): + self._remote = (remote, branch.replace('refs/', + 'refs/remotes/')) + else: + self._remote = (remote, 'refs/remotes/%s/%s' % (remote, branch)) + return self._remote - def GetRemoteUrl(self) -> Optional[str]: - """Return the configured remote URL, e.g. 'git://example.org/foo.git/'. + def GetRemoteUrl(self) -> Optional[str]: + """Return the configured remote URL, e.g. 'git://example.org/foo.git/'. Returns None if there is no remote. """ - is_cached, value = self._cached_remote_url - if is_cached: - return value + is_cached, value = self._cached_remote_url + if is_cached: + return value - remote, _ = self.GetRemoteBranch() - url = scm.GIT.GetConfig(settings.GetRoot(), 'remote.%s.url' % remote, '') + remote, _ = self.GetRemoteBranch() + url = scm.GIT.GetConfig(settings.GetRoot(), 'remote.%s.url' % remote, + '') - # Check if the remote url can be parsed as an URL. - host = urllib.parse.urlparse(url).netloc - if host: - self._cached_remote_url = (True, url) - return url + # Check if the remote url can be parsed as an URL. + host = urllib.parse.urlparse(url).netloc + if host: + self._cached_remote_url = (True, url) + return url - # If it cannot be parsed as an url, assume it is a local directory, - # probably a git cache. - logging.warning('"%s" doesn\'t appear to point to a git host. ' - 'Interpreting it as a local directory.', url) - if not os.path.isdir(url): - logging.error( - 'Remote "%(remote)s" for branch "%(branch)s" points to "%(url)s", ' - 'but it doesn\'t exist.', - {'remote': remote, 'branch': self.GetBranch(), 'url': url}) - return None + # If it cannot be parsed as an url, assume it is a local directory, + # probably a git cache. + logging.warning( + '"%s" doesn\'t appear to point to a git host. ' + 'Interpreting it as a local directory.', url) + if not os.path.isdir(url): + logging.error( + 'Remote "%(remote)s" for branch "%(branch)s" points to "%(url)s", ' + 'but it doesn\'t exist.', { + 'remote': remote, + 'branch': self.GetBranch(), + 'url': url + }) + return None - cache_path = url - url = scm.GIT.GetConfig(url, 'remote.%s.url' % remote, '') + cache_path = url + url = scm.GIT.GetConfig(url, 'remote.%s.url' % remote, '') - host = urllib.parse.urlparse(url).netloc - if not host: - logging.error( - 'Remote "%(remote)s" for branch "%(branch)s" points to ' - '"%(cache_path)s", but it is misconfigured.\n' - '"%(cache_path)s" must be a git repo and must have a remote named ' - '"%(remote)s" pointing to the git host.', { - 'remote': remote, - 'cache_path': cache_path, - 'branch': self.GetBranch()}) - return None + host = urllib.parse.urlparse(url).netloc + if not host: + logging.error( + 'Remote "%(remote)s" for branch "%(branch)s" points to ' + '"%(cache_path)s", but it is misconfigured.\n' + '"%(cache_path)s" must be a git repo and must have a remote named ' + '"%(remote)s" pointing to the git host.', { + 'remote': remote, + 'cache_path': cache_path, + 'branch': self.GetBranch() + }) + return None - self._cached_remote_url = (True, url) - return url + self._cached_remote_url = (True, url) + return url - def GetIssue(self): - """Returns the issue number as a int or None if not set.""" - if self.issue is None and not self.lookedup_issue: - if self.GetBranch(): - self.issue = self._GitGetBranchConfigValue(ISSUE_CONFIG_KEY) - if self.issue is not None: - self.issue = int(self.issue) - self.lookedup_issue = True - return self.issue + def GetIssue(self): + """Returns the issue number as a int or None if not set.""" + if self.issue is None and not self.lookedup_issue: + if self.GetBranch(): + self.issue = self._GitGetBranchConfigValue(ISSUE_CONFIG_KEY) + if self.issue is not None: + self.issue = int(self.issue) + self.lookedup_issue = True + return self.issue - def GetIssueURL(self, short=False): - """Get the URL for a particular issue.""" - issue = self.GetIssue() - if not issue: - return None - server = self.GetCodereviewServer() - if short: - server = _KNOWN_GERRIT_TO_SHORT_URLS.get(server, server) - return '%s/%s' % (server, issue) + def GetIssueURL(self, short=False): + """Get the URL for a particular issue.""" + issue = self.GetIssue() + if not issue: + return None + server = self.GetCodereviewServer() + if short: + server = _KNOWN_GERRIT_TO_SHORT_URLS.get(server, server) + return '%s/%s' % (server, issue) - def FetchDescription(self, pretty=False): - assert self.GetIssue(), 'issue is required to query Gerrit' + def FetchDescription(self, pretty=False): + assert self.GetIssue(), 'issue is required to query Gerrit' - if self.description is None: - data = self._GetChangeDetail(['CURRENT_REVISION', 'CURRENT_COMMIT']) - current_rev = data['current_revision'] - self.description = data['revisions'][current_rev]['commit']['message'] + if self.description is None: + data = self._GetChangeDetail(['CURRENT_REVISION', 'CURRENT_COMMIT']) + current_rev = data['current_revision'] + self.description = data['revisions'][current_rev]['commit'][ + 'message'] - if not pretty: - return self.description + if not pretty: + return self.description - # Set width to 72 columns + 2 space indent. - wrapper = textwrap.TextWrapper(width=74, replace_whitespace=True) - wrapper.initial_indent = wrapper.subsequent_indent = ' ' - lines = self.description.splitlines() - return '\n'.join([wrapper.fill(line) for line in lines]) + # Set width to 72 columns + 2 space indent. + wrapper = textwrap.TextWrapper(width=74, replace_whitespace=True) + wrapper.initial_indent = wrapper.subsequent_indent = ' ' + lines = self.description.splitlines() + return '\n'.join([wrapper.fill(line) for line in lines]) - def GetPatchset(self): - """Returns the patchset number as a int or None if not set.""" - if self.patchset is None and not self.lookedup_patchset: - if self.GetBranch(): - self.patchset = self._GitGetBranchConfigValue(PATCHSET_CONFIG_KEY) - if self.patchset is not None: - self.patchset = int(self.patchset) - self.lookedup_patchset = True - return self.patchset + def GetPatchset(self): + """Returns the patchset number as a int or None if not set.""" + if self.patchset is None and not self.lookedup_patchset: + if self.GetBranch(): + self.patchset = self._GitGetBranchConfigValue( + PATCHSET_CONFIG_KEY) + if self.patchset is not None: + self.patchset = int(self.patchset) + self.lookedup_patchset = True + return self.patchset - def GetAuthor(self): - return scm.GIT.GetConfig(settings.GetRoot(), 'user.email') + def GetAuthor(self): + return scm.GIT.GetConfig(settings.GetRoot(), 'user.email') - def SetPatchset(self, patchset): - """Set this branch's patchset. If patchset=0, clears the patchset.""" - assert self.GetBranch() - if not patchset: - self.patchset = None - else: - self.patchset = int(patchset) - self._GitSetBranchConfigValue(PATCHSET_CONFIG_KEY, str(self.patchset)) + def SetPatchset(self, patchset): + """Set this branch's patchset. If patchset=0, clears the patchset.""" + assert self.GetBranch() + if not patchset: + self.patchset = None + else: + self.patchset = int(patchset) + self._GitSetBranchConfigValue(PATCHSET_CONFIG_KEY, str(self.patchset)) - def SetIssue(self, issue=None): - """Set this branch's issue. If issue isn't given, clears the issue.""" - assert self.GetBranch() - if issue: - issue = int(issue) - self._GitSetBranchConfigValue(ISSUE_CONFIG_KEY, str(issue)) - self.issue = issue - codereview_server = self.GetCodereviewServer() - if codereview_server: - self._GitSetBranchConfigValue( - CODEREVIEW_SERVER_CONFIG_KEY, codereview_server) - else: - # Reset all of these just to be clean. - reset_suffixes = [ - LAST_UPLOAD_HASH_CONFIG_KEY, - ISSUE_CONFIG_KEY, - PATCHSET_CONFIG_KEY, - CODEREVIEW_SERVER_CONFIG_KEY, - GERRIT_SQUASH_HASH_CONFIG_KEY, - ] - for prop in reset_suffixes: + def SetIssue(self, issue=None): + """Set this branch's issue. If issue isn't given, clears the issue.""" + assert self.GetBranch() + if issue: + issue = int(issue) + self._GitSetBranchConfigValue(ISSUE_CONFIG_KEY, str(issue)) + self.issue = issue + codereview_server = self.GetCodereviewServer() + if codereview_server: + self._GitSetBranchConfigValue(CODEREVIEW_SERVER_CONFIG_KEY, + codereview_server) + else: + # Reset all of these just to be clean. + reset_suffixes = [ + LAST_UPLOAD_HASH_CONFIG_KEY, + ISSUE_CONFIG_KEY, + PATCHSET_CONFIG_KEY, + CODEREVIEW_SERVER_CONFIG_KEY, + GERRIT_SQUASH_HASH_CONFIG_KEY, + ] + for prop in reset_suffixes: + try: + self._GitSetBranchConfigValue(prop, None) + except subprocess2.CalledProcessError: + pass + msg = RunGit(['log', '-1', '--format=%B']).strip() + if msg and git_footers.get_footer_change_id(msg): + print( + 'WARNING: The change patched into this branch has a Change-Id. ' + 'Removing it.') + RunGit([ + 'commit', '--amend', '-m', + git_footers.remove_footer(msg, 'Change-Id') + ]) + self.lookedup_issue = True + self.issue = None + self.patchset = None + + def GetAffectedFiles(self, + upstream: str, + end_commit: Optional[str] = None) -> Sequence[str]: + """Returns the list of affected files for the given commit range.""" try: - self._GitSetBranchConfigValue(prop, None) + return [ + f for _, f in scm.GIT.CaptureStatus( + settings.GetRoot(), upstream, end_commit=end_commit) + ] except subprocess2.CalledProcessError: - pass - msg = RunGit(['log', '-1', '--format=%B']).strip() - if msg and git_footers.get_footer_change_id(msg): - print('WARNING: The change patched into this branch has a Change-Id. ' - 'Removing it.') - RunGit(['commit', '--amend', '-m', - git_footers.remove_footer(msg, 'Change-Id')]) - self.lookedup_issue = True - self.issue = None - self.patchset = None + DieWithError( + ('\nFailed to diff against upstream branch %s\n\n' + 'This branch probably doesn\'t exist anymore. To reset the\n' + 'tracking branch, please run\n' + ' git branch --set-upstream-to origin/main %s\n' + 'or replace origin/main with the relevant branch') % + (upstream, self.GetBranch())) - def GetAffectedFiles(self, - upstream: str, - end_commit: Optional[str] = None) -> Sequence[str]: - """Returns the list of affected files for the given commit range.""" - try: - return [ - f for _, f in scm.GIT.CaptureStatus( - settings.GetRoot(), upstream, end_commit=end_commit) - ] - except subprocess2.CalledProcessError: - DieWithError( - ('\nFailed to diff against upstream branch %s\n\n' - 'This branch probably doesn\'t exist anymore. To reset the\n' - 'tracking branch, please run\n' - ' git branch --set-upstream-to origin/main %s\n' - 'or replace origin/main with the relevant branch') % - (upstream, self.GetBranch())) + def UpdateDescription(self, description, force=False): + assert self.GetIssue(), 'issue is required to update description' - def UpdateDescription(self, description, force=False): - assert self.GetIssue(), 'issue is required to update description' + if gerrit_util.HasPendingChangeEdit(self.GetGerritHost(), + self._GerritChangeIdentifier()): + if not force: + confirm_or_exit( + 'The description cannot be modified while the issue has a pending ' + 'unpublished edit. Either publish the edit in the Gerrit web UI ' + 'or delete it.\n\n', + action='delete the unpublished edit') - if gerrit_util.HasPendingChangeEdit( - self.GetGerritHost(), self._GerritChangeIdentifier()): - if not force: - confirm_or_exit( - 'The description cannot be modified while the issue has a pending ' - 'unpublished edit. Either publish the edit in the Gerrit web UI ' - 'or delete it.\n\n', action='delete the unpublished edit') + gerrit_util.DeletePendingChangeEdit(self.GetGerritHost(), + self._GerritChangeIdentifier()) + gerrit_util.SetCommitMessage(self.GetGerritHost(), + self._GerritChangeIdentifier(), + description, + notify='NONE') - gerrit_util.DeletePendingChangeEdit( - self.GetGerritHost(), self._GerritChangeIdentifier()) - gerrit_util.SetCommitMessage( - self.GetGerritHost(), self._GerritChangeIdentifier(), - description, notify='NONE') + self.description = description - self.description = description + def _GetCommonPresubmitArgs(self, verbose, upstream): + args = [ + '--root', + settings.GetRoot(), + '--upstream', + upstream, + ] - def _GetCommonPresubmitArgs(self, verbose, upstream): - args = [ - '--root', settings.GetRoot(), - '--upstream', upstream, - ] + args.extend(['--verbose'] * verbose) - args.extend(['--verbose'] * verbose) + remote, remote_branch = self.GetRemoteBranch() + target_ref = GetTargetRef(remote, remote_branch, None) + if settings.GetIsGerrit(): + args.extend(['--gerrit_url', self.GetCodereviewServer()]) + args.extend(['--gerrit_project', self.GetGerritProject()]) + args.extend(['--gerrit_branch', target_ref]) - remote, remote_branch = self.GetRemoteBranch() - target_ref = GetTargetRef(remote, remote_branch, None) - if settings.GetIsGerrit(): - args.extend(['--gerrit_url', self.GetCodereviewServer()]) - args.extend(['--gerrit_project', self.GetGerritProject()]) - args.extend(['--gerrit_branch', target_ref]) + author = self.GetAuthor() + issue = self.GetIssue() + patchset = self.GetPatchset() + if author: + args.extend(['--author', author]) + if issue: + args.extend(['--issue', str(issue)]) + if patchset: + args.extend(['--patchset', str(patchset)]) - author = self.GetAuthor() - issue = self.GetIssue() - patchset = self.GetPatchset() - if author: - args.extend(['--author', author]) - if issue: - args.extend(['--issue', str(issue)]) - if patchset: - args.extend(['--patchset', str(patchset)]) + return args - return args + def RunHook(self, + committing, + may_prompt, + verbose, + parallel, + upstream, + description, + all_files, + files=None, + resultdb=False, + realm=None): + """Calls sys.exit() if the hook fails; returns a HookResults otherwise.""" + args = self._GetCommonPresubmitArgs(verbose, upstream) + args.append('--commit' if committing else '--upload') + if may_prompt: + args.append('--may_prompt') + if parallel: + args.append('--parallel') + if all_files: + args.append('--all_files') + if files: + args.extend(files.split(';')) + args.append('--source_controlled_only') + if files or all_files: + args.append('--no_diffs') - def RunHook(self, - committing, - may_prompt, - verbose, - parallel, - upstream, - description, - all_files, - files=None, - resultdb=False, - realm=None): - """Calls sys.exit() if the hook fails; returns a HookResults otherwise.""" - args = self._GetCommonPresubmitArgs(verbose, upstream) - args.append('--commit' if committing else '--upload') - if may_prompt: - args.append('--may_prompt') - if parallel: - args.append('--parallel') - if all_files: - args.append('--all_files') - if files: - args.extend(files.split(';')) - args.append('--source_controlled_only') - if files or all_files: - args.append('--no_diffs') + if resultdb and not realm: + # TODO (crbug.com/1113463): store realm somewhere and look it up so + # it is not required to pass the realm flag + print( + 'Note: ResultDB reporting will NOT be performed because --realm' + ' was not specified. To enable ResultDB, please run the command' + ' again with the --realm argument to specify the LUCI realm.') - if resultdb and not realm: - # TODO (crbug.com/1113463): store realm somewhere and look it up so - # it is not required to pass the realm flag - print('Note: ResultDB reporting will NOT be performed because --realm' - ' was not specified. To enable ResultDB, please run the command' - ' again with the --realm argument to specify the LUCI realm.') + return self._RunPresubmit(args, + description, + resultdb=resultdb, + realm=realm) - return self._RunPresubmit(args, - description, - resultdb=resultdb, - realm=realm) + def _RunPresubmit(self, + args: Sequence[str], + description: str, + resultdb: bool = False, + realm: Optional[str] = None) -> Mapping[str, Any]: + args = list(args) - def _RunPresubmit(self, - args: Sequence[str], - description: str, - resultdb: bool = False, - realm: Optional[str] = None) -> Mapping[str, Any]: - args = list(args) + with gclient_utils.temporary_file() as description_file: + with gclient_utils.temporary_file() as json_output: + gclient_utils.FileWrite(description_file, description) + args.extend(['--json_output', json_output]) + args.extend(['--description_file', description_file]) + start = time_time() + cmd = ['vpython3', PRESUBMIT_SUPPORT] + args + if resultdb and realm: + cmd = ['rdb', 'stream', '-new', '-realm', realm, '--'] + cmd - with gclient_utils.temporary_file() as description_file: - with gclient_utils.temporary_file() as json_output: - gclient_utils.FileWrite(description_file, description) - args.extend(['--json_output', json_output]) - args.extend(['--description_file', description_file]) - start = time_time() - cmd = ['vpython3', PRESUBMIT_SUPPORT] + args - if resultdb and realm: - cmd = ['rdb', 'stream', '-new', '-realm', realm, '--'] + cmd + p = subprocess2.Popen(cmd) + exit_code = p.wait() - p = subprocess2.Popen(cmd) - exit_code = p.wait() + metrics.collector.add_repeated( + 'sub_commands', { + 'command': 'presubmit', + 'execution_time': time_time() - start, + 'exit_code': exit_code, + }) - metrics.collector.add_repeated('sub_commands', { - 'command': 'presubmit', - 'execution_time': time_time() - start, - 'exit_code': exit_code, - }) + if exit_code: + sys.exit(exit_code) - if exit_code: - sys.exit(exit_code) + json_results = gclient_utils.FileRead(json_output) + return json.loads(json_results) - json_results = gclient_utils.FileRead(json_output) - return json.loads(json_results) + def RunPostUploadHook(self, verbose, upstream, description): + args = self._GetCommonPresubmitArgs(verbose, upstream) + args.append('--post_upload') - def RunPostUploadHook(self, verbose, upstream, description): - args = self._GetCommonPresubmitArgs(verbose, upstream) - args.append('--post_upload') + with gclient_utils.temporary_file() as description_file: + gclient_utils.FileWrite(description_file, description) + args.extend(['--description_file', description_file]) + subprocess2.Popen(['vpython3', PRESUBMIT_SUPPORT] + args).wait() - with gclient_utils.temporary_file() as description_file: - gclient_utils.FileWrite(description_file, description) - args.extend(['--description_file', description_file]) - subprocess2.Popen(['vpython3', PRESUBMIT_SUPPORT] + args).wait() + def _GetDescriptionForUpload(self, options: optparse.Values, + git_diff_args: Sequence[str], + files: Sequence[str]) -> ChangeDescription: + """Get description message for upload.""" + if self.GetIssue(): + description = self.FetchDescription() + elif options.message: + description = options.message + else: + description = _create_description_from_log(git_diff_args) + if options.title and options.squash: + description = options.title + '\n\n' + description - def _GetDescriptionForUpload(self, options: optparse.Values, - git_diff_args: Sequence[str], - files: Sequence[str]) -> ChangeDescription: - """Get description message for upload.""" - if self.GetIssue(): - description = self.FetchDescription() - elif options.message: - description = options.message - else: - description = _create_description_from_log(git_diff_args) - if options.title and options.squash: - description = options.title + '\n\n' + description + bug = options.bug + fixed = options.fixed + if not self.GetIssue(): + # Extract bug number from branch name, but only if issue is being + # created. It must start with bug or fix, followed by _ or - and + # number. Optionally, it may contain _ or - after number with + # arbitrary text. Examples: bug-123 bug_123 fix-123 + # fix-123-some-description + branch = self.GetBranch() + if branch is not None: + match = re.match( + r'^(?Pbug|fix(?:e[sd])?)[_-]?(?P\d+)([-_]|$)', + branch) + if not bug and not fixed and match: + if match.group('type') == 'bug': + bug = match.group('bugnum') + else: + fixed = match.group('bugnum') - bug = options.bug - fixed = options.fixed - if not self.GetIssue(): - # Extract bug number from branch name, but only if issue is being created. - # It must start with bug or fix, followed by _ or - and number. - # Optionally, it may contain _ or - after number with arbitrary text. - # Examples: - # bug-123 - # bug_123 - # fix-123 - # fix-123-some-description - branch = self.GetBranch() - if branch is not None: - match = re.match( - r'^(?Pbug|fix(?:e[sd])?)[_-]?(?P\d+)([-_]|$)', branch) - if not bug and not fixed and match: - if match.group('type') == 'bug': - bug = match.group('bugnum') - else: - fixed = match.group('bugnum') + change_description = ChangeDescription(description, bug, fixed) - change_description = ChangeDescription(description, bug, fixed) + # Fill gaps in OWNERS coverage to reviewers if requested. + if options.add_owners_to: + assert options.add_owners_to in ('R'), options.add_owners_to + status = self.owners_client.GetFilesApprovalStatus( + files, [], options.reviewers) + missing_files = [ + f for f in files + if status[f] == self._owners_client.INSUFFICIENT_REVIEWERS + ] + owners = self.owners_client.SuggestOwners( + missing_files, exclude=[self.GetAuthor()]) + assert isinstance(options.reviewers, list), options.reviewers + options.reviewers.extend(owners) - # Fill gaps in OWNERS coverage to reviewers if requested. - if options.add_owners_to: - assert options.add_owners_to in ('R'), options.add_owners_to - status = self.owners_client.GetFilesApprovalStatus( - files, [], options.reviewers) - missing_files = [ - f for f in files - if status[f] == self._owners_client.INSUFFICIENT_REVIEWERS - ] - owners = self.owners_client.SuggestOwners( - missing_files, exclude=[self.GetAuthor()]) - assert isinstance(options.reviewers, list), options.reviewers - options.reviewers.extend(owners) + # Set the reviewer list now so that presubmit checks can access it. + if options.reviewers: + change_description.update_reviewers(options.reviewers) - # Set the reviewer list now so that presubmit checks can access it. - if options.reviewers: - change_description.update_reviewers(options.reviewers) + return change_description - return change_description + def _GetTitleForUpload(self, options, multi_change_upload=False): + # type: (optparse.Values, Optional[bool]) -> str - def _GetTitleForUpload(self, options, multi_change_upload=False): - # type: (optparse.Values, Optional[bool]) -> str + # Getting titles for multipl commits is not supported so we return the + # default. + if not options.squash or multi_change_upload or options.title: + return options.title - # Getting titles for multipl commits is not supported so we return the - # default. - if not options.squash or multi_change_upload or options.title: - return options.title + # On first upload, patchset title is always this string, while + # options.title gets converted to first line of message. + if not self.GetIssue(): + return 'Initial upload' - # On first upload, patchset title is always this string, while options.title - # gets converted to first line of message. - if not self.GetIssue(): - return 'Initial upload' + # When uploading subsequent patchsets, options.message is taken as the + # title if options.title is not provided. + if options.message: + return options.message.strip() - # When uploading subsequent patchsets, options.message is taken as the title - # if options.title is not provided. - if options.message: - return options.message.strip() + # Use the subject of the last commit as title by default. + title = RunGit(['show', '-s', '--format=%s', 'HEAD']).strip() + if options.force or options.skip_title: + return title + user_title = gclient_utils.AskForData('Title for patchset [%s]: ' % + title) - # Use the subject of the last commit as title by default. - title = RunGit(['show', '-s', '--format=%s', 'HEAD']).strip() - if options.force or options.skip_title: - return title - user_title = gclient_utils.AskForData('Title for patchset [%s]: ' % title) + # Use the default title if the user confirms the default with a 'y'. + if user_title.lower() == 'y': + return title + return user_title or title - # Use the default title if the user confirms the default with a 'y'. - if user_title.lower() == 'y': - return title - return user_title or title + def _GetRefSpecOptions(self, + options: optparse.Values, + change_desc: ChangeDescription, + multi_change_upload: bool = False, + dogfood_path: bool = False) -> List[str]: + # Extra options that can be specified at push time. Doc: + # https://gerrit-review.googlesource.com/Documentation/user-upload.html + refspec_opts = [] - def _GetRefSpecOptions(self, - options: optparse.Values, - change_desc: ChangeDescription, - multi_change_upload: bool = False, - dogfood_path: bool = False) -> List[str]: - # Extra options that can be specified at push time. Doc: - # https://gerrit-review.googlesource.com/Documentation/user-upload.html - refspec_opts = [] + # By default, new changes are started in WIP mode, and subsequent + # patchsets don't send email. At any time, passing --send-mail or + # --send-email will mark the change ready and send email for that + # particular patch. + if options.send_mail: + refspec_opts.append('ready') + refspec_opts.append('notify=ALL') + elif (not self.GetIssue() and options.squash and not dogfood_path): + refspec_opts.append('wip') + else: + refspec_opts.append('notify=NONE') - # By default, new changes are started in WIP mode, and subsequent patchsets - # don't send email. At any time, passing --send-mail or --send-email will - # mark the change ready and send email for that particular patch. - if options.send_mail: - refspec_opts.append('ready') - refspec_opts.append('notify=ALL') - elif (not self.GetIssue() and options.squash and not dogfood_path): - refspec_opts.append('wip') - else: - refspec_opts.append('notify=NONE') + # TODO(tandrii): options.message should be posted as a comment if + # --send-mail or --send-email is set on non-initial upload as Rietveld + # used to do it. - # TODO(tandrii): options.message should be posted as a comment if - # --send-mail or --send-email is set on non-initial upload as Rietveld used - # to do it. + # Set options.title in case user was prompted in _GetTitleForUpload and + # _CMDUploadChange needs to be called again. + options.title = self._GetTitleForUpload( + options, multi_change_upload=multi_change_upload) - # Set options.title in case user was prompted in _GetTitleForUpload and - # _CMDUploadChange needs to be called again. - options.title = self._GetTitleForUpload( - options, multi_change_upload=multi_change_upload) + if options.title: + # Punctuation and whitespace in |title| must be percent-encoded. + refspec_opts.append( + 'm=' + gerrit_util.PercentEncodeForGitRef(options.title)) - if options.title: - # Punctuation and whitespace in |title| must be percent-encoded. - refspec_opts.append('m=' + - gerrit_util.PercentEncodeForGitRef(options.title)) + if options.private: + refspec_opts.append('private') - if options.private: - refspec_opts.append('private') + if options.topic: + # Documentation on Gerrit topics is here: + # https://gerrit-review.googlesource.com/Documentation/user-upload.html#topic + refspec_opts.append('topic=%s' % options.topic) - if options.topic: - # Documentation on Gerrit topics is here: - # https://gerrit-review.googlesource.com/Documentation/user-upload.html#topic - refspec_opts.append('topic=%s' % options.topic) + if options.enable_auto_submit: + refspec_opts.append('l=Auto-Submit+1') + if options.set_bot_commit: + refspec_opts.append('l=Bot-Commit+1') + if options.use_commit_queue: + refspec_opts.append('l=Commit-Queue+2') + elif options.cq_dry_run: + refspec_opts.append('l=Commit-Queue+1') - if options.enable_auto_submit: - refspec_opts.append('l=Auto-Submit+1') - if options.set_bot_commit: - refspec_opts.append('l=Bot-Commit+1') - if options.use_commit_queue: - refspec_opts.append('l=Commit-Queue+2') - elif options.cq_dry_run: - refspec_opts.append('l=Commit-Queue+1') + if change_desc.get_reviewers(tbr_only=True): + score = gerrit_util.GetCodeReviewTbrScore(self.GetGerritHost(), + self.GetGerritProject()) + refspec_opts.append('l=Code-Review+%s' % score) - if change_desc.get_reviewers(tbr_only=True): - score = gerrit_util.GetCodeReviewTbrScore(self.GetGerritHost(), - self.GetGerritProject()) - refspec_opts.append('l=Code-Review+%s' % score) + # Gerrit sorts hashtags, so order is not important. + hashtags = {change_desc.sanitize_hash_tag(t) for t in options.hashtags} + # We check GetIssue because we only add hashtags from the + # description on the first upload. + # TODO(b/265929888): When we fully launch the new path: + # 1) remove fetching hashtags from description alltogether + # 2) Or use descrtiption hashtags for: + # `not (self.GetIssue() and multi_change_upload)` + # 3) Or enabled change description tags for multi and single changes + # by adding them post `git push`. + if not (self.GetIssue() and dogfood_path): + hashtags.update(change_desc.get_hash_tags()) + refspec_opts.extend(['hashtag=%s' % t for t in hashtags]) - # Gerrit sorts hashtags, so order is not important. - hashtags = {change_desc.sanitize_hash_tag(t) for t in options.hashtags} - # We check GetIssue because we only add hashtags from the - # description on the first upload. - # TODO(b/265929888): When we fully launch the new path: - # 1) remove fetching hashtags from description alltogether - # 2) Or use descrtiption hashtags for: - # `not (self.GetIssue() and multi_change_upload)` - # 3) Or enabled change description tags for multi and single changes - # by adding them post `git push`. - if not (self.GetIssue() and dogfood_path): - hashtags.update(change_desc.get_hash_tags()) - refspec_opts.extend(['hashtag=%s' % t for t in hashtags]) + # Note: Reviewers, and ccs are handled individually for each + # branch/change. + return refspec_opts - # Note: Reviewers, and ccs are handled individually for each - # branch/change. - return refspec_opts - - def PrepareSquashedCommit(self, - options: optparse.Values, - parent: str, - orig_parent: str, - end_commit: Optional[str] = None) -> _NewUpload: - """Create a squashed commit to upload. + def PrepareSquashedCommit(self, + options: optparse.Values, + parent: str, + orig_parent: str, + end_commit: Optional[str] = None) -> _NewUpload: + """Create a squashed commit to upload. Args: @@ -1886,439 +1944,451 @@ class Changelist(object): end_commit: The commit to use as the end of the new squashed commit. """ - if end_commit is None: - end_commit = RunGit(['rev-parse', self.branchref]).strip() + if end_commit is None: + end_commit = RunGit(['rev-parse', self.branchref]).strip() - reviewers, ccs, change_desc = self._PrepareChange(options, orig_parent, - end_commit) - latest_tree = RunGit(['rev-parse', end_commit + ':']).strip() - with gclient_utils.temporary_file() as desc_tempfile: - gclient_utils.FileWrite(desc_tempfile, change_desc.description) - commit_to_push = RunGit( - ['commit-tree', latest_tree, '-p', parent, '-F', - desc_tempfile]).strip() + reviewers, ccs, change_desc = self._PrepareChange( + options, orig_parent, end_commit) + latest_tree = RunGit(['rev-parse', end_commit + ':']).strip() + with gclient_utils.temporary_file() as desc_tempfile: + gclient_utils.FileWrite(desc_tempfile, change_desc.description) + commit_to_push = RunGit( + ['commit-tree', latest_tree, '-p', parent, '-F', + desc_tempfile]).strip() - # Gerrit may or may not update fast enough to return the correct patchset - # number after we push. Get the pre-upload patchset and increment later. - prev_patchset = self.GetMostRecentPatchset(update=False) or 0 - return _NewUpload(reviewers, ccs, commit_to_push, end_commit, parent, - change_desc, prev_patchset) + # Gerrit may or may not update fast enough to return the correct + # patchset number after we push. Get the pre-upload patchset and + # increment later. + prev_patchset = self.GetMostRecentPatchset(update=False) or 0 + return _NewUpload(reviewers, ccs, commit_to_push, end_commit, parent, + change_desc, prev_patchset) - def PrepareCherryPickSquashedCommit(self, options: optparse.Values, - parent: str) -> _NewUpload: - """Create a commit cherry-picked on parent to push.""" + def PrepareCherryPickSquashedCommit(self, options: optparse.Values, + parent: str) -> _NewUpload: + """Create a commit cherry-picked on parent to push.""" - # The `parent` is what we will cherry-pick on top of. - # The `cherry_pick_base` is the beginning range of what - # we are cherry-picking. - cherry_pick_base = self.GetCommonAncestorWithUpstream() - reviewers, ccs, change_desc = self._PrepareChange(options, cherry_pick_base, - self.branchref) + # The `parent` is what we will cherry-pick on top of. + # The `cherry_pick_base` is the beginning range of what + # we are cherry-picking. + cherry_pick_base = self.GetCommonAncestorWithUpstream() + reviewers, ccs, change_desc = self._PrepareChange( + options, cherry_pick_base, self.branchref) - new_upload_hash = RunGit(['rev-parse', self.branchref]).strip() - latest_tree = RunGit(['rev-parse', self.branchref + ':']).strip() - with gclient_utils.temporary_file() as desc_tempfile: - gclient_utils.FileWrite(desc_tempfile, change_desc.description) - commit_to_cp = RunGit([ - 'commit-tree', latest_tree, '-p', cherry_pick_base, '-F', - desc_tempfile - ]).strip() + new_upload_hash = RunGit(['rev-parse', self.branchref]).strip() + latest_tree = RunGit(['rev-parse', self.branchref + ':']).strip() + with gclient_utils.temporary_file() as desc_tempfile: + gclient_utils.FileWrite(desc_tempfile, change_desc.description) + commit_to_cp = RunGit([ + 'commit-tree', latest_tree, '-p', cherry_pick_base, '-F', + desc_tempfile + ]).strip() - RunGit(['checkout', '-q', parent]) - ret, _out = RunGitWithCode(['cherry-pick', commit_to_cp]) - if ret: - RunGit(['cherry-pick', '--abort']) - RunGit(['checkout', '-q', self.branch]) - DieWithError('Could not cleanly cherry-pick') + RunGit(['checkout', '-q', parent]) + ret, _out = RunGitWithCode(['cherry-pick', commit_to_cp]) + if ret: + RunGit(['cherry-pick', '--abort']) + RunGit(['checkout', '-q', self.branch]) + DieWithError('Could not cleanly cherry-pick') - commit_to_push = RunGit(['rev-parse', 'HEAD']).strip() - RunGit(['checkout', '-q', self.branch]) + commit_to_push = RunGit(['rev-parse', 'HEAD']).strip() + RunGit(['checkout', '-q', self.branch]) - # Gerrit may or may not update fast enough to return the correct patchset - # number after we push. Get the pre-upload patchset and increment later. - prev_patchset = self.GetMostRecentPatchset(update=False) or 0 - return _NewUpload(reviewers, ccs, commit_to_push, new_upload_hash, - cherry_pick_base, change_desc, prev_patchset) + # Gerrit may or may not update fast enough to return the correct + # patchset number after we push. Get the pre-upload patchset and + # increment later. + prev_patchset = self.GetMostRecentPatchset(update=False) or 0 + return _NewUpload(reviewers, ccs, commit_to_push, new_upload_hash, + cherry_pick_base, change_desc, prev_patchset) - def _PrepareChange( - self, options: optparse.Values, parent: str, end_commit: str - ) -> Tuple[Sequence[str], Sequence[str], ChangeDescription]: - """Prepares the change to be uploaded.""" - self.EnsureCanUploadPatchset(options.force) + def _PrepareChange( + self, options: optparse.Values, parent: str, end_commit: str + ) -> Tuple[Sequence[str], Sequence[str], ChangeDescription]: + """Prepares the change to be uploaded.""" + self.EnsureCanUploadPatchset(options.force) - files = self.GetAffectedFiles(parent, end_commit=end_commit) - change_desc = self._GetDescriptionForUpload(options, [parent, end_commit], - files) + files = self.GetAffectedFiles(parent, end_commit=end_commit) + change_desc = self._GetDescriptionForUpload(options, + [parent, end_commit], files) - watchlist = watchlists.Watchlists(settings.GetRoot()) - self.ExtendCC(watchlist.GetWatchersForPaths(files)) - if not options.bypass_hooks: - hook_results = self.RunHook(committing=False, - may_prompt=not options.force, - verbose=options.verbose, - parallel=options.parallel, - upstream=parent, - description=change_desc.description, - all_files=False) - self.ExtendCC(hook_results['more_cc']) + watchlist = watchlists.Watchlists(settings.GetRoot()) + self.ExtendCC(watchlist.GetWatchersForPaths(files)) + if not options.bypass_hooks: + hook_results = self.RunHook(committing=False, + may_prompt=not options.force, + verbose=options.verbose, + parallel=options.parallel, + upstream=parent, + description=change_desc.description, + all_files=False) + self.ExtendCC(hook_results['more_cc']) - # Update the change description and ensure we have a Change Id. - if self.GetIssue(): - if options.edit_description: - change_desc.prompt() - change_detail = self._GetChangeDetail(['CURRENT_REVISION']) - change_id = change_detail['change_id'] - change_desc.ensure_change_id(change_id) + # Update the change description and ensure we have a Change Id. + if self.GetIssue(): + if options.edit_description: + change_desc.prompt() + change_detail = self._GetChangeDetail(['CURRENT_REVISION']) + change_id = change_detail['change_id'] + change_desc.ensure_change_id(change_id) - else: # No change issue. First time uploading - if not options.force and not options.message_file: - change_desc.prompt() + else: # No change issue. First time uploading + if not options.force and not options.message_file: + change_desc.prompt() - # Check if user added a change_id in the descripiton. - change_ids = git_footers.get_footer_change_id(change_desc.description) - if len(change_ids) == 1: - change_id = change_ids[0] - else: - change_id = GenerateGerritChangeId(change_desc.description) - change_desc.ensure_change_id(change_id) + # Check if user added a change_id in the descripiton. + change_ids = git_footers.get_footer_change_id( + change_desc.description) + if len(change_ids) == 1: + change_id = change_ids[0] + else: + change_id = GenerateGerritChangeId(change_desc.description) + change_desc.ensure_change_id(change_id) - if options.preserve_tryjobs: - change_desc.set_preserve_tryjobs() + if options.preserve_tryjobs: + change_desc.set_preserve_tryjobs() - SaveDescriptionBackup(change_desc) + SaveDescriptionBackup(change_desc) - # Add ccs - ccs = [] - # Add default, watchlist, presubmit ccs if this is the initial upload - # and CL is not private and auto-ccing has not been disabled. - if not options.private and not options.no_autocc and not self.GetIssue(): - ccs = self.GetCCList().split(',') - if len(ccs) > 100: - lsc = ('https://chromium.googlesource.com/chromium/src/+/HEAD/docs/' - 'process/lsc/lsc_workflow.md') - print('WARNING: This will auto-CC %s users.' % len(ccs)) - print('LSC may be more appropriate: %s' % lsc) - print('You can also use the --no-autocc flag to disable auto-CC.') - confirm_or_exit(action='continue') + # Add ccs + ccs = [] + # Add default, watchlist, presubmit ccs if this is the initial upload + # and CL is not private and auto-ccing has not been disabled. + if not options.private and not options.no_autocc and not self.GetIssue( + ): + ccs = self.GetCCList().split(',') + if len(ccs) > 100: + lsc = ( + 'https://chromium.googlesource.com/chromium/src/+/HEAD/docs/' + 'process/lsc/lsc_workflow.md') + print('WARNING: This will auto-CC %s users.' % len(ccs)) + print('LSC may be more appropriate: %s' % lsc) + print( + 'You can also use the --no-autocc flag to disable auto-CC.') + confirm_or_exit(action='continue') - # Add ccs from the --cc flag. - if options.cc: - ccs.extend(options.cc) + # Add ccs from the --cc flag. + if options.cc: + ccs.extend(options.cc) - ccs = [email.strip() for email in ccs if email.strip()] - if change_desc.get_cced(): - ccs.extend(change_desc.get_cced()) + ccs = [email.strip() for email in ccs if email.strip()] + if change_desc.get_cced(): + ccs.extend(change_desc.get_cced()) - return change_desc.get_reviewers(), ccs, change_desc + return change_desc.get_reviewers(), ccs, change_desc - def PostUploadUpdates(self, options: optparse.Values, new_upload: _NewUpload, - change_number: str) -> None: - """Makes necessary post upload changes to the local and remote cl.""" - if not self.GetIssue(): - self.SetIssue(change_number) + def PostUploadUpdates(self, options: optparse.Values, + new_upload: _NewUpload, change_number: str) -> None: + """Makes necessary post upload changes to the local and remote cl.""" + if not self.GetIssue(): + self.SetIssue(change_number) - self.SetPatchset(new_upload.prev_patchset + 1) + self.SetPatchset(new_upload.prev_patchset + 1) - self._GitSetBranchConfigValue(GERRIT_SQUASH_HASH_CONFIG_KEY, - new_upload.commit_to_push) - self._GitSetBranchConfigValue(LAST_UPLOAD_HASH_CONFIG_KEY, - new_upload.new_last_uploaded_commit) + self._GitSetBranchConfigValue(GERRIT_SQUASH_HASH_CONFIG_KEY, + new_upload.commit_to_push) + self._GitSetBranchConfigValue(LAST_UPLOAD_HASH_CONFIG_KEY, + new_upload.new_last_uploaded_commit) - if settings.GetRunPostUploadHook(): - self.RunPostUploadHook(options.verbose, new_upload.parent, - new_upload.change_desc.description) + if settings.GetRunPostUploadHook(): + self.RunPostUploadHook(options.verbose, new_upload.parent, + new_upload.change_desc.description) - if new_upload.reviewers or new_upload.ccs: - gerrit_util.AddReviewers(self.GetGerritHost(), - self._GerritChangeIdentifier(), - reviewers=new_upload.reviewers, - ccs=new_upload.ccs, - notify=bool(options.send_mail)) + if new_upload.reviewers or new_upload.ccs: + gerrit_util.AddReviewers(self.GetGerritHost(), + self._GerritChangeIdentifier(), + reviewers=new_upload.reviewers, + ccs=new_upload.ccs, + notify=bool(options.send_mail)) - def CMDUpload(self, options, git_diff_args, orig_args): - """Uploads a change to codereview.""" - custom_cl_base = None - if git_diff_args: - custom_cl_base = base_branch = git_diff_args[0] - else: - if self.GetBranch() is None: - DieWithError('Can\'t upload from detached HEAD state. Get on a branch!') + def CMDUpload(self, options, git_diff_args, orig_args): + """Uploads a change to codereview.""" + custom_cl_base = None + if git_diff_args: + custom_cl_base = base_branch = git_diff_args[0] + else: + if self.GetBranch() is None: + DieWithError( + 'Can\'t upload from detached HEAD state. Get on a branch!') - # Default to diffing against common ancestor of upstream branch - base_branch = self.GetCommonAncestorWithUpstream() - git_diff_args = [base_branch, 'HEAD'] + # Default to diffing against common ancestor of upstream branch + base_branch = self.GetCommonAncestorWithUpstream() + git_diff_args = [base_branch, 'HEAD'] - # Fast best-effort checks to abort before running potentially expensive - # hooks if uploading is likely to fail anyway. Passing these checks does - # not guarantee that uploading will not fail. - self.EnsureAuthenticated(force=options.force) - self.EnsureCanUploadPatchset(force=options.force) + # Fast best-effort checks to abort before running potentially expensive + # hooks if uploading is likely to fail anyway. Passing these checks does + # not guarantee that uploading will not fail. + self.EnsureAuthenticated(force=options.force) + self.EnsureCanUploadPatchset(force=options.force) - print(f'Processing {_GetCommitCountSummary(*git_diff_args)}...') + print(f'Processing {_GetCommitCountSummary(*git_diff_args)}...') - # Apply watchlists on upload. - watchlist = watchlists.Watchlists(settings.GetRoot()) - files = self.GetAffectedFiles(base_branch) - if not options.bypass_watchlists: - self.ExtendCC(watchlist.GetWatchersForPaths(files)) + # Apply watchlists on upload. + watchlist = watchlists.Watchlists(settings.GetRoot()) + files = self.GetAffectedFiles(base_branch) + if not options.bypass_watchlists: + self.ExtendCC(watchlist.GetWatchersForPaths(files)) - change_desc = self._GetDescriptionForUpload(options, git_diff_args, files) - if not options.bypass_hooks: - hook_results = self.RunHook(committing=False, - may_prompt=not options.force, - verbose=options.verbose, - parallel=options.parallel, - upstream=base_branch, - description=change_desc.description, - all_files=False) - self.ExtendCC(hook_results['more_cc']) + change_desc = self._GetDescriptionForUpload(options, git_diff_args, + files) + if not options.bypass_hooks: + hook_results = self.RunHook(committing=False, + may_prompt=not options.force, + verbose=options.verbose, + parallel=options.parallel, + upstream=base_branch, + description=change_desc.description, + all_files=False) + self.ExtendCC(hook_results['more_cc']) - print_stats(git_diff_args) - ret = self.CMDUploadChange( - options, git_diff_args, custom_cl_base, change_desc) - if not ret: - if self.GetBranch() is not None: - self._GitSetBranchConfigValue( - LAST_UPLOAD_HASH_CONFIG_KEY, - scm.GIT.ResolveCommit(settings.GetRoot(), 'HEAD')) - # Run post upload hooks, if specified. - if settings.GetRunPostUploadHook(): - self.RunPostUploadHook(options.verbose, base_branch, - change_desc.description) + print_stats(git_diff_args) + ret = self.CMDUploadChange(options, git_diff_args, custom_cl_base, + change_desc) + if not ret: + if self.GetBranch() is not None: + self._GitSetBranchConfigValue( + LAST_UPLOAD_HASH_CONFIG_KEY, + scm.GIT.ResolveCommit(settings.GetRoot(), 'HEAD')) + # Run post upload hooks, if specified. + if settings.GetRunPostUploadHook(): + self.RunPostUploadHook(options.verbose, base_branch, + change_desc.description) - # Upload all dependencies if specified. - if options.dependencies: - print() - print('--dependencies has been specified.') - print('All dependent local branches will be re-uploaded.') - print() - # Remove the dependencies flag from args so that we do not end up in a - # loop. - orig_args.remove('--dependencies') - ret = upload_branch_deps(self, orig_args, options.force) - return ret + # Upload all dependencies if specified. + if options.dependencies: + print() + print('--dependencies has been specified.') + print('All dependent local branches will be re-uploaded.') + print() + # Remove the dependencies flag from args so that we do not end + # up in a loop. + orig_args.remove('--dependencies') + ret = upload_branch_deps(self, orig_args, options.force) + return ret - def SetCQState(self, new_state): - """Updates the CQ state for the latest patchset. + def SetCQState(self, new_state): + """Updates the CQ state for the latest patchset. Issue must have been already uploaded and known. """ - assert new_state in _CQState.ALL_STATES - assert self.GetIssue() - try: - vote_map = { - _CQState.NONE: 0, - _CQState.DRY_RUN: 1, - _CQState.COMMIT: 2, - } - labels = {'Commit-Queue': vote_map[new_state]} - notify = False if new_state == _CQState.DRY_RUN else None - gerrit_util.SetReview( - self.GetGerritHost(), self._GerritChangeIdentifier(), - labels=labels, notify=notify) - return 0 - except KeyboardInterrupt: - raise - except: - print('WARNING: Failed to %s.\n' - 'Either:\n' - ' * Your project has no CQ,\n' - ' * You don\'t have permission to change the CQ state,\n' - ' * There\'s a bug in this code (see stack trace below).\n' - 'Consider specifying which bots to trigger manually or asking your ' - 'project owners for permissions or contacting Chrome Infra at:\n' - 'https://www.chromium.org/infra\n\n' % - ('cancel CQ' if new_state == _CQState.NONE else 'trigger CQ')) - # Still raise exception so that stack trace is printed. - raise + assert new_state in _CQState.ALL_STATES + assert self.GetIssue() + try: + vote_map = { + _CQState.NONE: 0, + _CQState.DRY_RUN: 1, + _CQState.COMMIT: 2, + } + labels = {'Commit-Queue': vote_map[new_state]} + notify = False if new_state == _CQState.DRY_RUN else None + gerrit_util.SetReview(self.GetGerritHost(), + self._GerritChangeIdentifier(), + labels=labels, + notify=notify) + return 0 + except KeyboardInterrupt: + raise + except: + print( + 'WARNING: Failed to %s.\n' + 'Either:\n' + ' * Your project has no CQ,\n' + ' * You don\'t have permission to change the CQ state,\n' + ' * There\'s a bug in this code (see stack trace below).\n' + 'Consider specifying which bots to trigger manually or asking your ' + 'project owners for permissions or contacting Chrome Infra at:\n' + 'https://www.chromium.org/infra\n\n' % + ('cancel CQ' if new_state == _CQState.NONE else 'trigger CQ')) + # Still raise exception so that stack trace is printed. + raise - def GetGerritHost(self): - # Lazy load of configs. - self.GetCodereviewServer() + def GetGerritHost(self): + # Lazy load of configs. + self.GetCodereviewServer() - if self._gerrit_host and '.' not in self._gerrit_host: - # Abbreviated domain like "chromium" instead of chromium.googlesource.com. - parsed = urllib.parse.urlparse(self.GetRemoteUrl()) - if parsed.scheme == 'sso': - self._gerrit_host = '%s.googlesource.com' % self._gerrit_host - self._gerrit_server = 'https://%s' % self._gerrit_host + if self._gerrit_host and '.' not in self._gerrit_host: + # Abbreviated domain like "chromium" instead of + # chromium.googlesource.com. + parsed = urllib.parse.urlparse(self.GetRemoteUrl()) + if parsed.scheme == 'sso': + self._gerrit_host = '%s.googlesource.com' % self._gerrit_host + self._gerrit_server = 'https://%s' % self._gerrit_host - return self._gerrit_host + return self._gerrit_host - def _GetGitHost(self): - """Returns git host to be used when uploading change to Gerrit.""" - remote_url = self.GetRemoteUrl() - if not remote_url: - return None - return urllib.parse.urlparse(remote_url).netloc + def _GetGitHost(self): + """Returns git host to be used when uploading change to Gerrit.""" + remote_url = self.GetRemoteUrl() + if not remote_url: + return None + return urllib.parse.urlparse(remote_url).netloc - def GetCodereviewServer(self): - if not self._gerrit_server: - # If we're on a branch then get the server potentially associated - # with that branch. - if self.GetIssue() and self.GetBranch(): - self._gerrit_server = self._GitGetBranchConfigValue( - CODEREVIEW_SERVER_CONFIG_KEY) - if self._gerrit_server: - self._gerrit_host = urllib.parse.urlparse(self._gerrit_server).netloc - if not self._gerrit_server: - url = urllib.parse.urlparse(self.GetRemoteUrl()) - parts = url.netloc.split('.') + def GetCodereviewServer(self): + if not self._gerrit_server: + # If we're on a branch then get the server potentially associated + # with that branch. + if self.GetIssue() and self.GetBranch(): + self._gerrit_server = self._GitGetBranchConfigValue( + CODEREVIEW_SERVER_CONFIG_KEY) + if self._gerrit_server: + self._gerrit_host = urllib.parse.urlparse( + self._gerrit_server).netloc + if not self._gerrit_server: + url = urllib.parse.urlparse(self.GetRemoteUrl()) + parts = url.netloc.split('.') - # We assume repo to be hosted on Gerrit, and hence Gerrit server - # has "-review" suffix for lowest level subdomain. - parts[0] = parts[0] + '-review' + # We assume repo to be hosted on Gerrit, and hence Gerrit server + # has "-review" suffix for lowest level subdomain. + parts[0] = parts[0] + '-review' - if url.scheme == 'sso' and len(parts) == 1: - # sso:// uses abbreivated hosts, eg. sso://chromium instead of - # chromium.googlesource.com. Hence, for code review server, they need - # to be expanded. - parts[0] += '.googlesource.com' + if url.scheme == 'sso' and len(parts) == 1: + # sso:// uses abbreivated hosts, eg. sso://chromium instead + # of chromium.googlesource.com. Hence, for code review + # server, they need to be expanded. + parts[0] += '.googlesource.com' - self._gerrit_host = '.'.join(parts) - self._gerrit_server = 'https://%s' % self._gerrit_host - return self._gerrit_server + self._gerrit_host = '.'.join(parts) + self._gerrit_server = 'https://%s' % self._gerrit_host + return self._gerrit_server - def GetGerritProject(self): - """Returns Gerrit project name based on remote git URL.""" - remote_url = self.GetRemoteUrl() - if remote_url is None: - logging.warning('can\'t detect Gerrit project.') - return None - project = urllib.parse.urlparse(remote_url).path.strip('/') - if project.endswith('.git'): - project = project[:-len('.git')] - # *.googlesource.com hosts ensure that Git/Gerrit projects don't start with - # 'a/' prefix, because 'a/' prefix is used to force authentication in - # gitiles/git-over-https protocol. E.g., - # https://chromium.googlesource.com/a/v8/v8 refers to the same repo/project - # as - # https://chromium.googlesource.com/v8/v8 - if project.startswith('a/'): - project = project[len('a/'):] - return project + def GetGerritProject(self): + """Returns Gerrit project name based on remote git URL.""" + remote_url = self.GetRemoteUrl() + if remote_url is None: + logging.warning('can\'t detect Gerrit project.') + return None + project = urllib.parse.urlparse(remote_url).path.strip('/') + if project.endswith('.git'): + project = project[:-len('.git')] + # *.googlesource.com hosts ensure that Git/Gerrit projects don't start + # with 'a/' prefix, because 'a/' prefix is used to force authentication + # in gitiles/git-over-https protocol. E.g., + # https://chromium.googlesource.com/a/v8/v8 refers to the same + # repo/project as https://chromium.googlesource.com/v8/v8 + if project.startswith('a/'): + project = project[len('a/'):] + return project - def _GerritChangeIdentifier(self): - """Handy method for gerrit_util.ChangeIdentifier for a given CL. + def _GerritChangeIdentifier(self): + """Handy method for gerrit_util.ChangeIdentifier for a given CL. Not to be confused by value of "Change-Id:" footer. If Gerrit project can be determined, this will speed up Gerrit HTTP API RPC. """ - project = self.GetGerritProject() - if project: - return gerrit_util.ChangeIdentifier(project, self.GetIssue()) - # Fall back on still unique, but less efficient change number. - return str(self.GetIssue()) + project = self.GetGerritProject() + if project: + return gerrit_util.ChangeIdentifier(project, self.GetIssue()) + # Fall back on still unique, but less efficient change number. + return str(self.GetIssue()) - def EnsureAuthenticated(self, force, refresh=None): - """Best effort check that user is authenticated with Gerrit server.""" - if settings.GetGerritSkipEnsureAuthenticated(): - # For projects with unusual authentication schemes. - # See http://crbug.com/603378. - return + def EnsureAuthenticated(self, force, refresh=None): + """Best effort check that user is authenticated with Gerrit server.""" + if settings.GetGerritSkipEnsureAuthenticated(): + # For projects with unusual authentication schemes. + # See http://crbug.com/603378. + return - # Check presence of cookies only if using cookies-based auth method. - cookie_auth = gerrit_util.Authenticator.get() - if not isinstance(cookie_auth, gerrit_util.CookiesAuthenticator): - return + # Check presence of cookies only if using cookies-based auth method. + cookie_auth = gerrit_util.Authenticator.get() + if not isinstance(cookie_auth, gerrit_util.CookiesAuthenticator): + return - remote_url = self.GetRemoteUrl() - if remote_url is None: - logging.warning('invalid remote') - return - if urllib.parse.urlparse(remote_url).scheme not in ['https', 'sso']: - logging.warning( - 'Ignoring branch %(branch)s with non-https/sso remote ' - '%(remote)s', { - 'branch': self.branch, - 'remote': self.GetRemoteUrl() - }) - return + remote_url = self.GetRemoteUrl() + if remote_url is None: + logging.warning('invalid remote') + return + if urllib.parse.urlparse(remote_url).scheme not in ['https', 'sso']: + logging.warning( + 'Ignoring branch %(branch)s with non-https/sso remote ' + '%(remote)s', { + 'branch': self.branch, + 'remote': self.GetRemoteUrl() + }) + return - # Lazy-loader to identify Gerrit and Git hosts. - self.GetCodereviewServer() - git_host = self._GetGitHost() - assert self._gerrit_server and self._gerrit_host and git_host + # Lazy-loader to identify Gerrit and Git hosts. + self.GetCodereviewServer() + git_host = self._GetGitHost() + assert self._gerrit_server and self._gerrit_host and git_host - gerrit_auth = cookie_auth.get_auth_header(self._gerrit_host) - git_auth = cookie_auth.get_auth_header(git_host) - if gerrit_auth and git_auth: - if gerrit_auth == git_auth: - return - all_gsrc = cookie_auth.get_auth_header('d0esN0tEx1st.googlesource.com') - print( - 'WARNING: You have different credentials for Gerrit and git hosts:\n' - ' %s\n' - ' %s\n' - ' Consider running the following command:\n' - ' git cl creds-check\n' - ' %s\n' - ' %s' % - (git_host, self._gerrit_host, - ('Hint: delete creds for .googlesource.com' if all_gsrc else ''), - cookie_auth.get_new_password_message(git_host))) - if not force: - confirm_or_exit('If you know what you are doing', action='continue') - return + gerrit_auth = cookie_auth.get_auth_header(self._gerrit_host) + git_auth = cookie_auth.get_auth_header(git_host) + if gerrit_auth and git_auth: + if gerrit_auth == git_auth: + return + all_gsrc = cookie_auth.get_auth_header( + 'd0esN0tEx1st.googlesource.com') + print( + 'WARNING: You have different credentials for Gerrit and git hosts:\n' + ' %s\n' + ' %s\n' + ' Consider running the following command:\n' + ' git cl creds-check\n' + ' %s\n' + ' %s' % + (git_host, self._gerrit_host, + ('Hint: delete creds for .googlesource.com' if all_gsrc else + ''), cookie_auth.get_new_password_message(git_host))) + if not force: + confirm_or_exit('If you know what you are doing', + action='continue') + return - missing = ( - ([] if gerrit_auth else [self._gerrit_host]) + - ([] if git_auth else [git_host])) - DieWithError('Credentials for the following hosts are required:\n' - ' %s\n' - 'These are read from %s (or legacy %s)\n' - '%s' % ( - '\n '.join(missing), - cookie_auth.get_gitcookies_path(), - cookie_auth.get_netrc_path(), - cookie_auth.get_new_password_message(git_host))) + missing = (([] if gerrit_auth else [self._gerrit_host]) + + ([] if git_auth else [git_host])) + DieWithError('Credentials for the following hosts are required:\n' + ' %s\n' + 'These are read from %s (or legacy %s)\n' + '%s' % + ('\n '.join(missing), cookie_auth.get_gitcookies_path(), + cookie_auth.get_netrc_path(), + cookie_auth.get_new_password_message(git_host))) - def EnsureCanUploadPatchset(self, force): - if not self.GetIssue(): - return + def EnsureCanUploadPatchset(self, force): + if not self.GetIssue(): + return - status = self._GetChangeDetail()['status'] - if status == 'ABANDONED': - DieWithError( - 'Change %s has been abandoned, new uploads are not allowed' % - (self.GetIssueURL())) - if status == 'MERGED': - answer = gclient_utils.AskForData( - 'Change %s has been submitted, new uploads are not allowed. ' - 'Would you like to start a new change (Y/n)?' % self.GetIssueURL() - ).lower() - if answer not in ('y', ''): - DieWithError('New uploads are not allowed.') - self.SetIssue() - return + status = self._GetChangeDetail()['status'] + if status == 'ABANDONED': + DieWithError( + 'Change %s has been abandoned, new uploads are not allowed' % + (self.GetIssueURL())) + if status == 'MERGED': + answer = gclient_utils.AskForData( + 'Change %s has been submitted, new uploads are not allowed. ' + 'Would you like to start a new change (Y/n)?' % + self.GetIssueURL()).lower() + if answer not in ('y', ''): + DieWithError('New uploads are not allowed.') + self.SetIssue() + return - # TODO(vadimsh): For some reason the chunk of code below was skipped if - # 'is_gce' is True. I'm just refactoring it to be 'skip if not cookies'. - # Apparently this check is not very important? Otherwise get_auth_email - # could have been added to other implementations of Authenticator. - cookies_auth = gerrit_util.Authenticator.get() - if not isinstance(cookies_auth, gerrit_util.CookiesAuthenticator): - return + # TODO(vadimsh): For some reason the chunk of code below was skipped if + # 'is_gce' is True. I'm just refactoring it to be 'skip if not cookies'. + # Apparently this check is not very important? Otherwise get_auth_email + # could have been added to other implementations of Authenticator. + cookies_auth = gerrit_util.Authenticator.get() + if not isinstance(cookies_auth, gerrit_util.CookiesAuthenticator): + return - cookies_user = cookies_auth.get_auth_email(self.GetGerritHost()) - if self.GetIssueOwner() == cookies_user: - return - logging.debug('change %s owner is %s, cookies user is %s', - self.GetIssue(), self.GetIssueOwner(), cookies_user) - # Maybe user has linked accounts or something like that, - # so ask what Gerrit thinks of this user. - details = gerrit_util.GetAccountDetails(self.GetGerritHost(), 'self') - if details['email'] == self.GetIssueOwner(): - return - if not force: - print('WARNING: Change %s is owned by %s, but you authenticate to Gerrit ' - 'as %s.\n' - 'Uploading may fail due to lack of permissions.' % - (self.GetIssue(), self.GetIssueOwner(), details['email'])) - confirm_or_exit(action='upload') + cookies_user = cookies_auth.get_auth_email(self.GetGerritHost()) + if self.GetIssueOwner() == cookies_user: + return + logging.debug('change %s owner is %s, cookies user is %s', + self.GetIssue(), self.GetIssueOwner(), cookies_user) + # Maybe user has linked accounts or something like that, + # so ask what Gerrit thinks of this user. + details = gerrit_util.GetAccountDetails(self.GetGerritHost(), 'self') + if details['email'] == self.GetIssueOwner(): + return + if not force: + print( + 'WARNING: Change %s is owned by %s, but you authenticate to Gerrit ' + 'as %s.\n' + 'Uploading may fail due to lack of permissions.' % + (self.GetIssue(), self.GetIssueOwner(), details['email'])) + confirm_or_exit(action='upload') - def GetStatus(self): - """Applies a rough heuristic to give a simple summary of an issue's review + def GetStatus(self): + """Applies a rough heuristic to give a simple summary of an issue's review or CQ status, assuming adherence to a common workflow. Returns None if no issue for this branch, or one of the following keywords: @@ -2331,1021 +2401,1076 @@ class Changelist(object): * 'commit' - in the CQ * 'closed' - successfully submitted or abandoned """ - if not self.GetIssue(): - return None + if not self.GetIssue(): + return None - try: - data = self._GetChangeDetail([ - 'DETAILED_LABELS', 'CURRENT_REVISION', 'SUBMITTABLE']) - except GerritChangeNotExists: - return 'error' + try: + data = self._GetChangeDetail( + ['DETAILED_LABELS', 'CURRENT_REVISION', 'SUBMITTABLE']) + except GerritChangeNotExists: + return 'error' - if data['status'] in ('ABANDONED', 'MERGED'): - return 'closed' + if data['status'] in ('ABANDONED', 'MERGED'): + return 'closed' - cq_label = data['labels'].get('Commit-Queue', {}) - max_cq_vote = 0 - for vote in cq_label.get('all', []): - max_cq_vote = max(max_cq_vote, vote.get('value', 0)) - if max_cq_vote == 2: - return 'commit' - if max_cq_vote == 1: - return 'dry-run' + cq_label = data['labels'].get('Commit-Queue', {}) + max_cq_vote = 0 + for vote in cq_label.get('all', []): + max_cq_vote = max(max_cq_vote, vote.get('value', 0)) + if max_cq_vote == 2: + return 'commit' + if max_cq_vote == 1: + return 'dry-run' - if data['labels'].get('Code-Review', {}).get('approved'): - return 'lgtm' + if data['labels'].get('Code-Review', {}).get('approved'): + return 'lgtm' - if not data.get('reviewers', {}).get('REVIEWER', []): - return 'unsent' + if not data.get('reviewers', {}).get('REVIEWER', []): + return 'unsent' - owner = data['owner'].get('_account_id') - messages = sorted(data.get('messages', []), key=lambda m: m.get('date')) - while messages: - m = messages.pop() - if (m.get('tag', '').startswith('autogenerated:cq') or - m.get('tag', '').startswith('autogenerated:cv')): - # Ignore replies from LUCI CV/CQ. - continue - if m.get('author', {}).get('_account_id') == owner: - # Most recent message was by owner. - return 'waiting' + owner = data['owner'].get('_account_id') + messages = sorted(data.get('messages', []), key=lambda m: m.get('date')) + while messages: + m = messages.pop() + if (m.get('tag', '').startswith('autogenerated:cq') + or m.get('tag', '').startswith('autogenerated:cv')): + # Ignore replies from LUCI CV/CQ. + continue + if m.get('author', {}).get('_account_id') == owner: + # Most recent message was by owner. + return 'waiting' - # Some reply from non-owner. - return 'reply' + # Some reply from non-owner. + return 'reply' - # Somehow there are no messages even though there are reviewers. - return 'unsent' + # Somehow there are no messages even though there are reviewers. + return 'unsent' - def GetMostRecentPatchset(self, update=True): - if not self.GetIssue(): - return None + def GetMostRecentPatchset(self, update=True): + if not self.GetIssue(): + return None - data = self._GetChangeDetail(['CURRENT_REVISION']) - patchset = data['revisions'][data['current_revision']]['_number'] - if update: - self.SetPatchset(patchset) - return patchset + data = self._GetChangeDetail(['CURRENT_REVISION']) + patchset = data['revisions'][data['current_revision']]['_number'] + if update: + self.SetPatchset(patchset) + return patchset - def _IsPatchsetRangeSignificant(self, lower, upper): - """Returns True if the inclusive range of patchsets contains any reworks or + def _IsPatchsetRangeSignificant(self, lower, upper): + """Returns True if the inclusive range of patchsets contains any reworks or rebases.""" - if not self.GetIssue(): - return False + if not self.GetIssue(): + return False - data = self._GetChangeDetail(['ALL_REVISIONS']) - ps_kind = {} - for rev_info in data.get('revisions', {}).values(): - ps_kind[rev_info['_number']] = rev_info.get('kind', '') + data = self._GetChangeDetail(['ALL_REVISIONS']) + ps_kind = {} + for rev_info in data.get('revisions', {}).values(): + ps_kind[rev_info['_number']] = rev_info.get('kind', '') - for ps in range(lower, upper + 1): - assert ps in ps_kind, 'expected patchset %d in change detail' % ps - if ps_kind[ps] not in ('NO_CHANGE', 'NO_CODE_CHANGE'): - return True - return False + for ps in range(lower, upper + 1): + assert ps in ps_kind, 'expected patchset %d in change detail' % ps + if ps_kind[ps] not in ('NO_CHANGE', 'NO_CODE_CHANGE'): + return True + return False - def GetMostRecentDryRunPatchset(self): - """Get patchsets equivalent to the most recent patchset and return + def GetMostRecentDryRunPatchset(self): + """Get patchsets equivalent to the most recent patchset and return the patchset with the latest dry run. If none have been dry run, return the latest patchset.""" - if not self.GetIssue(): - return None + if not self.GetIssue(): + return None - data = self._GetChangeDetail(['ALL_REVISIONS']) - patchset = data['revisions'][data['current_revision']]['_number'] - dry_run = {int(m['_revision_number']) - for m in data.get('messages', []) - if m.get('tag', '').endswith('dry-run')} + data = self._GetChangeDetail(['ALL_REVISIONS']) + patchset = data['revisions'][data['current_revision']]['_number'] + dry_run = { + int(m['_revision_number']) + for m in data.get('messages', []) + if m.get('tag', '').endswith('dry-run') + } - for revision_info in sorted(data.get('revisions', {}).values(), - key=lambda c: c['_number'], reverse=True): - if revision_info['_number'] in dry_run: - patchset = revision_info['_number'] - break - if revision_info.get('kind', '') not in \ - ('NO_CHANGE', 'NO_CODE_CHANGE', 'TRIVIAL_REBASE'): - break - self.SetPatchset(patchset) - return patchset + for revision_info in sorted(data.get('revisions', {}).values(), + key=lambda c: c['_number'], + reverse=True): + if revision_info['_number'] in dry_run: + patchset = revision_info['_number'] + break + if revision_info.get('kind', '') not in \ + ('NO_CHANGE', 'NO_CODE_CHANGE', 'TRIVIAL_REBASE'): + break + self.SetPatchset(patchset) + return patchset - def AddComment(self, message, publish=None): - gerrit_util.SetReview( - self.GetGerritHost(), self._GerritChangeIdentifier(), - msg=message, ready=publish) + def AddComment(self, message, publish=None): + gerrit_util.SetReview(self.GetGerritHost(), + self._GerritChangeIdentifier(), + msg=message, + ready=publish) - def GetCommentsSummary(self, readable=True): - # DETAILED_ACCOUNTS is to get emails in accounts. - # CURRENT_REVISION is included to get the latest patchset so that - # only the robot comments from the latest patchset can be shown. - messages = self._GetChangeDetail( - options=['MESSAGES', 'DETAILED_ACCOUNTS', - 'CURRENT_REVISION']).get('messages', []) - file_comments = gerrit_util.GetChangeComments( - self.GetGerritHost(), self._GerritChangeIdentifier()) - robot_file_comments = gerrit_util.GetChangeRobotComments( - self.GetGerritHost(), self._GerritChangeIdentifier()) + def GetCommentsSummary(self, readable=True): + # DETAILED_ACCOUNTS is to get emails in accounts. + # CURRENT_REVISION is included to get the latest patchset so that + # only the robot comments from the latest patchset can be shown. + messages = self._GetChangeDetail( + options=['MESSAGES', 'DETAILED_ACCOUNTS', 'CURRENT_REVISION']).get( + 'messages', []) + file_comments = gerrit_util.GetChangeComments( + self.GetGerritHost(), self._GerritChangeIdentifier()) + robot_file_comments = gerrit_util.GetChangeRobotComments( + self.GetGerritHost(), self._GerritChangeIdentifier()) - # Add the robot comments onto the list of comments, but only - # keep those that are from the latest patchset. - latest_patch_set = self.GetMostRecentPatchset() - for path, robot_comments in robot_file_comments.items(): - line_comments = file_comments.setdefault(path, []) - line_comments.extend( - [c for c in robot_comments if c['patch_set'] == latest_patch_set]) + # Add the robot comments onto the list of comments, but only + # keep those that are from the latest patchset. + latest_patch_set = self.GetMostRecentPatchset() + for path, robot_comments in robot_file_comments.items(): + line_comments = file_comments.setdefault(path, []) + line_comments.extend([ + c for c in robot_comments if c['patch_set'] == latest_patch_set + ]) - # Build dictionary of file comments for easy access and sorting later. - # {author+date: {path: {patchset: {line: url+message}}}} - comments = collections.defaultdict( - lambda: collections.defaultdict(lambda: collections.defaultdict(dict))) + # Build dictionary of file comments for easy access and sorting later. + # {author+date: {path: {patchset: {line: url+message}}}} + comments = collections.defaultdict(lambda: collections.defaultdict( + lambda: collections.defaultdict(dict))) - server = self.GetCodereviewServer() - if server in _KNOWN_GERRIT_TO_SHORT_URLS: - # /c/ is automatically added by short URL server. - url_prefix = '%s/%s' % (_KNOWN_GERRIT_TO_SHORT_URLS[server], - self.GetIssue()) - else: - url_prefix = '%s/c/%s' % (server, self.GetIssue()) - - for path, line_comments in file_comments.items(): - for comment in line_comments: - tag = comment.get('tag', '') - if tag.startswith('autogenerated') and 'robot_id' not in comment: - continue - key = (comment['author']['email'], comment['updated']) - if comment.get('side', 'REVISION') == 'PARENT': - patchset = 'Base' + server = self.GetCodereviewServer() + if server in _KNOWN_GERRIT_TO_SHORT_URLS: + # /c/ is automatically added by short URL server. + url_prefix = '%s/%s' % (_KNOWN_GERRIT_TO_SHORT_URLS[server], + self.GetIssue()) else: - patchset = 'PS%d' % comment['patch_set'] - line = comment.get('line', 0) - url = ('%s/%s/%s#%s%s' % - (url_prefix, comment['patch_set'], path, - 'b' if comment.get('side') == 'PARENT' else '', - str(line) if line else '')) - comments[key][path][patchset][line] = (url, comment['message']) + url_prefix = '%s/c/%s' % (server, self.GetIssue()) - summaries = [] - for msg in messages: - summary = self._BuildCommentSummary(msg, comments, readable) - if summary: - summaries.append(summary) - return summaries + for path, line_comments in file_comments.items(): + for comment in line_comments: + tag = comment.get('tag', '') + if tag.startswith( + 'autogenerated') and 'robot_id' not in comment: + continue + key = (comment['author']['email'], comment['updated']) + if comment.get('side', 'REVISION') == 'PARENT': + patchset = 'Base' + else: + patchset = 'PS%d' % comment['patch_set'] + line = comment.get('line', 0) + url = ('%s/%s/%s#%s%s' % + (url_prefix, comment['patch_set'], + path, 'b' if comment.get('side') == 'PARENT' else '', + str(line) if line else '')) + comments[key][path][patchset][line] = (url, comment['message']) - @staticmethod - def _BuildCommentSummary(msg, comments, readable): - if 'email' not in msg['author']: - # Some bot accounts may not have an email associated. - return None + summaries = [] + for msg in messages: + summary = self._BuildCommentSummary(msg, comments, readable) + if summary: + summaries.append(summary) + return summaries - key = (msg['author']['email'], msg['date']) - # Don't bother showing autogenerated messages that don't have associated - # file or line comments. this will filter out most autogenerated - # messages, but will keep robot comments like those from Tricium. - is_autogenerated = msg.get('tag', '').startswith('autogenerated') - if is_autogenerated and not comments.get(key): - return None - message = msg['message'] - # Gerrit spits out nanoseconds. - assert len(msg['date'].split('.')[-1]) == 9 - date = datetime.datetime.strptime(msg['date'][:-3], - '%Y-%m-%d %H:%M:%S.%f') - if key in comments: - message += '\n' - for path, patchsets in sorted(comments.get(key, {}).items()): - if readable: - message += '\n%s' % path - for patchset, lines in sorted(patchsets.items()): - for line, (url, content) in sorted(lines.items()): - if line: - line_str = 'Line %d' % line - path_str = '%s:%d:' % (path, line) - else: - line_str = 'File comment' - path_str = '%s:0:' % path - if readable: - message += '\n %s, %s: %s' % (patchset, line_str, url) - message += '\n %s\n' % content - else: - message += '\n%s ' % path_str - message += '\n%s\n' % content + @staticmethod + def _BuildCommentSummary(msg, comments, readable): + if 'email' not in msg['author']: + # Some bot accounts may not have an email associated. + return None - return _CommentSummary( - date=date, - message=message, - sender=msg['author']['email'], - autogenerated=is_autogenerated, - # These could be inferred from the text messages and correlated with - # Code-Review label maximum, however this is not reliable. - # Leaving as is until the need arises. - approval=False, - disapproval=False, - ) + key = (msg['author']['email'], msg['date']) + # Don't bother showing autogenerated messages that don't have associated + # file or line comments. this will filter out most autogenerated + # messages, but will keep robot comments like those from Tricium. + is_autogenerated = msg.get('tag', '').startswith('autogenerated') + if is_autogenerated and not comments.get(key): + return None + message = msg['message'] + # Gerrit spits out nanoseconds. + assert len(msg['date'].split('.')[-1]) == 9 + date = datetime.datetime.strptime(msg['date'][:-3], + '%Y-%m-%d %H:%M:%S.%f') + if key in comments: + message += '\n' + for path, patchsets in sorted(comments.get(key, {}).items()): + if readable: + message += '\n%s' % path + for patchset, lines in sorted(patchsets.items()): + for line, (url, content) in sorted(lines.items()): + if line: + line_str = 'Line %d' % line + path_str = '%s:%d:' % (path, line) + else: + line_str = 'File comment' + path_str = '%s:0:' % path + if readable: + message += '\n %s, %s: %s' % (patchset, line_str, url) + message += '\n %s\n' % content + else: + message += '\n%s ' % path_str + message += '\n%s\n' % content - def CloseIssue(self): - gerrit_util.AbandonChange( - self.GetGerritHost(), self._GerritChangeIdentifier(), msg='') + return _CommentSummary( + date=date, + message=message, + sender=msg['author']['email'], + autogenerated=is_autogenerated, + # These could be inferred from the text messages and correlated with + # Code-Review label maximum, however this is not reliable. + # Leaving as is until the need arises. + approval=False, + disapproval=False, + ) - def SubmitIssue(self): - gerrit_util.SubmitChange( - self.GetGerritHost(), self._GerritChangeIdentifier()) + def CloseIssue(self): + gerrit_util.AbandonChange(self.GetGerritHost(), + self._GerritChangeIdentifier(), + msg='') - def _GetChangeDetail(self, options=None): - """Returns details of associated Gerrit change and caching results.""" - options = options or [] - assert self.GetIssue(), 'issue is required to query Gerrit' + def SubmitIssue(self): + gerrit_util.SubmitChange(self.GetGerritHost(), + self._GerritChangeIdentifier()) - # Optimization to avoid multiple RPCs: - if 'CURRENT_REVISION' in options or 'ALL_REVISIONS' in options: - options.append('CURRENT_COMMIT') + def _GetChangeDetail(self, options=None): + """Returns details of associated Gerrit change and caching results.""" + options = options or [] + assert self.GetIssue(), 'issue is required to query Gerrit' - # Normalize issue and options for consistent keys in cache. - cache_key = str(self.GetIssue()) - options_set = frozenset(o.upper() for o in options) + # Optimization to avoid multiple RPCs: + if 'CURRENT_REVISION' in options or 'ALL_REVISIONS' in options: + options.append('CURRENT_COMMIT') - for cached_options_set, data in self._detail_cache.get(cache_key, []): - # Assumption: data fetched before with extra options is suitable - # for return for a smaller set of options. - # For example, if we cached data for - # options=[CURRENT_REVISION, DETAILED_FOOTERS] - # and request is for options=[CURRENT_REVISION], - # THEN we can return prior cached data. - if options_set.issubset(cached_options_set): + # Normalize issue and options for consistent keys in cache. + cache_key = str(self.GetIssue()) + options_set = frozenset(o.upper() for o in options) + + for cached_options_set, data in self._detail_cache.get(cache_key, []): + # Assumption: data fetched before with extra options is suitable + # for return for a smaller set of options. + # For example, if we cached data for + # options=[CURRENT_REVISION, DETAILED_FOOTERS] + # and request is for options=[CURRENT_REVISION], + # THEN we can return prior cached data. + if options_set.issubset(cached_options_set): + return data + + try: + data = gerrit_util.GetChangeDetail(self.GetGerritHost(), + self._GerritChangeIdentifier(), + options_set) + except gerrit_util.GerritError as e: + if e.http_status == 404: + raise GerritChangeNotExists(self.GetIssue(), + self.GetCodereviewServer()) + raise + + self._detail_cache.setdefault(cache_key, []).append((options_set, data)) return data - try: - data = gerrit_util.GetChangeDetail( - self.GetGerritHost(), self._GerritChangeIdentifier(), options_set) - except gerrit_util.GerritError as e: - if e.http_status == 404: - raise GerritChangeNotExists(self.GetIssue(), self.GetCodereviewServer()) - raise + def _GetChangeCommit(self, revision='current'): + assert self.GetIssue(), 'issue must be set to query Gerrit' + try: + data = gerrit_util.GetChangeCommit(self.GetGerritHost(), + self._GerritChangeIdentifier(), + revision) + except gerrit_util.GerritError as e: + if e.http_status == 404: + raise GerritChangeNotExists(self.GetIssue(), + self.GetCodereviewServer()) + raise + return data - self._detail_cache.setdefault(cache_key, []).append((options_set, data)) - return data + def _IsCqConfigured(self): + detail = self._GetChangeDetail(['LABELS']) + return u'Commit-Queue' in detail.get('labels', {}) - def _GetChangeCommit(self, revision='current'): - assert self.GetIssue(), 'issue must be set to query Gerrit' - try: - data = gerrit_util.GetChangeCommit(self.GetGerritHost(), - self._GerritChangeIdentifier(), - revision) - except gerrit_util.GerritError as e: - if e.http_status == 404: - raise GerritChangeNotExists(self.GetIssue(), self.GetCodereviewServer()) - raise - return data + def CMDLand(self, force, bypass_hooks, verbose, parallel, resultdb, realm): + if git_common.is_dirty_git_tree('land'): + return 1 - def _IsCqConfigured(self): - detail = self._GetChangeDetail(['LABELS']) - return u'Commit-Queue' in detail.get('labels', {}) - - def CMDLand(self, force, bypass_hooks, verbose, parallel, resultdb, realm): - if git_common.is_dirty_git_tree('land'): - return 1 - - detail = self._GetChangeDetail(['CURRENT_REVISION', 'LABELS']) - if not force and self._IsCqConfigured(): - confirm_or_exit('\nIt seems this repository has a CQ, ' - 'which can test and land changes for you. ' - 'Are you sure you wish to bypass it?\n', - action='bypass CQ') - differs = True - last_upload = self._GitGetBranchConfigValue(GERRIT_SQUASH_HASH_CONFIG_KEY) - # Note: git diff outputs nothing if there is no diff. - if not last_upload or RunGit(['diff', last_upload]).strip(): - print('WARNING: Some changes from local branch haven\'t been uploaded.') - else: - if detail['current_revision'] == last_upload: - differs = False - else: - print('WARNING: Local branch contents differ from latest uploaded ' - 'patchset.') - if differs: - if not force: - confirm_or_exit( - 'Do you want to submit latest Gerrit patchset and bypass hooks?\n', - action='submit') - print('WARNING: Bypassing hooks and submitting latest uploaded patchset.') - elif not bypass_hooks: - upstream = self.GetCommonAncestorWithUpstream() - if self.GetIssue(): - description = self.FetchDescription() - else: - description = _create_description_from_log([upstream]) - self.RunHook( - committing=True, - may_prompt=not force, - verbose=verbose, - parallel=parallel, - upstream=upstream, - description=description, - all_files=False, - resultdb=resultdb, - realm=realm) - - self.SubmitIssue() - print('Issue %s has been submitted.' % self.GetIssueURL()) - links = self._GetChangeCommit().get('web_links', []) - for link in links: - if link.get('name') in ['gitiles', 'browse'] and link.get('url'): - print('Landed as: %s' % link.get('url')) - break - return 0 - - def CMDPatchWithParsedIssue(self, parsed_issue_arg, nocommit, force, - newbranch): - assert parsed_issue_arg.valid - - self.issue = parsed_issue_arg.issue - - if parsed_issue_arg.hostname: - self._gerrit_host = parsed_issue_arg.hostname - self._gerrit_server = 'https://%s' % self._gerrit_host - - try: - detail = self._GetChangeDetail(['ALL_REVISIONS']) - except GerritChangeNotExists as e: - DieWithError(str(e)) - - if not parsed_issue_arg.patchset: - # Use current revision by default. - revision_info = detail['revisions'][detail['current_revision']] - patchset = int(revision_info['_number']) - else: - patchset = parsed_issue_arg.patchset - for revision_info in detail['revisions'].values(): - if int(revision_info['_number']) == parsed_issue_arg.patchset: - break - else: - DieWithError('Couldn\'t find patchset %i in change %i' % - (parsed_issue_arg.patchset, self.GetIssue())) - - remote_url = self.GetRemoteUrl() - if remote_url.endswith('.git'): - remote_url = remote_url[:-len('.git')] - remote_url = remote_url.rstrip('/') - - fetch_info = revision_info['fetch']['http'] - fetch_info['url'] = fetch_info['url'].rstrip('/') - - if remote_url != fetch_info['url']: - DieWithError('Trying to patch a change from %s but this repo appears ' - 'to be %s.' % (fetch_info['url'], remote_url)) - - RunGit(['fetch', fetch_info['url'], fetch_info['ref']]) - - # Set issue immediately in case the cherry-pick fails, which happens - # when resolving conflicts. - if self.GetBranch(): - self.SetIssue(parsed_issue_arg.issue) - - if force: - RunGit(['reset', '--hard', 'FETCH_HEAD']) - print('Checked out commit for change %i patchset %i locally' % - (parsed_issue_arg.issue, patchset)) - elif nocommit: - RunGit(['cherry-pick', '--no-commit', 'FETCH_HEAD']) - print('Patch applied to index.') - else: - RunGit(['cherry-pick', 'FETCH_HEAD']) - print('Committed patch for change %i patchset %i locally.' % - (parsed_issue_arg.issue, patchset)) - print('Note: this created a local commit which does not have ' - 'the same hash as the one uploaded for review. This will make ' - 'uploading changes based on top of this branch difficult.\n' - 'If you want to do that, use "git cl patch --force" instead.') - - if self.GetBranch(): - self.SetPatchset(patchset) - fetched_hash = scm.GIT.ResolveCommit(settings.GetRoot(), 'FETCH_HEAD') - self._GitSetBranchConfigValue(LAST_UPLOAD_HASH_CONFIG_KEY, fetched_hash) - self._GitSetBranchConfigValue(GERRIT_SQUASH_HASH_CONFIG_KEY, fetched_hash) - else: - print('WARNING: You are in detached HEAD state.\n' - 'The patch has been applied to your checkout, but you will not be ' - 'able to upload a new patch set to the gerrit issue.\n' - 'Try using the \'-b\' option if you would like to work on a ' - 'branch and/or upload a new patch set.') - - return 0 - - @staticmethod - def _GerritCommitMsgHookCheck(offer_removal): - # type: (bool) -> None - """Checks for the gerrit's commit-msg hook and removes it if necessary.""" - hook = os.path.join(settings.GetRoot(), '.git', 'hooks', 'commit-msg') - if not os.path.exists(hook): - return - # Crude attempt to distinguish Gerrit Codereview hook from a potentially - # custom developer-made one. - data = gclient_utils.FileRead(hook) - if not('From Gerrit Code Review' in data and 'add_ChangeId()' in data): - return - print('WARNING: You have Gerrit commit-msg hook installed.\n' - 'It is not necessary for uploading with git cl in squash mode, ' - 'and may interfere with it in subtle ways.\n' - 'We recommend you remove the commit-msg hook.') - if offer_removal: - if ask_for_explicit_yes('Do you want to remove it now?'): - gclient_utils.rm_file_or_tree(hook) - print('Gerrit commit-msg hook removed.') - else: - print('OK, will keep Gerrit commit-msg hook in place.') - - def _CleanUpOldTraces(self): - """Keep only the last |MAX_TRACES| traces.""" - try: - traces = sorted([ - os.path.join(TRACES_DIR, f) - for f in os.listdir(TRACES_DIR) - if (os.path.isfile(os.path.join(TRACES_DIR, f)) - and not f.startswith('tmp')) - ]) - traces_to_delete = traces[:-MAX_TRACES] - for trace in traces_to_delete: - os.remove(trace) - except OSError: - print('WARNING: Failed to remove old git traces from\n' - ' %s' - 'Consider removing them manually.' % TRACES_DIR) - - def _WriteGitPushTraces(self, trace_name, traces_dir, git_push_metadata): - """Zip and write the git push traces stored in traces_dir.""" - gclient_utils.safe_makedirs(TRACES_DIR) - traces_zip = trace_name + '-traces' - traces_readme = trace_name + '-README' - # Create a temporary dir to store git config and gitcookies in. It will be - # compressed and stored next to the traces. - git_info_dir = tempfile.mkdtemp() - git_info_zip = trace_name + '-git-info' - - git_push_metadata['now'] = datetime_now().strftime('%Y-%m-%dT%H:%M:%S.%f') - - git_push_metadata['trace_name'] = trace_name - gclient_utils.FileWrite( - traces_readme, TRACES_README_FORMAT % git_push_metadata) - - # Keep only the first 6 characters of the git hashes on the packet - # trace. This greatly decreases size after compression. - packet_traces = os.path.join(traces_dir, 'trace-packet') - if os.path.isfile(packet_traces): - contents = gclient_utils.FileRead(packet_traces) - gclient_utils.FileWrite( - packet_traces, GIT_HASH_RE.sub(r'\1', contents)) - shutil.make_archive(traces_zip, 'zip', traces_dir) - - # Collect and compress the git config and gitcookies. - git_config = RunGit(['config', '-l']) - gclient_utils.FileWrite( - os.path.join(git_info_dir, 'git-config'), - git_config) - - cookie_auth = gerrit_util.Authenticator.get() - if isinstance(cookie_auth, gerrit_util.CookiesAuthenticator): - gitcookies_path = cookie_auth.get_gitcookies_path() - if os.path.isfile(gitcookies_path): - gitcookies = gclient_utils.FileRead(gitcookies_path) - gclient_utils.FileWrite( - os.path.join(git_info_dir, 'gitcookies'), - GITCOOKIES_REDACT_RE.sub('REDACTED', gitcookies)) - shutil.make_archive(git_info_zip, 'zip', git_info_dir) - - gclient_utils.rmtree(git_info_dir) - - def _RunGitPushWithTraces(self, - refspec, - refspec_opts, - git_push_metadata, - git_push_options=None): - """Run git push and collect the traces resulting from the execution.""" - # Create a temporary directory to store traces in. Traces will be compressed - # and stored in a 'traces' dir inside depot_tools. - traces_dir = tempfile.mkdtemp() - trace_name = os.path.join( - TRACES_DIR, datetime_now().strftime('%Y%m%dT%H%M%S.%f')) - - env = os.environ.copy() - env['GIT_REDACT_COOKIES'] = 'o,SSO,GSSO_Uberproxy' - env['GIT_TR2_EVENT'] = os.path.join(traces_dir, 'tr2-event') - env['GIT_TRACE2_EVENT'] = os.path.join(traces_dir, 'tr2-event') - env['GIT_TRACE_CURL'] = os.path.join(traces_dir, 'trace-curl') - env['GIT_TRACE_CURL_NO_DATA'] = '1' - env['GIT_TRACE_PACKET'] = os.path.join(traces_dir, 'trace-packet') - - push_returncode = 0 - before_push = time_time() - try: - remote_url = self.GetRemoteUrl() - push_cmd = ['git', 'push', remote_url, refspec] - if git_push_options: - for opt in git_push_options: - push_cmd.extend(['-o', opt]) - - push_stdout = gclient_utils.CheckCallAndFilter( - push_cmd, - env=env, - print_stdout=True, - # Flush after every line: useful for seeing progress when running as - # recipe. - filter_fn=lambda _: sys.stdout.flush()) - push_stdout = push_stdout.decode('utf-8', 'replace') - except subprocess2.CalledProcessError as e: - push_returncode = e.returncode - if 'blocked keyword' in str(e.stdout) or 'banned word' in str(e.stdout): - raise GitPushError( - 'Failed to create a change, very likely due to blocked keyword. ' - 'Please examine output above for the reason of the failure.\n' - 'If this is a false positive, you can try to bypass blocked ' - 'keyword by using push option ' - '-o banned-words~skip, e.g.:\n' - 'git cl upload -o banned-words~skip\n\n' - 'If git-cl is not working correctly, file a bug under the ' - 'Infra>SDK component.') - if 'git push -o nokeycheck' in str(e.stdout): - raise GitPushError( - 'Failed to create a change, very likely due to a private key being ' - 'detected. Please examine output above for the reason of the ' - 'failure.\n' - 'If this is a false positive, you can try to bypass private key ' - 'detection by using push option ' - '-o nokeycheck, e.g.:\n' - 'git cl upload -o nokeycheck\n\n' - 'If git-cl is not working correctly, file a bug under the ' - 'Infra>SDK component.') - - raise GitPushError( - 'Failed to create a change. Please examine output above for the ' - 'reason of the failure.\n' - 'For emergencies, Googlers can escalate to ' - 'go/gob-support or go/notify#gob\n' - 'Hint: run command below to diagnose common Git/Gerrit ' - 'credential problems:\n' - ' git cl creds-check\n' - '\n' - 'If git-cl is not working correctly, file a bug under the Infra>SDK ' - 'component including the files below.\n' - 'Review the files before upload, since they might contain sensitive ' - 'information.\n' - 'Set the Restrict-View-Google label so that they are not publicly ' - 'accessible.\n' + TRACES_MESSAGE % {'trace_name': trace_name}) - finally: - execution_time = time_time() - before_push - metrics.collector.add_repeated('sub_commands', { - 'command': 'git push', - 'execution_time': execution_time, - 'exit_code': push_returncode, - 'arguments': metrics_utils.extract_known_subcommand_args(refspec_opts), - }) - - git_push_metadata['execution_time'] = execution_time - git_push_metadata['exit_code'] = push_returncode - self._WriteGitPushTraces(trace_name, traces_dir, git_push_metadata) - - self._CleanUpOldTraces() - gclient_utils.rmtree(traces_dir) - - return push_stdout - - def CMDUploadChange(self, options, git_diff_args, custom_cl_base, - change_desc): - """Upload the current branch to Gerrit, retry if new remote HEAD is - found. options and change_desc may be mutated.""" - remote, remote_branch = self.GetRemoteBranch() - branch = GetTargetRef(remote, remote_branch, options.target_branch) - - try: - return self._CMDUploadChange(options, git_diff_args, custom_cl_base, - change_desc, branch) - except GitPushError as e: - # Repository might be in the middle of transition to main branch as - # default, and uploads to old default might be blocked. - if remote_branch not in [DEFAULT_OLD_BRANCH, DEFAULT_NEW_BRANCH]: - DieWithError(str(e), change_desc) - - project_head = gerrit_util.GetProjectHead(self._gerrit_host, - self.GetGerritProject()) - if project_head == branch: - DieWithError(str(e), change_desc) - branch = project_head - - print("WARNING: Fetching remote state and retrying upload to default " - "branch...") - RunGit(['fetch', '--prune', remote]) - options.edit_description = False - options.force = True - try: - self._CMDUploadChange(options, git_diff_args, custom_cl_base, - change_desc, branch) - except GitPushError as e: - DieWithError(str(e), change_desc) - - def _CMDUploadChange(self, options, git_diff_args, custom_cl_base, - change_desc, branch): - """Upload the current branch to Gerrit.""" - if options.squash: - Changelist._GerritCommitMsgHookCheck(offer_removal=not options.force) - external_parent = None - if self.GetIssue(): - # User requested to change description - if options.edit_description: - change_desc.prompt() - change_detail = self._GetChangeDetail(['CURRENT_REVISION']) - change_id = change_detail['change_id'] - change_desc.ensure_change_id(change_id) - - # Check if changes outside of this workspace have been uploaded. - current_rev = change_detail['current_revision'] - last_uploaded_rev = self._GitGetBranchConfigValue( + detail = self._GetChangeDetail(['CURRENT_REVISION', 'LABELS']) + if not force and self._IsCqConfigured(): + confirm_or_exit( + '\nIt seems this repository has a CQ, ' + 'which can test and land changes for you. ' + 'Are you sure you wish to bypass it?\n', + action='bypass CQ') + differs = True + last_upload = self._GitGetBranchConfigValue( GERRIT_SQUASH_HASH_CONFIG_KEY) - if last_uploaded_rev and current_rev != last_uploaded_rev: - external_parent = self._UpdateWithExternalChanges() - else: # if not self.GetIssue() - if not options.force and not options.message_file: - change_desc.prompt() - change_ids = git_footers.get_footer_change_id(change_desc.description) - if len(change_ids) == 1: - change_id = change_ids[0] + # Note: git diff outputs nothing if there is no diff. + if not last_upload or RunGit(['diff', last_upload]).strip(): + print( + 'WARNING: Some changes from local branch haven\'t been uploaded.' + ) else: - change_id = GenerateGerritChangeId(change_desc.description) - change_desc.ensure_change_id(change_id) + if detail['current_revision'] == last_upload: + differs = False + else: + print( + 'WARNING: Local branch contents differ from latest uploaded ' + 'patchset.') + if differs: + if not force: + confirm_or_exit( + 'Do you want to submit latest Gerrit patchset and bypass hooks?\n', + action='submit') + print( + 'WARNING: Bypassing hooks and submitting latest uploaded patchset.' + ) + elif not bypass_hooks: + upstream = self.GetCommonAncestorWithUpstream() + if self.GetIssue(): + description = self.FetchDescription() + else: + description = _create_description_from_log([upstream]) + self.RunHook(committing=True, + may_prompt=not force, + verbose=verbose, + parallel=parallel, + upstream=upstream, + description=description, + all_files=False, + resultdb=resultdb, + realm=realm) - if options.preserve_tryjobs: - change_desc.set_preserve_tryjobs() + self.SubmitIssue() + print('Issue %s has been submitted.' % self.GetIssueURL()) + links = self._GetChangeCommit().get('web_links', []) + for link in links: + if link.get('name') in ['gitiles', 'browse'] and link.get('url'): + print('Landed as: %s' % link.get('url')) + break + return 0 - remote, upstream_branch = self.FetchUpstreamTuple(self.GetBranch()) - parent = external_parent or self._ComputeParent( - remote, upstream_branch, custom_cl_base, options.force, change_desc) - tree = RunGit(['rev-parse', 'HEAD:']).strip() - with gclient_utils.temporary_file() as desc_tempfile: - gclient_utils.FileWrite(desc_tempfile, change_desc.description) - ref_to_push = RunGit( - ['commit-tree', tree, '-p', parent, '-F', desc_tempfile]).strip() - else: # if not options.squash - if options.no_add_changeid: - pass - else: # adding Change-Ids is okay. - if not git_footers.get_footer_change_id(change_desc.description): - DownloadGerritHook(False) - change_desc.set_description( - self._AddChangeIdToCommitMessage(change_desc.description, - git_diff_args)) - ref_to_push = 'HEAD' - # For no-squash mode, we assume the remote called "origin" is the one we - # want. It is not worthwhile to support different workflows for - # no-squash mode. - parent = 'origin/%s' % branch - # attempt to extract the changeid from the current description - # fail informatively if not possible. - change_id_candidates = git_footers.get_footer_change_id( - change_desc.description) - if not change_id_candidates: - DieWithError("Unable to extract change-id from message.") - change_id = change_id_candidates[0] + def CMDPatchWithParsedIssue(self, parsed_issue_arg, nocommit, force, + newbranch): + assert parsed_issue_arg.valid - SaveDescriptionBackup(change_desc) - commits = RunGitSilent(['rev-list', '%s..%s' % (parent, - ref_to_push)]).splitlines() - if len(commits) > 1: - print('WARNING: This will upload %d commits. Run the following command ' - 'to see which commits will be uploaded: ' % len(commits)) - print('git log %s..%s' % (parent, ref_to_push)) - print('You can also use `git squash-branch` to squash these into a ' - 'single commit.') - confirm_or_exit(action='upload') + self.issue = parsed_issue_arg.issue - reviewers = sorted(change_desc.get_reviewers()) - cc = [] - # Add default, watchlist, presubmit ccs if this is the initial upload - # and CL is not private and auto-ccing has not been disabled. - if not options.private and not options.no_autocc and not self.GetIssue(): - cc = self.GetCCList().split(',') - if len(cc) > 100: - lsc = ('https://chromium.googlesource.com/chromium/src/+/HEAD/docs/' - 'process/lsc/lsc_workflow.md') - print('WARNING: This will auto-CC %s users.' % len(cc)) - print('LSC may be more appropriate: %s' % lsc) - print('You can also use the --no-autocc flag to disable auto-CC.') - confirm_or_exit(action='continue') - # Add cc's from the --cc flag. - if options.cc: - cc.extend(options.cc) - cc = [email.strip() for email in cc if email.strip()] - if change_desc.get_cced(): - cc.extend(change_desc.get_cced()) - if self.GetGerritHost() == 'chromium-review.googlesource.com': - valid_accounts = set(reviewers + cc) - # TODO(crbug/877717): relax this for all hosts. - else: - valid_accounts = gerrit_util.ValidAccounts( - self.GetGerritHost(), reviewers + cc) - logging.info('accounts %s are recognized, %s invalid', - sorted(valid_accounts), - set(reviewers + cc).difference(set(valid_accounts))) + if parsed_issue_arg.hostname: + self._gerrit_host = parsed_issue_arg.hostname + self._gerrit_server = 'https://%s' % self._gerrit_host - # Extra options that can be specified at push time. Doc: - # https://gerrit-review.googlesource.com/Documentation/user-upload.html - refspec_opts = self._GetRefSpecOptions(options, change_desc) + try: + detail = self._GetChangeDetail(['ALL_REVISIONS']) + except GerritChangeNotExists as e: + DieWithError(str(e)) - for r in sorted(reviewers): - if r in valid_accounts: - refspec_opts.append('r=%s' % r) - reviewers.remove(r) - else: - # TODO(tandrii): this should probably be a hard failure. - print('WARNING: reviewer %s doesn\'t have a Gerrit account, skipping' - % r) - for c in sorted(cc): - # refspec option will be rejected if cc doesn't correspond to an - # account, even though REST call to add such arbitrary cc may succeed. - if c in valid_accounts: - refspec_opts.append('cc=%s' % c) - cc.remove(c) + if not parsed_issue_arg.patchset: + # Use current revision by default. + revision_info = detail['revisions'][detail['current_revision']] + patchset = int(revision_info['_number']) + else: + patchset = parsed_issue_arg.patchset + for revision_info in detail['revisions'].values(): + if int(revision_info['_number']) == parsed_issue_arg.patchset: + break + else: + DieWithError('Couldn\'t find patchset %i in change %i' % + (parsed_issue_arg.patchset, self.GetIssue())) - refspec_suffix = '' - if refspec_opts: - refspec_suffix = '%' + ','.join(refspec_opts) - assert ' ' not in refspec_suffix, ( - 'spaces not allowed in refspec: "%s"' % refspec_suffix) - refspec = '%s:refs/for/%s%s' % (ref_to_push, branch, refspec_suffix) + remote_url = self.GetRemoteUrl() + if remote_url.endswith('.git'): + remote_url = remote_url[:-len('.git')] + remote_url = remote_url.rstrip('/') - git_push_metadata = { - 'gerrit_host': self.GetGerritHost(), - 'title': options.title or '', - 'change_id': change_id, - 'description': change_desc.description, - } + fetch_info = revision_info['fetch']['http'] + fetch_info['url'] = fetch_info['url'].rstrip('/') - # Gerrit may or may not update fast enough to return the correct patchset - # number after we push. Get the pre-upload patchset and increment later. - latest_ps = self.GetMostRecentPatchset(update=False) or 0 + if remote_url != fetch_info['url']: + DieWithError( + 'Trying to patch a change from %s but this repo appears ' + 'to be %s.' % (fetch_info['url'], remote_url)) - push_stdout = self._RunGitPushWithTraces(refspec, refspec_opts, - git_push_metadata, - options.push_options) + RunGit(['fetch', fetch_info['url'], fetch_info['ref']]) - if options.squash: - regex = re.compile(r'remote:\s+https?://[\w\-\.\+\/#]*/(\d+)\s.*') - change_numbers = [m.group(1) - for m in map(regex.match, push_stdout.splitlines()) - if m] - if len(change_numbers) != 1: - DieWithError( - ('Created|Updated %d issues on Gerrit, but only 1 expected.\n' - 'Change-Id: %s') % (len(change_numbers), change_id), change_desc) - self.SetIssue(change_numbers[0]) - self.SetPatchset(latest_ps + 1) - self._GitSetBranchConfigValue(GERRIT_SQUASH_HASH_CONFIG_KEY, ref_to_push) + # Set issue immediately in case the cherry-pick fails, which happens + # when resolving conflicts. + if self.GetBranch(): + self.SetIssue(parsed_issue_arg.issue) - if self.GetIssue() and (reviewers or cc): - # GetIssue() is not set in case of non-squash uploads according to tests. - # TODO(crbug.com/751901): non-squash uploads in git cl should be removed. - gerrit_util.AddReviewers(self.GetGerritHost(), - self._GerritChangeIdentifier(), - reviewers, - cc, - notify=bool(options.send_mail)) + if force: + RunGit(['reset', '--hard', 'FETCH_HEAD']) + print('Checked out commit for change %i patchset %i locally' % + (parsed_issue_arg.issue, patchset)) + elif nocommit: + RunGit(['cherry-pick', '--no-commit', 'FETCH_HEAD']) + print('Patch applied to index.') + else: + RunGit(['cherry-pick', 'FETCH_HEAD']) + print('Committed patch for change %i patchset %i locally.' % + (parsed_issue_arg.issue, patchset)) + print( + 'Note: this created a local commit which does not have ' + 'the same hash as the one uploaded for review. This will make ' + 'uploading changes based on top of this branch difficult.\n' + 'If you want to do that, use "git cl patch --force" instead.') - return 0 + if self.GetBranch(): + self.SetPatchset(patchset) + fetched_hash = scm.GIT.ResolveCommit(settings.GetRoot(), + 'FETCH_HEAD') + self._GitSetBranchConfigValue(LAST_UPLOAD_HASH_CONFIG_KEY, + fetched_hash) + self._GitSetBranchConfigValue(GERRIT_SQUASH_HASH_CONFIG_KEY, + fetched_hash) + else: + print( + 'WARNING: You are in detached HEAD state.\n' + 'The patch has been applied to your checkout, but you will not be ' + 'able to upload a new patch set to the gerrit issue.\n' + 'Try using the \'-b\' option if you would like to work on a ' + 'branch and/or upload a new patch set.') - def _ComputeParent(self, remote, upstream_branch, custom_cl_base, force, - change_desc): - """Computes parent of the generated commit to be uploaded to Gerrit. + return 0 + + @staticmethod + def _GerritCommitMsgHookCheck(offer_removal): + # type: (bool) -> None + """Checks for the gerrit's commit-msg hook and removes it if necessary.""" + hook = os.path.join(settings.GetRoot(), '.git', 'hooks', 'commit-msg') + if not os.path.exists(hook): + return + # Crude attempt to distinguish Gerrit Codereview hook from a potentially + # custom developer-made one. + data = gclient_utils.FileRead(hook) + if not ('From Gerrit Code Review' in data and 'add_ChangeId()' in data): + return + print('WARNING: You have Gerrit commit-msg hook installed.\n' + 'It is not necessary for uploading with git cl in squash mode, ' + 'and may interfere with it in subtle ways.\n' + 'We recommend you remove the commit-msg hook.') + if offer_removal: + if ask_for_explicit_yes('Do you want to remove it now?'): + gclient_utils.rm_file_or_tree(hook) + print('Gerrit commit-msg hook removed.') + else: + print('OK, will keep Gerrit commit-msg hook in place.') + + def _CleanUpOldTraces(self): + """Keep only the last |MAX_TRACES| traces.""" + try: + traces = sorted([ + os.path.join(TRACES_DIR, f) for f in os.listdir(TRACES_DIR) + if (os.path.isfile(os.path.join(TRACES_DIR, f)) + and not f.startswith('tmp')) + ]) + traces_to_delete = traces[:-MAX_TRACES] + for trace in traces_to_delete: + os.remove(trace) + except OSError: + print('WARNING: Failed to remove old git traces from\n' + ' %s' + 'Consider removing them manually.' % TRACES_DIR) + + def _WriteGitPushTraces(self, trace_name, traces_dir, git_push_metadata): + """Zip and write the git push traces stored in traces_dir.""" + gclient_utils.safe_makedirs(TRACES_DIR) + traces_zip = trace_name + '-traces' + traces_readme = trace_name + '-README' + # Create a temporary dir to store git config and gitcookies in. It will + # be compressed and stored next to the traces. + git_info_dir = tempfile.mkdtemp() + git_info_zip = trace_name + '-git-info' + + git_push_metadata['now'] = datetime_now().strftime( + '%Y-%m-%dT%H:%M:%S.%f') + + git_push_metadata['trace_name'] = trace_name + gclient_utils.FileWrite(traces_readme, + TRACES_README_FORMAT % git_push_metadata) + + # Keep only the first 6 characters of the git hashes on the packet + # trace. This greatly decreases size after compression. + packet_traces = os.path.join(traces_dir, 'trace-packet') + if os.path.isfile(packet_traces): + contents = gclient_utils.FileRead(packet_traces) + gclient_utils.FileWrite(packet_traces, + GIT_HASH_RE.sub(r'\1', contents)) + shutil.make_archive(traces_zip, 'zip', traces_dir) + + # Collect and compress the git config and gitcookies. + git_config = RunGit(['config', '-l']) + gclient_utils.FileWrite(os.path.join(git_info_dir, 'git-config'), + git_config) + + cookie_auth = gerrit_util.Authenticator.get() + if isinstance(cookie_auth, gerrit_util.CookiesAuthenticator): + gitcookies_path = cookie_auth.get_gitcookies_path() + if os.path.isfile(gitcookies_path): + gitcookies = gclient_utils.FileRead(gitcookies_path) + gclient_utils.FileWrite( + os.path.join(git_info_dir, 'gitcookies'), + GITCOOKIES_REDACT_RE.sub('REDACTED', gitcookies)) + shutil.make_archive(git_info_zip, 'zip', git_info_dir) + + gclient_utils.rmtree(git_info_dir) + + def _RunGitPushWithTraces(self, + refspec, + refspec_opts, + git_push_metadata, + git_push_options=None): + """Run git push and collect the traces resulting from the execution.""" + # Create a temporary directory to store traces in. Traces will be + # compressed and stored in a 'traces' dir inside depot_tools. + traces_dir = tempfile.mkdtemp() + trace_name = os.path.join(TRACES_DIR, + datetime_now().strftime('%Y%m%dT%H%M%S.%f')) + + env = os.environ.copy() + env['GIT_REDACT_COOKIES'] = 'o,SSO,GSSO_Uberproxy' + env['GIT_TR2_EVENT'] = os.path.join(traces_dir, 'tr2-event') + env['GIT_TRACE2_EVENT'] = os.path.join(traces_dir, 'tr2-event') + env['GIT_TRACE_CURL'] = os.path.join(traces_dir, 'trace-curl') + env['GIT_TRACE_CURL_NO_DATA'] = '1' + env['GIT_TRACE_PACKET'] = os.path.join(traces_dir, 'trace-packet') + + push_returncode = 0 + before_push = time_time() + try: + remote_url = self.GetRemoteUrl() + push_cmd = ['git', 'push', remote_url, refspec] + if git_push_options: + for opt in git_push_options: + push_cmd.extend(['-o', opt]) + + push_stdout = gclient_utils.CheckCallAndFilter( + push_cmd, + env=env, + print_stdout=True, + # Flush after every line: useful for seeing progress when + # running as recipe. + filter_fn=lambda _: sys.stdout.flush()) + push_stdout = push_stdout.decode('utf-8', 'replace') + except subprocess2.CalledProcessError as e: + push_returncode = e.returncode + if 'blocked keyword' in str(e.stdout) or 'banned word' in str( + e.stdout): + raise GitPushError( + 'Failed to create a change, very likely due to blocked keyword. ' + 'Please examine output above for the reason of the failure.\n' + 'If this is a false positive, you can try to bypass blocked ' + 'keyword by using push option ' + '-o banned-words~skip, e.g.:\n' + 'git cl upload -o banned-words~skip\n\n' + 'If git-cl is not working correctly, file a bug under the ' + 'Infra>SDK component.') + if 'git push -o nokeycheck' in str(e.stdout): + raise GitPushError( + 'Failed to create a change, very likely due to a private key being ' + 'detected. Please examine output above for the reason of the ' + 'failure.\n' + 'If this is a false positive, you can try to bypass private key ' + 'detection by using push option ' + '-o nokeycheck, e.g.:\n' + 'git cl upload -o nokeycheck\n\n' + 'If git-cl is not working correctly, file a bug under the ' + 'Infra>SDK component.') + + raise GitPushError( + 'Failed to create a change. Please examine output above for the ' + 'reason of the failure.\n' + 'For emergencies, Googlers can escalate to ' + 'go/gob-support or go/notify#gob\n' + 'Hint: run command below to diagnose common Git/Gerrit ' + 'credential problems:\n' + ' git cl creds-check\n' + '\n' + 'If git-cl is not working correctly, file a bug under the Infra>SDK ' + 'component including the files below.\n' + 'Review the files before upload, since they might contain sensitive ' + 'information.\n' + 'Set the Restrict-View-Google label so that they are not publicly ' + 'accessible.\n' + TRACES_MESSAGE % {'trace_name': trace_name}) + finally: + execution_time = time_time() - before_push + metrics.collector.add_repeated( + 'sub_commands', { + 'command': + 'git push', + 'execution_time': + execution_time, + 'exit_code': + push_returncode, + 'arguments': + metrics_utils.extract_known_subcommand_args(refspec_opts), + }) + + git_push_metadata['execution_time'] = execution_time + git_push_metadata['exit_code'] = push_returncode + self._WriteGitPushTraces(trace_name, traces_dir, git_push_metadata) + + self._CleanUpOldTraces() + gclient_utils.rmtree(traces_dir) + + return push_stdout + + def CMDUploadChange(self, options, git_diff_args, custom_cl_base, + change_desc): + """Upload the current branch to Gerrit, retry if new remote HEAD is + found. options and change_desc may be mutated.""" + remote, remote_branch = self.GetRemoteBranch() + branch = GetTargetRef(remote, remote_branch, options.target_branch) + + try: + return self._CMDUploadChange(options, git_diff_args, custom_cl_base, + change_desc, branch) + except GitPushError as e: + # Repository might be in the middle of transition to main branch as + # default, and uploads to old default might be blocked. + if remote_branch not in [DEFAULT_OLD_BRANCH, DEFAULT_NEW_BRANCH]: + DieWithError(str(e), change_desc) + + project_head = gerrit_util.GetProjectHead(self._gerrit_host, + self.GetGerritProject()) + if project_head == branch: + DieWithError(str(e), change_desc) + branch = project_head + + print("WARNING: Fetching remote state and retrying upload to default " + "branch...") + RunGit(['fetch', '--prune', remote]) + options.edit_description = False + options.force = True + try: + self._CMDUploadChange(options, git_diff_args, custom_cl_base, + change_desc, branch) + except GitPushError as e: + DieWithError(str(e), change_desc) + + def _CMDUploadChange(self, options, git_diff_args, custom_cl_base, + change_desc, branch): + """Upload the current branch to Gerrit.""" + if options.squash: + Changelist._GerritCommitMsgHookCheck( + offer_removal=not options.force) + external_parent = None + if self.GetIssue(): + # User requested to change description + if options.edit_description: + change_desc.prompt() + change_detail = self._GetChangeDetail(['CURRENT_REVISION']) + change_id = change_detail['change_id'] + change_desc.ensure_change_id(change_id) + + # Check if changes outside of this workspace have been uploaded. + current_rev = change_detail['current_revision'] + last_uploaded_rev = self._GitGetBranchConfigValue( + GERRIT_SQUASH_HASH_CONFIG_KEY) + if last_uploaded_rev and current_rev != last_uploaded_rev: + external_parent = self._UpdateWithExternalChanges() + else: # if not self.GetIssue() + if not options.force and not options.message_file: + change_desc.prompt() + change_ids = git_footers.get_footer_change_id( + change_desc.description) + if len(change_ids) == 1: + change_id = change_ids[0] + else: + change_id = GenerateGerritChangeId(change_desc.description) + change_desc.ensure_change_id(change_id) + + if options.preserve_tryjobs: + change_desc.set_preserve_tryjobs() + + remote, upstream_branch = self.FetchUpstreamTuple(self.GetBranch()) + parent = external_parent or self._ComputeParent( + remote, upstream_branch, custom_cl_base, options.force, + change_desc) + tree = RunGit(['rev-parse', 'HEAD:']).strip() + with gclient_utils.temporary_file() as desc_tempfile: + gclient_utils.FileWrite(desc_tempfile, change_desc.description) + ref_to_push = RunGit( + ['commit-tree', tree, '-p', parent, '-F', + desc_tempfile]).strip() + else: # if not options.squash + if options.no_add_changeid: + pass + else: # adding Change-Ids is okay. + if not git_footers.get_footer_change_id( + change_desc.description): + DownloadGerritHook(False) + change_desc.set_description( + self._AddChangeIdToCommitMessage( + change_desc.description, git_diff_args)) + ref_to_push = 'HEAD' + # For no-squash mode, we assume the remote called "origin" is the + # one we want. It is not worthwhile to support different workflows + # for no-squash mode. + parent = 'origin/%s' % branch + # attempt to extract the changeid from the current description + # fail informatively if not possible. + change_id_candidates = git_footers.get_footer_change_id( + change_desc.description) + if not change_id_candidates: + DieWithError("Unable to extract change-id from message.") + change_id = change_id_candidates[0] + + SaveDescriptionBackup(change_desc) + commits = RunGitSilent(['rev-list', + '%s..%s' % (parent, ref_to_push)]).splitlines() + if len(commits) > 1: + print( + 'WARNING: This will upload %d commits. Run the following command ' + 'to see which commits will be uploaded: ' % len(commits)) + print('git log %s..%s' % (parent, ref_to_push)) + print('You can also use `git squash-branch` to squash these into a ' + 'single commit.') + confirm_or_exit(action='upload') + + reviewers = sorted(change_desc.get_reviewers()) + cc = [] + # Add default, watchlist, presubmit ccs if this is the initial upload + # and CL is not private and auto-ccing has not been disabled. + if not options.private and not options.no_autocc and not self.GetIssue( + ): + cc = self.GetCCList().split(',') + if len(cc) > 100: + lsc = ('https://chromium.googlesource.com/chromium/src/+/HEAD/docs/' + 'process/lsc/lsc_workflow.md') + print('WARNING: This will auto-CC %s users.' % len(cc)) + print('LSC may be more appropriate: %s' % lsc) + print('You can also use the --no-autocc flag to disable auto-CC.') + confirm_or_exit(action='continue') + # Add cc's from the --cc flag. + if options.cc: + cc.extend(options.cc) + cc = [email.strip() for email in cc if email.strip()] + if change_desc.get_cced(): + cc.extend(change_desc.get_cced()) + if self.GetGerritHost() == 'chromium-review.googlesource.com': + valid_accounts = set(reviewers + cc) + # TODO(crbug/877717): relax this for all hosts. + else: + valid_accounts = gerrit_util.ValidAccounts(self.GetGerritHost(), + reviewers + cc) + logging.info('accounts %s are recognized, %s invalid', + sorted(valid_accounts), + set(reviewers + cc).difference(set(valid_accounts))) + + # Extra options that can be specified at push time. Doc: + # https://gerrit-review.googlesource.com/Documentation/user-upload.html + refspec_opts = self._GetRefSpecOptions(options, change_desc) + + for r in sorted(reviewers): + if r in valid_accounts: + refspec_opts.append('r=%s' % r) + reviewers.remove(r) + else: + # TODO(tandrii): this should probably be a hard failure. + print( + 'WARNING: reviewer %s doesn\'t have a Gerrit account, skipping' + % r) + for c in sorted(cc): + # refspec option will be rejected if cc doesn't correspond to an + # account, even though REST call to add such arbitrary cc may + # succeed. + if c in valid_accounts: + refspec_opts.append('cc=%s' % c) + cc.remove(c) + + refspec_suffix = '' + if refspec_opts: + refspec_suffix = '%' + ','.join(refspec_opts) + assert ' ' not in refspec_suffix, ( + 'spaces not allowed in refspec: "%s"' % refspec_suffix) + refspec = '%s:refs/for/%s%s' % (ref_to_push, branch, refspec_suffix) + + git_push_metadata = { + 'gerrit_host': self.GetGerritHost(), + 'title': options.title or '', + 'change_id': change_id, + 'description': change_desc.description, + } + + # Gerrit may or may not update fast enough to return the correct + # patchset number after we push. Get the pre-upload patchset and + # increment later. + latest_ps = self.GetMostRecentPatchset(update=False) or 0 + + push_stdout = self._RunGitPushWithTraces(refspec, refspec_opts, + git_push_metadata, + options.push_options) + + if options.squash: + regex = re.compile(r'remote:\s+https?://[\w\-\.\+\/#]*/(\d+)\s.*') + change_numbers = [ + m.group(1) for m in map(regex.match, push_stdout.splitlines()) + if m + ] + if len(change_numbers) != 1: + DieWithError(( + 'Created|Updated %d issues on Gerrit, but only 1 expected.\n' + 'Change-Id: %s') % (len(change_numbers), change_id), + change_desc) + self.SetIssue(change_numbers[0]) + self.SetPatchset(latest_ps + 1) + self._GitSetBranchConfigValue(GERRIT_SQUASH_HASH_CONFIG_KEY, + ref_to_push) + + if self.GetIssue() and (reviewers or cc): + # GetIssue() is not set in case of non-squash uploads according to + # tests. TODO(crbug.com/751901): non-squash uploads in git cl should + # be removed. + gerrit_util.AddReviewers(self.GetGerritHost(), + self._GerritChangeIdentifier(), + reviewers, + cc, + notify=bool(options.send_mail)) + + return 0 + + def _ComputeParent(self, remote, upstream_branch, custom_cl_base, force, + change_desc): + """Computes parent of the generated commit to be uploaded to Gerrit. Returns revision or a ref name. """ - if custom_cl_base: - # Try to avoid creating additional unintended CLs when uploading, unless - # user wants to take this risk. - local_ref_of_target_remote = self.GetRemoteBranch()[1] - code, _ = RunGitWithCode(['merge-base', '--is-ancestor', custom_cl_base, - local_ref_of_target_remote]) - if code == 1: - print('\nWARNING: Manually specified base of this CL `%s` ' - 'doesn\'t seem to belong to target remote branch `%s`.\n\n' - 'If you proceed with upload, more than 1 CL may be created by ' - 'Gerrit as a result, in turn confusing or crashing git cl.\n\n' - 'If you are certain that specified base `%s` has already been ' - 'uploaded to Gerrit as another CL, you may proceed.\n' % - (custom_cl_base, local_ref_of_target_remote, custom_cl_base)) - if not force: - confirm_or_exit( - 'Do you take responsibility for cleaning up potential mess ' - 'resulting from proceeding with upload?', - action='upload') - return custom_cl_base + if custom_cl_base: + # Try to avoid creating additional unintended CLs when uploading, + # unless user wants to take this risk. + local_ref_of_target_remote = self.GetRemoteBranch()[1] + code, _ = RunGitWithCode([ + 'merge-base', '--is-ancestor', custom_cl_base, + local_ref_of_target_remote + ]) + if code == 1: + print( + '\nWARNING: Manually specified base of this CL `%s` ' + 'doesn\'t seem to belong to target remote branch `%s`.\n\n' + 'If you proceed with upload, more than 1 CL may be created by ' + 'Gerrit as a result, in turn confusing or crashing git cl.\n\n' + 'If you are certain that specified base `%s` has already been ' + 'uploaded to Gerrit as another CL, you may proceed.\n' % + (custom_cl_base, local_ref_of_target_remote, + custom_cl_base)) + if not force: + confirm_or_exit( + 'Do you take responsibility for cleaning up potential mess ' + 'resulting from proceeding with upload?', + action='upload') + return custom_cl_base - if remote != '.': - return self.GetCommonAncestorWithUpstream() + if remote != '.': + return self.GetCommonAncestorWithUpstream() - # If our upstream branch is local, we base our squashed commit on its - # squashed version. - upstream_branch_name = scm.GIT.ShortBranchName(upstream_branch) + # If our upstream branch is local, we base our squashed commit on its + # squashed version. + upstream_branch_name = scm.GIT.ShortBranchName(upstream_branch) - if upstream_branch_name == 'master': - return self.GetCommonAncestorWithUpstream() - if upstream_branch_name == 'main': - return self.GetCommonAncestorWithUpstream() + if upstream_branch_name == 'master': + return self.GetCommonAncestorWithUpstream() + if upstream_branch_name == 'main': + return self.GetCommonAncestorWithUpstream() - # Check the squashed hash of the parent. - # TODO(tandrii): consider checking parent change in Gerrit and using its - # hash if tree hash of latest parent revision (patchset) in Gerrit matches - # the tree hash of the parent branch. The upside is less likely bogus - # requests to reupload parent change just because it's uploadhash is - # missing, yet the downside likely exists, too (albeit unknown to me yet). - parent = scm.GIT.GetBranchConfig(settings.GetRoot(), upstream_branch_name, - GERRIT_SQUASH_HASH_CONFIG_KEY) - # Verify that the upstream branch has been uploaded too, otherwise - # Gerrit will create additional CLs when uploading. - if not parent or (RunGitSilent(['rev-parse', upstream_branch + ':']) != - RunGitSilent(['rev-parse', parent + ':'])): - DieWithError( - '\nUpload upstream branch %s first.\n' - 'It is likely that this branch has been rebased since its last ' - 'upload, so you just need to upload it again.\n' - '(If you uploaded it with --no-squash, then branch dependencies ' - 'are not supported, and you should reupload with --squash.)' - % upstream_branch_name, - change_desc) - return parent + # Check the squashed hash of the parent. + # TODO(tandrii): consider checking parent change in Gerrit and using its + # hash if tree hash of latest parent revision (patchset) in Gerrit + # matches the tree hash of the parent branch. The upside is less likely + # bogus requests to reupload parent change just because it's uploadhash + # is missing, yet the downside likely exists, too (albeit unknown to me + # yet). + parent = scm.GIT.GetBranchConfig(settings.GetRoot(), + upstream_branch_name, + GERRIT_SQUASH_HASH_CONFIG_KEY) + # Verify that the upstream branch has been uploaded too, otherwise + # Gerrit will create additional CLs when uploading. + if not parent or (RunGitSilent(['rev-parse', upstream_branch + ':']) != + RunGitSilent(['rev-parse', parent + ':'])): + DieWithError( + '\nUpload upstream branch %s first.\n' + 'It is likely that this branch has been rebased since its last ' + 'upload, so you just need to upload it again.\n' + '(If you uploaded it with --no-squash, then branch dependencies ' + 'are not supported, and you should reupload with --squash.)' % + upstream_branch_name, change_desc) + return parent - def _UpdateWithExternalChanges(self): - """Updates workspace with external changes. + def _UpdateWithExternalChanges(self): + """Updates workspace with external changes. Returns the commit hash that should be used as the merge base on upload. """ - local_ps = self.GetPatchset() - if local_ps is None: - return + local_ps = self.GetPatchset() + if local_ps is None: + return - external_ps = self.GetMostRecentPatchset(update=False) - if external_ps is None or local_ps == external_ps or \ - not self._IsPatchsetRangeSignificant(local_ps + 1, external_ps): - return + external_ps = self.GetMostRecentPatchset(update=False) + if external_ps is None or local_ps == external_ps or \ + not self._IsPatchsetRangeSignificant(local_ps + 1, external_ps): + return - num_changes = external_ps - local_ps - if num_changes > 1: - change_words = 'changes were' - else: - change_words = 'change was' - print('\n%d external %s published to %s:\n' % - (num_changes, change_words, self.GetIssueURL(short=True))) + num_changes = external_ps - local_ps + if num_changes > 1: + change_words = 'changes were' + else: + change_words = 'change was' + print('\n%d external %s published to %s:\n' % + (num_changes, change_words, self.GetIssueURL(short=True))) - # Print an overview of external changes. - ps_to_commit = {} - ps_to_info = {} - revisions = self._GetChangeDetail(['ALL_REVISIONS']) - for commit_id, revision_info in revisions.get('revisions', {}).items(): - ps_num = revision_info['_number'] - ps_to_commit[ps_num] = commit_id - ps_to_info[ps_num] = revision_info + # Print an overview of external changes. + ps_to_commit = {} + ps_to_info = {} + revisions = self._GetChangeDetail(['ALL_REVISIONS']) + for commit_id, revision_info in revisions.get('revisions', {}).items(): + ps_num = revision_info['_number'] + ps_to_commit[ps_num] = commit_id + ps_to_info[ps_num] = revision_info - for ps in range(external_ps, local_ps, -1): - commit = ps_to_commit[ps][:8] - desc = ps_to_info[ps].get('description', '') - print('Patchset %d [%s] %s' % (ps, commit, desc)) + for ps in range(external_ps, local_ps, -1): + commit = ps_to_commit[ps][:8] + desc = ps_to_info[ps].get('description', '') + print('Patchset %d [%s] %s' % (ps, commit, desc)) - print('\nSee diff at: %s/%d..%d' % - (self.GetIssueURL(short=True), local_ps, external_ps)) - print('\nUploading without applying patches will override them.') + print('\nSee diff at: %s/%d..%d' % + (self.GetIssueURL(short=True), local_ps, external_ps)) + print('\nUploading without applying patches will override them.') - if not ask_for_explicit_yes('Get the latest changes and apply on top?'): - return + if not ask_for_explicit_yes('Get the latest changes and apply on top?'): + return - # Get latest Gerrit merge base. Use the first parent even if multiple exist. - external_parent = self._GetChangeCommit(revision=external_ps)['parents'][0] - external_base = external_parent['commit'] + # Get latest Gerrit merge base. Use the first parent even if multiple + # exist. + external_parent = self._GetChangeCommit( + revision=external_ps)['parents'][0] + external_base = external_parent['commit'] - branch = git_common.current_branch() - local_base = self.GetCommonAncestorWithUpstream() - if local_base != external_base: - print('\nLocal merge base %s is different from Gerrit %s.\n' % - (local_base, external_base)) - if git_common.upstream(branch): - confirm_or_exit('Can\'t apply the latest changes from Gerrit.\n' - 'Continue with upload and override the latest changes?') - return - print('No upstream branch set. Continuing upload with Gerrit merge base.') + branch = git_common.current_branch() + local_base = self.GetCommonAncestorWithUpstream() + if local_base != external_base: + print('\nLocal merge base %s is different from Gerrit %s.\n' % + (local_base, external_base)) + if git_common.upstream(branch): + confirm_or_exit( + 'Can\'t apply the latest changes from Gerrit.\n' + 'Continue with upload and override the latest changes?') + return + print( + 'No upstream branch set. Continuing upload with Gerrit merge base.' + ) - external_parent_last_uploaded = self._GetChangeCommit( - revision=local_ps)['parents'][0] - external_base_last_uploaded = external_parent_last_uploaded['commit'] + external_parent_last_uploaded = self._GetChangeCommit( + revision=local_ps)['parents'][0] + external_base_last_uploaded = external_parent_last_uploaded['commit'] - if external_base != external_base_last_uploaded: - print('\nPatch set merge bases are different (%s, %s).\n' % - (external_base_last_uploaded, external_base)) - confirm_or_exit('Can\'t apply the latest changes from Gerrit.\n' - 'Continue with upload and override the latest changes?') - return + if external_base != external_base_last_uploaded: + print('\nPatch set merge bases are different (%s, %s).\n' % + (external_base_last_uploaded, external_base)) + confirm_or_exit( + 'Can\'t apply the latest changes from Gerrit.\n' + 'Continue with upload and override the latest changes?') + return - # Fetch Gerrit's CL base if it doesn't exist locally. - remote, _ = self.GetRemoteBranch() - if not scm.GIT.IsValidRevision(settings.GetRoot(), external_base): - RunGitSilent(['fetch', remote, external_base]) + # Fetch Gerrit's CL base if it doesn't exist locally. + remote, _ = self.GetRemoteBranch() + if not scm.GIT.IsValidRevision(settings.GetRoot(), external_base): + RunGitSilent(['fetch', remote, external_base]) - # Get the diff between local_ps and external_ps. - print('Fetching changes...') - issue = self.GetIssue() - changes_ref = 'refs/changes/%02d/%d/' % (issue % 100, issue) - RunGitSilent(['fetch', remote, changes_ref + str(local_ps)]) - last_uploaded = RunGitSilent(['rev-parse', 'FETCH_HEAD']).strip() - RunGitSilent(['fetch', remote, changes_ref + str(external_ps)]) - latest_external = RunGitSilent(['rev-parse', 'FETCH_HEAD']).strip() + # Get the diff between local_ps and external_ps. + print('Fetching changes...') + issue = self.GetIssue() + changes_ref = 'refs/changes/%02d/%d/' % (issue % 100, issue) + RunGitSilent(['fetch', remote, changes_ref + str(local_ps)]) + last_uploaded = RunGitSilent(['rev-parse', 'FETCH_HEAD']).strip() + RunGitSilent(['fetch', remote, changes_ref + str(external_ps)]) + latest_external = RunGitSilent(['rev-parse', 'FETCH_HEAD']).strip() - # If the commit parents are different, don't apply the diff as it very - # likely contains many more changes not relevant to this CL. - parents = RunGitSilent( - ['rev-parse', - '%s~1' % (last_uploaded), - '%s~1' % (latest_external)]).strip().split() - assert len(parents) == 2, 'Expected two parents.' - if parents[0] != parents[1]: - confirm_or_exit( - 'Can\'t apply the latest changes from Gerrit (parent mismatch ' - 'between PS).\n' - 'Continue with upload and override the latest changes?') - return + # If the commit parents are different, don't apply the diff as it very + # likely contains many more changes not relevant to this CL. + parents = RunGitSilent( + ['rev-parse', + '%s~1' % (last_uploaded), + '%s~1' % (latest_external)]).strip().split() + assert len(parents) == 2, 'Expected two parents.' + if parents[0] != parents[1]: + confirm_or_exit( + 'Can\'t apply the latest changes from Gerrit (parent mismatch ' + 'between PS).\n' + 'Continue with upload and override the latest changes?') + return - diff = RunGitSilent(['diff', '%s..%s' % (last_uploaded, latest_external)]) + diff = RunGitSilent( + ['diff', '%s..%s' % (last_uploaded, latest_external)]) - # Diff can be empty in the case of trivial rebases. - if not diff: - return external_base + # Diff can be empty in the case of trivial rebases. + if not diff: + return external_base - # Apply the diff. - with gclient_utils.temporary_file() as diff_tempfile: - gclient_utils.FileWrite(diff_tempfile, diff) - clean_patch = RunGitWithCode(['apply', '--check', diff_tempfile])[0] == 0 - RunGitSilent(['apply', '-3', '--intent-to-add', diff_tempfile]) - if not clean_patch: - # Normally patchset is set after upload. But because we exit, that never - # happens. Updating here makes sure that subsequent uploads don't need - # to fetch/apply the same diff again. - self.SetPatchset(external_ps) - DieWithError('\nPatch did not apply cleanly. Please resolve any ' - 'conflicts and reupload.') + # Apply the diff. + with gclient_utils.temporary_file() as diff_tempfile: + gclient_utils.FileWrite(diff_tempfile, diff) + clean_patch = RunGitWithCode(['apply', '--check', + diff_tempfile])[0] == 0 + RunGitSilent(['apply', '-3', '--intent-to-add', diff_tempfile]) + if not clean_patch: + # Normally patchset is set after upload. But because we exit, + # that never happens. Updating here makes sure that subsequent + # uploads don't need to fetch/apply the same diff again. + self.SetPatchset(external_ps) + DieWithError( + '\nPatch did not apply cleanly. Please resolve any ' + 'conflicts and reupload.') - message = 'Incorporate external changes from ' - if num_changes == 1: - message += 'patchset %d' % external_ps - else: - message += 'patchsets %d to %d' % (local_ps + 1, external_ps) - RunGitSilent(['commit', '-am', message]) - # TODO(crbug.com/1382528): Use the previous commit's message as a default - # patchset title instead of this 'Incorporate' message. - return external_base + message = 'Incorporate external changes from ' + if num_changes == 1: + message += 'patchset %d' % external_ps + else: + message += 'patchsets %d to %d' % (local_ps + 1, external_ps) + RunGitSilent(['commit', '-am', message]) + # TODO(crbug.com/1382528): Use the previous commit's message as a + # default patchset title instead of this 'Incorporate' message. + return external_base - def _AddChangeIdToCommitMessage(self, log_desc, args): - """Re-commits using the current message, assumes the commit hook is in + def _AddChangeIdToCommitMessage(self, log_desc, args): + """Re-commits using the current message, assumes the commit hook is in place. """ - RunGit(['commit', '--amend', '-m', log_desc]) - new_log_desc = _create_description_from_log(args) - if git_footers.get_footer_change_id(new_log_desc): - print('git-cl: Added Change-Id to commit message.') - return new_log_desc + RunGit(['commit', '--amend', '-m', log_desc]) + new_log_desc = _create_description_from_log(args) + if git_footers.get_footer_change_id(new_log_desc): + print('git-cl: Added Change-Id to commit message.') + return new_log_desc - DieWithError('ERROR: Gerrit commit-msg hook not installed.') + DieWithError('ERROR: Gerrit commit-msg hook not installed.') - def CannotTriggerTryJobReason(self): - try: - data = self._GetChangeDetail() - except GerritChangeNotExists: - return 'Gerrit doesn\'t know about your change %s' % self.GetIssue() + def CannotTriggerTryJobReason(self): + try: + data = self._GetChangeDetail() + except GerritChangeNotExists: + return 'Gerrit doesn\'t know about your change %s' % self.GetIssue() - if data['status'] in ('ABANDONED', 'MERGED'): - return 'CL %s is closed' % self.GetIssue() + if data['status'] in ('ABANDONED', 'MERGED'): + return 'CL %s is closed' % self.GetIssue() - def GetGerritChange(self, patchset=None): - """Returns a buildbucket.v2.GerritChange message for the current issue.""" - host = urllib.parse.urlparse(self.GetCodereviewServer()).hostname - issue = self.GetIssue() - patchset = int(patchset or self.GetPatchset()) - data = self._GetChangeDetail(['ALL_REVISIONS']) + def GetGerritChange(self, patchset=None): + """Returns a buildbucket.v2.GerritChange message for the current issue.""" + host = urllib.parse.urlparse(self.GetCodereviewServer()).hostname + issue = self.GetIssue() + patchset = int(patchset or self.GetPatchset()) + data = self._GetChangeDetail(['ALL_REVISIONS']) - assert host and issue and patchset, 'CL must be uploaded first' + assert host and issue and patchset, 'CL must be uploaded first' - has_patchset = any( - int(revision_data['_number']) == patchset - for revision_data in data['revisions'].values()) - if not has_patchset: - raise Exception('Patchset %d is not known in Gerrit change %d' % - (patchset, self.GetIssue())) + has_patchset = any( + int(revision_data['_number']) == patchset + for revision_data in data['revisions'].values()) + if not has_patchset: + raise Exception('Patchset %d is not known in Gerrit change %d' % + (patchset, self.GetIssue())) - return { - 'host': host, - 'change': issue, - 'project': data['project'], - 'patchset': patchset, - } + return { + 'host': host, + 'change': issue, + 'project': data['project'], + 'patchset': patchset, + } - def GetIssueOwner(self): - return self._GetChangeDetail(['DETAILED_ACCOUNTS'])['owner']['email'] + def GetIssueOwner(self): + return self._GetChangeDetail(['DETAILED_ACCOUNTS'])['owner']['email'] - def GetReviewers(self): - details = self._GetChangeDetail(['DETAILED_ACCOUNTS']) - return [r['email'] for r in details['reviewers'].get('REVIEWER', [])] + def GetReviewers(self): + details = self._GetChangeDetail(['DETAILED_ACCOUNTS']) + return [r['email'] for r in details['reviewers'].get('REVIEWER', [])] def _get_bug_line_values(default_project_prefix, bugs): - """Given default_project_prefix and comma separated list of bugs, yields bug + """Given default_project_prefix and comma separated list of bugs, yields bug line values. Each bug can be either: @@ -3358,424 +3483,438 @@ def _get_bug_line_values(default_project_prefix, bugs): >>> list(_get_bug_line_values('v8:', '123,chromium:789')) ['v8:123', 'chromium:789'] """ - default_bugs = [] - others = [] - for bug in bugs.split(','): - bug = bug.strip() - if bug: - try: - default_bugs.append(int(bug)) - except ValueError: - others.append(bug) + default_bugs = [] + others = [] + for bug in bugs.split(','): + bug = bug.strip() + if bug: + try: + default_bugs.append(int(bug)) + except ValueError: + others.append(bug) - if default_bugs: - default_bugs = ','.join(map(str, default_bugs)) - if default_project_prefix: - if not default_project_prefix.endswith(':'): - default_project_prefix += ':' - yield '%s%s' % (default_project_prefix, default_bugs) - else: - yield default_bugs - for other in sorted(others): - # Don't bother finding common prefixes, CLs with >2 bugs are very very rare. - yield other + if default_bugs: + default_bugs = ','.join(map(str, default_bugs)) + if default_project_prefix: + if not default_project_prefix.endswith(':'): + default_project_prefix += ':' + yield '%s%s' % (default_project_prefix, default_bugs) + else: + yield default_bugs + for other in sorted(others): + # Don't bother finding common prefixes, CLs with >2 bugs are very very + # rare. + yield other def FindCodereviewSettingsFile(filename='codereview.settings'): - """Finds the given file starting in the cwd and going up. + """Finds the given file starting in the cwd and going up. Only looks up to the top of the repository unless an 'inherit-review-settings-ok' file exists in the root of the repository. """ - inherit_ok_file = 'inherit-review-settings-ok' - cwd = os.getcwd() - root = settings.GetRoot() - if os.path.isfile(os.path.join(root, inherit_ok_file)): - root = None - while True: - if os.path.isfile(os.path.join(cwd, filename)): - return open(os.path.join(cwd, filename)) - if cwd == root: - break - parent_dir = os.path.dirname(cwd) - if parent_dir == cwd: - # We hit the system root directory. - break - cwd = parent_dir - return None + inherit_ok_file = 'inherit-review-settings-ok' + cwd = os.getcwd() + root = settings.GetRoot() + if os.path.isfile(os.path.join(root, inherit_ok_file)): + root = None + while True: + if os.path.isfile(os.path.join(cwd, filename)): + return open(os.path.join(cwd, filename)) + if cwd == root: + break + parent_dir = os.path.dirname(cwd) + if parent_dir == cwd: + # We hit the system root directory. + break + cwd = parent_dir + return None def LoadCodereviewSettingsFromFile(fileobj): - """Parses a codereview.settings file and updates hooks.""" - keyvals = gclient_utils.ParseCodereviewSettingsContent(fileobj.read()) + """Parses a codereview.settings file and updates hooks.""" + keyvals = gclient_utils.ParseCodereviewSettingsContent(fileobj.read()) - def SetProperty(name, setting, unset_error_ok=False): - fullname = 'rietveld.' + name - if setting in keyvals: - RunGit(['config', fullname, keyvals[setting]]) - else: - RunGit(['config', '--unset-all', fullname], error_ok=unset_error_ok) + def SetProperty(name, setting, unset_error_ok=False): + fullname = 'rietveld.' + name + if setting in keyvals: + RunGit(['config', fullname, keyvals[setting]]) + else: + RunGit(['config', '--unset-all', fullname], error_ok=unset_error_ok) - if not keyvals.get('GERRIT_HOST', False): - SetProperty('server', 'CODE_REVIEW_SERVER') - # Only server setting is required. Other settings can be absent. - # In that case, we ignore errors raised during option deletion attempt. - SetProperty('cc', 'CC_LIST', unset_error_ok=True) - SetProperty('tree-status-url', 'STATUS', unset_error_ok=True) - SetProperty('viewvc-url', 'VIEW_VC', unset_error_ok=True) - SetProperty('bug-prefix', 'BUG_PREFIX', unset_error_ok=True) - SetProperty('cpplint-regex', 'LINT_REGEX', unset_error_ok=True) - SetProperty('cpplint-ignore-regex', 'LINT_IGNORE_REGEX', unset_error_ok=True) - SetProperty('run-post-upload-hook', 'RUN_POST_UPLOAD_HOOK', - unset_error_ok=True) - SetProperty( - 'format-full-by-default', 'FORMAT_FULL_BY_DEFAULT', unset_error_ok=True) + if not keyvals.get('GERRIT_HOST', False): + SetProperty('server', 'CODE_REVIEW_SERVER') + # Only server setting is required. Other settings can be absent. + # In that case, we ignore errors raised during option deletion attempt. + SetProperty('cc', 'CC_LIST', unset_error_ok=True) + SetProperty('tree-status-url', 'STATUS', unset_error_ok=True) + SetProperty('viewvc-url', 'VIEW_VC', unset_error_ok=True) + SetProperty('bug-prefix', 'BUG_PREFIX', unset_error_ok=True) + SetProperty('cpplint-regex', 'LINT_REGEX', unset_error_ok=True) + SetProperty('cpplint-ignore-regex', + 'LINT_IGNORE_REGEX', + unset_error_ok=True) + SetProperty('run-post-upload-hook', + 'RUN_POST_UPLOAD_HOOK', + unset_error_ok=True) + SetProperty('format-full-by-default', + 'FORMAT_FULL_BY_DEFAULT', + unset_error_ok=True) - if 'GERRIT_HOST' in keyvals: - RunGit(['config', 'gerrit.host', keyvals['GERRIT_HOST']]) + if 'GERRIT_HOST' in keyvals: + RunGit(['config', 'gerrit.host', keyvals['GERRIT_HOST']]) - if 'GERRIT_SQUASH_UPLOADS' in keyvals: - RunGit(['config', 'gerrit.squash-uploads', - keyvals['GERRIT_SQUASH_UPLOADS']]) + if 'GERRIT_SQUASH_UPLOADS' in keyvals: + RunGit([ + 'config', 'gerrit.squash-uploads', keyvals['GERRIT_SQUASH_UPLOADS'] + ]) - if 'GERRIT_SKIP_ENSURE_AUTHENTICATED' in keyvals: - RunGit(['config', 'gerrit.skip-ensure-authenticated', - keyvals['GERRIT_SKIP_ENSURE_AUTHENTICATED']]) + if 'GERRIT_SKIP_ENSURE_AUTHENTICATED' in keyvals: + RunGit([ + 'config', 'gerrit.skip-ensure-authenticated', + keyvals['GERRIT_SKIP_ENSURE_AUTHENTICATED'] + ]) - if 'PUSH_URL_CONFIG' in keyvals and 'ORIGIN_URL_CONFIG' in keyvals: - # should be of the form - # PUSH_URL_CONFIG: url.ssh://gitrw.chromium.org.pushinsteadof - # ORIGIN_URL_CONFIG: http://src.chromium.org/git - RunGit(['config', keyvals['PUSH_URL_CONFIG'], - keyvals['ORIGIN_URL_CONFIG']]) + if 'PUSH_URL_CONFIG' in keyvals and 'ORIGIN_URL_CONFIG' in keyvals: + # should be of the form + # PUSH_URL_CONFIG: url.ssh://gitrw.chromium.org.pushinsteadof + # ORIGIN_URL_CONFIG: http://src.chromium.org/git + RunGit([ + 'config', keyvals['PUSH_URL_CONFIG'], keyvals['ORIGIN_URL_CONFIG'] + ]) def urlretrieve(source, destination): - """Downloads a network object to a local file, like urllib.urlretrieve. + """Downloads a network object to a local file, like urllib.urlretrieve. This is necessary because urllib is broken for SSL connections via a proxy. """ - with open(destination, 'wb') as f: - f.write(urllib.request.urlopen(source).read()) + with open(destination, 'wb') as f: + f.write(urllib.request.urlopen(source).read()) def hasSheBang(fname): - """Checks fname is a #! script.""" - with open(fname) as f: - return f.read(2).startswith('#!') + """Checks fname is a #! script.""" + with open(fname) as f: + return f.read(2).startswith('#!') def DownloadGerritHook(force): - """Downloads and installs a Gerrit commit-msg hook. + """Downloads and installs a Gerrit commit-msg hook. Args: force: True to update hooks. False to install hooks if not present. """ - src = 'https://gerrit-review.googlesource.com/tools/hooks/commit-msg' - dst = os.path.join(settings.GetRoot(), '.git', 'hooks', 'commit-msg') - if not os.access(dst, os.X_OK): - if os.path.exists(dst): - if not force: - return - try: - urlretrieve(src, dst) - if not hasSheBang(dst): - DieWithError('Not a script: %s\n' - 'You need to download from\n%s\n' - 'into .git/hooks/commit-msg and ' - 'chmod +x .git/hooks/commit-msg' % (dst, src)) - os.chmod(dst, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) - except Exception: - if os.path.exists(dst): - os.remove(dst) - DieWithError('\nFailed to download hooks.\n' - 'You need to download from\n%s\n' - 'into .git/hooks/commit-msg and ' - 'chmod +x .git/hooks/commit-msg' % src) + src = 'https://gerrit-review.googlesource.com/tools/hooks/commit-msg' + dst = os.path.join(settings.GetRoot(), '.git', 'hooks', 'commit-msg') + if not os.access(dst, os.X_OK): + if os.path.exists(dst): + if not force: + return + try: + urlretrieve(src, dst) + if not hasSheBang(dst): + DieWithError('Not a script: %s\n' + 'You need to download from\n%s\n' + 'into .git/hooks/commit-msg and ' + 'chmod +x .git/hooks/commit-msg' % (dst, src)) + os.chmod(dst, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + except Exception: + if os.path.exists(dst): + os.remove(dst) + DieWithError('\nFailed to download hooks.\n' + 'You need to download from\n%s\n' + 'into .git/hooks/commit-msg and ' + 'chmod +x .git/hooks/commit-msg' % src) class _GitCookiesChecker(object): - """Provides facilities for validating and suggesting fixes to .gitcookies.""" + """Provides facilities for validating and suggesting fixes to .gitcookies.""" + def __init__(self): + # Cached list of [host, identity, source], where source is either + # .gitcookies or .netrc. + self._all_hosts = None - def __init__(self): - # Cached list of [host, identity, source], where source is either - # .gitcookies or .netrc. - self._all_hosts = None - - def ensure_configured_gitcookies(self): - """Runs checks and suggests fixes to make git use .gitcookies from default + def ensure_configured_gitcookies(self): + """Runs checks and suggests fixes to make git use .gitcookies from default path.""" - default = gerrit_util.CookiesAuthenticator.get_gitcookies_path() - configured_path = RunGitSilent( - ['config', '--global', 'http.cookiefile']).strip() - configured_path = os.path.expanduser(configured_path) - if configured_path: - self._ensure_default_gitcookies_path(configured_path, default) - else: - self._configure_gitcookies_path(default) + default = gerrit_util.CookiesAuthenticator.get_gitcookies_path() + configured_path = RunGitSilent( + ['config', '--global', 'http.cookiefile']).strip() + configured_path = os.path.expanduser(configured_path) + if configured_path: + self._ensure_default_gitcookies_path(configured_path, default) + else: + self._configure_gitcookies_path(default) - @staticmethod - def _ensure_default_gitcookies_path(configured_path, default_path): - assert configured_path - if configured_path == default_path: - print('git is already configured to use your .gitcookies from %s' % - configured_path) - return + @staticmethod + def _ensure_default_gitcookies_path(configured_path, default_path): + assert configured_path + if configured_path == default_path: + print('git is already configured to use your .gitcookies from %s' % + configured_path) + return - print('WARNING: You have configured custom path to .gitcookies: %s\n' - 'Gerrit and other depot_tools expect .gitcookies at %s\n' % - (configured_path, default_path)) + print('WARNING: You have configured custom path to .gitcookies: %s\n' + 'Gerrit and other depot_tools expect .gitcookies at %s\n' % + (configured_path, default_path)) - if not os.path.exists(configured_path): - print('However, your configured .gitcookies file is missing.') - confirm_or_exit('Reconfigure git to use default .gitcookies?', - action='reconfigure') - RunGit(['config', '--global', 'http.cookiefile', default_path]) - return + if not os.path.exists(configured_path): + print('However, your configured .gitcookies file is missing.') + confirm_or_exit('Reconfigure git to use default .gitcookies?', + action='reconfigure') + RunGit(['config', '--global', 'http.cookiefile', default_path]) + return - if os.path.exists(default_path): - print('WARNING: default .gitcookies file already exists %s' % - default_path) - DieWithError('Please delete %s manually and re-run git cl creds-check' % - default_path) + if os.path.exists(default_path): + print('WARNING: default .gitcookies file already exists %s' % + default_path) + DieWithError( + 'Please delete %s manually and re-run git cl creds-check' % + default_path) - confirm_or_exit('Move existing .gitcookies to default location?', - action='move') - shutil.move(configured_path, default_path) - RunGit(['config', '--global', 'http.cookiefile', default_path]) - print('Moved and reconfigured git to use .gitcookies from %s' % - default_path) + confirm_or_exit('Move existing .gitcookies to default location?', + action='move') + shutil.move(configured_path, default_path) + RunGit(['config', '--global', 'http.cookiefile', default_path]) + print('Moved and reconfigured git to use .gitcookies from %s' % + default_path) - @staticmethod - def _configure_gitcookies_path(default_path): - netrc_path = gerrit_util.CookiesAuthenticator.get_netrc_path() - if os.path.exists(netrc_path): - print('You seem to be using outdated .netrc for git credentials: %s' % - netrc_path) - print('This tool will guide you through setting up recommended ' - '.gitcookies store for git credentials.\n' - '\n' - 'IMPORTANT: If something goes wrong and you decide to go back, do:\n' - ' git config --global --unset http.cookiefile\n' - ' mv %s %s.backup\n\n' % (default_path, default_path)) - confirm_or_exit(action='setup .gitcookies') - RunGit(['config', '--global', 'http.cookiefile', default_path]) - print('Configured git to use .gitcookies from %s' % default_path) + @staticmethod + def _configure_gitcookies_path(default_path): + netrc_path = gerrit_util.CookiesAuthenticator.get_netrc_path() + if os.path.exists(netrc_path): + print( + 'You seem to be using outdated .netrc for git credentials: %s' % + netrc_path) + print( + 'This tool will guide you through setting up recommended ' + '.gitcookies store for git credentials.\n' + '\n' + 'IMPORTANT: If something goes wrong and you decide to go back, do:\n' + ' git config --global --unset http.cookiefile\n' + ' mv %s %s.backup\n\n' % (default_path, default_path)) + confirm_or_exit(action='setup .gitcookies') + RunGit(['config', '--global', 'http.cookiefile', default_path]) + print('Configured git to use .gitcookies from %s' % default_path) - def get_hosts_with_creds(self, include_netrc=False): - if self._all_hosts is None: - a = gerrit_util.CookiesAuthenticator() - self._all_hosts = [(h, u, s) for h, u, s in itertools.chain(( - (h, u, '.netrc') for h, (u, _, _) in a.netrc.hosts.items()), ( - (h, u, '.gitcookies') for h, (u, _) in a.gitcookies.items())) - if h.endswith(_GOOGLESOURCE)] + def get_hosts_with_creds(self, include_netrc=False): + if self._all_hosts is None: + a = gerrit_util.CookiesAuthenticator() + self._all_hosts = [(h, u, s) for h, u, s in itertools.chain(( + (h, u, '.netrc') for h, (u, _, _) in a.netrc.hosts.items()), ( + (h, u, '.gitcookies') + for h, (u, _) in a.gitcookies.items())) + if h.endswith(_GOOGLESOURCE)] - if include_netrc: - return self._all_hosts - return [(h, u, s) for h, u, s in self._all_hosts if s != '.netrc'] + if include_netrc: + return self._all_hosts + return [(h, u, s) for h, u, s in self._all_hosts if s != '.netrc'] - def print_current_creds(self, include_netrc=False): - hosts = sorted(self.get_hosts_with_creds(include_netrc=include_netrc)) - if not hosts: - print('No Git/Gerrit credentials found') - return - lengths = [max(map(len, (row[i] for row in hosts))) for i in range(3)] - header = [('Host', 'User', 'Which file'), - ['=' * l for l in lengths]] - for row in (header + hosts): - print('\t'.join((('%%+%ds' % l) % s) - for l, s in zip(lengths, row))) + def print_current_creds(self, include_netrc=False): + hosts = sorted(self.get_hosts_with_creds(include_netrc=include_netrc)) + if not hosts: + print('No Git/Gerrit credentials found') + return + lengths = [max(map(len, (row[i] for row in hosts))) for i in range(3)] + header = [('Host', 'User', 'Which file'), ['=' * l for l in lengths]] + for row in (header + hosts): + print('\t'.join((('%%+%ds' % l) % s) for l, s in zip(lengths, row))) - @staticmethod - def _parse_identity(identity): - """Parses identity "git-.domain" into and domain.""" - # Special case: usernames that contain ".", which are generally not - # distinguishable from sub-domains. But we do know typical domains: - if identity.endswith('.chromium.org'): - domain = 'chromium.org' - username = identity[:-len('.chromium.org')] - else: - username, domain = identity.split('.', 1) - if username.startswith('git-'): - username = username[len('git-'):] - return username, domain + @staticmethod + def _parse_identity(identity): + """Parses identity "git-.domain" into and domain.""" + # Special case: usernames that contain ".", which are generally not + # distinguishable from sub-domains. But we do know typical domains: + if identity.endswith('.chromium.org'): + domain = 'chromium.org' + username = identity[:-len('.chromium.org')] + else: + username, domain = identity.split('.', 1) + if username.startswith('git-'): + username = username[len('git-'):] + return username, domain - def has_generic_host(self): - """Returns whether generic .googlesource.com has been configured. + def has_generic_host(self): + """Returns whether generic .googlesource.com has been configured. Chrome Infra recommends to use explicit ${host}.googlesource.com instead. """ - for host, _, _ in self.get_hosts_with_creds(include_netrc=False): - if host == '.' + _GOOGLESOURCE: - return True - return False + for host, _, _ in self.get_hosts_with_creds(include_netrc=False): + if host == '.' + _GOOGLESOURCE: + return True + return False - def _get_git_gerrit_identity_pairs(self): - """Returns map from canonic host to pair of identities (Git, Gerrit). + def _get_git_gerrit_identity_pairs(self): + """Returns map from canonic host to pair of identities (Git, Gerrit). One of identities might be None, meaning not configured. """ - host_to_identity_pairs = {} - for host, identity, _ in self.get_hosts_with_creds(): - canonical = _canonical_git_googlesource_host(host) - pair = host_to_identity_pairs.setdefault(canonical, [None, None]) - idx = 0 if canonical == host else 1 - pair[idx] = identity - return host_to_identity_pairs + host_to_identity_pairs = {} + for host, identity, _ in self.get_hosts_with_creds(): + canonical = _canonical_git_googlesource_host(host) + pair = host_to_identity_pairs.setdefault(canonical, [None, None]) + idx = 0 if canonical == host else 1 + pair[idx] = identity + return host_to_identity_pairs - def get_partially_configured_hosts(self): - return set( - (host if i1 else _canonical_gerrit_googlesource_host(host)) - for host, (i1, i2) in self._get_git_gerrit_identity_pairs().items() - if None in (i1, i2) and host != '.' + _GOOGLESOURCE) + def get_partially_configured_hosts(self): + return set( + (host if i1 else _canonical_gerrit_googlesource_host(host)) + for host, (i1, i2) in self._get_git_gerrit_identity_pairs().items() + if None in (i1, i2) and host != '.' + _GOOGLESOURCE) - def get_conflicting_hosts(self): - return set( - host - for host, (i1, i2) in self._get_git_gerrit_identity_pairs().items() - if None not in (i1, i2) and i1 != i2) + def get_conflicting_hosts(self): + return set( + host + for host, (i1, i2) in self._get_git_gerrit_identity_pairs().items() + if None not in (i1, i2) and i1 != i2) - def get_duplicated_hosts(self): - counters = collections.Counter(h for h, _, _ in self.get_hosts_with_creds()) - return set(host for host, count in counters.items() if count > 1) + def get_duplicated_hosts(self): + counters = collections.Counter( + h for h, _, _ in self.get_hosts_with_creds()) + return set(host for host, count in counters.items() if count > 1) + @staticmethod + def _format_hosts(hosts, extra_column_func=None): + hosts = sorted(hosts) + assert hosts + if extra_column_func is None: + extras = [''] * len(hosts) + else: + extras = [extra_column_func(host) for host in hosts] + tmpl = '%%-%ds %%-%ds' % (max(map(len, hosts)), max(map(len, + extras))) + lines = [] + for he in zip(hosts, extras): + lines.append(tmpl % he) + return lines - @staticmethod - def _format_hosts(hosts, extra_column_func=None): - hosts = sorted(hosts) - assert hosts - if extra_column_func is None: - extras = [''] * len(hosts) - else: - extras = [extra_column_func(host) for host in hosts] - tmpl = '%%-%ds %%-%ds' % (max(map(len, hosts)), max(map(len, extras))) - lines = [] - for he in zip(hosts, extras): - lines.append(tmpl % he) - return lines + def _find_problems(self): + if self.has_generic_host(): + yield ('.googlesource.com wildcard record detected', [ + 'Chrome Infrastructure team recommends to list full host names ' + 'explicitly.' + ], None) - def _find_problems(self): - if self.has_generic_host(): - yield ('.googlesource.com wildcard record detected', - ['Chrome Infrastructure team recommends to list full host names ' - 'explicitly.'], - None) + dups = self.get_duplicated_hosts() + if dups: + yield ('The following hosts were defined twice', + self._format_hosts(dups), None) - dups = self.get_duplicated_hosts() - if dups: - yield ('The following hosts were defined twice', - self._format_hosts(dups), - None) + partial = self.get_partially_configured_hosts() + if partial: + yield ('Credentials should come in pairs for Git and Gerrit hosts. ' + 'These hosts are missing', + self._format_hosts( + partial, lambda host: 'but %s defined' % + _get_counterpart_host(host)), partial) - partial = self.get_partially_configured_hosts() - if partial: - yield ('Credentials should come in pairs for Git and Gerrit hosts. ' - 'These hosts are missing', - self._format_hosts( - partial, lambda host: 'but %s defined' % _get_counterpart_host( - host)), partial) + conflicting = self.get_conflicting_hosts() + if conflicting: + yield ( + 'The following Git hosts have differing credentials from their ' + 'Gerrit counterparts', + self._format_hosts( + conflicting, lambda host: '%s vs %s' % tuple( + self._get_git_gerrit_identity_pairs()[host])), + conflicting) - conflicting = self.get_conflicting_hosts() - if conflicting: - yield ('The following Git hosts have differing credentials from their ' - 'Gerrit counterparts', - self._format_hosts(conflicting, lambda host: '%s vs %s' % - tuple(self._get_git_gerrit_identity_pairs()[host])), - conflicting) + def find_and_report_problems(self): + """Returns True if there was at least one problem, else False.""" + found = False + bad_hosts = set() + for title, sublines, hosts in self._find_problems(): + if not found: + found = True + print('\n\n.gitcookies problem report:\n') + bad_hosts.update(hosts or []) + print(' %s%s' % (title, (':' if sublines else ''))) + if sublines: + print() + print(' %s' % '\n '.join(sublines)) + print() - def find_and_report_problems(self): - """Returns True if there was at least one problem, else False.""" - found = False - bad_hosts = set() - for title, sublines, hosts in self._find_problems(): - if not found: - found = True - print('\n\n.gitcookies problem report:\n') - bad_hosts.update(hosts or []) - print(' %s%s' % (title, (':' if sublines else ''))) - if sublines: - print() - print(' %s' % '\n '.join(sublines)) - print() - - if bad_hosts: - assert found - print(' You can manually remove corresponding lines in your %s file and ' - 'visit the following URLs with correct account to generate ' - 'correct credential lines:\n' % - gerrit_util.CookiesAuthenticator.get_gitcookies_path()) - print(' %s' % '\n '.join( - sorted( - set(gerrit_util.CookiesAuthenticator().get_new_password_url( - _canonical_git_googlesource_host(host)) - for host in bad_hosts)))) - return found + if bad_hosts: + assert found + print( + ' You can manually remove corresponding lines in your %s file and ' + 'visit the following URLs with correct account to generate ' + 'correct credential lines:\n' % + gerrit_util.CookiesAuthenticator.get_gitcookies_path()) + print(' %s' % '\n '.join( + sorted( + set(gerrit_util.CookiesAuthenticator().get_new_password_url( + _canonical_git_googlesource_host(host)) + for host in bad_hosts)))) + return found @metrics.collector.collect_metrics('git cl creds-check') def CMDcreds_check(parser, args): - """Checks credentials and suggests changes.""" - _, _ = parser.parse_args(args) + """Checks credentials and suggests changes.""" + _, _ = parser.parse_args(args) - # Code below checks .gitcookies. Abort if using something else. - authn = gerrit_util.Authenticator.get() - if not isinstance(authn, gerrit_util.CookiesAuthenticator): - message = ( - 'This command is not designed for bot environment. It checks ' - '~/.gitcookies file not generally used on bots.') - # TODO(crbug.com/1059384): Automatically detect when running on cloudtop. - if isinstance(authn, gerrit_util.GceAuthenticator): - message += ( - '\n' - 'If you need to run this on GCE or a cloudtop instance, ' - 'export SKIP_GCE_AUTH_FOR_GIT=1 in your env.') - DieWithError(message) + # Code below checks .gitcookies. Abort if using something else. + authn = gerrit_util.Authenticator.get() + if not isinstance(authn, gerrit_util.CookiesAuthenticator): + message = ( + 'This command is not designed for bot environment. It checks ' + '~/.gitcookies file not generally used on bots.') + # TODO(crbug.com/1059384): Automatically detect when running on + # cloudtop. + if isinstance(authn, gerrit_util.GceAuthenticator): + message += ( + '\n' + 'If you need to run this on GCE or a cloudtop instance, ' + 'export SKIP_GCE_AUTH_FOR_GIT=1 in your env.') + DieWithError(message) - checker = _GitCookiesChecker() - checker.ensure_configured_gitcookies() + checker = _GitCookiesChecker() + checker.ensure_configured_gitcookies() - print('Your .netrc and .gitcookies have credentials for these hosts:') - checker.print_current_creds(include_netrc=True) + print('Your .netrc and .gitcookies have credentials for these hosts:') + checker.print_current_creds(include_netrc=True) - if not checker.find_and_report_problems(): - print('\nNo problems detected in your .gitcookies file.') - return 0 - return 1 + if not checker.find_and_report_problems(): + print('\nNo problems detected in your .gitcookies file.') + return 0 + return 1 @metrics.collector.collect_metrics('git cl baseurl') def CMDbaseurl(parser, args): - """Gets or sets base-url for this branch.""" - _, args = parser.parse_args(args) - branchref = scm.GIT.GetBranchRef(settings.GetRoot()) - branch = scm.GIT.ShortBranchName(branchref) - if not args: - print('Current base-url:') - return RunGit(['config', 'branch.%s.base-url' % branch], - error_ok=False).strip() + """Gets or sets base-url for this branch.""" + _, args = parser.parse_args(args) + branchref = scm.GIT.GetBranchRef(settings.GetRoot()) + branch = scm.GIT.ShortBranchName(branchref) + if not args: + print('Current base-url:') + return RunGit(['config', 'branch.%s.base-url' % branch], + error_ok=False).strip() - print('Setting base-url to %s' % args[0]) - return RunGit(['config', 'branch.%s.base-url' % branch, args[0]], - error_ok=False).strip() + print('Setting base-url to %s' % args[0]) + return RunGit(['config', 'branch.%s.base-url' % branch, args[0]], + error_ok=False).strip() def color_for_status(status): - """Maps a Changelist status to color, for CMDstatus and other tools.""" - BOLD = '\033[1m' - return { - 'unsent': BOLD + Fore.YELLOW, - 'waiting': BOLD + Fore.RED, - 'reply': BOLD + Fore.YELLOW, - 'not lgtm': BOLD + Fore.RED, - 'lgtm': BOLD + Fore.GREEN, - 'commit': BOLD + Fore.MAGENTA, - 'closed': BOLD + Fore.CYAN, - 'error': BOLD + Fore.WHITE, - }.get(status, Fore.WHITE) + """Maps a Changelist status to color, for CMDstatus and other tools.""" + BOLD = '\033[1m' + return { + 'unsent': BOLD + Fore.YELLOW, + 'waiting': BOLD + Fore.RED, + 'reply': BOLD + Fore.YELLOW, + 'not lgtm': BOLD + Fore.RED, + 'lgtm': BOLD + Fore.GREEN, + 'commit': BOLD + Fore.MAGENTA, + 'closed': BOLD + Fore.CYAN, + 'error': BOLD + Fore.WHITE, + }.get(status, Fore.WHITE) def get_cl_statuses(changes, fine_grained, max_processes=None): - """Returns a blocking iterable of (cl, status) for given branches. + """Returns a blocking iterable of (cl, status) for given branches. If fine_grained is true, this will fetch CL statuses from the server. Otherwise, simply indicate if there's a matching url for the given branches. @@ -3786,55 +3925,58 @@ def get_cl_statuses(changes, fine_grained, max_processes=None): See GetStatus() for a list of possible statuses. """ - if not changes: - return + if not changes: + return - if not fine_grained: - # Fast path which doesn't involve querying codereview servers. - # Do not use get_approving_reviewers(), since it requires an HTTP request. + if not fine_grained: + # Fast path which doesn't involve querying codereview servers. + # Do not use get_approving_reviewers(), since it requires an HTTP + # request. + for cl in changes: + yield (cl, 'waiting' if cl.GetIssueURL() else 'error') + return + + # First, sort out authentication issues. + logging.debug('ensuring credentials exist') for cl in changes: - yield (cl, 'waiting' if cl.GetIssueURL() else 'error') - return + cl.EnsureAuthenticated(force=False, refresh=True) - # First, sort out authentication issues. - logging.debug('ensuring credentials exist') - for cl in changes: - cl.EnsureAuthenticated(force=False, refresh=True) + def fetch(cl): + try: + return (cl, cl.GetStatus()) + except: + # See http://crbug.com/629863. + logging.exception('failed to fetch status for cl %s:', + cl.GetIssue()) + raise - def fetch(cl): + threads_count = len(changes) + if max_processes: + threads_count = max(1, min(threads_count, max_processes)) + logging.debug('querying %d CLs using %d threads', len(changes), + threads_count) + + pool = multiprocessing.pool.ThreadPool(threads_count) + fetched_cls = set() try: - return (cl, cl.GetStatus()) - except: - # See http://crbug.com/629863. - logging.exception('failed to fetch status for cl %s:', cl.GetIssue()) - raise + it = pool.imap_unordered(fetch, changes).__iter__() + while True: + try: + cl, status = it.next(timeout=5) + except (multiprocessing.TimeoutError, StopIteration): + break + fetched_cls.add(cl) + yield cl, status + finally: + pool.close() - threads_count = len(changes) - if max_processes: - threads_count = max(1, min(threads_count, max_processes)) - logging.debug('querying %d CLs using %d threads', len(changes), threads_count) - - pool = multiprocessing.pool.ThreadPool(threads_count) - fetched_cls = set() - try: - it = pool.imap_unordered(fetch, changes).__iter__() - while True: - try: - cl, status = it.next(timeout=5) - except (multiprocessing.TimeoutError, StopIteration): - break - fetched_cls.add(cl) - yield cl, status - finally: - pool.close() - - # Add any branches that failed to fetch. - for cl in set(changes) - fetched_cls: - yield (cl, 'error') + # Add any branches that failed to fetch. + for cl in set(changes) - fetched_cls: + yield (cl, 'error') def upload_branch_deps(cl, args, force=False): - """Uploads CLs of local branches that are dependents of the current branch. + """Uploads CLs of local branches that are dependents of the current branch. If the local branch dependency tree looks like: @@ -3851,198 +3993,206 @@ def upload_branch_deps(cl, args, force=False): with its dependent branches, and you would like their dependencies updated in Rietveld. """ - if git_common.is_dirty_git_tree('upload-branch-deps'): - return 1 + if git_common.is_dirty_git_tree('upload-branch-deps'): + return 1 - root_branch = cl.GetBranch() - if root_branch is None: - DieWithError('Can\'t find dependent branches from detached HEAD state. ' - 'Get on a branch!') - if not cl.GetIssue(): - DieWithError('Current branch does not have an uploaded CL. We cannot set ' - 'patchset dependencies without an uploaded CL.') + root_branch = cl.GetBranch() + if root_branch is None: + DieWithError('Can\'t find dependent branches from detached HEAD state. ' + 'Get on a branch!') + if not cl.GetIssue(): + DieWithError( + 'Current branch does not have an uploaded CL. We cannot set ' + 'patchset dependencies without an uploaded CL.') - branches = RunGit(['for-each-ref', - '--format=%(refname:short) %(upstream:short)', - 'refs/heads']) - if not branches: - print('No local branches found.') - return 0 + branches = RunGit([ + 'for-each-ref', '--format=%(refname:short) %(upstream:short)', + 'refs/heads' + ]) + if not branches: + print('No local branches found.') + return 0 - # Create a dictionary of all local branches to the branches that are - # dependent on it. - tracked_to_dependents = collections.defaultdict(list) - for b in branches.splitlines(): - tokens = b.split() - if len(tokens) == 2: - branch_name, tracked = tokens - tracked_to_dependents[tracked].append(branch_name) + # Create a dictionary of all local branches to the branches that are + # dependent on it. + tracked_to_dependents = collections.defaultdict(list) + for b in branches.splitlines(): + tokens = b.split() + if len(tokens) == 2: + branch_name, tracked = tokens + tracked_to_dependents[tracked].append(branch_name) - print() - print('The dependent local branches of %s are:' % root_branch) - dependents = [] + print() + print('The dependent local branches of %s are:' % root_branch) + dependents = [] - def traverse_dependents_preorder(branch, padding=''): - dependents_to_process = tracked_to_dependents.get(branch, []) - padding += ' ' - for dependent in dependents_to_process: - print('%s%s' % (padding, dependent)) - dependents.append(dependent) - traverse_dependents_preorder(dependent, padding) + def traverse_dependents_preorder(branch, padding=''): + dependents_to_process = tracked_to_dependents.get(branch, []) + padding += ' ' + for dependent in dependents_to_process: + print('%s%s' % (padding, dependent)) + dependents.append(dependent) + traverse_dependents_preorder(dependent, padding) - traverse_dependents_preorder(root_branch) - print() + traverse_dependents_preorder(root_branch) + print() - if not dependents: - print('There are no dependent local branches for %s' % root_branch) - return 0 + if not dependents: + print('There are no dependent local branches for %s' % root_branch) + return 0 - # Record all dependents that failed to upload. - failures = {} - # Go through all dependents, checkout the branch and upload. - try: + # Record all dependents that failed to upload. + failures = {} + # Go through all dependents, checkout the branch and upload. + try: + for dependent_branch in dependents: + print() + print('--------------------------------------') + print('Running "git cl upload" from %s:' % dependent_branch) + RunGit(['checkout', '-q', dependent_branch]) + print() + try: + if CMDupload(OptionParser(), args) != 0: + print('Upload failed for %s!' % dependent_branch) + failures[dependent_branch] = 1 + except: # pylint: disable=bare-except + failures[dependent_branch] = 1 + print() + finally: + # Swap back to the original root branch. + RunGit(['checkout', '-q', root_branch]) + + print() + print('Upload complete for dependent branches!') for dependent_branch in dependents: - print() - print('--------------------------------------') - print('Running "git cl upload" from %s:' % dependent_branch) - RunGit(['checkout', '-q', dependent_branch]) - print() - try: - if CMDupload(OptionParser(), args) != 0: - print('Upload failed for %s!' % dependent_branch) - failures[dependent_branch] = 1 - except: # pylint: disable=bare-except - failures[dependent_branch] = 1 - print() - finally: - # Swap back to the original root branch. - RunGit(['checkout', '-q', root_branch]) + upload_status = 'failed' if failures.get( + dependent_branch) else 'succeeded' + print(' %s : %s' % (dependent_branch, upload_status)) + print() - print() - print('Upload complete for dependent branches!') - for dependent_branch in dependents: - upload_status = 'failed' if failures.get(dependent_branch) else 'succeeded' - print(' %s : %s' % (dependent_branch, upload_status)) - print() - - return 0 + return 0 def GetArchiveTagForBranch(issue_num, branch_name, existing_tags, pattern): - """Given a proposed tag name, returns a tag name that is guaranteed to be + """Given a proposed tag name, returns a tag name that is guaranteed to be unique. If 'foo' is proposed but already exists, then 'foo-2' is used, or 'foo-3', and so on.""" - proposed_tag = pattern.format(**{'issue': issue_num, 'branch': branch_name}) - for suffix_num in itertools.count(1): - if suffix_num == 1: - to_check = proposed_tag - else: - to_check = '%s-%d' % (proposed_tag, suffix_num) + proposed_tag = pattern.format(**{'issue': issue_num, 'branch': branch_name}) + for suffix_num in itertools.count(1): + if suffix_num == 1: + to_check = proposed_tag + else: + to_check = '%s-%d' % (proposed_tag, suffix_num) - if to_check not in existing_tags: - return to_check + if to_check not in existing_tags: + return to_check @metrics.collector.collect_metrics('git cl archive') def CMDarchive(parser, args): - """Archives and deletes branches associated with closed changelists.""" - parser.add_option( - '-j', '--maxjobs', action='store', type=int, - help='The maximum number of jobs to use when retrieving review status.') - parser.add_option( - '-f', '--force', action='store_true', - help='Bypasses the confirmation prompt.') - parser.add_option( - '-d', '--dry-run', action='store_true', - help='Skip the branch tagging and removal steps.') - parser.add_option( - '-t', '--notags', action='store_true', - help='Do not tag archived branches. ' - 'Note: local commit history may be lost.') - parser.add_option( - '-p', - '--pattern', - default='git-cl-archived-{issue}-{branch}', - help='Format string for archive tags. ' - 'E.g. \'archived-{issue}-{branch}\'.') + """Archives and deletes branches associated with closed changelists.""" + parser.add_option( + '-j', + '--maxjobs', + action='store', + type=int, + help='The maximum number of jobs to use when retrieving review status.') + parser.add_option('-f', + '--force', + action='store_true', + help='Bypasses the confirmation prompt.') + parser.add_option('-d', + '--dry-run', + action='store_true', + help='Skip the branch tagging and removal steps.') + parser.add_option('-t', + '--notags', + action='store_true', + help='Do not tag archived branches. ' + 'Note: local commit history may be lost.') + parser.add_option('-p', + '--pattern', + default='git-cl-archived-{issue}-{branch}', + help='Format string for archive tags. ' + 'E.g. \'archived-{issue}-{branch}\'.') - options, args = parser.parse_args(args) - if args: - parser.error('Unsupported args: %s' % ' '.join(args)) + options, args = parser.parse_args(args) + if args: + parser.error('Unsupported args: %s' % ' '.join(args)) + + branches = RunGit(['for-each-ref', '--format=%(refname)', 'refs/heads']) + if not branches: + return 0 + + tags = RunGit(['for-each-ref', '--format=%(refname)', 'refs/tags' + ]).splitlines() or [] + tags = [t.split('/')[-1] for t in tags] + + print('Finding all branches associated with closed issues...') + changes = [Changelist(branchref=b) for b in branches.splitlines()] + alignment = max(5, max(len(c.GetBranch()) for c in changes)) + statuses = get_cl_statuses(changes, + fine_grained=True, + max_processes=options.maxjobs) + proposal = [(cl.GetBranch(), + GetArchiveTagForBranch(cl.GetIssue(), cl.GetBranch(), tags, + options.pattern)) + for cl, status in statuses + if status in ('closed', 'rietveld-not-supported')] + proposal.sort() + + if not proposal: + print('No branches with closed codereview issues found.') + return 0 + + current_branch = scm.GIT.GetBranch(settings.GetRoot()) + + print('\nBranches with closed issues that will be archived:\n') + if options.notags: + for next_item in proposal: + print(' ' + next_item[0]) + else: + print('%*s | %s' % (alignment, 'Branch name', 'Archival tag name')) + for next_item in proposal: + print('%*s %s' % (alignment, next_item[0], next_item[1])) + + # Quit now on precondition failure or if instructed by the user, either + # via an interactive prompt or by command line flags. + if options.dry_run: + print('\nNo changes were made (dry run).\n') + return 0 + + if any(branch == current_branch for branch, _ in proposal): + print('You are currently on a branch \'%s\' which is associated with a ' + 'closed codereview issue, so archive cannot proceed. Please ' + 'checkout another branch and run this command again.' % + current_branch) + return 1 + + if not options.force: + answer = gclient_utils.AskForData( + '\nProceed with deletion (Y/n)? ').lower() + if answer not in ('y', ''): + print('Aborted.') + return 1 + + for branch, tagname in proposal: + if not options.notags: + RunGit(['tag', tagname, branch]) + + if RunGitWithCode(['branch', '-D', branch])[0] != 0: + # Clean up the tag if we failed to delete the branch. + RunGit(['tag', '-d', tagname]) + + print('\nJob\'s done!') - branches = RunGit(['for-each-ref', '--format=%(refname)', 'refs/heads']) - if not branches: return 0 - tags = RunGit(['for-each-ref', '--format=%(refname)', - 'refs/tags']).splitlines() or [] - tags = [t.split('/')[-1] for t in tags] - - print('Finding all branches associated with closed issues...') - changes = [Changelist(branchref=b) - for b in branches.splitlines()] - alignment = max(5, max(len(c.GetBranch()) for c in changes)) - statuses = get_cl_statuses(changes, - fine_grained=True, - max_processes=options.maxjobs) - proposal = [(cl.GetBranch(), - GetArchiveTagForBranch(cl.GetIssue(), cl.GetBranch(), tags, - options.pattern)) - for cl, status in statuses - if status in ('closed', 'rietveld-not-supported')] - proposal.sort() - - if not proposal: - print('No branches with closed codereview issues found.') - return 0 - - current_branch = scm.GIT.GetBranch(settings.GetRoot()) - - print('\nBranches with closed issues that will be archived:\n') - if options.notags: - for next_item in proposal: - print(' ' + next_item[0]) - else: - print('%*s | %s' % (alignment, 'Branch name', 'Archival tag name')) - for next_item in proposal: - print('%*s %s' % (alignment, next_item[0], next_item[1])) - - # Quit now on precondition failure or if instructed by the user, either - # via an interactive prompt or by command line flags. - if options.dry_run: - print('\nNo changes were made (dry run).\n') - return 0 - - if any(branch == current_branch for branch, _ in proposal): - print('You are currently on a branch \'%s\' which is associated with a ' - 'closed codereview issue, so archive cannot proceed. Please ' - 'checkout another branch and run this command again.' % - current_branch) - return 1 - - if not options.force: - answer = gclient_utils.AskForData('\nProceed with deletion (Y/n)? ').lower() - if answer not in ('y', ''): - print('Aborted.') - return 1 - - for branch, tagname in proposal: - if not options.notags: - RunGit(['tag', tagname, branch]) - - if RunGitWithCode(['branch', '-D', branch])[0] != 0: - # Clean up the tag if we failed to delete the branch. - RunGit(['tag', '-d', tagname]) - - print('\nJob\'s done!') - - return 0 - @metrics.collector.collect_metrics('git cl status') def CMDstatus(parser, args): - """Show status of changelists. + """Show status of changelists. Colors are used to tell the state of the CL unless --fast is used: - Blue waiting for review @@ -4055,471 +4205,519 @@ def CMDstatus(parser, args): Also see 'git cl comments'. """ - parser.add_option( - '--no-branch-color', - action='store_true', - help='Disable colorized branch names') - parser.add_option('--field', - help='print only specific field (desc|id|patch|status|url)') - parser.add_option('-f', '--fast', action='store_true', - help='Do not retrieve review status') - parser.add_option( - '-j', '--maxjobs', action='store', type=int, - help='The maximum number of jobs to use when retrieving review status') - parser.add_option( - '-i', '--issue', type=int, - help='Operate on this issue instead of the current branch\'s implicit ' - 'issue. Requires --field to be set.') - parser.add_option('-d', - '--date-order', - action='store_true', - help='Order branches by committer date.') - options, args = parser.parse_args(args) - if args: - parser.error('Unsupported args: %s' % args) + parser.add_option('--no-branch-color', + action='store_true', + help='Disable colorized branch names') + parser.add_option( + '--field', help='print only specific field (desc|id|patch|status|url)') + parser.add_option('-f', + '--fast', + action='store_true', + help='Do not retrieve review status') + parser.add_option( + '-j', + '--maxjobs', + action='store', + type=int, + help='The maximum number of jobs to use when retrieving review status') + parser.add_option( + '-i', + '--issue', + type=int, + help='Operate on this issue instead of the current branch\'s implicit ' + 'issue. Requires --field to be set.') + parser.add_option('-d', + '--date-order', + action='store_true', + help='Order branches by committer date.') + options, args = parser.parse_args(args) + if args: + parser.error('Unsupported args: %s' % args) - if options.issue is not None and not options.field: - parser.error('--field must be given when --issue is set.') + if options.issue is not None and not options.field: + parser.error('--field must be given when --issue is set.') - if options.field: - cl = Changelist(issue=options.issue) - if options.field.startswith('desc'): - if cl.GetIssue(): - print(cl.FetchDescription()) - elif options.field == 'id': - issueid = cl.GetIssue() - if issueid: - print(issueid) - elif options.field == 'patch': - patchset = cl.GetMostRecentPatchset() - if patchset: - print(patchset) - elif options.field == 'status': - print(cl.GetStatus()) - elif options.field == 'url': - url = cl.GetIssueURL() - if url: - print(url) - return 0 + if options.field: + cl = Changelist(issue=options.issue) + if options.field.startswith('desc'): + if cl.GetIssue(): + print(cl.FetchDescription()) + elif options.field == 'id': + issueid = cl.GetIssue() + if issueid: + print(issueid) + elif options.field == 'patch': + patchset = cl.GetMostRecentPatchset() + if patchset: + print(patchset) + elif options.field == 'status': + print(cl.GetStatus()) + elif options.field == 'url': + url = cl.GetIssueURL() + if url: + print(url) + return 0 - branches = RunGit([ - 'for-each-ref', '--format=%(refname) %(committerdate:unix)', 'refs/heads' - ]) - if not branches: - print('No local branch found.') - return 0 + branches = RunGit([ + 'for-each-ref', '--format=%(refname) %(committerdate:unix)', + 'refs/heads' + ]) + if not branches: + print('No local branch found.') + return 0 - changes = [ - Changelist(branchref=b, commit_date=ct) - for b, ct in map(lambda line: line.split(' '), branches.splitlines()) - ] - print('Branches associated with reviews:') - output = get_cl_statuses(changes, - fine_grained=not options.fast, - max_processes=options.maxjobs) + changes = [ + Changelist(branchref=b, commit_date=ct) + for b, ct in map(lambda line: line.split(' '), branches.splitlines()) + ] + print('Branches associated with reviews:') + output = get_cl_statuses(changes, + fine_grained=not options.fast, + max_processes=options.maxjobs) - current_branch = scm.GIT.GetBranch(settings.GetRoot()) + current_branch = scm.GIT.GetBranch(settings.GetRoot()) - def FormatBranchName(branch, colorize=False): - """Simulates 'git branch' behavior. Colorizes and prefixes branch name with + def FormatBranchName(branch, colorize=False): + """Simulates 'git branch' behavior. Colorizes and prefixes branch name with an asterisk when it is the current branch.""" - asterisk = "" - color = Fore.RESET - if branch == current_branch: - asterisk = "* " - color = Fore.GREEN - branch_name = scm.GIT.ShortBranchName(branch) + asterisk = "" + color = Fore.RESET + if branch == current_branch: + asterisk = "* " + color = Fore.GREEN + branch_name = scm.GIT.ShortBranchName(branch) - if colorize: - return asterisk + color + branch_name + Fore.RESET - return asterisk + branch_name + if colorize: + return asterisk + color + branch_name + Fore.RESET + return asterisk + branch_name - branch_statuses = {} + branch_statuses = {} - alignment = max(5, max(len(FormatBranchName(c.GetBranch())) for c in changes)) + alignment = max(5, + max(len(FormatBranchName(c.GetBranch())) for c in changes)) - if options.date_order or settings.IsStatusCommitOrderByDate(): - sorted_changes = sorted(changes, - key=lambda c: c.GetCommitDate(), - reverse=True) - else: - sorted_changes = sorted(changes, key=lambda c: c.GetBranch()) - for cl in sorted_changes: - branch = cl.GetBranch() - while branch not in branch_statuses: - c, status = next(output) - branch_statuses[c.GetBranch()] = status - status = branch_statuses.pop(branch) - url = cl.GetIssueURL(short=True) - if url and (not status or status == 'error'): - # The issue probably doesn't exist anymore. - url += ' (broken)' + if options.date_order or settings.IsStatusCommitOrderByDate(): + sorted_changes = sorted(changes, + key=lambda c: c.GetCommitDate(), + reverse=True) + else: + sorted_changes = sorted(changes, key=lambda c: c.GetBranch()) + for cl in sorted_changes: + branch = cl.GetBranch() + while branch not in branch_statuses: + c, status = next(output) + branch_statuses[c.GetBranch()] = status + status = branch_statuses.pop(branch) + url = cl.GetIssueURL(short=True) + if url and (not status or status == 'error'): + # The issue probably doesn't exist anymore. + url += ' (broken)' - color = color_for_status(status) - # Turn off bold as well as colors. - END = '\033[0m' - reset = Fore.RESET + END - if not setup_color.IS_TTY: - color = '' - reset = '' - status_str = '(%s)' % status if status else '' + color = color_for_status(status) + # Turn off bold as well as colors. + END = '\033[0m' + reset = Fore.RESET + END + if not setup_color.IS_TTY: + color = '' + reset = '' + status_str = '(%s)' % status if status else '' - branch_display = FormatBranchName(branch) - padding = ' ' * (alignment - len(branch_display)) - if not options.no_branch_color: - branch_display = FormatBranchName(branch, colorize=True) + branch_display = FormatBranchName(branch) + padding = ' ' * (alignment - len(branch_display)) + if not options.no_branch_color: + branch_display = FormatBranchName(branch, colorize=True) - print(' %s : %s%s %s%s' % (padding + branch_display, color, url, - status_str, reset)) + print(' %s : %s%s %s%s' % + (padding + branch_display, color, url, status_str, reset)) - print() - print('Current branch: %s' % current_branch) - for cl in changes: - if cl.GetBranch() == current_branch: - break - if not cl.GetIssue(): - print('No issue assigned.') + print() + print('Current branch: %s' % current_branch) + for cl in changes: + if cl.GetBranch() == current_branch: + break + if not cl.GetIssue(): + print('No issue assigned.') + return 0 + print('Issue number: %s (%s)' % (cl.GetIssue(), cl.GetIssueURL())) + if not options.fast: + print('Issue description:') + print(cl.FetchDescription(pretty=True)) return 0 - print('Issue number: %s (%s)' % (cl.GetIssue(), cl.GetIssueURL())) - if not options.fast: - print('Issue description:') - print(cl.FetchDescription(pretty=True)) - return 0 def colorize_CMDstatus_doc(): - """To be called once in main() to add colors to git cl status help.""" - colors = [i for i in dir(Fore) if i[0].isupper()] + """To be called once in main() to add colors to git cl status help.""" + colors = [i for i in dir(Fore) if i[0].isupper()] - def colorize_line(line): - for color in colors: - if color in line.upper(): - # Extract whitespace first and the leading '-'. - indent = len(line) - len(line.lstrip(' ')) + 1 - return line[:indent] + getattr(Fore, color) + line[indent:] + Fore.RESET - return line + def colorize_line(line): + for color in colors: + if color in line.upper(): + # Extract whitespace first and the leading '-'. + indent = len(line) - len(line.lstrip(' ')) + 1 + return line[:indent] + getattr( + Fore, color) + line[indent:] + Fore.RESET + return line - lines = CMDstatus.__doc__.splitlines() - CMDstatus.__doc__ = '\n'.join(colorize_line(l) for l in lines) + lines = CMDstatus.__doc__.splitlines() + CMDstatus.__doc__ = '\n'.join(colorize_line(l) for l in lines) def write_json(path, contents): - if path == '-': - json.dump(contents, sys.stdout) - else: - with open(path, 'w') as f: - json.dump(contents, f) + if path == '-': + json.dump(contents, sys.stdout) + else: + with open(path, 'w') as f: + json.dump(contents, f) @subcommand.usage('[issue_number]') @metrics.collector.collect_metrics('git cl issue') def CMDissue(parser, args): - """Sets or displays the current code review issue number. + """Sets or displays the current code review issue number. Pass issue number 0 to clear the current issue. """ - parser.add_option('-r', '--reverse', action='store_true', - help='Lookup the branch(es) for the specified issues. If ' - 'no issues are specified, all branches with mapped ' - 'issues will be listed.') - parser.add_option('--json', - help='Path to JSON output file, or "-" for stdout.') - options, args = parser.parse_args(args) + parser.add_option('-r', + '--reverse', + action='store_true', + help='Lookup the branch(es) for the specified issues. If ' + 'no issues are specified, all branches with mapped ' + 'issues will be listed.') + parser.add_option('--json', + help='Path to JSON output file, or "-" for stdout.') + options, args = parser.parse_args(args) - if options.reverse: - branches = RunGit(['for-each-ref', 'refs/heads', - '--format=%(refname)']).splitlines() - # Reverse issue lookup. - issue_branch_map = {} + if options.reverse: + branches = RunGit(['for-each-ref', 'refs/heads', + '--format=%(refname)']).splitlines() + # Reverse issue lookup. + issue_branch_map = {} - git_config = {} - for config in RunGit(['config', '--get-regexp', - r'branch\..*issue']).splitlines(): - name, _space, val = config.partition(' ') - git_config[name] = val + git_config = {} + for config in RunGit(['config', '--get-regexp', + r'branch\..*issue']).splitlines(): + name, _space, val = config.partition(' ') + git_config[name] = val - for branch in branches: - issue = git_config.get( - 'branch.%s.%s' % (scm.GIT.ShortBranchName(branch), ISSUE_CONFIG_KEY)) - if issue: - issue_branch_map.setdefault(int(issue), []).append(branch) - if not args: - args = sorted(issue_branch_map.keys()) - result = {} - for issue in args: - try: - issue_num = int(issue) - except ValueError: - print('ERROR cannot parse issue number: %s' % issue, file=sys.stderr) - continue - result[issue_num] = issue_branch_map.get(issue_num) - print('Branch for issue number %s: %s' % - (issue, ', '.join(issue_branch_map.get(issue_num) or ('None', )))) + for branch in branches: + issue = git_config.get( + 'branch.%s.%s' % + (scm.GIT.ShortBranchName(branch), ISSUE_CONFIG_KEY)) + if issue: + issue_branch_map.setdefault(int(issue), []).append(branch) + if not args: + args = sorted(issue_branch_map.keys()) + result = {} + for issue in args: + try: + issue_num = int(issue) + except ValueError: + print('ERROR cannot parse issue number: %s' % issue, + file=sys.stderr) + continue + result[issue_num] = issue_branch_map.get(issue_num) + print('Branch for issue number %s: %s' % (issue, ', '.join( + issue_branch_map.get(issue_num) or ('None', )))) + if options.json: + write_json(options.json, result) + return 0 + + if len(args) > 0: + issue = ParseIssueNumberArgument(args[0]) + if not issue.valid: + DieWithError( + 'Pass a url or number to set the issue, 0 to unset it, ' + 'or no argument to list it.\n' + 'Maybe you want to run git cl status?') + cl = Changelist() + cl.SetIssue(issue.issue) + else: + cl = Changelist() + print('Issue number: %s (%s)' % (cl.GetIssue(), cl.GetIssueURL())) if options.json: - write_json(options.json, result) + write_json( + options.json, { + 'gerrit_host': cl.GetGerritHost(), + 'gerrit_project': cl.GetGerritProject(), + 'issue_url': cl.GetIssueURL(), + 'issue': cl.GetIssue(), + }) return 0 - if len(args) > 0: - issue = ParseIssueNumberArgument(args[0]) - if not issue.valid: - DieWithError('Pass a url or number to set the issue, 0 to unset it, ' - 'or no argument to list it.\n' - 'Maybe you want to run git cl status?') - cl = Changelist() - cl.SetIssue(issue.issue) - else: - cl = Changelist() - print('Issue number: %s (%s)' % (cl.GetIssue(), cl.GetIssueURL())) - if options.json: - write_json(options.json, { - 'gerrit_host': cl.GetGerritHost(), - 'gerrit_project': cl.GetGerritProject(), - 'issue_url': cl.GetIssueURL(), - 'issue': cl.GetIssue(), - }) - return 0 - @metrics.collector.collect_metrics('git cl comments') def CMDcomments(parser, args): - """Shows or posts review comments for any changelist.""" - parser.add_option('-a', '--add-comment', dest='comment', - help='comment to add to an issue') - parser.add_option('-p', '--publish', action='store_true', - help='marks CL as ready and sends comment to reviewers') - parser.add_option('-i', '--issue', dest='issue', - help='review issue id (defaults to current issue).') - parser.add_option('-m', '--machine-readable', dest='readable', - action='store_false', default=True, - help='output comments in a format compatible with ' - 'editor parsing') - parser.add_option('-j', '--json-file', - help='File to write JSON summary to, or "-" for stdout') - options, args = parser.parse_args(args) + """Shows or posts review comments for any changelist.""" + parser.add_option('-a', + '--add-comment', + dest='comment', + help='comment to add to an issue') + parser.add_option('-p', + '--publish', + action='store_true', + help='marks CL as ready and sends comment to reviewers') + parser.add_option('-i', + '--issue', + dest='issue', + help='review issue id (defaults to current issue).') + parser.add_option('-m', + '--machine-readable', + dest='readable', + action='store_false', + default=True, + help='output comments in a format compatible with ' + 'editor parsing') + parser.add_option('-j', + '--json-file', + help='File to write JSON summary to, or "-" for stdout') + options, args = parser.parse_args(args) - issue = None - if options.issue: - try: - issue = int(options.issue) - except ValueError: - DieWithError('A review issue ID is expected to be a number.') + issue = None + if options.issue: + try: + issue = int(options.issue) + except ValueError: + DieWithError('A review issue ID is expected to be a number.') - cl = Changelist(issue=issue) + cl = Changelist(issue=issue) - if options.comment: - cl.AddComment(options.comment, options.publish) + if options.comment: + cl.AddComment(options.comment, options.publish) + return 0 + + summary = sorted(cl.GetCommentsSummary(readable=options.readable), + key=lambda c: c.date) + for comment in summary: + if comment.disapproval: + color = Fore.RED + elif comment.approval: + color = Fore.GREEN + elif comment.sender == cl.GetIssueOwner(): + color = Fore.MAGENTA + elif comment.autogenerated: + color = Fore.CYAN + else: + color = Fore.BLUE + print('\n%s%s %s%s\n%s' % + (color, comment.date.strftime('%Y-%m-%d %H:%M:%S UTC'), + comment.sender, Fore.RESET, '\n'.join( + ' ' + l for l in comment.message.strip().splitlines()))) + + if options.json_file: + + def pre_serialize(c): + dct = c._asdict().copy() + dct['date'] = dct['date'].strftime('%Y-%m-%d %H:%M:%S.%f') + return dct + + write_json(options.json_file, [pre_serialize(x) for x in summary]) return 0 - summary = sorted(cl.GetCommentsSummary(readable=options.readable), - key=lambda c: c.date) - for comment in summary: - if comment.disapproval: - color = Fore.RED - elif comment.approval: - color = Fore.GREEN - elif comment.sender == cl.GetIssueOwner(): - color = Fore.MAGENTA - elif comment.autogenerated: - color = Fore.CYAN - else: - color = Fore.BLUE - print('\n%s%s %s%s\n%s' % ( - color, - comment.date.strftime('%Y-%m-%d %H:%M:%S UTC'), - comment.sender, - Fore.RESET, - '\n'.join(' ' + l for l in comment.message.strip().splitlines()))) - - if options.json_file: - def pre_serialize(c): - dct = c._asdict().copy() - dct['date'] = dct['date'].strftime('%Y-%m-%d %H:%M:%S.%f') - return dct - write_json(options.json_file, [pre_serialize(x) for x in summary]) - return 0 - @subcommand.usage('[codereview url or issue id]') @metrics.collector.collect_metrics('git cl description') def CMDdescription(parser, args): - """Brings up the editor for the current CL's description.""" - parser.add_option('-d', '--display', action='store_true', - help='Display the description instead of opening an editor') - parser.add_option('-n', '--new-description', - help='New description to set for this issue (- for stdin, ' - '+ to load from local commit HEAD)') - parser.add_option('-f', '--force', action='store_true', - help='Delete any unpublished Gerrit edits for this issue ' - 'without prompting') + """Brings up the editor for the current CL's description.""" + parser.add_option( + '-d', + '--display', + action='store_true', + help='Display the description instead of opening an editor') + parser.add_option( + '-n', + '--new-description', + help='New description to set for this issue (- for stdin, ' + '+ to load from local commit HEAD)') + parser.add_option('-f', + '--force', + action='store_true', + help='Delete any unpublished Gerrit edits for this issue ' + 'without prompting') - options, args = parser.parse_args(args) + options, args = parser.parse_args(args) - target_issue_arg = None - if len(args) > 0: - target_issue_arg = ParseIssueNumberArgument(args[0]) - if not target_issue_arg.valid: - parser.error('Invalid issue ID or URL.') + target_issue_arg = None + if len(args) > 0: + target_issue_arg = ParseIssueNumberArgument(args[0]) + if not target_issue_arg.valid: + parser.error('Invalid issue ID or URL.') - kwargs = {} - if target_issue_arg: - kwargs['issue'] = target_issue_arg.issue - kwargs['codereview_host'] = target_issue_arg.hostname + kwargs = {} + if target_issue_arg: + kwargs['issue'] = target_issue_arg.issue + kwargs['codereview_host'] = target_issue_arg.hostname - cl = Changelist(**kwargs) - if not cl.GetIssue(): - DieWithError('This branch has no associated changelist.') + cl = Changelist(**kwargs) + if not cl.GetIssue(): + DieWithError('This branch has no associated changelist.') - if args and not args[0].isdigit(): - logging.info('canonical issue/change URL: %s\n', cl.GetIssueURL()) + if args and not args[0].isdigit(): + logging.info('canonical issue/change URL: %s\n', cl.GetIssueURL()) - description = ChangeDescription(cl.FetchDescription()) + description = ChangeDescription(cl.FetchDescription()) - if options.display: - print(description.description) + if options.display: + print(description.description) + return 0 + + if options.new_description: + text = options.new_description + if text == '-': + text = '\n'.join(l.rstrip() for l in sys.stdin) + elif text == '+': + base_branch = cl.GetCommonAncestorWithUpstream() + text = _create_description_from_log([base_branch]) + + description.set_description(text) + else: + description.prompt() + if cl.FetchDescription().strip() != description.description: + cl.UpdateDescription(description.description, force=options.force) return 0 - if options.new_description: - text = options.new_description - if text == '-': - text = '\n'.join(l.rstrip() for l in sys.stdin) - elif text == '+': - base_branch = cl.GetCommonAncestorWithUpstream() - text = _create_description_from_log([base_branch]) - - description.set_description(text) - else: - description.prompt() - if cl.FetchDescription().strip() != description.description: - cl.UpdateDescription(description.description, force=options.force) - return 0 - @metrics.collector.collect_metrics('git cl lint') def CMDlint(parser, args): - """Runs cpplint on the current changelist.""" - parser.add_option('--filter', action='append', metavar='-x,+y', - help='Comma-separated list of cpplint\'s category-filters') - options, args = parser.parse_args(args) + """Runs cpplint on the current changelist.""" + parser.add_option( + '--filter', + action='append', + metavar='-x,+y', + help='Comma-separated list of cpplint\'s category-filters') + options, args = parser.parse_args(args) - # Access to a protected member _XX of a client class - # pylint: disable=protected-access - try: - import cpplint - import cpplint_chromium - except ImportError: - print('Your depot_tools is missing cpplint.py and/or cpplint_chromium.py.') - return 1 + # Access to a protected member _XX of a client class + # pylint: disable=protected-access + try: + import cpplint + import cpplint_chromium + except ImportError: + print( + 'Your depot_tools is missing cpplint.py and/or cpplint_chromium.py.' + ) + return 1 - # Change the current working directory before calling lint so that it - # shows the correct base. - previous_cwd = os.getcwd() - os.chdir(settings.GetRoot()) - try: - cl = Changelist() - files = cl.GetAffectedFiles(cl.GetCommonAncestorWithUpstream()) - if not files: - print('Cannot lint an empty CL') - return 1 + # Change the current working directory before calling lint so that it + # shows the correct base. + previous_cwd = os.getcwd() + os.chdir(settings.GetRoot()) + try: + cl = Changelist() + files = cl.GetAffectedFiles(cl.GetCommonAncestorWithUpstream()) + if not files: + print('Cannot lint an empty CL') + return 1 - # Process cpplint arguments, if any. - filters = presubmit_canned_checks.GetCppLintFilters(options.filter) - command = ['--filter=' + ','.join(filters)] - command.extend(args) - command.extend(files) - filenames = cpplint.ParseArguments(command) + # Process cpplint arguments, if any. + filters = presubmit_canned_checks.GetCppLintFilters(options.filter) + command = ['--filter=' + ','.join(filters)] + command.extend(args) + command.extend(files) + filenames = cpplint.ParseArguments(command) - include_regex = re.compile(settings.GetLintRegex()) - ignore_regex = re.compile(settings.GetLintIgnoreRegex()) - extra_check_functions = [cpplint_chromium.CheckPointerDeclarationWhitespace] - for filename in filenames: - if not include_regex.match(filename): - print('Skipping file %s' % filename) - continue + include_regex = re.compile(settings.GetLintRegex()) + ignore_regex = re.compile(settings.GetLintIgnoreRegex()) + extra_check_functions = [ + cpplint_chromium.CheckPointerDeclarationWhitespace + ] + for filename in filenames: + if not include_regex.match(filename): + print('Skipping file %s' % filename) + continue - if ignore_regex.match(filename): - print('Ignoring file %s' % filename) - continue + if ignore_regex.match(filename): + print('Ignoring file %s' % filename) + continue - cpplint.ProcessFile(filename, cpplint._cpplint_state.verbose_level, - extra_check_functions) - finally: - os.chdir(previous_cwd) - print('Total errors found: %d\n' % cpplint._cpplint_state.error_count) - if cpplint._cpplint_state.error_count != 0: - return 1 - return 0 + cpplint.ProcessFile(filename, cpplint._cpplint_state.verbose_level, + extra_check_functions) + finally: + os.chdir(previous_cwd) + print('Total errors found: %d\n' % cpplint._cpplint_state.error_count) + if cpplint._cpplint_state.error_count != 0: + return 1 + return 0 @metrics.collector.collect_metrics('git cl presubmit') @subcommand.usage('[base branch]') def CMDpresubmit(parser, args): - """Runs presubmit tests on the current changelist.""" - parser.add_option('-u', '--upload', action='store_true', - help='Run upload hook instead of the push hook') - parser.add_option('-f', '--force', action='store_true', - help='Run checks even if tree is dirty') - parser.add_option('--all', action='store_true', - help='Run checks against all files, not just modified ones') - parser.add_option('--files', - nargs=1, - help='Semicolon-separated list of files to be marked as ' - 'modified when executing presubmit or post-upload hooks. ' - 'fnmatch wildcards can also be used.') - parser.add_option('--parallel', action='store_true', - help='Run all tests specified by input_api.RunTests in all ' - 'PRESUBMIT files in parallel.') - parser.add_option('--resultdb', action='store_true', - help='Run presubmit checks in the ResultSink environment ' - 'and send results to the ResultDB database.') - parser.add_option('--realm', help='LUCI realm if reporting to ResultDB') - options, args = parser.parse_args(args) + """Runs presubmit tests on the current changelist.""" + parser.add_option('-u', + '--upload', + action='store_true', + help='Run upload hook instead of the push hook') + parser.add_option('-f', + '--force', + action='store_true', + help='Run checks even if tree is dirty') + parser.add_option( + '--all', + action='store_true', + help='Run checks against all files, not just modified ones') + parser.add_option('--files', + nargs=1, + help='Semicolon-separated list of files to be marked as ' + 'modified when executing presubmit or post-upload hooks. ' + 'fnmatch wildcards can also be used.') + parser.add_option( + '--parallel', + action='store_true', + help='Run all tests specified by input_api.RunTests in all ' + 'PRESUBMIT files in parallel.') + parser.add_option('--resultdb', + action='store_true', + help='Run presubmit checks in the ResultSink environment ' + 'and send results to the ResultDB database.') + parser.add_option('--realm', help='LUCI realm if reporting to ResultDB') + options, args = parser.parse_args(args) - if not options.force and git_common.is_dirty_git_tree('presubmit'): - print('use --force to check even if tree is dirty.') - return 1 + if not options.force and git_common.is_dirty_git_tree('presubmit'): + print('use --force to check even if tree is dirty.') + return 1 - cl = Changelist() - if args: - base_branch = args[0] - else: - # Default to diffing against the common ancestor of the upstream branch. - base_branch = cl.GetCommonAncestorWithUpstream() - - start = time.time() - try: - if not 'PRESUBMIT_SKIP_NETWORK' in os.environ and cl.GetIssue(): - description = cl.FetchDescription() + cl = Changelist() + if args: + base_branch = args[0] else: - description = _create_description_from_log([base_branch]) - except Exception as e: - print('Failed to fetch CL description - %s' % str(e)) - description = _create_description_from_log([base_branch]) - elapsed = time.time() - start - if elapsed > 5: - print('%.1f s to get CL description.' % elapsed) + # Default to diffing against the common ancestor of the upstream branch. + base_branch = cl.GetCommonAncestorWithUpstream() - if not base_branch: - if not options.force: - print('use --force to check even when not on a branch.') - return 1 - base_branch = 'HEAD' + start = time.time() + try: + if not 'PRESUBMIT_SKIP_NETWORK' in os.environ and cl.GetIssue(): + description = cl.FetchDescription() + else: + description = _create_description_from_log([base_branch]) + except Exception as e: + print('Failed to fetch CL description - %s' % str(e)) + description = _create_description_from_log([base_branch]) + elapsed = time.time() - start + if elapsed > 5: + print('%.1f s to get CL description.' % elapsed) - cl.RunHook(committing=not options.upload, - may_prompt=False, - verbose=options.verbose, - parallel=options.parallel, - upstream=base_branch, - description=description, - all_files=options.all, - files=options.files, - resultdb=options.resultdb, - realm=options.realm) - return 0 + if not base_branch: + if not options.force: + print('use --force to check even when not on a branch.') + return 1 + base_branch = 'HEAD' + + cl.RunHook(committing=not options.upload, + may_prompt=False, + verbose=options.verbose, + parallel=options.parallel, + upstream=base_branch, + description=description, + all_files=options.all, + files=options.files, + resultdb=options.resultdb, + realm=options.realm) + return 0 def GenerateGerritChangeId(message): - """Returns the Change ID footer value (Ixxxxxx...xxx). + """Returns the Change ID footer value (Ixxxxxx...xxx). Works the same way as https://gerrit-review.googlesource.com/tools/hooks/commit-msg @@ -4528,95 +4726,99 @@ def GenerateGerritChangeId(message): The basic idea is to generate git hash of a state of the tree, original commit message, author/committer info and timestamps. """ - lines = [] - tree_hash = RunGitSilent(['write-tree']) - lines.append('tree %s' % tree_hash.strip()) - code, parent = RunGitWithCode(['rev-parse', 'HEAD~0'], suppress_stderr=False) - if code == 0: - lines.append('parent %s' % parent.strip()) - author = RunGitSilent(['var', 'GIT_AUTHOR_IDENT']) - lines.append('author %s' % author.strip()) - committer = RunGitSilent(['var', 'GIT_COMMITTER_IDENT']) - lines.append('committer %s' % committer.strip()) - lines.append('') - # Note: Gerrit's commit-hook actually cleans message of some lines and - # whitespace. This code is not doing this, but it clearly won't decrease - # entropy. - lines.append(message) - change_hash = RunCommand(['git', 'hash-object', '-t', 'commit', '--stdin'], - stdin=('\n'.join(lines)).encode()) - return 'I%s' % change_hash.strip() + lines = [] + tree_hash = RunGitSilent(['write-tree']) + lines.append('tree %s' % tree_hash.strip()) + code, parent = RunGitWithCode(['rev-parse', 'HEAD~0'], + suppress_stderr=False) + if code == 0: + lines.append('parent %s' % parent.strip()) + author = RunGitSilent(['var', 'GIT_AUTHOR_IDENT']) + lines.append('author %s' % author.strip()) + committer = RunGitSilent(['var', 'GIT_COMMITTER_IDENT']) + lines.append('committer %s' % committer.strip()) + lines.append('') + # Note: Gerrit's commit-hook actually cleans message of some lines and + # whitespace. This code is not doing this, but it clearly won't decrease + # entropy. + lines.append(message) + change_hash = RunCommand(['git', 'hash-object', '-t', 'commit', '--stdin'], + stdin=('\n'.join(lines)).encode()) + return 'I%s' % change_hash.strip() def GetTargetRef(remote, remote_branch, target_branch): - """Computes the remote branch ref to use for the CL. + """Computes the remote branch ref to use for the CL. Args: remote (str): The git remote for the CL. remote_branch (str): The git remote branch for the CL. target_branch (str): The target branch specified by the user. """ - if not (remote and remote_branch): - return None + if not (remote and remote_branch): + return None - if target_branch: - # Canonicalize branch references to the equivalent local full symbolic - # refs, which are then translated into the remote full symbolic refs - # below. - if '/' not in target_branch: - remote_branch = 'refs/remotes/%s/%s' % (remote, target_branch) - else: - prefix_replacements = ( - ('^((refs/)?remotes/)?branch-heads/', 'refs/remotes/branch-heads/'), - ('^((refs/)?remotes/)?%s/' % remote, 'refs/remotes/%s/' % remote), - ('^(refs/)?heads/', 'refs/remotes/%s/' % remote), - ) - match = None - for regex, replacement in prefix_replacements: - match = re.search(regex, target_branch) - if match: - remote_branch = target_branch.replace(match.group(0), replacement) - break - if not match: - # This is a branch path but not one we recognize; use as-is. - remote_branch = target_branch - # pylint: disable=consider-using-get - elif remote_branch in REFS_THAT_ALIAS_TO_OTHER_REFS: - # pylint: enable=consider-using-get - # Handle the refs that need to land in different refs. - remote_branch = REFS_THAT_ALIAS_TO_OTHER_REFS[remote_branch] + if target_branch: + # Canonicalize branch references to the equivalent local full symbolic + # refs, which are then translated into the remote full symbolic refs + # below. + if '/' not in target_branch: + remote_branch = 'refs/remotes/%s/%s' % (remote, target_branch) + else: + prefix_replacements = ( + ('^((refs/)?remotes/)?branch-heads/', + 'refs/remotes/branch-heads/'), + ('^((refs/)?remotes/)?%s/' % remote, + 'refs/remotes/%s/' % remote), + ('^(refs/)?heads/', 'refs/remotes/%s/' % remote), + ) + match = None + for regex, replacement in prefix_replacements: + match = re.search(regex, target_branch) + if match: + remote_branch = target_branch.replace( + match.group(0), replacement) + break + if not match: + # This is a branch path but not one we recognize; use as-is. + remote_branch = target_branch + # pylint: disable=consider-using-get + elif remote_branch in REFS_THAT_ALIAS_TO_OTHER_REFS: + # pylint: enable=consider-using-get + # Handle the refs that need to land in different refs. + remote_branch = REFS_THAT_ALIAS_TO_OTHER_REFS[remote_branch] - # Create the true path to the remote branch. - # Does the following translation: - # * refs/remotes/origin/refs/diff/test -> refs/diff/test - # * refs/remotes/origin/main -> refs/heads/main - # * refs/remotes/branch-heads/test -> refs/branch-heads/test - if remote_branch.startswith('refs/remotes/%s/refs/' % remote): - remote_branch = remote_branch.replace('refs/remotes/%s/' % remote, '') - elif remote_branch.startswith('refs/remotes/%s/' % remote): - remote_branch = remote_branch.replace('refs/remotes/%s/' % remote, - 'refs/heads/') - elif remote_branch.startswith('refs/remotes/branch-heads'): - remote_branch = remote_branch.replace('refs/remotes/', 'refs/') + # Create the true path to the remote branch. + # Does the following translation: + # * refs/remotes/origin/refs/diff/test -> refs/diff/test + # * refs/remotes/origin/main -> refs/heads/main + # * refs/remotes/branch-heads/test -> refs/branch-heads/test + if remote_branch.startswith('refs/remotes/%s/refs/' % remote): + remote_branch = remote_branch.replace('refs/remotes/%s/' % remote, '') + elif remote_branch.startswith('refs/remotes/%s/' % remote): + remote_branch = remote_branch.replace('refs/remotes/%s/' % remote, + 'refs/heads/') + elif remote_branch.startswith('refs/remotes/branch-heads'): + remote_branch = remote_branch.replace('refs/remotes/', 'refs/') - return remote_branch + return remote_branch def cleanup_list(l): - """Fixes a list so that comma separated items are put as individual items. + """Fixes a list so that comma separated items are put as individual items. So that "--reviewers joe@c,john@c --reviewers joa@c" results in options.reviewers == sorted(['joe@c', 'john@c', 'joa@c']). """ - items = sum((i.split(',') for i in l), []) - stripped_items = (i.strip() for i in items) - return sorted(filter(None, stripped_items)) + items = sum((i.split(',') for i in l), []) + stripped_items = (i.strip() for i in items) + return sorted(filter(None, stripped_items)) @subcommand.usage('[flags]') @metrics.collector.collect_metrics('git cl upload') def CMDupload(parser, args): - """Uploads the current changelist to codereview. + """Uploads the current changelist to codereview. Can skip dependency patchset uploads for a branch by running: git config branch.branch_name.skip-deps-uploads True @@ -4634,1675 +4836,1811 @@ def CMDupload(parser, args): Foo bar: implement foo will be hashtagged with "git-cl" and "foo-bar" respectively. """ - parser.add_option('--bypass-hooks', action='store_true', dest='bypass_hooks', - help='bypass upload presubmit hook') - parser.add_option('--bypass-watchlists', action='store_true', - dest='bypass_watchlists', - help='bypass watchlists auto CC-ing reviewers') - parser.add_option('-f', '--force', action='store_true', dest='force', - help="force yes to questions (don't prompt)") - parser.add_option('--message', '-m', dest='message', - help='message for patchset') - parser.add_option('-b', '--bug', - help='pre-populate the bug number(s) for this issue. ' - 'If several, separate with commas') - parser.add_option('--message-file', dest='message_file', - help='file which contains message for patchset') - parser.add_option('--title', '-t', dest='title', - help='title for patchset') - parser.add_option('-T', '--skip-title', action='store_true', - dest='skip_title', - help='Use the most recent commit message as the title of ' - 'the patchset') - parser.add_option('-r', '--reviewers', - action='append', default=[], - help='reviewer email addresses') - parser.add_option('--cc', - action='append', default=[], - help='cc email addresses') - parser.add_option('--hashtag', dest='hashtags', - action='append', default=[], - help=('Gerrit hashtag for new CL; ' - 'can be applied multiple times')) - parser.add_option('-s', - '--send-mail', - '--send-email', - dest='send_mail', - action='store_true', - help='send email to reviewer(s) and cc(s) immediately') - parser.add_option('--target_branch', - '--target-branch', - metavar='TARGET', - help='Apply CL to remote ref TARGET. ' + - 'Default: remote branch head, or main') - parser.add_option('--squash', action='store_true', - help='Squash multiple commits into one') - parser.add_option('--no-squash', action='store_false', dest='squash', - help='Don\'t squash multiple commits into one') - parser.add_option('--topic', default=None, - help='Topic to specify when uploading') - parser.add_option('--r-owners', dest='add_owners_to', action='store_const', - const='R', help='add a set of OWNERS to R') - parser.add_option('-c', - '--use-commit-queue', - action='store_true', - default=False, - help='tell the CQ to commit this patchset; ' - 'implies --send-mail') - parser.add_option('-d', '--cq-dry-run', - action='store_true', default=False, - help='Send the patchset to do a CQ dry run right after ' - 'upload.') - parser.add_option('--set-bot-commit', action='store_true', - help=optparse.SUPPRESS_HELP) - parser.add_option('--preserve-tryjobs', action='store_true', - help='instruct the CQ to let tryjobs running even after ' - 'new patchsets are uploaded instead of canceling ' - 'prior patchset\' tryjobs') - parser.add_option('--dependencies', action='store_true', - help='Uploads CLs of all the local branches that depend on ' - 'the current branch') - parser.add_option('-a', '--enable-auto-submit', action='store_true', - help='Sends your change to the CQ after an approval. Only ' - 'works on repos that have the Auto-Submit label ' - 'enabled') - parser.add_option('--parallel', action='store_true', - help='Run all tests specified by input_api.RunTests in all ' - 'PRESUBMIT files in parallel.') - parser.add_option('--no-autocc', action='store_true', - help='Disables automatic addition of CC emails') - parser.add_option('--private', action='store_true', - help='Set the review private. This implies --no-autocc.') - parser.add_option('-R', '--retry-failed', action='store_true', - help='Retry failed tryjobs from old patchset immediately ' - 'after uploading new patchset. Cannot be used with ' - '--use-commit-queue or --cq-dry-run.') - parser.add_option('--fixed', '-x', - help='List of bugs that will be commented on and marked ' - 'fixed (pre-populates "Fixed:" tag). Same format as ' - '-b option / "Bug:" tag. If fixing several issues, ' - 'separate with commas.') - parser.add_option('--edit-description', action='store_true', default=False, - help='Modify description before upload. Cannot be used ' - 'with --force. It is a noop when --no-squash is set ' - 'or a new commit is created.') - parser.add_option('--git-completion-helper', action="store_true", - help=optparse.SUPPRESS_HELP) - parser.add_option('-o', - '--push-options', - action='append', - default=[], - help='Transmit the given string to the server when ' - 'performing git push (pass-through). See git-push ' - 'documentation for more details.') - parser.add_option('--no-add-changeid', - action='store_true', - dest='no_add_changeid', - help='Do not add change-ids to messages.') - parser.add_option('--cherry-pick-stacked', - '--cp', - dest='cherry_pick_stacked', - action='store_true', - help='If parent branch has un-uploaded updates, ' - 'automatically skip parent branches and just upload ' - 'the current branch cherry-pick on its parent\'s last ' - 'uploaded commit. Allows users to skip the potential ' - 'interactive confirmation step.') - # TODO(b/265929888): Add --wip option of --cl-status option. + parser.add_option('--bypass-hooks', + action='store_true', + dest='bypass_hooks', + help='bypass upload presubmit hook') + parser.add_option('--bypass-watchlists', + action='store_true', + dest='bypass_watchlists', + help='bypass watchlists auto CC-ing reviewers') + parser.add_option('-f', + '--force', + action='store_true', + dest='force', + help="force yes to questions (don't prompt)") + parser.add_option('--message', + '-m', + dest='message', + help='message for patchset') + parser.add_option('-b', + '--bug', + help='pre-populate the bug number(s) for this issue. ' + 'If several, separate with commas') + parser.add_option('--message-file', + dest='message_file', + help='file which contains message for patchset') + parser.add_option('--title', '-t', dest='title', help='title for patchset') + parser.add_option('-T', + '--skip-title', + action='store_true', + dest='skip_title', + help='Use the most recent commit message as the title of ' + 'the patchset') + parser.add_option('-r', + '--reviewers', + action='append', + default=[], + help='reviewer email addresses') + parser.add_option('--cc', + action='append', + default=[], + help='cc email addresses') + parser.add_option('--hashtag', + dest='hashtags', + action='append', + default=[], + help=('Gerrit hashtag for new CL; ' + 'can be applied multiple times')) + parser.add_option('-s', + '--send-mail', + '--send-email', + dest='send_mail', + action='store_true', + help='send email to reviewer(s) and cc(s) immediately') + parser.add_option('--target_branch', + '--target-branch', + metavar='TARGET', + help='Apply CL to remote ref TARGET. ' + + 'Default: remote branch head, or main') + parser.add_option('--squash', + action='store_true', + help='Squash multiple commits into one') + parser.add_option('--no-squash', + action='store_false', + dest='squash', + help='Don\'t squash multiple commits into one') + parser.add_option('--topic', + default=None, + help='Topic to specify when uploading') + parser.add_option('--r-owners', + dest='add_owners_to', + action='store_const', + const='R', + help='add a set of OWNERS to R') + parser.add_option('-c', + '--use-commit-queue', + action='store_true', + default=False, + help='tell the CQ to commit this patchset; ' + 'implies --send-mail') + parser.add_option('-d', + '--cq-dry-run', + action='store_true', + default=False, + help='Send the patchset to do a CQ dry run right after ' + 'upload.') + parser.add_option('--set-bot-commit', + action='store_true', + help=optparse.SUPPRESS_HELP) + parser.add_option('--preserve-tryjobs', + action='store_true', + help='instruct the CQ to let tryjobs running even after ' + 'new patchsets are uploaded instead of canceling ' + 'prior patchset\' tryjobs') + parser.add_option( + '--dependencies', + action='store_true', + help='Uploads CLs of all the local branches that depend on ' + 'the current branch') + parser.add_option( + '-a', + '--enable-auto-submit', + action='store_true', + help='Sends your change to the CQ after an approval. Only ' + 'works on repos that have the Auto-Submit label ' + 'enabled') + parser.add_option( + '--parallel', + action='store_true', + help='Run all tests specified by input_api.RunTests in all ' + 'PRESUBMIT files in parallel.') + parser.add_option('--no-autocc', + action='store_true', + help='Disables automatic addition of CC emails') + parser.add_option('--private', + action='store_true', + help='Set the review private. This implies --no-autocc.') + parser.add_option('-R', + '--retry-failed', + action='store_true', + help='Retry failed tryjobs from old patchset immediately ' + 'after uploading new patchset. Cannot be used with ' + '--use-commit-queue or --cq-dry-run.') + parser.add_option('--fixed', + '-x', + help='List of bugs that will be commented on and marked ' + 'fixed (pre-populates "Fixed:" tag). Same format as ' + '-b option / "Bug:" tag. If fixing several issues, ' + 'separate with commas.') + parser.add_option('--edit-description', + action='store_true', + default=False, + help='Modify description before upload. Cannot be used ' + 'with --force. It is a noop when --no-squash is set ' + 'or a new commit is created.') + parser.add_option('--git-completion-helper', + action="store_true", + help=optparse.SUPPRESS_HELP) + parser.add_option('-o', + '--push-options', + action='append', + default=[], + help='Transmit the given string to the server when ' + 'performing git push (pass-through). See git-push ' + 'documentation for more details.') + parser.add_option('--no-add-changeid', + action='store_true', + dest='no_add_changeid', + help='Do not add change-ids to messages.') + parser.add_option('--cherry-pick-stacked', + '--cp', + dest='cherry_pick_stacked', + action='store_true', + help='If parent branch has un-uploaded updates, ' + 'automatically skip parent branches and just upload ' + 'the current branch cherry-pick on its parent\'s last ' + 'uploaded commit. Allows users to skip the potential ' + 'interactive confirmation step.') + # TODO(b/265929888): Add --wip option of --cl-status option. - orig_args = args - (options, args) = parser.parse_args(args) + orig_args = args + (options, args) = parser.parse_args(args) - if options.git_completion_helper: - print(' '.join(opt.get_opt_string() for opt in parser.option_list - if opt.help != optparse.SUPPRESS_HELP)) - return + if options.git_completion_helper: + print(' '.join(opt.get_opt_string() for opt in parser.option_list + if opt.help != optparse.SUPPRESS_HELP)) + return - # TODO(crbug.com/1475405): Warn users if the project uses submodules and - # they have fsmonitor enabled. - if os.path.isfile('.gitmodules'): - git_common.warn_submodule() + # TODO(crbug.com/1475405): Warn users if the project uses submodules and + # they have fsmonitor enabled. + if os.path.isfile('.gitmodules'): + git_common.warn_submodule() - if git_common.is_dirty_git_tree('upload'): - return 1 + if git_common.is_dirty_git_tree('upload'): + return 1 - options.reviewers = cleanup_list(options.reviewers) - options.cc = cleanup_list(options.cc) + options.reviewers = cleanup_list(options.reviewers) + options.cc = cleanup_list(options.cc) - if options.edit_description and options.force: - parser.error('Only one of --force and --edit-description allowed') + if options.edit_description and options.force: + parser.error('Only one of --force and --edit-description allowed') - if options.message_file: - if options.message: - parser.error('Only one of --message and --message-file allowed.') - options.message = gclient_utils.FileRead(options.message_file) + if options.message_file: + if options.message: + parser.error('Only one of --message and --message-file allowed.') + options.message = gclient_utils.FileRead(options.message_file) - if ([options.cq_dry_run, - options.use_commit_queue, - options.retry_failed].count(True) > 1): - parser.error('Only one of --use-commit-queue, --cq-dry-run or ' - '--retry-failed is allowed.') + if ([options.cq_dry_run, options.use_commit_queue, options.retry_failed + ].count(True) > 1): + parser.error('Only one of --use-commit-queue, --cq-dry-run or ' + '--retry-failed is allowed.') - if options.skip_title and options.title: - parser.error('Only one of --title and --skip-title allowed.') + if options.skip_title and options.title: + parser.error('Only one of --title and --skip-title allowed.') - if options.use_commit_queue: - options.send_mail = True + if options.use_commit_queue: + options.send_mail = True - if options.squash is None: - # Load default for user, repo, squash=true, in this order. - options.squash = settings.GetSquashGerritUploads() + if options.squash is None: + # Load default for user, repo, squash=true, in this order. + options.squash = settings.GetSquashGerritUploads() - cl = Changelist(branchref=options.target_branch) + cl = Changelist(branchref=options.target_branch) - # Warm change details cache now to avoid RPCs later, reducing latency for - # developers. - if cl.GetIssue(): - cl._GetChangeDetail( - ['DETAILED_ACCOUNTS', 'CURRENT_REVISION', 'CURRENT_COMMIT', 'LABELS']) + # Warm change details cache now to avoid RPCs later, reducing latency for + # developers. + if cl.GetIssue(): + cl._GetChangeDetail([ + 'DETAILED_ACCOUNTS', 'CURRENT_REVISION', 'CURRENT_COMMIT', 'LABELS' + ]) - if options.retry_failed and not cl.GetIssue(): - print('No previous patchsets, so --retry-failed has no effect.') - options.retry_failed = False + if options.retry_failed and not cl.GetIssue(): + print('No previous patchsets, so --retry-failed has no effect.') + options.retry_failed = False + disable_dogfood_stacked_changes = os.environ.get( + DOGFOOD_STACKED_CHANGES_VAR) == '0' + dogfood_stacked_changes = os.environ.get(DOGFOOD_STACKED_CHANGES_VAR) == '1' - disable_dogfood_stacked_changes = os.environ.get( - DOGFOOD_STACKED_CHANGES_VAR) == '0' - dogfood_stacked_changes = os.environ.get(DOGFOOD_STACKED_CHANGES_VAR) == '1' + # Only print message for folks who don't have DOGFOOD_STACKED_CHANGES set + # to an expected value. + if (options.squash and not dogfood_stacked_changes + and not disable_dogfood_stacked_changes): + print( + 'This repo has been enrolled in the stacked changes dogfood.\n' + '`git cl upload` now uploads the current branch and all upstream ' + 'branches that have un-uploaded updates.\n' + 'Patches can now be reapplied with --force:\n' + '`git cl patch --reapply --force`.\n' + 'Googlers may visit go/stacked-changes-dogfood for more information.\n' + '\n' + 'Depot Tools no longer sets new uploads to "WIP". Please update the\n' + '"Set new changes to "work in progress" by default" checkbox at\n' + 'https://-review.googlesource.com/settings/\n' + '\n' + 'To opt-out use `export DOGFOOD_STACKED_CHANGES=0`.\n' + 'To hide this message use `export DOGFOOD_STACKED_CHANGES=1`.\n' + 'File bugs at https://bit.ly/3Y6opoI\n') - # Only print message for folks who don't have DOGFOOD_STACKED_CHANGES set - # to an expected value. - if (options.squash and not dogfood_stacked_changes - and not disable_dogfood_stacked_changes): - print( - 'This repo has been enrolled in the stacked changes dogfood.\n' - '`git cl upload` now uploads the current branch and all upstream ' - 'branches that have un-uploaded updates.\n' - 'Patches can now be reapplied with --force:\n' - '`git cl patch --reapply --force`.\n' - 'Googlers may visit go/stacked-changes-dogfood for more information.\n' - '\n' - 'Depot Tools no longer sets new uploads to "WIP". Please update the\n' - '"Set new changes to "work in progress" by default" checkbox at\n' - 'https://-review.googlesource.com/settings/\n' - '\n' - 'To opt-out use `export DOGFOOD_STACKED_CHANGES=0`.\n' - 'To hide this message use `export DOGFOOD_STACKED_CHANGES=1`.\n' - 'File bugs at https://bit.ly/3Y6opoI\n') + if options.squash and not disable_dogfood_stacked_changes: + if options.dependencies: + parser.error( + '--dependencies is not available for this dogfood workflow.') - if options.squash and not disable_dogfood_stacked_changes: - if options.dependencies: - parser.error('--dependencies is not available for this dogfood workflow.') + if options.cherry_pick_stacked: + try: + orig_args.remove('--cherry-pick-stacked') + except ValueError: + orig_args.remove('--cp') + UploadAllSquashed(options, orig_args) + return 0 if options.cherry_pick_stacked: - try: - orig_args.remove('--cherry-pick-stacked') - except ValueError: - orig_args.remove('--cp') - UploadAllSquashed(options, orig_args) - return 0 + parser.error( + '--cherry-pick-stacked is not available for this workflow.') - if options.cherry_pick_stacked: - parser.error('--cherry-pick-stacked is not available for this workflow.') + # cl.GetMostRecentPatchset uses cached information, and can return the last + # patchset before upload. Calling it here makes it clear that it's the + # last patchset before upload. Note that GetMostRecentPatchset will fail + # if no CL has been uploaded yet. + if options.retry_failed: + patchset = cl.GetMostRecentPatchset() - # cl.GetMostRecentPatchset uses cached information, and can return the last - # patchset before upload. Calling it here makes it clear that it's the - # last patchset before upload. Note that GetMostRecentPatchset will fail - # if no CL has been uploaded yet. - if options.retry_failed: - patchset = cl.GetMostRecentPatchset() + ret = cl.CMDUpload(options, args, orig_args) - ret = cl.CMDUpload(options, args, orig_args) + if options.retry_failed: + if ret != 0: + print('Upload failed, so --retry-failed has no effect.') + return ret + builds, _ = _fetch_latest_builds(cl, + DEFAULT_BUILDBUCKET_HOST, + latest_patchset=patchset) + jobs = _filter_failed_for_retry(builds) + if len(jobs) == 0: + print('No failed tryjobs, so --retry-failed has no effect.') + return ret + _trigger_tryjobs(cl, jobs, options, patchset + 1) - if options.retry_failed: - if ret != 0: - print('Upload failed, so --retry-failed has no effect.') - return ret - builds, _ = _fetch_latest_builds(cl, - DEFAULT_BUILDBUCKET_HOST, - latest_patchset=patchset) - jobs = _filter_failed_for_retry(builds) - if len(jobs) == 0: - print('No failed tryjobs, so --retry-failed has no effect.') - return ret - _trigger_tryjobs(cl, jobs, options, patchset + 1) - - return ret + return ret def UploadAllSquashed(options: optparse.Values, orig_args: Sequence[str]) -> int: - """Uploads the current and upstream branches (if necessary).""" - cls, cherry_pick_current = _UploadAllPrecheck(options, orig_args) + """Uploads the current and upstream branches (if necessary).""" + cls, cherry_pick_current = _UploadAllPrecheck(options, orig_args) - # Create commits. - uploads_by_cl: List[Tuple[Changelist, _NewUpload]] = [] - if cherry_pick_current: - parent = cls[1]._GitGetBranchConfigValue(GERRIT_SQUASH_HASH_CONFIG_KEY) - new_upload = cls[0].PrepareCherryPickSquashedCommit(options, parent) - uploads_by_cl.append((cls[0], new_upload)) - else: - ordered_cls = list(reversed(cls)) + # Create commits. + uploads_by_cl: List[Tuple[Changelist, _NewUpload]] = [] + if cherry_pick_current: + parent = cls[1]._GitGetBranchConfigValue(GERRIT_SQUASH_HASH_CONFIG_KEY) + new_upload = cls[0].PrepareCherryPickSquashedCommit(options, parent) + uploads_by_cl.append((cls[0], new_upload)) + else: + ordered_cls = list(reversed(cls)) - cl = ordered_cls[0] - # We can only support external changes when we're only uploading one - # branch. - parent = cl._UpdateWithExternalChanges() if len(ordered_cls) == 1 else None - orig_parent = None - if parent is None: - origin = '.' - branch = cl.GetBranch() + cl = ordered_cls[0] + # We can only support external changes when we're only uploading one + # branch. + parent = cl._UpdateWithExternalChanges() if len( + ordered_cls) == 1 else None + orig_parent = None + if parent is None: + origin = '.' + branch = cl.GetBranch() - while origin == '.': - # Search for cl's closest ancestor with a gerrit hash. - origin, upstream_branch_ref = Changelist.FetchUpstreamTuple(branch) - if origin == '.': - upstream_branch = scm.GIT.ShortBranchName(upstream_branch_ref) + while origin == '.': + # Search for cl's closest ancestor with a gerrit hash. + origin, upstream_branch_ref = Changelist.FetchUpstreamTuple( + branch) + if origin == '.': + upstream_branch = scm.GIT.ShortBranchName( + upstream_branch_ref) - # Support the `git merge` and `git pull` workflow. - if upstream_branch in ['master', 'main']: - parent = cl.GetCommonAncestorWithUpstream() - else: - orig_parent = scm.GIT.GetBranchConfig(settings.GetRoot(), - upstream_branch, - LAST_UPLOAD_HASH_CONFIG_KEY) - parent = scm.GIT.GetBranchConfig(settings.GetRoot(), - upstream_branch, - GERRIT_SQUASH_HASH_CONFIG_KEY) - if parent: - break - branch = upstream_branch - else: - # Either the root of the tree is the cl's direct parent and the while - # loop above only found empty branches between cl and the root of the - # tree. - parent = cl.GetCommonAncestorWithUpstream() + # Support the `git merge` and `git pull` workflow. + if upstream_branch in ['master', 'main']: + parent = cl.GetCommonAncestorWithUpstream() + else: + orig_parent = scm.GIT.GetBranchConfig( + settings.GetRoot(), upstream_branch, + LAST_UPLOAD_HASH_CONFIG_KEY) + parent = scm.GIT.GetBranchConfig( + settings.GetRoot(), upstream_branch, + GERRIT_SQUASH_HASH_CONFIG_KEY) + if parent: + break + branch = upstream_branch + else: + # Either the root of the tree is the cl's direct parent and the + # while loop above only found empty branches between cl and the + # root of the tree. + parent = cl.GetCommonAncestorWithUpstream() - if orig_parent is None: - orig_parent = parent - for i, cl in enumerate(ordered_cls): - # If we're in the middle of the stack, set end_commit to downstream's - # direct ancestor. - if i + 1 < len(ordered_cls): - child_base_commit = ordered_cls[i + 1].GetCommonAncestorWithUpstream() - else: - child_base_commit = None - new_upload = cl.PrepareSquashedCommit(options, - parent, - orig_parent, - end_commit=child_base_commit) - uploads_by_cl.append((cl, new_upload)) - parent = new_upload.commit_to_push - orig_parent = child_base_commit + if orig_parent is None: + orig_parent = parent + for i, cl in enumerate(ordered_cls): + # If we're in the middle of the stack, set end_commit to + # downstream's direct ancestor. + if i + 1 < len(ordered_cls): + child_base_commit = ordered_cls[ + i + 1].GetCommonAncestorWithUpstream() + else: + child_base_commit = None + new_upload = cl.PrepareSquashedCommit(options, + parent, + orig_parent, + end_commit=child_base_commit) + uploads_by_cl.append((cl, new_upload)) + parent = new_upload.commit_to_push + orig_parent = child_base_commit - # Create refspec options - cl, new_upload = uploads_by_cl[-1] - refspec_opts = cl._GetRefSpecOptions( - options, - new_upload.change_desc, - multi_change_upload=len(uploads_by_cl) > 1, - dogfood_path=True) - refspec_suffix = '' - if refspec_opts: - refspec_suffix = '%' + ','.join(refspec_opts) - assert ' ' not in refspec_suffix, ('spaces not allowed in refspec: "%s"' % - refspec_suffix) + # Create refspec options + cl, new_upload = uploads_by_cl[-1] + refspec_opts = cl._GetRefSpecOptions( + options, + new_upload.change_desc, + multi_change_upload=len(uploads_by_cl) > 1, + dogfood_path=True) + refspec_suffix = '' + if refspec_opts: + refspec_suffix = '%' + ','.join(refspec_opts) + assert ' ' not in refspec_suffix, ( + 'spaces not allowed in refspec: "%s"' % refspec_suffix) - remote, remote_branch = cl.GetRemoteBranch() - branch = GetTargetRef(remote, remote_branch, options.target_branch) - refspec = '%s:refs/for/%s%s' % (new_upload.commit_to_push, branch, - refspec_suffix) + remote, remote_branch = cl.GetRemoteBranch() + branch = GetTargetRef(remote, remote_branch, options.target_branch) + refspec = '%s:refs/for/%s%s' % (new_upload.commit_to_push, branch, + refspec_suffix) - # Git push - git_push_metadata = { - 'gerrit_host': - cl.GetGerritHost(), - 'title': - options.title or '', - 'change_id': - git_footers.get_footer_change_id(new_upload.change_desc.description), - 'description': - new_upload.change_desc.description, - } - push_stdout = cl._RunGitPushWithTraces(refspec, refspec_opts, - git_push_metadata, - options.push_options) + # Git push + git_push_metadata = { + 'gerrit_host': + cl.GetGerritHost(), + 'title': + options.title or '', + 'change_id': + git_footers.get_footer_change_id(new_upload.change_desc.description), + 'description': + new_upload.change_desc.description, + } + push_stdout = cl._RunGitPushWithTraces(refspec, refspec_opts, + git_push_metadata, + options.push_options) - # Post push updates - regex = re.compile(r'remote:\s+https?://[\w\-\.\+\/#]*/(\d+)\s.*') - change_numbers = [ - m.group(1) for m in map(regex.match, push_stdout.splitlines()) if m - ] + # Post push updates + regex = re.compile(r'remote:\s+https?://[\w\-\.\+\/#]*/(\d+)\s.*') + change_numbers = [ + m.group(1) for m in map(regex.match, push_stdout.splitlines()) if m + ] - for i, (cl, new_upload) in enumerate(uploads_by_cl): - cl.PostUploadUpdates(options, new_upload, change_numbers[i]) + for i, (cl, new_upload) in enumerate(uploads_by_cl): + cl.PostUploadUpdates(options, new_upload, change_numbers[i]) - return 0 + return 0 def _UploadAllPrecheck(options, orig_args): - # type: (optparse.Values, Sequence[str]) -> Tuple[Sequence[Changelist], bool] - """Checks the state of the tree and gives the user uploading options + # type: (optparse.Values, Sequence[str]) -> Tuple[Sequence[Changelist], + # bool] + """Checks the state of the tree and gives the user uploading options Returns: A tuple of the ordered list of changes that have new commits since their last upload and a boolean of whether the user wants to cherry-pick and upload the current branch instead of uploading all cls. """ - cl = Changelist() - if cl.GetBranch() is None: - DieWithError('Can\'t upload from detached HEAD state. Get on a branch!') + cl = Changelist() + if cl.GetBranch() is None: + DieWithError('Can\'t upload from detached HEAD state. Get on a branch!') - branch_ref = None - cls = [] - must_upload_upstream = False - first_pass = True + branch_ref = None + cls = [] + must_upload_upstream = False + first_pass = True - Changelist._GerritCommitMsgHookCheck(offer_removal=not options.force) + Changelist._GerritCommitMsgHookCheck(offer_removal=not options.force) - while True: - if len(cls) > _MAX_STACKED_BRANCHES_UPLOAD: - DieWithError( - 'More than %s branches in the stack have not been uploaded.\n' - 'Are your branches in a misconfigured state?\n' - 'If not, please upload some upstream changes first.' % - (_MAX_STACKED_BRANCHES_UPLOAD)) + while True: + if len(cls) > _MAX_STACKED_BRANCHES_UPLOAD: + DieWithError( + 'More than %s branches in the stack have not been uploaded.\n' + 'Are your branches in a misconfigured state?\n' + 'If not, please upload some upstream changes first.' % + (_MAX_STACKED_BRANCHES_UPLOAD)) - cl = Changelist(branchref=branch_ref) + cl = Changelist(branchref=branch_ref) - # Only add CL if it has anything to commit. - base_commit = cl.GetCommonAncestorWithUpstream() - end_commit = RunGit(['rev-parse', cl.GetBranchRef()]).strip() + # Only add CL if it has anything to commit. + base_commit = cl.GetCommonAncestorWithUpstream() + end_commit = RunGit(['rev-parse', cl.GetBranchRef()]).strip() - commit_summary = _GetCommitCountSummary(base_commit, end_commit) - if commit_summary: - cls.append(cl) - if (not first_pass and - cl._GitGetBranchConfigValue(GERRIT_SQUASH_HASH_CONFIG_KEY) is None): - # We are mid-stack and the user must upload their upstream branches. - must_upload_upstream = True - print(f'Found change with {commit_summary}...') - elif first_pass: # The current branch has nothing to commit. Exit. - DieWithError('Branch %s has nothing to commit' % cl.GetBranch()) - # Else: A mid-stack branch has nothing to commit. We do not add it to cls. - first_pass = False + commit_summary = _GetCommitCountSummary(base_commit, end_commit) + if commit_summary: + cls.append(cl) + if (not first_pass and + cl._GitGetBranchConfigValue(GERRIT_SQUASH_HASH_CONFIG_KEY) + is None): + # We are mid-stack and the user must upload their upstream + # branches. + must_upload_upstream = True + print(f'Found change with {commit_summary}...') + elif first_pass: # The current branch has nothing to commit. Exit. + DieWithError('Branch %s has nothing to commit' % cl.GetBranch()) + # Else: A mid-stack branch has nothing to commit. We do not add it to + # cls. + first_pass = False - # Cases below determine if we should continue to traverse up the tree. - origin, upstream_branch_ref = Changelist.FetchUpstreamTuple(cl.GetBranch()) - branch_ref = upstream_branch_ref # set branch for next run. + # Cases below determine if we should continue to traverse up the tree. + origin, upstream_branch_ref = Changelist.FetchUpstreamTuple( + cl.GetBranch()) + branch_ref = upstream_branch_ref # set branch for next run. - upstream_branch = scm.GIT.ShortBranchName(upstream_branch_ref) - upstream_last_upload = scm.GIT.GetBranchConfig(settings.GetRoot(), - upstream_branch, - LAST_UPLOAD_HASH_CONFIG_KEY) + upstream_branch = scm.GIT.ShortBranchName(upstream_branch_ref) + upstream_last_upload = scm.GIT.GetBranchConfig( + settings.GetRoot(), upstream_branch, LAST_UPLOAD_HASH_CONFIG_KEY) - # Case 1: We've reached the beginning of the tree. - if origin != '.': - break + # Case 1: We've reached the beginning of the tree. + if origin != '.': + break - # Case 2: If any upstream branches have never been uploaded, - # the user MUST upload them unless they are empty. Continue to - # next loop to add upstream if it is not empty. - if not upstream_last_upload: - continue + # Case 2: If any upstream branches have never been uploaded, + # the user MUST upload them unless they are empty. Continue to + # next loop to add upstream if it is not empty. + if not upstream_last_upload: + continue - # Case 3: If upstream's last_upload == cl.base_commit we do - # not need to upload any more upstreams from this point on. - # (Even if there may be diverged branches higher up the tree) - if base_commit == upstream_last_upload: - break + # Case 3: If upstream's last_upload == cl.base_commit we do + # not need to upload any more upstreams from this point on. + # (Even if there may be diverged branches higher up the tree) + if base_commit == upstream_last_upload: + break - # Case 4: If upstream's last_upload < cl.base_commit we are - # uploading cl and upstream_cl. - # Continue up the tree to check other branch relations. - if scm.GIT.IsAncestor(upstream_last_upload, base_commit): - continue + # Case 4: If upstream's last_upload < cl.base_commit we are + # uploading cl and upstream_cl. + # Continue up the tree to check other branch relations. + if scm.GIT.IsAncestor(upstream_last_upload, base_commit): + continue - # Case 5: If cl.base_commit < upstream's last_upload the user - # must rebase before uploading. - if scm.GIT.IsAncestor(base_commit, upstream_last_upload): - DieWithError( - 'At least one branch in the stack has diverged from its upstream ' - 'branch and does not contain its upstream\'s last upload.\n' - 'Please rebase the stack with `git rebase-update` before uploading.') + # Case 5: If cl.base_commit < upstream's last_upload the user + # must rebase before uploading. + if scm.GIT.IsAncestor(base_commit, upstream_last_upload): + DieWithError( + 'At least one branch in the stack has diverged from its upstream ' + 'branch and does not contain its upstream\'s last upload.\n' + 'Please rebase the stack with `git rebase-update` before uploading.' + ) - # The tree went through a rebase. LAST_UPLOAD_HASH_CONFIG_KEY no longer has - # any relation to commits in the tree. Continue up the tree until we hit - # the root. + # The tree went through a rebase. LAST_UPLOAD_HASH_CONFIG_KEY no longer + # has any relation to commits in the tree. Continue up the tree until we + # hit the root. - # We assume all cls in the stack have the same auth requirements and only - # check this once. - cls[0].EnsureAuthenticated(force=options.force) + # We assume all cls in the stack have the same auth requirements and only + # check this once. + cls[0].EnsureAuthenticated(force=options.force) - cherry_pick = False - if len(cls) > 1: - opt_message = '' - branches = ', '.join([cl.branch for cl in cls]) - if len(orig_args): - opt_message = ('options %s will be used for all uploads.\n' % orig_args) - if must_upload_upstream: - msg = ('At least one parent branch in `%s` has never been uploaded ' - 'and must be uploaded before/with `%s`.\n' % - (branches, cls[1].branch)) - if options.cherry_pick_stacked: - DieWithError(msg) - if not options.force: - confirm_or_exit('\n' + opt_message + msg) - else: - if options.cherry_pick_stacked: - print('cherry-picking `%s` on %s\'s last upload' % - (cls[0].branch, cls[1].branch)) - cherry_pick = True - elif not options.force: - answer = gclient_utils.AskForData( - '\n' + opt_message + - 'Press enter to update branches %s.\nOr type `n` to upload only ' - '`%s` cherry-picked on %s\'s last upload:' % - (branches, cls[0].branch, cls[1].branch)) - if answer.lower() == 'n': - cherry_pick = True - return cls, cherry_pick + cherry_pick = False + if len(cls) > 1: + opt_message = '' + branches = ', '.join([cl.branch for cl in cls]) + if len(orig_args): + opt_message = ('options %s will be used for all uploads.\n' % + orig_args) + if must_upload_upstream: + msg = ('At least one parent branch in `%s` has never been uploaded ' + 'and must be uploaded before/with `%s`.\n' % + (branches, cls[1].branch)) + if options.cherry_pick_stacked: + DieWithError(msg) + if not options.force: + confirm_or_exit('\n' + opt_message + msg) + else: + if options.cherry_pick_stacked: + print('cherry-picking `%s` on %s\'s last upload' % + (cls[0].branch, cls[1].branch)) + cherry_pick = True + elif not options.force: + answer = gclient_utils.AskForData( + '\n' + opt_message + + 'Press enter to update branches %s.\nOr type `n` to upload only ' + '`%s` cherry-picked on %s\'s last upload:' % + (branches, cls[0].branch, cls[1].branch)) + if answer.lower() == 'n': + cherry_pick = True + return cls, cherry_pick @subcommand.usage('--description=') @metrics.collector.collect_metrics('git cl split') def CMDsplit(parser, args): - """Splits a branch into smaller branches and uploads CLs. + """Splits a branch into smaller branches and uploads CLs. Creates a branch and uploads a CL for each group of files modified in the current branch that share a common OWNERS file. In the CL description and comment, the string '$directory', is replaced with the directory containing the shared OWNERS file. """ - parser.add_option('-d', '--description', dest='description_file', - help='A text file containing a CL description in which ' - '$directory will be replaced by each CL\'s directory.') - parser.add_option('-c', '--comment', dest='comment_file', - help='A text file containing a CL comment.') - parser.add_option('-n', '--dry-run', dest='dry_run', action='store_true', - default=False, - help='List the files and reviewers for each CL that would ' - 'be created, but don\'t create branches or CLs.') - parser.add_option('--cq-dry-run', action='store_true', - help='If set, will do a cq dry run for each uploaded CL. ' - 'Please be careful when doing this; more than ~10 CLs ' - 'has the potential to overload our build ' - 'infrastructure. Try to upload these not during high ' - 'load times (usually 11-3 Mountain View time). Email ' - 'infra-dev@chromium.org with any questions.') - parser.add_option('-a', - '--enable-auto-submit', - action='store_true', - dest='enable_auto_submit', - default=True, - help='Sends your change to the CQ after an approval. Only ' - 'works on repos that have the Auto-Submit label ' - 'enabled') - parser.add_option('--disable-auto-submit', - action='store_false', - dest='enable_auto_submit', - help='Disables automatic sending of the changes to the CQ ' - 'after approval. Note that auto-submit only works for ' - 'repos that have the Auto-Submit label enabled.') - parser.add_option('--max-depth', - type='int', - default=0, - help='The max depth to look for OWNERS files. Useful for ' - 'controlling the granularity of the split CLs, e.g. ' - '--max-depth=1 will only split by top-level ' - 'directory. Specifying a value less than 1 means no ' - 'limit on max depth.') - parser.add_option('--topic', - default=None, - help='Topic to specify when uploading') - options, _ = parser.parse_args(args) + parser.add_option('-d', + '--description', + dest='description_file', + help='A text file containing a CL description in which ' + '$directory will be replaced by each CL\'s directory.') + parser.add_option('-c', + '--comment', + dest='comment_file', + help='A text file containing a CL comment.') + parser.add_option( + '-n', + '--dry-run', + dest='dry_run', + action='store_true', + default=False, + help='List the files and reviewers for each CL that would ' + 'be created, but don\'t create branches or CLs.') + parser.add_option('--cq-dry-run', + action='store_true', + help='If set, will do a cq dry run for each uploaded CL. ' + 'Please be careful when doing this; more than ~10 CLs ' + 'has the potential to overload our build ' + 'infrastructure. Try to upload these not during high ' + 'load times (usually 11-3 Mountain View time). Email ' + 'infra-dev@chromium.org with any questions.') + parser.add_option( + '-a', + '--enable-auto-submit', + action='store_true', + dest='enable_auto_submit', + default=True, + help='Sends your change to the CQ after an approval. Only ' + 'works on repos that have the Auto-Submit label ' + 'enabled') + parser.add_option( + '--disable-auto-submit', + action='store_false', + dest='enable_auto_submit', + help='Disables automatic sending of the changes to the CQ ' + 'after approval. Note that auto-submit only works for ' + 'repos that have the Auto-Submit label enabled.') + parser.add_option('--max-depth', + type='int', + default=0, + help='The max depth to look for OWNERS files. Useful for ' + 'controlling the granularity of the split CLs, e.g. ' + '--max-depth=1 will only split by top-level ' + 'directory. Specifying a value less than 1 means no ' + 'limit on max depth.') + parser.add_option('--topic', + default=None, + help='Topic to specify when uploading') + options, _ = parser.parse_args(args) - if not options.description_file: - parser.error('No --description flag specified.') + if not options.description_file: + parser.error('No --description flag specified.') - def WrappedCMDupload(args): - return CMDupload(OptionParser(), args) + def WrappedCMDupload(args): + return CMDupload(OptionParser(), args) - return split_cl.SplitCl(options.description_file, options.comment_file, - Changelist, WrappedCMDupload, options.dry_run, - options.cq_dry_run, options.enable_auto_submit, - options.max_depth, options.topic, settings.GetRoot()) + return split_cl.SplitCl(options.description_file, options.comment_file, + Changelist, WrappedCMDupload, options.dry_run, + options.cq_dry_run, options.enable_auto_submit, + options.max_depth, options.topic, + settings.GetRoot()) @subcommand.usage('DEPRECATED') @metrics.collector.collect_metrics('git cl commit') def CMDdcommit(parser, args): - """DEPRECATED: Used to commit the current changelist via git-svn.""" - message = ('git-cl no longer supports committing to SVN repositories via ' - 'git-svn. You probably want to use `git cl land` instead.') - print(message) - return 1 + """DEPRECATED: Used to commit the current changelist via git-svn.""" + message = ('git-cl no longer supports committing to SVN repositories via ' + 'git-svn. You probably want to use `git cl land` instead.') + print(message) + return 1 @subcommand.usage('[upstream branch to apply against]') @metrics.collector.collect_metrics('git cl land') def CMDland(parser, args): - """Commits the current changelist via git. + """Commits the current changelist via git. In case of Gerrit, uses Gerrit REST api to "submit" the issue, which pushes upstream and closes the issue automatically and atomically. """ - parser.add_option('--bypass-hooks', action='store_true', dest='bypass_hooks', - help='bypass upload presubmit hook') - parser.add_option('-f', '--force', action='store_true', dest='force', - help="force yes to questions (don't prompt)") - parser.add_option('--parallel', action='store_true', - help='Run all tests specified by input_api.RunTests in all ' - 'PRESUBMIT files in parallel.') - parser.add_option('--resultdb', action='store_true', - help='Run presubmit checks in the ResultSink environment ' - 'and send results to the ResultDB database.') - parser.add_option('--realm', help='LUCI realm if reporting to ResultDB') - (options, args) = parser.parse_args(args) + parser.add_option('--bypass-hooks', + action='store_true', + dest='bypass_hooks', + help='bypass upload presubmit hook') + parser.add_option('-f', + '--force', + action='store_true', + dest='force', + help="force yes to questions (don't prompt)") + parser.add_option( + '--parallel', + action='store_true', + help='Run all tests specified by input_api.RunTests in all ' + 'PRESUBMIT files in parallel.') + parser.add_option('--resultdb', + action='store_true', + help='Run presubmit checks in the ResultSink environment ' + 'and send results to the ResultDB database.') + parser.add_option('--realm', help='LUCI realm if reporting to ResultDB') + (options, args) = parser.parse_args(args) - cl = Changelist() + cl = Changelist() - if not cl.GetIssue(): - DieWithError('You must upload the change first to Gerrit.\n' - ' If you would rather have `git cl land` upload ' - 'automatically for you, see http://crbug.com/642759') - return cl.CMDLand(options.force, options.bypass_hooks, options.verbose, - options.parallel, options.resultdb, options.realm) + if not cl.GetIssue(): + DieWithError('You must upload the change first to Gerrit.\n' + ' If you would rather have `git cl land` upload ' + 'automatically for you, see http://crbug.com/642759') + return cl.CMDLand(options.force, options.bypass_hooks, options.verbose, + options.parallel, options.resultdb, options.realm) @subcommand.usage('') @metrics.collector.collect_metrics('git cl patch') def CMDpatch(parser, args): - """Applies (cherry-picks) a Gerrit changelist locally.""" - parser.add_option('-b', dest='newbranch', - help='create a new branch off trunk for the patch') - parser.add_option('-f', '--force', action='store_true', - help='overwrite state on the current or chosen branch') - parser.add_option('-n', '--no-commit', action='store_true', dest='nocommit', - help='don\'t commit after patch applies.') + """Applies (cherry-picks) a Gerrit changelist locally.""" + parser.add_option('-b', + dest='newbranch', + help='create a new branch off trunk for the patch') + parser.add_option('-f', + '--force', + action='store_true', + help='overwrite state on the current or chosen branch') + parser.add_option('-n', + '--no-commit', + action='store_true', + dest='nocommit', + help='don\'t commit after patch applies.') - group = optparse.OptionGroup( - parser, - 'Options for continuing work on the current issue uploaded from a ' - 'different clone (e.g. different machine). Must be used independently ' - 'from the other options. No issue number should be specified, and the ' - 'branch must have an issue number associated with it') - group.add_option('--reapply', action='store_true', dest='reapply', - help='Reset the branch and reapply the issue.\n' - 'CAUTION: This will undo any local changes in this ' - 'branch') + group = optparse.OptionGroup( + parser, + 'Options for continuing work on the current issue uploaded from a ' + 'different clone (e.g. different machine). Must be used independently ' + 'from the other options. No issue number should be specified, and the ' + 'branch must have an issue number associated with it') + group.add_option('--reapply', + action='store_true', + dest='reapply', + help='Reset the branch and reapply the issue.\n' + 'CAUTION: This will undo any local changes in this ' + 'branch') - group.add_option('--pull', action='store_true', dest='pull', - help='Performs a pull before reapplying.') - parser.add_option_group(group) + group.add_option('--pull', + action='store_true', + dest='pull', + help='Performs a pull before reapplying.') + parser.add_option_group(group) - (options, args) = parser.parse_args(args) + (options, args) = parser.parse_args(args) + + if options.reapply: + if options.newbranch: + parser.error('--reapply works on the current branch only.') + if len(args) > 0: + parser.error('--reapply implies no additional arguments.') + + cl = Changelist() + if not cl.GetIssue(): + parser.error('Current branch must have an associated issue.') + + upstream = cl.GetUpstreamBranch() + if upstream is None: + parser.error('No upstream branch specified. Cannot reset branch.') + + RunGit(['reset', '--hard', upstream]) + if options.pull: + RunGit(['pull']) + + target_issue_arg = ParseIssueNumberArgument(cl.GetIssue()) + return cl.CMDPatchWithParsedIssue(target_issue_arg, options.nocommit, + options.force, False) + + if len(args) != 1 or not args[0]: + parser.error('Must specify issue number or URL.') + + target_issue_arg = ParseIssueNumberArgument(args[0]) + if not target_issue_arg.valid: + parser.error('Invalid issue ID or URL.') + + # We don't want uncommitted changes mixed up with the patch. + if git_common.is_dirty_git_tree('patch'): + return 1 - if options.reapply: if options.newbranch: - parser.error('--reapply works on the current branch only.') - if len(args) > 0: - parser.error('--reapply implies no additional arguments.') + if options.force: + RunGit(['branch', '-D', options.newbranch], + stderr=subprocess2.PIPE, + error_ok=True) + git_new_branch.create_new_branch(options.newbranch) - cl = Changelist() - if not cl.GetIssue(): - parser.error('Current branch must have an associated issue.') + cl = Changelist(codereview_host=target_issue_arg.hostname, + issue=target_issue_arg.issue) - upstream = cl.GetUpstreamBranch() - if upstream is None: - parser.error('No upstream branch specified. Cannot reset branch.') + if not args[0].isdigit(): + print('canonical issue/change URL: %s\n' % cl.GetIssueURL()) - RunGit(['reset', '--hard', upstream]) - if options.pull: - RunGit(['pull']) - - target_issue_arg = ParseIssueNumberArgument(cl.GetIssue()) return cl.CMDPatchWithParsedIssue(target_issue_arg, options.nocommit, - options.force, False) - - if len(args) != 1 or not args[0]: - parser.error('Must specify issue number or URL.') - - target_issue_arg = ParseIssueNumberArgument(args[0]) - if not target_issue_arg.valid: - parser.error('Invalid issue ID or URL.') - - # We don't want uncommitted changes mixed up with the patch. - if git_common.is_dirty_git_tree('patch'): - return 1 - - if options.newbranch: - if options.force: - RunGit(['branch', '-D', options.newbranch], - stderr=subprocess2.PIPE, error_ok=True) - git_new_branch.create_new_branch(options.newbranch) - - cl = Changelist( - codereview_host=target_issue_arg.hostname, issue=target_issue_arg.issue) - - if not args[0].isdigit(): - print('canonical issue/change URL: %s\n' % cl.GetIssueURL()) - - return cl.CMDPatchWithParsedIssue(target_issue_arg, options.nocommit, - options.force, options.newbranch) + options.force, options.newbranch) def GetTreeStatus(url=None): - """Fetches the tree status and returns either 'open', 'closed', + """Fetches the tree status and returns either 'open', 'closed', 'unknown' or 'unset'.""" - url = url or settings.GetTreeStatusUrl(error_ok=True) - if url: - status = str(urllib.request.urlopen(url).read().lower()) - if status.find('closed') != -1 or status == '0': - return 'closed' + url = url or settings.GetTreeStatusUrl(error_ok=True) + if url: + status = str(urllib.request.urlopen(url).read().lower()) + if status.find('closed') != -1 or status == '0': + return 'closed' - if status.find('open') != -1 or status == '1': - return 'open' + if status.find('open') != -1 or status == '1': + return 'open' - return 'unknown' - return 'unset' + return 'unknown' + return 'unset' def GetTreeStatusReason(): - """Fetches the tree status from a json url and returns the message + """Fetches the tree status from a json url and returns the message with the reason for the tree to be opened or closed.""" - url = settings.GetTreeStatusUrl() - json_url = urllib.parse.urljoin(url, '/current?format=json') - connection = urllib.request.urlopen(json_url) - status = json.loads(connection.read()) - connection.close() - return status['message'] + url = settings.GetTreeStatusUrl() + json_url = urllib.parse.urljoin(url, '/current?format=json') + connection = urllib.request.urlopen(json_url) + status = json.loads(connection.read()) + connection.close() + return status['message'] @metrics.collector.collect_metrics('git cl tree') def CMDtree(parser, args): - """Shows the status of the tree.""" - _, args = parser.parse_args(args) - status = GetTreeStatus() - if 'unset' == status: - print('You must configure your tree status URL by running "git cl config".') - return 2 + """Shows the status of the tree.""" + _, args = parser.parse_args(args) + status = GetTreeStatus() + if 'unset' == status: + print( + 'You must configure your tree status URL by running "git cl config".' + ) + return 2 - print('The tree is %s' % status) - print() - print(GetTreeStatusReason()) - if status != 'open': - return 1 - return 0 + print('The tree is %s' % status) + print() + print(GetTreeStatusReason()) + if status != 'open': + return 1 + return 0 @metrics.collector.collect_metrics('git cl try') def CMDtry(parser, args): - """Triggers tryjobs using either Buildbucket or CQ dry run.""" - group = optparse.OptionGroup(parser, 'Tryjob options') - group.add_option( - '-b', '--bot', action='append', - help=('IMPORTANT: specify ONE builder per --bot flag. Use it multiple ' - 'times to specify multiple builders. ex: ' - '"-b win_rel -b win_layout". See ' - 'the try server waterfall for the builders name and the tests ' - 'available.')) - group.add_option( - '-B', '--bucket', default='', - help=('Buildbucket bucket to send the try requests. Format: ' - '"luci.$LUCI_PROJECT.$LUCI_BUCKET". eg: "luci.chromium.try"')) - group.add_option( - '-r', '--revision', - help='Revision to use for the tryjob; default: the revision will ' - 'be determined by the try recipe that builder runs, which usually ' - 'defaults to HEAD of origin/master or origin/main') - group.add_option( - '-c', '--clobber', action='store_true', default=False, - help='Force a clobber before building; that is don\'t do an ' - 'incremental build') - group.add_option( - '--category', default='git_cl_try', help='Specify custom build category.') - group.add_option( - '--project', - help='Override which project to use. Projects are defined ' - 'in recipe to determine to which repository or directory to ' - 'apply the patch') - group.add_option( - '-p', '--property', dest='properties', action='append', default=[], - help='Specify generic properties in the form -p key1=value1 -p ' - 'key2=value2 etc. The value will be treated as ' - 'json if decodable, or as string otherwise. ' - 'NOTE: using this may make your tryjob not usable for CQ, ' - 'which will then schedule another tryjob with default properties') - group.add_option( - '--buildbucket-host', default='cr-buildbucket.appspot.com', - help='Host of buildbucket. The default host is %default.') - parser.add_option_group(group) - parser.add_option( - '-R', '--retry-failed', action='store_true', default=False, - help='Retry failed jobs from the latest set of tryjobs. ' - 'Not allowed with --bucket and --bot options.') - parser.add_option( - '-i', '--issue', type=int, - help='Operate on this issue instead of the current branch\'s implicit ' - 'issue.') - options, args = parser.parse_args(args) + """Triggers tryjobs using either Buildbucket or CQ dry run.""" + group = optparse.OptionGroup(parser, 'Tryjob options') + group.add_option( + '-b', + '--bot', + action='append', + help=('IMPORTANT: specify ONE builder per --bot flag. Use it multiple ' + 'times to specify multiple builders. ex: ' + '"-b win_rel -b win_layout". See ' + 'the try server waterfall for the builders name and the tests ' + 'available.')) + group.add_option( + '-B', + '--bucket', + default='', + help=('Buildbucket bucket to send the try requests. Format: ' + '"luci.$LUCI_PROJECT.$LUCI_BUCKET". eg: "luci.chromium.try"')) + group.add_option( + '-r', + '--revision', + help='Revision to use for the tryjob; default: the revision will ' + 'be determined by the try recipe that builder runs, which usually ' + 'defaults to HEAD of origin/master or origin/main') + group.add_option( + '-c', + '--clobber', + action='store_true', + default=False, + help='Force a clobber before building; that is don\'t do an ' + 'incremental build') + group.add_option('--category', + default='git_cl_try', + help='Specify custom build category.') + group.add_option( + '--project', + help='Override which project to use. Projects are defined ' + 'in recipe to determine to which repository or directory to ' + 'apply the patch') + group.add_option( + '-p', + '--property', + dest='properties', + action='append', + default=[], + help='Specify generic properties in the form -p key1=value1 -p ' + 'key2=value2 etc. The value will be treated as ' + 'json if decodable, or as string otherwise. ' + 'NOTE: using this may make your tryjob not usable for CQ, ' + 'which will then schedule another tryjob with default properties') + group.add_option('--buildbucket-host', + default='cr-buildbucket.appspot.com', + help='Host of buildbucket. The default host is %default.') + parser.add_option_group(group) + parser.add_option('-R', + '--retry-failed', + action='store_true', + default=False, + help='Retry failed jobs from the latest set of tryjobs. ' + 'Not allowed with --bucket and --bot options.') + parser.add_option( + '-i', + '--issue', + type=int, + help='Operate on this issue instead of the current branch\'s implicit ' + 'issue.') + options, args = parser.parse_args(args) - # Make sure that all properties are prop=value pairs. - bad_params = [x for x in options.properties if '=' not in x] - if bad_params: - parser.error('Got properties with missing "=": %s' % bad_params) + # Make sure that all properties are prop=value pairs. + bad_params = [x for x in options.properties if '=' not in x] + if bad_params: + parser.error('Got properties with missing "=": %s' % bad_params) - if args: - parser.error('Unknown arguments: %s' % args) + if args: + parser.error('Unknown arguments: %s' % args) - cl = Changelist(issue=options.issue) - if not cl.GetIssue(): - parser.error('Need to upload first.') + cl = Changelist(issue=options.issue) + if not cl.GetIssue(): + parser.error('Need to upload first.') - # HACK: warm up Gerrit change detail cache to save on RPCs. - cl._GetChangeDetail(['DETAILED_ACCOUNTS', 'ALL_REVISIONS']) + # HACK: warm up Gerrit change detail cache to save on RPCs. + cl._GetChangeDetail(['DETAILED_ACCOUNTS', 'ALL_REVISIONS']) - error_message = cl.CannotTriggerTryJobReason() - if error_message: - parser.error('Can\'t trigger tryjobs: %s' % error_message) + error_message = cl.CannotTriggerTryJobReason() + if error_message: + parser.error('Can\'t trigger tryjobs: %s' % error_message) - if options.bot: - if options.retry_failed: - parser.error('--bot is not compatible with --retry-failed.') - if not options.bucket: - parser.error('A bucket (e.g. "chromium/try") is required.') + if options.bot: + if options.retry_failed: + parser.error('--bot is not compatible with --retry-failed.') + if not options.bucket: + parser.error('A bucket (e.g. "chromium/try") is required.') - triggered = [b for b in options.bot if 'triggered' in b] - if triggered: - parser.error( - 'Cannot schedule builds on triggered bots: %s.\n' - 'This type of bot requires an initial job from a parent (usually a ' - 'builder). Schedule a job on the parent instead.\n' % triggered) + triggered = [b for b in options.bot if 'triggered' in b] + if triggered: + parser.error( + 'Cannot schedule builds on triggered bots: %s.\n' + 'This type of bot requires an initial job from a parent (usually a ' + 'builder). Schedule a job on the parent instead.\n' % triggered) - if options.bucket.startswith('.master'): - parser.error('Buildbot masters are not supported.') + if options.bucket.startswith('.master'): + parser.error('Buildbot masters are not supported.') - project, bucket = _parse_bucket(options.bucket) - if project is None or bucket is None: - parser.error('Invalid bucket: %s.' % options.bucket) - jobs = sorted((project, bucket, bot) for bot in options.bot) - elif options.retry_failed: - print('Searching for failed tryjobs...') - builds, patchset = _fetch_latest_builds(cl, DEFAULT_BUILDBUCKET_HOST) - if options.verbose: - print('Got %d builds in patchset #%d' % (len(builds), patchset)) - jobs = _filter_failed_for_retry(builds) - if not jobs: - print('There are no failed jobs in the latest set of jobs ' - '(patchset #%d), doing nothing.' % patchset) - return 0 - num_builders = len(jobs) - if num_builders > 10: - confirm_or_exit('There are %d builders with failed builds.' - % num_builders, action='continue') - else: - if options.verbose: - print('git cl try with no bots now defaults to CQ dry run.') - print('Scheduling CQ dry run on: %s' % cl.GetIssueURL()) - return cl.SetCQState(_CQState.DRY_RUN) + project, bucket = _parse_bucket(options.bucket) + if project is None or bucket is None: + parser.error('Invalid bucket: %s.' % options.bucket) + jobs = sorted((project, bucket, bot) for bot in options.bot) + elif options.retry_failed: + print('Searching for failed tryjobs...') + builds, patchset = _fetch_latest_builds(cl, DEFAULT_BUILDBUCKET_HOST) + if options.verbose: + print('Got %d builds in patchset #%d' % (len(builds), patchset)) + jobs = _filter_failed_for_retry(builds) + if not jobs: + print('There are no failed jobs in the latest set of jobs ' + '(patchset #%d), doing nothing.' % patchset) + return 0 + num_builders = len(jobs) + if num_builders > 10: + confirm_or_exit('There are %d builders with failed builds.' % + num_builders, + action='continue') + else: + if options.verbose: + print('git cl try with no bots now defaults to CQ dry run.') + print('Scheduling CQ dry run on: %s' % cl.GetIssueURL()) + return cl.SetCQState(_CQState.DRY_RUN) - patchset = cl.GetMostRecentPatchset() - try: - _trigger_tryjobs(cl, jobs, options, patchset) - except BuildbucketResponseException as ex: - print('ERROR: %s' % ex) - return 1 - return 0 + patchset = cl.GetMostRecentPatchset() + try: + _trigger_tryjobs(cl, jobs, options, patchset) + except BuildbucketResponseException as ex: + print('ERROR: %s' % ex) + return 1 + return 0 @metrics.collector.collect_metrics('git cl try-results') def CMDtry_results(parser, args): - """Prints info about results for tryjobs associated with the current CL.""" - group = optparse.OptionGroup(parser, 'Tryjob results options') - group.add_option( - '-p', '--patchset', type=int, help='patchset number if not current.') - group.add_option( - '--print-master', action='store_true', help='print master name as well.') - group.add_option( - '--color', action='store_true', default=setup_color.IS_TTY, - help='force color output, useful when piping output.') - group.add_option( - '--buildbucket-host', default='cr-buildbucket.appspot.com', - help='Host of buildbucket. The default host is %default.') - group.add_option( - '--json', help=('Path of JSON output file to write tryjob results to,' - 'or "-" for stdout.')) - parser.add_option_group(group) - parser.add_option( - '-i', '--issue', type=int, - help='Operate on this issue instead of the current branch\'s implicit ' - 'issue.') - options, args = parser.parse_args(args) - if args: - parser.error('Unrecognized args: %s' % ' '.join(args)) + """Prints info about results for tryjobs associated with the current CL.""" + group = optparse.OptionGroup(parser, 'Tryjob results options') + group.add_option('-p', + '--patchset', + type=int, + help='patchset number if not current.') + group.add_option('--print-master', + action='store_true', + help='print master name as well.') + group.add_option('--color', + action='store_true', + default=setup_color.IS_TTY, + help='force color output, useful when piping output.') + group.add_option('--buildbucket-host', + default='cr-buildbucket.appspot.com', + help='Host of buildbucket. The default host is %default.') + group.add_option( + '--json', + help=('Path of JSON output file to write tryjob results to,' + 'or "-" for stdout.')) + parser.add_option_group(group) + parser.add_option( + '-i', + '--issue', + type=int, + help='Operate on this issue instead of the current branch\'s implicit ' + 'issue.') + options, args = parser.parse_args(args) + if args: + parser.error('Unrecognized args: %s' % ' '.join(args)) - cl = Changelist(issue=options.issue) - if not cl.GetIssue(): - parser.error('Need to upload first.') + cl = Changelist(issue=options.issue) + if not cl.GetIssue(): + parser.error('Need to upload first.') - patchset = options.patchset - if not patchset: - patchset = cl.GetMostRecentDryRunPatchset() + patchset = options.patchset if not patchset: - parser.error('Code review host doesn\'t know about issue %s. ' - 'No access to issue or wrong issue number?\n' - 'Either upload first, or pass --patchset explicitly.' % - cl.GetIssue()) + patchset = cl.GetMostRecentDryRunPatchset() + if not patchset: + parser.error('Code review host doesn\'t know about issue %s. ' + 'No access to issue or wrong issue number?\n' + 'Either upload first, or pass --patchset explicitly.' % + cl.GetIssue()) - try: - jobs = _fetch_tryjobs(cl, DEFAULT_BUILDBUCKET_HOST, patchset) - except BuildbucketResponseException as ex: - print('Buildbucket error: %s' % ex) - return 1 - if options.json: - write_json(options.json, jobs) - else: - _print_tryjobs(options, jobs) - return 0 + try: + jobs = _fetch_tryjobs(cl, DEFAULT_BUILDBUCKET_HOST, patchset) + except BuildbucketResponseException as ex: + print('Buildbucket error: %s' % ex) + return 1 + if options.json: + write_json(options.json, jobs) + else: + _print_tryjobs(options, jobs) + return 0 @subcommand.usage('[new upstream branch]') @metrics.collector.collect_metrics('git cl upstream') def CMDupstream(parser, args): - """Prints or sets the name of the upstream branch, if any.""" - _, args = parser.parse_args(args) - if len(args) > 1: - parser.error('Unrecognized args: %s' % ' '.join(args)) + """Prints or sets the name of the upstream branch, if any.""" + _, args = parser.parse_args(args) + if len(args) > 1: + parser.error('Unrecognized args: %s' % ' '.join(args)) - cl = Changelist() - if args: - # One arg means set upstream branch. - branch = cl.GetBranch() - RunGit(['branch', '--set-upstream-to', args[0], branch]) cl = Changelist() - print('Upstream branch set to %s' % (cl.GetUpstreamBranch(),)) + if args: + # One arg means set upstream branch. + branch = cl.GetBranch() + RunGit(['branch', '--set-upstream-to', args[0], branch]) + cl = Changelist() + print('Upstream branch set to %s' % (cl.GetUpstreamBranch(), )) - # Clear configured merge-base, if there is one. - git_common.remove_merge_base(branch) - else: - print(cl.GetUpstreamBranch()) - return 0 + # Clear configured merge-base, if there is one. + git_common.remove_merge_base(branch) + else: + print(cl.GetUpstreamBranch()) + return 0 @metrics.collector.collect_metrics('git cl web') def CMDweb(parser, args): - """Opens the current CL in the web browser.""" - parser.add_option('-p', - '--print-only', - action='store_true', - dest='print_only', - help='Only print the Gerrit URL, don\'t open it in the ' - 'browser.') - (options, args) = parser.parse_args(args) - if args: - parser.error('Unrecognized args: %s' % ' '.join(args)) + """Opens the current CL in the web browser.""" + parser.add_option('-p', + '--print-only', + action='store_true', + dest='print_only', + help='Only print the Gerrit URL, don\'t open it in the ' + 'browser.') + (options, args) = parser.parse_args(args) + if args: + parser.error('Unrecognized args: %s' % ' '.join(args)) - issue_url = Changelist().GetIssueURL() - if not issue_url: - print('ERROR No issue to open', file=sys.stderr) - return 1 + issue_url = Changelist().GetIssueURL() + if not issue_url: + print('ERROR No issue to open', file=sys.stderr) + return 1 - if options.print_only: - print(issue_url) + if options.print_only: + print(issue_url) + return 0 + + # Redirect I/O before invoking browser to hide its output. For example, this + # allows us to hide the "Created new window in existing browser session." + # message from Chrome. Based on https://stackoverflow.com/a/2323563. + saved_stdout = os.dup(1) + saved_stderr = os.dup(2) + os.close(1) + os.close(2) + os.open(os.devnull, os.O_RDWR) + try: + webbrowser.open(issue_url) + finally: + os.dup2(saved_stdout, 1) + os.dup2(saved_stderr, 2) return 0 - # Redirect I/O before invoking browser to hide its output. For example, this - # allows us to hide the "Created new window in existing browser session." - # message from Chrome. Based on https://stackoverflow.com/a/2323563. - saved_stdout = os.dup(1) - saved_stderr = os.dup(2) - os.close(1) - os.close(2) - os.open(os.devnull, os.O_RDWR) - try: - webbrowser.open(issue_url) - finally: - os.dup2(saved_stdout, 1) - os.dup2(saved_stderr, 2) - return 0 - @metrics.collector.collect_metrics('git cl set-commit') def CMDset_commit(parser, args): - """Sets the commit bit to trigger the CQ.""" - parser.add_option('-d', '--dry-run', action='store_true', - help='trigger in dry run mode') - parser.add_option('-c', '--clear', action='store_true', - help='stop CQ run, if any') - parser.add_option( - '-i', '--issue', type=int, - help='Operate on this issue instead of the current branch\'s implicit ' - 'issue.') - options, args = parser.parse_args(args) - if args: - parser.error('Unrecognized args: %s' % ' '.join(args)) - if [options.dry_run, options.clear].count(True) > 1: - parser.error('Only one of --dry-run, and --clear are allowed.') + """Sets the commit bit to trigger the CQ.""" + parser.add_option('-d', + '--dry-run', + action='store_true', + help='trigger in dry run mode') + parser.add_option('-c', + '--clear', + action='store_true', + help='stop CQ run, if any') + parser.add_option( + '-i', + '--issue', + type=int, + help='Operate on this issue instead of the current branch\'s implicit ' + 'issue.') + options, args = parser.parse_args(args) + if args: + parser.error('Unrecognized args: %s' % ' '.join(args)) + if [options.dry_run, options.clear].count(True) > 1: + parser.error('Only one of --dry-run, and --clear are allowed.') - cl = Changelist(issue=options.issue) - if not cl.GetIssue(): - parser.error('Must upload the issue first.') + cl = Changelist(issue=options.issue) + if not cl.GetIssue(): + parser.error('Must upload the issue first.') - if options.clear: - state = _CQState.NONE - elif options.dry_run: - state = _CQState.DRY_RUN - else: - state = _CQState.COMMIT - cl.SetCQState(state) - return 0 + if options.clear: + state = _CQState.NONE + elif options.dry_run: + state = _CQState.DRY_RUN + else: + state = _CQState.COMMIT + cl.SetCQState(state) + return 0 @metrics.collector.collect_metrics('git cl set-close') def CMDset_close(parser, args): - """Closes the issue.""" - parser.add_option( - '-i', '--issue', type=int, - help='Operate on this issue instead of the current branch\'s implicit ' - 'issue.') - options, args = parser.parse_args(args) - if args: - parser.error('Unrecognized args: %s' % ' '.join(args)) - cl = Changelist(issue=options.issue) - # Ensure there actually is an issue to close. - if not cl.GetIssue(): - DieWithError('ERROR: No issue to close.') - cl.CloseIssue() - return 0 + """Closes the issue.""" + parser.add_option( + '-i', + '--issue', + type=int, + help='Operate on this issue instead of the current branch\'s implicit ' + 'issue.') + options, args = parser.parse_args(args) + if args: + parser.error('Unrecognized args: %s' % ' '.join(args)) + cl = Changelist(issue=options.issue) + # Ensure there actually is an issue to close. + if not cl.GetIssue(): + DieWithError('ERROR: No issue to close.') + cl.CloseIssue() + return 0 @metrics.collector.collect_metrics('git cl diff') def CMDdiff(parser, args): - """Shows differences between local tree and last upload.""" - parser.add_option( - '--stat', - action='store_true', - dest='stat', - help='Generate a diffstat') - options, args = parser.parse_args(args) - if args: - parser.error('Unrecognized args: %s' % ' '.join(args)) + """Shows differences between local tree and last upload.""" + parser.add_option('--stat', + action='store_true', + dest='stat', + help='Generate a diffstat') + options, args = parser.parse_args(args) + if args: + parser.error('Unrecognized args: %s' % ' '.join(args)) - cl = Changelist() - issue = cl.GetIssue() - branch = cl.GetBranch() - if not issue: - DieWithError('No issue found for current branch (%s)' % branch) + cl = Changelist() + issue = cl.GetIssue() + branch = cl.GetBranch() + if not issue: + DieWithError('No issue found for current branch (%s)' % branch) - base = cl._GitGetBranchConfigValue(LAST_UPLOAD_HASH_CONFIG_KEY) - if not base: - base = cl._GitGetBranchConfigValue(GERRIT_SQUASH_HASH_CONFIG_KEY) - if not base: - detail = cl._GetChangeDetail(['CURRENT_REVISION', 'CURRENT_COMMIT']) - revision_info = detail['revisions'][detail['current_revision']] - fetch_info = revision_info['fetch']['http'] - RunGit(['fetch', fetch_info['url'], fetch_info['ref']]) - base = 'FETCH_HEAD' + base = cl._GitGetBranchConfigValue(LAST_UPLOAD_HASH_CONFIG_KEY) + if not base: + base = cl._GitGetBranchConfigValue(GERRIT_SQUASH_HASH_CONFIG_KEY) + if not base: + detail = cl._GetChangeDetail(['CURRENT_REVISION', 'CURRENT_COMMIT']) + revision_info = detail['revisions'][detail['current_revision']] + fetch_info = revision_info['fetch']['http'] + RunGit(['fetch', fetch_info['url'], fetch_info['ref']]) + base = 'FETCH_HEAD' - cmd = ['git', 'diff'] - if options.stat: - cmd.append('--stat') - cmd.append(base) - subprocess2.check_call(cmd) + cmd = ['git', 'diff'] + if options.stat: + cmd.append('--stat') + cmd.append(base) + subprocess2.check_call(cmd) - return 0 + return 0 @metrics.collector.collect_metrics('git cl owners') def CMDowners(parser, args): - """Finds potential owners for reviewing.""" - parser.add_option( - '--ignore-current', - action='store_true', - help='Ignore the CL\'s current reviewers and start from scratch.') - parser.add_option( - '--ignore-self', - action='store_true', - help='Do not consider CL\'s author as an owners.') - parser.add_option( - '--no-color', - action='store_true', - help='Use this option to disable color output') - parser.add_option( - '--batch', - action='store_true', - help='Do not run interactively, just suggest some') - # TODO: Consider moving this to another command, since other - # git-cl owners commands deal with owners for a given CL. - parser.add_option( - '--show-all', - action='store_true', - help='Show all owners for a particular file') - options, args = parser.parse_args(args) + """Finds potential owners for reviewing.""" + parser.add_option( + '--ignore-current', + action='store_true', + help='Ignore the CL\'s current reviewers and start from scratch.') + parser.add_option('--ignore-self', + action='store_true', + help='Do not consider CL\'s author as an owners.') + parser.add_option('--no-color', + action='store_true', + help='Use this option to disable color output') + parser.add_option('--batch', + action='store_true', + help='Do not run interactively, just suggest some') + # TODO: Consider moving this to another command, since other + # git-cl owners commands deal with owners for a given CL. + parser.add_option('--show-all', + action='store_true', + help='Show all owners for a particular file') + options, args = parser.parse_args(args) - cl = Changelist() - author = cl.GetAuthor() + cl = Changelist() + author = cl.GetAuthor() - if options.show_all: - if len(args) == 0: - print('No files specified for --show-all. Nothing to do.') - return 0 - owners_by_path = cl.owners_client.BatchListOwners(args) - for path in args: - print('Owners for %s:' % path) - print('\n'.join( - ' - %s' % owner - for owner in owners_by_path.get(path, ['No owners found']))) - return 0 + if options.show_all: + if len(args) == 0: + print('No files specified for --show-all. Nothing to do.') + return 0 + owners_by_path = cl.owners_client.BatchListOwners(args) + for path in args: + print('Owners for %s:' % path) + print('\n'.join( + ' - %s' % owner + for owner in owners_by_path.get(path, ['No owners found']))) + return 0 - if args: - if len(args) > 1: - parser.error('Unknown args.') - base_branch = args[0] - else: - # Default to diffing against the common ancestor of the upstream branch. - base_branch = cl.GetCommonAncestorWithUpstream() + if args: + if len(args) > 1: + parser.error('Unknown args.') + base_branch = args[0] + else: + # Default to diffing against the common ancestor of the upstream branch. + base_branch = cl.GetCommonAncestorWithUpstream() - affected_files = cl.GetAffectedFiles(base_branch) + affected_files = cl.GetAffectedFiles(base_branch) - if options.batch: - owners = cl.owners_client.SuggestOwners(affected_files, exclude=[author]) - print('\n'.join(owners)) - return 0 + if options.batch: + owners = cl.owners_client.SuggestOwners(affected_files, + exclude=[author]) + print('\n'.join(owners)) + return 0 - return owners_finder.OwnersFinder( - affected_files, - author, - [] if options.ignore_current else cl.GetReviewers(), - cl.owners_client, - disable_color=options.no_color, - ignore_author=options.ignore_self).run() + return owners_finder.OwnersFinder( + affected_files, + author, [] if options.ignore_current else cl.GetReviewers(), + cl.owners_client, + disable_color=options.no_color, + ignore_author=options.ignore_self).run() def BuildGitDiffCmd(diff_type, upstream_commit, args, allow_prefix=False): - """Generates a diff command.""" - # Generate diff for the current branch's changes. - diff_cmd = ['-c', 'core.quotePath=false', 'diff', '--no-ext-diff'] + """Generates a diff command.""" + # Generate diff for the current branch's changes. + diff_cmd = ['-c', 'core.quotePath=false', 'diff', '--no-ext-diff'] - if allow_prefix: - # explicitly setting --src-prefix and --dst-prefix is necessary in the - # case that diff.noprefix is set in the user's git config. - diff_cmd += ['--src-prefix=a/', '--dst-prefix=b/'] - else: - diff_cmd += ['--no-prefix'] + if allow_prefix: + # explicitly setting --src-prefix and --dst-prefix is necessary in the + # case that diff.noprefix is set in the user's git config. + diff_cmd += ['--src-prefix=a/', '--dst-prefix=b/'] + else: + diff_cmd += ['--no-prefix'] - diff_cmd += [diff_type, upstream_commit, '--'] + diff_cmd += [diff_type, upstream_commit, '--'] - if args: - for arg in args: - if os.path.isdir(arg) or os.path.isfile(arg): - diff_cmd.append(arg) - else: - DieWithError('Argument "%s" is not a file or a directory' % arg) + if args: + for arg in args: + if os.path.isdir(arg) or os.path.isfile(arg): + diff_cmd.append(arg) + else: + DieWithError('Argument "%s" is not a file or a directory' % arg) - return diff_cmd + return diff_cmd def _RunClangFormatDiff(opts, clang_diff_files, top_dir, upstream_commit): - """Runs clang-format-diff and sets a return value if necessary.""" + """Runs clang-format-diff and sets a return value if necessary.""" - if not clang_diff_files: - return 0 + if not clang_diff_files: + return 0 - # Set to 2 to signal to CheckPatchFormatted() that this patch isn't - # formatted. This is used to block during the presubmit. - return_value = 0 + # Set to 2 to signal to CheckPatchFormatted() that this patch isn't + # formatted. This is used to block during the presubmit. + return_value = 0 - # Locate the clang-format binary in the checkout - try: - clang_format_tool = clang_format.FindClangFormatToolInChromiumTree() - except clang_format.NotFoundError as e: - DieWithError(e) - - if opts.full or settings.GetFormatFullByDefault(): - cmd = [clang_format_tool] - if not opts.dry_run and not opts.diff: - cmd.append('-i') - if opts.dry_run: - for diff_file in clang_diff_files: - with open(diff_file, 'r') as myfile: - code = myfile.read().replace('\r\n', '\n') - stdout = RunCommand(cmd + [diff_file], cwd=top_dir) - stdout = stdout.replace('\r\n', '\n') - if opts.diff: - sys.stdout.write(stdout) - if code != stdout: - return_value = 2 - else: - stdout = RunCommand(cmd + clang_diff_files, cwd=top_dir) - if opts.diff: - sys.stdout.write(stdout) - else: + # Locate the clang-format binary in the checkout try: - script = clang_format.FindClangFormatScriptInChromiumTree( - 'clang-format-diff.py') + clang_format_tool = clang_format.FindClangFormatToolInChromiumTree() except clang_format.NotFoundError as e: - DieWithError(e) + DieWithError(e) - cmd = ['vpython3', script, '-p0'] - if not opts.dry_run and not opts.diff: - cmd.append('-i') + if opts.full or settings.GetFormatFullByDefault(): + cmd = [clang_format_tool] + if not opts.dry_run and not opts.diff: + cmd.append('-i') + if opts.dry_run: + for diff_file in clang_diff_files: + with open(diff_file, 'r') as myfile: + code = myfile.read().replace('\r\n', '\n') + stdout = RunCommand(cmd + [diff_file], cwd=top_dir) + stdout = stdout.replace('\r\n', '\n') + if opts.diff: + sys.stdout.write(stdout) + if code != stdout: + return_value = 2 + else: + stdout = RunCommand(cmd + clang_diff_files, cwd=top_dir) + if opts.diff: + sys.stdout.write(stdout) + else: + try: + script = clang_format.FindClangFormatScriptInChromiumTree( + 'clang-format-diff.py') + except clang_format.NotFoundError as e: + DieWithError(e) - diff_cmd = BuildGitDiffCmd('-U0', upstream_commit, clang_diff_files) - diff_output = RunGit(diff_cmd).encode('utf-8') + cmd = ['vpython3', script, '-p0'] + if not opts.dry_run and not opts.diff: + cmd.append('-i') - env = os.environ.copy() - env['PATH'] = ( - str(os.path.dirname(clang_format_tool)) + os.pathsep + env['PATH']) - stdout = RunCommand( - cmd, stdin=diff_output, cwd=top_dir, env=env, - shell=sys.platform.startswith('win32')) - if opts.diff: - sys.stdout.write(stdout) - if opts.dry_run and len(stdout) > 0: - return_value = 2 + diff_cmd = BuildGitDiffCmd('-U0', upstream_commit, clang_diff_files) + diff_output = RunGit(diff_cmd).encode('utf-8') - return return_value + env = os.environ.copy() + env['PATH'] = (str(os.path.dirname(clang_format_tool)) + os.pathsep + + env['PATH']) + stdout = RunCommand(cmd, + stdin=diff_output, + cwd=top_dir, + env=env, + shell=sys.platform.startswith('win32')) + if opts.diff: + sys.stdout.write(stdout) + if opts.dry_run and len(stdout) > 0: + return_value = 2 + + return return_value def _RunRustFmt(opts, rust_diff_files, top_dir, upstream_commit): - """Runs rustfmt. Just like _RunClangFormatDiff returns 2 to indicate that + """Runs rustfmt. Just like _RunClangFormatDiff returns 2 to indicate that presubmit checks have failed (and returns 0 otherwise).""" - if not rust_diff_files: + if not rust_diff_files: + return 0 + + # Locate the rustfmt binary. + try: + rustfmt_tool = rustfmt.FindRustfmtToolInChromiumTree() + except rustfmt.NotFoundError as e: + DieWithError(e) + + # TODO(crbug.com/1440869): Support formatting only the changed lines + # if `opts.full or settings.GetFormatFullByDefault()` is False. + cmd = [rustfmt_tool] + if opts.dry_run: + cmd.append('--check') + cmd += rust_diff_files + rustfmt_exitcode = subprocess2.call(cmd) + + if opts.presubmit and rustfmt_exitcode != 0: + return 2 + return 0 - # Locate the rustfmt binary. - try: - rustfmt_tool = rustfmt.FindRustfmtToolInChromiumTree() - except rustfmt.NotFoundError as e: - DieWithError(e) - - # TODO(crbug.com/1440869): Support formatting only the changed lines - # if `opts.full or settings.GetFormatFullByDefault()` is False. - cmd = [rustfmt_tool] - if opts.dry_run: - cmd.append('--check') - cmd += rust_diff_files - rustfmt_exitcode = subprocess2.call(cmd) - - if opts.presubmit and rustfmt_exitcode != 0: - return 2 - - return 0 - def _RunSwiftFormat(opts, swift_diff_files, top_dir, upstream_commit): - """Runs swift-format. Just like _RunClangFormatDiff returns 2 to indicate + """Runs swift-format. Just like _RunClangFormatDiff returns 2 to indicate that presubmit checks have failed (and returns 0 otherwise).""" - if not swift_diff_files: + if not swift_diff_files: + return 0 + + # Locate the swift-format binary. + try: + swift_format_tool = swift_format.FindSwiftFormatToolInChromiumTree() + except swift_format.NotFoundError as e: + DieWithError(e) + + cmd = [swift_format_tool] + if opts.dry_run: + cmd += ['lint', '-s'] + else: + cmd += ['format', '-i'] + cmd += swift_diff_files + swift_format_exitcode = subprocess2.call(cmd) + + if opts.presubmit and swift_format_exitcode != 0: + return 2 + return 0 - # Locate the swift-format binary. - try: - swift_format_tool = swift_format.FindSwiftFormatToolInChromiumTree() - except swift_format.NotFoundError as e: - DieWithError(e) - - cmd = [swift_format_tool] - if opts.dry_run: - cmd += ['lint', '-s'] - else: - cmd += ['format', '-i'] - cmd += swift_diff_files - swift_format_exitcode = subprocess2.call(cmd) - - if opts.presubmit and swift_format_exitcode != 0: - return 2 - - return 0 - def MatchingFileType(file_name, extensions): - """Returns True if the file name ends with one of the given extensions.""" - return bool([ext for ext in extensions if file_name.lower().endswith(ext)]) + """Returns True if the file name ends with one of the given extensions.""" + return bool([ext for ext in extensions if file_name.lower().endswith(ext)]) @subcommand.usage('[files or directories to diff]') @metrics.collector.collect_metrics('git cl format') def CMDformat(parser, args): - """Runs auto-formatting tools (clang-format etc.) on the diff.""" - CLANG_EXTS = ['.cc', '.cpp', '.h', '.m', '.mm', '.proto', '.java'] - GN_EXTS = ['.gn', '.gni', '.typemap'] - RUST_EXTS = ['.rs'] - SWIFT_EXTS = ['.swift'] - parser.add_option('--full', action='store_true', - help='Reformat the full content of all touched files') - parser.add_option('--upstream', help='Branch to check against') - parser.add_option('--dry-run', action='store_true', - help='Don\'t modify any file on disk.') - parser.add_option( - '--no-clang-format', - dest='clang_format', - action='store_false', - default=True, - help='Disables formatting of various file types using clang-format.') - parser.add_option( - '--python', - action='store_true', - default=None, - help='Enables python formatting on all python files.') - parser.add_option( - '--no-python', - action='store_true', - default=False, - help='Disables python formatting on all python files. ' - 'If neither --python or --no-python are set, python files that have a ' - '.style.yapf file in an ancestor directory will be formatted. ' - 'It is an error to set both.') - parser.add_option( - '--js', - action='store_true', - help='Format javascript code with clang-format. ' - 'Has no effect if --no-clang-format is set.') - parser.add_option('--diff', action='store_true', - help='Print diff to stdout rather than modifying files.') - parser.add_option('--presubmit', action='store_true', - help='Used when running the script from a presubmit.') + """Runs auto-formatting tools (clang-format etc.) on the diff.""" + CLANG_EXTS = ['.cc', '.cpp', '.h', '.m', '.mm', '.proto', '.java'] + GN_EXTS = ['.gn', '.gni', '.typemap'] + RUST_EXTS = ['.rs'] + SWIFT_EXTS = ['.swift'] + parser.add_option('--full', + action='store_true', + help='Reformat the full content of all touched files') + parser.add_option('--upstream', help='Branch to check against') + parser.add_option('--dry-run', + action='store_true', + help='Don\'t modify any file on disk.') + parser.add_option( + '--no-clang-format', + dest='clang_format', + action='store_false', + default=True, + help='Disables formatting of various file types using clang-format.') + parser.add_option('--python', + action='store_true', + default=None, + help='Enables python formatting on all python files.') + parser.add_option( + '--no-python', + action='store_true', + default=False, + help='Disables python formatting on all python files. ' + 'If neither --python or --no-python are set, python files that have a ' + '.style.yapf file in an ancestor directory will be formatted. ' + 'It is an error to set both.') + parser.add_option('--js', + action='store_true', + help='Format javascript code with clang-format. ' + 'Has no effect if --no-clang-format is set.') + parser.add_option('--diff', + action='store_true', + help='Print diff to stdout rather than modifying files.') + parser.add_option('--presubmit', + action='store_true', + help='Used when running the script from a presubmit.') - parser.add_option('--rust-fmt', - dest='use_rust_fmt', - action='store_true', - default=rustfmt.IsRustfmtSupported(), - help='Enables formatting of Rust file types using rustfmt.') - parser.add_option( - '--no-rust-fmt', - dest='use_rust_fmt', - action='store_false', - help='Disables formatting of Rust file types using rustfmt.') + parser.add_option( + '--rust-fmt', + dest='use_rust_fmt', + action='store_true', + default=rustfmt.IsRustfmtSupported(), + help='Enables formatting of Rust file types using rustfmt.') + parser.add_option( + '--no-rust-fmt', + dest='use_rust_fmt', + action='store_false', + help='Disables formatting of Rust file types using rustfmt.') - parser.add_option( - '--swift-format', - dest='use_swift_format', - action='store_true', - default=swift_format.IsSwiftFormatSupported(), - help='Enables formatting of Swift file types using swift-format ' - '(macOS host only).') - parser.add_option( - '--no-swift-format', - dest='use_swift_format', - action='store_false', - help='Disables formatting of Swift file types using swift-format.') + parser.add_option( + '--swift-format', + dest='use_swift_format', + action='store_true', + default=swift_format.IsSwiftFormatSupported(), + help='Enables formatting of Swift file types using swift-format ' + '(macOS host only).') + parser.add_option( + '--no-swift-format', + dest='use_swift_format', + action='store_false', + help='Disables formatting of Swift file types using swift-format.') - opts, args = parser.parse_args(args) + opts, args = parser.parse_args(args) - if opts.python is not None and opts.no_python: - raise parser.error('Cannot set both --python and --no-python') - if opts.no_python: - opts.python = False + if opts.python is not None and opts.no_python: + raise parser.error('Cannot set both --python and --no-python') + if opts.no_python: + opts.python = False - # Normalize any remaining args against the current path, so paths relative to - # the current directory are still resolved as expected. - args = [os.path.join(os.getcwd(), arg) for arg in args] + # Normalize any remaining args against the current path, so paths relative + # to the current directory are still resolved as expected. + args = [os.path.join(os.getcwd(), arg) for arg in args] - # git diff generates paths against the root of the repository. Change - # to that directory so clang-format can find files even within subdirs. - rel_base_path = settings.GetRelativeRoot() - if rel_base_path: - os.chdir(rel_base_path) + # git diff generates paths against the root of the repository. Change + # to that directory so clang-format can find files even within subdirs. + rel_base_path = settings.GetRelativeRoot() + if rel_base_path: + os.chdir(rel_base_path) - # Grab the merge-base commit, i.e. the upstream commit of the current - # branch when it was created or the last time it was rebased. This is - # to cover the case where the user may have called "git fetch origin", - # moving the origin branch to a newer commit, but hasn't rebased yet. - upstream_commit = None - upstream_branch = opts.upstream - if not upstream_branch: - cl = Changelist() - upstream_branch = cl.GetUpstreamBranch() - if upstream_branch: - upstream_commit = RunGit(['merge-base', 'HEAD', upstream_branch]) - upstream_commit = upstream_commit.strip() + # Grab the merge-base commit, i.e. the upstream commit of the current + # branch when it was created or the last time it was rebased. This is + # to cover the case where the user may have called "git fetch origin", + # moving the origin branch to a newer commit, but hasn't rebased yet. + upstream_commit = None + upstream_branch = opts.upstream + if not upstream_branch: + cl = Changelist() + upstream_branch = cl.GetUpstreamBranch() + if upstream_branch: + upstream_commit = RunGit(['merge-base', 'HEAD', upstream_branch]) + upstream_commit = upstream_commit.strip() - if not upstream_commit: - DieWithError('Could not find base commit for this branch. ' - 'Are you in detached state?') + if not upstream_commit: + DieWithError('Could not find base commit for this branch. ' + 'Are you in detached state?') - changed_files_cmd = BuildGitDiffCmd('--name-only', upstream_commit, args) - diff_output = RunGit(changed_files_cmd) - diff_files = diff_output.splitlines() - # Filter out files deleted by this CL - diff_files = [x for x in diff_files if os.path.isfile(x)] + changed_files_cmd = BuildGitDiffCmd('--name-only', upstream_commit, args) + diff_output = RunGit(changed_files_cmd) + diff_files = diff_output.splitlines() + # Filter out files deleted by this CL + diff_files = [x for x in diff_files if os.path.isfile(x)] - if opts.js: - CLANG_EXTS.extend(['.js', '.ts']) + if opts.js: + CLANG_EXTS.extend(['.js', '.ts']) - clang_diff_files = [] - if opts.clang_format: - clang_diff_files = [ - x for x in diff_files if MatchingFileType(x, CLANG_EXTS) + clang_diff_files = [] + if opts.clang_format: + clang_diff_files = [ + x for x in diff_files if MatchingFileType(x, CLANG_EXTS) + ] + python_diff_files = [x for x in diff_files if MatchingFileType(x, ['.py'])] + rust_diff_files = [x for x in diff_files if MatchingFileType(x, RUST_EXTS)] + swift_diff_files = [ + x for x in diff_files if MatchingFileType(x, SWIFT_EXTS) ] - python_diff_files = [x for x in diff_files if MatchingFileType(x, ['.py'])] - rust_diff_files = [x for x in diff_files if MatchingFileType(x, RUST_EXTS)] - swift_diff_files = [x for x in diff_files if MatchingFileType(x, SWIFT_EXTS)] - gn_diff_files = [x for x in diff_files if MatchingFileType(x, GN_EXTS)] + gn_diff_files = [x for x in diff_files if MatchingFileType(x, GN_EXTS)] - top_dir = settings.GetRoot() + top_dir = settings.GetRoot() - return_value = _RunClangFormatDiff(opts, clang_diff_files, top_dir, - upstream_commit) + return_value = _RunClangFormatDiff(opts, clang_diff_files, top_dir, + upstream_commit) - if opts.use_rust_fmt: - rust_fmt_return_value = _RunRustFmt(opts, rust_diff_files, top_dir, - upstream_commit) - if rust_fmt_return_value == 2: - return_value = 2 + if opts.use_rust_fmt: + rust_fmt_return_value = _RunRustFmt(opts, rust_diff_files, top_dir, + upstream_commit) + if rust_fmt_return_value == 2: + return_value = 2 - if opts.use_swift_format: - if sys.platform != 'darwin': - DieWithError('swift-format is only supported on macOS.') - swift_format_return_value = _RunSwiftFormat(opts, swift_diff_files, top_dir, - upstream_commit) - if swift_format_return_value == 2: - return_value = 2 + if opts.use_swift_format: + if sys.platform != 'darwin': + DieWithError('swift-format is only supported on macOS.') + swift_format_return_value = _RunSwiftFormat(opts, swift_diff_files, + top_dir, upstream_commit) + if swift_format_return_value == 2: + return_value = 2 - # Similar code to above, but using yapf on .py files rather than clang-format - # on C/C++ files - py_explicitly_disabled = opts.python is not None and not opts.python - if python_diff_files and not py_explicitly_disabled: - depot_tools_path = os.path.dirname(os.path.abspath(__file__)) - yapf_tool = os.path.join(depot_tools_path, 'yapf') + # Similar code to above, but using yapf on .py files rather than + # clang-format on C/C++ files + py_explicitly_disabled = opts.python is not None and not opts.python + if python_diff_files and not py_explicitly_disabled: + depot_tools_path = os.path.dirname(os.path.abspath(__file__)) + yapf_tool = os.path.join(depot_tools_path, 'yapf') - # Used for caching. - yapf_configs = {} - for f in python_diff_files: - # Find the yapf style config for the current file, defaults to depot - # tools default. - _FindYapfConfigFile(f, yapf_configs, top_dir) + # Used for caching. + yapf_configs = {} + for f in python_diff_files: + # Find the yapf style config for the current file, defaults to depot + # tools default. + _FindYapfConfigFile(f, yapf_configs, top_dir) - # Turn on python formatting by default if a yapf config is specified. - # This breaks in the case of this repo though since the specified - # style file is also the global default. - if opts.python is None: - filtered_py_files = [] - for f in python_diff_files: - if _FindYapfConfigFile(f, yapf_configs, top_dir) is not None: - filtered_py_files.append(f) - else: - filtered_py_files = python_diff_files - - # Note: yapf still seems to fix indentation of the entire file - # even if line ranges are specified. - # See https://github.com/google/yapf/issues/499 - if not opts.full and filtered_py_files: - py_line_diffs = _ComputeDiffLineRanges(filtered_py_files, upstream_commit) - - yapfignore_patterns = _GetYapfIgnorePatterns(top_dir) - filtered_py_files = _FilterYapfIgnoredFiles(filtered_py_files, - yapfignore_patterns) - - for f in filtered_py_files: - yapf_style = _FindYapfConfigFile(f, yapf_configs, top_dir) - # Default to pep8 if not .style.yapf is found. - if not yapf_style: - yapf_style = 'pep8' - - with open(f, 'r') as py_f: - if 'python2' in py_f.readline(): - vpython_script = 'vpython' + # Turn on python formatting by default if a yapf config is specified. + # This breaks in the case of this repo though since the specified + # style file is also the global default. + if opts.python is None: + filtered_py_files = [] + for f in python_diff_files: + if _FindYapfConfigFile(f, yapf_configs, top_dir) is not None: + filtered_py_files.append(f) else: - vpython_script = 'vpython3' + filtered_py_files = python_diff_files - cmd = [vpython_script, yapf_tool, '--style', yapf_style, f] + # Note: yapf still seems to fix indentation of the entire file + # even if line ranges are specified. + # See https://github.com/google/yapf/issues/499 + if not opts.full and filtered_py_files: + py_line_diffs = _ComputeDiffLineRanges(filtered_py_files, + upstream_commit) - has_formattable_lines = False - if not opts.full: - # Only run yapf over changed line ranges. - for diff_start, diff_len in py_line_diffs[f]: - diff_end = diff_start + diff_len - 1 - # Yapf errors out if diff_end < diff_start but this - # is a valid line range diff for a removal. - if diff_end >= diff_start: - has_formattable_lines = True - cmd += ['-l', '{}-{}'.format(diff_start, diff_end)] - # If all line diffs were removals we have nothing to format. - if not has_formattable_lines: - continue + yapfignore_patterns = _GetYapfIgnorePatterns(top_dir) + filtered_py_files = _FilterYapfIgnoredFiles(filtered_py_files, + yapfignore_patterns) - if opts.diff or opts.dry_run: - cmd += ['--diff'] - # Will return non-zero exit code if non-empty diff. - stdout = RunCommand(cmd, - error_ok=True, - stderr=subprocess2.PIPE, - cwd=top_dir, - shell=sys.platform.startswith('win32')) - if opts.diff: - sys.stdout.write(stdout) - elif len(stdout) > 0: - return_value = 2 - else: - cmd += ['-i'] - RunCommand(cmd, cwd=top_dir, shell=sys.platform.startswith('win32')) + for f in filtered_py_files: + yapf_style = _FindYapfConfigFile(f, yapf_configs, top_dir) + # Default to pep8 if not .style.yapf is found. + if not yapf_style: + yapf_style = 'pep8' - # Format GN build files. Always run on full build files for canonical form. - if gn_diff_files: - cmd = ['gn', 'format'] - if opts.dry_run or opts.diff: - cmd.append('--dry-run') - for gn_diff_file in gn_diff_files: - gn_ret = subprocess2.call(cmd + [gn_diff_file], - shell=sys.platform.startswith('win'), - cwd=top_dir) - if opts.dry_run and gn_ret == 2: - return_value = 2 # Not formatted. - elif opts.diff and gn_ret == 2: - # TODO this should compute and print the actual diff. - print('This change has GN build file diff for ' + gn_diff_file) - elif gn_ret != 0: - # For non-dry run cases (and non-2 return values for dry-run), a - # nonzero error code indicates a failure, probably because the file - # doesn't parse. - DieWithError('gn format failed on ' + gn_diff_file + - '\nTry running `gn format` on this file manually.') + with open(f, 'r') as py_f: + if 'python2' in py_f.readline(): + vpython_script = 'vpython' + else: + vpython_script = 'vpython3' - # Skip the metrics formatting from the global presubmit hook. These files have - # a separate presubmit hook that issues an error if the files need formatting, - # whereas the top-level presubmit script merely issues a warning. Formatting - # these files is somewhat slow, so it's important not to duplicate the work. - if not opts.presubmit: - for diff_xml in GetDiffXMLs(diff_files): - xml_dir = GetMetricsDir(diff_xml) - if not xml_dir: - continue + cmd = [vpython_script, yapf_tool, '--style', yapf_style, f] - tool_dir = os.path.join(top_dir, xml_dir) - pretty_print_tool = os.path.join(tool_dir, 'pretty_print.py') - cmd = ['vpython3', pretty_print_tool, '--non-interactive'] + has_formattable_lines = False + if not opts.full: + # Only run yapf over changed line ranges. + for diff_start, diff_len in py_line_diffs[f]: + diff_end = diff_start + diff_len - 1 + # Yapf errors out if diff_end < diff_start but this + # is a valid line range diff for a removal. + if diff_end >= diff_start: + has_formattable_lines = True + cmd += ['-l', '{}-{}'.format(diff_start, diff_end)] + # If all line diffs were removals we have nothing to format. + if not has_formattable_lines: + continue - # If the XML file is histograms.xml or enums.xml, add the xml path to the - # command as histograms/pretty_print.py now needs a relative path argument - # after splitting the histograms into multiple directories. - # For example, in tools/metrics/ukm, pretty-print could be run using: - # $ python pretty_print.py - # But in tools/metrics/histogrmas, pretty-print should be run with an - # additional relative path argument, like: - # $ python pretty_print.py metadata/UMA/histograms.xml - # $ python pretty_print.py enums.xml + if opts.diff or opts.dry_run: + cmd += ['--diff'] + # Will return non-zero exit code if non-empty diff. + stdout = RunCommand(cmd, + error_ok=True, + stderr=subprocess2.PIPE, + cwd=top_dir, + shell=sys.platform.startswith('win32')) + if opts.diff: + sys.stdout.write(stdout) + elif len(stdout) > 0: + return_value = 2 + else: + cmd += ['-i'] + RunCommand(cmd, + cwd=top_dir, + shell=sys.platform.startswith('win32')) - if xml_dir == os.path.join('tools', 'metrics', 'histograms'): - if os.path.basename(diff_xml) not in ('histograms.xml', 'enums.xml', - 'histogram_suffixes_list.xml'): - # Skip this XML file if it's not one of the known types. - continue - cmd.append(diff_xml) + # Format GN build files. Always run on full build files for canonical form. + if gn_diff_files: + cmd = ['gn', 'format'] + if opts.dry_run or opts.diff: + cmd.append('--dry-run') + for gn_diff_file in gn_diff_files: + gn_ret = subprocess2.call(cmd + [gn_diff_file], + shell=sys.platform.startswith('win'), + cwd=top_dir) + if opts.dry_run and gn_ret == 2: + return_value = 2 # Not formatted. + elif opts.diff and gn_ret == 2: + # TODO this should compute and print the actual diff. + print('This change has GN build file diff for ' + gn_diff_file) + elif gn_ret != 0: + # For non-dry run cases (and non-2 return values for dry-run), a + # nonzero error code indicates a failure, probably because the + # file doesn't parse. + DieWithError('gn format failed on ' + gn_diff_file + + '\nTry running `gn format` on this file manually.') - if opts.dry_run or opts.diff: - cmd.append('--diff') + # Skip the metrics formatting from the global presubmit hook. These files + # have a separate presubmit hook that issues an error if the files need + # formatting, whereas the top-level presubmit script merely issues a + # warning. Formatting these files is somewhat slow, so it's important not to + # duplicate the work. + if not opts.presubmit: + for diff_xml in GetDiffXMLs(diff_files): + xml_dir = GetMetricsDir(diff_xml) + if not xml_dir: + continue - # TODO(isherman): Once this file runs only on Python 3.3+, drop the - # `shell` param and instead replace `'vpython'` with - # `shutil.which('frob')` above: https://stackoverflow.com/a/32799942 - stdout = RunCommand(cmd, - cwd=top_dir, - shell=sys.platform.startswith('win32')) - if opts.diff: - sys.stdout.write(stdout) - if opts.dry_run and stdout: - return_value = 2 # Not formatted. + tool_dir = os.path.join(top_dir, xml_dir) + pretty_print_tool = os.path.join(tool_dir, 'pretty_print.py') + cmd = ['vpython3', pretty_print_tool, '--non-interactive'] - return return_value + # If the XML file is histograms.xml or enums.xml, add the xml path + # to the command as histograms/pretty_print.py now needs a relative + # path argument after splitting the histograms into multiple + # directories. For example, in tools/metrics/ukm, pretty-print could + # be run using: $ python pretty_print.py But in + # tools/metrics/histogrmas, pretty-print should be run with an + # additional relative path argument, like: $ python pretty_print.py + # metadata/UMA/histograms.xml $ python pretty_print.py enums.xml + + if xml_dir == os.path.join('tools', 'metrics', 'histograms'): + if os.path.basename(diff_xml) not in ( + 'histograms.xml', 'enums.xml', + 'histogram_suffixes_list.xml'): + # Skip this XML file if it's not one of the known types. + continue + cmd.append(diff_xml) + + if opts.dry_run or opts.diff: + cmd.append('--diff') + + # TODO(isherman): Once this file runs only on Python 3.3+, drop the + # `shell` param and instead replace `'vpython'` with + # `shutil.which('frob')` above: https://stackoverflow.com/a/32799942 + stdout = RunCommand(cmd, + cwd=top_dir, + shell=sys.platform.startswith('win32')) + if opts.diff: + sys.stdout.write(stdout) + if opts.dry_run and stdout: + return_value = 2 # Not formatted. + + return return_value def GetDiffXMLs(diff_files): - return [ - os.path.normpath(x) for x in diff_files if MatchingFileType(x, ['.xml']) - ] + return [ + os.path.normpath(x) for x in diff_files + if MatchingFileType(x, ['.xml']) + ] def GetMetricsDir(diff_xml): - metrics_xml_dirs = [ - os.path.join('tools', 'metrics', 'actions'), - os.path.join('tools', 'metrics', 'histograms'), - os.path.join('tools', 'metrics', 'structured'), - os.path.join('tools', 'metrics', 'ukm'), - ] - for xml_dir in metrics_xml_dirs: - if diff_xml.startswith(xml_dir): - return xml_dir - return None + metrics_xml_dirs = [ + os.path.join('tools', 'metrics', 'actions'), + os.path.join('tools', 'metrics', 'histograms'), + os.path.join('tools', 'metrics', 'structured'), + os.path.join('tools', 'metrics', 'ukm'), + ] + for xml_dir in metrics_xml_dirs: + if diff_xml.startswith(xml_dir): + return xml_dir + return None @subcommand.usage('') @metrics.collector.collect_metrics('git cl checkout') def CMDcheckout(parser, args): - """Checks out a branch associated with a given Gerrit issue.""" - _, args = parser.parse_args(args) + """Checks out a branch associated with a given Gerrit issue.""" + _, args = parser.parse_args(args) - if len(args) != 1: - parser.print_help() - return 1 + if len(args) != 1: + parser.print_help() + return 1 - issue_arg = ParseIssueNumberArgument(args[0]) - if not issue_arg.valid: - parser.error('Invalid issue ID or URL.') + issue_arg = ParseIssueNumberArgument(args[0]) + if not issue_arg.valid: + parser.error('Invalid issue ID or URL.') - target_issue = str(issue_arg.issue) + target_issue = str(issue_arg.issue) - output = RunGit(['config', '--local', '--get-regexp', - r'branch\..*\.' + ISSUE_CONFIG_KEY], - error_ok=True) + output = RunGit([ + 'config', '--local', '--get-regexp', r'branch\..*\.' + ISSUE_CONFIG_KEY + ], + error_ok=True) - branches = [] - for key, issue in [x.split() for x in output.splitlines()]: - if issue == target_issue: - branches.append(re.sub(r'branch\.(.*)\.' + ISSUE_CONFIG_KEY, r'\1', key)) + branches = [] + for key, issue in [x.split() for x in output.splitlines()]: + if issue == target_issue: + branches.append( + re.sub(r'branch\.(.*)\.' + ISSUE_CONFIG_KEY, r'\1', key)) - if len(branches) == 0: - print('No branch found for issue %s.' % target_issue) - return 1 - if len(branches) == 1: - RunGit(['checkout', branches[0]]) - else: - print('Multiple branches match issue %s:' % target_issue) - for i in range(len(branches)): - print('%d: %s' % (i, branches[i])) - which = gclient_utils.AskForData('Choose by index: ') - try: - RunGit(['checkout', branches[int(which)]]) - except (IndexError, ValueError): - print('Invalid selection, not checking out any branch.') - return 1 + if len(branches) == 0: + print('No branch found for issue %s.' % target_issue) + return 1 + if len(branches) == 1: + RunGit(['checkout', branches[0]]) + else: + print('Multiple branches match issue %s:' % target_issue) + for i in range(len(branches)): + print('%d: %s' % (i, branches[i])) + which = gclient_utils.AskForData('Choose by index: ') + try: + RunGit(['checkout', branches[int(which)]]) + except (IndexError, ValueError): + print('Invalid selection, not checking out any branch.') + return 1 - return 0 + return 0 def CMDlol(parser, args): - # This command is intentionally undocumented. - print(zlib.decompress(base64.b64decode( - 'eNptkLEOwyAMRHe+wupCIqW57v0Vq84WqWtXyrcXnCBsmgMJ+/SSAxMZgRB6NzE' - 'E2ObgCKJooYdu4uAQVffUEoE1sRQLxAcqzd7uK2gmStrll1ucV3uZyaY5sXyDd9' - 'JAnN+lAXsOMJ90GANAi43mq5/VeeacylKVgi8o6F1SC63FxnagHfJUTfUYdCR/W' - 'Ofe+0dHL7PicpytKP750Fh1q2qnLVof4w8OZWNY')).decode('utf-8')) - return 0 + # This command is intentionally undocumented. + print( + zlib.decompress( + base64.b64decode( + 'eNptkLEOwyAMRHe+wupCIqW57v0Vq84WqWtXyrcXnCBsmgMJ+/SSAxMZgRB6NzE' + 'E2ObgCKJooYdu4uAQVffUEoE1sRQLxAcqzd7uK2gmStrll1ucV3uZyaY5sXyDd9' + 'JAnN+lAXsOMJ90GANAi43mq5/VeeacylKVgi8o6F1SC63FxnagHfJUTfUYdCR/W' + 'Ofe+0dHL7PicpytKP750Fh1q2qnLVof4w8OZWNY')).decode('utf-8')) + return 0 def CMDversion(parser, args): - import utils - print(utils.depot_tools_version()) + import utils + print(utils.depot_tools_version()) class OptionParser(optparse.OptionParser): - """Creates the option parse and add --verbose support.""" + """Creates the option parse and add --verbose support.""" + def __init__(self, *args, **kwargs): + optparse.OptionParser.__init__(self, + *args, + prog='git cl', + version=__version__, + **kwargs) + self.add_option('-v', + '--verbose', + action='count', + default=0, + help='Use 2 times for more debugging info') - def __init__(self, *args, **kwargs): - optparse.OptionParser.__init__( - self, *args, prog='git cl', version=__version__, **kwargs) - self.add_option( - '-v', '--verbose', action='count', default=0, - help='Use 2 times for more debugging info') - - def parse_args(self, args=None, _values=None): - try: - return self._parse_args(args) - finally: - # Regardless of success or failure of args parsing, we want to report - # metrics, but only after logging has been initialized (if parsing - # succeeded). - global settings - settings = Settings() - - if metrics.collector.config.should_collect_metrics: + def parse_args(self, args=None, _values=None): try: - # GetViewVCUrl ultimately calls logging method. - project_url = settings.GetViewVCUrl().strip('/+') - if project_url in metrics_utils.KNOWN_PROJECT_URLS: - metrics.collector.add('project_urls', [project_url]) - except subprocess2.CalledProcessError: - # Occurs when command is not executed in a git repository - # We should not fail here. If the command needs to be executed - # in a repo, it will be raised later. - pass + return self._parse_args(args) + finally: + # Regardless of success or failure of args parsing, we want to + # report metrics, but only after logging has been initialized (if + # parsing succeeded). + global settings + settings = Settings() - def _parse_args(self, args=None): - # Create an optparse.Values object that will store only the actual passed - # options, without the defaults. - actual_options = optparse.Values() - _, args = optparse.OptionParser.parse_args(self, args, actual_options) - # Create an optparse.Values object with the default options. - options = optparse.Values(self.get_default_values().__dict__) - # Update it with the options passed by the user. - options._update_careful(actual_options.__dict__) - # Store the options passed by the user in an _actual_options attribute. - # We store only the keys, and not the values, since the values can contain - # arbitrary information, which might be PII. - metrics.collector.add('arguments', list(actual_options.__dict__.keys())) + if metrics.collector.config.should_collect_metrics: + try: + # GetViewVCUrl ultimately calls logging method. + project_url = settings.GetViewVCUrl().strip('/+') + if project_url in metrics_utils.KNOWN_PROJECT_URLS: + metrics.collector.add('project_urls', [project_url]) + except subprocess2.CalledProcessError: + # Occurs when command is not executed in a git repository + # We should not fail here. If the command needs to be + # executed in a repo, it will be raised later. + pass - levels = [logging.WARNING, logging.INFO, logging.DEBUG] - logging.basicConfig( - level=levels[min(options.verbose, len(levels) - 1)], - format='[%(levelname).1s%(asctime)s %(process)d %(thread)d ' - '%(filename)s] %(message)s') + def _parse_args(self, args=None): + # Create an optparse.Values object that will store only the actual + # passed options, without the defaults. + actual_options = optparse.Values() + _, args = optparse.OptionParser.parse_args(self, args, actual_options) + # Create an optparse.Values object with the default options. + options = optparse.Values(self.get_default_values().__dict__) + # Update it with the options passed by the user. + options._update_careful(actual_options.__dict__) + # Store the options passed by the user in an _actual_options attribute. + # We store only the keys, and not the values, since the values can + # contain arbitrary information, which might be PII. + metrics.collector.add('arguments', list(actual_options.__dict__.keys())) - return options, args + levels = [logging.WARNING, logging.INFO, logging.DEBUG] + logging.basicConfig( + level=levels[min(options.verbose, + len(levels) - 1)], + format='[%(levelname).1s%(asctime)s %(process)d %(thread)d ' + '%(filename)s] %(message)s') + + return options, args def main(argv): - if sys.hexversion < 0x02060000: - print('\nYour Python version %s is unsupported, please upgrade.\n' % - (sys.version.split(' ', 1)[0],), file=sys.stderr) - return 2 + if sys.hexversion < 0x02060000: + print('\nYour Python version %s is unsupported, please upgrade.\n' % + (sys.version.split(' ', 1)[0], ), + file=sys.stderr) + return 2 - colorize_CMDstatus_doc() - dispatcher = subcommand.CommandDispatcher(__name__) - try: - return dispatcher.execute(OptionParser(), argv) - except auth.LoginRequiredError as e: - DieWithError(str(e)) - except urllib.error.HTTPError as e: - if e.code != 500: - raise - DieWithError( - ('App Engine is misbehaving and returned HTTP %d, again. Keep faith ' - 'and retry or visit go/isgaeup.\n%s') % (e.code, str(e))) - return 0 + colorize_CMDstatus_doc() + dispatcher = subcommand.CommandDispatcher(__name__) + try: + return dispatcher.execute(OptionParser(), argv) + except auth.LoginRequiredError as e: + DieWithError(str(e)) + except urllib.error.HTTPError as e: + if e.code != 500: + raise + DieWithError(( + 'App Engine is misbehaving and returned HTTP %d, again. Keep faith ' + 'and retry or visit go/isgaeup.\n%s') % (e.code, str(e))) + return 0 if __name__ == '__main__': - # These affect sys.stdout, so do it outside of main() to simplify mocks in - # the unit tests. - fix_encoding.fix_encoding() - setup_color.init() - with metrics.collector.print_notice_and_exit(): - sys.exit(main(sys.argv[1:])) + # These affect sys.stdout, so do it outside of main() to simplify mocks in + # the unit tests. + fix_encoding.fix_encoding() + setup_color.init() + with metrics.collector.print_notice_and_exit(): + sys.exit(main(sys.argv[1:])) diff --git a/git_common.py b/git_common.py index e439465a8c..592e9313e7 100644 --- a/git_common.py +++ b/git_common.py @@ -15,15 +15,16 @@ from third_party import colorama def wrapper(func): - def wrap(self, timeout=None): - return func(self, timeout=timeout or threading.TIMEOUT_MAX) + def wrap(self, timeout=None): + return func(self, timeout=timeout or threading.TIMEOUT_MAX) + + return wrap + - return wrap IMapIterator.next = wrapper(IMapIterator.next) IMapIterator.__next__ = IMapIterator.next # TODO(iannucci): Monkeypatch all other 'wait' methods too. - import binascii import collections import contextlib @@ -41,32 +42,26 @@ import subprocess2 from io import BytesIO - ROOT = os.path.abspath(os.path.dirname(__file__)) IS_WIN = sys.platform == 'win32' TEST_MODE = False def win_find_git(): - for elem in os.environ.get('PATH', '').split(os.pathsep): - for candidate in ('git.exe', 'git.bat'): - path = os.path.join(elem, candidate) - if os.path.isfile(path): - return path - raise ValueError('Could not find Git on PATH.') + for elem in os.environ.get('PATH', '').split(os.pathsep): + for candidate in ('git.exe', 'git.bat'): + path = os.path.join(elem, candidate) + if os.path.isfile(path): + return path + raise ValueError('Could not find Git on PATH.') GIT_EXE = 'git' if not IS_WIN else win_find_git() - FREEZE = 'FREEZE' -FREEZE_SECTIONS = { - 'indexed': 'soft', - 'unindexed': 'mixed' -} +FREEZE_SECTIONS = {'indexed': 'soft', 'unindexed': 'mixed'} FREEZE_MATCHER = re.compile(r'%s.(%s)' % (FREEZE, '|'.join(FREEZE_SECTIONS))) - # NOTE: This list is DEPRECATED in favor of the Infra Git wrapper: # https://chromium.googlesource.com/infra/infra/+/HEAD/go/src/infra/tools/git # @@ -119,11 +114,8 @@ GIT_TRANSIENT_ERRORS = ( # crbug.com/430343 # TODO(dnj): Resync with Chromite. r'The requested URL returned error: 5\d+', - r'Connection reset by peer', - r'Unable to look up', - r'Couldn\'t resolve host', ) @@ -135,15 +127,15 @@ GIT_TRANSIENT_ERRORS_RE = re.compile('|'.join(GIT_TRANSIENT_ERRORS), # See git commit b6160d95 for more information. MIN_UPSTREAM_TRACK_GIT_VERSION = (2, 3) + class BadCommitRefException(Exception): - def __init__(self, refs): - msg = ('one of %s does not seem to be a valid commitref.' % - str(refs)) - super(BadCommitRefException, self).__init__(msg) + def __init__(self, refs): + msg = ('one of %s does not seem to be a valid commitref.' % str(refs)) + super(BadCommitRefException, self).__init__(msg) def memoize_one(**kwargs): - """Memoizes a single-argument pure function. + """Memoizes a single-argument pure function. Values of None are not cached. @@ -158,64 +150,69 @@ def memoize_one(**kwargs): * clear() - Drops the entire contents of the cache. Useful for unittests. * update(other) - Updates the contents of the cache from another dict. """ - assert 'threadsafe' in kwargs, 'Must specify threadsafe={True,False}' - threadsafe = kwargs['threadsafe'] + assert 'threadsafe' in kwargs, 'Must specify threadsafe={True,False}' + threadsafe = kwargs['threadsafe'] - if threadsafe: - def withlock(lock, f): - def inner(*args, **kwargs): - with lock: - return f(*args, **kwargs) - return inner - else: - def withlock(_lock, f): - return f + if threadsafe: - def decorator(f): - # Instantiate the lock in decorator, in case users of memoize_one do: - # - # memoizer = memoize_one(threadsafe=True) - # - # @memoizer - # def fn1(val): ... - # - # @memoizer - # def fn2(val): ... + def withlock(lock, f): + def inner(*args, **kwargs): + with lock: + return f(*args, **kwargs) - lock = threading.Lock() if threadsafe else None - cache = {} - _get = withlock(lock, cache.get) - _set = withlock(lock, cache.__setitem__) + return inner + else: - @functools.wraps(f) - def inner(arg): - ret = _get(arg) - if ret is None: - ret = f(arg) - if ret is not None: - _set(arg, ret) - return ret - inner.get = _get - inner.set = _set - inner.clear = withlock(lock, cache.clear) - inner.update = withlock(lock, cache.update) - return inner - return decorator + def withlock(_lock, f): + return f + + def decorator(f): + # Instantiate the lock in decorator, in case users of memoize_one do: + # + # memoizer = memoize_one(threadsafe=True) + # + # @memoizer + # def fn1(val): ... + # + # @memoizer + # def fn2(val): ... + + lock = threading.Lock() if threadsafe else None + cache = {} + _get = withlock(lock, cache.get) + _set = withlock(lock, cache.__setitem__) + + @functools.wraps(f) + def inner(arg): + ret = _get(arg) + if ret is None: + ret = f(arg) + if ret is not None: + _set(arg, ret) + return ret + + inner.get = _get + inner.set = _set + inner.clear = withlock(lock, cache.clear) + inner.update = withlock(lock, cache.update) + return inner + + return decorator def _ScopedPool_initer(orig, orig_args): # pragma: no cover - """Initializer method for ScopedPool's subprocesses. + """Initializer method for ScopedPool's subprocesses. This helps ScopedPool handle Ctrl-C's correctly. """ - signal.signal(signal.SIGINT, signal.SIG_IGN) - if orig: - orig(*orig_args) + signal.signal(signal.SIGINT, signal.SIG_IGN) + if orig: + orig(*orig_args) @contextlib.contextmanager def ScopedPool(*args, **kwargs): - """Context Manager which returns a multiprocessing.pool instance which + """Context Manager which returns a multiprocessing.pool instance which correctly deals with thrown exceptions. *args - Arguments to multiprocessing.pool @@ -224,28 +221,28 @@ def ScopedPool(*args, **kwargs): kind ('threads', 'procs') - The type of underlying coprocess to use. **etc - Arguments to multiprocessing.pool """ - if kwargs.pop('kind', None) == 'threads': - pool = multiprocessing.pool.ThreadPool(*args, **kwargs) - else: - orig, orig_args = kwargs.get('initializer'), kwargs.get('initargs', ()) - kwargs['initializer'] = _ScopedPool_initer - kwargs['initargs'] = orig, orig_args - pool = multiprocessing.pool.Pool(*args, **kwargs) + if kwargs.pop('kind', None) == 'threads': + pool = multiprocessing.pool.ThreadPool(*args, **kwargs) + else: + orig, orig_args = kwargs.get('initializer'), kwargs.get('initargs', ()) + kwargs['initializer'] = _ScopedPool_initer + kwargs['initargs'] = orig, orig_args + pool = multiprocessing.pool.Pool(*args, **kwargs) - try: - yield pool - pool.close() - except: - pool.terminate() - raise - finally: - pool.join() + try: + yield pool + pool.close() + except: + pool.terminate() + raise + finally: + pool.join() class ProgressPrinter(object): - """Threaded single-stat status message printer.""" - def __init__(self, fmt, enabled=None, fout=sys.stderr, period=0.5): - """Create a ProgressPrinter. + """Threaded single-stat status message printer.""" + def __init__(self, fmt, enabled=None, fout=sys.stderr, period=0.5): + """Create a ProgressPrinter. Use it as a context manager which produces a simple 'increment' method: @@ -264,107 +261,111 @@ class ProgressPrinter(object): period (float) - The time in seconds for the printer thread to wait between printing. """ - self.fmt = fmt - if enabled is None: # pragma: no cover - self.enabled = logging.getLogger().isEnabledFor(logging.INFO) - else: - self.enabled = enabled + self.fmt = fmt + if enabled is None: # pragma: no cover + self.enabled = logging.getLogger().isEnabledFor(logging.INFO) + else: + self.enabled = enabled - self._count = 0 - self._dead = False - self._dead_cond = threading.Condition() - self._stream = fout - self._thread = threading.Thread(target=self._run) - self._period = period + self._count = 0 + self._dead = False + self._dead_cond = threading.Condition() + self._stream = fout + self._thread = threading.Thread(target=self._run) + self._period = period - def _emit(self, s): - if self.enabled: - self._stream.write('\r' + s) - self._stream.flush() + def _emit(self, s): + if self.enabled: + self._stream.write('\r' + s) + self._stream.flush() - def _run(self): - with self._dead_cond: - while not self._dead: - self._emit(self.fmt % {'count': self._count}) - self._dead_cond.wait(self._period) - self._emit((self.fmt + '\n') % {'count': self._count}) + def _run(self): + with self._dead_cond: + while not self._dead: + self._emit(self.fmt % {'count': self._count}) + self._dead_cond.wait(self._period) + self._emit((self.fmt + '\n') % {'count': self._count}) - def inc(self, amount=1): - self._count += amount + def inc(self, amount=1): + self._count += amount - def __enter__(self): - self._thread.start() - return self.inc + def __enter__(self): + self._thread.start() + return self.inc - def __exit__(self, _exc_type, _exc_value, _traceback): - self._dead = True - with self._dead_cond: - self._dead_cond.notifyAll() - self._thread.join() - del self._thread + def __exit__(self, _exc_type, _exc_value, _traceback): + self._dead = True + with self._dead_cond: + self._dead_cond.notifyAll() + self._thread.join() + del self._thread def once(function): - """@Decorates |function| so that it only performs its action once, no matter + """@Decorates |function| so that it only performs its action once, no matter how many times the decorated |function| is called.""" - has_run = [False] - def _wrapper(*args, **kwargs): - if not has_run[0]: - has_run[0] = True - function(*args, **kwargs) - return _wrapper + has_run = [False] + + def _wrapper(*args, **kwargs): + if not has_run[0]: + has_run[0] = True + function(*args, **kwargs) + + return _wrapper def unicode_repr(s): - result = repr(s) - return result[1:] if result.startswith('u') else result + result = repr(s) + return result[1:] if result.startswith('u') else result ## Git functions + def die(message, *args): - print(textwrap.dedent(message % args), file=sys.stderr) - sys.exit(1) + print(textwrap.dedent(message % args), file=sys.stderr) + sys.exit(1) def blame(filename, revision=None, porcelain=False, abbrev=None, *_args): - command = ['blame'] - if porcelain: - command.append('-p') - if revision is not None: - command.append(revision) - if abbrev is not None: - command.append('--abbrev=%d' % abbrev) - command.extend(['--', filename]) - return run(*command) + command = ['blame'] + if porcelain: + command.append('-p') + if revision is not None: + command.append(revision) + if abbrev is not None: + command.append('--abbrev=%d' % abbrev) + command.extend(['--', filename]) + return run(*command) def branch_config(branch, option, default=None): - return get_config('branch.%s.%s' % (branch, option), default=default) + return get_config('branch.%s.%s' % (branch, option), default=default) def branch_config_map(option): - """Return {branch: <|option| value>} for all branches.""" - try: - reg = re.compile(r'^branch\.(.*)\.%s$' % option) - lines = get_config_regexp(reg.pattern) - return {reg.match(k).group(1): v for k, v in (l.split() for l in lines)} - except subprocess2.CalledProcessError: - return {} + """Return {branch: <|option| value>} for all branches.""" + try: + reg = re.compile(r'^branch\.(.*)\.%s$' % option) + lines = get_config_regexp(reg.pattern) + return {reg.match(k).group(1): v for k, v in (l.split() for l in lines)} + except subprocess2.CalledProcessError: + return {} def branches(use_limit=True, *args): - NO_BRANCH = ('* (no branch', '* (detached', '* (HEAD detached') + NO_BRANCH = ('* (no branch', '* (detached', '* (HEAD detached') - key = 'depot-tools.branch-limit' - limit = get_config_int(key, 20) + key = 'depot-tools.branch-limit' + limit = get_config_int(key, 20) - raw_branches = run('branch', *args).splitlines() + raw_branches = run('branch', *args).splitlines() - num = len(raw_branches) + num = len(raw_branches) - if use_limit and num > limit: - die("""\ + if use_limit and num > limit: + die( + """\ Your git repo has too many branches (%d/%d) for this tool to work well. You may adjust this limit by running: @@ -374,126 +375,126 @@ def branches(use_limit=True, *args): git cl archive """, num, limit, key) - for line in raw_branches: - if line.startswith(NO_BRANCH): - continue - yield line.split()[-1] + for line in raw_branches: + if line.startswith(NO_BRANCH): + continue + yield line.split()[-1] def get_config(option, default=None): - try: - return run('config', '--get', option) or default - except subprocess2.CalledProcessError: - return default + try: + return run('config', '--get', option) or default + except subprocess2.CalledProcessError: + return default def get_config_int(option, default=0): - assert isinstance(default, int) - try: - return int(get_config(option, default)) - except ValueError: - return default + assert isinstance(default, int) + try: + return int(get_config(option, default)) + except ValueError: + return default def get_config_list(option): - try: - return run('config', '--get-all', option).split() - except subprocess2.CalledProcessError: - return [] + try: + return run('config', '--get-all', option).split() + except subprocess2.CalledProcessError: + return [] def get_config_regexp(pattern): - if IS_WIN: # pragma: no cover - # this madness is because we call git.bat which calls git.exe which calls - # bash.exe (or something to that effect). Each layer divides the number of - # ^'s by 2. - pattern = pattern.replace('^', '^' * 8) - return run('config', '--get-regexp', pattern).splitlines() + if IS_WIN: # pragma: no cover + # this madness is because we call git.bat which calls git.exe which + # calls bash.exe (or something to that effect). Each layer divides the + # number of ^'s by 2. + pattern = pattern.replace('^', '^' * 8) + return run('config', '--get-regexp', pattern).splitlines() def is_fsmonitor_enabled(): - """Returns true if core.fsmonitor is enabled in git config.""" - fsmonitor = get_config('core.fsmonitor', 'False') - return fsmonitor.strip().lower() == 'true' + """Returns true if core.fsmonitor is enabled in git config.""" + fsmonitor = get_config('core.fsmonitor', 'False') + return fsmonitor.strip().lower() == 'true' def warn_submodule(): - """Print warnings for submodules.""" - # TODO(crbug.com/1475405): Warn users if the project uses submodules and - # they have fsmonitor enabled. - if sys.platform.startswith('darwin') and is_fsmonitor_enabled(): - print(colorama.Fore.RED) - print('WARNING: You have fsmonitor enabled. There is a major issue ' - 'resulting in git diff-index returning wrong results. Please ' - 'disable it by running:') - print(' git config core.fsmonitor false') - print('We will remove this warning once https://crbug.com/1475405 is ' - 'fixed.') - print(colorama.Style.RESET_ALL) + """Print warnings for submodules.""" + # TODO(crbug.com/1475405): Warn users if the project uses submodules and + # they have fsmonitor enabled. + if sys.platform.startswith('darwin') and is_fsmonitor_enabled(): + print(colorama.Fore.RED) + print('WARNING: You have fsmonitor enabled. There is a major issue ' + 'resulting in git diff-index returning wrong results. Please ' + 'disable it by running:') + print(' git config core.fsmonitor false') + print('We will remove this warning once https://crbug.com/1475405 is ' + 'fixed.') + print(colorama.Style.RESET_ALL) def current_branch(): - try: - return run('rev-parse', '--abbrev-ref', 'HEAD') - except subprocess2.CalledProcessError: - return None + try: + return run('rev-parse', '--abbrev-ref', 'HEAD') + except subprocess2.CalledProcessError: + return None def del_branch_config(branch, option, scope='local'): - del_config('branch.%s.%s' % (branch, option), scope=scope) + del_config('branch.%s.%s' % (branch, option), scope=scope) def del_config(option, scope='local'): - try: - run('config', '--' + scope, '--unset', option) - except subprocess2.CalledProcessError: - pass + try: + run('config', '--' + scope, '--unset', option) + except subprocess2.CalledProcessError: + pass def diff(oldrev, newrev, *args): - return run('diff', oldrev, newrev, *args) + return run('diff', oldrev, newrev, *args) def freeze(): - took_action = False - key = 'depot-tools.freeze-size-limit' - MB = 2**20 - limit_mb = get_config_int(key, 100) - untracked_bytes = 0 + took_action = False + key = 'depot-tools.freeze-size-limit' + MB = 2**20 + limit_mb = get_config_int(key, 100) + untracked_bytes = 0 - root_path = repo_root() + root_path = repo_root() - # unindexed tracks all the files which are unindexed but we want to add to - # the `FREEZE.unindexed` commit. - unindexed = [] + # unindexed tracks all the files which are unindexed but we want to add to + # the `FREEZE.unindexed` commit. + unindexed = [] - # will be set to true if there are any indexed files to commit. - have_indexed_files = False + # will be set to true if there are any indexed files to commit. + have_indexed_files = False - for f, s in status(ignore_submodules='all'): - if is_unmerged(s): - die("Cannot freeze unmerged changes!") - if s.lstat not in ' ?': - # This covers all changes to indexed files. - # lstat = ' ' means that the file is tracked and modified, but wasn't - # added yet. - # lstat = '?' means that the file is untracked. - have_indexed_files = True + for f, s in status(ignore_submodules='all'): + if is_unmerged(s): + die("Cannot freeze unmerged changes!") + if s.lstat not in ' ?': + # This covers all changes to indexed files. + # lstat = ' ' means that the file is tracked and modified, but + # wasn't added yet. lstat = '?' means that the file is untracked. + have_indexed_files = True - # If the file has both indexed and unindexed changes. - # rstat shows the status of the working tree. If the file also has changes - # in the working tree, it should be tracked both in indexed and unindexed - # changes. - if s.rstat != ' ': - unindexed.append(f.encode('utf-8')) - else: - unindexed.append(f.encode('utf-8')) + # If the file has both indexed and unindexed changes. + # rstat shows the status of the working tree. If the file also has + # changes in the working tree, it should be tracked both in indexed + # and unindexed changes. + if s.rstat != ' ': + unindexed.append(f.encode('utf-8')) + else: + unindexed.append(f.encode('utf-8')) - if s.lstat == '?' and limit_mb > 0: - untracked_bytes += os.lstat(os.path.join(root_path, f)).st_size + if s.lstat == '?' and limit_mb > 0: + untracked_bytes += os.lstat(os.path.join(root_path, f)).st_size - if limit_mb > 0 and untracked_bytes > limit_mb * MB: - die("""\ + if limit_mb > 0 and untracked_bytes > limit_mb * MB: + die( + """\ You appear to have too much untracked+unignored data in your git checkout: %.1f / %d MB. @@ -510,116 +511,117 @@ def freeze(): freeze limit by running: git config %s Where is an integer threshold in megabytes.""", - untracked_bytes / (MB * 1.0), limit_mb, key) + untracked_bytes / (MB * 1.0), limit_mb, key) - if have_indexed_files: - try: - run('commit', '--no-verify', '-m', f'{FREEZE}.indexed') - took_action = True - except subprocess2.CalledProcessError: - pass + if have_indexed_files: + try: + run('commit', '--no-verify', '-m', f'{FREEZE}.indexed') + took_action = True + except subprocess2.CalledProcessError: + pass - add_errors = False - if unindexed: - try: - run('add', - '--pathspec-from-file', - '-', - '--ignore-errors', - indata=b'\n'.join(unindexed), - cwd=root_path) - except subprocess2.CalledProcessError: - add_errors = True + add_errors = False + if unindexed: + try: + run('add', + '--pathspec-from-file', + '-', + '--ignore-errors', + indata=b'\n'.join(unindexed), + cwd=root_path) + except subprocess2.CalledProcessError: + add_errors = True - try: - run('commit', '--no-verify', '-m', f'{FREEZE}.unindexed') - took_action = True - except subprocess2.CalledProcessError: - pass + try: + run('commit', '--no-verify', '-m', f'{FREEZE}.unindexed') + took_action = True + except subprocess2.CalledProcessError: + pass - ret = [] - if add_errors: - ret.append('Failed to index some unindexed files.') - if not took_action: - ret.append('Nothing to freeze.') - return ' '.join(ret) or None + ret = [] + if add_errors: + ret.append('Failed to index some unindexed files.') + if not took_action: + ret.append('Nothing to freeze.') + return ' '.join(ret) or None def get_branch_tree(use_limit=False): - """Get the dictionary of {branch: parent}, compatible with topo_iter. + """Get the dictionary of {branch: parent}, compatible with topo_iter. Returns a tuple of (skipped, ) where skipped is a set of branches without upstream branches defined. """ - skipped = set() - branch_tree = {} + skipped = set() + branch_tree = {} - for branch in branches(use_limit=use_limit): - parent = upstream(branch) - if not parent: - skipped.add(branch) - continue - branch_tree[branch] = parent + for branch in branches(use_limit=use_limit): + parent = upstream(branch) + if not parent: + skipped.add(branch) + continue + branch_tree[branch] = parent - return skipped, branch_tree + return skipped, branch_tree def get_or_create_merge_base(branch, parent=None): - """Finds the configured merge base for branch. + """Finds the configured merge base for branch. If parent is supplied, it's used instead of calling upstream(branch). """ - base = branch_config(branch, 'base') - base_upstream = branch_config(branch, 'base-upstream') - parent = parent or upstream(branch) - if parent is None or branch is None: - return None - actual_merge_base = run('merge-base', parent, branch) + base = branch_config(branch, 'base') + base_upstream = branch_config(branch, 'base-upstream') + parent = parent or upstream(branch) + if parent is None or branch is None: + return None + actual_merge_base = run('merge-base', parent, branch) - if base_upstream != parent: - base = None - base_upstream = None + if base_upstream != parent: + base = None + base_upstream = None - def is_ancestor(a, b): - return run_with_retcode('merge-base', '--is-ancestor', a, b) == 0 + def is_ancestor(a, b): + return run_with_retcode('merge-base', '--is-ancestor', a, b) == 0 - if base and base != actual_merge_base: - if not is_ancestor(base, branch): - logging.debug('Found WRONG pre-set merge-base for %s: %s', branch, base) - base = None - elif is_ancestor(base, actual_merge_base): - logging.debug('Found OLD pre-set merge-base for %s: %s', branch, base) - base = None - else: - logging.debug('Found pre-set merge-base for %s: %s', branch, base) + if base and base != actual_merge_base: + if not is_ancestor(base, branch): + logging.debug('Found WRONG pre-set merge-base for %s: %s', branch, + base) + base = None + elif is_ancestor(base, actual_merge_base): + logging.debug('Found OLD pre-set merge-base for %s: %s', branch, + base) + base = None + else: + logging.debug('Found pre-set merge-base for %s: %s', branch, base) - if not base: - base = actual_merge_base - manual_merge_base(branch, base, parent) + if not base: + base = actual_merge_base + manual_merge_base(branch, base, parent) - return base + return base def hash_multi(*reflike): - return run('rev-parse', *reflike).splitlines() + return run('rev-parse', *reflike).splitlines() def hash_one(reflike, short=False): - args = ['rev-parse', reflike] - if short: - args.insert(1, '--short') - return run(*args) + args = ['rev-parse', reflike] + if short: + args.insert(1, '--short') + return run(*args) def in_rebase(): - git_dir = run('rev-parse', '--git-dir') - return ( - os.path.exists(os.path.join(git_dir, 'rebase-merge')) or - os.path.exists(os.path.join(git_dir, 'rebase-apply'))) + git_dir = run('rev-parse', '--git-dir') + return (os.path.exists(os.path.join(git_dir, 'rebase-merge')) + or os.path.exists(os.path.join(git_dir, 'rebase-apply'))) def intern_f(f, kind='blob'): - """Interns a file object into the git object store. + """Interns a file object into the git object store. Args: f (file-like object) - The file-like object to intern @@ -627,62 +629,61 @@ def intern_f(f, kind='blob'): Returns the git hash of the interned object (hex encoded). """ - ret = run('hash-object', '-t', kind, '-w', '--stdin', stdin=f) - f.close() - return ret + ret = run('hash-object', '-t', kind, '-w', '--stdin', stdin=f) + f.close() + return ret def is_dormant(branch): - # TODO(iannucci): Do an oldness check? - return branch_config(branch, 'dormant', 'false') != 'false' + # TODO(iannucci): Do an oldness check? + return branch_config(branch, 'dormant', 'false') != 'false' def is_unmerged(stat_value): - return ( - 'U' in (stat_value.lstat, stat_value.rstat) or - ((stat_value.lstat == stat_value.rstat) and stat_value.lstat in 'AD') - ) + return ('U' in (stat_value.lstat, stat_value.rstat) + or ((stat_value.lstat == stat_value.rstat) + and stat_value.lstat in 'AD')) def manual_merge_base(branch, base, parent): - set_branch_config(branch, 'base', base) - set_branch_config(branch, 'base-upstream', parent) + set_branch_config(branch, 'base', base) + set_branch_config(branch, 'base-upstream', parent) def mktree(treedict): - """Makes a git tree object and returns its hash. + """Makes a git tree object and returns its hash. See |tree()| for the values of mode, type, and ref. Args: treedict - { name: (mode, type, ref) } """ - with tempfile.TemporaryFile() as f: - for name, (mode, typ, ref) in treedict.items(): - f.write(('%s %s %s\t%s\0' % (mode, typ, ref, name)).encode('utf-8')) - f.seek(0) - return run('mktree', '-z', stdin=f) + with tempfile.TemporaryFile() as f: + for name, (mode, typ, ref) in treedict.items(): + f.write(('%s %s %s\t%s\0' % (mode, typ, ref, name)).encode('utf-8')) + f.seek(0) + return run('mktree', '-z', stdin=f) def parse_commitrefs(*commitrefs): - """Returns binary encoded commit hashes for one or more commitrefs. + """Returns binary encoded commit hashes for one or more commitrefs. A commitref is anything which can resolve to a commit. Popular examples: * 'HEAD' * 'origin/main' * 'cool_branch~2' """ - try: - return [binascii.unhexlify(h) for h in hash_multi(*commitrefs)] - except subprocess2.CalledProcessError: - raise BadCommitRefException(commitrefs) + try: + return [binascii.unhexlify(h) for h in hash_multi(*commitrefs)] + except subprocess2.CalledProcessError: + raise BadCommitRefException(commitrefs) RebaseRet = collections.namedtuple('RebaseRet', 'success stdout stderr') def rebase(parent, start, branch, abort=False, allow_gc=False): - """Rebases |start|..|branch| onto the branch |parent|. + """Rebases |start|..|branch| onto the branch |parent|. Sets 'gc.auto=0' for the duration of this call to prevent the rebase from running a potentially slow garbage collection cycle. @@ -704,140 +705,145 @@ def rebase(parent, start, branch, abort=False, allow_gc=False): message - if the rebase failed, this contains the stdout of the failed rebase. """ - try: - args = [ - '-c', 'gc.auto={}'.format('1' if allow_gc else '0'), - 'rebase', - ] - if TEST_MODE: - args.append('--committer-date-is-author-date') - args += [ - '--onto', parent, start, branch, - ] - run(*args) - return RebaseRet(True, '', '') - except subprocess2.CalledProcessError as cpe: - if abort: - run_with_retcode('rebase', '--abort') # ignore failure - return RebaseRet(False, cpe.stdout.decode('utf-8', 'replace'), - cpe.stderr.decode('utf-8', 'replace')) + try: + args = [ + '-c', + 'gc.auto={}'.format('1' if allow_gc else '0'), + 'rebase', + ] + if TEST_MODE: + args.append('--committer-date-is-author-date') + args += [ + '--onto', + parent, + start, + branch, + ] + run(*args) + return RebaseRet(True, '', '') + except subprocess2.CalledProcessError as cpe: + if abort: + run_with_retcode('rebase', '--abort') # ignore failure + return RebaseRet(False, cpe.stdout.decode('utf-8', 'replace'), + cpe.stderr.decode('utf-8', 'replace')) def remove_merge_base(branch): - del_branch_config(branch, 'base') - del_branch_config(branch, 'base-upstream') + del_branch_config(branch, 'base') + del_branch_config(branch, 'base-upstream') def repo_root(): - """Returns the absolute path to the repository root.""" - return run('rev-parse', '--show-toplevel') + """Returns the absolute path to the repository root.""" + return run('rev-parse', '--show-toplevel') def upstream_default(): - """Returns the default branch name of the origin repository.""" - try: - ret = run('rev-parse', '--abbrev-ref', 'origin/HEAD') - # Detect if the repository migrated to main branch - if ret == 'origin/master': - try: - ret = run('rev-parse', '--abbrev-ref', 'origin/main') - run('remote', 'set-head', '-a', 'origin') + """Returns the default branch name of the origin repository.""" + try: ret = run('rev-parse', '--abbrev-ref', 'origin/HEAD') - except subprocess2.CalledProcessError: - pass - return ret - except subprocess2.CalledProcessError: - return 'origin/main' + # Detect if the repository migrated to main branch + if ret == 'origin/master': + try: + ret = run('rev-parse', '--abbrev-ref', 'origin/main') + run('remote', 'set-head', '-a', 'origin') + ret = run('rev-parse', '--abbrev-ref', 'origin/HEAD') + except subprocess2.CalledProcessError: + pass + return ret + except subprocess2.CalledProcessError: + return 'origin/main' def root(): - return get_config('depot-tools.upstream', upstream_default()) + return get_config('depot-tools.upstream', upstream_default()) @contextlib.contextmanager def less(): # pragma: no cover - """Runs 'less' as context manager yielding its stdin as a PIPE. + """Runs 'less' as context manager yielding its stdin as a PIPE. Automatically checks if sys.stdout is a non-TTY stream. If so, it avoids running less and just yields sys.stdout. The returned PIPE is opened on binary mode. """ - if not setup_color.IS_TTY: - # On Python 3, sys.stdout doesn't accept bytes, and sys.stdout.buffer must - # be used. - yield getattr(sys.stdout, 'buffer', sys.stdout) - return + if not setup_color.IS_TTY: + # On Python 3, sys.stdout doesn't accept bytes, and sys.stdout.buffer + # must be used. + yield getattr(sys.stdout, 'buffer', sys.stdout) + return - # Run with the same options that git uses (see setup_pager in git repo). - # -F: Automatically quit if the output is less than one screen. - # -R: Don't escape ANSI color codes. - # -X: Don't clear the screen before starting. - cmd = ('less', '-FRX') - try: - proc = subprocess2.Popen(cmd, stdin=subprocess2.PIPE) - yield proc.stdin - finally: + # Run with the same options that git uses (see setup_pager in git repo). + # -F: Automatically quit if the output is less than one screen. + # -R: Don't escape ANSI color codes. + # -X: Don't clear the screen before starting. + cmd = ('less', '-FRX') try: - proc.stdin.close() - except BrokenPipeError: - # BrokenPipeError is raised if proc has already completed, - pass - proc.wait() + proc = subprocess2.Popen(cmd, stdin=subprocess2.PIPE) + yield proc.stdin + finally: + try: + proc.stdin.close() + except BrokenPipeError: + # BrokenPipeError is raised if proc has already completed, + pass + proc.wait() def run(*cmd, **kwargs): - """The same as run_with_stderr, except it only returns stdout.""" - return run_with_stderr(*cmd, **kwargs)[0] + """The same as run_with_stderr, except it only returns stdout.""" + return run_with_stderr(*cmd, **kwargs)[0] def run_with_retcode(*cmd, **kwargs): - """Run a command but only return the status code.""" - try: - run(*cmd, **kwargs) - return 0 - except subprocess2.CalledProcessError as cpe: - return cpe.returncode + """Run a command but only return the status code.""" + try: + run(*cmd, **kwargs) + return 0 + except subprocess2.CalledProcessError as cpe: + return cpe.returncode + def run_stream(*cmd, **kwargs): - """Runs a git command. Returns stdout as a PIPE (file-like object). + """Runs a git command. Returns stdout as a PIPE (file-like object). stderr is dropped to avoid races if the process outputs to both stdout and stderr. """ - kwargs.setdefault('stderr', subprocess2.DEVNULL) - kwargs.setdefault('stdout', subprocess2.PIPE) - kwargs.setdefault('shell', False) - cmd = (GIT_EXE, '-c', 'color.ui=never') + cmd - proc = subprocess2.Popen(cmd, **kwargs) - return proc.stdout + kwargs.setdefault('stderr', subprocess2.DEVNULL) + kwargs.setdefault('stdout', subprocess2.PIPE) + kwargs.setdefault('shell', False) + cmd = (GIT_EXE, '-c', 'color.ui=never') + cmd + proc = subprocess2.Popen(cmd, **kwargs) + return proc.stdout @contextlib.contextmanager def run_stream_with_retcode(*cmd, **kwargs): - """Runs a git command as context manager yielding stdout as a PIPE. + """Runs a git command as context manager yielding stdout as a PIPE. stderr is dropped to avoid races if the process outputs to both stdout and stderr. Raises subprocess2.CalledProcessError on nonzero return code. """ - kwargs.setdefault('stderr', subprocess2.DEVNULL) - kwargs.setdefault('stdout', subprocess2.PIPE) - kwargs.setdefault('shell', False) - cmd = (GIT_EXE, '-c', 'color.ui=never') + cmd - try: - proc = subprocess2.Popen(cmd, **kwargs) - yield proc.stdout - finally: - retcode = proc.wait() - if retcode != 0: - raise subprocess2.CalledProcessError(retcode, cmd, os.getcwd(), - b'', b'') + kwargs.setdefault('stderr', subprocess2.DEVNULL) + kwargs.setdefault('stdout', subprocess2.PIPE) + kwargs.setdefault('shell', False) + cmd = (GIT_EXE, '-c', 'color.ui=never') + cmd + try: + proc = subprocess2.Popen(cmd, **kwargs) + yield proc.stdout + finally: + retcode = proc.wait() + if retcode != 0: + raise subprocess2.CalledProcessError(retcode, cmd, os.getcwd(), b'', + b'') def run_with_stderr(*cmd, **kwargs): - """Runs a git command. + """Runs a git command. Returns (stdout, stderr) as a pair of strings. @@ -845,64 +851,66 @@ def run_with_stderr(*cmd, **kwargs): autostrip (bool) - Strip the output. Defaults to True. indata (str) - Specifies stdin data for the process. """ - kwargs.setdefault('stdin', subprocess2.PIPE) - kwargs.setdefault('stdout', subprocess2.PIPE) - kwargs.setdefault('stderr', subprocess2.PIPE) - kwargs.setdefault('shell', False) - autostrip = kwargs.pop('autostrip', True) - indata = kwargs.pop('indata', None) - decode = kwargs.pop('decode', True) - accepted_retcodes = kwargs.pop('accepted_retcodes', [0]) + kwargs.setdefault('stdin', subprocess2.PIPE) + kwargs.setdefault('stdout', subprocess2.PIPE) + kwargs.setdefault('stderr', subprocess2.PIPE) + kwargs.setdefault('shell', False) + autostrip = kwargs.pop('autostrip', True) + indata = kwargs.pop('indata', None) + decode = kwargs.pop('decode', True) + accepted_retcodes = kwargs.pop('accepted_retcodes', [0]) - cmd = (GIT_EXE, '-c', 'color.ui=never') + cmd - proc = subprocess2.Popen(cmd, **kwargs) - ret, err = proc.communicate(indata) - retcode = proc.wait() - if retcode not in accepted_retcodes: - raise subprocess2.CalledProcessError(retcode, cmd, os.getcwd(), ret, err) + cmd = (GIT_EXE, '-c', 'color.ui=never') + cmd + proc = subprocess2.Popen(cmd, **kwargs) + ret, err = proc.communicate(indata) + retcode = proc.wait() + if retcode not in accepted_retcodes: + raise subprocess2.CalledProcessError(retcode, cmd, os.getcwd(), ret, + err) - if autostrip: - ret = (ret or b'').strip() - err = (err or b'').strip() + if autostrip: + ret = (ret or b'').strip() + err = (err or b'').strip() - if decode: - ret = ret.decode('utf-8', 'replace') - err = err.decode('utf-8', 'replace') + if decode: + ret = ret.decode('utf-8', 'replace') + err = err.decode('utf-8', 'replace') - return ret, err + return ret, err def set_branch_config(branch, option, value, scope='local'): - set_config('branch.%s.%s' % (branch, option), value, scope=scope) + set_config('branch.%s.%s' % (branch, option), value, scope=scope) def set_config(option, value, scope='local'): - run('config', '--' + scope, option, value) + run('config', '--' + scope, option, value) def get_dirty_files(): - # Make sure index is up-to-date before running diff-index. - run_with_retcode('update-index', '--refresh', '-q') - return run('diff-index', '--ignore-submodules', '--name-status', 'HEAD') + # Make sure index is up-to-date before running diff-index. + run_with_retcode('update-index', '--refresh', '-q') + return run('diff-index', '--ignore-submodules', '--name-status', 'HEAD') def is_dirty_git_tree(cmd): - w = lambda s: sys.stderr.write(s+"\n") + w = lambda s: sys.stderr.write(s + "\n") - dirty = get_dirty_files() - if dirty: - w('Cannot %s with a dirty tree. Commit%s or stash your changes first.' % - (cmd, '' if cmd == 'upload' else ', freeze')) - w('Uncommitted files: (git diff-index --name-status HEAD)') - w(dirty[:4096]) - if len(dirty) > 4096: # pragma: no cover - w('... (run "git diff-index --name-status HEAD" to see full output).') - return True - return False + dirty = get_dirty_files() + if dirty: + w('Cannot %s with a dirty tree. Commit%s or stash your changes first.' % + (cmd, '' if cmd == 'upload' else ', freeze')) + w('Uncommitted files: (git diff-index --name-status HEAD)') + w(dirty[:4096]) + if len(dirty) > 4096: # pragma: no cover + w('... (run "git diff-index --name-status HEAD" to see full ' + 'output).') + return True + return False def status(ignore_submodules=None): - """Returns a parsed version of git-status. + """Returns a parsed version of git-status. Args: ignore_submodules (str|None): "all", "none", or None. @@ -916,86 +924,93 @@ def status(ignore_submodules=None): if lstat == 'R' """ - ignore_submodules = ignore_submodules or 'none' - assert ignore_submodules in ( - 'all', 'none'), f'ignore_submodules value {ignore_submodules} is invalid' + ignore_submodules = ignore_submodules or 'none' + assert ignore_submodules in ( + 'all', + 'none'), f'ignore_submodules value {ignore_submodules} is invalid' - stat_entry = collections.namedtuple('stat_entry', 'lstat rstat src') + stat_entry = collections.namedtuple('stat_entry', 'lstat rstat src') - def tokenizer(stream): - acc = BytesIO() - c = None - while c != b'': - c = stream.read(1) - if c in (None, b'', b'\0'): - if len(acc.getvalue()) > 0: - yield acc.getvalue() - acc = BytesIO() - else: - acc.write(c) + def tokenizer(stream): + acc = BytesIO() + c = None + while c != b'': + c = stream.read(1) + if c in (None, b'', b'\0'): + if len(acc.getvalue()) > 0: + yield acc.getvalue() + acc = BytesIO() + else: + acc.write(c) - def parser(tokens): - while True: - try: - status_dest = next(tokens).decode('utf-8') - except StopIteration: - return - stat, dest = status_dest[:2], status_dest[3:] - lstat, rstat = stat - if lstat == 'R': - src = next(tokens).decode('utf-8') - else: - src = dest - yield (dest, stat_entry(lstat, rstat, src)) + def parser(tokens): + while True: + try: + status_dest = next(tokens).decode('utf-8') + except StopIteration: + return + stat, dest = status_dest[:2], status_dest[3:] + lstat, rstat = stat + if lstat == 'R': + src = next(tokens).decode('utf-8') + else: + src = dest + yield (dest, stat_entry(lstat, rstat, src)) - return parser( - tokenizer( - run_stream('status', - '-z', - f'--ignore-submodules={ignore_submodules}', - bufsize=-1))) + return parser( + tokenizer( + run_stream('status', + '-z', + f'--ignore-submodules={ignore_submodules}', + bufsize=-1))) def squash_current_branch(header=None, merge_base=None): - header = header or 'git squash commit for %s.' % current_branch() - merge_base = merge_base or get_or_create_merge_base(current_branch()) - log_msg = header + '\n' - if log_msg: - log_msg += '\n' - log_msg += run('log', '--reverse', '--format=%H%n%B', '%s..HEAD' % merge_base) - run('reset', '--soft', merge_base) + header = header or 'git squash commit for %s.' % current_branch() + merge_base = merge_base or get_or_create_merge_base(current_branch()) + log_msg = header + '\n' + if log_msg: + log_msg += '\n' + log_msg += run('log', '--reverse', '--format=%H%n%B', + '%s..HEAD' % merge_base) + run('reset', '--soft', merge_base) - if not get_dirty_files(): - # Sometimes the squash can result in the same tree, meaning that there is - # nothing to commit at this point. - print('Nothing to commit; squashed branch is empty') - return False - run('commit', '--no-verify', '-a', '-F', '-', indata=log_msg.encode('utf-8')) - return True + if not get_dirty_files(): + # Sometimes the squash can result in the same tree, meaning that there + # is nothing to commit at this point. + print('Nothing to commit; squashed branch is empty') + return False + run('commit', + '--no-verify', + '-a', + '-F', + '-', + indata=log_msg.encode('utf-8')) + return True def tags(*args): - return run('tag', *args).splitlines() + return run('tag', *args).splitlines() def thaw(): - took_action = False - with run_stream('rev-list', 'HEAD') as stream: - for sha in stream: - sha = sha.strip().decode('utf-8') - msg = run('show', '--format=%f%b', '-s', 'HEAD') - match = FREEZE_MATCHER.match(msg) - if not match: - if not took_action: - return 'Nothing to thaw.' - break + took_action = False + with run_stream('rev-list', 'HEAD') as stream: + for sha in stream: + sha = sha.strip().decode('utf-8') + msg = run('show', '--format=%f%b', '-s', 'HEAD') + match = FREEZE_MATCHER.match(msg) + if not match: + if not took_action: + return 'Nothing to thaw.' + break - run('reset', '--' + FREEZE_SECTIONS[match.group(1)], sha) - took_action = True + run('reset', '--' + FREEZE_SECTIONS[match.group(1)], sha) + took_action = True def topo_iter(branch_tree, top_down=True): - """Generates (branch, parent) in topographical order for a branch tree. + """Generates (branch, parent) in topographical order for a branch tree. Given a tree: @@ -1018,34 +1033,34 @@ def topo_iter(branch_tree, top_down=True): if top_down is True, items are yielded from A->D. Otherwise they're yielded from D->A. Within a layer the branches will be yielded in sorted order. """ - branch_tree = branch_tree.copy() + branch_tree = branch_tree.copy() - # TODO(iannucci): There is probably a more efficient way to do these. - if top_down: - while branch_tree: - this_pass = [(b, p) for b, p in branch_tree.items() - if p not in branch_tree] - assert this_pass, "Branch tree has cycles: %r" % branch_tree - for branch, parent in sorted(this_pass): - yield branch, parent - del branch_tree[branch] - else: - parent_to_branches = collections.defaultdict(set) - for branch, parent in branch_tree.items(): - parent_to_branches[parent].add(branch) + # TODO(iannucci): There is probably a more efficient way to do these. + if top_down: + while branch_tree: + this_pass = [(b, p) for b, p in branch_tree.items() + if p not in branch_tree] + assert this_pass, "Branch tree has cycles: %r" % branch_tree + for branch, parent in sorted(this_pass): + yield branch, parent + del branch_tree[branch] + else: + parent_to_branches = collections.defaultdict(set) + for branch, parent in branch_tree.items(): + parent_to_branches[parent].add(branch) - while branch_tree: - this_pass = [(b, p) for b, p in branch_tree.items() - if not parent_to_branches[b]] - assert this_pass, "Branch tree has cycles: %r" % branch_tree - for branch, parent in sorted(this_pass): - yield branch, parent - parent_to_branches[parent].discard(branch) - del branch_tree[branch] + while branch_tree: + this_pass = [(b, p) for b, p in branch_tree.items() + if not parent_to_branches[b]] + assert this_pass, "Branch tree has cycles: %r" % branch_tree + for branch, parent in sorted(this_pass): + yield branch, parent + parent_to_branches[parent].discard(branch) + del branch_tree[branch] def tree(treeref, recurse=False): - """Returns a dict representation of a git tree object. + """Returns a dict representation of a git tree object. Args: treeref (str) - a git ref which resolves to a tree (commits count as trees). @@ -1067,122 +1082,129 @@ def tree(treeref, recurse=False): ref is the hex encoded hash of the entry. """ - ret = {} - opts = ['ls-tree', '--full-tree'] - if recurse: - opts.append('-r') - opts.append(treeref) - try: - for line in run(*opts).splitlines(): - mode, typ, ref, name = line.split(None, 3) - ret[name] = (mode, typ, ref) - except subprocess2.CalledProcessError: - return None - return ret + ret = {} + opts = ['ls-tree', '--full-tree'] + if recurse: + opts.append('-r') + opts.append(treeref) + try: + for line in run(*opts).splitlines(): + mode, typ, ref, name = line.split(None, 3) + ret[name] = (mode, typ, ref) + except subprocess2.CalledProcessError: + return None + return ret def get_remote_url(remote='origin'): - try: - return run('config', 'remote.%s.url' % remote) - except subprocess2.CalledProcessError: - return None + try: + return run('config', 'remote.%s.url' % remote) + except subprocess2.CalledProcessError: + return None def upstream(branch): - try: - return run('rev-parse', '--abbrev-ref', '--symbolic-full-name', - branch+'@{upstream}') - except subprocess2.CalledProcessError: - return None + try: + return run('rev-parse', '--abbrev-ref', '--symbolic-full-name', + branch + '@{upstream}') + except subprocess2.CalledProcessError: + return None def get_git_version(): - """Returns a tuple that contains the numeric components of the current git + """Returns a tuple that contains the numeric components of the current git version.""" - version_string = run('--version') - version_match = re.search(r'(\d+.)+(\d+)', version_string) - version = version_match.group() if version_match else '' + version_string = run('--version') + version_match = re.search(r'(\d+.)+(\d+)', version_string) + version = version_match.group() if version_match else '' - return tuple(int(x) for x in version.split('.')) + return tuple(int(x) for x in version.split('.')) def get_branches_info(include_tracking_status): - format_string = ( - '--format=%(refname:short):%(objectname:short):%(upstream:short):') + format_string = ( + '--format=%(refname:short):%(objectname:short):%(upstream:short):') - # This is not covered by the depot_tools CQ which only has git version 1.8. - if (include_tracking_status and - get_git_version() >= MIN_UPSTREAM_TRACK_GIT_VERSION): # pragma: no cover - format_string += '%(upstream:track)' + # This is not covered by the depot_tools CQ which only has git version 1.8. + if (include_tracking_status and get_git_version() >= + MIN_UPSTREAM_TRACK_GIT_VERSION): # pragma: no cover + format_string += '%(upstream:track)' - info_map = {} - data = run('for-each-ref', format_string, 'refs/heads') - BranchesInfo = collections.namedtuple( - 'BranchesInfo', 'hash upstream commits behind') - for line in data.splitlines(): - (branch, branch_hash, upstream_branch, tracking_status) = line.split(':') + info_map = {} + data = run('for-each-ref', format_string, 'refs/heads') + BranchesInfo = collections.namedtuple('BranchesInfo', + 'hash upstream commits behind') + for line in data.splitlines(): + (branch, branch_hash, upstream_branch, + tracking_status) = line.split(':') - commits = None - if include_tracking_status: - base = get_or_create_merge_base(branch) - if base: - commits_list = run('rev-list', '--count', branch, '^%s' % base, '--') - commits = int(commits_list) or None + commits = None + if include_tracking_status: + base = get_or_create_merge_base(branch) + if base: + commits_list = run('rev-list', '--count', branch, '^%s' % base, + '--') + commits = int(commits_list) or None - behind_match = re.search(r'behind (\d+)', tracking_status) - behind = int(behind_match.group(1)) if behind_match else None + behind_match = re.search(r'behind (\d+)', tracking_status) + behind = int(behind_match.group(1)) if behind_match else None - info_map[branch] = BranchesInfo( - hash=branch_hash, upstream=upstream_branch, commits=commits, - behind=behind) + info_map[branch] = BranchesInfo(hash=branch_hash, + upstream=upstream_branch, + commits=commits, + behind=behind) - # Set None for upstreams which are not branches (e.g empty upstream, remotes - # and deleted upstream branches). - missing_upstreams = {} - for info in info_map.values(): - if info.upstream not in info_map and info.upstream not in missing_upstreams: - missing_upstreams[info.upstream] = None + # Set None for upstreams which are not branches (e.g empty upstream, remotes + # and deleted upstream branches). + missing_upstreams = {} + for info in info_map.values(): + if (info.upstream not in info_map + and info.upstream not in missing_upstreams): + missing_upstreams[info.upstream] = None - result = info_map.copy() - result.update(missing_upstreams) - return result + result = info_map.copy() + result.update(missing_upstreams) + return result -def make_workdir_common(repository, new_workdir, files_to_symlink, - files_to_copy, symlink=None): - if not symlink: - symlink = os.symlink - os.makedirs(new_workdir) - for entry in files_to_symlink: - clone_file(repository, new_workdir, entry, symlink) - for entry in files_to_copy: - clone_file(repository, new_workdir, entry, shutil.copy) +def make_workdir_common(repository, + new_workdir, + files_to_symlink, + files_to_copy, + symlink=None): + if not symlink: + symlink = os.symlink + os.makedirs(new_workdir) + for entry in files_to_symlink: + clone_file(repository, new_workdir, entry, symlink) + for entry in files_to_copy: + clone_file(repository, new_workdir, entry, shutil.copy) def make_workdir(repository, new_workdir): - GIT_DIRECTORY_WHITELIST = [ - 'config', - 'info', - 'hooks', - 'logs/refs', - 'objects', - 'packed-refs', - 'refs', - 'remotes', - 'rr-cache', - 'shallow', - ] - make_workdir_common(repository, new_workdir, GIT_DIRECTORY_WHITELIST, - ['HEAD']) + GIT_DIRECTORY_WHITELIST = [ + 'config', + 'info', + 'hooks', + 'logs/refs', + 'objects', + 'packed-refs', + 'refs', + 'remotes', + 'rr-cache', + 'shallow', + ] + make_workdir_common(repository, new_workdir, GIT_DIRECTORY_WHITELIST, + ['HEAD']) def clone_file(repository, new_workdir, link, operation): - if not os.path.exists(os.path.join(repository, link)): - return - link_dir = os.path.dirname(os.path.join(new_workdir, link)) - if not os.path.exists(link_dir): - os.makedirs(link_dir) - src = os.path.join(repository, link) - if os.path.islink(src): - src = os.path.realpath(src) - operation(src, os.path.join(new_workdir, link)) + if not os.path.exists(os.path.join(repository, link)): + return + link_dir = os.path.dirname(os.path.join(new_workdir, link)) + if not os.path.exists(link_dir): + os.makedirs(link_dir) + src = os.path.join(repository, link) + if os.path.islink(src): + src = os.path.realpath(src) + operation(src, os.path.join(new_workdir, link)) diff --git a/git_dates.py b/git_dates.py index 41cf3acf98..140e267f5f 100644 --- a/git_dates.py +++ b/git_dates.py @@ -1,14 +1,13 @@ # Copyright 2016 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Utility module for dealing with Git timestamps.""" import datetime def timestamp_offset_to_datetime(timestamp, offset): - """Converts a timestamp + offset into a datetime.datetime. + """Converts a timestamp + offset into a datetime.datetime. Useful for dealing with the output of porcelain commands, which provide times as timestamp and offset strings. @@ -20,43 +19,43 @@ def timestamp_offset_to_datetime(timestamp, offset): Returns: A tz-aware datetime.datetime for this timestamp. """ - timestamp = int(timestamp) - tz = FixedOffsetTZ.from_offset_string(offset) - return datetime.datetime.fromtimestamp(timestamp, tz) + timestamp = int(timestamp) + tz = FixedOffsetTZ.from_offset_string(offset) + return datetime.datetime.fromtimestamp(timestamp, tz) def datetime_string(dt): - """Converts a tz-aware datetime.datetime into a string in git format.""" - return dt.strftime('%Y-%m-%d %H:%M:%S %z') + """Converts a tz-aware datetime.datetime into a string in git format.""" + return dt.strftime('%Y-%m-%d %H:%M:%S %z') # Adapted from: https://docs.python.org/2/library/datetime.html#tzinfo-objects class FixedOffsetTZ(datetime.tzinfo): - def __init__(self, offset, name): - datetime.tzinfo.__init__(self) - self.__offset = offset - self.__name = name + def __init__(self, offset, name): + datetime.tzinfo.__init__(self) + self.__offset = offset + self.__name = name - def __repr__(self): # pragma: no cover - return '{}({!r}, {!r})'.format(type(self).__name__, self.__offset, - self.__name) + def __repr__(self): # pragma: no cover + return '{}({!r}, {!r})'.format( + type(self).__name__, self.__offset, self.__name) - @classmethod - def from_offset_string(cls, offset): - try: - hours = int(offset[:-2]) - minutes = int(offset[-2:]) - except ValueError: - return cls(datetime.timedelta(0), 'UTC') + @classmethod + def from_offset_string(cls, offset): + try: + hours = int(offset[:-2]) + minutes = int(offset[-2:]) + except ValueError: + return cls(datetime.timedelta(0), 'UTC') - delta = datetime.timedelta(hours=hours, minutes=minutes) - return cls(delta, offset) + delta = datetime.timedelta(hours=hours, minutes=minutes) + return cls(delta, offset) - def utcoffset(self, dt): - return self.__offset + def utcoffset(self, dt): + return self.__offset - def tzname(self, dt): - return self.__name + def tzname(self, dt): + return self.__name - def dst(self, dt): - return datetime.timedelta(0) + def dst(self, dt): + return datetime.timedelta(0) diff --git a/git_drover.py b/git_drover.py index 7fb089763c..d0853074d7 100755 --- a/git_drover.py +++ b/git_drover.py @@ -5,7 +5,6 @@ import argparse - _HELP_MESSAGE = """\ git drover has been deprecated in favor of cherry-picking using Gerrit. Try it, it's faster! @@ -23,24 +22,26 @@ https://www.chromium.org/developers/how-tos/get-the-code/multiple-working-direct def main(): - parser = argparse.ArgumentParser(description=_HELP_MESSAGE) - parser.add_argument( - '--branch', - default='BRANCH', - metavar='BRANCH', - type=str, - help='the name of the branch to which to cherry-pick; e.g. 1234') - parser.add_argument( - '--cherry-pick', - default='HASH_OF_THE_COMMIT_TO_CHERRY_PICK', - metavar='HASH_OF_THE_COMMIT_TO_CHERRY_PICK', - type=str, - help=('the change to cherry-pick; this can be any string ' - 'that unambiguosly refers to a revision not involving HEAD')) - options, _ = parser.parse_known_args() + parser = argparse.ArgumentParser(description=_HELP_MESSAGE) + parser.add_argument( + '--branch', + default='BRANCH', + metavar='BRANCH', + type=str, + help='the name of the branch to which to cherry-pick; e.g. 1234') + parser.add_argument( + '--cherry-pick', + default='HASH_OF_THE_COMMIT_TO_CHERRY_PICK', + metavar='HASH_OF_THE_COMMIT_TO_CHERRY_PICK', + type=str, + help=('the change to cherry-pick; this can be any string ' + 'that unambiguosly refers to a revision not involving HEAD')) + options, _ = parser.parse_known_args() + + print( + _HELP_MESSAGE.format(branch=options.branch, + cherry_pick=options.cherry_pick)) - print(_HELP_MESSAGE.format( - branch=options.branch, cherry_pick=options.cherry_pick)) if __name__ == '__main__': - main() + main() diff --git a/git_find_releases.py b/git_find_releases.py index 64de921bd3..12670a242a 100755 --- a/git_find_releases.py +++ b/git_find_releases.py @@ -19,47 +19,51 @@ import git_common as git def GetNameForCommit(sha1): - name = re.sub(r'~.*$', '', git.run('name-rev', '--tags', '--name-only', sha1)) - if name == 'undefined': - name = git.run( - 'name-rev', '--refs', 'remotes/branch-heads/*', '--name-only', - sha1) + ' [untagged]' - return name + name = re.sub(r'~.*$', '', git.run('name-rev', '--tags', '--name-only', + sha1)) + if name == 'undefined': + name = git.run('name-rev', '--refs', 'remotes/branch-heads/*', + '--name-only', sha1) + ' [untagged]' + return name def GetMergesForCommit(sha1): - return [c.split()[0] for c in - git.run('log', '--oneline', '-F', '--all', '--no-abbrev', '--grep', - 'cherry picked from commit %s' % sha1).splitlines()] + return [ + c.split()[0] + for c in git.run('log', '--oneline', '-F', '--all', '--no-abbrev', + '--grep', 'cherry picked from commit %s' % + sha1).splitlines() + ] def main(args): - parser = optparse.OptionParser(usage=sys.modules[__name__].__doc__) - _, args = parser.parse_args(args) + parser = optparse.OptionParser(usage=sys.modules[__name__].__doc__) + _, args = parser.parse_args(args) - if len(args) == 0: - parser.error('Need at least one commit.') + if len(args) == 0: + parser.error('Need at least one commit.') - for arg in args: - commit_name = GetNameForCommit(arg) - if not commit_name: - print('%s not found' % arg) - return 1 - print('commit %s was:' % arg) - print(' initially in ' + commit_name) - merges = GetMergesForCommit(arg) - for merge in merges: - print(' merged to ' + GetNameForCommit(merge) + ' (as ' + merge + ')') - if not merges: - print('No merges found. If this seems wrong, be sure that you did:') - print(' git fetch origin && gclient sync --with_branch_heads') + for arg in args: + commit_name = GetNameForCommit(arg) + if not commit_name: + print('%s not found' % arg) + return 1 + print('commit %s was:' % arg) + print(' initially in ' + commit_name) + merges = GetMergesForCommit(arg) + for merge in merges: + print(' merged to ' + GetNameForCommit(merge) + ' (as ' + merge + + ')') + if not merges: + print('No merges found. If this seems wrong, be sure that you did:') + print(' git fetch origin && gclient sync --with_branch_heads') - return 0 + return 0 if __name__ == '__main__': - try: - sys.exit(main(sys.argv[1:])) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(1) + try: + sys.exit(main(sys.argv[1:])) + except KeyboardInterrupt: + sys.stderr.write('interrupted\n') + sys.exit(1) diff --git a/git_footers.py b/git_footers.py index b8c1db7d83..b70b248f0d 100755 --- a/git_footers.py +++ b/git_footers.py @@ -12,49 +12,48 @@ from collections import defaultdict import git_common as git - FOOTER_PATTERN = re.compile(r'^\s*([\w-]+): *(.*)$') CHROME_COMMIT_POSITION_PATTERN = re.compile(r'^([\w/\-\.]+)@{#(\d+)}$') FOOTER_KEY_BLOCKLIST = set(['http', 'https']) def normalize_name(header): - return '-'.join([ word.title() for word in header.strip().split('-') ]) + return '-'.join([word.title() for word in header.strip().split('-')]) def parse_footer(line): - """Returns footer's (key, value) if footer is valid, else None.""" - match = FOOTER_PATTERN.match(line) - if match and match.group(1) not in FOOTER_KEY_BLOCKLIST: - return (match.group(1), match.group(2)) - return None + """Returns footer's (key, value) if footer is valid, else None.""" + match = FOOTER_PATTERN.match(line) + if match and match.group(1) not in FOOTER_KEY_BLOCKLIST: + return (match.group(1), match.group(2)) + return None def parse_footers(message): - """Parses a git commit message into a multimap of footers.""" - _, _, parsed_footers = split_footers(message) - footer_map = defaultdict(list) - if parsed_footers: - # Read footers from bottom to top, because latter takes precedense, - # and we want it to be first in the multimap value. - for (k, v) in reversed(parsed_footers): - footer_map[normalize_name(k)].append(v.strip()) - return footer_map + """Parses a git commit message into a multimap of footers.""" + _, _, parsed_footers = split_footers(message) + footer_map = defaultdict(list) + if parsed_footers: + # Read footers from bottom to top, because latter takes precedense, + # and we want it to be first in the multimap value. + for (k, v) in reversed(parsed_footers): + footer_map[normalize_name(k)].append(v.strip()) + return footer_map def matches_footer_key(line, key): - """Returns whether line is a valid footer whose key matches a given one. + """Returns whether line is a valid footer whose key matches a given one. Keys are compared in normalized form. """ - r = parse_footer(line) - if r is None: - return False - return normalize_name(r[0]) == normalize_name(key) + r = parse_footer(line) + if r is None: + return False + return normalize_name(r[0]) == normalize_name(key) def split_footers(message): - """Returns (non_footer_lines, footer_lines, parsed footers). + """Returns (non_footer_lines, footer_lines, parsed footers). Guarantees that: (non_footer_lines + footer_lines) ~= message.splitlines(), with at @@ -63,57 +62,59 @@ def split_footers(message): There could be fewer parsed_footers than footer lines if some lines in last paragraph are malformed. """ - message_lines = list(message.rstrip().splitlines()) - footer_lines = [] - maybe_footer_lines = [] - for line in reversed(message_lines): - if line == '' or line.isspace(): - break - - if parse_footer(line): - footer_lines.extend(maybe_footer_lines) - maybe_footer_lines = [] - footer_lines.append(line) - else: - # We only want to include malformed lines if they are preceded by - # well-formed lines. So keep them in holding until we see a well-formed - # line (case above). - maybe_footer_lines.append(line) - else: - # The whole description was consisting of footers, - # which means those aren't footers. + message_lines = list(message.rstrip().splitlines()) footer_lines = [] + maybe_footer_lines = [] + for line in reversed(message_lines): + if line == '' or line.isspace(): + break - footer_lines.reverse() - footers = [footer for footer in map(parse_footer, footer_lines) if footer] - if not footers: - return message_lines, [], [] - if maybe_footer_lines: - # If some malformed lines were left over, add a newline to split them - # from the well-formed ones. - return message_lines[:-len(footer_lines)] + [''], footer_lines, footers - return message_lines[:-len(footer_lines)], footer_lines, footers + if parse_footer(line): + footer_lines.extend(maybe_footer_lines) + maybe_footer_lines = [] + footer_lines.append(line) + else: + # We only want to include malformed lines if they are preceded by + # well-formed lines. So keep them in holding until we see a + # well-formed line (case above). + maybe_footer_lines.append(line) + else: + # The whole description was consisting of footers, + # which means those aren't footers. + footer_lines = [] + + footer_lines.reverse() + footers = [footer for footer in map(parse_footer, footer_lines) if footer] + if not footers: + return message_lines, [], [] + if maybe_footer_lines: + # If some malformed lines were left over, add a newline to split them + # from the well-formed ones. + return message_lines[:-len(footer_lines)] + [''], footer_lines, footers + return message_lines[:-len(footer_lines)], footer_lines, footers def get_footer_change_id(message): - """Returns a list of Gerrit's ChangeId from given commit message.""" - return parse_footers(message).get(normalize_name('Change-Id'), []) + """Returns a list of Gerrit's ChangeId from given commit message.""" + return parse_footers(message).get(normalize_name('Change-Id'), []) def add_footer_change_id(message, change_id): - """Returns message with Change-ID footer in it. + """Returns message with Change-ID footer in it. Assumes that Change-Id is not yet in footers, which is then inserted at earliest footer line which is after all of these footers: Bug|Issue|Test|Feature. """ - assert 'Change-Id' not in parse_footers(message) - return add_footer(message, 'Change-Id', change_id, - after_keys=['Bug', 'Issue', 'Test', 'Feature']) + assert 'Change-Id' not in parse_footers(message) + return add_footer(message, + 'Change-Id', + change_id, + after_keys=['Bug', 'Issue', 'Test', 'Feature']) def add_footer(message, key, value, after_keys=None, before_keys=None): - """Returns a message with given footer appended. + """Returns a message with given footer appended. If after_keys and before_keys are both None (default), appends footer last. If after_keys is provided and matches footers already present, inserts footer @@ -127,66 +128,69 @@ def add_footer(message, key, value, after_keys=None, before_keys=None): after_keys=['Bug', 'Issue'] the new footer will be inserted between Bug and Verified-By existing footers. """ - assert key == normalize_name(key), 'Use normalized key' - new_footer = '%s: %s' % (key, value) - if not FOOTER_PATTERN.match(new_footer): - raise ValueError('Invalid footer %r' % new_footer) + assert key == normalize_name(key), 'Use normalized key' + new_footer = '%s: %s' % (key, value) + if not FOOTER_PATTERN.match(new_footer): + raise ValueError('Invalid footer %r' % new_footer) - top_lines, footer_lines, _ = split_footers(message) - if not footer_lines: - if not top_lines or top_lines[-1] != '': - top_lines.append('') - footer_lines = [new_footer] - else: - after_keys = set(map(normalize_name, after_keys or [])) - after_indices = [ - footer_lines.index(x) for x in footer_lines for k in after_keys - if matches_footer_key(x, k)] - before_keys = set(map(normalize_name, before_keys or [])) - before_indices = [ - footer_lines.index(x) for x in footer_lines for k in before_keys - if matches_footer_key(x, k)] - if after_indices: - # after_keys takes precedence, even if there's a conflict. - insert_idx = max(after_indices) + 1 - elif before_indices: - insert_idx = min(before_indices) + top_lines, footer_lines, _ = split_footers(message) + if not footer_lines: + if not top_lines or top_lines[-1] != '': + top_lines.append('') + footer_lines = [new_footer] else: - insert_idx = len(footer_lines) - footer_lines.insert(insert_idx, new_footer) - return '\n'.join(top_lines + footer_lines) + after_keys = set(map(normalize_name, after_keys or [])) + after_indices = [ + footer_lines.index(x) for x in footer_lines for k in after_keys + if matches_footer_key(x, k) + ] + before_keys = set(map(normalize_name, before_keys or [])) + before_indices = [ + footer_lines.index(x) for x in footer_lines for k in before_keys + if matches_footer_key(x, k) + ] + if after_indices: + # after_keys takes precedence, even if there's a conflict. + insert_idx = max(after_indices) + 1 + elif before_indices: + insert_idx = min(before_indices) + else: + insert_idx = len(footer_lines) + footer_lines.insert(insert_idx, new_footer) + return '\n'.join(top_lines + footer_lines) def remove_footer(message, key): - """Returns a message with all instances of given footer removed.""" - key = normalize_name(key) - top_lines, footer_lines, _ = split_footers(message) - if not footer_lines: - return message - new_footer_lines = [] - for line in footer_lines: - try: - f = normalize_name(parse_footer(line)[0]) - if f != key: - new_footer_lines.append(line) - except TypeError: - # If the footer doesn't parse (i.e. is malformed), just let it carry over. - new_footer_lines.append(line) - return '\n'.join(top_lines + new_footer_lines) + """Returns a message with all instances of given footer removed.""" + key = normalize_name(key) + top_lines, footer_lines, _ = split_footers(message) + if not footer_lines: + return message + new_footer_lines = [] + for line in footer_lines: + try: + f = normalize_name(parse_footer(line)[0]) + if f != key: + new_footer_lines.append(line) + except TypeError: + # If the footer doesn't parse (i.e. is malformed), just let it carry + # over. + new_footer_lines.append(line) + return '\n'.join(top_lines + new_footer_lines) def get_unique(footers, key): - key = normalize_name(key) - values = footers[key] - assert len(values) <= 1, 'Multiple %s footers' % key - if values: - return values[0] + key = normalize_name(key) + values = footers[key] + assert len(values) <= 1, 'Multiple %s footers' % key + if values: + return values[0] - return None + return None def get_position(footers): - """Get the commit position from the footers multimap using a heuristic. + """Get the commit position from the footers multimap using a heuristic. Returns: A tuple of the branch and the position on that branch. For example, @@ -196,65 +200,68 @@ def get_position(footers): would give the return value ('refs/heads/main', 292272). """ - position = get_unique(footers, 'Cr-Commit-Position') - if position: - match = CHROME_COMMIT_POSITION_PATTERN.match(position) - assert match, 'Invalid Cr-Commit-Position value: %s' % position - return (match.group(1), match.group(2)) + position = get_unique(footers, 'Cr-Commit-Position') + if position: + match = CHROME_COMMIT_POSITION_PATTERN.match(position) + assert match, 'Invalid Cr-Commit-Position value: %s' % position + return (match.group(1), match.group(2)) - raise ValueError('Unable to infer commit position from footers') + raise ValueError('Unable to infer commit position from footers') def main(args): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument('ref', nargs='?', help='Git ref to retrieve footers from.' - ' Omit to parse stdin.') + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('ref', + nargs='?', + help='Git ref to retrieve footers from.' + ' Omit to parse stdin.') - g = parser.add_mutually_exclusive_group() - g.add_argument('--key', metavar='KEY', - help='Get all values for the given footer name, one per ' - 'line (case insensitive)') - g.add_argument('--position', action='store_true') - g.add_argument('--position-ref', action='store_true') - g.add_argument('--position-num', action='store_true') - g.add_argument('--json', help='filename to dump JSON serialized footers to.') + g = parser.add_mutually_exclusive_group() + g.add_argument('--key', + metavar='KEY', + help='Get all values for the given footer name, one per ' + 'line (case insensitive)') + g.add_argument('--position', action='store_true') + g.add_argument('--position-ref', action='store_true') + g.add_argument('--position-num', action='store_true') + g.add_argument('--json', + help='filename to dump JSON serialized footers to.') - opts = parser.parse_args(args) + opts = parser.parse_args(args) - if opts.ref: - message = git.run('log', '-1', '--format=%B', opts.ref) - else: - message = sys.stdin.read() + if opts.ref: + message = git.run('log', '-1', '--format=%B', opts.ref) + else: + message = sys.stdin.read() - footers = parse_footers(message) + footers = parse_footers(message) - if opts.key: - for v in footers.get(normalize_name(opts.key), []): - print(v) - elif opts.position: - pos = get_position(footers) - print('%s@{#%s}' % (pos[0], pos[1] or '?')) - elif opts.position_ref: - print(get_position(footers)[0]) - elif opts.position_num: - pos = get_position(footers) - assert pos[1], 'No valid position for commit' - print(pos[1]) - elif opts.json: - with open(opts.json, 'w') as f: - json.dump(footers, f) - else: - for k in footers.keys(): - for v in footers[k]: - print('%s: %s' % (k, v)) - return 0 + if opts.key: + for v in footers.get(normalize_name(opts.key), []): + print(v) + elif opts.position: + pos = get_position(footers) + print('%s@{#%s}' % (pos[0], pos[1] or '?')) + elif opts.position_ref: + print(get_position(footers)[0]) + elif opts.position_num: + pos = get_position(footers) + assert pos[1], 'No valid position for commit' + print(pos[1]) + elif opts.json: + with open(opts.json, 'w') as f: + json.dump(footers, f) + else: + for k in footers.keys(): + for v in footers[k]: + print('%s: %s' % (k, v)) + return 0 if __name__ == '__main__': - try: - sys.exit(main(sys.argv[1:])) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(1) + try: + sys.exit(main(sys.argv[1:])) + except KeyboardInterrupt: + sys.stderr.write('interrupted\n') + sys.exit(1) diff --git a/git_freezer.py b/git_freezer.py index e4b1d8b162..cf8ec442cd 100755 --- a/git_freezer.py +++ b/git_freezer.py @@ -12,28 +12,28 @@ from git_common import freeze, thaw def CMDfreeze(parser, args): - """Freeze a branch's changes, excluding unstaged gitlinks changes.""" - parser.parse_args(args) - return freeze() + """Freeze a branch's changes, excluding unstaged gitlinks changes.""" + parser.parse_args(args) + return freeze() def CMDthaw(parser, args): - """Returns a frozen branch to the state before it was frozen.""" - parser.parse_args(args) - return thaw() + """Returns a frozen branch to the state before it was frozen.""" + parser.parse_args(args) + return thaw() def main(args): - dispatcher = subcommand.CommandDispatcher(__name__) - ret = dispatcher.execute(optparse.OptionParser(), args) - if ret: - print(ret) - return 0 + dispatcher = subcommand.CommandDispatcher(__name__) + ret = dispatcher.execute(optparse.OptionParser(), args) + if ret: + print(ret) + return 0 if __name__ == '__main__': - try: - sys.exit(main(sys.argv[1:])) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(1) + try: + sys.exit(main(sys.argv[1:])) + except KeyboardInterrupt: + sys.stderr.write('interrupted\n') + sys.exit(1) diff --git a/git_hyper_blame.py b/git_hyper_blame.py index 560a3bf34b..e335c788ca 100755 --- a/git_hyper_blame.py +++ b/git_hyper_blame.py @@ -2,7 +2,6 @@ # Copyright 2016 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Wrapper around git blame that ignores certain commits. """ @@ -17,135 +16,133 @@ import git_common import git_dates import setup_color - logging.getLogger().setLevel(logging.INFO) - DEFAULT_IGNORE_FILE_NAME = '.git-blame-ignore-revs' class Commit(object): - """Info about a commit.""" - def __init__(self, commithash): - self.commithash = commithash - self.author = None - self.author_mail = None - self.author_time = None - self.author_tz = None - self.committer = None - self.committer_mail = None - self.committer_time = None - self.committer_tz = None - self.summary = None - self.boundary = None - self.previous = None - self.filename = None + """Info about a commit.""" + def __init__(self, commithash): + self.commithash = commithash + self.author = None + self.author_mail = None + self.author_time = None + self.author_tz = None + self.committer = None + self.committer_mail = None + self.committer_time = None + self.committer_tz = None + self.summary = None + self.boundary = None + self.previous = None + self.filename = None - def __repr__(self): # pragma: no cover - return '' % self.commithash + def __repr__(self): # pragma: no cover + return '' % self.commithash BlameLine = collections.namedtuple( - 'BlameLine', - 'commit context lineno_then lineno_now modified') + 'BlameLine', 'commit context lineno_then lineno_now modified') def parse_blame(blameoutput): - """Parses the output of git blame -p into a data structure.""" - lines = blameoutput.split('\n') - i = 0 - commits = {} + """Parses the output of git blame -p into a data structure.""" + lines = blameoutput.split('\n') + i = 0 + commits = {} - while i < len(lines): - # Read a commit line and parse it. - line = lines[i] - i += 1 - if not line.strip(): - continue - commitline = line.split() - commithash = commitline[0] - lineno_then = int(commitline[1]) - lineno_now = int(commitline[2]) - - try: - commit = commits[commithash] - except KeyError: - commit = Commit(commithash) - commits[commithash] = commit - - # Read commit details until we find a context line. while i < len(lines): - line = lines[i] - i += 1 - if line.startswith('\t'): - break + # Read a commit line and parse it. + line = lines[i] + i += 1 + if not line.strip(): + continue + commitline = line.split() + commithash = commitline[0] + lineno_then = int(commitline[1]) + lineno_now = int(commitline[2]) - try: - key, value = line.split(' ', 1) - except ValueError: - key = line - value = True - setattr(commit, key.replace('-', '_'), value) + try: + commit = commits[commithash] + except KeyError: + commit = Commit(commithash) + commits[commithash] = commit - context = line[1:] + # Read commit details until we find a context line. + while i < len(lines): + line = lines[i] + i += 1 + if line.startswith('\t'): + break - yield BlameLine(commit, context, lineno_then, lineno_now, False) + try: + key, value = line.split(' ', 1) + except ValueError: + key = line + value = True + setattr(commit, key.replace('-', '_'), value) + + context = line[1:] + + yield BlameLine(commit, context, lineno_then, lineno_now, False) def print_table(outbuf, table, align): - """Print a 2D rectangular array, aligning columns with spaces. + """Print a 2D rectangular array, aligning columns with spaces. Args: align: string of 'l' and 'r', designating whether each column is left- or right-aligned. """ - if len(table) == 0: - return + if len(table) == 0: + return - colwidths = None - for row in table: - if colwidths is None: - colwidths = [len(x) for x in row] - else: - colwidths = [max(colwidths[i], len(x)) for i, x in enumerate(row)] + colwidths = None + for row in table: + if colwidths is None: + colwidths = [len(x) for x in row] + else: + colwidths = [max(colwidths[i], len(x)) for i, x in enumerate(row)] - for row in table: - cells = [] - for i, cell in enumerate(row): - padding = ' ' * (colwidths[i] - len(cell)) - if align[i] == 'r': - cell = padding + cell - elif i < len(row) - 1: - # Do not pad the final column if left-aligned. - cell += padding - cells.append(cell.encode('utf-8', 'replace')) - try: - outbuf.write(b' '.join(cells) + b'\n') - except IOError: # pragma: no cover - # Can happen on Windows if the pipe is closed early. - pass + for row in table: + cells = [] + for i, cell in enumerate(row): + padding = ' ' * (colwidths[i] - len(cell)) + if align[i] == 'r': + cell = padding + cell + elif i < len(row) - 1: + # Do not pad the final column if left-aligned. + cell += padding + cells.append(cell.encode('utf-8', 'replace')) + try: + outbuf.write(b' '.join(cells) + b'\n') + except IOError: # pragma: no cover + # Can happen on Windows if the pipe is closed early. + pass def pretty_print(outbuf, parsedblame, show_filenames=False): - """Pretty-prints the output of parse_blame.""" - table = [] - for line in parsedblame: - author_time = git_dates.timestamp_offset_to_datetime( - line.commit.author_time, line.commit.author_tz) - row = [line.commit.commithash[:8], - '(' + line.commit.author, - git_dates.datetime_string(author_time), - str(line.lineno_now) + ('*' if line.modified else '') + ')', - line.context] - if show_filenames: - row.insert(1, line.commit.filename) - table.append(row) - print_table(outbuf, table, align='llllrl' if show_filenames else 'lllrl') + """Pretty-prints the output of parse_blame.""" + table = [] + for line in parsedblame: + author_time = git_dates.timestamp_offset_to_datetime( + line.commit.author_time, line.commit.author_tz) + row = [ + line.commit.commithash[:8], '(' + line.commit.author, + git_dates.datetime_string(author_time), + str(line.lineno_now) + ('*' if line.modified else '') + ')', + line.context + ] + if show_filenames: + row.insert(1, line.commit.filename) + table.append(row) + print_table(outbuf, table, align='llllrl' if show_filenames else 'lllrl') def get_parsed_blame(filename, revision='HEAD'): - blame = git_common.blame(filename, revision=revision, porcelain=True) - return list(parse_blame(blame)) + blame = git_common.blame(filename, revision=revision, porcelain=True) + return list(parse_blame(blame)) # Map from (oldrev, newrev) to hunk list (caching the results of git diff, but @@ -156,41 +153,41 @@ diff_hunks_cache = {} def cache_diff_hunks(oldrev, newrev): - def parse_start_length(s): - # Chop the '-' or '+'. - s = s[1:] - # Length is optional (defaults to 1). + def parse_start_length(s): + # Chop the '-' or '+'. + s = s[1:] + # Length is optional (defaults to 1). + try: + start, length = s.split(',') + except ValueError: + start = s + length = 1 + return int(start), int(length) + try: - start, length = s.split(',') - except ValueError: - start = s - length = 1 - return int(start), int(length) + return diff_hunks_cache[(oldrev, newrev)] + except KeyError: + pass - try: - return diff_hunks_cache[(oldrev, newrev)] - except KeyError: - pass + # Use -U0 to get the smallest possible hunks. + diff = git_common.diff(oldrev, newrev, '-U0') - # Use -U0 to get the smallest possible hunks. - diff = git_common.diff(oldrev, newrev, '-U0') + # Get all the hunks. + hunks = [] + for line in diff.split('\n'): + if not line.startswith('@@'): + continue + ranges = line.split(' ', 3)[1:3] + ranges = tuple(parse_start_length(r) for r in ranges) + hunks.append(ranges) - # Get all the hunks. - hunks = [] - for line in diff.split('\n'): - if not line.startswith('@@'): - continue - ranges = line.split(' ', 3)[1:3] - ranges = tuple(parse_start_length(r) for r in ranges) - hunks.append(ranges) - - diff_hunks_cache[(oldrev, newrev)] = hunks - return hunks + diff_hunks_cache[(oldrev, newrev)] = hunks + return hunks def approx_lineno_across_revs(filename, newfilename, revision, newrevision, lineno): - """Computes the approximate movement of a line number between two revisions. + """Computes the approximate movement of a line number between two revisions. Consider line |lineno| in |filename| at |revision|. This function computes the line number of that line in |newfilename| at |newrevision|. This is @@ -206,183 +203,200 @@ def approx_lineno_across_revs(filename, newfilename, revision, newrevision, Returns: Line number within |newfilename| at |newrevision|. """ - # This doesn't work that well if there are a lot of line changes within the - # hunk (demonstrated by GitHyperBlameLineMotionTest.testIntraHunkLineMotion). - # A fuzzy heuristic that takes the text of the new line and tries to find a - # deleted line within the hunk that mostly matches the new line could help. + # This doesn't work that well if there are a lot of line changes within the + # hunk (demonstrated by + # GitHyperBlameLineMotionTest.testIntraHunkLineMotion). A fuzzy heuristic + # that takes the text of the new line and tries to find a deleted line + # within the hunk that mostly matches the new line could help. - # Use the : syntax to diff between two blobs. This is the - # only way to diff a file that has been renamed. - old = '%s:%s' % (revision, filename) - new = '%s:%s' % (newrevision, newfilename) - hunks = cache_diff_hunks(old, new) + # Use the : syntax to diff between two blobs. This is + # the only way to diff a file that has been renamed. + old = '%s:%s' % (revision, filename) + new = '%s:%s' % (newrevision, newfilename) + hunks = cache_diff_hunks(old, new) - cumulative_offset = 0 + cumulative_offset = 0 - # Find the hunk containing lineno (if any). - for (oldstart, oldlength), (newstart, newlength) in hunks: - cumulative_offset += newlength - oldlength + # Find the hunk containing lineno (if any). + for (oldstart, oldlength), (newstart, newlength) in hunks: + cumulative_offset += newlength - oldlength - if lineno >= oldstart + oldlength: - # Not there yet. - continue + if lineno >= oldstart + oldlength: + # Not there yet. + continue - if lineno < oldstart: - # Gone too far. - break + if lineno < oldstart: + # Gone too far. + break - # lineno is in [oldstart, oldlength] at revision; [newstart, newlength] at - # newrevision. + # lineno is in [oldstart, oldlength] at revision; [newstart, newlength] + # at newrevision. - # If newlength == 0, newstart will be the line before the deleted hunk. - # Since the line must have been deleted, just return that as the nearest - # line in the new file. Caution: newstart can be 0 in this case. - if newlength == 0: - return max(1, newstart) + # If newlength == 0, newstart will be the line before the deleted hunk. + # Since the line must have been deleted, just return that as the nearest + # line in the new file. Caution: newstart can be 0 in this case. + if newlength == 0: + return max(1, newstart) - newend = newstart + newlength - 1 + newend = newstart + newlength - 1 - # Move lineno based on the amount the entire hunk shifted. - lineno = lineno + newstart - oldstart - # Constrain the output within the range [newstart, newend]. - return min(newend, max(newstart, lineno)) + # Move lineno based on the amount the entire hunk shifted. + lineno = lineno + newstart - oldstart + # Constrain the output within the range [newstart, newend]. + return min(newend, max(newstart, lineno)) - # Wasn't in a hunk. Figure out the line motion based on the difference in - # length between the hunks seen so far. - return lineno + cumulative_offset + # Wasn't in a hunk. Figure out the line motion based on the difference in + # length between the hunks seen so far. + return lineno + cumulative_offset def hyper_blame(outbuf, ignored, filename, revision): - # Map from commit to parsed blame from that commit. - blame_from = {} - filename = os.path.normpath(filename) + # Map from commit to parsed blame from that commit. + blame_from = {} + filename = os.path.normpath(filename) + + def cache_blame_from(filename, commithash): + try: + return blame_from[commithash] + except KeyError: + parsed = get_parsed_blame(filename, commithash) + blame_from[commithash] = parsed + return parsed - def cache_blame_from(filename, commithash): try: - return blame_from[commithash] - except KeyError: - parsed = get_parsed_blame(filename, commithash) - blame_from[commithash] = parsed - return parsed + parsed = cache_blame_from(filename, git_common.hash_one(revision)) + except subprocess2.CalledProcessError as e: + sys.stderr.write(e.stderr.decode()) + return e.returncode - try: - parsed = cache_blame_from(filename, git_common.hash_one(revision)) - except subprocess2.CalledProcessError as e: - sys.stderr.write(e.stderr.decode()) - return e.returncode + new_parsed = [] - new_parsed = [] + # We don't show filenames in blame output unless we have to. + show_filenames = False - # We don't show filenames in blame output unless we have to. - show_filenames = False + for line in parsed: + # If a line references an ignored commit, blame that commit's parent + # repeatedly until we find a non-ignored commit. + while line.commit.commithash in ignored: + if line.commit.previous is None: + # You can't ignore the commit that added this file. + break - for line in parsed: - # If a line references an ignored commit, blame that commit's parent - # repeatedly until we find a non-ignored commit. - while line.commit.commithash in ignored: - if line.commit.previous is None: - # You can't ignore the commit that added this file. - break + previouscommit, previousfilename = line.commit.previous.split( + ' ', 1) + parent_blame = cache_blame_from(previousfilename, previouscommit) - previouscommit, previousfilename = line.commit.previous.split(' ', 1) - parent_blame = cache_blame_from(previousfilename, previouscommit) + if len(parent_blame) == 0: + # The previous version of this file was empty, therefore, you + # can't ignore this commit. + break - if len(parent_blame) == 0: - # The previous version of this file was empty, therefore, you can't - # ignore this commit. - break + # line.lineno_then is the line number in question at line.commit. We + # need to translate that line number so that it refers to the + # position of the same line on previouscommit. + lineno_previous = approx_lineno_across_revs(line.commit.filename, + previousfilename, + line.commit.commithash, + previouscommit, + line.lineno_then) + logging.debug('ignore commit %s on line p%d/t%d/n%d', + line.commit.commithash, lineno_previous, + line.lineno_then, line.lineno_now) - # line.lineno_then is the line number in question at line.commit. We need - # to translate that line number so that it refers to the position of the - # same line on previouscommit. - lineno_previous = approx_lineno_across_revs( - line.commit.filename, previousfilename, line.commit.commithash, - previouscommit, line.lineno_then) - logging.debug('ignore commit %s on line p%d/t%d/n%d', - line.commit.commithash, lineno_previous, line.lineno_then, - line.lineno_now) + # Get the line at lineno_previous in the parent commit. + assert 1 <= lineno_previous <= len(parent_blame) + newline = parent_blame[lineno_previous - 1] - # Get the line at lineno_previous in the parent commit. - assert 1 <= lineno_previous <= len(parent_blame) - newline = parent_blame[lineno_previous - 1] + # Replace the commit and lineno_then, but not the lineno_now or + # context. + line = BlameLine(newline.commit, line.context, newline.lineno_then, + line.lineno_now, True) + logging.debug(' replacing with %r', line) - # Replace the commit and lineno_then, but not the lineno_now or context. - line = BlameLine(newline.commit, line.context, newline.lineno_then, - line.lineno_now, True) - logging.debug(' replacing with %r', line) + # If any line has a different filename to the file's current name, turn + # on filename display for the entire blame output. Use normpath to make + # variable consistent across platforms. + if os.path.normpath(line.commit.filename) != filename: + show_filenames = True - # If any line has a different filename to the file's current name, turn on - # filename display for the entire blame output. - # Use normpath to make variable consistent across platforms. - if os.path.normpath(line.commit.filename) != filename: - show_filenames = True + new_parsed.append(line) - new_parsed.append(line) + pretty_print(outbuf, new_parsed, show_filenames=show_filenames) - pretty_print(outbuf, new_parsed, show_filenames=show_filenames) - - return 0 + return 0 def parse_ignore_file(ignore_file): - for line in ignore_file: - line = line.split('#', 1)[0].strip() - if line: - yield line + for line in ignore_file: + line = line.split('#', 1)[0].strip() + if line: + yield line def main(args, outbuf): - parser = argparse.ArgumentParser( - prog='git hyper-blame', - description='git blame with support for ignoring certain commits.') - parser.add_argument('-i', metavar='REVISION', action='append', dest='ignored', - default=[], help='a revision to ignore') - parser.add_argument('--ignore-file', metavar='FILE', dest='ignore_file', - help='a file containing a list of revisions to ignore') - parser.add_argument('--no-default-ignores', dest='no_default_ignores', - action='store_true', - help='Do not ignore commits from .git-blame-ignore-revs.') - parser.add_argument('revision', nargs='?', default='HEAD', metavar='REVISION', - help='revision to look at') - parser.add_argument('filename', metavar='FILE', help='filename to blame') + parser = argparse.ArgumentParser( + prog='git hyper-blame', + description='git blame with support for ignoring certain commits.') + parser.add_argument('-i', + metavar='REVISION', + action='append', + dest='ignored', + default=[], + help='a revision to ignore') + parser.add_argument('--ignore-file', + metavar='FILE', + dest='ignore_file', + help='a file containing a list of revisions to ignore') + parser.add_argument( + '--no-default-ignores', + dest='no_default_ignores', + action='store_true', + help='Do not ignore commits from .git-blame-ignore-revs.') + parser.add_argument('revision', + nargs='?', + default='HEAD', + metavar='REVISION', + help='revision to look at') + parser.add_argument('filename', metavar='FILE', help='filename to blame') - args = parser.parse_args(args) - try: - repo_root = git_common.repo_root() - except subprocess2.CalledProcessError as e: - sys.stderr.write(e.stderr.decode()) - return e.returncode - - # Make filename relative to the repository root, and cd to the root dir (so - # all filenames throughout this script are relative to the root). - filename = os.path.relpath(args.filename, repo_root) - os.chdir(repo_root) - - # Normalize filename so we can compare it to other filenames git gives us. - filename = os.path.normpath(filename) - filename = os.path.normcase(filename) - - ignored_list = list(args.ignored) - if not args.no_default_ignores and os.path.exists(DEFAULT_IGNORE_FILE_NAME): - with open(DEFAULT_IGNORE_FILE_NAME) as ignore_file: - ignored_list.extend(parse_ignore_file(ignore_file)) - - if args.ignore_file: - with open(args.ignore_file) as ignore_file: - ignored_list.extend(parse_ignore_file(ignore_file)) - - ignored = set() - for c in ignored_list: + args = parser.parse_args(args) try: - ignored.add(git_common.hash_one(c)) + repo_root = git_common.repo_root() except subprocess2.CalledProcessError as e: - # Custom warning string (the message from git-rev-parse is inappropriate). - sys.stderr.write('warning: unknown revision \'%s\'.\n' % c) + sys.stderr.write(e.stderr.decode()) + return e.returncode - return hyper_blame(outbuf, ignored, filename, args.revision) + # Make filename relative to the repository root, and cd to the root dir (so + # all filenames throughout this script are relative to the root). + filename = os.path.relpath(args.filename, repo_root) + os.chdir(repo_root) + + # Normalize filename so we can compare it to other filenames git gives us. + filename = os.path.normpath(filename) + filename = os.path.normcase(filename) + + ignored_list = list(args.ignored) + if not args.no_default_ignores and os.path.exists(DEFAULT_IGNORE_FILE_NAME): + with open(DEFAULT_IGNORE_FILE_NAME) as ignore_file: + ignored_list.extend(parse_ignore_file(ignore_file)) + + if args.ignore_file: + with open(args.ignore_file) as ignore_file: + ignored_list.extend(parse_ignore_file(ignore_file)) + + ignored = set() + for c in ignored_list: + try: + ignored.add(git_common.hash_one(c)) + except subprocess2.CalledProcessError as e: + # Custom warning string (the message from git-rev-parse is + # inappropriate). + sys.stderr.write('warning: unknown revision \'%s\'.\n' % c) + + return hyper_blame(outbuf, ignored, filename, args.revision) if __name__ == '__main__': # pragma: no cover - setup_color.init() - with git_common.less() as less_input: - sys.exit(main(sys.argv[1:], less_input)) + setup_color.init() + with git_common.less() as less_input: + sys.exit(main(sys.argv[1:], less_input)) diff --git a/git_map.py b/git_map.py index e693fbece9..8eba3e3daa 100755 --- a/git_map.py +++ b/git_map.py @@ -2,7 +2,6 @@ # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """ usage: git map [-h] [--help] [] @@ -26,7 +25,6 @@ import subprocess2 from third_party import colorama - RESET = colorama.Fore.RESET + colorama.Back.RESET + colorama.Style.RESET_ALL BRIGHT = colorama.Style.BRIGHT @@ -41,119 +39,122 @@ YELLOW = colorama.Fore.YELLOW def _print_help(outbuf): - names = { - 'Cyan': CYAN, - 'Green': GREEN, - 'Magenta': MAGENTA, - 'Red': RED, - 'White': WHITE, - 'Blue background': BLUE_BACK, - } - msg = '' - for line in __doc__.splitlines(): - for name, color in names.items(): - if name in line: - msg += line.replace('* ' + name, color + '* ' + name + RESET) + '\n' - break - else: - msg += line + '\n' - outbuf.write(msg.encode('utf-8', 'replace')) + names = { + 'Cyan': CYAN, + 'Green': GREEN, + 'Magenta': MAGENTA, + 'Red': RED, + 'White': WHITE, + 'Blue background': BLUE_BACK, + } + msg = '' + for line in __doc__.splitlines(): + for name, color in names.items(): + if name in line: + msg += line.replace('* ' + name, + color + '* ' + name + RESET) + '\n' + break + else: + msg += line + '\n' + outbuf.write(msg.encode('utf-8', 'replace')) def _color_branch(branch, all_branches, all_tags, current): - if branch in (current, 'HEAD -> ' + current): - color = CYAN - current = None - elif branch in all_branches: - color = GREEN - all_branches.remove(branch) - elif branch in all_tags: - color = MAGENTA - elif branch.startswith('tag: '): - color = MAGENTA - branch = branch[len('tag: '):] - else: - color = RED - return color + branch + RESET + if branch in (current, 'HEAD -> ' + current): + color = CYAN + current = None + elif branch in all_branches: + color = GREEN + all_branches.remove(branch) + elif branch in all_tags: + color = MAGENTA + elif branch.startswith('tag: '): + color = MAGENTA + branch = branch[len('tag: '):] + else: + color = RED + return color + branch + RESET def _color_branch_list(branch_list, all_branches, all_tags, current): - if not branch_list: - return '' - colored_branches = (GREEN + ', ').join( - _color_branch(branch, all_branches, all_tags, current) - for branch in branch_list if branch != 'HEAD') - return (GREEN + '(' + colored_branches + GREEN + ') ' + RESET) + if not branch_list: + return '' + colored_branches = (GREEN + ', ').join( + _color_branch(branch, all_branches, all_tags, current) + for branch in branch_list if branch != 'HEAD') + return (GREEN + '(' + colored_branches + GREEN + ') ' + RESET) def _parse_log_line(line): - graph, branch_list, commit_date, subject = ( - line.decode('utf-8', 'replace').strip().split('\x00')) - branch_list = [] if not branch_list else branch_list.split(', ') - commit = graph.split()[-1] - graph = graph[:-len(commit)] - return graph, commit, branch_list, commit_date, subject + graph, branch_list, commit_date, subject = (line.decode( + 'utf-8', 'replace').strip().split('\x00')) + branch_list = [] if not branch_list else branch_list.split(', ') + commit = graph.split()[-1] + graph = graph[:-len(commit)] + return graph, commit, branch_list, commit_date, subject def main(argv, outbuf): - if '-h' in argv or '--help' in argv: - _print_help(outbuf) + if '-h' in argv or '--help' in argv: + _print_help(outbuf) + return 0 + + map_extra = git_common.get_config_list('depot_tools.map_extra') + cmd = [ + git_common.GIT_EXE, 'log', + git_common.root(), '--graph', '--branches', '--tags', '--color=always', + '--date=short', '--pretty=format:%H%x00%D%x00%cd%x00%s' + ] + map_extra + argv + + log_proc = subprocess2.Popen(cmd, stdout=subprocess2.PIPE, shell=False) + + current = git_common.current_branch() + all_tags = set(git_common.tags()) + all_branches = set(git_common.branches()) + if current in all_branches: + all_branches.remove(current) + + merge_base_map = {} + for branch in all_branches: + merge_base = git_common.get_or_create_merge_base(branch) + if merge_base: + merge_base_map.setdefault(merge_base, set()).add(branch) + + for merge_base, branches in merge_base_map.items(): + merge_base_map[merge_base] = ', '.join(branches) + + try: + for line in log_proc.stdout: + if b'\x00' not in line: + outbuf.write(line) + continue + + graph, commit, branch_list, commit_date, subject = _parse_log_line( + line) + + if 'HEAD' in branch_list: + graph = graph.replace('*', BLUE_BACK + '*') + + line = '{graph}{commit}\t{branches}{date} ~ {subject}'.format( + graph=graph, + commit=BRIGHT_RED + commit[:10] + RESET, + branches=_color_branch_list(branch_list, all_branches, all_tags, + current), + date=YELLOW + commit_date + RESET, + subject=subject) + + if commit in merge_base_map: + line += ' <({})'.format(WHITE + merge_base_map[commit] + + RESET) + + line += os.linesep + outbuf.write(line.encode('utf-8', 'replace')) + except (BrokenPipeError, KeyboardInterrupt): + pass return 0 - map_extra = git_common.get_config_list('depot_tools.map_extra') - cmd = [ - git_common.GIT_EXE, 'log', git_common.root(), - '--graph', '--branches', '--tags', '--color=always', '--date=short', - '--pretty=format:%H%x00%D%x00%cd%x00%s' - ] + map_extra + argv - - log_proc = subprocess2.Popen(cmd, stdout=subprocess2.PIPE, shell=False) - - current = git_common.current_branch() - all_tags = set(git_common.tags()) - all_branches = set(git_common.branches()) - if current in all_branches: - all_branches.remove(current) - - merge_base_map = {} - for branch in all_branches: - merge_base = git_common.get_or_create_merge_base(branch) - if merge_base: - merge_base_map.setdefault(merge_base, set()).add(branch) - - for merge_base, branches in merge_base_map.items(): - merge_base_map[merge_base] = ', '.join(branches) - - try: - for line in log_proc.stdout: - if b'\x00' not in line: - outbuf.write(line) - continue - - graph, commit, branch_list, commit_date, subject = _parse_log_line(line) - - if 'HEAD' in branch_list: - graph = graph.replace('*', BLUE_BACK + '*') - - line = '{graph}{commit}\t{branches}{date} ~ {subject}'.format( - graph=graph, - commit=BRIGHT_RED + commit[:10] + RESET, - branches=_color_branch_list( - branch_list, all_branches, all_tags, current), - date=YELLOW + commit_date + RESET, - subject=subject) - - if commit in merge_base_map: - line += ' <({})'.format(WHITE + merge_base_map[commit] + RESET) - - line += os.linesep - outbuf.write(line.encode('utf-8', 'replace')) - except (BrokenPipeError, KeyboardInterrupt): - pass - return 0 - if __name__ == '__main__': - setup_color.init() - with git_common.less() as less_input: - sys.exit(main(sys.argv[1:], less_input)) + setup_color.init() + with git_common.less() as less_input: + sys.exit(main(sys.argv[1:], less_input)) diff --git a/git_map_branches.py b/git_map_branches.py index 2fb89a66c3..eb68074195 100755 --- a/git_map_branches.py +++ b/git_map_branches.py @@ -2,7 +2,6 @@ # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Print dependency tree of branches in local repo. Example: @@ -42,349 +41,364 @@ DEFAULT_SEPARATOR = ' ' * 4 class OutputManager(object): - """Manages a number of OutputLines and formats them into aligned columns.""" + """Manages a number of OutputLines and formats them into aligned columns.""" + def __init__(self): + self.lines = [] + self.nocolor = False + self.max_column_lengths = [] + self.num_columns = None - def __init__(self): - self.lines = [] - self.nocolor = False - self.max_column_lengths = [] - self.num_columns = None + def append(self, line): + # All lines must have the same number of columns. + if not self.num_columns: + self.num_columns = len(line.columns) + self.max_column_lengths = [0] * self.num_columns + assert self.num_columns == len(line.columns) - def append(self, line): - # All lines must have the same number of columns. - if not self.num_columns: - self.num_columns = len(line.columns) - self.max_column_lengths = [0] * self.num_columns - assert self.num_columns == len(line.columns) + if self.nocolor: + line.colors = [''] * self.num_columns - if self.nocolor: - line.colors = [''] * self.num_columns + self.lines.append(line) - self.lines.append(line) + # Update maximum column lengths. + for i, col in enumerate(line.columns): + self.max_column_lengths[i] = max(self.max_column_lengths[i], + len(col)) - # Update maximum column lengths. - for i, col in enumerate(line.columns): - self.max_column_lengths[i] = max(self.max_column_lengths[i], len(col)) + def merge(self, other): + for line in other.lines: + self.append(line) - def merge(self, other): - for line in other.lines: - self.append(line) - - def as_formatted_string(self): - return '\n'.join( - l.as_padded_string(self.max_column_lengths) for l in self.lines) + def as_formatted_string(self): + return '\n'.join( + l.as_padded_string(self.max_column_lengths) for l in self.lines) class OutputLine(object): - """A single line of data. + """A single line of data. This consists of an equal number of columns, colors and separators.""" + def __init__(self): + self.columns = [] + self.separators = [] + self.colors = [] - def __init__(self): - self.columns = [] - self.separators = [] - self.colors = [] + def append(self, data, separator=DEFAULT_SEPARATOR, color=Fore.WHITE): + self.columns.append(data) + self.separators.append(separator) + self.colors.append(color) - def append(self, data, separator=DEFAULT_SEPARATOR, color=Fore.WHITE): - self.columns.append(data) - self.separators.append(separator) - self.colors.append(color) - - def as_padded_string(self, max_column_lengths): - """"Returns the data as a string with each column padded to + def as_padded_string(self, max_column_lengths): + """"Returns the data as a string with each column padded to |max_column_lengths|.""" - output_string = '' - for i, (color, data, separator) in enumerate( - zip(self.colors, self.columns, self.separators)): - if max_column_lengths[i] == 0: - continue + output_string = '' + for i, (color, data, separator) in enumerate( + zip(self.colors, self.columns, self.separators)): + if max_column_lengths[i] == 0: + continue - padding = (max_column_lengths[i] - len(data)) * ' ' - output_string += color + data + padding + separator + padding = (max_column_lengths[i] - len(data)) * ' ' + output_string += color + data + padding + separator - return output_string.rstrip() + return output_string.rstrip() class BranchMapper(object): - """A class which constructs output representing the tree's branch structure. + """A class which constructs output representing the tree's branch structure. Attributes: __branches_info: a map of branches to their BranchesInfo objects which consist of the branch hash, upstream and ahead/behind status. __gone_branches: a set of upstreams which are not fetchable by git""" + def __init__(self): + self.verbosity = 0 + self.maxjobs = 0 + self.show_subject = False + self.hide_dormant = False + self.output = OutputManager() + self.__gone_branches = set() + self.__branches_info = None + self.__parent_map = collections.defaultdict(list) + self.__current_branch = None + self.__current_hash = None + self.__tag_set = None + self.__status_info = {} - def __init__(self): - self.verbosity = 0 - self.maxjobs = 0 - self.show_subject = False - self.hide_dormant = False - self.output = OutputManager() - self.__gone_branches = set() - self.__branches_info = None - self.__parent_map = collections.defaultdict(list) - self.__current_branch = None - self.__current_hash = None - self.__tag_set = None - self.__status_info = {} + def start(self): + self.__branches_info = get_branches_info( + include_tracking_status=self.verbosity >= 1) + if (self.verbosity >= 2): + # Avoid heavy import unless necessary. + from git_cl import get_cl_statuses, color_for_status, Changelist - def start(self): - self.__branches_info = get_branches_info( - include_tracking_status=self.verbosity >= 1) - if (self.verbosity >= 2): - # Avoid heavy import unless necessary. - from git_cl import get_cl_statuses, color_for_status, Changelist + change_cls = [ + Changelist(branchref='refs/heads/' + b) + for b in self.__branches_info.keys() if b + ] + status_info = get_cl_statuses(change_cls, + fine_grained=self.verbosity > 2, + max_processes=self.maxjobs) - change_cls = [Changelist(branchref='refs/heads/'+b) - for b in self.__branches_info.keys() if b] - status_info = get_cl_statuses(change_cls, - fine_grained=self.verbosity > 2, - max_processes=self.maxjobs) + # This is a blocking get which waits for the remote CL status to be + # retrieved. + for cl, status in status_info: + self.__status_info[cl.GetBranch()] = (cl.GetIssueURL( + short=True), color_for_status(status), status) - # This is a blocking get which waits for the remote CL status to be - # retrieved. - for cl, status in status_info: - self.__status_info[cl.GetBranch()] = (cl.GetIssueURL(short=True), - color_for_status(status), status) + roots = set() - roots = set() + # A map of parents to a list of their children. + for branch, branch_info in self.__branches_info.items(): + if not branch_info: + continue - # A map of parents to a list of their children. - for branch, branch_info in self.__branches_info.items(): - if not branch_info: - continue + parent = branch_info.upstream + if self.__check_cycle(branch): + continue + if not self.__branches_info[parent]: + branch_upstream = upstream(branch) + # If git can't find the upstream, mark the upstream as gone. + if branch_upstream: + parent = branch_upstream + else: + self.__gone_branches.add(parent) + # A parent that isn't in the branches info is a root. + roots.add(parent) - parent = branch_info.upstream - if self.__check_cycle(branch): - continue - if not self.__branches_info[parent]: - branch_upstream = upstream(branch) - # If git can't find the upstream, mark the upstream as gone. - if branch_upstream: - parent = branch_upstream + self.__parent_map[parent].append(branch) + + self.__current_branch = current_branch() + self.__current_hash = hash_one('HEAD', short=True) + self.__tag_set = tags() + + if roots: + for root in sorted(roots): + self.__append_branch(root, self.output) else: - self.__gone_branches.add(parent) - # A parent that isn't in the branches info is a root. - roots.add(parent) + no_branches = OutputLine() + no_branches.append('No User Branches') + self.output.append(no_branches) - self.__parent_map[parent].append(branch) + def __check_cycle(self, branch): + # Maximum length of the cycle is `num_branches`. This limit avoids + # running into a cycle which does *not* contain `branch`. + num_branches = len(self.__branches_info) + cycle = [branch] + while len(cycle) < num_branches and self.__branches_info[cycle[-1]]: + parent = self.__branches_info[cycle[-1]].upstream + cycle.append(parent) + if parent == branch: + print('Warning: Detected cycle in branches: {}'.format( + ' -> '.join(cycle)), + file=sys.stderr) + return True + return False - self.__current_branch = current_branch() - self.__current_hash = hash_one('HEAD', short=True) - self.__tag_set = tags() + def __is_invalid_parent(self, parent): + return not parent or parent in self.__gone_branches - if roots: - for root in sorted(roots): - self.__append_branch(root, self.output) - else: - no_branches = OutputLine() - no_branches.append('No User Branches') - self.output.append(no_branches) + def __color_for_branch(self, branch, branch_hash): + if branch.startswith('origin/'): + color = Fore.RED + elif branch.startswith('branch-heads'): + color = Fore.BLUE + elif self.__is_invalid_parent(branch) or branch in self.__tag_set: + color = Fore.MAGENTA + elif self.__current_hash.startswith(branch_hash): + color = Fore.CYAN + else: + color = Fore.GREEN - def __check_cycle(self, branch): - # Maximum length of the cycle is `num_branches`. This limit avoids running - # into a cycle which does *not* contain `branch`. - num_branches = len(self.__branches_info) - cycle = [branch] - while len(cycle) < num_branches and self.__branches_info[cycle[-1]]: - parent = self.__branches_info[cycle[-1]].upstream - cycle.append(parent) - if parent == branch: - print('Warning: Detected cycle in branches: {}'.format( - ' -> '.join(cycle)), file=sys.stderr) - return True - return False + if branch_hash and self.__current_hash.startswith(branch_hash): + color += Style.BRIGHT + else: + color += Style.NORMAL - def __is_invalid_parent(self, parent): - return not parent or parent in self.__gone_branches + return color - def __color_for_branch(self, branch, branch_hash): - if branch.startswith('origin/'): - color = Fore.RED - elif branch.startswith('branch-heads'): - color = Fore.BLUE - elif self.__is_invalid_parent(branch) or branch in self.__tag_set: - color = Fore.MAGENTA - elif self.__current_hash.startswith(branch_hash): - color = Fore.CYAN - else: - color = Fore.GREEN + def __is_dormant_branch(self, branch): + if '/' in branch: + return False - if branch_hash and self.__current_hash.startswith(branch_hash): - color += Style.BRIGHT - else: - color += Style.NORMAL + is_dormant = run('config', + '--get', + 'branch.{}.dormant'.format(branch), + accepted_retcodes=[0, 1]) + return is_dormant == 'true' - return color - - def __is_dormant_branch(self, branch): - if '/' in branch: - return False - - is_dormant = run('config', - '--get', - 'branch.{}.dormant'.format(branch), - accepted_retcodes=[0, 1]) - return is_dormant == 'true' - - def __append_branch(self, branch, output, depth=0): - """Recurses through the tree structure and appends an OutputLine to the + def __append_branch(self, branch, output, depth=0): + """Recurses through the tree structure and appends an OutputLine to the OutputManager for each branch.""" - child_output = OutputManager() - for child in sorted(self.__parent_map.pop(branch, ())): - self.__append_branch(child, child_output, depth=depth + 1) + child_output = OutputManager() + for child in sorted(self.__parent_map.pop(branch, ())): + self.__append_branch(child, child_output, depth=depth + 1) - is_dormant_branch = self.__is_dormant_branch(branch) - if self.hide_dormant and is_dormant_branch and not child_output.lines: - return + is_dormant_branch = self.__is_dormant_branch(branch) + if self.hide_dormant and is_dormant_branch and not child_output.lines: + return - branch_info = self.__branches_info[branch] - if branch_info: - branch_hash = branch_info.hash - else: - try: - branch_hash = hash_one(branch, short=True) - except subprocess2.CalledProcessError: - branch_hash = None + branch_info = self.__branches_info[branch] + if branch_info: + branch_hash = branch_info.hash + else: + try: + branch_hash = hash_one(branch, short=True) + except subprocess2.CalledProcessError: + branch_hash = None - line = OutputLine() + line = OutputLine() - # The branch name with appropriate indentation. - suffix = '' - if branch == self.__current_branch or ( - self.__current_branch == 'HEAD' and branch == self.__current_hash): - suffix = ' *' - branch_string = branch - if branch in self.__gone_branches: - branch_string = '{%s:GONE}' % branch - if not branch: - branch_string = '{NO_UPSTREAM}' - main_string = ' ' * depth + branch_string + suffix - line.append( - main_string, - color=self.__color_for_branch(branch, branch_hash)) + # The branch name with appropriate indentation. + suffix = '' + if branch == self.__current_branch or (self.__current_branch == 'HEAD' + and branch + == self.__current_hash): + suffix = ' *' + branch_string = branch + if branch in self.__gone_branches: + branch_string = '{%s:GONE}' % branch + if not branch: + branch_string = '{NO_UPSTREAM}' + main_string = ' ' * depth + branch_string + suffix + line.append(main_string, + color=self.__color_for_branch(branch, branch_hash)) - # The branch hash. - if self.verbosity >= 2: - line.append(branch_hash or '', separator=' ', color=Fore.RED) + # The branch hash. + if self.verbosity >= 2: + line.append(branch_hash or '', separator=' ', color=Fore.RED) - # The branch tracking status. - if self.verbosity >= 1: - commits_string = '' - behind_string = '' - front_separator = '' - center_separator = '' - back_separator = '' - if branch_info and not self.__is_invalid_parent(branch_info.upstream): - behind = branch_info.behind - commits = branch_info.commits + # The branch tracking status. + if self.verbosity >= 1: + commits_string = '' + behind_string = '' + front_separator = '' + center_separator = '' + back_separator = '' + if branch_info and not self.__is_invalid_parent( + branch_info.upstream): + behind = branch_info.behind + commits = branch_info.commits - if commits: - commits_string = '%d commit' % commits - commits_string += 's' if commits > 1 else ' ' - if behind: - behind_string = 'behind %d' % behind + if commits: + commits_string = '%d commit' % commits + commits_string += 's' if commits > 1 else ' ' + if behind: + behind_string = 'behind %d' % behind - if commits or behind: - front_separator = '[' - back_separator = ']' + if commits or behind: + front_separator = '[' + back_separator = ']' - if commits and behind: - center_separator = '|' + if commits and behind: + center_separator = '|' - line.append(front_separator, separator=' ') - line.append(commits_string, separator=' ', color=Fore.MAGENTA) - line.append(center_separator, separator=' ') - line.append(behind_string, separator=' ', color=Fore.MAGENTA) - line.append(back_separator) + line.append(front_separator, separator=' ') + line.append(commits_string, separator=' ', color=Fore.MAGENTA) + line.append(center_separator, separator=' ') + line.append(behind_string, separator=' ', color=Fore.MAGENTA) + line.append(back_separator) - if self.verbosity >= 4: - line.append(' (dormant)' if is_dormant_branch else ' ', - separator=' ', - color=Fore.RED) + if self.verbosity >= 4: + line.append(' (dormant)' if is_dormant_branch else ' ', + separator=' ', + color=Fore.RED) - # The Rietveld issue associated with the branch. - if self.verbosity >= 2: - (url, color, status) = ('', '', '') if self.__is_invalid_parent(branch) \ - else self.__status_info[branch] - if self.verbosity > 2: - line.append('{} ({})'.format(url, status) if url else '', color=color) - else: - line.append(url or '', color=color) + # The Rietveld issue associated with the branch. + if self.verbosity >= 2: + (url, color, + status) = (('', '', '') if self.__is_invalid_parent(branch) else + self.__status_info[branch]) + if self.verbosity > 2: + line.append('{} ({})'.format(url, status) if url else '', + color=color) + else: + line.append(url or '', color=color) - # The subject of the most recent commit on the branch. - if self.show_subject: - if not self.__is_invalid_parent(branch): - line.append(run('log', '-n1', '--format=%s', branch, '--')) - else: - line.append('') + # The subject of the most recent commit on the branch. + if self.show_subject: + if not self.__is_invalid_parent(branch): + line.append(run('log', '-n1', '--format=%s', branch, '--')) + else: + line.append('') - output.append(line) + output.append(line) - output.merge(child_output) + output.merge(child_output) def print_desc(): - for line in __doc__.splitlines(): - starpos = line.find('* ') - if starpos == -1 or '-' not in line: - print(line) - else: - _, color, rest = line.split(None, 2) - outline = line[:starpos+1] - outline += getattr(Fore, color.upper()) + " " + color + " " + Fore.RESET - outline += rest - print(outline) - print('') + for line in __doc__.splitlines(): + starpos = line.find('* ') + if starpos == -1 or '-' not in line: + print(line) + else: + _, color, rest = line.split(None, 2) + outline = line[:starpos + 1] + outline += getattr(Fore, + color.upper()) + " " + color + " " + Fore.RESET + outline += rest + print(outline) + print('') + @metrics.collector.collect_metrics('git map-branches') def main(argv): - setup_color.init() - if get_git_version() < MIN_UPSTREAM_TRACK_GIT_VERSION: - print( - 'This tool will not show all tracking information for git version ' - 'earlier than ' + - '.'.join(str(x) for x in MIN_UPSTREAM_TRACK_GIT_VERSION) + - '. Please consider upgrading.', file=sys.stderr) + setup_color.init() + if get_git_version() < MIN_UPSTREAM_TRACK_GIT_VERSION: + print( + 'This tool will not show all tracking information for git version ' + 'earlier than ' + + '.'.join(str(x) for x in MIN_UPSTREAM_TRACK_GIT_VERSION) + + '. Please consider upgrading.', + file=sys.stderr) - if '-h' in argv: - print_desc() + if '-h' in argv: + print_desc() - parser = argparse.ArgumentParser() - parser.add_argument('-v', - action='count', - default=0, - help=('Pass once to show tracking info, ' - 'twice for hash and review url, ' - 'thrice for review status, ' - 'four times to mark dormant branches')) - parser.add_argument('--no-color', action='store_true', dest='nocolor', - help='Turn off colors.') - parser.add_argument( - '-j', '--maxjobs', action='store', type=int, - help='The number of jobs to use when retrieving review status') - parser.add_argument('--show-subject', action='store_true', - dest='show_subject', help='Show the commit subject.') - parser.add_argument('--hide-dormant', - action='store_true', - dest='hide_dormant', - help='Hides dormant branches.') + parser = argparse.ArgumentParser() + parser.add_argument('-v', + action='count', + default=0, + help=('Pass once to show tracking info, ' + 'twice for hash and review url, ' + 'thrice for review status, ' + 'four times to mark dormant branches')) + parser.add_argument('--no-color', + action='store_true', + dest='nocolor', + help='Turn off colors.') + parser.add_argument( + '-j', + '--maxjobs', + action='store', + type=int, + help='The number of jobs to use when retrieving review status') + parser.add_argument('--show-subject', + action='store_true', + dest='show_subject', + help='Show the commit subject.') + parser.add_argument('--hide-dormant', + action='store_true', + dest='hide_dormant', + help='Hides dormant branches.') - opts = parser.parse_args(argv) + opts = parser.parse_args(argv) + + mapper = BranchMapper() + mapper.verbosity = opts.v + mapper.output.nocolor = opts.nocolor + mapper.maxjobs = opts.maxjobs + mapper.show_subject = opts.show_subject + mapper.hide_dormant = opts.hide_dormant + mapper.start() + print(mapper.output.as_formatted_string()) + return 0 - mapper = BranchMapper() - mapper.verbosity = opts.v - mapper.output.nocolor = opts.nocolor - mapper.maxjobs = opts.maxjobs - mapper.show_subject = opts.show_subject - mapper.hide_dormant = opts.hide_dormant - mapper.start() - print(mapper.output.as_formatted_string()) - return 0 if __name__ == '__main__': - try: - with metrics.collector.print_notice_and_exit(): - sys.exit(main(sys.argv[1:])) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(1) + try: + with metrics.collector.print_notice_and_exit(): + sys.exit(main(sys.argv[1:])) + except KeyboardInterrupt: + sys.stderr.write('interrupted\n') + sys.exit(1) diff --git a/git_mark_merge_base.py b/git_mark_merge_base.py index 2a45bae139..5ec6419198 100755 --- a/git_mark_merge_base.py +++ b/git_mark_merge_base.py @@ -2,7 +2,6 @@ # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """ Explicitly set/remove/print the merge-base for the current branch. @@ -21,51 +20,52 @@ from git_common import get_or_create_merge_base, hash_one, upstream def main(argv): - parser = argparse.ArgumentParser( - description=__doc__.strip().splitlines()[0], - epilog=' '.join(__doc__.strip().splitlines()[1:])) - g = parser.add_mutually_exclusive_group() - g.add_argument( - 'merge_base', nargs='?', - help='The new hash to use as the merge base for the current branch' - ) - g.add_argument('--delete', '-d', action='store_true', - help='Remove the set mark.') - opts = parser.parse_args(argv) + parser = argparse.ArgumentParser( + description=__doc__.strip().splitlines()[0], + epilog=' '.join(__doc__.strip().splitlines()[1:])) + g = parser.add_mutually_exclusive_group() + g.add_argument( + 'merge_base', + nargs='?', + help='The new hash to use as the merge base for the current branch') + g.add_argument('--delete', + '-d', + action='store_true', + help='Remove the set mark.') + opts = parser.parse_args(argv) - cur = current_branch() + cur = current_branch() - if opts.delete: - try: - remove_merge_base(cur) - except CalledProcessError: - print('No merge base currently exists for %s.' % cur) - return 0 + if opts.delete: + try: + remove_merge_base(cur) + except CalledProcessError: + print('No merge base currently exists for %s.' % cur) + return 0 - if opts.merge_base: - try: - opts.merge_base = hash_one(opts.merge_base) - except CalledProcessError: - print( - 'fatal: could not resolve %s as a commit' % opts.merge_base, - file=sys.stderr) - return 1 + if opts.merge_base: + try: + opts.merge_base = hash_one(opts.merge_base) + except CalledProcessError: + print('fatal: could not resolve %s as a commit' % opts.merge_base, + file=sys.stderr) + return 1 - manual_merge_base(cur, opts.merge_base, upstream(cur)) + manual_merge_base(cur, opts.merge_base, upstream(cur)) - ret = 0 - actual = get_or_create_merge_base(cur) - if opts.merge_base and opts.merge_base != actual: - ret = 1 - print("Invalid merge_base %s" % opts.merge_base) + ret = 0 + actual = get_or_create_merge_base(cur) + if opts.merge_base and opts.merge_base != actual: + ret = 1 + print("Invalid merge_base %s" % opts.merge_base) - print("merge_base(%s): %s" % (cur, actual)) - return ret + print("merge_base(%s): %s" % (cur, actual)) + return ret if __name__ == '__main__': - try: - sys.exit(main(sys.argv[1:])) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(1) + try: + sys.exit(main(sys.argv[1:])) + except KeyboardInterrupt: + sys.stderr.write('interrupted\n') + sys.exit(1) diff --git a/git_migrate_default_branch.py b/git_migrate_default_branch.py index 11ff4b611b..735f049cc4 100644 --- a/git_migrate_default_branch.py +++ b/git_migrate_default_branch.py @@ -15,88 +15,89 @@ import urllib.parse def GetGerritProject(remote_url): - """Returns Gerrit project name based on remote git URL.""" - if remote_url is None: - raise RuntimeError('can\'t detect Gerrit project.') - project = urllib.parse.urlparse(remote_url).path.strip('/') - if project.endswith('.git'): - project = project[:-len('.git')] - # *.googlesource.com hosts ensure that Git/Gerrit projects don't start with - # 'a/' prefix, because 'a/' prefix is used to force authentication in - # gitiles/git-over-https protocol. E.g., - # https://chromium.googlesource.com/a/v8/v8 refers to the same repo/project - # as - # https://chromium.googlesource.com/v8/v8 - if project.startswith('a/'): - project = project[len('a/'):] - return project + """Returns Gerrit project name based on remote git URL.""" + if remote_url is None: + raise RuntimeError('can\'t detect Gerrit project.') + project = urllib.parse.urlparse(remote_url).path.strip('/') + if project.endswith('.git'): + project = project[:-len('.git')] + # *.googlesource.com hosts ensure that Git/Gerrit projects don't start with + # 'a/' prefix, because 'a/' prefix is used to force authentication in + # gitiles/git-over-https protocol. E.g., + # https://chromium.googlesource.com/a/v8/v8 refers to the same repo/project + # as + # https://chromium.googlesource.com/v8/v8 + if project.startswith('a/'): + project = project[len('a/'):] + return project def GetGerritHost(git_host): - parts = git_host.split('.') - parts[0] = parts[0] + '-review' - return '.'.join(parts) + parts = git_host.split('.') + parts[0] = parts[0] + '-review' + return '.'.join(parts) def main(): - remote = git_common.run('remote') - # Use first remote as source of truth - remote = remote.split("\n")[0] - if not remote: - raise RuntimeError('Could not find any remote') - url = scm.GIT.GetConfig(git_common.repo_root(), 'remote.%s.url' % remote) - host = urllib.parse.urlparse(url).netloc - if not host: - raise RuntimeError('Could not find remote host') + remote = git_common.run('remote') + # Use first remote as source of truth + remote = remote.split("\n")[0] + if not remote: + raise RuntimeError('Could not find any remote') + url = scm.GIT.GetConfig(git_common.repo_root(), 'remote.%s.url' % remote) + host = urllib.parse.urlparse(url).netloc + if not host: + raise RuntimeError('Could not find remote host') - project_head = gerrit_util.GetProjectHead(GetGerritHost(host), - GetGerritProject(url)) - if project_head != 'refs/heads/main': - raise RuntimeError("The repository is not migrated yet.") + project_head = gerrit_util.GetProjectHead(GetGerritHost(host), + GetGerritProject(url)) + if project_head != 'refs/heads/main': + raise RuntimeError("The repository is not migrated yet.") - # User may have set to fetch only old default branch. Ensure fetch is tracking - # main too. - git_common.run('config', '--unset-all', - 'remote.origin.fetch', 'refs/heads/*') - git_common.run('config', '--add', - 'remote.origin.fetch', '+refs/heads/*:refs/remotes/origin/*') - logging.info("Running fetch...") - git_common.run('fetch', remote) - logging.info("Updating remote HEAD...") - git_common.run('remote', 'set-head', '-a', remote) + # User may have set to fetch only old default branch. Ensure fetch is + # tracking main too. + git_common.run('config', '--unset-all', 'remote.origin.fetch', + 'refs/heads/*') + git_common.run('config', '--add', 'remote.origin.fetch', + '+refs/heads/*:refs/remotes/origin/*') + logging.info("Running fetch...") + git_common.run('fetch', remote) + logging.info("Updating remote HEAD...") + git_common.run('remote', 'set-head', '-a', remote) - branches = git_common.get_branches_info(True) - - if 'master' in branches: - logging.info("Migrating master branch...") - if 'main' in branches: - logging.info('You already have master and main branch, consider removing ' - 'master manually:\n' - ' $ git branch -d master\n') - else: - git_common.run('branch', '-m', 'master', 'main') branches = git_common.get_branches_info(True) - for name in branches: - branch = branches[name] - if not branch: - continue + if 'master' in branches: + logging.info("Migrating master branch...") + if 'main' in branches: + logging.info( + 'You already have master and main branch, consider removing ' + 'master manually:\n' + ' $ git branch -d master\n') + else: + git_common.run('branch', '-m', 'master', 'main') + branches = git_common.get_branches_info(True) - if 'master' in branch.upstream: - logging.info("Migrating %s branch..." % name) - new_upstream = branch.upstream.replace('master', 'main') - git_common.run('branch', '--set-upstream-to', new_upstream, name) - git_common.remove_merge_base(name) + for name in branches: + branch = branches[name] + if not branch: + continue + + if 'master' in branch.upstream: + logging.info("Migrating %s branch..." % name) + new_upstream = branch.upstream.replace('master', 'main') + git_common.run('branch', '--set-upstream-to', new_upstream, name) + git_common.remove_merge_base(name) if __name__ == '__main__': - fix_encoding.fix_encoding() - logging.basicConfig(level=logging.INFO) - with metrics.collector.print_notice_and_exit(): - try: - logging.info("Starting migration") - main() - logging.info("Migration completed") - except RuntimeError as e: - logging.error("Error %s" % str(e)) - sys.exit(1) + fix_encoding.fix_encoding() + logging.basicConfig(level=logging.INFO) + with metrics.collector.print_notice_and_exit(): + try: + logging.info("Starting migration") + main() + logging.info("Migration completed") + except RuntimeError as e: + logging.error("Error %s" % str(e)) + sys.exit(1) diff --git a/git_nav_downstream.py b/git_nav_downstream.py index a378707a69..7834d8a77b 100755 --- a/git_nav_downstream.py +++ b/git_nav_downstream.py @@ -2,7 +2,6 @@ # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """ Checks out a downstream branch from the currently checked out branch. If there is more than one downstream branch, then this script will prompt you to select @@ -19,50 +18,53 @@ import metrics @metrics.collector.collect_metrics('git nav-downstream') def main(args): - parser = argparse.ArgumentParser() - parser.add_argument('--pick', - help=( - 'The number to pick if this command would ' - 'prompt')) - opts = parser.parse_args(args) + parser = argparse.ArgumentParser() + parser.add_argument('--pick', + help=('The number to pick if this command would ' + 'prompt')) + opts = parser.parse_args(args) - upfn = upstream - cur = current_branch() - if cur == 'HEAD': - def _upfn(b): - parent = upstream(b) - if parent: - return hash_one(parent) - upfn = _upfn - cur = hash_one(cur) - downstreams = [b for b in branches() if upfn(b) == cur] - if not downstreams: - print("No downstream branches") - return 1 + upfn = upstream + cur = current_branch() + if cur == 'HEAD': - if len(downstreams) == 1: - run('checkout', downstreams[0], stdout=sys.stdout, stderr=sys.stderr) - else: - high = len(downstreams) - 1 - while True: - print("Please select a downstream branch") - for i, b in enumerate(downstreams): - print(" %d. %s" % (i, b)) - prompt = "Selection (0-%d)[0]: " % high - r = opts.pick - if r: - print(prompt + r) - else: - r = gclient_utils.AskForData(prompt).strip() or '0' - if not r.isdigit() or (0 > int(r) > high): - print("Invalid choice.") - else: - run('checkout', downstreams[int(r)], stdout=sys.stdout, - stderr=sys.stderr) - break - return 0 + def _upfn(b): + parent = upstream(b) + if parent: + return hash_one(parent) + + upfn = _upfn + cur = hash_one(cur) + downstreams = [b for b in branches() if upfn(b) == cur] + if not downstreams: + print("No downstream branches") + return 1 + + if len(downstreams) == 1: + run('checkout', downstreams[0], stdout=sys.stdout, stderr=sys.stderr) + else: + high = len(downstreams) - 1 + while True: + print("Please select a downstream branch") + for i, b in enumerate(downstreams): + print(" %d. %s" % (i, b)) + prompt = "Selection (0-%d)[0]: " % high + r = opts.pick + if r: + print(prompt + r) + else: + r = gclient_utils.AskForData(prompt).strip() or '0' + if not r.isdigit() or (0 > int(r) > high): + print("Invalid choice.") + else: + run('checkout', + downstreams[int(r)], + stdout=sys.stdout, + stderr=sys.stderr) + break + return 0 if __name__ == '__main__': - with metrics.collector.print_notice_and_exit(): - sys.exit(main(sys.argv[1:])) + with metrics.collector.print_notice_and_exit(): + sys.exit(main(sys.argv[1:])) diff --git a/git_new_branch.py b/git_new_branch.py index bdf8bcf2ff..3087588c98 100755 --- a/git_new_branch.py +++ b/git_new_branch.py @@ -13,70 +13,78 @@ import git_common import subprocess2 -def create_new_branch( - branch_name, upstream_current=False, upstream=None, inject_current=False): - upstream = upstream or git_common.root() - try: - if inject_current: - below = git_common.current_branch() - if below is None: - raise Exception('no current branch') - above = git_common.upstream(below) - if above is None: - raise Exception('branch %s has no upstream' % (below)) - git_common.run('checkout', '--track', above, '-b', branch_name) - git_common.run('branch', '--set-upstream-to', branch_name, below) - elif upstream_current: - git_common.run('checkout', '--track', '-b', branch_name) - else: - if upstream in git_common.tags(): - # TODO(iannucci): ensure that basis_ref is an ancestor of HEAD? - git_common.run( - 'checkout', '--no-track', '-b', branch_name, - git_common.hash_one(upstream)) - git_common.set_config('branch.%s.remote' % branch_name, '.') - git_common.set_config('branch.%s.merge' % branch_name, upstream) - else: - # TODO(iannucci): Detect unclean workdir then stash+pop if we need to - # teleport to a conflicting portion of history? - git_common.run('checkout', '--track', upstream, '-b', branch_name) - git_common.get_or_create_merge_base(branch_name) - except subprocess2.CalledProcessError as cpe: - sys.stdout.write(cpe.stdout.decode('utf-8', 'replace')) - sys.stderr.write(cpe.stderr.decode('utf-8', 'replace')) - return 1 - sys.stderr.write('Switched to branch %s.\n' % branch_name) - return 0 +def create_new_branch(branch_name, + upstream_current=False, + upstream=None, + inject_current=False): + upstream = upstream or git_common.root() + try: + if inject_current: + below = git_common.current_branch() + if below is None: + raise Exception('no current branch') + above = git_common.upstream(below) + if above is None: + raise Exception('branch %s has no upstream' % (below)) + git_common.run('checkout', '--track', above, '-b', branch_name) + git_common.run('branch', '--set-upstream-to', branch_name, below) + elif upstream_current: + git_common.run('checkout', '--track', '-b', branch_name) + else: + if upstream in git_common.tags(): + # TODO(iannucci): ensure that basis_ref is an ancestor of HEAD? + git_common.run('checkout', '--no-track', '-b', branch_name, + git_common.hash_one(upstream)) + git_common.set_config('branch.%s.remote' % branch_name, '.') + git_common.set_config('branch.%s.merge' % branch_name, upstream) + else: + # TODO(iannucci): Detect unclean workdir then stash+pop if we + # need to teleport to a conflicting portion of history? + git_common.run('checkout', '--track', upstream, '-b', + branch_name) + git_common.get_or_create_merge_base(branch_name) + except subprocess2.CalledProcessError as cpe: + sys.stdout.write(cpe.stdout.decode('utf-8', 'replace')) + sys.stderr.write(cpe.stderr.decode('utf-8', 'replace')) + return 1 + sys.stderr.write('Switched to branch %s.\n' % branch_name) + return 0 + def main(args): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description=__doc__, - ) - parser.add_argument('branch_name') - g = parser.add_mutually_exclusive_group() - g.add_argument('--upstream-current', '--upstream_current', - action='store_true', - help='set upstream branch to current branch.') - g.add_argument('--upstream', metavar='REF', - help='upstream branch (or tag) to track.') - g.add_argument('--inject-current', '--inject_current', - action='store_true', - help='new branch adopts current branch\'s upstream,' + - ' and new branch becomes current branch\'s upstream.') - g.add_argument('--lkgr', action='store_const', const='lkgr', dest='upstream', - help='set basis ref for new branch to lkgr.') + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description=__doc__, + ) + parser.add_argument('branch_name') + g = parser.add_mutually_exclusive_group() + g.add_argument('--upstream-current', + '--upstream_current', + action='store_true', + help='set upstream branch to current branch.') + g.add_argument('--upstream', + metavar='REF', + help='upstream branch (or tag) to track.') + g.add_argument('--inject-current', + '--inject_current', + action='store_true', + help='new branch adopts current branch\'s upstream,' + + ' and new branch becomes current branch\'s upstream.') + g.add_argument('--lkgr', + action='store_const', + const='lkgr', + dest='upstream', + help='set basis ref for new branch to lkgr.') - opts = parser.parse_args(args) + opts = parser.parse_args(args) - return create_new_branch( - opts.branch_name, opts.upstream_current, opts.upstream, - opts.inject_current) + return create_new_branch(opts.branch_name, opts.upstream_current, + opts.upstream, opts.inject_current) if __name__ == '__main__': # pragma: no cover - try: - sys.exit(main(sys.argv[1:])) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(1) + try: + sys.exit(main(sys.argv[1:])) + except KeyboardInterrupt: + sys.stderr.write('interrupted\n') + sys.exit(1) diff --git a/git_number.py b/git_number.py index 864f3ee8cc..beae4ac7a6 100755 --- a/git_number.py +++ b/git_number.py @@ -2,7 +2,6 @@ # Copyright 2013 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Usage: %prog [options] []* If no 's are supplied, it defaults to HEAD. @@ -51,18 +50,18 @@ POOL_KIND = 'procs' def pathlify(hash_prefix): - """Converts a binary object hash prefix into a posix path, one folder per + """Converts a binary object hash prefix into a posix path, one folder per byte. >>> pathlify('\xDE\xAD') 'de/ad' """ - return '/'.join('%02x' % b for b in hash_prefix) + return '/'.join('%02x' % b for b in hash_prefix) @git.memoize_one(threadsafe=False) def get_number_tree(prefix_bytes): - """Returns a dictionary of the git-number registry specified by + """Returns a dictionary of the git-number registry specified by |prefix_bytes|. This is in the form of {: ...} @@ -70,36 +69,37 @@ def get_number_tree(prefix_bytes): >>> get_number_tree('\x83\xb4') {'\x83\xb4\xe3\xe4W\xf9J*\x8f/c\x16\xecD\xd1\x04\x8b\xa9qz': 169, ...} """ - ref = '%s:%s' % (REF, pathlify(prefix_bytes)) + ref = '%s:%s' % (REF, pathlify(prefix_bytes)) - try: - raw = git.run('cat-file', 'blob', ref, autostrip=False, decode=False) - return dict(struct.unpack_from(CHUNK_FMT, raw, i * CHUNK_SIZE) - for i in range(len(raw) // CHUNK_SIZE)) - except subprocess2.CalledProcessError: - return {} + try: + raw = git.run('cat-file', 'blob', ref, autostrip=False, decode=False) + return dict( + struct.unpack_from(CHUNK_FMT, raw, i * CHUNK_SIZE) + for i in range(len(raw) // CHUNK_SIZE)) + except subprocess2.CalledProcessError: + return {} @git.memoize_one(threadsafe=False) def get_num(commit_hash): - """Returns the generation number for a commit. + """Returns the generation number for a commit. Returns None if the generation number for this commit hasn't been calculated yet (see load_generation_numbers()). """ - return get_number_tree(commit_hash[:PREFIX_LEN]).get(commit_hash) + return get_number_tree(commit_hash[:PREFIX_LEN]).get(commit_hash) def clear_caches(on_disk=False): - """Clears in-process caches for e.g. unit testing.""" - get_number_tree.clear() - get_num.clear() - if on_disk: - git.run('update-ref', '-d', REF) + """Clears in-process caches for e.g. unit testing.""" + get_number_tree.clear() + get_num.clear() + if on_disk: + git.run('update-ref', '-d', REF) def intern_number_tree(tree): - """Transforms a number tree (in the form returned by |get_number_tree|) into + """Transforms a number tree (in the form returned by |get_number_tree|) into a git blob. Returns the git blob id as hex-encoded string. @@ -108,88 +108,95 @@ def intern_number_tree(tree): >>> intern_number_tree(d) 'c552317aa95ca8c3f6aae3357a4be299fbcb25ce' """ - with tempfile.TemporaryFile() as f: - for k, v in sorted(tree.items()): - f.write(struct.pack(CHUNK_FMT, k, v)) - f.seek(0) - return git.intern_f(f) + with tempfile.TemporaryFile() as f: + for k, v in sorted(tree.items()): + f.write(struct.pack(CHUNK_FMT, k, v)) + f.seek(0) + return git.intern_f(f) def leaf_map_fn(pre_tree): - """Converts a prefix and number tree into a git index line.""" - pre, tree = pre_tree - return '100644 blob %s\t%s\0' % (intern_number_tree(tree), pathlify(pre)) + """Converts a prefix and number tree into a git index line.""" + pre, tree = pre_tree + return '100644 blob %s\t%s\0' % (intern_number_tree(tree), pathlify(pre)) def finalize(targets): - """Saves all cache data to the git repository. + """Saves all cache data to the git repository. After calculating the generation number for |targets|, call finalize() to save all the work to the git repository. This in particular saves the trees referred to by DIRTY_TREES. """ - if not DIRTY_TREES: - return + if not DIRTY_TREES: + return - msg = 'git-number Added %s numbers' % sum(DIRTY_TREES.values()) + msg = 'git-number Added %s numbers' % sum(DIRTY_TREES.values()) - idx = os.path.join(git.run('rev-parse', '--git-dir'), 'number.idx') - env = os.environ.copy() - env['GIT_INDEX_FILE'] = str(idx) + idx = os.path.join(git.run('rev-parse', '--git-dir'), 'number.idx') + env = os.environ.copy() + env['GIT_INDEX_FILE'] = str(idx) - progress_message = 'Finalizing: (%%(count)d/%d)' % len(DIRTY_TREES) - with git.ProgressPrinter(progress_message) as inc: - git.run('read-tree', REF, env=env) + progress_message = 'Finalizing: (%%(count)d/%d)' % len(DIRTY_TREES) + with git.ProgressPrinter(progress_message) as inc: + git.run('read-tree', REF, env=env) - prefixes_trees = ((p, get_number_tree(p)) for p in sorted(DIRTY_TREES)) - updater = subprocess2.Popen(['git', 'update-index', '-z', '--index-info'], - stdin=subprocess2.PIPE, env=env) + prefixes_trees = ((p, get_number_tree(p)) for p in sorted(DIRTY_TREES)) + updater = subprocess2.Popen( + ['git', 'update-index', '-z', '--index-info'], + stdin=subprocess2.PIPE, + env=env) - with git.ScopedPool(kind=POOL_KIND) as leaf_pool: - for item in leaf_pool.imap(leaf_map_fn, prefixes_trees): - updater.stdin.write(item.encode()) - inc() + with git.ScopedPool(kind=POOL_KIND) as leaf_pool: + for item in leaf_pool.imap(leaf_map_fn, prefixes_trees): + updater.stdin.write(item.encode()) + inc() - updater.stdin.close() - updater.wait() - assert updater.returncode == 0 + updater.stdin.close() + updater.wait() + assert updater.returncode == 0 - tree_id = git.run('write-tree', env=env) - commit_cmd = [ - # Git user.name and/or user.email may not be configured, so specifying - # them explicitly. They are not used, but required by Git. - '-c', 'user.name=%s' % AUTHOR_NAME, - '-c', 'user.email=%s' % AUTHOR_EMAIL, - 'commit-tree', - '-m', msg, - '-p'] + git.hash_multi(REF) - for t in targets: - commit_cmd.extend(['-p', binascii.hexlify(t).decode()]) - commit_cmd.append(tree_id) - commit_hash = git.run(*commit_cmd) - git.run('update-ref', REF, commit_hash) - DIRTY_TREES.clear() + tree_id = git.run('write-tree', env=env) + commit_cmd = [ + # Git user.name and/or user.email may not be configured, so + # specifying them explicitly. They are not used, but required by + # Git. + '-c', + 'user.name=%s' % AUTHOR_NAME, + '-c', + 'user.email=%s' % AUTHOR_EMAIL, + 'commit-tree', + '-m', + msg, + '-p' + ] + git.hash_multi(REF) + for t in targets: + commit_cmd.extend(['-p', binascii.hexlify(t).decode()]) + commit_cmd.append(tree_id) + commit_hash = git.run(*commit_cmd) + git.run('update-ref', REF, commit_hash) + DIRTY_TREES.clear() def preload_tree(prefix): - """Returns the prefix and parsed tree object for the specified prefix.""" - return prefix, get_number_tree(prefix) + """Returns the prefix and parsed tree object for the specified prefix.""" + return prefix, get_number_tree(prefix) def all_prefixes(depth=PREFIX_LEN): - prefixes = [bytes([i]) for i in range(255)] - for x in prefixes: - # This isn't covered because PREFIX_LEN currently == 1 - if depth > 1: # pragma: no cover - for r in all_prefixes(depth - 1): - yield x + r - else: - yield x + prefixes = [bytes([i]) for i in range(255)] + for x in prefixes: + # This isn't covered because PREFIX_LEN currently == 1 + if depth > 1: # pragma: no cover + for r in all_prefixes(depth - 1): + yield x + r + else: + yield x def load_generation_numbers(targets): - """Populates the caches of get_num and get_number_tree so they contain + """Populates the caches of get_num and get_number_tree so they contain the results for |targets|. Loads cached numbers from disk, and calculates missing numbers if one or @@ -198,95 +205,109 @@ def load_generation_numbers(targets): Args: targets - An iterable of binary-encoded full git commit hashes. """ - # In case they pass us a generator, listify targets. - targets = list(targets) + # In case they pass us a generator, listify targets. + targets = list(targets) - if all(get_num(t) is not None for t in targets): - return + if all(get_num(t) is not None for t in targets): + return - if git.tree(REF) is None: - empty = git.mktree({}) - commit_hash = git.run( - # Git user.name and/or user.email may not be configured, so specifying - # them explicitly. They are not used, but required by Git. - '-c', 'user.name=%s' % AUTHOR_NAME, - '-c', 'user.email=%s' % AUTHOR_EMAIL, - 'commit-tree', - '-m', 'Initial commit from git-number', - empty) - git.run('update-ref', REF, commit_hash) + if git.tree(REF) is None: + empty = git.mktree({}) + commit_hash = git.run( + # Git user.name and/or user.email may not be configured, so + # specifying them explicitly. They are not used, but required by + # Git. + '-c', + 'user.name=%s' % AUTHOR_NAME, + '-c', + 'user.email=%s' % AUTHOR_EMAIL, + 'commit-tree', + '-m', + 'Initial commit from git-number', + empty) + git.run('update-ref', REF, commit_hash) - with git.ScopedPool(kind=POOL_KIND) as pool: - preload_iter = pool.imap_unordered(preload_tree, all_prefixes()) + with git.ScopedPool(kind=POOL_KIND) as pool: + preload_iter = pool.imap_unordered(preload_tree, all_prefixes()) - rev_list = [] + rev_list = [] - with git.ProgressPrinter('Loading commits: %(count)d') as inc: - # Curiously, buffering the list into memory seems to be the fastest - # approach in python (as opposed to iterating over the lines in the - # stdout as they're produced). GIL strikes again :/ - cmd = [ - 'rev-list', '--topo-order', '--parents', '--reverse', '^' + REF, - ] + [binascii.hexlify(target).decode() for target in targets] - for line in git.run(*cmd).splitlines(): - tokens = [binascii.unhexlify(token) for token in line.split()] - rev_list.append((tokens[0], tokens[1:])) - inc() + with git.ProgressPrinter('Loading commits: %(count)d') as inc: + # Curiously, buffering the list into memory seems to be the fastest + # approach in python (as opposed to iterating over the lines in the + # stdout as they're produced). GIL strikes again :/ + cmd = [ + 'rev-list', + '--topo-order', + '--parents', + '--reverse', + '^' + REF, + ] + [binascii.hexlify(target).decode() for target in targets] + for line in git.run(*cmd).splitlines(): + tokens = [binascii.unhexlify(token) for token in line.split()] + rev_list.append((tokens[0], tokens[1:])) + inc() - get_number_tree.update(preload_iter) + get_number_tree.update(preload_iter) - with git.ProgressPrinter('Counting: %%(count)d/%d' % len(rev_list)) as inc: - for commit_hash, pars in rev_list: - num = max(map(get_num, pars)) + 1 if pars else 0 + with git.ProgressPrinter('Counting: %%(count)d/%d' % len(rev_list)) as inc: + for commit_hash, pars in rev_list: + num = max(map(get_num, pars)) + 1 if pars else 0 - prefix = commit_hash[:PREFIX_LEN] - get_number_tree(prefix)[commit_hash] = num - DIRTY_TREES[prefix] += 1 - get_num.set(commit_hash, num) + prefix = commit_hash[:PREFIX_LEN] + get_number_tree(prefix)[commit_hash] = num + DIRTY_TREES[prefix] += 1 + get_num.set(commit_hash, num) - inc() + inc() def main(): # pragma: no cover - parser = optparse.OptionParser(usage=sys.modules[__name__].__doc__) - parser.add_option('--no-cache', action='store_true', - help='Do not actually cache anything we calculate.') - parser.add_option('--reset', action='store_true', - help='Reset the generation number cache and quit.') - parser.add_option('-v', '--verbose', action='count', default=0, - help='Be verbose. Use more times for more verbosity.') - opts, args = parser.parse_args() + parser = optparse.OptionParser(usage=sys.modules[__name__].__doc__) + parser.add_option('--no-cache', + action='store_true', + help='Do not actually cache anything we calculate.') + parser.add_option('--reset', + action='store_true', + help='Reset the generation number cache and quit.') + parser.add_option('-v', + '--verbose', + action='count', + default=0, + help='Be verbose. Use more times for more verbosity.') + opts, args = parser.parse_args() - levels = [logging.ERROR, logging.INFO, logging.DEBUG] - logging.basicConfig(level=levels[min(opts.verbose, len(levels) - 1)]) + levels = [logging.ERROR, logging.INFO, logging.DEBUG] + logging.basicConfig(level=levels[min(opts.verbose, len(levels) - 1)]) - # 'git number' should only be used on bots. - if os.getenv('CHROME_HEADLESS') != '1': - logging.error("'git-number' is an infrastructure tool that is only " - "intended to be used internally by bots. Developers should " - "use the 'Cr-Commit-Position' value in the commit's message.") - return 1 + # 'git number' should only be used on bots. + if os.getenv('CHROME_HEADLESS') != '1': + logging.error( + "'git-number' is an infrastructure tool that is only " + "intended to be used internally by bots. Developers should " + "use the 'Cr-Commit-Position' value in the commit's message.") + return 1 - if opts.reset: - clear_caches(on_disk=True) - return + if opts.reset: + clear_caches(on_disk=True) + return - try: - targets = git.parse_commitrefs(*(args or ['HEAD'])) - except git.BadCommitRefException as e: - parser.error(e) + try: + targets = git.parse_commitrefs(*(args or ['HEAD'])) + except git.BadCommitRefException as e: + parser.error(e) - load_generation_numbers(targets) - if not opts.no_cache: - finalize(targets) + load_generation_numbers(targets) + if not opts.no_cache: + finalize(targets) - print('\n'.join(map(str, map(get_num, targets)))) - return 0 + print('\n'.join(map(str, map(get_num, targets)))) + return 0 if __name__ == '__main__': # pragma: no cover - try: - sys.exit(main()) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(1) + try: + sys.exit(main()) + except KeyboardInterrupt: + sys.stderr.write('interrupted\n') + sys.exit(1) diff --git a/git_rebase_update.py b/git_rebase_update.py index 9c9881ab6a..13be0d6ab8 100755 --- a/git_rebase_update.py +++ b/git_rebase_update.py @@ -2,7 +2,6 @@ # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """ Tool to update all branches to have the latest changes from their upstreams. """ @@ -19,185 +18,192 @@ from pprint import pformat import git_common as git - STARTING_BRANCH_KEY = 'depot-tools.rebase-update.starting-branch' STARTING_WORKDIR_KEY = 'depot-tools.rebase-update.starting-workdir' def find_return_branch_workdir(): - """Finds the branch and working directory which we should return to after + """Finds the branch and working directory which we should return to after rebase-update completes. These values may persist across multiple invocations of rebase-update, if rebase-update runs into a conflict mid-way. """ - return_branch = git.get_config(STARTING_BRANCH_KEY) - workdir = git.get_config(STARTING_WORKDIR_KEY) - if not return_branch: - workdir = os.getcwd() - git.set_config(STARTING_WORKDIR_KEY, workdir) - return_branch = git.current_branch() - if return_branch != 'HEAD': - git.set_config(STARTING_BRANCH_KEY, return_branch) + return_branch = git.get_config(STARTING_BRANCH_KEY) + workdir = git.get_config(STARTING_WORKDIR_KEY) + if not return_branch: + workdir = os.getcwd() + git.set_config(STARTING_WORKDIR_KEY, workdir) + return_branch = git.current_branch() + if return_branch != 'HEAD': + git.set_config(STARTING_BRANCH_KEY, return_branch) - return return_branch, workdir + return return_branch, workdir def fetch_remotes(branch_tree): - """Fetches all remotes which are needed to update |branch_tree|.""" - fetch_tags = False - remotes = set() - tag_set = git.tags() - fetchspec_map = {} - all_fetchspec_configs = git.get_config_regexp(r'^remote\..*\.fetch') - for fetchspec_config in all_fetchspec_configs: - key, _, fetchspec = fetchspec_config.partition(' ') - dest_spec = fetchspec.partition(':')[2] - remote_name = key.split('.')[1] - fetchspec_map[dest_spec] = remote_name - for parent in branch_tree.values(): - if parent in tag_set: - fetch_tags = True + """Fetches all remotes which are needed to update |branch_tree|.""" + fetch_tags = False + remotes = set() + tag_set = git.tags() + fetchspec_map = {} + all_fetchspec_configs = git.get_config_regexp(r'^remote\..*\.fetch') + for fetchspec_config in all_fetchspec_configs: + key, _, fetchspec = fetchspec_config.partition(' ') + dest_spec = fetchspec.partition(':')[2] + remote_name = key.split('.')[1] + fetchspec_map[dest_spec] = remote_name + for parent in branch_tree.values(): + if parent in tag_set: + fetch_tags = True + else: + full_ref = git.run('rev-parse', '--symbolic-full-name', parent) + for dest_spec, remote_name in fetchspec_map.items(): + if fnmatch(full_ref, dest_spec): + remotes.add(remote_name) + break + + fetch_args = [] + if fetch_tags: + # Need to fetch all because we don't know what remote the tag comes from + # :( TODO(iannucci): assert that the tags are in the remote fetch + # refspec + fetch_args = ['--all'] else: - full_ref = git.run('rev-parse', '--symbolic-full-name', parent) - for dest_spec, remote_name in fetchspec_map.items(): - if fnmatch(full_ref, dest_spec): - remotes.add(remote_name) - break + fetch_args.append('--multiple') + fetch_args.extend(remotes) + # TODO(iannucci): Should we fetch git-svn? - fetch_args = [] - if fetch_tags: - # Need to fetch all because we don't know what remote the tag comes from :( - # TODO(iannucci): assert that the tags are in the remote fetch refspec - fetch_args = ['--all'] - else: - fetch_args.append('--multiple') - fetch_args.extend(remotes) - # TODO(iannucci): Should we fetch git-svn? - - if not fetch_args: # pragma: no cover - print('Nothing to fetch.') - else: - git.run_with_stderr('fetch', *fetch_args, stdout=sys.stdout, - stderr=sys.stderr) + if not fetch_args: # pragma: no cover + print('Nothing to fetch.') + else: + git.run_with_stderr('fetch', + *fetch_args, + stdout=sys.stdout, + stderr=sys.stderr) def remove_empty_branches(branch_tree): - tag_set = git.tags() - ensure_root_checkout = git.once(lambda: git.run('checkout', git.root())) + tag_set = git.tags() + ensure_root_checkout = git.once(lambda: git.run('checkout', git.root())) - deletions = {} - reparents = {} - downstreams = collections.defaultdict(list) - for branch, parent in git.topo_iter(branch_tree, top_down=False): - if git.is_dormant(branch): - continue + deletions = {} + reparents = {} + downstreams = collections.defaultdict(list) + for branch, parent in git.topo_iter(branch_tree, top_down=False): + if git.is_dormant(branch): + continue - downstreams[parent].append(branch) + downstreams[parent].append(branch) - # If branch and parent have the same tree, then branch has to be marked - # for deletion and its children and grand-children reparented to parent. - if git.hash_one(branch+":") == git.hash_one(parent+":"): - ensure_root_checkout() + # If branch and parent have the same tree, then branch has to be marked + # for deletion and its children and grand-children reparented to parent. + if git.hash_one(branch + ":") == git.hash_one(parent + ":"): + ensure_root_checkout() - logging.debug('branch %s merged to %s', branch, parent) + logging.debug('branch %s merged to %s', branch, parent) - # Mark branch for deletion while remembering the ordering, then add all - # its children as grand-children of its parent and record reparenting - # information if necessary. - deletions[branch] = len(deletions) + # Mark branch for deletion while remembering the ordering, then add + # all its children as grand-children of its parent and record + # reparenting information if necessary. + deletions[branch] = len(deletions) - for down in downstreams[branch]: - if down in deletions: - continue + for down in downstreams[branch]: + if down in deletions: + continue - # Record the new and old parent for down, or update such a record - # if it already exists. Keep track of the ordering so that reparenting - # happen in topological order. - downstreams[parent].append(down) - if down not in reparents: - reparents[down] = (len(reparents), parent, branch) + # Record the new and old parent for down, or update such a + # record if it already exists. Keep track of the ordering so + # that reparenting happen in topological order. + downstreams[parent].append(down) + if down not in reparents: + reparents[down] = (len(reparents), parent, branch) + else: + order, _, old_parent = reparents[down] + reparents[down] = (order, parent, old_parent) + + # Apply all reparenting recorded, in order. + for branch, value in sorted(reparents.items(), key=lambda x: x[1][0]): + _, parent, old_parent = value + if parent in tag_set: + git.set_branch_config(branch, 'remote', '.') + git.set_branch_config(branch, 'merge', 'refs/tags/%s' % parent) + print('Reparented %s to track %s [tag] (was tracking %s)' % + (branch, parent, old_parent)) else: - order, _, old_parent = reparents[down] - reparents[down] = (order, parent, old_parent) + git.run('branch', '--set-upstream-to', parent, branch) + print('Reparented %s to track %s (was tracking %s)' % + (branch, parent, old_parent)) - # Apply all reparenting recorded, in order. - for branch, value in sorted(reparents.items(), key=lambda x:x[1][0]): - _, parent, old_parent = value - if parent in tag_set: - git.set_branch_config(branch, 'remote', '.') - git.set_branch_config(branch, 'merge', 'refs/tags/%s' % parent) - print('Reparented %s to track %s [tag] (was tracking %s)' % - (branch, parent, old_parent)) - else: - git.run('branch', '--set-upstream-to', parent, branch) - print('Reparented %s to track %s (was tracking %s)' % (branch, parent, - old_parent)) - - # Apply all deletions recorded, in order. - for branch, _ in sorted(deletions.items(), key=lambda x: x[1]): - print(git.run('branch', '-d', branch)) + # Apply all deletions recorded, in order. + for branch, _ in sorted(deletions.items(), key=lambda x: x[1]): + print(git.run('branch', '-d', branch)) def rebase_branch(branch, parent, start_hash): - logging.debug('considering %s(%s) -> %s(%s) : %s', - branch, git.hash_one(branch), parent, git.hash_one(parent), - start_hash) + logging.debug('considering %s(%s) -> %s(%s) : %s', branch, + git.hash_one(branch), parent, git.hash_one(parent), + start_hash) - # If parent has FROZEN commits, don't base branch on top of them. Instead, - # base branch on top of whatever commit is before them. - back_ups = 0 - orig_parent = parent - while git.run('log', '-n1', '--format=%s', - parent, '--').startswith(git.FREEZE): - back_ups += 1 - parent = git.run('rev-parse', parent+'~') + # If parent has FROZEN commits, don't base branch on top of them. Instead, + # base branch on top of whatever commit is before them. + back_ups = 0 + orig_parent = parent + while git.run('log', '-n1', '--format=%s', parent, + '--').startswith(git.FREEZE): + back_ups += 1 + parent = git.run('rev-parse', parent + '~') - if back_ups: - logging.debug('Backed parent up by %d from %s to %s', - back_ups, orig_parent, parent) + if back_ups: + logging.debug('Backed parent up by %d from %s to %s', back_ups, + orig_parent, parent) - if git.hash_one(parent) != start_hash: - # Try a plain rebase first - print('Rebasing:', branch) - rebase_ret = git.rebase(parent, start_hash, branch, abort=True) - if not rebase_ret.success: - # TODO(iannucci): Find collapsible branches in a smarter way? - print("Failed! Attempting to squash", branch, "...", end=' ') - sys.stdout.flush() - squash_branch = branch+"_squash_attempt" - git.run('checkout', '-b', squash_branch) - git.squash_current_branch(merge_base=start_hash) + if git.hash_one(parent) != start_hash: + # Try a plain rebase first + print('Rebasing:', branch) + rebase_ret = git.rebase(parent, start_hash, branch, abort=True) + if not rebase_ret.success: + # TODO(iannucci): Find collapsible branches in a smarter way? + print("Failed! Attempting to squash", branch, "...", end=' ') + sys.stdout.flush() + squash_branch = branch + "_squash_attempt" + git.run('checkout', '-b', squash_branch) + git.squash_current_branch(merge_base=start_hash) - # Try to rebase the branch_squash_attempt branch to see if it's empty. - squash_ret = git.rebase(parent, start_hash, squash_branch, abort=True) - empty_rebase = git.hash_one(squash_branch) == git.hash_one(parent) - git.run('checkout', branch) - git.run('branch', '-D', squash_branch) - if squash_ret.success and empty_rebase: - print('Success!') - git.squash_current_branch(merge_base=start_hash) - git.rebase(parent, start_hash, branch) - else: - print("Failed!") - print() + # Try to rebase the branch_squash_attempt branch to see if it's + # empty. + squash_ret = git.rebase(parent, + start_hash, + squash_branch, + abort=True) + empty_rebase = git.hash_one(squash_branch) == git.hash_one(parent) + git.run('checkout', branch) + git.run('branch', '-D', squash_branch) + if squash_ret.success and empty_rebase: + print('Success!') + git.squash_current_branch(merge_base=start_hash) + git.rebase(parent, start_hash, branch) + else: + print("Failed!") + print() - # rebase and leave in mid-rebase state. - # This second rebase attempt should always fail in the same - # way that the first one does. If it magically succeeds then - # something very strange has happened. - second_rebase_ret = git.rebase(parent, start_hash, branch) - if second_rebase_ret.success: # pragma: no cover - print("Second rebase succeeded unexpectedly!") - print("Please see: http://crbug.com/425696") - print("First rebased failed with:") - print(rebase_ret.stderr) - else: - print("Here's what git-rebase (squashed) had to say:") - print() - print(squash_ret.stdout) - print(squash_ret.stderr) - print(textwrap.dedent("""\ + # rebase and leave in mid-rebase state. + # This second rebase attempt should always fail in the same + # way that the first one does. If it magically succeeds then + # something very strange has happened. + second_rebase_ret = git.rebase(parent, start_hash, branch) + if second_rebase_ret.success: # pragma: no cover + print("Second rebase succeeded unexpectedly!") + print("Please see: http://crbug.com/425696") + print("First rebased failed with:") + print(rebase_ret.stderr) + else: + print("Here's what git-rebase (squashed) had to say:") + print() + print(squash_ret.stdout) + print(squash_ret.stderr) + print( + textwrap.dedent("""\ Squashing failed. You probably have a real merge conflict. Your working copy is in mid-rebase. Either: @@ -208,147 +214,161 @@ def rebase_branch(branch, parent, start_hash): And then run `git rebase-update -n` to resume. """ % branch)) - return False - else: - print('%s up-to-date' % branch) + return False + else: + print('%s up-to-date' % branch) - git.remove_merge_base(branch) - git.get_or_create_merge_base(branch) + git.remove_merge_base(branch) + git.get_or_create_merge_base(branch) - return True + return True def main(args=None): - parser = argparse.ArgumentParser() - parser.add_argument('--verbose', '-v', action='store_true') - parser.add_argument('--keep-going', '-k', action='store_true', - help='Keep processing past failed rebases.') - parser.add_argument('--no_fetch', '--no-fetch', '-n', - action='store_true', - help='Skip fetching remotes.') - parser.add_argument( - '--current', action='store_true', help='Only rebase the current branch.') - parser.add_argument('branches', nargs='*', - help='Branches to be rebased. All branches are assumed ' - 'if none specified.') - parser.add_argument('--keep-empty', '-e', action='store_true', - help='Do not automatically delete empty branches.') - opts = parser.parse_args(args) + parser = argparse.ArgumentParser() + parser.add_argument('--verbose', '-v', action='store_true') + parser.add_argument('--keep-going', + '-k', + action='store_true', + help='Keep processing past failed rebases.') + parser.add_argument('--no_fetch', + '--no-fetch', + '-n', + action='store_true', + help='Skip fetching remotes.') + parser.add_argument('--current', + action='store_true', + help='Only rebase the current branch.') + parser.add_argument('branches', + nargs='*', + help='Branches to be rebased. All branches are assumed ' + 'if none specified.') + parser.add_argument('--keep-empty', + '-e', + action='store_true', + help='Do not automatically delete empty branches.') + opts = parser.parse_args(args) - if opts.verbose: # pragma: no cover - logging.getLogger().setLevel(logging.DEBUG) + if opts.verbose: # pragma: no cover + logging.getLogger().setLevel(logging.DEBUG) - # TODO(iannucci): snapshot all branches somehow, so we can implement - # `git rebase-update --undo`. - # * Perhaps just copy packed-refs + refs/ + logs/ to the side? - # * commit them to a secret ref? - # * Then we could view a summary of each run as a - # `diff --stat` on that secret ref. + # TODO(iannucci): snapshot all branches somehow, so we can implement + # `git rebase-update --undo`. + # * Perhaps just copy packed-refs + refs/ + logs/ to the side? + # * commit them to a secret ref? + # * Then we could view a summary of each run as a + # `diff --stat` on that secret ref. - if git.in_rebase(): - # TODO(iannucci): Be able to resume rebase with flags like --continue, - # etc. - print('Rebase in progress. Please complete the rebase before running ' - '`git rebase-update`.') - return 1 + if git.in_rebase(): + # TODO(iannucci): Be able to resume rebase with flags like --continue, + # etc. + print('Rebase in progress. Please complete the rebase before running ' + '`git rebase-update`.') + return 1 - return_branch, return_workdir = find_return_branch_workdir() - os.chdir(git.run('rev-parse', '--show-toplevel')) + return_branch, return_workdir = find_return_branch_workdir() + os.chdir(git.run('rev-parse', '--show-toplevel')) - if git.current_branch() == 'HEAD': - if git.run('status', '--porcelain', '--ignore-submodules=all'): - print('Cannot rebase-update with detached head + uncommitted changes.') - return 1 - else: - git.freeze() # just in case there are any local changes. - - branches_to_rebase = set(opts.branches) - if opts.current: - branches_to_rebase.add(git.current_branch()) - - skipped, branch_tree = git.get_branch_tree(use_limit=not opts.current) - if branches_to_rebase: - skipped = set(skipped).intersection(branches_to_rebase) - for branch in skipped: - print('Skipping %s: No upstream specified' % branch) - - if not opts.no_fetch: - fetch_remotes(branch_tree) - - merge_base = {} - for branch, parent in branch_tree.items(): - merge_base[branch] = git.get_or_create_merge_base(branch, parent) - - logging.debug('branch_tree: %s' % pformat(branch_tree)) - logging.debug('merge_base: %s' % pformat(merge_base)) - - retcode = 0 - unrebased_branches = [] - # Rebase each branch starting with the root-most branches and working - # towards the leaves. - for branch, parent in git.topo_iter(branch_tree): - # Only rebase specified branches, unless none specified. - if branches_to_rebase and branch not in branches_to_rebase: - continue - if git.is_dormant(branch): - print('Skipping dormant branch', branch) + if git.current_branch() == 'HEAD': + if git.run('status', '--porcelain', '--ignore-submodules=all'): + print( + 'Cannot rebase-update with detached head + uncommitted changes.' + ) + return 1 else: - ret = rebase_branch(branch, parent, merge_base[branch]) - if not ret: - retcode = 1 + git.freeze() # just in case there are any local changes. - if opts.keep_going: - print('--keep-going set, continuing with next branch.') - unrebased_branches.append(branch) - if git.in_rebase(): - git.run_with_retcode('rebase', '--abort') - if git.in_rebase(): # pragma: no cover - print('Failed to abort rebase. Something is really wrong.') - break + branches_to_rebase = set(opts.branches) + if opts.current: + branches_to_rebase.add(git.current_branch()) + + skipped, branch_tree = git.get_branch_tree(use_limit=not opts.current) + if branches_to_rebase: + skipped = set(skipped).intersection(branches_to_rebase) + for branch in skipped: + print('Skipping %s: No upstream specified' % branch) + + if not opts.no_fetch: + fetch_remotes(branch_tree) + + merge_base = {} + for branch, parent in branch_tree.items(): + merge_base[branch] = git.get_or_create_merge_base(branch, parent) + + logging.debug('branch_tree: %s' % pformat(branch_tree)) + logging.debug('merge_base: %s' % pformat(merge_base)) + + retcode = 0 + unrebased_branches = [] + # Rebase each branch starting with the root-most branches and working + # towards the leaves. + for branch, parent in git.topo_iter(branch_tree): + # Only rebase specified branches, unless none specified. + if branches_to_rebase and branch not in branches_to_rebase: + continue + if git.is_dormant(branch): + print('Skipping dormant branch', branch) else: - break + ret = rebase_branch(branch, parent, merge_base[branch]) + if not ret: + retcode = 1 - if unrebased_branches: - print() - print('The following branches could not be cleanly rebased:') - for branch in unrebased_branches: - print(' %s' % branch) + if opts.keep_going: + print('--keep-going set, continuing with next branch.') + unrebased_branches.append(branch) + if git.in_rebase(): + git.run_with_retcode('rebase', '--abort') + if git.in_rebase(): # pragma: no cover + print( + 'Failed to abort rebase. Something is really wrong.' + ) + break + else: + break - if not retcode: - if not opts.keep_empty: - remove_empty_branches(branch_tree) + if unrebased_branches: + print() + print('The following branches could not be cleanly rebased:') + for branch in unrebased_branches: + print(' %s' % branch) - # return_branch may not be there any more. - if return_branch in git.branches(use_limit=False): - git.run('checkout', return_branch) - git.thaw() - else: - root_branch = git.root() - if return_branch != 'HEAD': - print("%s was merged with its parent, checking out %s instead." % - (git.unicode_repr(return_branch), git.unicode_repr(root_branch))) - git.run('checkout', root_branch) + if not retcode: + if not opts.keep_empty: + remove_empty_branches(branch_tree) - # return_workdir may also not be there any more. - if return_workdir: - try: - os.chdir(return_workdir) - except OSError as e: - print( - "Unable to return to original workdir %r: %s" % (return_workdir, e)) - git.set_config(STARTING_BRANCH_KEY, '') - git.set_config(STARTING_WORKDIR_KEY, '') + # return_branch may not be there any more. + if return_branch in git.branches(use_limit=False): + git.run('checkout', return_branch) + git.thaw() + else: + root_branch = git.root() + if return_branch != 'HEAD': + print( + "%s was merged with its parent, checking out %s instead." % + (git.unicode_repr(return_branch), + git.unicode_repr(root_branch))) + git.run('checkout', root_branch) - print() - print("Running `git gc --auto` - Ctrl-C to abort is OK.") - git.run('gc', '--auto') + # return_workdir may also not be there any more. + if return_workdir: + try: + os.chdir(return_workdir) + except OSError as e: + print("Unable to return to original workdir %r: %s" % + (return_workdir, e)) + git.set_config(STARTING_BRANCH_KEY, '') + git.set_config(STARTING_WORKDIR_KEY, '') - return retcode + print() + print("Running `git gc --auto` - Ctrl-C to abort is OK.") + git.run('gc', '--auto') + + return retcode if __name__ == '__main__': # pragma: no cover - try: - sys.exit(main()) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(1) + try: + sys.exit(main()) + except KeyboardInterrupt: + sys.stderr.write('interrupted\n') + sys.exit(1) diff --git a/git_rename_branch.py b/git_rename_branch.py index 95094e85ea..a600eb2427 100755 --- a/git_rename_branch.py +++ b/git_rename_branch.py @@ -2,7 +2,6 @@ # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Rename the current branch while maintaining correct dependencies.""" import argparse @@ -13,43 +12,47 @@ import subprocess2 from git_common import current_branch, run, set_branch_config, branch_config from git_common import branch_config_map + def main(args): - current = current_branch() - if current == 'HEAD': - current = None - old_name_help = 'The old branch to rename.' - if current: - old_name_help += ' (default %(default)r)' + current = current_branch() + if current == 'HEAD': + current = None + old_name_help = 'The old branch to rename.' + if current: + old_name_help += ' (default %(default)r)' - parser = argparse.ArgumentParser() - parser.add_argument('old_name', nargs=('?' if current else 1), - help=old_name_help, default=current) - parser.add_argument('new_name', help='The new branch name.') + parser = argparse.ArgumentParser() + parser.add_argument('old_name', + nargs=('?' if current else 1), + help=old_name_help, + default=current) + parser.add_argument('new_name', help='The new branch name.') - opts = parser.parse_args(args) + opts = parser.parse_args(args) - # when nargs=1, we get a list :( - if isinstance(opts.old_name, list): - opts.old_name = opts.old_name[0] + # when nargs=1, we get a list :( + if isinstance(opts.old_name, list): + opts.old_name = opts.old_name[0] - try: - run('branch', '-m', opts.old_name, opts.new_name) + try: + run('branch', '-m', opts.old_name, opts.new_name) - # update the downstreams - for branch, merge in branch_config_map('merge').items(): - if merge == 'refs/heads/' + opts.old_name: - # Only care about local branches - if branch_config(branch, 'remote') == '.': - set_branch_config(branch, 'merge', 'refs/heads/' + opts.new_name) - except subprocess2.CalledProcessError as cpe: - sys.stderr.write(cpe.stderr.decode('utf-8', 'replace')) - return 1 - return 0 + # update the downstreams + for branch, merge in branch_config_map('merge').items(): + if merge == 'refs/heads/' + opts.old_name: + # Only care about local branches + if branch_config(branch, 'remote') == '.': + set_branch_config(branch, 'merge', + 'refs/heads/' + opts.new_name) + except subprocess2.CalledProcessError as cpe: + sys.stderr.write(cpe.stderr.decode('utf-8', 'replace')) + return 1 + return 0 if __name__ == '__main__': # pragma: no cover - try: - sys.exit(main(sys.argv[1:])) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(1) + try: + sys.exit(main(sys.argv[1:])) + except KeyboardInterrupt: + sys.stderr.write('interrupted\n') + sys.exit(1) diff --git a/git_reparent_branch.py b/git_reparent_branch.py index e84fdd74f1..3366a3ec3b 100755 --- a/git_reparent_branch.py +++ b/git_reparent_branch.py @@ -2,7 +2,6 @@ # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Change the upstream of the current branch.""" import argparse @@ -17,86 +16,90 @@ from git_common import get_branch_tree, topo_iter import git_rebase_update import metrics + @metrics.collector.collect_metrics('git reparent-branch') def main(args): - root_ref = root() + root_ref = root() - parser = argparse.ArgumentParser() - g = parser.add_mutually_exclusive_group() - g.add_argument('new_parent', nargs='?', - help='New parent branch (or tag) to reparent to.') - g.add_argument('--root', action='store_true', - help='Reparent to the configured root branch (%s).' % root_ref) - g.add_argument('--lkgr', action='store_true', - help='Reparent to the lkgr tag.') - opts = parser.parse_args(args) + parser = argparse.ArgumentParser() + g = parser.add_mutually_exclusive_group() + g.add_argument('new_parent', + nargs='?', + help='New parent branch (or tag) to reparent to.') + g.add_argument('--root', + action='store_true', + help='Reparent to the configured root branch (%s).' % + root_ref) + g.add_argument('--lkgr', + action='store_true', + help='Reparent to the lkgr tag.') + opts = parser.parse_args(args) - # TODO(iannucci): Allow specification of the branch-to-reparent + # TODO(iannucci): Allow specification of the branch-to-reparent - branch = current_branch() + branch = current_branch() - if opts.root: - new_parent = root_ref - elif opts.lkgr: - new_parent = 'lkgr' - else: - if not opts.new_parent: - parser.error('Must specify new parent somehow') - new_parent = opts.new_parent - cur_parent = upstream(branch) + if opts.root: + new_parent = root_ref + elif opts.lkgr: + new_parent = 'lkgr' + else: + if not opts.new_parent: + parser.error('Must specify new parent somehow') + new_parent = opts.new_parent + cur_parent = upstream(branch) - if branch == 'HEAD' or not branch: - parser.error('Must be on the branch you want to reparent') - if new_parent == cur_parent: - parser.error('Cannot reparent a branch to its existing parent') + if branch == 'HEAD' or not branch: + parser.error('Must be on the branch you want to reparent') + if new_parent == cur_parent: + parser.error('Cannot reparent a branch to its existing parent') - if not cur_parent: - msg = ( - "Unable to determine %s@{upstream}.\n\nThis can happen if you didn't use " - "`git new-branch` to create the branch and haven't used " - "`git branch --set-upstream-to` to assign it one.\n\nPlease assign an " - "upstream branch and then run this command again." - ) - print(msg % branch, file=sys.stderr) - return 1 + if not cur_parent: + msg = ( + "Unable to determine %s@{upstream}.\n\nThis can happen if you " + "didn't use `git new-branch` to create the branch and haven't used " + "`git branch --set-upstream-to` to assign it one.\n\nPlease assign " + "an upstream branch and then run this command again.") + print(msg % branch, file=sys.stderr) + return 1 - mbase = get_or_create_merge_base(branch, cur_parent) + mbase = get_or_create_merge_base(branch, cur_parent) - all_tags = tags() - if cur_parent in all_tags: - cur_parent += ' [tag]' + all_tags = tags() + if cur_parent in all_tags: + cur_parent += ' [tag]' - try: - run('show-ref', new_parent) - except subprocess2.CalledProcessError: - print('fatal: invalid reference: %s' % new_parent, file=sys.stderr) - return 1 + try: + run('show-ref', new_parent) + except subprocess2.CalledProcessError: + print('fatal: invalid reference: %s' % new_parent, file=sys.stderr) + return 1 - if new_parent in all_tags: - print("Reparenting %s to track %s [tag] (was %s)" % (branch, new_parent, - cur_parent)) - set_branch_config(branch, 'remote', '.') - set_branch_config(branch, 'merge', new_parent) - else: - print("Reparenting %s to track %s (was %s)" % (branch, new_parent, - cur_parent)) - run('branch', '--set-upstream-to', new_parent, branch) + if new_parent in all_tags: + print("Reparenting %s to track %s [tag] (was %s)" % + (branch, new_parent, cur_parent)) + set_branch_config(branch, 'remote', '.') + set_branch_config(branch, 'merge', new_parent) + else: + print("Reparenting %s to track %s (was %s)" % + (branch, new_parent, cur_parent)) + run('branch', '--set-upstream-to', new_parent, branch) - manual_merge_base(branch, mbase, new_parent) + manual_merge_base(branch, mbase, new_parent) - # ONLY rebase-update the branch which moved (and dependants) - _, branch_tree = get_branch_tree() - branches = [branch] - for branch, parent in topo_iter(branch_tree): - if parent in branches: - branches.append(branch) - return git_rebase_update.main(['--no-fetch', '--keep-empty'] + branches) + # ONLY rebase-update the branch which moved (and dependants) + _, branch_tree = get_branch_tree() + branches = [branch] + for branch, parent in topo_iter(branch_tree): + if parent in branches: + branches.append(branch) + return git_rebase_update.main(['--no-fetch', '--keep-empty'] + branches) if __name__ == '__main__': # pragma: no cover - try: - with metrics.collector.print_notice_and_exit(): - sys.exit(main(sys.argv[1:])) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(1) + try: + with metrics.collector.print_notice_and_exit(): + sys.exit(main(sys.argv[1:])) + except KeyboardInterrupt: + sys.stderr.write('interrupted\n') + sys.exit(1) diff --git a/git_retry.py b/git_retry.py index 1ae63f8abf..f8db8b518a 100755 --- a/git_retry.py +++ b/git_retry.py @@ -2,8 +2,6 @@ # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - - """Generic retry wrapper for Git operations. This is largely DEPRECATED in favor of the Infra Git wrapper: @@ -22,63 +20,62 @@ from git_common import GIT_EXE, GIT_TRANSIENT_ERRORS_RE class TeeThread(threading.Thread): + def __init__(self, fd, out_fd, name): + super(TeeThread, self).__init__(name='git-retry.tee.%s' % (name, )) + self.data = None + self.fd = fd + self.out_fd = out_fd - def __init__(self, fd, out_fd, name): - super(TeeThread, self).__init__(name='git-retry.tee.%s' % (name,)) - self.data = None - self.fd = fd - self.out_fd = out_fd - - def run(self): - chunks = [] - for line in self.fd: - line = line.decode('utf-8') - chunks.append(line) - self.out_fd.write(line) - self.data = ''.join(chunks) + def run(self): + chunks = [] + for line in self.fd: + line = line.decode('utf-8') + chunks.append(line) + self.out_fd.write(line) + self.data = ''.join(chunks) class GitRetry(object): - logger = logging.getLogger('git-retry') - DEFAULT_DELAY_SECS = 3.0 - DEFAULT_RETRY_COUNT = 5 + logger = logging.getLogger('git-retry') + DEFAULT_DELAY_SECS = 3.0 + DEFAULT_RETRY_COUNT = 5 - def __init__(self, retry_count=None, delay=None, delay_factor=None): - self.retry_count = retry_count or self.DEFAULT_RETRY_COUNT - self.delay = max(delay, 0) if delay else 0 - self.delay_factor = max(delay_factor, 0) if delay_factor else 0 + def __init__(self, retry_count=None, delay=None, delay_factor=None): + self.retry_count = retry_count or self.DEFAULT_RETRY_COUNT + self.delay = max(delay, 0) if delay else 0 + self.delay_factor = max(delay_factor, 0) if delay_factor else 0 - def shouldRetry(self, stderr): - m = GIT_TRANSIENT_ERRORS_RE.search(stderr) - if not m: - return False - self.logger.info("Encountered known transient error: [%s]", - stderr[m.start(): m.end()]) - return True + def shouldRetry(self, stderr): + m = GIT_TRANSIENT_ERRORS_RE.search(stderr) + if not m: + return False + self.logger.info("Encountered known transient error: [%s]", + stderr[m.start():m.end()]) + return True - @staticmethod - def execute(*args): - args = (GIT_EXE,) + args - proc = subprocess.Popen( - args, - stderr=subprocess.PIPE, - ) - stderr_tee = TeeThread(proc.stderr, sys.stderr, 'stderr') + @staticmethod + def execute(*args): + args = (GIT_EXE, ) + args + proc = subprocess.Popen( + args, + stderr=subprocess.PIPE, + ) + stderr_tee = TeeThread(proc.stderr, sys.stderr, 'stderr') - # Start our process. Collect/tee 'stdout' and 'stderr'. - stderr_tee.start() - try: - proc.wait() - except KeyboardInterrupt: - proc.kill() - raise - finally: - stderr_tee.join() - return proc.returncode, None, stderr_tee.data + # Start our process. Collect/tee 'stdout' and 'stderr'. + stderr_tee.start() + try: + proc.wait() + except KeyboardInterrupt: + proc.kill() + raise + finally: + stderr_tee.join() + return proc.returncode, None, stderr_tee.data - def computeDelay(self, iteration): - """Returns: the delay (in seconds) for a given iteration + def computeDelay(self, iteration): + """Returns: the delay (in seconds) for a given iteration The first iteration has a delay of '0'. @@ -86,97 +83,113 @@ class GitRetry(object): iteration: (int) The iteration index (starting with zero as the first iteration) """ - if (not self.delay) or (iteration == 0): - return 0 - if self.delay_factor == 0: - # Linear delay - return iteration * self.delay - # Exponential delay - return (self.delay_factor ** (iteration - 1)) * self.delay + if (not self.delay) or (iteration == 0): + return 0 + if self.delay_factor == 0: + # Linear delay + return iteration * self.delay + # Exponential delay + return (self.delay_factor**(iteration - 1)) * self.delay - def __call__(self, *args): - returncode = 0 - for i in range(self.retry_count): - # If the previous run failed and a delay is configured, delay before the - # next run. - delay = self.computeDelay(i) - if delay > 0: - self.logger.info("Delaying for [%s second(s)] until next retry", delay) - time.sleep(delay) + def __call__(self, *args): + returncode = 0 + for i in range(self.retry_count): + # If the previous run failed and a delay is configured, delay before + # the next run. + delay = self.computeDelay(i) + if delay > 0: + self.logger.info("Delaying for [%s second(s)] until next retry", + delay) + time.sleep(delay) - self.logger.debug("Executing subprocess (%d/%d) with arguments: %s", - (i+1), self.retry_count, args) - returncode, _, stderr = self.execute(*args) + self.logger.debug("Executing subprocess (%d/%d) with arguments: %s", + (i + 1), self.retry_count, args) + returncode, _, stderr = self.execute(*args) - self.logger.debug("Process terminated with return code: %d", returncode) - if returncode == 0: - break + self.logger.debug("Process terminated with return code: %d", + returncode) + if returncode == 0: + break - if not self.shouldRetry(stderr): - self.logger.error("Process failure was not known to be transient; " - "terminating with return code %d", returncode) - break - return returncode + if not self.shouldRetry(stderr): + self.logger.error( + "Process failure was not known to be transient; " + "terminating with return code %d", returncode) + break + return returncode def main(args): - # If we're using the Infra Git wrapper, do nothing here. - # https://chromium.googlesource.com/infra/infra/+/HEAD/go/src/infra/tools/git - if 'INFRA_GIT_WRAPPER' in os.environ: - # Remove Git's execution path from PATH so that our call-through re-invokes - # the Git wrapper. - # See crbug.com/721450 - env = os.environ.copy() - git_exec = subprocess.check_output([GIT_EXE, '--exec-path']).strip() - env['PATH'] = os.pathsep.join([ - elem for elem in env.get('PATH', '').split(os.pathsep) - if elem != git_exec]) - return subprocess.call([GIT_EXE] + args, env=env) + # If we're using the Infra Git wrapper, do nothing here. + # https://chromium.googlesource.com/infra/infra/+/HEAD/go/src/infra/tools/git + if 'INFRA_GIT_WRAPPER' in os.environ: + # Remove Git's execution path from PATH so that our call-through + # re-invokes the Git wrapper. See crbug.com/721450 + env = os.environ.copy() + git_exec = subprocess.check_output([GIT_EXE, '--exec-path']).strip() + env['PATH'] = os.pathsep.join([ + elem for elem in env.get('PATH', '').split(os.pathsep) + if elem != git_exec + ]) + return subprocess.call([GIT_EXE] + args, env=env) - parser = optparse.OptionParser() - parser.disable_interspersed_args() - parser.add_option('-v', '--verbose', - action='count', default=0, - help="Increase verbosity; can be specified multiple times") - parser.add_option('-c', '--retry-count', metavar='COUNT', - type=int, default=GitRetry.DEFAULT_RETRY_COUNT, - help="Number of times to retry (default=%default)") - parser.add_option('-d', '--delay', metavar='SECONDS', - type=float, default=GitRetry.DEFAULT_DELAY_SECS, - help="Specifies the amount of time (in seconds) to wait " - "between successive retries (default=%default). This " - "can be zero.") - parser.add_option('-D', '--delay-factor', metavar='FACTOR', - type=int, default=2, - help="The exponential factor to apply to delays in between " - "successive failures (default=%default). If this is " - "zero, delays will increase linearly. Set this to " - "one to have a constant (non-increasing) delay.") + parser = optparse.OptionParser() + parser.disable_interspersed_args() + parser.add_option( + '-v', + '--verbose', + action='count', + default=0, + help="Increase verbosity; can be specified multiple times") + parser.add_option('-c', + '--retry-count', + metavar='COUNT', + type=int, + default=GitRetry.DEFAULT_RETRY_COUNT, + help="Number of times to retry (default=%default)") + parser.add_option('-d', + '--delay', + metavar='SECONDS', + type=float, + default=GitRetry.DEFAULT_DELAY_SECS, + help="Specifies the amount of time (in seconds) to wait " + "between successive retries (default=%default). This " + "can be zero.") + parser.add_option( + '-D', + '--delay-factor', + metavar='FACTOR', + type=int, + default=2, + help="The exponential factor to apply to delays in between " + "successive failures (default=%default). If this is " + "zero, delays will increase linearly. Set this to " + "one to have a constant (non-increasing) delay.") - opts, args = parser.parse_args(args) + opts, args = parser.parse_args(args) - # Configure logging verbosity - if opts.verbose == 0: - logging.getLogger().setLevel(logging.WARNING) - elif opts.verbose == 1: - logging.getLogger().setLevel(logging.INFO) - else: - logging.getLogger().setLevel(logging.DEBUG) + # Configure logging verbosity + if opts.verbose == 0: + logging.getLogger().setLevel(logging.WARNING) + elif opts.verbose == 1: + logging.getLogger().setLevel(logging.INFO) + else: + logging.getLogger().setLevel(logging.DEBUG) - # Execute retries - retry = GitRetry( - retry_count=opts.retry_count, - delay=opts.delay, - delay_factor=opts.delay_factor, - ) - return retry(*args) + # Execute retries + retry = GitRetry( + retry_count=opts.retry_count, + delay=opts.delay, + delay_factor=opts.delay_factor, + ) + return retry(*args) if __name__ == '__main__': - logging.basicConfig() - logging.getLogger().setLevel(logging.WARNING) - try: - sys.exit(main(sys.argv[2:])) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(1) + logging.basicConfig() + logging.getLogger().setLevel(logging.WARNING) + try: + sys.exit(main(sys.argv[2:])) + except KeyboardInterrupt: + sys.stderr.write('interrupted\n') + sys.exit(1) diff --git a/git_squash_branch.py b/git_squash_branch.py index 9b9485ab07..7f54443855 100755 --- a/git_squash_branch.py +++ b/git_squash_branch.py @@ -8,21 +8,25 @@ import sys import git_common + def main(args): - parser = argparse.ArgumentParser() - parser.add_argument( - '-m', '--message', metavar='', default=None, - help='Use the given as the first line of the commit message.') - opts = parser.parse_args(args) - if git_common.is_dirty_git_tree('squash-branch'): - return 1 - git_common.squash_current_branch(opts.message) - return 0 + parser = argparse.ArgumentParser() + parser.add_argument( + '-m', + '--message', + metavar='', + default=None, + help='Use the given as the first line of the commit message.') + opts = parser.parse_args(args) + if git_common.is_dirty_git_tree('squash-branch'): + return 1 + git_common.squash_current_branch(opts.message) + return 0 if __name__ == '__main__': - try: - sys.exit(main(sys.argv[1:])) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(1) + try: + sys.exit(main(sys.argv[1:])) + except KeyboardInterrupt: + sys.stderr.write('interrupted\n') + sys.exit(1) diff --git a/git_upstream_diff.py b/git_upstream_diff.py index 4fd9da5cc3..8074e38c9e 100755 --- a/git_upstream_diff.py +++ b/git_upstream_diff.py @@ -10,53 +10,58 @@ import subprocess2 import git_common as git + def main(args): - default_args = git.get_config_list('depot-tools.upstream-diff.default-args') - args = default_args + args + default_args = git.get_config_list('depot-tools.upstream-diff.default-args') + args = default_args + args - current_branch = git.current_branch() + current_branch = git.current_branch() - parser = argparse.ArgumentParser() - parser.add_argument('--wordwise', action='store_true', default=False, - help=( - 'Print a colorized wordwise diff ' - 'instead of line-wise diff')) - parser.add_argument('--branch', default=current_branch, - help='Show changes from a different branch. Passing ' - '"HEAD" is the same as omitting this option (it ' - 'diffs against the current branch)') - opts, extra_args = parser.parse_known_args(args) + parser = argparse.ArgumentParser() + parser.add_argument('--wordwise', + action='store_true', + default=False, + help=('Print a colorized wordwise diff ' + 'instead of line-wise diff')) + parser.add_argument('--branch', + default=current_branch, + help='Show changes from a different branch. Passing ' + '"HEAD" is the same as omitting this option (it ' + 'diffs against the current branch)') + opts, extra_args = parser.parse_known_args(args) - if opts.branch == 'HEAD': - opts.branch = current_branch + if opts.branch == 'HEAD': + opts.branch = current_branch - if not opts.branch or opts.branch == 'HEAD': - print('fatal: Cannot perform git-upstream-diff while not on a branch') - return 1 + if not opts.branch or opts.branch == 'HEAD': + print('fatal: Cannot perform git-upstream-diff while not on a branch') + return 1 - par = git.upstream(opts.branch) - if not par: - print('fatal: No upstream configured for branch \'%s\'' % opts.branch) - return 1 + par = git.upstream(opts.branch) + if not par: + print('fatal: No upstream configured for branch \'%s\'' % opts.branch) + return 1 - cmd = [git.GIT_EXE, '-c', 'core.quotePath=false', - 'diff', '--patience', '-C', '-C'] - if opts.wordwise: - cmd += ['--word-diff=color', r'--word-diff-regex=(\w+|[^[:space:]])'] - cmd += [git.get_or_create_merge_base(opts.branch, par)] - # Only specify the end commit if it is not the current branch, this lets the - # diff include uncommitted changes when diffing the current branch. - if opts.branch != current_branch: - cmd += [opts.branch] + cmd = [ + git.GIT_EXE, '-c', 'core.quotePath=false', 'diff', '--patience', '-C', + '-C' + ] + if opts.wordwise: + cmd += ['--word-diff=color', r'--word-diff-regex=(\w+|[^[:space:]])'] + cmd += [git.get_or_create_merge_base(opts.branch, par)] + # Only specify the end commit if it is not the current branch, this lets the + # diff include uncommitted changes when diffing the current branch. + if opts.branch != current_branch: + cmd += [opts.branch] - cmd += extra_args + cmd += extra_args - return subprocess2.check_call(cmd) + return subprocess2.check_call(cmd) if __name__ == '__main__': - try: - sys.exit(main(sys.argv[1:])) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(1) + try: + sys.exit(main(sys.argv[1:])) + except KeyboardInterrupt: + sys.stderr.write('interrupted\n') + sys.exit(1) diff --git a/gn.py b/gn.py index 0cf2538d14..1a1e3fa3fb 100755 --- a/gn.py +++ b/gn.py @@ -2,7 +2,6 @@ # Copyright 2013 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """This script is a wrapper around the GN binary that is pulled from Google Cloud Storage when you sync Chrome. The binaries go into platform-specific subdirectories in the source tree. @@ -21,57 +20,60 @@ import sys def PruneVirtualEnv(): - # Set by VirtualEnv, no need to keep it. - os.environ.pop('VIRTUAL_ENV', None) + # Set by VirtualEnv, no need to keep it. + os.environ.pop('VIRTUAL_ENV', None) - # Set by VPython, if scripts want it back they have to set it explicitly. - os.environ.pop('PYTHONNOUSERSITE', None) + # Set by VPython, if scripts want it back they have to set it explicitly. + os.environ.pop('PYTHONNOUSERSITE', None) - # Look for "activate_this.py" in this path, which is installed by VirtualEnv. - # This mechanism is used by vpython as well to sanitize VirtualEnvs from - # $PATH. - os.environ['PATH'] = os.pathsep.join([ - p for p in os.environ.get('PATH', '').split(os.pathsep) - if not os.path.isfile(os.path.join(p, 'activate_this.py')) - ]) + # Look for "activate_this.py" in this path, which is installed by + # VirtualEnv. This mechanism is used by vpython as well to sanitize + # VirtualEnvs from $PATH. + os.environ['PATH'] = os.pathsep.join([ + p for p in os.environ.get('PATH', '').split(os.pathsep) + if not os.path.isfile(os.path.join(p, 'activate_this.py')) + ]) def main(args): - # Prune all evidence of VPython/VirtualEnv out of the environment. This means - # that we 'unwrap' vpython VirtualEnv path/env manipulation. Invocations of - # `python` from GN should never inherit the gn.py's own VirtualEnv. This also - # helps to ensure that generated ninja files do not reference python.exe from - # the VirtualEnv generated from depot_tools' own .vpython file (or lack - # thereof), but instead reference the default python from the PATH. - PruneVirtualEnv() + # Prune all evidence of VPython/VirtualEnv out of the environment. This + # means that we 'unwrap' vpython VirtualEnv path/env manipulation. + # Invocations of `python` from GN should never inherit the gn.py's own + # VirtualEnv. This also helps to ensure that generated ninja files do not + # reference python.exe from the VirtualEnv generated from depot_tools' own + # .vpython file (or lack thereof), but instead reference the default python + # from the PATH. + PruneVirtualEnv() - # Try in primary solution location first, with the gn binary having been - # downloaded by cipd in the projects DEPS. - primary_solution_path = gclient_paths.GetPrimarySolutionPath() - if primary_solution_path: - gn_path = os.path.join(primary_solution_path, 'third_party', - 'gn', 'gn' + gclient_paths.GetExeSuffix()) - if os.path.exists(gn_path): - return subprocess.call([gn_path] + args[1:]) + # Try in primary solution location first, with the gn binary having been + # downloaded by cipd in the projects DEPS. + primary_solution_path = gclient_paths.GetPrimarySolutionPath() + if primary_solution_path: + gn_path = os.path.join(primary_solution_path, 'third_party', 'gn', + 'gn' + gclient_paths.GetExeSuffix()) + if os.path.exists(gn_path): + return subprocess.call([gn_path] + args[1:]) - # Otherwise try the old .sha1 and download_from_google_storage locations - # inside of buildtools. - bin_path = gclient_paths.GetBuildtoolsPlatformBinaryPath() - if not bin_path: - print('gn.py: Could not find checkout in any parent of the current path.\n' - 'This must be run inside a checkout.', file=sys.stderr) - return 1 - gn_path = os.path.join(bin_path, 'gn' + gclient_paths.GetExeSuffix()) - if not os.path.exists(gn_path): - print( - 'gn.py: Could not find gn executable at: %s' % gn_path, file=sys.stderr) - return 2 - return subprocess.call([gn_path] + args[1:]) + # Otherwise try the old .sha1 and download_from_google_storage locations + # inside of buildtools. + bin_path = gclient_paths.GetBuildtoolsPlatformBinaryPath() + if not bin_path: + print( + 'gn.py: Could not find checkout in any parent of the current ' + 'path.\nThis must be run inside a checkout.', + file=sys.stderr) + return 1 + gn_path = os.path.join(bin_path, 'gn' + gclient_paths.GetExeSuffix()) + if not os.path.exists(gn_path): + print('gn.py: Could not find gn executable at: %s' % gn_path, + file=sys.stderr) + return 2 + return subprocess.call([gn_path] + args[1:]) if __name__ == '__main__': - try: - sys.exit(main(sys.argv)) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(1) + try: + sys.exit(main(sys.argv)) + except KeyboardInterrupt: + sys.stderr.write('interrupted\n') + sys.exit(1) diff --git a/gsutil.py b/gsutil.py index c8eb0ff15c..1e90f56617 100755 --- a/gsutil.py +++ b/gsutil.py @@ -2,7 +2,6 @@ # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Run a pinned gsutil.""" from __future__ import print_function @@ -22,7 +21,6 @@ import urllib.request import zipfile - GSUTIL_URL = 'https://storage.googleapis.com/pub/' API_URL = 'https://www.googleapis.com/storage/v1/b/pub/o/' @@ -41,273 +39,278 @@ LUCI_AUTH_SCOPES = [ class InvalidGsutilError(Exception): - pass + pass def download_gsutil(version, target_dir): - """Downloads gsutil into the target_dir.""" - filename = 'gsutil_%s.zip' % version - target_filename = os.path.join(target_dir, filename) + """Downloads gsutil into the target_dir.""" + filename = 'gsutil_%s.zip' % version + target_filename = os.path.join(target_dir, filename) - # Check if the target exists already. - if os.path.exists(target_filename): - md5_calc = hashlib.md5() - with open(target_filename, 'rb') as f: - while True: - buf = f.read(4096) - if not buf: - break - md5_calc.update(buf) - local_md5 = md5_calc.hexdigest() + # Check if the target exists already. + if os.path.exists(target_filename): + md5_calc = hashlib.md5() + with open(target_filename, 'rb') as f: + while True: + buf = f.read(4096) + if not buf: + break + md5_calc.update(buf) + local_md5 = md5_calc.hexdigest() - metadata_url = '%s%s' % (API_URL, filename) - metadata = json.load(urllib.request.urlopen(metadata_url)) - remote_md5 = base64.b64decode(metadata['md5Hash']).decode('utf-8') + metadata_url = '%s%s' % (API_URL, filename) + metadata = json.load(urllib.request.urlopen(metadata_url)) + remote_md5 = base64.b64decode(metadata['md5Hash']).decode('utf-8') - if local_md5 == remote_md5: - return target_filename - os.remove(target_filename) + if local_md5 == remote_md5: + return target_filename + os.remove(target_filename) - # Do the download. - url = '%s%s' % (GSUTIL_URL, filename) - u = urllib.request.urlopen(url) - with open(target_filename, 'wb') as f: - while True: - buf = u.read(4096) - if not buf: - break - f.write(buf) - return target_filename + # Do the download. + url = '%s%s' % (GSUTIL_URL, filename) + u = urllib.request.urlopen(url) + with open(target_filename, 'wb') as f: + while True: + buf = u.read(4096) + if not buf: + break + f.write(buf) + return target_filename @contextlib.contextmanager def temporary_directory(base): - tmpdir = tempfile.mkdtemp(prefix='t', dir=base) - try: - yield tmpdir - finally: - if os.path.isdir(tmpdir): - shutil.rmtree(tmpdir) + tmpdir = tempfile.mkdtemp(prefix='t', dir=base) + try: + yield tmpdir + finally: + if os.path.isdir(tmpdir): + shutil.rmtree(tmpdir) def ensure_gsutil(version, target, clean): - bin_dir = os.path.join(target, 'gsutil_%s' % version) - gsutil_bin = os.path.join(bin_dir, 'gsutil', 'gsutil') - gsutil_flag = os.path.join(bin_dir, 'gsutil', 'install.flag') - # We assume that if gsutil_flag exists, then we have a good version - # of the gsutil package. - if not clean and os.path.isfile(gsutil_flag): - # Everything is awesome! we're all done here. + bin_dir = os.path.join(target, 'gsutil_%s' % version) + gsutil_bin = os.path.join(bin_dir, 'gsutil', 'gsutil') + gsutil_flag = os.path.join(bin_dir, 'gsutil', 'install.flag') + # We assume that if gsutil_flag exists, then we have a good version + # of the gsutil package. + if not clean and os.path.isfile(gsutil_flag): + # Everything is awesome! we're all done here. + return gsutil_bin + + if not os.path.exists(target): + try: + os.makedirs(target) + except FileExistsError: + # Another process is prepping workspace, so let's check if + # gsutil_bin is present. If after several checks it's still not, + # continue with downloading gsutil. + delay = 2 # base delay, in seconds + for _ in range(3): # make N attempts + # sleep first as it's not expected to have file ready just yet. + time.sleep(delay) + delay *= 1.5 # next delay increased by that factor + if os.path.isfile(gsutil_bin): + return gsutil_bin + + with temporary_directory(target) as instance_dir: + # Clean up if we're redownloading a corrupted gsutil. + cleanup_path = os.path.join(instance_dir, 'clean') + try: + os.rename(bin_dir, cleanup_path) + except (OSError, IOError): + cleanup_path = None + if cleanup_path: + shutil.rmtree(cleanup_path) + + download_dir = os.path.join(instance_dir, 'd') + target_zip_filename = download_gsutil(version, instance_dir) + with zipfile.ZipFile(target_zip_filename, 'r') as target_zip: + target_zip.extractall(download_dir) + + shutil.move(download_dir, bin_dir) + # Final check that the gsutil bin exists. This should never fail. + if not os.path.isfile(gsutil_bin): + raise InvalidGsutilError() + # Drop a flag file. + with open(gsutil_flag, 'w') as f: + f.write('This flag file is dropped by gsutil.py') + return gsutil_bin - if not os.path.exists(target): - try: - os.makedirs(target) - except FileExistsError: - # Another process is prepping workspace, so let's check if gsutil_bin is - # present. If after several checks it's still not, continue with - # downloading gsutil. - delay = 2 # base delay, in seconds - for _ in range(3): # make N attempts - # sleep first as it's not expected to have file ready just yet. - time.sleep(delay) - delay *= 1.5 # next delay increased by that factor - if os.path.isfile(gsutil_bin): - return gsutil_bin - - with temporary_directory(target) as instance_dir: - # Clean up if we're redownloading a corrupted gsutil. - cleanup_path = os.path.join(instance_dir, 'clean') - try: - os.rename(bin_dir, cleanup_path) - except (OSError, IOError): - cleanup_path = None - if cleanup_path: - shutil.rmtree(cleanup_path) - - download_dir = os.path.join(instance_dir, 'd') - target_zip_filename = download_gsutil(version, instance_dir) - with zipfile.ZipFile(target_zip_filename, 'r') as target_zip: - target_zip.extractall(download_dir) - - shutil.move(download_dir, bin_dir) - # Final check that the gsutil bin exists. This should never fail. - if not os.path.isfile(gsutil_bin): - raise InvalidGsutilError() - # Drop a flag file. - with open(gsutil_flag, 'w') as f: - f.write('This flag file is dropped by gsutil.py') - - return gsutil_bin - def _is_luci_context(): - """Returns True if the script is run within luci-context""" - if os.getenv('SWARMING_HEADLESS') == '1': - return True + """Returns True if the script is run within luci-context""" + if os.getenv('SWARMING_HEADLESS') == '1': + return True - luci_context_env = os.getenv('LUCI_CONTEXT') - if not luci_context_env: - return False + luci_context_env = os.getenv('LUCI_CONTEXT') + if not luci_context_env: + return False - try: - with open(luci_context_env) as f: - luci_context_json = json.load(f) - return 'local_auth' in luci_context_json - except (ValueError, FileNotFoundError): - return False + try: + with open(luci_context_env) as f: + luci_context_json = json.load(f) + return 'local_auth' in luci_context_json + except (ValueError, FileNotFoundError): + return False def luci_context(cmd): - """Helper to call`luci-auth context`.""" - p = _luci_auth_cmd('context', wrapped_cmds=cmd) + """Helper to call`luci-auth context`.""" + p = _luci_auth_cmd('context', wrapped_cmds=cmd) - # If luci-auth is not logged in, fallback to normal execution. - if b'Not logged in.' in p.stderr: - return _run_subprocess(cmd, interactive=True) + # If luci-auth is not logged in, fallback to normal execution. + if b'Not logged in.' in p.stderr: + return _run_subprocess(cmd, interactive=True) - _print_subprocess_result(p) - return p + _print_subprocess_result(p) + return p def luci_login(): - """Helper to run `luci-auth login`.""" - # luci-auth requires interactive shell. - return _luci_auth_cmd('login', interactive=True) + """Helper to run `luci-auth login`.""" + # luci-auth requires interactive shell. + return _luci_auth_cmd('login', interactive=True) def _luci_auth_cmd(luci_cmd, wrapped_cmds=None, interactive=False): - """Helper to call luci-auth command.""" - cmd = ['luci-auth', luci_cmd, '-scopes', ' '.join(LUCI_AUTH_SCOPES)] - if wrapped_cmds: - cmd += ['--'] + wrapped_cmds + """Helper to call luci-auth command.""" + cmd = ['luci-auth', luci_cmd, '-scopes', ' '.join(LUCI_AUTH_SCOPES)] + if wrapped_cmds: + cmd += ['--'] + wrapped_cmds - return _run_subprocess(cmd, interactive) + return _run_subprocess(cmd, interactive) def _run_subprocess(cmd, interactive=False, env=None): - """Wrapper to run the given command within a subprocess.""" - kwargs = {'shell': IS_WINDOWS} + """Wrapper to run the given command within a subprocess.""" + kwargs = {'shell': IS_WINDOWS} - if env: - kwargs['env'] = dict(os.environ, **env) + if env: + kwargs['env'] = dict(os.environ, **env) - if not interactive: - kwargs['stdout'] = subprocess.PIPE - kwargs['stderr'] = subprocess.PIPE + if not interactive: + kwargs['stdout'] = subprocess.PIPE + kwargs['stderr'] = subprocess.PIPE - return subprocess.run(cmd, **kwargs) + return subprocess.run(cmd, **kwargs) def _print_subprocess_result(p): - """Prints the subprocess result to stdout & stderr.""" - if p.stdout: - sys.stdout.buffer.write(p.stdout) + """Prints the subprocess result to stdout & stderr.""" + if p.stdout: + sys.stdout.buffer.write(p.stdout) - if p.stderr: - sys.stderr.buffer.write(p.stderr) + if p.stderr: + sys.stderr.buffer.write(p.stderr) def is_boto_present(): - """Returns true if the .boto file is present in the default path.""" - return os.getenv('BOTO_CONFIG') or os.getenv( - 'AWS_CREDENTIAL_FILE') or os.path.isfile( - os.path.join(os.path.expanduser('~'), '.boto')) + """Returns true if the .boto file is present in the default path.""" + return os.getenv('BOTO_CONFIG') or os.getenv( + 'AWS_CREDENTIAL_FILE') or os.path.isfile( + os.path.join(os.path.expanduser('~'), '.boto')) def run_gsutil(target, args, clean=False): - # Redirect gsutil config calls to luci-auth. - if 'config' in args: - return luci_login().returncode + # Redirect gsutil config calls to luci-auth. + if 'config' in args: + return luci_login().returncode - gsutil_bin = ensure_gsutil(VERSION, target, clean) - args_opt = ['-o', 'GSUtil:software_update_check_period=0'] + gsutil_bin = ensure_gsutil(VERSION, target, clean) + args_opt = ['-o', 'GSUtil:software_update_check_period=0'] - if sys.platform == 'darwin': - # We are experiencing problems with multiprocessing on MacOS where gsutil.py - # may hang. - # This behavior is documented in gsutil codebase, and recommendation is to - # set GSUtil:parallel_process_count=1. - # https://github.com/GoogleCloudPlatform/gsutil/blob/06efc9dc23719fab4fd5fadb506d252bbd3fe0dd/gslib/command.py#L1331 - # https://github.com/GoogleCloudPlatform/gsutil/issues/1100 - args_opt.extend(['-o', 'GSUtil:parallel_process_count=1']) - if sys.platform == 'cygwin': - # This script requires Windows Python, so invoke with depot_tools' - # Python. - def winpath(path): - stdout = subprocess.check_output(['cygpath', '-w', path]) - return stdout.strip().decode('utf-8', 'replace') - cmd = ['python.bat', winpath(__file__)] - cmd.extend(args) - sys.exit(subprocess.call(cmd)) - assert sys.platform != 'cygwin' + if sys.platform == 'darwin': + # We are experiencing problems with multiprocessing on MacOS where + # gsutil.py may hang. This behavior is documented in gsutil codebase, + # and recommendation is to set GSUtil:parallel_process_count=1. + # https://github.com/GoogleCloudPlatform/gsutil/blob/06efc9dc23719fab4fd5fadb506d252bbd3fe0dd/gslib/command.py#L1331 + # https://github.com/GoogleCloudPlatform/gsutil/issues/1100 + args_opt.extend(['-o', 'GSUtil:parallel_process_count=1']) + if sys.platform == 'cygwin': + # This script requires Windows Python, so invoke with depot_tools' + # Python. + def winpath(path): + stdout = subprocess.check_output(['cygpath', '-w', path]) + return stdout.strip().decode('utf-8', 'replace') - cmd = [ - 'vpython3', - '-vpython-spec', os.path.join(THIS_DIR, 'gsutil.vpython3'), - '--', - gsutil_bin - ] + args_opt + args + cmd = ['python.bat', winpath(__file__)] + cmd.extend(args) + sys.exit(subprocess.call(cmd)) + assert sys.platform != 'cygwin' - # When .boto is present, try without additional wrappers and handle specific - # errors. - if is_boto_present(): - p = _run_subprocess(cmd) + cmd = [ + 'vpython3', '-vpython-spec', + os.path.join(THIS_DIR, 'gsutil.vpython3'), '--', gsutil_bin + ] + args_opt + args - # Notify user that their .boto file might be outdated. - if b'Your credentials are invalid.' in p.stderr: - # Make sure this error message is visible when invoked by gclient runhooks - separator = '*' * 80 - print('\n' + separator + '\n' + - 'Warning: You might have an outdated .boto file. If this issue ' - 'persists after running `gsutil.py config`, try removing your ' - '.boto, usually located in your home directory.\n' + separator + - '\n', - file=sys.stderr) + # When .boto is present, try without additional wrappers and handle specific + # errors. + if is_boto_present(): + p = _run_subprocess(cmd) - _print_subprocess_result(p) - return p.returncode + # Notify user that their .boto file might be outdated. + if b'Your credentials are invalid.' in p.stderr: + # Make sure this error message is visible when invoked by gclient + # runhooks + separator = '*' * 80 + print( + '\n' + separator + '\n' + + 'Warning: You might have an outdated .boto file. If this issue ' + 'persists after running `gsutil.py config`, try removing your ' + '.boto, usually located in your home directory.\n' + separator + + '\n', + file=sys.stderr) - # Skip wrapping commands if luci-auth is already being - if _is_luci_context(): - return _run_subprocess(cmd, interactive=True).returncode + _print_subprocess_result(p) + return p.returncode - # Wrap gsutil with luci-auth context. - return luci_context(cmd).returncode + # Skip wrapping commands if luci-auth is already being + if _is_luci_context(): + return _run_subprocess(cmd, interactive=True).returncode + + # Wrap gsutil with luci-auth context. + return luci_context(cmd).returncode def parse_args(): - bin_dir = os.environ.get('DEPOT_TOOLS_GSUTIL_BIN_DIR', DEFAULT_BIN_DIR) + bin_dir = os.environ.get('DEPOT_TOOLS_GSUTIL_BIN_DIR', DEFAULT_BIN_DIR) - # Help is disabled as it conflicts with gsutil -h, which controls headers. - parser = argparse.ArgumentParser(add_help=False) + # Help is disabled as it conflicts with gsutil -h, which controls headers. + parser = argparse.ArgumentParser(add_help=False) - parser.add_argument('--clean', action='store_true', - help='Clear any existing gsutil package, forcing a new download.') - parser.add_argument('--target', default=bin_dir, - help='The target directory to download/store a gsutil version in. ' - '(default is %(default)s).') + parser.add_argument( + '--clean', + action='store_true', + help='Clear any existing gsutil package, forcing a new download.') + parser.add_argument( + '--target', + default=bin_dir, + help='The target directory to download/store a gsutil version in. ' + '(default is %(default)s).') - # These two args exist for backwards-compatibility but are no-ops. - parser.add_argument('--force-version', default=VERSION, - help='(deprecated, this flag has no effect)') - parser.add_argument('--fallback', - help='(deprecated, this flag has no effect)') + # These two args exist for backwards-compatibility but are no-ops. + parser.add_argument('--force-version', + default=VERSION, + help='(deprecated, this flag has no effect)') + parser.add_argument('--fallback', + help='(deprecated, this flag has no effect)') - parser.add_argument('args', nargs=argparse.REMAINDER) + parser.add_argument('args', nargs=argparse.REMAINDER) - args, extras = parser.parse_known_args() - if args.args and args.args[0] == '--': - args.args.pop(0) - if extras: - args.args = extras + args.args - return args + args, extras = parser.parse_known_args() + if args.args and args.args[0] == '--': + args.args.pop(0) + if extras: + args.args = extras + args.args + return args def main(): - args = parse_args() - return run_gsutil(args.target, args.args, clean=args.clean) + args = parse_args() + return run_gsutil(args.target, args.args, clean=args.clean) if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/infra_to_superproject.py b/infra_to_superproject.py index 0140d43474..f17585869b 100644 --- a/infra_to_superproject.py +++ b/infra_to_superproject.py @@ -23,44 +23,44 @@ import shutil def main(argv): - source = os.getcwd() + source = os.getcwd() - parser = argparse.ArgumentParser(description=__doc__.strip().splitlines()[0], - epilog=' '.join( - __doc__.strip().splitlines()[1:])) - parser.add_argument('-n', - '--no-backup', - action='store_true', - help='NOT RECOMMENDED. Skips copying the current ' - 'checkout (which can take up to ~15 min) to ' - 'a backup before starting the migration.') - args = parser.parse_args(argv) + parser = argparse.ArgumentParser( + description=__doc__.strip().splitlines()[0], + epilog=' '.join(__doc__.strip().splitlines()[1:])) + parser.add_argument('-n', + '--no-backup', + action='store_true', + help='NOT RECOMMENDED. Skips copying the current ' + 'checkout (which can take up to ~15 min) to ' + 'a backup before starting the migration.') + args = parser.parse_args(argv) - if not args.no_backup: - backup = source + '_backup' - print(f'Creating backup in {backup}') - print('May take up to ~15 minutes...') - shutil.copytree(source, backup, symlinks=True, dirs_exist_ok=True) - print('backup complete') + if not args.no_backup: + backup = source + '_backup' + print(f'Creating backup in {backup}') + print('May take up to ~15 minutes...') + shutil.copytree(source, backup, symlinks=True, dirs_exist_ok=True) + print('backup complete') - print(f'Deleting old {source}/.gclient file') - gclient_file = os.path.join(source, '.gclient') - with open(gclient_file, 'r') as file: - data = file.read() - internal = "infra_internal" in data - os.remove(gclient_file) + print(f'Deleting old {source}/.gclient file') + gclient_file = os.path.join(source, '.gclient') + with open(gclient_file, 'r') as file: + data = file.read() + internal = "infra_internal" in data + os.remove(gclient_file) - print('Migrating to infra/infra_superproject') - cmds = ['fetch', '--force'] - if internal: - cmds.append('infra_internal') - print('including internal code in checkout') - else: - cmds.append('infra') - shell = sys.platform == 'win32' - fetch = subprocess.Popen(cmds, cwd=source, shell=shell) - fetch.wait() + print('Migrating to infra/infra_superproject') + cmds = ['fetch', '--force'] + if internal: + cmds.append('infra_internal') + print('including internal code in checkout') + else: + cmds.append('infra') + shell = sys.platform == 'win32' + fetch = subprocess.Popen(cmds, cwd=source, shell=shell) + fetch.wait() if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) + sys.exit(main(sys.argv[1:])) diff --git a/isort b/isort index ccf1640d6f..f8d5b5271f 100755 --- a/isort +++ b/isort @@ -19,7 +19,6 @@ from pathlib import Path import sys - THIS_DIR = Path(__file__).resolve().parent RC_FILE = THIS_DIR / '.isort.cfg' diff --git a/lockfile.py b/lockfile.py index 9e41635860..3cd5670365 100644 --- a/lockfile.py +++ b/lockfile.py @@ -13,94 +13,95 @@ import time class LockError(Exception): - pass + pass if sys.platform.startswith('win'): - # Windows implementation - import win32imports + # Windows implementation + import win32imports - BYTES_TO_LOCK = 1 + BYTES_TO_LOCK = 1 - def _open_file(lockfile): - return win32imports.Handle( - win32imports.CreateFileW( - lockfile, # lpFileName - win32imports.GENERIC_WRITE, # dwDesiredAccess - 0, # dwShareMode=prevent others from opening file - None, # lpSecurityAttributes - win32imports.CREATE_ALWAYS, # dwCreationDisposition - win32imports.FILE_ATTRIBUTE_NORMAL, # dwFlagsAndAttributes - None # hTemplateFile - )) + def _open_file(lockfile): + return win32imports.Handle( + win32imports.CreateFileW( + lockfile, # lpFileName + win32imports.GENERIC_WRITE, # dwDesiredAccess + 0, # dwShareMode=prevent others from opening file + None, # lpSecurityAttributes + win32imports.CREATE_ALWAYS, # dwCreationDisposition + win32imports.FILE_ATTRIBUTE_NORMAL, # dwFlagsAndAttributes + None # hTemplateFile + )) - def _close_file(handle): - # CloseHandle releases lock too. - win32imports.CloseHandle(handle) + def _close_file(handle): + # CloseHandle releases lock too. + win32imports.CloseHandle(handle) - def _lock_file(handle): - ret = win32imports.LockFileEx( - handle, # hFile - win32imports.LOCKFILE_FAIL_IMMEDIATELY - | win32imports.LOCKFILE_EXCLUSIVE_LOCK, # dwFlags - 0, #dwReserved - BYTES_TO_LOCK, # nNumberOfBytesToLockLow - 0, # nNumberOfBytesToLockHigh - win32imports.Overlapped() # lpOverlapped - ) - # LockFileEx returns result as bool, which is converted into an integer - # (1 == successful; 0 == not successful) - if ret == 0: - error_code = win32imports.GetLastError() - raise OSError('Failed to lock handle (error code: %d).' % error_code) + def _lock_file(handle): + ret = win32imports.LockFileEx( + handle, # hFile + win32imports.LOCKFILE_FAIL_IMMEDIATELY + | win32imports.LOCKFILE_EXCLUSIVE_LOCK, # dwFlags + 0, #dwReserved + BYTES_TO_LOCK, # nNumberOfBytesToLockLow + 0, # nNumberOfBytesToLockHigh + win32imports.Overlapped() # lpOverlapped + ) + # LockFileEx returns result as bool, which is converted into an integer + # (1 == successful; 0 == not successful) + if ret == 0: + error_code = win32imports.GetLastError() + raise OSError('Failed to lock handle (error code: %d).' % + error_code) else: - # Unix implementation - import fcntl + # Unix implementation + import fcntl - def _open_file(lockfile): - open_flags = (os.O_CREAT | os.O_WRONLY) - return os.open(lockfile, open_flags, 0o644) + def _open_file(lockfile): + open_flags = (os.O_CREAT | os.O_WRONLY) + return os.open(lockfile, open_flags, 0o644) - def _close_file(fd): - os.close(fd) + def _close_file(fd): + os.close(fd) - def _lock_file(fd): - fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + def _lock_file(fd): + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) def _try_lock(lockfile): - f = _open_file(lockfile) - try: - _lock_file(f) - except Exception: - _close_file(f) - raise - return lambda: _close_file(f) + f = _open_file(lockfile) + try: + _lock_file(f) + except Exception: + _close_file(f) + raise + return lambda: _close_file(f) def _lock(path, timeout=0): - """_lock returns function to release the lock if locking was successful. + """_lock returns function to release the lock if locking was successful. _lock also implements simple retry logic.""" - elapsed = 0 - while True: - try: - return _try_lock(path + '.locked') - except (OSError, IOError) as e: - if elapsed < timeout: - sleep_time = min(10, timeout - elapsed) - logging.info( - 'Could not create git cache lockfile; ' - 'will retry after sleep(%d).', sleep_time) - elapsed += sleep_time - time.sleep(sleep_time) - continue - raise LockError("Error locking %s (err: %s)" % (path, str(e))) + elapsed = 0 + while True: + try: + return _try_lock(path + '.locked') + except (OSError, IOError) as e: + if elapsed < timeout: + sleep_time = min(10, timeout - elapsed) + logging.info( + 'Could not create git cache lockfile; ' + 'will retry after sleep(%d).', sleep_time) + elapsed += sleep_time + time.sleep(sleep_time) + continue + raise LockError("Error locking %s (err: %s)" % (path, str(e))) @contextlib.contextmanager def lock(path, timeout=0): - """Get exclusive lock to path. + """Get exclusive lock to path. Usage: import lockfile @@ -109,8 +110,8 @@ def lock(path, timeout=0): pass """ - release_fn = _lock(path, timeout) - try: - yield - finally: - release_fn() + release_fn = _lock(path, timeout) + try: + yield + finally: + release_fn() diff --git a/man/src/filter_demo_output.py b/man/src/filter_demo_output.py index d2aa44ab5a..3222064d52 100755 --- a/man/src/filter_demo_output.py +++ b/man/src/filter_demo_output.py @@ -14,122 +14,123 @@ from xml.sax.saxutils import escape from io import StringIO if not os.path.exists('ansi2html'): - print('You must run ./make_docs.sh once before running this script.') - sys.exit(1) + print('You must run ./make_docs.sh once before running this script.') + sys.exit(1) # This dependency is pulled in by make_docs.sh # if it doesn't exist, run ./make_docs.sh first sys.path.insert(0, 'ansi2html') -import ansi2html # pylint: disable=import-error, W0611 +import ansi2html # pylint: disable=import-error, W0611 import ansi2html.converter # pylint: disable=import-error, W0611 + def simpleXML(string): - BRIGHT = 1 - DIM = 2 - NORMAL = 22 - RESET = 0 - ESC_RE = re.compile('(\x1B\\[[^m]*?)m') + BRIGHT = 1 + DIM = 2 + NORMAL = 22 + RESET = 0 + ESC_RE = re.compile('(\x1B\\[[^m]*?)m') - ret = StringIO() - boldstate = False + ret = StringIO() + boldstate = False - for tok in ESC_RE.split(string): - if not tok: - continue - if tok[0] == '\x1b': - codes = map(int, filter(bool, tok[2:].split(';'))) - if not codes: - codes = [RESET] - for code in codes: - # only care about Bright - if code == BRIGHT and boldstate is False: - boldstate = True - ret.write('') - elif code in (DIM, NORMAL, RESET) and boldstate: - boldstate = False - ret.write('') - else: - ret.write(escape(tok)) + for tok in ESC_RE.split(string): + if not tok: + continue + if tok[0] == '\x1b': + codes = map(int, filter(bool, tok[2:].split(';'))) + if not codes: + codes = [RESET] + for code in codes: + # only care about Bright + if code == BRIGHT and boldstate is False: + boldstate = True + ret.write('') + elif code in (DIM, NORMAL, RESET) and boldstate: + boldstate = False + ret.write('') + else: + ret.write(escape(tok)) - if boldstate: - ret.write('') + if boldstate: + ret.write('') - return ret.getvalue() + return ret.getvalue() def main(): - backend = sys.argv[1] - output = sys.stdin.read().rstrip() + backend = sys.argv[1] + output = sys.stdin.read().rstrip() - callout_re = re.compile(r'\x1b\[(\d+)c\n') - callouts = collections.defaultdict(int) - for i, line in enumerate(output.splitlines(True)): - m = callout_re.match(line) - if m: - callouts[i + int(m.group(1)) - len(callouts)] += 1 + callout_re = re.compile(r'\x1b\[(\d+)c\n') + callouts = collections.defaultdict(int) + for i, line in enumerate(output.splitlines(True)): + m = callout_re.match(line) + if m: + callouts[i + int(m.group(1)) - len(callouts)] += 1 - output = callout_re.sub('', output) + output = callout_re.sub('', output) - w = sys.stdout.write + w = sys.stdout.write - comment_marker = '###COMMENT###' + comment_marker = '###COMMENT###' - callout_counter = 1 - if backend == 'xhtml11': - preamble = ( - '

'
-    )
-    postamble = '

' - c = ansi2html.Ansi2HTMLConverter(inline=True, scheme='dracula') + callout_counter = 1 + if backend == 'xhtml11': + preamble = ( + '

'
+        )
+        postamble = '

' + c = ansi2html.Ansi2HTMLConverter(inline=True, scheme='dracula') - in_code = False - body = c.convert(output, full=False) - for i, line in enumerate(body.splitlines()): - if line.startswith(comment_marker): + in_code = False + body = c.convert(output, full=False) + for i, line in enumerate(body.splitlines()): + if line.startswith(comment_marker): + if in_code: + w(postamble) + in_code = False + w(line[len(comment_marker):]) + else: + if not in_code: + w(preamble) + in_code = True + ext = '' + for _ in range(callouts[i]): + if not ext: + ext += '' + ext += ' <%d>' % callout_counter + callout_counter += 1 + if ext: + ext += '' + w(line + ext + '\n') if in_code: - w(postamble) - in_code = False - w(line[len(comment_marker):]) - else: - if not in_code: - w(preamble) - in_code = True - ext = '' - for _ in range(callouts[i]): - if not ext: - ext += '' - ext += ' <%d>' % callout_counter - callout_counter += 1 - if ext: - ext += '' - w(line + ext + '\n') - if in_code: - w(postamble) - else: - preamble = '' - postamble = '' + w(postamble) + else: + preamble = '' + postamble = '' - in_code = False - body = simpleXML(output) - for i, line in enumerate(body.splitlines()): - if line.startswith(comment_marker): + in_code = False + body = simpleXML(output) + for i, line in enumerate(body.splitlines()): + if line.startswith(comment_marker): + if in_code: + w(postamble) + in_code = False + w(line[len(comment_marker):]) + else: + if not in_code: + w(preamble) + in_code = True + ext = '' + for _ in range(callouts[i]): + ext += ' (%d)' % callout_counter + callout_counter += 1 + w(line + ext + '\n') if in_code: - w(postamble) - in_code = False - w(line[len(comment_marker):]) - else: - if not in_code: - w(preamble) - in_code = True - ext = '' - for _ in range(callouts[i]): - ext += ' (%d)' % callout_counter - callout_counter += 1 - w(line + ext + '\n') - if in_code: - w(postamble) + w(postamble) if __name__ == '__main__': - main() + main() diff --git a/metadata/.style.yapf b/metadata/.style.yapf deleted file mode 100644 index 557fa7bf84..0000000000 --- a/metadata/.style.yapf +++ /dev/null @@ -1,2 +0,0 @@ -[style] -based_on_style = pep8 diff --git a/metadata/fields/field_types.py b/metadata/fields/field_types.py index b1667f6010..609bc0edc1 100644 --- a/metadata/fields/field_types.py +++ b/metadata/fields/field_types.py @@ -58,8 +58,7 @@ class MetadataField: Raises: NotImplementedError if called. This method must be overridden with the actual validation of the field. """ - raise NotImplementedError( - f"{self._name} field validation not defined.") + raise NotImplementedError(f"{self._name} field validation not defined.") class FreeformTextField(MetadataField): diff --git a/metadata/tests/dependency_metadata_test.py b/metadata/tests/dependency_metadata_test.py index 22a0a9564a..a0ec1a20ed 100644 --- a/metadata/tests/dependency_metadata_test.py +++ b/metadata/tests/dependency_metadata_test.py @@ -52,8 +52,7 @@ class DependencyValidationTest(unittest.TestCase): dependency.add_entry(known_fields.LICENSE_FILE.get_name(), "LICENSE") dependency.add_entry(known_fields.LICENSE.get_name(), "Public Domain") dependency.add_entry(known_fields.VERSION.get_name(), "1.0.0") - dependency.add_entry(known_fields.NAME.get_name(), - "Test missing field") + dependency.add_entry(known_fields.NAME.get_name(), "Test missing field") # Leave URL field unspecified. results = dependency.validate( @@ -70,8 +69,7 @@ class DependencyValidationTest(unittest.TestCase): dependency = dm.DependencyMetadata() dependency.add_entry(known_fields.URL.get_name(), "https://www.example.com") - dependency.add_entry(known_fields.NAME.get_name(), - "Test invalid field") + dependency.add_entry(known_fields.NAME.get_name(), "Test invalid field") dependency.add_entry(known_fields.VERSION.get_name(), "1.0.0") dependency.add_entry(known_fields.LICENSE_FILE.get_name(), "LICENSE") dependency.add_entry(known_fields.LICENSE.get_name(), "Public domain") diff --git a/metadata/tests/validate_test.py b/metadata/tests/validate_test.py index 42afad08f0..657f2f5ffe 100644 --- a/metadata/tests/validate_test.py +++ b/metadata/tests/validate_test.py @@ -21,8 +21,8 @@ import metadata.validate _SOURCE_FILE_DIR = os.path.join(_THIS_DIR, "data") _VALID_METADATA_FILEPATH = os.path.join(_THIS_DIR, "data", "README.chromium.test.multi-valid") -_INVALID_METADATA_FILEPATH = os.path.join( - _THIS_DIR, "data", "README.chromium.test.multi-invalid") +_INVALID_METADATA_FILEPATH = os.path.join(_THIS_DIR, "data", + "README.chromium.test.multi-invalid") class ValidateContentTest(unittest.TestCase): diff --git a/metadata/validate.py b/metadata/validate.py index 9c21d16172..72a5b431a3 100644 --- a/metadata/validate.py +++ b/metadata/validate.py @@ -47,8 +47,7 @@ def validate_content(content: str, source_file_dir: str, return results -def _construct_file_read_error(filepath: str, - cause: str) -> vr.ValidationError: +def _construct_file_read_error(filepath: str, cause: str) -> vr.ValidationError: """Helper function to create a validation error for a file reading issue. """ diff --git a/metadata/validation_result.py b/metadata/validation_result.py index 6c171d3867..59ce5f5c77 100644 --- a/metadata/validation_result.py +++ b/metadata/validation_result.py @@ -7,9 +7,8 @@ import textwrap from typing import Dict, List, Union _CHROMIUM_METADATA_PRESCRIPT = "Third party metadata issue:" -_CHROMIUM_METADATA_POSTSCRIPT = ( - "Check //third_party/README.chromium.template " - "for details.") +_CHROMIUM_METADATA_POSTSCRIPT = ("Check //third_party/README.chromium.template " + "for details.") class ValidationResult: diff --git a/metrics.py b/metrics.py index 4641bf2d5f..3ccbfc9a11 100644 --- a/metrics.py +++ b/metrics.py @@ -20,7 +20,6 @@ import gclient_utils import metrics_utils import subprocess2 - DEPOT_TOOLS = os.path.dirname(os.path.abspath(__file__)) CONFIG_FILE = os.path.join(DEPOT_TOOLS, 'metrics.cfg') UPLOAD_SCRIPT = os.path.join(DEPOT_TOOLS, 'upload_metrics.py') @@ -32,294 +31,297 @@ DEPOT_TOOLS_ENV = ['DOGFOOD_STACKED_CHANGES'] INVALID_CONFIG_WARNING = ( 'WARNING: Your metrics.cfg file was invalid or nonexistent. A new one will ' - 'be created.' -) + 'be created.') PERMISSION_DENIED_WARNING = ( 'Could not write the metrics collection config:\n\t%s\n' - 'Metrics collection will be disabled.' -) + 'Metrics collection will be disabled.') class _Config(object): - def __init__(self): - self._initialized = False - self._config = {} + def __init__(self): + self._initialized = False + self._config = {} - def _ensure_initialized(self): - if self._initialized: - return + def _ensure_initialized(self): + if self._initialized: + return - # Metrics collection is disabled, so don't collect any metrics. - if not metrics_utils.COLLECT_METRICS: - self._config = { - 'is-googler': False, - 'countdown': 0, - 'opt-in': False, - 'version': metrics_utils.CURRENT_VERSION, - } - self._initialized = True - return + # Metrics collection is disabled, so don't collect any metrics. + if not metrics_utils.COLLECT_METRICS: + self._config = { + 'is-googler': False, + 'countdown': 0, + 'opt-in': False, + 'version': metrics_utils.CURRENT_VERSION, + } + self._initialized = True + return - # We are running on a bot. Ignore config and collect metrics. - if metrics_utils.REPORT_BUILD: - self._config = { - 'is-googler': True, - 'countdown': 0, - 'opt-in': True, - 'version': metrics_utils.CURRENT_VERSION, - } - self._initialized = True - return + # We are running on a bot. Ignore config and collect metrics. + if metrics_utils.REPORT_BUILD: + self._config = { + 'is-googler': True, + 'countdown': 0, + 'opt-in': True, + 'version': metrics_utils.CURRENT_VERSION, + } + self._initialized = True + return - try: - config = json.loads(gclient_utils.FileRead(CONFIG_FILE)) - except (IOError, ValueError): - config = {} + try: + config = json.loads(gclient_utils.FileRead(CONFIG_FILE)) + except (IOError, ValueError): + config = {} - self._config = config.copy() + self._config = config.copy() - if 'is-googler' not in self._config: - # /should-upload is only accessible from Google IPs, so we only need to - # check if we can reach the page. An external developer would get access - # denied. - try: - req = urllib.request.urlopen(metrics_utils.APP_URL + '/should-upload') - self._config['is-googler'] = req.getcode() == 200 - except (urllib.request.URLError, urllib.request.HTTPError): - self._config['is-googler'] = False + if 'is-googler' not in self._config: + # /should-upload is only accessible from Google IPs, so we only need + # to check if we can reach the page. An external developer would get + # access denied. + try: + req = urllib.request.urlopen(metrics_utils.APP_URL + + '/should-upload') + self._config['is-googler'] = req.getcode() == 200 + except (urllib.request.URLError, urllib.request.HTTPError): + self._config['is-googler'] = False - # Make sure the config variables we need are present, and initialize them to - # safe values otherwise. - self._config.setdefault('countdown', DEFAULT_COUNTDOWN) - self._config.setdefault('opt-in', None) - self._config.setdefault('version', metrics_utils.CURRENT_VERSION) + # Make sure the config variables we need are present, and initialize + # them to safe values otherwise. + self._config.setdefault('countdown', DEFAULT_COUNTDOWN) + self._config.setdefault('opt-in', None) + self._config.setdefault('version', metrics_utils.CURRENT_VERSION) - if config != self._config: - print(INVALID_CONFIG_WARNING, file=sys.stderr) - self._write_config() + if config != self._config: + print(INVALID_CONFIG_WARNING, file=sys.stderr) + self._write_config() - self._initialized = True + self._initialized = True - def _write_config(self): - try: - gclient_utils.FileWrite(CONFIG_FILE, json.dumps(self._config)) - except IOError as e: - print(PERMISSION_DENIED_WARNING % e, file=sys.stderr) - self._config['opt-in'] = False + def _write_config(self): + try: + gclient_utils.FileWrite(CONFIG_FILE, json.dumps(self._config)) + except IOError as e: + print(PERMISSION_DENIED_WARNING % e, file=sys.stderr) + self._config['opt-in'] = False - @property - def version(self): - self._ensure_initialized() - return self._config['version'] + @property + def version(self): + self._ensure_initialized() + return self._config['version'] - @property - def is_googler(self): - self._ensure_initialized() - return self._config['is-googler'] + @property + def is_googler(self): + self._ensure_initialized() + return self._config['is-googler'] - @property - def opted_in(self): - self._ensure_initialized() - return self._config['opt-in'] + @property + def opted_in(self): + self._ensure_initialized() + return self._config['opt-in'] - @opted_in.setter - def opted_in(self, value): - self._ensure_initialized() - self._config['opt-in'] = value - self._config['version'] = metrics_utils.CURRENT_VERSION - self._write_config() + @opted_in.setter + def opted_in(self, value): + self._ensure_initialized() + self._config['opt-in'] = value + self._config['version'] = metrics_utils.CURRENT_VERSION + self._write_config() - @property - def countdown(self): - self._ensure_initialized() - return self._config['countdown'] + @property + def countdown(self): + self._ensure_initialized() + return self._config['countdown'] - @property - def should_collect_metrics(self): - # Don't report metrics if user is not a Googler. - if not self.is_googler: - return False - # Don't report metrics if user has opted out. - if self.opted_in is False: - return False - # Don't report metrics if countdown hasn't reached 0. - if self.opted_in is None and self.countdown > 0: - return False - return True + @property + def should_collect_metrics(self): + # Don't report metrics if user is not a Googler. + if not self.is_googler: + return False + # Don't report metrics if user has opted out. + if self.opted_in is False: + return False + # Don't report metrics if countdown hasn't reached 0. + if self.opted_in is None and self.countdown > 0: + return False + return True - def decrease_countdown(self): - self._ensure_initialized() - if self.countdown == 0: - return - self._config['countdown'] -= 1 - if self.countdown == 0: - self._config['version'] = metrics_utils.CURRENT_VERSION - self._write_config() + def decrease_countdown(self): + self._ensure_initialized() + if self.countdown == 0: + return + self._config['countdown'] -= 1 + if self.countdown == 0: + self._config['version'] = metrics_utils.CURRENT_VERSION + self._write_config() - def reset_config(self): - # Only reset countdown if we're already collecting metrics. - if self.should_collect_metrics: - self._ensure_initialized() - self._config['countdown'] = DEFAULT_COUNTDOWN - self._config['opt-in'] = None + def reset_config(self): + # Only reset countdown if we're already collecting metrics. + if self.should_collect_metrics: + self._ensure_initialized() + self._config['countdown'] = DEFAULT_COUNTDOWN + self._config['opt-in'] = None class MetricsCollector(object): - def __init__(self): - self._metrics_lock = threading.Lock() - self._reported_metrics = {} - self._config = _Config() - self._collecting_metrics = False - self._collect_custom_metrics = True + def __init__(self): + self._metrics_lock = threading.Lock() + self._reported_metrics = {} + self._config = _Config() + self._collecting_metrics = False + self._collect_custom_metrics = True - @property - def config(self): - return self._config + @property + def config(self): + return self._config - @property - def collecting_metrics(self): - return self._collecting_metrics + @property + def collecting_metrics(self): + return self._collecting_metrics - def add(self, name, value): - if self._collect_custom_metrics: - with self._metrics_lock: - self._reported_metrics[name] = value + def add(self, name, value): + if self._collect_custom_metrics: + with self._metrics_lock: + self._reported_metrics[name] = value - def add_repeated(self, name, value): - if self._collect_custom_metrics: - with self._metrics_lock: - self._reported_metrics.setdefault(name, []).append(value) + def add_repeated(self, name, value): + if self._collect_custom_metrics: + with self._metrics_lock: + self._reported_metrics.setdefault(name, []).append(value) - @contextlib.contextmanager - def pause_metrics_collection(self): - collect_custom_metrics = self._collect_custom_metrics - self._collect_custom_metrics = False - try: - yield - finally: - self._collect_custom_metrics = collect_custom_metrics + @contextlib.contextmanager + def pause_metrics_collection(self): + collect_custom_metrics = self._collect_custom_metrics + self._collect_custom_metrics = False + try: + yield + finally: + self._collect_custom_metrics = collect_custom_metrics - def _upload_metrics_data(self): - """Upload the metrics data to the AppEngine app.""" - p = subprocess2.Popen(['vpython3', UPLOAD_SCRIPT], stdin=subprocess2.PIPE) - # We invoke a subprocess, and use stdin.write instead of communicate(), - # so that we are able to return immediately, leaving the upload running in - # the background. - p.stdin.write(json.dumps(self._reported_metrics).encode('utf-8')) - # ... but if we're running on a bot, wait until upload has completed. - if metrics_utils.REPORT_BUILD: - p.communicate() + def _upload_metrics_data(self): + """Upload the metrics data to the AppEngine app.""" + p = subprocess2.Popen(['vpython3', UPLOAD_SCRIPT], + stdin=subprocess2.PIPE) + # We invoke a subprocess, and use stdin.write instead of communicate(), + # so that we are able to return immediately, leaving the upload running + # in the background. + p.stdin.write(json.dumps(self._reported_metrics).encode('utf-8')) + # ... but if we're running on a bot, wait until upload has completed. + if metrics_utils.REPORT_BUILD: + p.communicate() - def _collect_metrics(self, func, command_name, *args, **kwargs): - # If we're already collecting metrics, just execute the function. - # e.g. git-cl split invokes git-cl upload several times to upload each - # split CL. - if self.collecting_metrics: - # Don't collect metrics for this function. - # e.g. Don't record the arguments git-cl split passes to git-cl upload. - with self.pause_metrics_collection(): - return func(*args, **kwargs) + def _collect_metrics(self, func, command_name, *args, **kwargs): + # If we're already collecting metrics, just execute the function. + # e.g. git-cl split invokes git-cl upload several times to upload each + # split CL. + if self.collecting_metrics: + # Don't collect metrics for this function. + # e.g. Don't record the arguments git-cl split passes to git-cl + # upload. + with self.pause_metrics_collection(): + return func(*args, **kwargs) - self._collecting_metrics = True - self.add('metrics_version', metrics_utils.CURRENT_VERSION) - self.add('command', command_name) - for env in DEPOT_TOOLS_ENV: - if env in os.environ: - self.add_repeated('env_vars', { - 'name': env, - 'value': os.environ.get(env) - }) + self._collecting_metrics = True + self.add('metrics_version', metrics_utils.CURRENT_VERSION) + self.add('command', command_name) + for env in DEPOT_TOOLS_ENV: + if env in os.environ: + self.add_repeated('env_vars', { + 'name': env, + 'value': os.environ.get(env) + }) - try: - start = time.time() - result = func(*args, **kwargs) - exception = None - # pylint: disable=bare-except - except: - exception = sys.exc_info() - finally: - self.add('execution_time', time.time() - start) + try: + start = time.time() + result = func(*args, **kwargs) + exception = None + # pylint: disable=bare-except + except: + exception = sys.exc_info() + finally: + self.add('execution_time', time.time() - start) - exit_code = metrics_utils.return_code_from_exception(exception) - self.add('exit_code', exit_code) + exit_code = metrics_utils.return_code_from_exception(exception) + self.add('exit_code', exit_code) - # Add metrics regarding environment information. - self.add('timestamp', int(time.time())) - self.add('python_version', metrics_utils.get_python_version()) - self.add('host_os', gclient_utils.GetOperatingSystem()) - self.add('host_arch', detect_host_arch.HostArch()) + # Add metrics regarding environment information. + self.add('timestamp', int(time.time())) + self.add('python_version', metrics_utils.get_python_version()) + self.add('host_os', gclient_utils.GetOperatingSystem()) + self.add('host_arch', detect_host_arch.HostArch()) - depot_tools_age = metrics_utils.get_repo_timestamp(DEPOT_TOOLS) - if depot_tools_age is not None: - self.add('depot_tools_age', int(depot_tools_age)) + depot_tools_age = metrics_utils.get_repo_timestamp(DEPOT_TOOLS) + if depot_tools_age is not None: + self.add('depot_tools_age', int(depot_tools_age)) - git_version = metrics_utils.get_git_version() - if git_version: - self.add('git_version', git_version) + git_version = metrics_utils.get_git_version() + if git_version: + self.add('git_version', git_version) - bot_metrics = metrics_utils.get_bot_metrics() - if bot_metrics: - self.add('bot_metrics', bot_metrics) + bot_metrics = metrics_utils.get_bot_metrics() + if bot_metrics: + self.add('bot_metrics', bot_metrics) - self._upload_metrics_data() - if exception: - gclient_utils.reraise(exception[0], exception[1], exception[2]) - return result + self._upload_metrics_data() + if exception: + gclient_utils.reraise(exception[0], exception[1], exception[2]) + return result - def collect_metrics(self, command_name): - """A decorator used to collect metrics over the life of a function. + def collect_metrics(self, command_name): + """A decorator used to collect metrics over the life of a function. This decorator executes the function and collects metrics about the system environment and the function performance. """ - def _decorator(func): - if not self.config.should_collect_metrics: - return func - # Needed to preserve the __name__ and __doc__ attributes of func. - @functools.wraps(func) - def _inner(*args, **kwargs): - return self._collect_metrics(func, command_name, *args, **kwargs) - return _inner - return _decorator + def _decorator(func): + if not self.config.should_collect_metrics: + return func + # Needed to preserve the __name__ and __doc__ attributes of func. + @functools.wraps(func) + def _inner(*args, **kwargs): + return self._collect_metrics(func, command_name, *args, + **kwargs) - @contextlib.contextmanager - def print_notice_and_exit(self): - """A context manager used to print the notice and terminate execution. + return _inner + + return _decorator + + @contextlib.contextmanager + def print_notice_and_exit(self): + """A context manager used to print the notice and terminate execution. This decorator executes the function and prints the monitoring notice if necessary. If an exception is raised, we will catch it, and print it before printing the metrics collection notice. This will call sys.exit() with an appropriate exit code to ensure the notice is the last thing printed.""" - # Needed to preserve the __name__ and __doc__ attributes of func. - try: - yield - exception = None - # pylint: disable=bare-except - except: - exception = sys.exc_info() + # Needed to preserve the __name__ and __doc__ attributes of func. + try: + yield + exception = None + # pylint: disable=bare-except + except: + exception = sys.exc_info() - # Print the exception before the metrics notice, so that the notice is - # clearly visible even if gclient fails. - if exception: - if isinstance(exception[1], KeyboardInterrupt): - sys.stderr.write('Interrupted\n') - elif not isinstance(exception[1], SystemExit): - traceback.print_exception(*exception) + # Print the exception before the metrics notice, so that the notice is + # clearly visible even if gclient fails. + if exception: + if isinstance(exception[1], KeyboardInterrupt): + sys.stderr.write('Interrupted\n') + elif not isinstance(exception[1], SystemExit): + traceback.print_exception(*exception) - # Check if the version has changed - if (self.config.is_googler - and self.config.opted_in is not False - and self.config.version != metrics_utils.CURRENT_VERSION): - metrics_utils.print_version_change(self.config.version) - self.config.reset_config() + # Check if the version has changed + if (self.config.is_googler and self.config.opted_in is not False + and self.config.version != metrics_utils.CURRENT_VERSION): + metrics_utils.print_version_change(self.config.version) + self.config.reset_config() - # Print the notice - if self.config.is_googler and self.config.opted_in is None: - metrics_utils.print_notice(self.config.countdown) - self.config.decrease_countdown() + # Print the notice + if self.config.is_googler and self.config.opted_in is None: + metrics_utils.print_notice(self.config.countdown) + self.config.decrease_countdown() - sys.exit(metrics_utils.return_code_from_exception(exception)) + sys.exit(metrics_utils.return_code_from_exception(exception)) collector = MetricsCollector() diff --git a/metrics_utils.py b/metrics_utils.py index 931cb38e03..0664e39698 100644 --- a/metrics_utils.py +++ b/metrics_utils.py @@ -12,7 +12,6 @@ import subprocess2 import sys import urllib.parse - # Current version of metrics recording. # When we add new metrics, the version number will be increased, we display the # user what has changed, and ask the user to agree again. @@ -21,223 +20,198 @@ CURRENT_VERSION = 2 APP_URL = 'https://cit-cli-metrics.appspot.com' REPORT_BUILD = os.getenv('DEPOT_TOOLS_REPORT_BUILD') -COLLECT_METRICS = ( - os.getenv('DEPOT_TOOLS_COLLECT_METRICS') != '0' - and os.getenv('DEPOT_TOOLS_METRICS') != '0') +COLLECT_METRICS = (os.getenv('DEPOT_TOOLS_COLLECT_METRICS') != '0' + and os.getenv('DEPOT_TOOLS_METRICS') != '0') SYNC_STATUS_SUCCESS = 'SYNC_STATUS_SUCCESS' SYNC_STATUS_FAILURE = 'SYNC_STATUS_FAILURE' def get_notice_countdown_header(countdown): - if countdown == 0: - yield ' METRICS COLLECTION IS TAKING PLACE' - else: - yield ' METRICS COLLECTION WILL START IN %d EXECUTIONS' % countdown + if countdown == 0: + yield ' METRICS COLLECTION IS TAKING PLACE' + else: + yield ' METRICS COLLECTION WILL START IN %d EXECUTIONS' % countdown + def get_notice_version_change_header(): - yield ' WE ARE COLLECTING ADDITIONAL METRICS' - yield '' - yield ' Please review the changes and opt-in again.' + yield ' WE ARE COLLECTING ADDITIONAL METRICS' + yield '' + yield ' Please review the changes and opt-in again.' + def get_notice_footer(): - yield 'To suppress this message opt in or out using:' - yield '$ gclient metrics [--opt-in] [--opt-out]' - yield 'For more information please see metrics.README.md' - yield 'in your depot_tools checkout or visit' - yield 'https://bit.ly/3MpLAYM.' + yield 'To suppress this message opt in or out using:' + yield '$ gclient metrics [--opt-in] [--opt-out]' + yield 'For more information please see metrics.README.md' + yield 'in your depot_tools checkout or visit' + yield 'https://bit.ly/3MpLAYM.' + def get_change_notice(version): - if version == 0: - return [] # No changes for version 0 + if version == 0: + return [] # No changes for version 0 - if version == 1: - return [ - 'We want to collect the Git version.', - 'We want to collect information about the HTTP', - 'requests that depot_tools makes, and the git and', - 'cipd commands it executes.', - '', - 'We only collect known strings to make sure we', - 'don\'t record PII.', - ] + if version == 1: + return [ + 'We want to collect the Git version.', + 'We want to collect information about the HTTP', + 'requests that depot_tools makes, and the git and', + 'cipd commands it executes.', + '', + 'We only collect known strings to make sure we', + 'don\'t record PII.', + ] - if version == 2: - return [ - 'We will start collecting metrics from bots.', - 'There are no changes for developers.', - 'If the DEPOT_TOOLS_REPORT_BUILD environment variable is set,', - 'we will report information about the current build', - '(e.g. buildbucket project, bucket, builder and build id),', - 'and authenticate to the metrics collection server.', - 'This information will only be recorded for requests', - 'authenticated as bot service accounts.', - ] + if version == 2: + return [ + 'We will start collecting metrics from bots.', + 'There are no changes for developers.', + 'If the DEPOT_TOOLS_REPORT_BUILD environment variable is set,', + 'we will report information about the current build', + '(e.g. buildbucket project, bucket, builder and build id),', + 'and authenticate to the metrics collection server.', + 'This information will only be recorded for requests', + 'authenticated as bot service accounts.', + ] KNOWN_PROJECT_URLS = { - 'https://chrome-internal.googlesource.com/chrome/ios_internal', - 'https://chrome-internal.googlesource.com/infra/infra_internal', - 'https://chromium.googlesource.com/breakpad/breakpad', - 'https://chromium.googlesource.com/chromium/src', - 'https://chromium.googlesource.com/chromium/tools/depot_tools', - 'https://chromium.googlesource.com/crashpad/crashpad', - 'https://chromium.googlesource.com/external/gyp', - 'https://chromium.googlesource.com/external/naclports', - 'https://chromium.googlesource.com/infra/goma/client', - 'https://chromium.googlesource.com/infra/infra', - 'https://chromium.googlesource.com/native_client/', - 'https://chromium.googlesource.com/syzygy', - 'https://chromium.googlesource.com/v8/v8', - 'https://dart.googlesource.com/sdk', - 'https://pdfium.googlesource.com/pdfium', - 'https://skia.googlesource.com/buildbot', - 'https://skia.googlesource.com/skia', - 'https://webrtc.googlesource.com/src', + 'https://chrome-internal.googlesource.com/chrome/ios_internal', + 'https://chrome-internal.googlesource.com/infra/infra_internal', + 'https://chromium.googlesource.com/breakpad/breakpad', + 'https://chromium.googlesource.com/chromium/src', + 'https://chromium.googlesource.com/chromium/tools/depot_tools', + 'https://chromium.googlesource.com/crashpad/crashpad', + 'https://chromium.googlesource.com/external/gyp', + 'https://chromium.googlesource.com/external/naclports', + 'https://chromium.googlesource.com/infra/goma/client', + 'https://chromium.googlesource.com/infra/infra', + 'https://chromium.googlesource.com/native_client/', + 'https://chromium.googlesource.com/syzygy', + 'https://chromium.googlesource.com/v8/v8', + 'https://dart.googlesource.com/sdk', + 'https://pdfium.googlesource.com/pdfium', + 'https://skia.googlesource.com/buildbot', + 'https://skia.googlesource.com/skia', + 'https://webrtc.googlesource.com/src', } KNOWN_HTTP_HOSTS = { - 'chrome-internal-review.googlesource.com', - 'chromium-review.googlesource.com', - 'dart-review.googlesource.com', - 'eu1-mirror-chromium-review.googlesource.com', - 'pdfium-review.googlesource.com', - 'skia-review.googlesource.com', - 'us1-mirror-chromium-review.googlesource.com', - 'us2-mirror-chromium-review.googlesource.com', - 'us3-mirror-chromium-review.googlesource.com', - 'webrtc-review.googlesource.com', + 'chrome-internal-review.googlesource.com', + 'chromium-review.googlesource.com', + 'dart-review.googlesource.com', + 'eu1-mirror-chromium-review.googlesource.com', + 'pdfium-review.googlesource.com', + 'skia-review.googlesource.com', + 'us1-mirror-chromium-review.googlesource.com', + 'us2-mirror-chromium-review.googlesource.com', + 'us3-mirror-chromium-review.googlesource.com', + 'webrtc-review.googlesource.com', } KNOWN_HTTP_METHODS = { - 'DELETE', - 'GET', - 'PATCH', - 'POST', - 'PUT', + 'DELETE', + 'GET', + 'PATCH', + 'POST', + 'PUT', } KNOWN_HTTP_PATHS = { - 'accounts': - re.compile(r'(/a)?/accounts/.*'), - 'changes': - re.compile(r'(/a)?/changes/([^/]+)?$'), - 'changes/abandon': - re.compile(r'(/a)?/changes/.*/abandon'), - 'changes/comments': - re.compile(r'(/a)?/changes/.*/comments'), - 'changes/detail': - re.compile(r'(/a)?/changes/.*/detail'), - 'changes/edit': - re.compile(r'(/a)?/changes/.*/edit'), - 'changes/message': - re.compile(r'(/a)?/changes/.*/message'), - 'changes/restore': - re.compile(r'(/a)?/changes/.*/restore'), - 'changes/reviewers': - re.compile(r'(/a)?/changes/.*/reviewers/.*'), - 'changes/revisions/commit': - re.compile(r'(/a)?/changes/.*/revisions/.*/commit'), - 'changes/revisions/review': - re.compile(r'(/a)?/changes/.*/revisions/.*/review'), - 'changes/submit': - re.compile(r'(/a)?/changes/.*/submit'), - 'projects/branches': - re.compile(r'(/a)?/projects/.*/branches/.*'), + 'accounts': re.compile(r'(/a)?/accounts/.*'), + 'changes': re.compile(r'(/a)?/changes/([^/]+)?$'), + 'changes/abandon': re.compile(r'(/a)?/changes/.*/abandon'), + 'changes/comments': re.compile(r'(/a)?/changes/.*/comments'), + 'changes/detail': re.compile(r'(/a)?/changes/.*/detail'), + 'changes/edit': re.compile(r'(/a)?/changes/.*/edit'), + 'changes/message': re.compile(r'(/a)?/changes/.*/message'), + 'changes/restore': re.compile(r'(/a)?/changes/.*/restore'), + 'changes/reviewers': re.compile(r'(/a)?/changes/.*/reviewers/.*'), + 'changes/revisions/commit': + re.compile(r'(/a)?/changes/.*/revisions/.*/commit'), + 'changes/revisions/review': + re.compile(r'(/a)?/changes/.*/revisions/.*/review'), + 'changes/submit': re.compile(r'(/a)?/changes/.*/submit'), + 'projects/branches': re.compile(r'(/a)?/projects/.*/branches/.*'), } KNOWN_HTTP_ARGS = { - 'ALL_REVISIONS', - 'CURRENT_COMMIT', - 'CURRENT_REVISION', - 'DETAILED_ACCOUNTS', - 'LABELS', + 'ALL_REVISIONS', + 'CURRENT_COMMIT', + 'CURRENT_REVISION', + 'DETAILED_ACCOUNTS', + 'LABELS', } -GIT_VERSION_RE = re.compile( - r'git version (\d)\.(\d{0,2})\.(\d{0,2})' -) +GIT_VERSION_RE = re.compile(r'git version (\d)\.(\d{0,2})\.(\d{0,2})') KNOWN_SUBCOMMAND_ARGS = { - 'cc', - 'hashtag', - 'l=Auto-Submit+1', - 'l=Code-Review+1', - 'l=Code-Review+2', - 'l=Commit-Queue+1', - 'l=Commit-Queue+2', - 'label', - 'm', - 'notify=ALL', - 'notify=NONE', - 'private', - 'r', - 'ready', - 'topic', - 'wip' + 'cc', 'hashtag', 'l=Auto-Submit+1', 'l=Code-Review+1', 'l=Code-Review+2', + 'l=Commit-Queue+1', 'l=Commit-Queue+2', 'label', 'm', 'notify=ALL', + 'notify=NONE', 'private', 'r', 'ready', 'topic', 'wip' } def get_python_version(): - """Return the python version in the major.minor.micro format.""" - return '{v.major}.{v.minor}.{v.micro}'.format(v=sys.version_info) + """Return the python version in the major.minor.micro format.""" + return '{v.major}.{v.minor}.{v.micro}'.format(v=sys.version_info) def get_git_version(): - """Return the Git version in the major.minor.micro format.""" - p = subprocess2.Popen( - ['git', '--version'], - stdout=subprocess2.PIPE, stderr=subprocess2.PIPE) - stdout, _ = p.communicate() - match = GIT_VERSION_RE.match(stdout.decode('utf-8')) - if not match: - return None - return '%s.%s.%s' % match.groups() + """Return the Git version in the major.minor.micro format.""" + p = subprocess2.Popen(['git', '--version'], + stdout=subprocess2.PIPE, + stderr=subprocess2.PIPE) + stdout, _ = p.communicate() + match = GIT_VERSION_RE.match(stdout.decode('utf-8')) + if not match: + return None + return '%s.%s.%s' % match.groups() def get_bot_metrics(): - try: - project, bucket, builder, build = REPORT_BUILD.split('/') - return { - 'build_id': int(build), - 'builder': { - 'project': project, - 'bucket': bucket, - 'builder': builder, - }, - } - except (AttributeError, ValueError): - return None - + try: + project, bucket, builder, build = REPORT_BUILD.split('/') + return { + 'build_id': int(build), + 'builder': { + 'project': project, + 'bucket': bucket, + 'builder': builder, + }, + } + except (AttributeError, ValueError): + return None def return_code_from_exception(exception): - """Returns the exit code that would result of raising the exception.""" - if exception is None: - return 0 - e = exception[1] - if isinstance(e, KeyboardInterrupt): - return 130 - if isinstance(e, SystemExit): - return e.code - return 1 + """Returns the exit code that would result of raising the exception.""" + if exception is None: + return 0 + e = exception[1] + if isinstance(e, KeyboardInterrupt): + return 130 + if isinstance(e, SystemExit): + return e.code + return 1 def extract_known_subcommand_args(args): - """Extract the known arguments from the passed list of args.""" - known_args = [] - for arg in args: - if arg in KNOWN_SUBCOMMAND_ARGS: - known_args.append(arg) - else: - arg = arg.split('=')[0] - if arg in KNOWN_SUBCOMMAND_ARGS: - known_args.append(arg) - return sorted(known_args) + """Extract the known arguments from the passed list of args.""" + known_args = [] + for arg in args: + if arg in KNOWN_SUBCOMMAND_ARGS: + known_args.append(arg) + else: + arg = arg.split('=')[0] + if arg in KNOWN_SUBCOMMAND_ARGS: + known_args.append(arg) + return sorted(known_args) def extract_http_metrics(request_uri, method, status, response_time): - """Extract metrics from the request URI. + """Extract metrics from the request URI. Extracts the host, path, and arguments from the request URI, and returns them along with the method, status and response time. @@ -253,81 +227,82 @@ def extract_http_metrics(request_uri, method, status, response_time): The regex defined in KNOWN_HTTP_PATH_RES are checked against the path, and those that match will be returned. """ - http_metrics = { - 'status': status, - 'response_time': response_time, - } + http_metrics = { + 'status': status, + 'response_time': response_time, + } - if method in KNOWN_HTTP_METHODS: - http_metrics['method'] = method + if method in KNOWN_HTTP_METHODS: + http_metrics['method'] = method - parsed_url = urllib.parse.urlparse(request_uri) + parsed_url = urllib.parse.urlparse(request_uri) - if parsed_url.netloc in KNOWN_HTTP_HOSTS: - http_metrics['host'] = parsed_url.netloc + if parsed_url.netloc in KNOWN_HTTP_HOSTS: + http_metrics['host'] = parsed_url.netloc - for name, path_re in KNOWN_HTTP_PATHS.items(): - if path_re.match(parsed_url.path): - http_metrics['path'] = name - break + for name, path_re in KNOWN_HTTP_PATHS.items(): + if path_re.match(parsed_url.path): + http_metrics['path'] = name + break - parsed_query = urllib.parse.parse_qs(parsed_url.query) + parsed_query = urllib.parse.parse_qs(parsed_url.query) - # Collect o-parameters from the request. - args = [ - arg for arg in parsed_query.get('o', []) - if arg in KNOWN_HTTP_ARGS - ] - if args: - http_metrics['arguments'] = args + # Collect o-parameters from the request. + args = [arg for arg in parsed_query.get('o', []) if arg in KNOWN_HTTP_ARGS] + if args: + http_metrics['arguments'] = args - return http_metrics + return http_metrics def get_repo_timestamp(path_to_repo): - """Get an approximate timestamp for the upstream of |path_to_repo|. + """Get an approximate timestamp for the upstream of |path_to_repo|. Returns the top two bits of the timestamp of the HEAD for the upstream of the branch path_to_repo is checked out at. """ - # Get the upstream for the current branch. If we're not in a branch, fallback - # to HEAD. - try: - upstream = scm.GIT.GetUpstreamBranch(path_to_repo) or 'HEAD' - except subprocess2.CalledProcessError: - upstream = 'HEAD' + # Get the upstream for the current branch. If we're not in a branch, + # fallback to HEAD. + try: + upstream = scm.GIT.GetUpstreamBranch(path_to_repo) or 'HEAD' + except subprocess2.CalledProcessError: + upstream = 'HEAD' - # Get the timestamp of the HEAD for the upstream of the current branch. - p = subprocess2.Popen( - ['git', '-C', path_to_repo, 'log', '-n1', upstream, '--format=%at'], - stdout=subprocess2.PIPE, stderr=subprocess2.PIPE) - stdout, _ = p.communicate() + # Get the timestamp of the HEAD for the upstream of the current branch. + p = subprocess2.Popen( + ['git', '-C', path_to_repo, 'log', '-n1', upstream, '--format=%at'], + stdout=subprocess2.PIPE, + stderr=subprocess2.PIPE) + stdout, _ = p.communicate() - # If there was an error, give up. - if p.returncode != 0: - return None + # If there was an error, give up. + if p.returncode != 0: + return None + + return stdout.strip() - return stdout.strip() def print_boxed_text(out, min_width, lines): - [EW, NS, SE, SW, NE, NW] = list('=|++++') - width = max(min_width, max(len(line) for line in lines)) - out(SE + EW * (width + 2) + SW + '\n') - for line in lines: - out('%s %-*s %s\n' % (NS, width, line, NS)) - out(NE + EW * (width + 2) + NW + '\n') + [EW, NS, SE, SW, NE, NW] = list('=|++++') + width = max(min_width, max(len(line) for line in lines)) + out(SE + EW * (width + 2) + SW + '\n') + for line in lines: + out('%s %-*s %s\n' % (NS, width, line, NS)) + out(NE + EW * (width + 2) + NW + '\n') + def print_notice(countdown): - """Print a notice to let the user know the status of metrics collection.""" - lines = list(get_notice_countdown_header(countdown)) - lines.append('') - lines += list(get_notice_footer()) - print_boxed_text(sys.stderr.write, 49, lines) + """Print a notice to let the user know the status of metrics collection.""" + lines = list(get_notice_countdown_header(countdown)) + lines.append('') + lines += list(get_notice_footer()) + print_boxed_text(sys.stderr.write, 49, lines) + def print_version_change(config_version): - """Print a notice to let the user know we are collecting more metrics.""" - lines = list(get_notice_version_change_header()) - for version in range(config_version + 1, CURRENT_VERSION + 1): - lines.append('') - lines += get_change_notice(version) - print_boxed_text(sys.stderr.write, 49, lines) + """Print a notice to let the user know we are collecting more metrics.""" + lines = list(get_notice_version_change_header()) + for version in range(config_version + 1, CURRENT_VERSION + 1): + lines.append('') + lines += get_change_notice(version) + print_boxed_text(sys.stderr.write, 49, lines) diff --git a/my_activity.py b/my_activity.py index 8db0b13f69..b1824dba67 100755 --- a/my_activity.py +++ b/my_activity.py @@ -2,7 +2,6 @@ # Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Get stats about your activity. Example: @@ -57,259 +56,271 @@ import gclient_utils import gerrit_util if sys.version_info.major == 2: - logging.critical( - 'Python 2 is not supported. Run my_activity.py using vpython3.') - + logging.critical( + 'Python 2 is not supported. Run my_activity.py using vpython3.') try: - import dateutil # pylint: disable=import-error - import dateutil.parser - from dateutil.relativedelta import relativedelta + import dateutil # pylint: disable=import-error + import dateutil.parser + from dateutil.relativedelta import relativedelta except ImportError: - logging.error('python-dateutil package required') - sys.exit(1) + logging.error('python-dateutil package required') + sys.exit(1) class DefaultFormatter(Formatter): - def __init__(self, default = ''): - super(DefaultFormatter, self).__init__() - self.default = default + def __init__(self, default=''): + super(DefaultFormatter, self).__init__() + self.default = default - def get_value(self, key, args, kwargs): - if isinstance(key, str) and key not in kwargs: - return self.default - return Formatter.get_value(self, key, args, kwargs) + def get_value(self, key, args, kwargs): + if isinstance(key, str) and key not in kwargs: + return self.default + return Formatter.get_value(self, key, args, kwargs) gerrit_instances = [ - { - 'url': 'android-review.googlesource.com', - 'shorturl': 'r.android.com', - 'short_url_protocol': 'https', - }, - { - 'url': 'gerrit-review.googlesource.com', - }, - { - 'url': 'chrome-internal-review.googlesource.com', - 'shorturl': 'crrev.com/i', - 'short_url_protocol': 'https', - }, - { - 'url': 'chromium-review.googlesource.com', - 'shorturl': 'crrev.com/c', - 'short_url_protocol': 'https', - }, - { - 'url': 'dawn-review.googlesource.com', - }, - { - 'url': 'pdfium-review.googlesource.com', - }, - { - 'url': 'skia-review.googlesource.com', - }, - { - 'url': 'review.coreboot.org', - }, + { + 'url': 'android-review.googlesource.com', + 'shorturl': 'r.android.com', + 'short_url_protocol': 'https', + }, + { + 'url': 'gerrit-review.googlesource.com', + }, + { + 'url': 'chrome-internal-review.googlesource.com', + 'shorturl': 'crrev.com/i', + 'short_url_protocol': 'https', + }, + { + 'url': 'chromium-review.googlesource.com', + 'shorturl': 'crrev.com/c', + 'short_url_protocol': 'https', + }, + { + 'url': 'dawn-review.googlesource.com', + }, + { + 'url': 'pdfium-review.googlesource.com', + }, + { + 'url': 'skia-review.googlesource.com', + }, + { + 'url': 'review.coreboot.org', + }, ] monorail_projects = { - 'angleproject': { - 'shorturl': 'anglebug.com', - 'short_url_protocol': 'http', - }, - 'chromium': { - 'shorturl': 'crbug.com', - 'short_url_protocol': 'https', - }, - 'dawn': {}, - 'google-breakpad': {}, - 'gyp': {}, - 'pdfium': { - 'shorturl': 'crbug.com/pdfium', - 'short_url_protocol': 'https', - }, - 'skia': {}, - 'tint': {}, - 'v8': { - 'shorturl': 'crbug.com/v8', - 'short_url_protocol': 'https', - }, + 'angleproject': { + 'shorturl': 'anglebug.com', + 'short_url_protocol': 'http', + }, + 'chromium': { + 'shorturl': 'crbug.com', + 'short_url_protocol': 'https', + }, + 'dawn': {}, + 'google-breakpad': {}, + 'gyp': {}, + 'pdfium': { + 'shorturl': 'crbug.com/pdfium', + 'short_url_protocol': 'https', + }, + 'skia': {}, + 'tint': {}, + 'v8': { + 'shorturl': 'crbug.com/v8', + 'short_url_protocol': 'https', + }, } + def username(email): - """Keeps the username of an email address.""" - return email and email.split('@', 1)[0] + """Keeps the username of an email address.""" + return email and email.split('@', 1)[0] def datetime_to_midnight(date): - return date - timedelta(hours=date.hour, minutes=date.minute, - seconds=date.second, microseconds=date.microsecond) + return date - timedelta(hours=date.hour, + minutes=date.minute, + seconds=date.second, + microseconds=date.microsecond) def get_quarter_of(date): - begin = (datetime_to_midnight(date) - - relativedelta(months=(date.month - 1) % 3, days=(date.day - 1))) - return begin, begin + relativedelta(months=3) + begin = (datetime_to_midnight(date) - + relativedelta(months=(date.month - 1) % 3, days=(date.day - 1))) + return begin, begin + relativedelta(months=3) def get_year_of(date): - begin = (datetime_to_midnight(date) - - relativedelta(months=(date.month - 1), days=(date.day - 1))) - return begin, begin + relativedelta(years=1) + begin = (datetime_to_midnight(date) - + relativedelta(months=(date.month - 1), days=(date.day - 1))) + return begin, begin + relativedelta(years=1) def get_week_of(date): - begin = (datetime_to_midnight(date) - timedelta(days=date.weekday())) - return begin, begin + timedelta(days=7) + begin = (datetime_to_midnight(date) - timedelta(days=date.weekday())) + return begin, begin + timedelta(days=7) def get_yes_or_no(msg): - while True: - response = gclient_utils.AskForData(msg + ' yes/no [no] ') - if response in ('y', 'yes'): - return True + while True: + response = gclient_utils.AskForData(msg + ' yes/no [no] ') + if response in ('y', 'yes'): + return True - if not response or response in ('n', 'no'): - return False + if not response or response in ('n', 'no'): + return False def datetime_from_gerrit(date_string): - return datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S.%f000') + return datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S.%f000') def datetime_from_monorail(date_string): - return datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%S') + return datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%S') + def extract_bug_numbers_from_description(issue): - # Getting the description for REST Gerrit - revision = issue['revisions'][issue['current_revision']] - description = revision['commit']['message'] + # Getting the description for REST Gerrit + revision = issue['revisions'][issue['current_revision']] + description = revision['commit']['message'] - bugs = [] - # Handle both "Bug: 99999" and "BUG=99999" bug notations - # Multiple bugs can be noted on a single line or in multiple ones. - matches = re.findall( - r'^(BUG=|(Bug|Fixed):\s*)((((?:[a-zA-Z0-9-]+:)?\d+)(,\s?)?)+)', - description, flags=re.IGNORECASE | re.MULTILINE) - if matches: - for match in matches: - bugs.extend(match[2].replace(' ', '').split(',')) - # Add default chromium: prefix if none specified. - bugs = [bug if ':' in bug else 'chromium:%s' % bug for bug in bugs] + bugs = [] + # Handle both "Bug: 99999" and "BUG=99999" bug notations + # Multiple bugs can be noted on a single line or in multiple ones. + matches = re.findall( + r'^(BUG=|(Bug|Fixed):\s*)((((?:[a-zA-Z0-9-]+:)?\d+)(,\s?)?)+)', + description, + flags=re.IGNORECASE | re.MULTILINE) + if matches: + for match in matches: + bugs.extend(match[2].replace(' ', '').split(',')) + # Add default chromium: prefix if none specified. + bugs = [bug if ':' in bug else 'chromium:%s' % bug for bug in bugs] + + return sorted(set(bugs)) - return sorted(set(bugs)) class MyActivity(object): - def __init__(self, options): - self.options = options - self.modified_after = options.begin - self.modified_before = options.end - self.user = options.user - self.changes = [] - self.reviews = [] - self.issues = [] - self.referenced_issues = [] - self.google_code_auth_token = None - self.access_errors = set() - self.skip_servers = (options.skip_servers.split(',')) + def __init__(self, options): + self.options = options + self.modified_after = options.begin + self.modified_before = options.end + self.user = options.user + self.changes = [] + self.reviews = [] + self.issues = [] + self.referenced_issues = [] + self.google_code_auth_token = None + self.access_errors = set() + self.skip_servers = (options.skip_servers.split(',')) - def show_progress(self, how='.'): - if sys.stdout.isatty(): - sys.stdout.write(how) - sys.stdout.flush() + def show_progress(self, how='.'): + if sys.stdout.isatty(): + sys.stdout.write(how) + sys.stdout.flush() - def gerrit_changes_over_rest(self, instance, filters): - # Convert the "key:value" filter to a list of (key, value) pairs. - req = list(f.split(':', 1) for f in filters) - try: - # Instantiate the generator to force all the requests now and catch the - # errors here. - return list(gerrit_util.GenerateAllChanges(instance['url'], req, - o_params=['MESSAGES', 'LABELS', 'DETAILED_ACCOUNTS', - 'CURRENT_REVISION', 'CURRENT_COMMIT'])) - except gerrit_util.GerritError as e: - error_message = 'Looking up %r: %s' % (instance['url'], e) - if error_message not in self.access_errors: - self.access_errors.add(error_message) - return [] + def gerrit_changes_over_rest(self, instance, filters): + # Convert the "key:value" filter to a list of (key, value) pairs. + req = list(f.split(':', 1) for f in filters) + try: + # Instantiate the generator to force all the requests now and catch + # the errors here. + return list( + gerrit_util.GenerateAllChanges(instance['url'], + req, + o_params=[ + 'MESSAGES', 'LABELS', + 'DETAILED_ACCOUNTS', + 'CURRENT_REVISION', + 'CURRENT_COMMIT' + ])) + except gerrit_util.GerritError as e: + error_message = 'Looking up %r: %s' % (instance['url'], e) + if error_message not in self.access_errors: + self.access_errors.add(error_message) + return [] - def gerrit_search(self, instance, owner=None, reviewer=None): - if instance['url'] in self.skip_servers: - return [] - max_age = datetime.today() - self.modified_after - filters = ['-age:%ss' % (max_age.days * 24 * 3600 + max_age.seconds)] - if owner: - assert not reviewer - filters.append('owner:%s' % owner) - else: - filters.extend(('-owner:%s' % reviewer, 'reviewer:%s' % reviewer)) - # TODO(cjhopman): Should abandoned changes be filtered out when - # merged_only is not enabled? - if self.options.merged_only: - filters.append('status:merged') + def gerrit_search(self, instance, owner=None, reviewer=None): + if instance['url'] in self.skip_servers: + return [] + max_age = datetime.today() - self.modified_after + filters = ['-age:%ss' % (max_age.days * 24 * 3600 + max_age.seconds)] + if owner: + assert not reviewer + filters.append('owner:%s' % owner) + else: + filters.extend(('-owner:%s' % reviewer, 'reviewer:%s' % reviewer)) + # TODO(cjhopman): Should abandoned changes be filtered out when + # merged_only is not enabled? + if self.options.merged_only: + filters.append('status:merged') - issues = self.gerrit_changes_over_rest(instance, filters) - self.show_progress() - issues = [self.process_gerrit_issue(instance, issue) - for issue in issues] + issues = self.gerrit_changes_over_rest(instance, filters) + self.show_progress() + issues = [ + self.process_gerrit_issue(instance, issue) for issue in issues + ] - issues = filter(self.filter_issue, issues) - issues = sorted(issues, key=lambda i: i['modified'], reverse=True) + issues = filter(self.filter_issue, issues) + issues = sorted(issues, key=lambda i: i['modified'], reverse=True) - return issues + return issues - def process_gerrit_issue(self, instance, issue): - ret = {} - if self.options.deltas: - ret['delta'] = DefaultFormatter().format( - '+{insertions},-{deletions}', - **issue) - ret['status'] = issue['status'] - if 'shorturl' in instance: - protocol = instance.get('short_url_protocol', 'http') - url = instance['shorturl'] - else: - protocol = 'https' - url = instance['url'] - ret['review_url'] = '%s://%s/%s' % (protocol, url, issue['_number']) + def process_gerrit_issue(self, instance, issue): + ret = {} + if self.options.deltas: + ret['delta'] = DefaultFormatter().format( + '+{insertions},-{deletions}', **issue) + ret['status'] = issue['status'] + if 'shorturl' in instance: + protocol = instance.get('short_url_protocol', 'http') + url = instance['shorturl'] + else: + protocol = 'https' + url = instance['url'] + ret['review_url'] = '%s://%s/%s' % (protocol, url, issue['_number']) - ret['header'] = issue['subject'] - ret['owner'] = issue['owner'].get('email', '') - ret['author'] = ret['owner'] - ret['created'] = datetime_from_gerrit(issue['created']) - ret['modified'] = datetime_from_gerrit(issue['updated']) - if 'messages' in issue: - ret['replies'] = self.process_gerrit_issue_replies(issue['messages']) - else: - ret['replies'] = [] - ret['reviewers'] = set(r['author'] for r in ret['replies']) - ret['reviewers'].discard(ret['author']) - ret['bugs'] = extract_bug_numbers_from_description(issue) - return ret + ret['header'] = issue['subject'] + ret['owner'] = issue['owner'].get('email', '') + ret['author'] = ret['owner'] + ret['created'] = datetime_from_gerrit(issue['created']) + ret['modified'] = datetime_from_gerrit(issue['updated']) + if 'messages' in issue: + ret['replies'] = self.process_gerrit_issue_replies( + issue['messages']) + else: + ret['replies'] = [] + ret['reviewers'] = set(r['author'] for r in ret['replies']) + ret['reviewers'].discard(ret['author']) + ret['bugs'] = extract_bug_numbers_from_description(issue) + return ret - @staticmethod - def process_gerrit_issue_replies(replies): - ret = [] - replies = filter(lambda r: 'author' in r and 'email' in r['author'], - replies) - for reply in replies: - ret.append({ - 'author': reply['author']['email'], - 'created': datetime_from_gerrit(reply['date']), - 'content': reply['message'], - }) - return ret + @staticmethod + def process_gerrit_issue_replies(replies): + ret = [] + replies = filter(lambda r: 'author' in r and 'email' in r['author'], + replies) + for reply in replies: + ret.append({ + 'author': reply['author']['email'], + 'created': datetime_from_gerrit(reply['date']), + 'content': reply['message'], + }) + return ret - def monorail_get_auth_http(self): - # Manually use a long timeout (10m); for some users who have a - # long history on the issue tracker, whatever the default timeout - # is is reached. - return auth.Authenticator().authorize(httplib2.Http(timeout=600)) + def monorail_get_auth_http(self): + # Manually use a long timeout (10m); for some users who have a + # long history on the issue tracker, whatever the default timeout + # is is reached. + return auth.Authenticator().authorize(httplib2.Http(timeout=600)) - def filter_modified_monorail_issue(self, issue): - """Precisely checks if an issue has been modified in the time range. + def filter_modified_monorail_issue(self, issue): + """Precisely checks if an issue has been modified in the time range. This fetches all issue comments to check if the issue has been modified in the time range specified by user. This is needed because monorail only @@ -324,682 +335,719 @@ class MyActivity(object): Returns: Passed issue if modified, None otherwise. """ - http = self.monorail_get_auth_http() - project, issue_id = issue['uid'].split(':') - url = ('https://monorail-prod.appspot.com/_ah/api/monorail/v1/projects' - '/%s/issues/%s/comments?maxResults=10000') % (project, issue_id) - _, body = http.request(url) - self.show_progress() - content = json.loads(body) - if not content: - logging.error('Unable to parse %s response from monorail.', project) - return issue + http = self.monorail_get_auth_http() + project, issue_id = issue['uid'].split(':') + url = ('https://monorail-prod.appspot.com/_ah/api/monorail/v1/projects' + '/%s/issues/%s/comments?maxResults=10000') % (project, issue_id) + _, body = http.request(url) + self.show_progress() + content = json.loads(body) + if not content: + logging.error('Unable to parse %s response from monorail.', project) + return issue - for item in content.get('items', []): - comment_published = datetime_from_monorail(item['published']) - if self.filter_modified(comment_published): - return issue + for item in content.get('items', []): + comment_published = datetime_from_monorail(item['published']) + if self.filter_modified(comment_published): + return issue - return None + return None - def monorail_query_issues(self, project, query): - http = self.monorail_get_auth_http() - url = ('https://monorail-prod.appspot.com/_ah/api/monorail/v1/projects' - '/%s/issues') % project - query_data = urllib.parse.urlencode(query) - url = url + '?' + query_data - _, body = http.request(url) - self.show_progress() - content = json.loads(body) - if not content: - logging.error('Unable to parse %s response from monorail.', project) - return [] + def monorail_query_issues(self, project, query): + http = self.monorail_get_auth_http() + url = ('https://monorail-prod.appspot.com/_ah/api/monorail/v1/projects' + '/%s/issues') % project + query_data = urllib.parse.urlencode(query) + url = url + '?' + query_data + _, body = http.request(url) + self.show_progress() + content = json.loads(body) + if not content: + logging.error('Unable to parse %s response from monorail.', project) + return [] - issues = [] - project_config = monorail_projects.get(project, {}) - for item in content.get('items', []): - if project_config.get('shorturl'): - protocol = project_config.get('short_url_protocol', 'http') - item_url = '%s://%s/%d' % ( - protocol, project_config['shorturl'], item['id']) - else: - item_url = 'https://bugs.chromium.org/p/%s/issues/detail?id=%d' % ( - project, item['id']) - issue = { - 'uid': '%s:%s' % (project, item['id']), - 'header': item['title'], - 'created': datetime_from_monorail(item['published']), - 'modified': datetime_from_monorail(item['updated']), - 'author': item['author']['name'], - 'url': item_url, - 'comments': [], - 'status': item['status'], - 'labels': [], - 'components': [] - } - if 'owner' in item: - issue['owner'] = item['owner']['name'] - else: - issue['owner'] = 'None' - if 'labels' in item: - issue['labels'] = item['labels'] - if 'components' in item: - issue['components'] = item['components'] - issues.append(issue) + issues = [] + project_config = monorail_projects.get(project, {}) + for item in content.get('items', []): + if project_config.get('shorturl'): + protocol = project_config.get('short_url_protocol', 'http') + item_url = '%s://%s/%d' % (protocol, project_config['shorturl'], + item['id']) + else: + item_url = ( + 'https://bugs.chromium.org/p/%s/issues/detail?id=%d' % + (project, item['id'])) + issue = { + 'uid': '%s:%s' % (project, item['id']), + 'header': item['title'], + 'created': datetime_from_monorail(item['published']), + 'modified': datetime_from_monorail(item['updated']), + 'author': item['author']['name'], + 'url': item_url, + 'comments': [], + 'status': item['status'], + 'labels': [], + 'components': [] + } + if 'owner' in item: + issue['owner'] = item['owner']['name'] + else: + issue['owner'] = 'None' + if 'labels' in item: + issue['labels'] = item['labels'] + if 'components' in item: + issue['components'] = item['components'] + issues.append(issue) - return issues + return issues - def monorail_issue_search(self, project): - epoch = datetime.utcfromtimestamp(0) - # Defaults to @chromium.org email if one wasn't provided on -u option. - user_str = (self.options.email if self.options.email.find('@') >= 0 - else '%s@chromium.org' % self.user) + def monorail_issue_search(self, project): + epoch = datetime.utcfromtimestamp(0) + # Defaults to @chromium.org email if one wasn't provided on -u option. + user_str = (self.options.email if self.options.email.find('@') >= 0 else + '%s@chromium.org' % self.user) - issues = self.monorail_query_issues(project, { - 'maxResults': 10000, - 'q': user_str, - 'publishedMax': '%d' % (self.modified_before - epoch).total_seconds(), - 'updatedMin': '%d' % (self.modified_after - epoch).total_seconds(), - }) + issues = self.monorail_query_issues( + project, { + 'maxResults': + 10000, + 'q': + user_str, + 'publishedMax': + '%d' % (self.modified_before - epoch).total_seconds(), + 'updatedMin': + '%d' % (self.modified_after - epoch).total_seconds(), + }) - if self.options.completed_issues: - return [ - issue for issue in issues - if (self.match(issue['owner']) and - issue['status'].lower() in ('verified', 'fixed')) - ] + if self.options.completed_issues: + return [ + issue for issue in issues + if (self.match(issue['owner']) and issue['status'].lower() in ( + 'verified', 'fixed')) + ] - return [ - issue for issue in issues - if user_str in (issue['author'], issue['owner'])] + return [ + issue for issue in issues + if user_str in (issue['author'], issue['owner']) + ] - def monorail_get_issues(self, project, issue_ids): - return self.monorail_query_issues(project, { - 'maxResults': 10000, - 'q': 'id:%s' % ','.join(issue_ids) - }) + def monorail_get_issues(self, project, issue_ids): + return self.monorail_query_issues(project, { + 'maxResults': 10000, + 'q': 'id:%s' % ','.join(issue_ids) + }) - def print_heading(self, heading): - print() - print(self.options.output_format_heading.format(heading=heading)) - - def match(self, author): - if '@' in self.user: - return author == self.user - return author.startswith(self.user + '@') - - def print_change(self, change): - activity = len([ - reply - for reply in change['replies'] - if self.match(reply['author']) - ]) - optional_values = { - 'created': change['created'].date().isoformat(), - 'modified': change['modified'].date().isoformat(), - 'reviewers': ', '.join(change['reviewers']), - 'status': change['status'], - 'activity': activity, - } - if self.options.deltas: - optional_values['delta'] = change['delta'] - - self.print_generic(self.options.output_format, - self.options.output_format_changes, - change['header'], - change['review_url'], - change['author'], - change['created'], - change['modified'], - optional_values) - - def print_issue(self, issue): - optional_values = { - 'created': issue['created'].date().isoformat(), - 'modified': issue['modified'].date().isoformat(), - 'owner': issue['owner'], - 'status': issue['status'], - } - self.print_generic(self.options.output_format, - self.options.output_format_issues, - issue['header'], - issue['url'], - issue['author'], - issue['created'], - issue['modified'], - optional_values) - - def print_review(self, review): - activity = len([ - reply - for reply in review['replies'] - if self.match(reply['author']) - ]) - optional_values = { - 'created': review['created'].date().isoformat(), - 'modified': review['modified'].date().isoformat(), - 'status': review['status'], - 'activity': activity, - } - if self.options.deltas: - optional_values['delta'] = review['delta'] - - self.print_generic(self.options.output_format, - self.options.output_format_reviews, - review['header'], - review['review_url'], - review['author'], - review['created'], - review['modified'], - optional_values) - - @staticmethod - def print_generic(default_fmt, specific_fmt, - title, url, author, created, modified, - optional_values=None): - output_format = specific_fmt if specific_fmt is not None else default_fmt - values = { - 'title': title, - 'url': url, - 'author': author, - 'created': created, - 'modified': modified, - } - if optional_values is not None: - values.update(optional_values) - print(DefaultFormatter().format(output_format, **values)) - - - def filter_issue(self, issue, should_filter_by_user=True): - def maybe_filter_username(email): - return not should_filter_by_user or username(email) == self.user - if (maybe_filter_username(issue['author']) and - self.filter_modified(issue['created'])): - return True - if (maybe_filter_username(issue['owner']) and - (self.filter_modified(issue['created']) or - self.filter_modified(issue['modified']))): - return True - for reply in issue['replies']: - if self.filter_modified(reply['created']): - if not should_filter_by_user: - break - if (username(reply['author']) == self.user - or (self.user + '@') in reply['content']): - break - else: - return False - return True - - def filter_modified(self, modified): - return self.modified_after < modified < self.modified_before - - def auth_for_changes(self): - #TODO(cjhopman): Move authentication check for getting changes here. - pass - - def auth_for_reviews(self): - # Reviews use all the same instances as changes so no authentication is - # required. - pass - - def get_changes(self): - num_instances = len(gerrit_instances) - with contextlib.closing(ThreadPool(num_instances)) as pool: - gerrit_changes = pool.map_async( - lambda instance: self.gerrit_search(instance, owner=self.user), - gerrit_instances) - gerrit_changes = itertools.chain.from_iterable(gerrit_changes.get()) - self.changes = list(gerrit_changes) - - def print_changes(self): - if self.changes: - self.print_heading('Changes') - for change in self.changes: - self.print_change(change) - - def print_access_errors(self): - if self.access_errors: - logging.error('Access Errors:') - for error in self.access_errors: - logging.error(error.rstrip()) - - def get_reviews(self): - num_instances = len(gerrit_instances) - with contextlib.closing(ThreadPool(num_instances)) as pool: - gerrit_reviews = pool.map_async( - lambda instance: self.gerrit_search(instance, reviewer=self.user), - gerrit_instances) - gerrit_reviews = itertools.chain.from_iterable(gerrit_reviews.get()) - self.reviews = list(gerrit_reviews) - - def print_reviews(self): - if self.reviews: - self.print_heading('Reviews') - for review in self.reviews: - self.print_review(review) - - def get_issues(self): - with contextlib.closing(ThreadPool(len(monorail_projects))) as pool: - monorail_issues = pool.map( - self.monorail_issue_search, monorail_projects.keys()) - monorail_issues = list(itertools.chain.from_iterable(monorail_issues)) - - if not monorail_issues: - return - - with contextlib.closing(ThreadPool(len(monorail_issues))) as pool: - filtered_issues = pool.map( - self.filter_modified_monorail_issue, monorail_issues) - self.issues = [issue for issue in filtered_issues if issue] - - def get_referenced_issues(self): - if not self.issues: - self.get_issues() - - if not self.changes: - self.get_changes() - - referenced_issue_uids = set(itertools.chain.from_iterable( - change['bugs'] for change in self.changes)) - fetched_issue_uids = set(issue['uid'] for issue in self.issues) - missing_issue_uids = referenced_issue_uids - fetched_issue_uids - - missing_issues_by_project = collections.defaultdict(list) - for issue_uid in missing_issue_uids: - project, issue_id = issue_uid.split(':') - missing_issues_by_project[project].append(issue_id) - - for project, issue_ids in missing_issues_by_project.items(): - self.referenced_issues += self.monorail_get_issues(project, issue_ids) - - def print_issues(self): - if self.issues: - self.print_heading('Issues') - for issue in self.issues: - self.print_issue(issue) - - def print_changes_by_issue(self, skip_empty_own): - if not self.issues or not self.changes: - return - - self.print_heading('Changes by referenced issue(s)') - issues = {issue['uid']: issue for issue in self.issues} - ref_issues = {issue['uid']: issue for issue in self.referenced_issues} - changes_by_issue_uid = collections.defaultdict(list) - changes_by_ref_issue_uid = collections.defaultdict(list) - changes_without_issue = [] - for change in self.changes: - added = False - for issue_uid in change['bugs']: - if issue_uid in issues: - changes_by_issue_uid[issue_uid].append(change) - added = True - if issue_uid in ref_issues: - changes_by_ref_issue_uid[issue_uid].append(change) - added = True - if not added: - changes_without_issue.append(change) - - # Changes referencing own issues. - for issue_uid in issues: - if changes_by_issue_uid[issue_uid] or not skip_empty_own: - self.print_issue(issues[issue_uid]) - if changes_by_issue_uid[issue_uid]: - print() - for change in changes_by_issue_uid[issue_uid]: - print(' ', end='') # this prints no newline - self.print_change(change) + def print_heading(self, heading): print() + print(self.options.output_format_heading.format(heading=heading)) - # Changes referencing others' issues. - for issue_uid in ref_issues: - assert changes_by_ref_issue_uid[issue_uid] - self.print_issue(ref_issues[issue_uid]) - for change in changes_by_ref_issue_uid[issue_uid]: - print('', end=' ') # this prints one space due to comma, but no newline - self.print_change(change) + def match(self, author): + if '@' in self.user: + return author == self.user + return author.startswith(self.user + '@') - # Changes referencing no issues. - if changes_without_issue: - print(self.options.output_format_no_url.format(title='Other changes')) - for change in changes_without_issue: - print('', end=' ') # this prints one space due to comma, but no newline - self.print_change(change) + def print_change(self, change): + activity = len([ + reply for reply in change['replies'] if self.match(reply['author']) + ]) + optional_values = { + 'created': change['created'].date().isoformat(), + 'modified': change['modified'].date().isoformat(), + 'reviewers': ', '.join(change['reviewers']), + 'status': change['status'], + 'activity': activity, + } + if self.options.deltas: + optional_values['delta'] = change['delta'] - def print_activity(self): - self.print_changes() - self.print_reviews() - self.print_issues() + self.print_generic(self.options.output_format, + self.options.output_format_changes, change['header'], + change['review_url'], change['author'], + change['created'], change['modified'], + optional_values) - def dump_json(self, ignore_keys=None): - if ignore_keys is None: - ignore_keys = ['replies'] + def print_issue(self, issue): + optional_values = { + 'created': issue['created'].date().isoformat(), + 'modified': issue['modified'].date().isoformat(), + 'owner': issue['owner'], + 'status': issue['status'], + } + self.print_generic(self.options.output_format, + self.options.output_format_issues, issue['header'], + issue['url'], issue['author'], issue['created'], + issue['modified'], optional_values) - def format_for_json_dump(in_array): - output = {} - for item in in_array: - url = item.get('url') or item.get('review_url') - if not url: - raise Exception('Dumped item %s does not specify url' % item) - output[url] = dict( - (k, v) for k,v in item.items() if k not in ignore_keys) - return output + def print_review(self, review): + activity = len([ + reply for reply in review['replies'] if self.match(reply['author']) + ]) + optional_values = { + 'created': review['created'].date().isoformat(), + 'modified': review['modified'].date().isoformat(), + 'status': review['status'], + 'activity': activity, + } + if self.options.deltas: + optional_values['delta'] = review['delta'] - class PythonObjectEncoder(json.JSONEncoder): - def default(self, o): # pylint: disable=method-hidden - if isinstance(o, datetime): - return o.isoformat() - if isinstance(o, set): - return list(o) - return json.JSONEncoder.default(self, o) + self.print_generic(self.options.output_format, + self.options.output_format_reviews, review['header'], + review['review_url'], review['author'], + review['created'], review['modified'], + optional_values) - output = { - 'reviews': format_for_json_dump(self.reviews), - 'changes': format_for_json_dump(self.changes), - 'issues': format_for_json_dump(self.issues) - } - print(json.dumps(output, indent=2, cls=PythonObjectEncoder)) + @staticmethod + def print_generic(default_fmt, + specific_fmt, + title, + url, + author, + created, + modified, + optional_values=None): + output_format = (specific_fmt + if specific_fmt is not None else default_fmt) + values = { + 'title': title, + 'url': url, + 'author': author, + 'created': created, + 'modified': modified, + } + if optional_values is not None: + values.update(optional_values) + print(DefaultFormatter().format(output_format, **values)) + + def filter_issue(self, issue, should_filter_by_user=True): + def maybe_filter_username(email): + return not should_filter_by_user or username(email) == self.user + + if (maybe_filter_username(issue['author']) + and self.filter_modified(issue['created'])): + return True + if (maybe_filter_username(issue['owner']) + and (self.filter_modified(issue['created']) + or self.filter_modified(issue['modified']))): + return True + for reply in issue['replies']: + if self.filter_modified(reply['created']): + if not should_filter_by_user: + break + if (username(reply['author']) == self.user + or (self.user + '@') in reply['content']): + break + else: + return False + return True + + def filter_modified(self, modified): + return self.modified_after < modified < self.modified_before + + def auth_for_changes(self): + #TODO(cjhopman): Move authentication check for getting changes here. + pass + + def auth_for_reviews(self): + # Reviews use all the same instances as changes so no authentication is + # required. + pass + + def get_changes(self): + num_instances = len(gerrit_instances) + with contextlib.closing(ThreadPool(num_instances)) as pool: + gerrit_changes = pool.map_async( + lambda instance: self.gerrit_search(instance, owner=self.user), + gerrit_instances) + gerrit_changes = itertools.chain.from_iterable(gerrit_changes.get()) + self.changes = list(gerrit_changes) + + def print_changes(self): + if self.changes: + self.print_heading('Changes') + for change in self.changes: + self.print_change(change) + + def print_access_errors(self): + if self.access_errors: + logging.error('Access Errors:') + for error in self.access_errors: + logging.error(error.rstrip()) + + def get_reviews(self): + num_instances = len(gerrit_instances) + with contextlib.closing(ThreadPool(num_instances)) as pool: + gerrit_reviews = pool.map_async( + lambda instance: self.gerrit_search(instance, + reviewer=self.user), + gerrit_instances) + gerrit_reviews = itertools.chain.from_iterable(gerrit_reviews.get()) + self.reviews = list(gerrit_reviews) + + def print_reviews(self): + if self.reviews: + self.print_heading('Reviews') + for review in self.reviews: + self.print_review(review) + + def get_issues(self): + with contextlib.closing(ThreadPool(len(monorail_projects))) as pool: + monorail_issues = pool.map(self.monorail_issue_search, + monorail_projects.keys()) + monorail_issues = list( + itertools.chain.from_iterable(monorail_issues)) + + if not monorail_issues: + return + + with contextlib.closing(ThreadPool(len(monorail_issues))) as pool: + filtered_issues = pool.map(self.filter_modified_monorail_issue, + monorail_issues) + self.issues = [issue for issue in filtered_issues if issue] + + def get_referenced_issues(self): + if not self.issues: + self.get_issues() + + if not self.changes: + self.get_changes() + + referenced_issue_uids = set( + itertools.chain.from_iterable(change['bugs'] + for change in self.changes)) + fetched_issue_uids = set(issue['uid'] for issue in self.issues) + missing_issue_uids = referenced_issue_uids - fetched_issue_uids + + missing_issues_by_project = collections.defaultdict(list) + for issue_uid in missing_issue_uids: + project, issue_id = issue_uid.split(':') + missing_issues_by_project[project].append(issue_id) + + for project, issue_ids in missing_issues_by_project.items(): + self.referenced_issues += self.monorail_get_issues( + project, issue_ids) + + def print_issues(self): + if self.issues: + self.print_heading('Issues') + for issue in self.issues: + self.print_issue(issue) + + def print_changes_by_issue(self, skip_empty_own): + if not self.issues or not self.changes: + return + + self.print_heading('Changes by referenced issue(s)') + issues = {issue['uid']: issue for issue in self.issues} + ref_issues = {issue['uid']: issue for issue in self.referenced_issues} + changes_by_issue_uid = collections.defaultdict(list) + changes_by_ref_issue_uid = collections.defaultdict(list) + changes_without_issue = [] + for change in self.changes: + added = False + for issue_uid in change['bugs']: + if issue_uid in issues: + changes_by_issue_uid[issue_uid].append(change) + added = True + if issue_uid in ref_issues: + changes_by_ref_issue_uid[issue_uid].append(change) + added = True + if not added: + changes_without_issue.append(change) + + # Changes referencing own issues. + for issue_uid in issues: + if changes_by_issue_uid[issue_uid] or not skip_empty_own: + self.print_issue(issues[issue_uid]) + if changes_by_issue_uid[issue_uid]: + print() + for change in changes_by_issue_uid[issue_uid]: + print(' ', end='') # this prints no newline + self.print_change(change) + print() + + # Changes referencing others' issues. + for issue_uid in ref_issues: + assert changes_by_ref_issue_uid[issue_uid] + self.print_issue(ref_issues[issue_uid]) + for change in changes_by_ref_issue_uid[issue_uid]: + print('', end=' ' + ) # this prints one space due to comma, but no newline + self.print_change(change) + + # Changes referencing no issues. + if changes_without_issue: + print( + self.options.output_format_no_url.format(title='Other changes')) + for change in changes_without_issue: + print('', end=' ' + ) # this prints one space due to comma, but no newline + self.print_change(change) + + def print_activity(self): + self.print_changes() + self.print_reviews() + self.print_issues() + + def dump_json(self, ignore_keys=None): + if ignore_keys is None: + ignore_keys = ['replies'] + + def format_for_json_dump(in_array): + output = {} + for item in in_array: + url = item.get('url') or item.get('review_url') + if not url: + raise Exception('Dumped item %s does not specify url' % + item) + output[url] = dict( + (k, v) for k, v in item.items() if k not in ignore_keys) + return output + + class PythonObjectEncoder(json.JSONEncoder): + def default(self, o): # pylint: disable=method-hidden + if isinstance(o, datetime): + return o.isoformat() + if isinstance(o, set): + return list(o) + return json.JSONEncoder.default(self, o) + + output = { + 'reviews': format_for_json_dump(self.reviews), + 'changes': format_for_json_dump(self.changes), + 'issues': format_for_json_dump(self.issues) + } + print(json.dumps(output, indent=2, cls=PythonObjectEncoder)) def main(): - parser = optparse.OptionParser(description=sys.modules[__name__].__doc__) - parser.add_option( - '-u', '--user', metavar='', - # Look for USER and USERNAME (Windows) environment variables. - default=os.environ.get('USER', os.environ.get('USERNAME')), - help='Filter on user, default=%default') - parser.add_option( - '-b', '--begin', metavar='', - help='Filter issues created after the date (mm/dd/yy)') - parser.add_option( - '-e', '--end', metavar='', - help='Filter issues created before the date (mm/dd/yy)') - quarter_begin, quarter_end = get_quarter_of(datetime.today() - - relativedelta(months=2)) - parser.add_option( - '-Q', '--last_quarter', action='store_true', - help='Use last quarter\'s dates, i.e. %s to %s' % ( - quarter_begin.strftime('%Y-%m-%d'), quarter_end.strftime('%Y-%m-%d'))) - parser.add_option( - '-Y', '--this_year', action='store_true', - help='Use this year\'s dates') - parser.add_option( - '-w', '--week_of', metavar='', - help='Show issues for week of the date (mm/dd/yy)') - parser.add_option( - '-W', '--last_week', action='count', - help='Show last week\'s issues. Use more times for more weeks.') - parser.add_option( - '-a', '--auth', - action='store_true', - help='Ask to authenticate for instances with no auth cookie') - parser.add_option( - '-d', '--deltas', - action='store_true', - help='Fetch deltas for changes.') - parser.add_option( - '--no-referenced-issues', - action='store_true', - help='Do not fetch issues referenced by owned changes. Useful in ' - 'combination with --changes-by-issue when you only want to list ' - 'issues that have also been modified in the same time period.') - parser.add_option( - '--skip_servers', - action='store', - default='', - help='A comma separated list of gerrit and rietveld servers to ignore') - parser.add_option( - '--skip-own-issues-without-changes', - action='store_true', - help='Skips listing own issues without changes when showing changes ' - 'grouped by referenced issue(s). See --changes-by-issue for more ' - 'details.') - parser.add_option( - '-F', '--config_file', metavar='', - help='Configuration file in JSON format, used to add additional gerrit ' - 'instances (see source code for an example).') + parser = optparse.OptionParser(description=sys.modules[__name__].__doc__) + parser.add_option( + '-u', + '--user', + metavar='', + # Look for USER and USERNAME (Windows) environment variables. + default=os.environ.get('USER', os.environ.get('USERNAME')), + help='Filter on user, default=%default') + parser.add_option('-b', + '--begin', + metavar='', + help='Filter issues created after the date (mm/dd/yy)') + parser.add_option('-e', + '--end', + metavar='', + help='Filter issues created before the date (mm/dd/yy)') + quarter_begin, quarter_end = get_quarter_of(datetime.today() - + relativedelta(months=2)) + parser.add_option( + '-Q', + '--last_quarter', + action='store_true', + help='Use last quarter\'s dates, i.e. %s to %s' % + (quarter_begin.strftime('%Y-%m-%d'), quarter_end.strftime('%Y-%m-%d'))) + parser.add_option('-Y', + '--this_year', + action='store_true', + help='Use this year\'s dates') + parser.add_option('-w', + '--week_of', + metavar='', + help='Show issues for week of the date (mm/dd/yy)') + parser.add_option( + '-W', + '--last_week', + action='count', + help='Show last week\'s issues. Use more times for more weeks.') + parser.add_option( + '-a', + '--auth', + action='store_true', + help='Ask to authenticate for instances with no auth cookie') + parser.add_option('-d', + '--deltas', + action='store_true', + help='Fetch deltas for changes.') + parser.add_option( + '--no-referenced-issues', + action='store_true', + help='Do not fetch issues referenced by owned changes. Useful in ' + 'combination with --changes-by-issue when you only want to list ' + 'issues that have also been modified in the same time period.') + parser.add_option( + '--skip_servers', + action='store', + default='', + help='A comma separated list of gerrit and rietveld servers to ignore') + parser.add_option( + '--skip-own-issues-without-changes', + action='store_true', + help='Skips listing own issues without changes when showing changes ' + 'grouped by referenced issue(s). See --changes-by-issue for more ' + 'details.') + parser.add_option( + '-F', + '--config_file', + metavar='', + help='Configuration file in JSON format, used to add additional gerrit ' + 'instances (see source code for an example).') - activity_types_group = optparse.OptionGroup(parser, 'Activity Types', - 'By default, all activity will be looked up and ' - 'printed. If any of these are specified, only ' - 'those specified will be searched.') - activity_types_group.add_option( - '-c', '--changes', - action='store_true', - help='Show changes.') - activity_types_group.add_option( - '-i', '--issues', - action='store_true', - help='Show issues.') - activity_types_group.add_option( - '-r', '--reviews', - action='store_true', - help='Show reviews.') - activity_types_group.add_option( - '--changes-by-issue', action='store_true', - help='Show changes grouped by referenced issue(s).') - parser.add_option_group(activity_types_group) + activity_types_group = optparse.OptionGroup( + parser, 'Activity Types', + 'By default, all activity will be looked up and ' + 'printed. If any of these are specified, only ' + 'those specified will be searched.') + activity_types_group.add_option('-c', + '--changes', + action='store_true', + help='Show changes.') + activity_types_group.add_option('-i', + '--issues', + action='store_true', + help='Show issues.') + activity_types_group.add_option('-r', + '--reviews', + action='store_true', + help='Show reviews.') + activity_types_group.add_option( + '--changes-by-issue', + action='store_true', + help='Show changes grouped by referenced issue(s).') + parser.add_option_group(activity_types_group) - output_format_group = optparse.OptionGroup(parser, 'Output Format', - 'By default, all activity will be printed in the ' - 'following format: {url} {title}. This can be ' - 'changed for either all activity types or ' - 'individually for each activity type. The format ' - 'is defined as documented for ' - 'string.format(...). The variables available for ' - 'all activity types are url, title, author, ' - 'created and modified. Format options for ' - 'specific activity types will override the ' - 'generic format.') - output_format_group.add_option( - '-f', '--output-format', metavar='', - default=u'{url} {title}', - help='Specifies the format to use when printing all your activity.') - output_format_group.add_option( - '--output-format-changes', metavar='', - default=None, - help='Specifies the format to use when printing changes. Supports the ' - 'additional variable {reviewers}') - output_format_group.add_option( - '--output-format-issues', metavar='', - default=None, - help='Specifies the format to use when printing issues. Supports the ' - 'additional variable {owner}.') - output_format_group.add_option( - '--output-format-reviews', metavar='', - default=None, - help='Specifies the format to use when printing reviews.') - output_format_group.add_option( - '--output-format-heading', metavar='', - default=u'{heading}:', - help='Specifies the format to use when printing headings. ' - 'Supports the variable {heading}.') - output_format_group.add_option( - '--output-format-no-url', default='{title}', - help='Specifies the format to use when printing activity without url.') - output_format_group.add_option( - '-m', '--markdown', action='store_true', - help='Use markdown-friendly output (overrides --output-format ' - 'and --output-format-heading)') - output_format_group.add_option( - '-j', '--json', action='store_true', - help='Output json data (overrides other format options)') - parser.add_option_group(output_format_group) + output_format_group = optparse.OptionGroup( + parser, 'Output Format', + 'By default, all activity will be printed in the ' + 'following format: {url} {title}. This can be ' + 'changed for either all activity types or ' + 'individually for each activity type. The format ' + 'is defined as documented for ' + 'string.format(...). The variables available for ' + 'all activity types are url, title, author, ' + 'created and modified. Format options for ' + 'specific activity types will override the ' + 'generic format.') + output_format_group.add_option( + '-f', + '--output-format', + metavar='', + default=u'{url} {title}', + help='Specifies the format to use when printing all your activity.') + output_format_group.add_option( + '--output-format-changes', + metavar='', + default=None, + help='Specifies the format to use when printing changes. Supports the ' + 'additional variable {reviewers}') + output_format_group.add_option( + '--output-format-issues', + metavar='', + default=None, + help='Specifies the format to use when printing issues. Supports the ' + 'additional variable {owner}.') + output_format_group.add_option( + '--output-format-reviews', + metavar='', + default=None, + help='Specifies the format to use when printing reviews.') + output_format_group.add_option( + '--output-format-heading', + metavar='', + default=u'{heading}:', + help='Specifies the format to use when printing headings. ' + 'Supports the variable {heading}.') + output_format_group.add_option( + '--output-format-no-url', + default='{title}', + help='Specifies the format to use when printing activity without url.') + output_format_group.add_option( + '-m', + '--markdown', + action='store_true', + help='Use markdown-friendly output (overrides --output-format ' + 'and --output-format-heading)') + output_format_group.add_option( + '-j', + '--json', + action='store_true', + help='Output json data (overrides other format options)') + parser.add_option_group(output_format_group) - parser.add_option( - '-v', '--verbose', - action='store_const', - dest='verbosity', - default=logging.WARN, - const=logging.INFO, - help='Output extra informational messages.' - ) - parser.add_option( - '-q', '--quiet', - action='store_const', - dest='verbosity', - const=logging.ERROR, - help='Suppress non-error messages.' - ) - parser.add_option( - '-M', '--merged-only', - action='store_true', - dest='merged_only', - default=False, - help='Shows only changes that have been merged.') - parser.add_option( - '-C', '--completed-issues', - action='store_true', - dest='completed_issues', - default=False, - help='Shows only monorail issues that have completed (Fixed|Verified) ' - 'by the user.') - parser.add_option( - '-o', '--output', metavar='', - help='Where to output the results. By default prints to stdout.') + parser.add_option('-v', + '--verbose', + action='store_const', + dest='verbosity', + default=logging.WARN, + const=logging.INFO, + help='Output extra informational messages.') + parser.add_option('-q', + '--quiet', + action='store_const', + dest='verbosity', + const=logging.ERROR, + help='Suppress non-error messages.') + parser.add_option('-M', + '--merged-only', + action='store_true', + dest='merged_only', + default=False, + help='Shows only changes that have been merged.') + parser.add_option( + '-C', + '--completed-issues', + action='store_true', + dest='completed_issues', + default=False, + help='Shows only monorail issues that have completed (Fixed|Verified) ' + 'by the user.') + parser.add_option( + '-o', + '--output', + metavar='', + help='Where to output the results. By default prints to stdout.') - # Remove description formatting - parser.format_description = ( - lambda _: parser.description) # pylint: disable=no-member + # Remove description formatting + parser.format_description = (lambda _: parser.description) # pylint: disable=no-member - options, args = parser.parse_args() - options.local_user = os.environ.get('USER') - if args: - parser.error('Args unsupported') - if not options.user: - parser.error('USER/USERNAME is not set, please use -u') - # Retains the original -u option as the email address. - options.email = options.user - options.user = username(options.email) + options, args = parser.parse_args() + options.local_user = os.environ.get('USER') + if args: + parser.error('Args unsupported') + if not options.user: + parser.error('USER/USERNAME is not set, please use -u') + # Retains the original -u option as the email address. + options.email = options.user + options.user = username(options.email) - logging.basicConfig(level=options.verbosity) + logging.basicConfig(level=options.verbosity) - # python-keyring provides easy access to the system keyring. - try: - import keyring # pylint: disable=unused-import,unused-variable,F0401 - except ImportError: - logging.warning('Consider installing python-keyring') + # python-keyring provides easy access to the system keyring. + try: + import keyring # pylint: disable=unused-import,unused-variable,F0401 + except ImportError: + logging.warning('Consider installing python-keyring') - if not options.begin: - if options.last_quarter: - begin, end = quarter_begin, quarter_end - elif options.this_year: - begin, end = get_year_of(datetime.today()) - elif options.week_of: - begin, end = (get_week_of(datetime.strptime(options.week_of, '%m/%d/%y'))) - elif options.last_week: - begin, end = (get_week_of(datetime.today() - - timedelta(days=1 + 7 * options.last_week))) - else: - begin, end = (get_week_of(datetime.today() - timedelta(days=1))) - else: - begin = dateutil.parser.parse(options.begin) - if options.end: - end = dateutil.parser.parse(options.end) - else: - end = datetime.today() - options.begin, options.end = begin, end - if begin >= end: - # The queries fail in peculiar ways when the begin date is in the future. - # Give a descriptive error message instead. - logging.error('Start date (%s) is the same or later than end date (%s)' % - (begin, end)) - return 1 - - if options.markdown: - options.output_format_heading = '### {heading}\n' - options.output_format = ' * [{title}]({url})' - options.output_format_no_url = ' * {title}' - logging.info('Searching for activity by %s', options.user) - logging.info('Using range %s to %s', options.begin, options.end) - - if options.config_file: - with open(options.config_file) as f: - config = json.load(f) - - for item, entries in config.items(): - if item == 'gerrit_instances': - for repo, dic in entries.items(): - # Use property name as URL - dic['url'] = repo - gerrit_instances.append(dic) - elif item == 'monorail_projects': - monorail_projects.append(entries) + if not options.begin: + if options.last_quarter: + begin, end = quarter_begin, quarter_end + elif options.this_year: + begin, end = get_year_of(datetime.today()) + elif options.week_of: + begin, end = (get_week_of( + datetime.strptime(options.week_of, '%m/%d/%y'))) + elif options.last_week: + begin, end = ( + get_week_of(datetime.today() - + timedelta(days=1 + 7 * options.last_week))) else: - logging.error('Invalid entry in config file.') - return 1 - - my_activity = MyActivity(options) - my_activity.show_progress('Loading data') - - if not (options.changes or options.reviews or options.issues or - options.changes_by_issue): - options.changes = True - options.issues = True - options.reviews = True - - # First do any required authentication so none of the user interaction has to - # wait for actual work. - if options.changes or options.changes_by_issue: - my_activity.auth_for_changes() - if options.reviews: - my_activity.auth_for_reviews() - - logging.info('Looking up activity.....') - - try: - if options.changes or options.changes_by_issue: - my_activity.get_changes() - if options.reviews: - my_activity.get_reviews() - if options.issues or options.changes_by_issue: - my_activity.get_issues() - if not options.no_referenced_issues: - my_activity.get_referenced_issues() - except auth.LoginRequiredError as e: - logging.error('auth.LoginRequiredError: %s', e) - - my_activity.show_progress('\n') - - my_activity.print_access_errors() - - output_file = None - try: - if options.output: - output_file = open(options.output, 'w') - logging.info('Printing output to "%s"', options.output) - sys.stdout = output_file - except (IOError, OSError) as e: - logging.error('Unable to write output: %s', e) - else: - if options.json: - my_activity.dump_json() + begin, end = (get_week_of(datetime.today() - timedelta(days=1))) else: - if options.changes: - my_activity.print_changes() - if options.reviews: - my_activity.print_reviews() - if options.issues: - my_activity.print_issues() - if options.changes_by_issue: - my_activity.print_changes_by_issue( - options.skip_own_issues_without_changes) - finally: - if output_file: - logging.info('Done printing to file.') - sys.stdout = sys.__stdout__ - output_file.close() + begin = dateutil.parser.parse(options.begin) + if options.end: + end = dateutil.parser.parse(options.end) + else: + end = datetime.today() + options.begin, options.end = begin, end + if begin >= end: + # The queries fail in peculiar ways when the begin date is in the + # future. Give a descriptive error message instead. + logging.error( + 'Start date (%s) is the same or later than end date (%s)' % + (begin, end)) + return 1 - return 0 + if options.markdown: + options.output_format_heading = '### {heading}\n' + options.output_format = ' * [{title}]({url})' + options.output_format_no_url = ' * {title}' + logging.info('Searching for activity by %s', options.user) + logging.info('Using range %s to %s', options.begin, options.end) + + if options.config_file: + with open(options.config_file) as f: + config = json.load(f) + + for item, entries in config.items(): + if item == 'gerrit_instances': + for repo, dic in entries.items(): + # Use property name as URL + dic['url'] = repo + gerrit_instances.append(dic) + elif item == 'monorail_projects': + monorail_projects.append(entries) + else: + logging.error('Invalid entry in config file.') + return 1 + + my_activity = MyActivity(options) + my_activity.show_progress('Loading data') + + if not (options.changes or options.reviews or options.issues + or options.changes_by_issue): + options.changes = True + options.issues = True + options.reviews = True + + # First do any required authentication so none of the user interaction has + # to wait for actual work. + if options.changes or options.changes_by_issue: + my_activity.auth_for_changes() + if options.reviews: + my_activity.auth_for_reviews() + + logging.info('Looking up activity.....') + + try: + if options.changes or options.changes_by_issue: + my_activity.get_changes() + if options.reviews: + my_activity.get_reviews() + if options.issues or options.changes_by_issue: + my_activity.get_issues() + if not options.no_referenced_issues: + my_activity.get_referenced_issues() + except auth.LoginRequiredError as e: + logging.error('auth.LoginRequiredError: %s', e) + + my_activity.show_progress('\n') + + my_activity.print_access_errors() + + output_file = None + try: + if options.output: + output_file = open(options.output, 'w') + logging.info('Printing output to "%s"', options.output) + sys.stdout = output_file + except (IOError, OSError) as e: + logging.error('Unable to write output: %s', e) + else: + if options.json: + my_activity.dump_json() + else: + if options.changes: + my_activity.print_changes() + if options.reviews: + my_activity.print_reviews() + if options.issues: + my_activity.print_issues() + if options.changes_by_issue: + my_activity.print_changes_by_issue( + options.skip_own_issues_without_changes) + finally: + if output_file: + logging.info('Done printing to file.') + sys.stdout = sys.__stdout__ + output_file.close() + + return 0 if __name__ == '__main__': - # Fix encoding to support non-ascii issue titles. - fix_encoding.fix_encoding() + # Fix encoding to support non-ascii issue titles. + fix_encoding.fix_encoding() - try: - sys.exit(main()) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(1) + try: + sys.exit(main()) + except KeyboardInterrupt: + sys.stderr.write('interrupted\n') + sys.exit(1) diff --git a/ninja.py b/ninja.py index 63b0a6af68..a54065f80c 100755 --- a/ninja.py +++ b/ninja.py @@ -15,76 +15,77 @@ import gclient_paths def findNinjaInPath(): - env_path = os.getenv('PATH') - if not env_path: - return - exe = 'ninja' - if sys.platform in ['win32', 'cygwin']: - exe += '.exe' - for bin_dir in env_path.split(os.pathsep): - if bin_dir.rstrip(os.sep).endswith('depot_tools'): - # skip depot_tools to avoid calling ninja.py infitely. - continue - ninja_path = os.path.join(bin_dir, exe) - if os.path.isfile(ninja_path): - return ninja_path + env_path = os.getenv('PATH') + if not env_path: + return + exe = 'ninja' + if sys.platform in ['win32', 'cygwin']: + exe += '.exe' + for bin_dir in env_path.split(os.pathsep): + if bin_dir.rstrip(os.sep).endswith('depot_tools'): + # skip depot_tools to avoid calling ninja.py infitely. + continue + ninja_path = os.path.join(bin_dir, exe) + if os.path.isfile(ninja_path): + return ninja_path def fallback(ninja_args): - # Try to find ninja in PATH. - ninja_path = findNinjaInPath() - if ninja_path: - return subprocess.call([ninja_path] + ninja_args) + # Try to find ninja in PATH. + ninja_path = findNinjaInPath() + if ninja_path: + return subprocess.call([ninja_path] + ninja_args) - print( - 'depot_tools/ninja.py: Could not find Ninja in the third_party of ' - 'the current project, nor in your PATH.\n' - 'Please take one of the following actions to install Ninja:\n' - '- If your project has DEPS, add a CIPD Ninja dependency to DEPS.\n' - '- Otherwise, add Ninja to your PATH *after* depot_tools.', - file=sys.stderr) - return 1 + print( + 'depot_tools/ninja.py: Could not find Ninja in the third_party of ' + 'the current project, nor in your PATH.\n' + 'Please take one of the following actions to install Ninja:\n' + '- If your project has DEPS, add a CIPD Ninja dependency to DEPS.\n' + '- Otherwise, add Ninja to your PATH *after* depot_tools.', + file=sys.stderr) + return 1 def main(args): - # On Windows the ninja.bat script passes along the arguments enclosed in - # double quotes. This prevents multiple levels of parsing of the special '^' - # characters needed when compiling a single file. When this case is detected, - # we need to split the argument. This means that arguments containing actual - # spaces are not supported by ninja.bat, but that is not a real limitation. - if (sys.platform.startswith('win') and len(args) == 2): - args = args[:1] + args[1].split() + # On Windows the ninja.bat script passes along the arguments enclosed in + # double quotes. This prevents multiple levels of parsing of the special '^' + # characters needed when compiling a single file. When this case is + # detected, we need to split the argument. This means that arguments + # containing actual spaces are not supported by ninja.bat, but that is not a + # real limitation. + if (sys.platform.startswith('win') and len(args) == 2): + args = args[:1] + args[1].split() - # macOS's python sets CPATH, LIBRARY_PATH, SDKROOT implicitly. - # https://openradar.appspot.com/radar?id=5608755232243712 - # - # Removing those environment variables to avoid affecting clang's behaviors. - if sys.platform == 'darwin': - os.environ.pop("CPATH", None) - os.environ.pop("LIBRARY_PATH", None) - os.environ.pop("SDKROOT", None) + # macOS's python sets CPATH, LIBRARY_PATH, SDKROOT implicitly. + # https://openradar.appspot.com/radar?id=5608755232243712 + # + # Removing those environment variables to avoid affecting clang's behaviors. + if sys.platform == 'darwin': + os.environ.pop("CPATH", None) + os.environ.pop("LIBRARY_PATH", None) + os.environ.pop("SDKROOT", None) - # Get gclient root + src. - primary_solution_path = gclient_paths.GetPrimarySolutionPath() - gclient_root_path = gclient_paths.FindGclientRoot(os.getcwd()) - gclient_src_root_path = None - if gclient_root_path: - gclient_src_root_path = os.path.join(gclient_root_path, 'src') + # Get gclient root + src. + primary_solution_path = gclient_paths.GetPrimarySolutionPath() + gclient_root_path = gclient_paths.FindGclientRoot(os.getcwd()) + gclient_src_root_path = None + if gclient_root_path: + gclient_src_root_path = os.path.join(gclient_root_path, 'src') - for base_path in set( - [primary_solution_path, gclient_root_path, gclient_src_root_path]): - if not base_path: - continue - ninja_path = os.path.join(base_path, 'third_party', 'ninja', - 'ninja' + gclient_paths.GetExeSuffix()) - if os.path.isfile(ninja_path): - return subprocess.call([ninja_path] + args[1:]) + for base_path in set( + [primary_solution_path, gclient_root_path, gclient_src_root_path]): + if not base_path: + continue + ninja_path = os.path.join(base_path, 'third_party', 'ninja', + 'ninja' + gclient_paths.GetExeSuffix()) + if os.path.isfile(ninja_path): + return subprocess.call([ninja_path] + args[1:]) - return fallback(args[1:]) + return fallback(args[1:]) if __name__ == '__main__': - try: - sys.exit(main(sys.argv)) - except KeyboardInterrupt: - sys.exit(1) + try: + sys.exit(main(sys.argv)) + except KeyboardInterrupt: + sys.exit(1) diff --git a/ninja_reclient.py b/ninja_reclient.py index bc4e79bab8..7ceaf7abdd 100755 --- a/ninja_reclient.py +++ b/ninja_reclient.py @@ -14,14 +14,14 @@ import reclient_helper def main(argv): - with reclient_helper.build_context(argv, 'ninja_reclient') as ret_code: - if ret_code: - return ret_code - try: - return ninja.main(argv) - except KeyboardInterrupt: - return 1 + with reclient_helper.build_context(argv, 'ninja_reclient') as ret_code: + if ret_code: + return ret_code + try: + return ninja.main(argv) + except KeyboardInterrupt: + return 1 if __name__ == '__main__': - sys.exit(main(sys.argv)) + sys.exit(main(sys.argv)) diff --git a/ninjalog_uploader.py b/ninjalog_uploader.py index d8566bfc67..e99bd86d64 100755 --- a/ninjalog_uploader.py +++ b/ninjalog_uploader.py @@ -39,92 +39,92 @@ ALLOWLISTED_CONFIGS = ('symbol_level', 'use_goma', 'is_debug', def IsGoogler(): - """Check whether this user is Googler or not.""" - p = subprocess.run('goma_auth info', - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - shell=True) - if p.returncode != 0: - return False - lines = p.stdout.splitlines() - if len(lines) == 0: - return False - l = lines[0] - # |l| will be like 'Login as @google.com' for googler using goma. - return l.startswith('Login as ') and l.endswith('@google.com') + """Check whether this user is Googler or not.""" + p = subprocess.run('goma_auth info', + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + shell=True) + if p.returncode != 0: + return False + lines = p.stdout.splitlines() + if len(lines) == 0: + return False + l = lines[0] + # |l| will be like 'Login as @google.com' for googler using goma. + return l.startswith('Login as ') and l.endswith('@google.com') def ParseGNArgs(gn_args): - """Parse gn_args as json and return config dictionary.""" - configs = json.loads(gn_args) - build_configs = {} + """Parse gn_args as json and return config dictionary.""" + configs = json.loads(gn_args) + build_configs = {} - for config in configs: - key = config["name"] - if key not in ALLOWLISTED_CONFIGS: - continue - if 'current' in config: - build_configs[key] = config['current']['value'] - else: - build_configs[key] = config['default']['value'] + for config in configs: + key = config["name"] + if key not in ALLOWLISTED_CONFIGS: + continue + if 'current' in config: + build_configs[key] = config['current']['value'] + else: + build_configs[key] = config['default']['value'] - return build_configs + return build_configs def GetBuildTargetFromCommandLine(cmdline): - """Get build targets from commandline.""" + """Get build targets from commandline.""" - # Skip argv0, argv1: ['/path/to/python3', '/path/to/depot_tools/ninja.py'] - idx = 2 + # Skip argv0, argv1: ['/path/to/python3', '/path/to/depot_tools/ninja.py'] + idx = 2 - # Skipping all args that involve these flags, and taking all remaining args - # as targets. - onearg_flags = ('-C', '-d', '-f', '-j', '-k', '-l', '-p', '-t', '-w') - zeroarg_flags = ('--version', '-n', '-v') + # Skipping all args that involve these flags, and taking all remaining args + # as targets. + onearg_flags = ('-C', '-d', '-f', '-j', '-k', '-l', '-p', '-t', '-w') + zeroarg_flags = ('--version', '-n', '-v') - targets = [] + targets = [] - while idx < len(cmdline): - arg = cmdline[idx] - if arg in onearg_flags: - idx += 2 - continue + while idx < len(cmdline): + arg = cmdline[idx] + if arg in onearg_flags: + idx += 2 + continue - if (arg[:2] in onearg_flags or arg in zeroarg_flags): - idx += 1 - continue + if (arg[:2] in onearg_flags or arg in zeroarg_flags): + idx += 1 + continue - # A target doesn't start with '-'. - if arg.startswith('-'): - idx += 1 - continue + # A target doesn't start with '-'. + if arg.startswith('-'): + idx += 1 + continue - # Avoid uploading absolute paths accidentally. e.g. b/270907050 - if os.path.isabs(arg): - idx += 1 - continue + # Avoid uploading absolute paths accidentally. e.g. b/270907050 + if os.path.isabs(arg): + idx += 1 + continue - targets.append(arg) - idx += 1 + targets.append(arg) + idx += 1 - return targets + return targets def GetJflag(cmdline): - """Parse cmdline to get flag value for -j""" + """Parse cmdline to get flag value for -j""" - for i in range(len(cmdline)): - if (cmdline[i] == '-j' and i + 1 < len(cmdline) - and cmdline[i + 1].isdigit()): - return int(cmdline[i + 1]) + for i in range(len(cmdline)): + if (cmdline[i] == '-j' and i + 1 < len(cmdline) + and cmdline[i + 1].isdigit()): + return int(cmdline[i + 1]) - if (cmdline[i].startswith('-j') and cmdline[i][len('-j'):].isdigit()): - return int(cmdline[i][len('-j'):]) + if (cmdline[i].startswith('-j') and cmdline[i][len('-j'):].isdigit()): + return int(cmdline[i][len('-j'):]) def GetMetadata(cmdline, ninjalog): - """Get metadata for uploaded ninjalog. + """Get metadata for uploaded ninjalog. Returned metadata has schema defined in https://cs.chromium.org?q="type+Metadata+struct+%7B"+file:%5Einfra/go/src/infra/appengine/chromium_build_stats/ninjalog/ @@ -132,120 +132,120 @@ def GetMetadata(cmdline, ninjalog): TODO(tikuta): Collect GOMA_* env var. """ - build_dir = os.path.dirname(ninjalog) + build_dir = os.path.dirname(ninjalog) - build_configs = {} - - try: - args = ['gn', 'args', build_dir, '--list', '--short', '--json'] - if sys.platform == 'win32': - # gn in PATH is bat file in windows environment (except cygwin). - args = ['cmd', '/c'] + args - - gn_args = subprocess.check_output(args) - build_configs = ParseGNArgs(gn_args) - except subprocess.CalledProcessError as e: - logging.error("Failed to call gn %s", e) build_configs = {} - # Stringify config. - for k in build_configs: - build_configs[k] = str(build_configs[k]) + try: + args = ['gn', 'args', build_dir, '--list', '--short', '--json'] + if sys.platform == 'win32': + # gn in PATH is bat file in windows environment (except cygwin). + args = ['cmd', '/c'] + args - metadata = { - 'platform': platform.system(), - 'cpu_core': multiprocessing.cpu_count(), - 'build_configs': build_configs, - 'targets': GetBuildTargetFromCommandLine(cmdline), - } + gn_args = subprocess.check_output(args) + build_configs = ParseGNArgs(gn_args) + except subprocess.CalledProcessError as e: + logging.error("Failed to call gn %s", e) + build_configs = {} - jflag = GetJflag(cmdline) - if jflag is not None: - metadata['jobs'] = jflag + # Stringify config. + for k in build_configs: + build_configs[k] = str(build_configs[k]) - return metadata + metadata = { + 'platform': platform.system(), + 'cpu_core': multiprocessing.cpu_count(), + 'build_configs': build_configs, + 'targets': GetBuildTargetFromCommandLine(cmdline), + } + + jflag = GetJflag(cmdline) + if jflag is not None: + metadata['jobs'] = jflag + + return metadata def GetNinjalog(cmdline): - """GetNinjalog returns the path to ninjalog from cmdline.""" - # ninjalog is in current working directory by default. - ninjalog_dir = '.' + """GetNinjalog returns the path to ninjalog from cmdline.""" + # ninjalog is in current working directory by default. + ninjalog_dir = '.' - i = 0 - while i < len(cmdline): - cmd = cmdline[i] - i += 1 - if cmd == '-C' and i < len(cmdline): - ninjalog_dir = cmdline[i] - i += 1 - continue + i = 0 + while i < len(cmdline): + cmd = cmdline[i] + i += 1 + if cmd == '-C' and i < len(cmdline): + ninjalog_dir = cmdline[i] + i += 1 + continue - if cmd.startswith('-C') and len(cmd) > len('-C'): - ninjalog_dir = cmd[len('-C'):] + if cmd.startswith('-C') and len(cmd) > len('-C'): + ninjalog_dir = cmd[len('-C'):] - return os.path.join(ninjalog_dir, '.ninja_log') + return os.path.join(ninjalog_dir, '.ninja_log') def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--server', - default='chromium-build-stats.appspot.com', - help='server to upload ninjalog file.') - parser.add_argument('--ninjalog', help='ninjalog file to upload.') - parser.add_argument('--verbose', - action='store_true', - help='Enable verbose logging.') - parser.add_argument('--cmdline', - required=True, - nargs=argparse.REMAINDER, - help='command line args passed to ninja.') + parser = argparse.ArgumentParser() + parser.add_argument('--server', + default='chromium-build-stats.appspot.com', + help='server to upload ninjalog file.') + parser.add_argument('--ninjalog', help='ninjalog file to upload.') + parser.add_argument('--verbose', + action='store_true', + help='Enable verbose logging.') + parser.add_argument('--cmdline', + required=True, + nargs=argparse.REMAINDER, + help='command line args passed to ninja.') - args = parser.parse_args() + args = parser.parse_args() - if args.verbose: - logging.basicConfig(level=logging.INFO) - else: - # Disable logging. - logging.disable(logging.CRITICAL) + if args.verbose: + logging.basicConfig(level=logging.INFO) + else: + # Disable logging. + logging.disable(logging.CRITICAL) - if not IsGoogler(): + if not IsGoogler(): + return 0 + + ninjalog = args.ninjalog or GetNinjalog(args.cmdline) + if not os.path.isfile(ninjalog): + logging.warning("ninjalog is not found in %s", ninjalog) + return 1 + + # We assume that each ninja invocation interval takes at least 2 seconds. + # This is not to have duplicate entry in server when current build is no-op. + if os.stat(ninjalog).st_mtime < time.time() - 2: + logging.info("ninjalog is not updated recently %s", ninjalog) + return 0 + + output = io.BytesIO() + + with open(ninjalog) as f: + with gzip.GzipFile(fileobj=output, mode='wb') as g: + g.write(f.read().encode()) + g.write(b'# end of ninja log\n') + + metadata = GetMetadata(args.cmdline, ninjalog) + logging.info('send metadata: %s', json.dumps(metadata)) + g.write(json.dumps(metadata).encode()) + + resp = request.urlopen( + request.Request('https://' + args.server + '/upload_ninja_log/', + data=output.getvalue(), + headers={'Content-Encoding': 'gzip'})) + + if resp.status != 200: + logging.warning("unexpected status code for response: %s", resp.status) + return 1 + + logging.info('response header: %s', resp.headers) + logging.info('response content: %s', resp.read()) return 0 - ninjalog = args.ninjalog or GetNinjalog(args.cmdline) - if not os.path.isfile(ninjalog): - logging.warning("ninjalog is not found in %s", ninjalog) - return 1 - - # We assume that each ninja invocation interval takes at least 2 seconds. - # This is not to have duplicate entry in server when current build is no-op. - if os.stat(ninjalog).st_mtime < time.time() - 2: - logging.info("ninjalog is not updated recently %s", ninjalog) - return 0 - - output = io.BytesIO() - - with open(ninjalog) as f: - with gzip.GzipFile(fileobj=output, mode='wb') as g: - g.write(f.read().encode()) - g.write(b'# end of ninja log\n') - - metadata = GetMetadata(args.cmdline, ninjalog) - logging.info('send metadata: %s', json.dumps(metadata)) - g.write(json.dumps(metadata).encode()) - - resp = request.urlopen( - request.Request('https://' + args.server + '/upload_ninja_log/', - data=output.getvalue(), - headers={'Content-Encoding': 'gzip'})) - - if resp.status != 200: - logging.warning("unexpected status code for response: %s", resp.status) - return 1 - - logging.info('response header: %s', resp.headers) - logging.info('response content: %s', resp.read()) - return 0 - if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/ninjalog_uploader_wrapper.py b/ninjalog_uploader_wrapper.py index 701978ce26..6a730699be 100755 --- a/ninjalog_uploader_wrapper.py +++ b/ninjalog_uploader_wrapper.py @@ -21,38 +21,38 @@ VERSION = 3 def LoadConfig(): - if os.path.isfile(CONFIG): - with open(CONFIG, 'r') as f: - try: - config = json.load(f) - except Exception: - # Set default value when failed to load config. - config = { - 'is-googler': ninjalog_uploader.IsGoogler(), - 'countdown': 10, - 'version': VERSION, - } + if os.path.isfile(CONFIG): + with open(CONFIG, 'r') as f: + try: + config = json.load(f) + except Exception: + # Set default value when failed to load config. + config = { + 'is-googler': ninjalog_uploader.IsGoogler(), + 'countdown': 10, + 'version': VERSION, + } - if config['version'] == VERSION: - config['countdown'] = max(0, config['countdown'] - 1) - return config + if config['version'] == VERSION: + config['countdown'] = max(0, config['countdown'] - 1) + return config - return { - 'is-googler': ninjalog_uploader.IsGoogler(), - 'countdown': 10, - 'version': VERSION, - } + return { + 'is-googler': ninjalog_uploader.IsGoogler(), + 'countdown': 10, + 'version': VERSION, + } def SaveConfig(config): - with open(CONFIG, 'w') as f: - json.dump(config, f) + with open(CONFIG, 'w') as f: + json.dump(config, f) def ShowMessage(countdown): - whitelisted = '\n'.join( - [' * %s' % config for config in ninjalog_uploader.ALLOWLISTED_CONFIGS]) - print(""" + whitelisted = '\n'.join( + [' * %s' % config for config in ninjalog_uploader.ALLOWLISTED_CONFIGS]) + print(""" Your ninjalog will be uploaded to build stats server. The uploaded log will be used to analyze user side build performance. @@ -85,51 +85,51 @@ https://chromium.googlesource.com/chromium/tools/depot_tools/+/main/ninjalog.REA def main(): - config = LoadConfig() + config = LoadConfig() - if len(sys.argv) == 2 and sys.argv[1] == 'opt-in': - config['opt-in'] = True - config['countdown'] = 0 - SaveConfig(config) - print('ninjalog upload is opted in.') - return 0 + if len(sys.argv) == 2 and sys.argv[1] == 'opt-in': + config['opt-in'] = True + config['countdown'] = 0 + SaveConfig(config) + print('ninjalog upload is opted in.') + return 0 - if len(sys.argv) == 2 and sys.argv[1] == 'opt-out': - config['opt-in'] = False - SaveConfig(config) - print('ninjalog upload is opted out.') - return 0 + if len(sys.argv) == 2 and sys.argv[1] == 'opt-out': + config['opt-in'] = False + SaveConfig(config) + print('ninjalog upload is opted out.') + return 0 - if 'opt-in' in config and not config['opt-in']: - # Upload is opted out. - return 0 + if 'opt-in' in config and not config['opt-in']: + # Upload is opted out. + return 0 - if not config.get("is-googler", False): - # Not googler. - return 0 + if not config.get("is-googler", False): + # Not googler. + return 0 - if config.get("countdown", 0) > 0: - # Need to show message. - ShowMessage(config["countdown"]) - # Only save config if something has meaningfully changed. - SaveConfig(config) - return 0 + if config.get("countdown", 0) > 0: + # Need to show message. + ShowMessage(config["countdown"]) + # Only save config if something has meaningfully changed. + SaveConfig(config) + return 0 - if len(sys.argv) == 1: - # dry-run for debugging. - print("upload ninjalog dry-run") - return 0 + if len(sys.argv) == 1: + # dry-run for debugging. + print("upload ninjalog dry-run") + return 0 - # Run upload script without wait. - devnull = open(os.devnull, "w") - creationnflags = 0 - if platform.system() == 'Windows': - creationnflags = subprocess.CREATE_NEW_PROCESS_GROUP - subprocess2.Popen([sys.executable, UPLOADER] + sys.argv[1:], - stdout=devnull, - stderr=devnull, - creationflags=creationnflags) + # Run upload script without wait. + devnull = open(os.devnull, "w") + creationnflags = 0 + if platform.system() == 'Windows': + creationnflags = subprocess.CREATE_NEW_PROCESS_GROUP + subprocess2.Popen([sys.executable, UPLOADER] + sys.argv[1:], + stdout=devnull, + stderr=devnull, + creationflags=creationnflags) if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/owners_client.py b/owners_client.py index cf5fb3c199..5f948e87eb 100644 --- a/owners_client.py +++ b/owners_client.py @@ -10,7 +10,7 @@ import git_common class OwnersClient(object): - """Interact with OWNERS files in a repository. + """Interact with OWNERS files in a repository. This class allows you to interact with OWNERS files in a repository both the Gerrit Code-Owners plugin REST API, and the owners database implemented by @@ -23,164 +23,167 @@ class OwnersClient(object): All code should use this class to interact with OWNERS files instead of the owners database in owners.py """ - # '*' means that everyone can approve. - EVERYONE = '*' + # '*' means that everyone can approve. + EVERYONE = '*' - # Possible status of a file. - # - INSUFFICIENT_REVIEWERS: The path needs owners approval, but none of its - # owners is currently a reviewer of the change. - # - PENDING: An owner of this path has been added as reviewer, but approval - # has not been given yet. - # - APPROVED: The path has been approved by an owner. - APPROVED = 'APPROVED' - PENDING = 'PENDING' - INSUFFICIENT_REVIEWERS = 'INSUFFICIENT_REVIEWERS' + # Possible status of a file. + # - INSUFFICIENT_REVIEWERS: The path needs owners approval, but none of its + # owners is currently a reviewer of the change. + # - PENDING: An owner of this path has been added as reviewer, but approval + # has not been given yet. + # - APPROVED: The path has been approved by an owner. + APPROVED = 'APPROVED' + PENDING = 'PENDING' + INSUFFICIENT_REVIEWERS = 'INSUFFICIENT_REVIEWERS' - def ListOwners(self, path): - """List all owners for a file. + def ListOwners(self, path): + """List all owners for a file. The returned list is sorted so that better owners appear first. """ - raise Exception('Not implemented') + raise Exception('Not implemented') - def BatchListOwners(self, paths): - """List all owners for a group of files. + def BatchListOwners(self, paths): + """List all owners for a group of files. Returns a dictionary {path: [owners]}. """ - with git_common.ScopedPool(kind='threads') as pool: - return dict(pool.imap_unordered( - lambda p: (p, self.ListOwners(p)), paths)) + with git_common.ScopedPool(kind='threads') as pool: + return dict( + pool.imap_unordered(lambda p: (p, self.ListOwners(p)), paths)) - def GetFilesApprovalStatus(self, paths, approvers, reviewers): - """Check the approval status for the given paths. + def GetFilesApprovalStatus(self, paths, approvers, reviewers): + """Check the approval status for the given paths. Utility method to check for approval status when a change has not yet been created, given reviewers and approvers. See GetChangeApprovalStatus for description of the returned value. """ - approvers = set(approvers) - if approvers: - approvers.add(self.EVERYONE) - reviewers = set(reviewers) - if reviewers: - reviewers.add(self.EVERYONE) - status = {} - owners_by_path = self.BatchListOwners(paths) - for path, owners in owners_by_path.items(): - owners = set(owners) - if owners.intersection(approvers): - status[path] = self.APPROVED - elif owners.intersection(reviewers): - status[path] = self.PENDING - else: - status[path] = self.INSUFFICIENT_REVIEWERS - return status + approvers = set(approvers) + if approvers: + approvers.add(self.EVERYONE) + reviewers = set(reviewers) + if reviewers: + reviewers.add(self.EVERYONE) + status = {} + owners_by_path = self.BatchListOwners(paths) + for path, owners in owners_by_path.items(): + owners = set(owners) + if owners.intersection(approvers): + status[path] = self.APPROVED + elif owners.intersection(reviewers): + status[path] = self.PENDING + else: + status[path] = self.INSUFFICIENT_REVIEWERS + return status - def ScoreOwners(self, paths, exclude=None): - """Get sorted list of owners for the given paths.""" - if not paths: - return [] - exclude = exclude or [] - owners = [] - queues = self.BatchListOwners(paths).values() - for i in range(max(len(q) for q in queues)): - for q in queues: - if i < len(q) and q[i] not in owners and q[i] not in exclude: - owners.append(q[i]) - return owners + def ScoreOwners(self, paths, exclude=None): + """Get sorted list of owners for the given paths.""" + if not paths: + return [] + exclude = exclude or [] + owners = [] + queues = self.BatchListOwners(paths).values() + for i in range(max(len(q) for q in queues)): + for q in queues: + if i < len(q) and q[i] not in owners and q[i] not in exclude: + owners.append(q[i]) + return owners - def SuggestOwners(self, paths, exclude=None): - """Suggest a set of owners for the given paths.""" - exclude = exclude or [] + def SuggestOwners(self, paths, exclude=None): + """Suggest a set of owners for the given paths.""" + exclude = exclude or [] - paths_by_owner = {} - owners_by_path = self.BatchListOwners(paths) - for path, owners in owners_by_path.items(): - for owner in owners: - paths_by_owner.setdefault(owner, set()).add(path) + paths_by_owner = {} + owners_by_path = self.BatchListOwners(paths) + for path, owners in owners_by_path.items(): + for owner in owners: + paths_by_owner.setdefault(owner, set()).add(path) - selected = [] - missing = set(paths) - for owner in self.ScoreOwners(paths, exclude=exclude): - missing_len = len(missing) - missing.difference_update(paths_by_owner[owner]) - if missing_len > len(missing): - selected.append(owner) - if not missing: - break + selected = [] + missing = set(paths) + for owner in self.ScoreOwners(paths, exclude=exclude): + missing_len = len(missing) + missing.difference_update(paths_by_owner[owner]) + if missing_len > len(missing): + selected.append(owner) + if not missing: + break + + return selected - return selected class GerritClient(OwnersClient): - """Implement OwnersClient using OWNERS REST API.""" - def __init__(self, host, project, branch): - super(GerritClient, self).__init__() + """Implement OwnersClient using OWNERS REST API.""" + def __init__(self, host, project, branch): + super(GerritClient, self).__init__() - self._host = host - self._project = project - self._branch = branch - self._owners_cache = {} - self._best_owners_cache = {} + self._host = host + self._project = project + self._branch = branch + self._owners_cache = {} + self._best_owners_cache = {} - # Seed used by Gerrit to shuffle code owners that have the same score. Can - # be used to make the sort order stable across several requests, e.g. to get - # the same set of random code owners for different file paths that have the - # same code owners. - self._seed = random.getrandbits(30) + # Seed used by Gerrit to shuffle code owners that have the same score. + # Can be used to make the sort order stable across several requests, + # e.g. to get the same set of random code owners for different file + # paths that have the same code owners. + self._seed = random.getrandbits(30) - def _FetchOwners(self, path, cache, highest_score_only=False): - # Always use slashes as separators. - path = path.replace(os.sep, '/') - if path not in cache: - # GetOwnersForFile returns a list of account details sorted by order of - # best reviewer for path. If owners have the same score, the order is - # random, seeded by `self._seed`. - data = gerrit_util.GetOwnersForFile(self._host, - self._project, - self._branch, - path, - resolve_all_users=False, - highest_score_only=highest_score_only, - seed=self._seed) - cache[path] = [ - d['account']['email'] for d in data['code_owners'] - if 'account' in d and 'email' in d['account'] - ] - # If owned_by_all_users is true, add everyone as an owner at the end of - # the owners list. - if data.get('owned_by_all_users', False): - cache[path].append(self.EVERYONE) - return cache[path] + def _FetchOwners(self, path, cache, highest_score_only=False): + # Always use slashes as separators. + path = path.replace(os.sep, '/') + if path not in cache: + # GetOwnersForFile returns a list of account details sorted by order + # of best reviewer for path. If owners have the same score, the + # order is random, seeded by `self._seed`. + data = gerrit_util.GetOwnersForFile( + self._host, + self._project, + self._branch, + path, + resolve_all_users=False, + highest_score_only=highest_score_only, + seed=self._seed) + cache[path] = [ + d['account']['email'] for d in data['code_owners'] + if 'account' in d and 'email' in d['account'] + ] + # If owned_by_all_users is true, add everyone as an owner at the end + # of the owners list. + if data.get('owned_by_all_users', False): + cache[path].append(self.EVERYONE) + return cache[path] - def ListOwners(self, path): - return self._FetchOwners(path, self._owners_cache) + def ListOwners(self, path): + return self._FetchOwners(path, self._owners_cache) - def ListBestOwners(self, path): - return self._FetchOwners(path, - self._best_owners_cache, - highest_score_only=True) + def ListBestOwners(self, path): + return self._FetchOwners(path, + self._best_owners_cache, + highest_score_only=True) - def BatchListBestOwners(self, paths): - """List only the higest-scoring owners for a group of files. + def BatchListBestOwners(self, paths): + """List only the higest-scoring owners for a group of files. Returns a dictionary {path: [owners]}. """ - with git_common.ScopedPool(kind='threads') as pool: - return dict( - pool.imap_unordered(lambda p: (p, self.ListBestOwners(p)), paths)) + with git_common.ScopedPool(kind='threads') as pool: + return dict( + pool.imap_unordered(lambda p: (p, self.ListBestOwners(p)), + paths)) def GetCodeOwnersClient(host, project, branch): - """Get a new OwnersClient. + """Get a new OwnersClient. Uses GerritClient and raises an exception if code-owners plugin is not available.""" - if gerrit_util.IsCodeOwnersEnabledOnHost(host): - return GerritClient(host, project, branch) - raise Exception( - 'code-owners plugin is not enabled. Ask your host admin to enable it ' - 'on %s. Read more about code-owners at ' - 'https://chromium-review.googlesource.com/' - 'plugins/code-owners/Documentation/index.html.' % host) + if gerrit_util.IsCodeOwnersEnabledOnHost(host): + return GerritClient(host, project, branch) + raise Exception( + 'code-owners plugin is not enabled. Ask your host admin to enable it ' + 'on %s. Read more about code-owners at ' + 'https://chromium-review.googlesource.com/' + 'plugins/code-owners/Documentation/index.html.' % host) diff --git a/owners_finder.py b/owners_finder.py index d43145ae95..3d6096f43d 100644 --- a/owners_finder.py +++ b/owners_finder.py @@ -1,7 +1,6 @@ # Copyright 2013 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Interactive tool for finding reviewers/owners for a change.""" from __future__ import print_function @@ -9,336 +8,344 @@ from __future__ import print_function import os import copy - import gclient_utils def first(iterable): - for element in iterable: - return element + for element in iterable: + return element class OwnersFinder(object): - COLOR_LINK = '\033[4m' - COLOR_BOLD = '\033[1;32m' - COLOR_GREY = '\033[0;37m' - COLOR_RESET = '\033[0m' + COLOR_LINK = '\033[4m' + COLOR_BOLD = '\033[1;32m' + COLOR_GREY = '\033[0;37m' + COLOR_RESET = '\033[0m' - indentation = 0 + indentation = 0 - def __init__(self, files, author, reviewers, owners_client, - email_postfix='@chromium.org', - disable_color=False, - ignore_author=False): - self.email_postfix = email_postfix + def __init__(self, + files, + author, + reviewers, + owners_client, + email_postfix='@chromium.org', + disable_color=False, + ignore_author=False): + self.email_postfix = email_postfix - if os.name == 'nt' or disable_color: - self.COLOR_LINK = '' - self.COLOR_BOLD = '' - self.COLOR_GREY = '' - self.COLOR_RESET = '' + if os.name == 'nt' or disable_color: + self.COLOR_LINK = '' + self.COLOR_BOLD = '' + self.COLOR_GREY = '' + self.COLOR_RESET = '' - self.author = author + self.author = author - filtered_files = files + filtered_files = files - reviewers = list(reviewers) - if author and not ignore_author: - reviewers.append(author) + reviewers = list(reviewers) + if author and not ignore_author: + reviewers.append(author) - # Eliminate files that existing reviewers can review. - self.owners_client = owners_client - approval_status = self.owners_client.GetFilesApprovalStatus( - filtered_files, reviewers, []) - filtered_files = [ - f for f in filtered_files - if approval_status[f] != self.owners_client.APPROVED] + # Eliminate files that existing reviewers can review. + self.owners_client = owners_client + approval_status = self.owners_client.GetFilesApprovalStatus( + filtered_files, reviewers, []) + filtered_files = [ + f for f in filtered_files + if approval_status[f] != self.owners_client.APPROVED + ] - # If some files are eliminated. - if len(filtered_files) != len(files): - files = filtered_files + # If some files are eliminated. + if len(filtered_files) != len(files): + files = filtered_files - self.files_to_owners = self.owners_client.BatchListOwners(files) + self.files_to_owners = self.owners_client.BatchListOwners(files) - self.owners_to_files = {} - self._map_owners_to_files() + self.owners_to_files = {} + self._map_owners_to_files() - self.original_files_to_owners = copy.deepcopy(self.files_to_owners) + self.original_files_to_owners = copy.deepcopy(self.files_to_owners) - # This is the queue that will be shown in the interactive questions. - # It is initially sorted by the score in descending order. In the - # interactive questions a user can choose to "defer" its decision, then the - # owner will be put to the end of the queue and shown later. - self.owners_queue = [] + # This is the queue that will be shown in the interactive questions. + # It is initially sorted by the score in descending order. In the + # interactive questions a user can choose to "defer" its decision, then + # the owner will be put to the end of the queue and shown later. + self.owners_queue = [] - self.unreviewed_files = set() - self.reviewed_by = {} - self.selected_owners = set() - self.deselected_owners = set() - self.reset() + self.unreviewed_files = set() + self.reviewed_by = {} + self.selected_owners = set() + self.deselected_owners = set() + self.reset() - def run(self): - self.reset() - while self.owners_queue and self.unreviewed_files: - owner = self.owners_queue[0] + def run(self): + self.reset() + while self.owners_queue and self.unreviewed_files: + owner = self.owners_queue[0] - if (owner in self.selected_owners) or (owner in self.deselected_owners): - continue + if (owner in self.selected_owners) or (owner + in self.deselected_owners): + continue - if not any((file_name in self.unreviewed_files) - for file_name in self.owners_to_files[owner]): - self.deselect_owner(owner) - continue + if not any((file_name in self.unreviewed_files) + for file_name in self.owners_to_files[owner]): + self.deselect_owner(owner) + continue - self.print_info(owner) + self.print_info(owner) - while True: - inp = self.input_command(owner) - if inp in ('y', 'yes'): - self.select_owner(owner) - break + while True: + inp = self.input_command(owner) + if inp in ('y', 'yes'): + self.select_owner(owner) + break - if inp in ('n', 'no'): - self.deselect_owner(owner) - break + if inp in ('n', 'no'): + self.deselect_owner(owner) + break - if inp in ('', 'd', 'defer'): - self.owners_queue.append(self.owners_queue.pop(0)) - break + if inp in ('', 'd', 'defer'): + self.owners_queue.append(self.owners_queue.pop(0)) + break - if inp in ('f', 'files'): - self.list_files() - break + if inp in ('f', 'files'): + self.list_files() + break - if inp in ('o', 'owners'): - self.list_owners(self.owners_queue) - break + if inp in ('o', 'owners'): + self.list_owners(self.owners_queue) + break - if inp in ('p', 'pick'): - self.pick_owner(gclient_utils.AskForData('Pick an owner: ')) - break + if inp in ('p', 'pick'): + self.pick_owner(gclient_utils.AskForData('Pick an owner: ')) + break - if inp.startswith('p ') or inp.startswith('pick '): - self.pick_owner(inp.split(' ', 2)[1].strip()) - break + if inp.startswith('p ') or inp.startswith('pick '): + self.pick_owner(inp.split(' ', 2)[1].strip()) + break - if inp in ('r', 'restart'): - self.reset() - break + if inp in ('r', 'restart'): + self.reset() + break - if inp in ('q', 'quit'): - # Exit with error - return 1 + if inp in ('q', 'quit'): + # Exit with error + return 1 - self.print_result() - return 0 + self.print_result() + return 0 - def _map_owners_to_files(self): - for file_name in self.files_to_owners: - for owner in self.files_to_owners[file_name]: - self.owners_to_files.setdefault(owner, set()) - self.owners_to_files[owner].add(file_name) + def _map_owners_to_files(self): + for file_name in self.files_to_owners: + for owner in self.files_to_owners[file_name]: + self.owners_to_files.setdefault(owner, set()) + self.owners_to_files[owner].add(file_name) - def reset(self): - self.files_to_owners = copy.deepcopy(self.original_files_to_owners) - self.unreviewed_files = set(self.files_to_owners.keys()) - self.reviewed_by = {} - self.selected_owners = set() - self.deselected_owners = set() + def reset(self): + self.files_to_owners = copy.deepcopy(self.original_files_to_owners) + self.unreviewed_files = set(self.files_to_owners.keys()) + self.reviewed_by = {} + self.selected_owners = set() + self.deselected_owners = set() - # Randomize owners' names so that if many reviewers have identical scores - # they will be randomly ordered to avoid bias. - owners = list(self.owners_client.ScoreOwners(self.files_to_owners.keys())) - if self.author and self.author in owners: - owners.remove(self.author) - self.owners_queue = owners - self.find_mandatory_owners() + # Randomize owners' names so that if many reviewers have identical + # scores they will be randomly ordered to avoid bias. + owners = list( + self.owners_client.ScoreOwners(self.files_to_owners.keys())) + if self.author and self.author in owners: + owners.remove(self.author) + self.owners_queue = owners + self.find_mandatory_owners() - def select_owner(self, owner, findMandatoryOwners=True): - if owner in self.selected_owners or owner in self.deselected_owners\ - or not (owner in self.owners_queue): - return - self.writeln('Selected: ' + owner) - self.owners_queue.remove(owner) - self.selected_owners.add(owner) - for file_name in filter( - lambda file_name: file_name in self.unreviewed_files, - self.owners_to_files[owner]): - self.unreviewed_files.remove(file_name) - self.reviewed_by[file_name] = owner - if findMandatoryOwners: - self.find_mandatory_owners() + def select_owner(self, owner, findMandatoryOwners=True): + if owner in self.selected_owners or owner in self.deselected_owners\ + or not (owner in self.owners_queue): + return + self.writeln('Selected: ' + owner) + self.owners_queue.remove(owner) + self.selected_owners.add(owner) + for file_name in filter( + lambda file_name: file_name in self.unreviewed_files, + self.owners_to_files[owner]): + self.unreviewed_files.remove(file_name) + self.reviewed_by[file_name] = owner + if findMandatoryOwners: + self.find_mandatory_owners() - def deselect_owner(self, owner, findMandatoryOwners=True): - if owner in self.selected_owners or owner in self.deselected_owners\ - or not (owner in self.owners_queue): - return - self.writeln('Deselected: ' + owner) - self.owners_queue.remove(owner) - self.deselected_owners.add(owner) - for file_name in self.owners_to_files[owner] & self.unreviewed_files: - self.files_to_owners[file_name].remove(owner) - if findMandatoryOwners: - self.find_mandatory_owners() + def deselect_owner(self, owner, findMandatoryOwners=True): + if owner in self.selected_owners or owner in self.deselected_owners\ + or not (owner in self.owners_queue): + return + self.writeln('Deselected: ' + owner) + self.owners_queue.remove(owner) + self.deselected_owners.add(owner) + for file_name in self.owners_to_files[owner] & self.unreviewed_files: + self.files_to_owners[file_name].remove(owner) + if findMandatoryOwners: + self.find_mandatory_owners() - def find_mandatory_owners(self): - continues = True - for owner in self.owners_queue: - if owner in self.selected_owners: - continue - if owner in self.deselected_owners: - continue - if len(self.owners_to_files[owner] & self.unreviewed_files) == 0: - self.deselect_owner(owner, False) - - while continues: - continues = False - for file_name in filter( - lambda file_name: len(self.files_to_owners[file_name]) == 1, - self.unreviewed_files): - owner = first(self.files_to_owners[file_name]) - self.select_owner(owner, False) + def find_mandatory_owners(self): continues = True - break + for owner in self.owners_queue: + if owner in self.selected_owners: + continue + if owner in self.deselected_owners: + continue + if len(self.owners_to_files[owner] & self.unreviewed_files) == 0: + self.deselect_owner(owner, False) - def print_file_info(self, file_name, except_owner=''): - if file_name not in self.unreviewed_files: - self.writeln(self.greyed(file_name + - ' (by ' + - self.bold_name(self.reviewed_by[file_name]) + - ')')) - else: - if len(self.files_to_owners[file_name]) <= 3: - other_owners = [] - for ow in self.files_to_owners[file_name]: - if ow != except_owner: - other_owners.append(self.bold_name(ow)) - self.writeln(file_name + - ' [' + (', '.join(other_owners)) + ']') - else: - self.writeln(file_name + ' [' + - self.bold(str(len(self.files_to_owners[file_name]))) + - ']') + while continues: + continues = False + for file_name in filter( + lambda file_name: len(self.files_to_owners[file_name]) == 1, + self.unreviewed_files): + owner = first(self.files_to_owners[file_name]) + self.select_owner(owner, False) + continues = True + break - def print_file_info_detailed(self, file_name): - self.writeln(file_name) - self.indent() - for ow in sorted(self.files_to_owners[file_name]): - if ow in self.deselected_owners: - self.writeln(self.bold_name(self.greyed(ow))) - elif ow in self.selected_owners: - self.writeln(self.bold_name(self.greyed(ow))) - else: - self.writeln(self.bold_name(ow)) - self.unindent() + def print_file_info(self, file_name, except_owner=''): + if file_name not in self.unreviewed_files: + self.writeln( + self.greyed(file_name + ' (by ' + + self.bold_name(self.reviewed_by[file_name]) + ')')) + else: + if len(self.files_to_owners[file_name]) <= 3: + other_owners = [] + for ow in self.files_to_owners[file_name]: + if ow != except_owner: + other_owners.append(self.bold_name(ow)) + self.writeln(file_name + ' [' + (', '.join(other_owners)) + ']') + else: + self.writeln( + file_name + ' [' + + self.bold(str(len(self.files_to_owners[file_name]))) + ']') - def print_owned_files_for(self, owner): - # Print owned files - self.writeln(self.bold_name(owner)) - self.writeln(self.bold_name(owner) + ' owns ' + - str(len(self.owners_to_files[owner])) + ' file(s):') - self.indent() - for file_name in sorted(self.owners_to_files[owner]): - self.print_file_info(file_name, owner) - self.unindent() - self.writeln() - - def list_owners(self, owners_queue): - if (len(self.owners_to_files) - len(self.deselected_owners) - - len(self.selected_owners)) > 3: - for ow in owners_queue: - if ow not in self.deselected_owners and ow not in self.selected_owners: - self.writeln(self.bold_name(ow)) - else: - for ow in owners_queue: - if ow not in self.deselected_owners and ow not in self.selected_owners: - self.writeln() - self.print_owned_files_for(ow) - - def list_files(self): - self.indent() - if len(self.unreviewed_files) > 5: - for file_name in sorted(self.unreviewed_files): - self.print_file_info(file_name) - else: - for file_name in self.unreviewed_files: - self.print_file_info_detailed(file_name) - self.unindent() - - def pick_owner(self, ow): - # Allowing to omit domain suffixes - if ow not in self.owners_to_files: - if ow + self.email_postfix in self.owners_to_files: - ow += self.email_postfix - - if ow not in self.owners_to_files: - self.writeln('You cannot pick ' + self.bold_name(ow) + ' manually. ' + - 'It\'s an invalid name or not related to the change list.') - return False - - if ow in self.selected_owners: - self.writeln('You cannot pick ' + self.bold_name(ow) + ' manually. ' + - 'It\'s already selected.') - return False - - if ow in self.deselected_owners: - self.writeln('You cannot pick ' + self.bold_name(ow) + ' manually.' + - 'It\'s already unselected.') - return False - - self.select_owner(ow) - return True - - def print_result(self): - # Print results - self.writeln() - self.writeln() - if len(self.selected_owners) == 0: - self.writeln('This change list already has owner-reviewers for all ' - 'files.') - self.writeln('Use --ignore-current if you want to ignore them.') - else: - self.writeln('** You selected these owners **') - self.writeln() - for owner in self.selected_owners: - self.writeln(self.bold_name(owner) + ':') + def print_file_info_detailed(self, file_name): + self.writeln(file_name) self.indent() - for file_name in sorted(self.owners_to_files[owner]): - self.writeln(file_name) + for ow in sorted(self.files_to_owners[file_name]): + if ow in self.deselected_owners: + self.writeln(self.bold_name(self.greyed(ow))) + elif ow in self.selected_owners: + self.writeln(self.bold_name(self.greyed(ow))) + else: + self.writeln(self.bold_name(ow)) self.unindent() - def bold(self, text): - return self.COLOR_BOLD + text + self.COLOR_RESET + def print_owned_files_for(self, owner): + # Print owned files + self.writeln(self.bold_name(owner)) + self.writeln( + self.bold_name(owner) + ' owns ' + + str(len(self.owners_to_files[owner])) + ' file(s):') + self.indent() + for file_name in sorted(self.owners_to_files[owner]): + self.print_file_info(file_name, owner) + self.unindent() + self.writeln() - def bold_name(self, name): - return (self.COLOR_BOLD + - name.replace(self.email_postfix, '') + self.COLOR_RESET) + def list_owners(self, owners_queue): + if (len(self.owners_to_files) - len(self.deselected_owners) - + len(self.selected_owners)) > 3: + for ow in owners_queue: + if (ow not in self.deselected_owners + and ow not in self.selected_owners): + self.writeln(self.bold_name(ow)) + else: + for ow in owners_queue: + if (ow not in self.deselected_owners + and ow not in self.selected_owners): + self.writeln() + self.print_owned_files_for(ow) - def greyed(self, text): - return self.COLOR_GREY + text + self.COLOR_RESET + def list_files(self): + self.indent() + if len(self.unreviewed_files) > 5: + for file_name in sorted(self.unreviewed_files): + self.print_file_info(file_name) + else: + for file_name in self.unreviewed_files: + self.print_file_info_detailed(file_name) + self.unindent() - def indent(self): - self.indentation += 1 + def pick_owner(self, ow): + # Allowing to omit domain suffixes + if ow not in self.owners_to_files: + if ow + self.email_postfix in self.owners_to_files: + ow += self.email_postfix - def unindent(self): - self.indentation -= 1 + if ow not in self.owners_to_files: + self.writeln( + 'You cannot pick ' + self.bold_name(ow) + ' manually. ' + + 'It\'s an invalid name or not related to the change list.') + return False - def print_indent(self): - return ' ' * self.indentation + if ow in self.selected_owners: + self.writeln('You cannot pick ' + self.bold_name(ow) + + ' manually. ' + 'It\'s already selected.') + return False - def writeln(self, text=''): - print(self.print_indent() + text) + if ow in self.deselected_owners: + self.writeln('You cannot pick ' + self.bold_name(ow) + + ' manually.' + 'It\'s already unselected.') + return False - def hr(self): - self.writeln('=====================') + self.select_owner(ow) + return True - def print_info(self, owner): - self.hr() - self.writeln( - self.bold(str(len(self.unreviewed_files))) + ' file(s) left.') - self.print_owned_files_for(owner) + def print_result(self): + # Print results + self.writeln() + self.writeln() + if len(self.selected_owners) == 0: + self.writeln('This change list already has owner-reviewers for all ' + 'files.') + self.writeln('Use --ignore-current if you want to ignore them.') + else: + self.writeln('** You selected these owners **') + self.writeln() + for owner in self.selected_owners: + self.writeln(self.bold_name(owner) + ':') + self.indent() + for file_name in sorted(self.owners_to_files[owner]): + self.writeln(file_name) + self.unindent() - def input_command(self, owner): - self.writeln('Add ' + self.bold_name(owner) + ' as your reviewer? ') - return gclient_utils.AskForData( - '[yes/no/Defer/pick/files/owners/quit/restart]: ').lower() + def bold(self, text): + return self.COLOR_BOLD + text + self.COLOR_RESET + + def bold_name(self, name): + return (self.COLOR_BOLD + name.replace(self.email_postfix, '') + + self.COLOR_RESET) + + def greyed(self, text): + return self.COLOR_GREY + text + self.COLOR_RESET + + def indent(self): + self.indentation += 1 + + def unindent(self): + self.indentation -= 1 + + def print_indent(self): + return ' ' * self.indentation + + def writeln(self, text=''): + print(self.print_indent() + text) + + def hr(self): + self.writeln('=====================') + + def print_info(self, owner): + self.hr() + self.writeln( + self.bold(str(len(self.unreviewed_files))) + ' file(s) left.') + self.print_owned_files_for(owner) + + def input_command(self, owner): + self.writeln('Add ' + self.bold_name(owner) + ' as your reviewer? ') + return gclient_utils.AskForData( + '[yes/no/Defer/pick/files/owners/quit/restart]: ').lower() diff --git a/post_build_ninja_summary.py b/post_build_ninja_summary.py index 8405639598..a36cb12beb 100755 --- a/post_build_ninja_summary.py +++ b/post_build_ninja_summary.py @@ -2,7 +2,6 @@ # Copyright (c) 2018 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Summarize the last ninja build, invoked with ninja's -C syntax. This script is designed to be automatically run after each ninja build in @@ -57,7 +56,6 @@ import fnmatch import os import sys - # The number of long build times to report: long_count = 10 # The number of long times by extension to report @@ -96,8 +94,8 @@ class Target: # Allow for modest floating-point errors epsilon = 0.000002 if (self.weighted_duration > self.Duration() + epsilon): - print('%s > %s?' % (self.weighted_duration, self.Duration())) - assert(self.weighted_duration <= self.Duration() + epsilon) + print('%s > %s?' % (self.weighted_duration, self.Duration())) + assert (self.weighted_duration <= self.Duration() + epsilon) return self.weighted_duration def DescribeTargets(self): @@ -108,7 +106,7 @@ class Target: result = ', '.join(self.targets) max_length = 65 if len(result) > max_length: - result = result[:max_length] + '...' + result = result[:max_length] + '...' return result @@ -125,10 +123,10 @@ def ReadTargets(log, show_all): for line in log: parts = line.strip().split('\t') if len(parts) != 5: - # If ninja.exe is rudely halted then the .ninja_log file may be - # corrupt. Silently continue. - continue - start, end, _, name, cmdhash = parts # Ignore restat. + # If ninja.exe is rudely halted then the .ninja_log file may be + # corrupt. Silently continue. + continue + start, end, _, name, cmdhash = parts # Ignore restat. # Convert from integral milliseconds to float seconds. start = int(start) / 1000.0 end = int(end) / 1000.0 @@ -142,68 +140,68 @@ def ReadTargets(log, show_all): targets_dict = {} target = None if cmdhash in targets_dict: - target = targets_dict[cmdhash] - if not show_all and (target.start != start or target.end != end): - # If several builds in a row just run one or two build steps then - # the end times may not go backwards so the last build may not be - # detected as such. However in many cases there will be a build step - # repeated in the two builds and the changed start/stop points for - # that command, identified by the hash, can be used to detect and - # reset the target dictionary. - targets_dict = {} - target = None + target = targets_dict[cmdhash] + if not show_all and (target.start != start or target.end != end): + # If several builds in a row just run one or two build steps + # then the end times may not go backwards so the last build may + # not be detected as such. However in many cases there will be a + # build step repeated in the two builds and the changed + # start/stop points for that command, identified by the hash, + # can be used to detect and reset the target dictionary. + targets_dict = {} + target = None if not target: - targets_dict[cmdhash] = target = Target(start, end) + targets_dict[cmdhash] = target = Target(start, end) last_end_seen = end target.targets.append(name) return list(targets_dict.values()) def GetExtension(target, extra_patterns): - """Return the file extension that best represents a target. + """Return the file extension that best represents a target. For targets that generate multiple outputs it is important to return a consistent 'canonical' extension. Ultimately the goal is to group build steps by type.""" - for output in target.targets: - if extra_patterns: - for fn_pattern in extra_patterns.split(';'): - if fnmatch.fnmatch(output, '*' + fn_pattern + '*'): - return fn_pattern - # Not a true extension, but a good grouping. - if output.endswith('type_mappings'): - extension = 'type_mappings' - break + for output in target.targets: + if extra_patterns: + for fn_pattern in extra_patterns.split(';'): + if fnmatch.fnmatch(output, '*' + fn_pattern + '*'): + return fn_pattern + # Not a true extension, but a good grouping. + if output.endswith('type_mappings'): + extension = 'type_mappings' + break - # Capture two extensions if present. For example: file.javac.jar should be - # distinguished from file.interface.jar. - root, ext1 = os.path.splitext(output) - _, ext2 = os.path.splitext(root) - extension = ext2 + ext1 # Preserve the order in the file name. + # Capture two extensions if present. For example: file.javac.jar should + # be distinguished from file.interface.jar. + root, ext1 = os.path.splitext(output) + _, ext2 = os.path.splitext(root) + extension = ext2 + ext1 # Preserve the order in the file name. - if len(extension) == 0: - extension = '(no extension found)' + if len(extension) == 0: + extension = '(no extension found)' - if ext1 in ['.pdb', '.dll', '.exe']: - extension = 'PEFile (linking)' - # Make sure that .dll and .exe are grouped together and that the - # .dll.lib files don't cause these to be listed as libraries - break - if ext1 in ['.so', '.TOC']: - extension = '.so (linking)' - # Attempt to identify linking, avoid identifying as '.TOC' - break - # Make sure .obj files don't get categorized as mojo files - if ext1 in ['.obj', '.o']: - break - # Jars are the canonical output of java targets. - if ext1 == '.jar': - break - # Normalize all mojo related outputs to 'mojo'. - if output.count('.mojom') > 0: - extension = 'mojo' - break - return extension + if ext1 in ['.pdb', '.dll', '.exe']: + extension = 'PEFile (linking)' + # Make sure that .dll and .exe are grouped together and that the + # .dll.lib files don't cause these to be listed as libraries + break + if ext1 in ['.so', '.TOC']: + extension = '.so (linking)' + # Attempt to identify linking, avoid identifying as '.TOC' + break + # Make sure .obj files don't get categorized as mojo files + if ext1 in ['.obj', '.o']: + break + # Jars are the canonical output of java targets. + if ext1 == '.jar': + break + # Normalize all mojo related outputs to 'mojo'. + if output.count('.mojom') > 0: + extension = 'mojo' + break + return extension def SummarizeEntries(entries, extra_step_types, elapsed_time_sorting): @@ -221,13 +219,13 @@ def SummarizeEntries(entries, extra_step_types, elapsed_time_sorting): latest = 0 total_cpu_time = 0 for target in entries: - if earliest < 0 or target.start < earliest: - earliest = target.start - if target.end > latest: - latest = target.end - total_cpu_time += target.Duration() - task_start_stop_times.append((target.start, 'start', target)) - task_start_stop_times.append((target.end, 'stop', target)) + if earliest < 0 or target.start < earliest: + earliest = target.start + if target.end > latest: + latest = target.end + total_cpu_time += target.Duration() + task_start_stop_times.append((target.start, 'start', target)) + task_start_stop_times.append((target.end, 'stop', target)) length = latest - earliest weighted_total = 0.0 @@ -247,40 +245,41 @@ def SummarizeEntries(entries, extra_step_types, elapsed_time_sorting): last_weighted_time = 0.0 # Scan all start/stop events. for event in task_start_stop_times: - time, action_name, target = event - # Accumulate weighted time up to now. - num_running = len(running_tasks) - if num_running > 0: - # Update the total weighted time up to this moment. - last_weighted_time += (time - last_time) / float(num_running) - if action_name == 'start': - # Record the total weighted task time when this task starts. - running_tasks[target] = last_weighted_time - if action_name == 'stop': - # Record the change in the total weighted task time while this task ran. - weighted_duration = last_weighted_time - running_tasks[target] - target.SetWeightedDuration(weighted_duration) - weighted_total += weighted_duration - del running_tasks[target] - last_time = time - assert(len(running_tasks) == 0) + time, action_name, target = event + # Accumulate weighted time up to now. + num_running = len(running_tasks) + if num_running > 0: + # Update the total weighted time up to this moment. + last_weighted_time += (time - last_time) / float(num_running) + if action_name == 'start': + # Record the total weighted task time when this task starts. + running_tasks[target] = last_weighted_time + if action_name == 'stop': + # Record the change in the total weighted task time while this task + # ran. + weighted_duration = last_weighted_time - running_tasks[target] + target.SetWeightedDuration(weighted_duration) + weighted_total += weighted_duration + del running_tasks[target] + last_time = time + assert (len(running_tasks) == 0) # Warn if the sum of weighted times is off by more than half a second. if abs(length - weighted_total) > 500: - print('Warning: Possible corrupt ninja log, results may be ' - 'untrustworthy. Length = %.3f, weighted total = %.3f' % ( - length, weighted_total)) + print('Warning: Possible corrupt ninja log, results may be ' + 'untrustworthy. Length = %.3f, weighted total = %.3f' % + (length, weighted_total)) # Print the slowest build steps: print(' Longest build steps:') if elapsed_time_sorting: - entries.sort(key=lambda x: x.Duration()) + entries.sort(key=lambda x: x.Duration()) else: - entries.sort(key=lambda x: x.WeightedDuration()) + entries.sort(key=lambda x: x.WeightedDuration()) for target in entries[-long_count:]: - print(' %8.1f weighted s to build %s (%.1f s elapsed time)' % ( - target.WeightedDuration(), - target.DescribeTargets(), target.Duration())) + print(' %8.1f weighted s to build %s (%.1f s elapsed time)' % + (target.WeightedDuration(), target.DescribeTargets(), + target.Duration())) # Sum up the time by file extension/type of the output file count_by_ext = {} @@ -288,38 +287,39 @@ def SummarizeEntries(entries, extra_step_types, elapsed_time_sorting): weighted_time_by_ext = {} # Scan through all of the targets to build up per-extension statistics. for target in entries: - extension = GetExtension(target, extra_step_types) - time_by_ext[extension] = time_by_ext.get(extension, 0) + target.Duration() - weighted_time_by_ext[extension] = weighted_time_by_ext.get(extension, - 0) + target.WeightedDuration() - count_by_ext[extension] = count_by_ext.get(extension, 0) + 1 + extension = GetExtension(target, extra_step_types) + time_by_ext[extension] = time_by_ext.get(extension, + 0) + target.Duration() + weighted_time_by_ext[extension] = weighted_time_by_ext.get( + extension, 0) + target.WeightedDuration() + count_by_ext[extension] = count_by_ext.get(extension, 0) + 1 print(' Time by build-step type:') # Copy to a list with extension name and total time swapped, to (time, ext) if elapsed_time_sorting: - weighted_time_by_ext_sorted = sorted((y, x) for (x, y) in - time_by_ext.items()) + weighted_time_by_ext_sorted = sorted( + (y, x) for (x, y) in time_by_ext.items()) else: - weighted_time_by_ext_sorted = sorted((y, x) for (x, y) in - weighted_time_by_ext.items()) + weighted_time_by_ext_sorted = sorted( + (y, x) for (x, y) in weighted_time_by_ext.items()) # Print the slowest build target types: for time, extension in weighted_time_by_ext_sorted[-long_ext_count:]: - print(' %8.1f s weighted time to generate %d %s files ' - '(%1.1f s elapsed time sum)' % (time, count_by_ext[extension], - extension, time_by_ext[extension])) + print( + ' %8.1f s weighted time to generate %d %s files ' + '(%1.1f s elapsed time sum)' % + (time, count_by_ext[extension], extension, time_by_ext[extension])) print(' %.1f s weighted time (%.1f s elapsed time sum, %1.1fx ' - 'parallelism)' % (length, total_cpu_time, - total_cpu_time * 1.0 / length)) - print(' %d build steps completed, average of %1.2f/s' % ( - len(entries), len(entries) / (length))) + 'parallelism)' % + (length, total_cpu_time, total_cpu_time * 1.0 / length)) + print(' %d build steps completed, average of %1.2f/s' % + (len(entries), len(entries) / (length))) def main(): log_file = '.ninja_log' parser = argparse.ArgumentParser() - parser.add_argument('-C', dest='build_directory', - help='Build directory.') + parser.add_argument('-C', dest='build_directory', help='Build directory.') parser.add_argument( '-s', '--step-types', @@ -338,22 +338,23 @@ def main(): if args.log_file: log_file = args.log_file if not args.step_types: - # Offer a convenient way to add extra step types automatically, including - # when this script is run by autoninja. get() returns None if the variable - # isn't set. - args.step_types = os.environ.get('chromium_step_types') + # Offer a convenient way to add extra step types automatically, + # including when this script is run by autoninja. get() returns None if + # the variable isn't set. + args.step_types = os.environ.get('chromium_step_types') if args.step_types: - # Make room for the extra build types. - global long_ext_count - long_ext_count += len(args.step_types.split(';')) + # Make room for the extra build types. + global long_ext_count + long_ext_count += len(args.step_types.split(';')) try: - with open(log_file, 'r') as log: - entries = ReadTargets(log, False) - SummarizeEntries(entries, args.step_types, args.elapsed_time_sorting) + with open(log_file, 'r') as log: + entries = ReadTargets(log, False) + SummarizeEntries(entries, args.step_types, + args.elapsed_time_sorting) except IOError: - print('Log file %r not found, no build summary created.' % log_file) - return errno.ENOENT + print('Log file %r not found, no build summary created.' % log_file) + return errno.ENOENT if __name__ == '__main__': diff --git a/presubmit_canned_checks.py b/presubmit_canned_checks.py index 2d116ad65a..7ccbe61cc0 100644 --- a/presubmit_canned_checks.py +++ b/presubmit_canned_checks.py @@ -1,7 +1,6 @@ # Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Generic presubmit checks that can be reused by other presubmit checks.""" from __future__ import print_function @@ -13,6 +12,9 @@ import zlib import metadata.discover import metadata.validate +# TODO: Should fix these warnings. +# pylint: disable=line-too-long + _HERE = _os.path.dirname(_os.path.abspath(__file__)) # These filters will be disabled if callers do not explicitly supply a @@ -30,12 +32,12 @@ _HERE = _os.path.dirname(_os.path.abspath(__file__)) # - runtime/int : Can be fixed long term; volume of errors too high # - whitespace/braces : We have a lot of explicit scoping in chrome code OFF_BY_DEFAULT_LINT_FILTERS = [ - '-build/include', - '-build/include_order', - '-build/namespaces', - '-readability/casting', - '-runtime/int', - '-whitespace/braces', + '-build/include', + '-build/include_order', + '-build/namespaces', + '-readability/casting', + '-runtime/int', + '-whitespace/braces', ] # These filters will be disabled unless callers explicitly enable them, because @@ -72,343 +74,367 @@ _CORP_LINK_KEYWORD = '.corp.google' def CheckChangeHasBugFieldFromChange(change, output_api, show_suggestions=True): - """Requires that the changelist have a Bug: field. If show_suggestions is + """Requires that the changelist have a Bug: field. If show_suggestions is False then only report on incorrect tags, not missing tags.""" - bugs = change.BugsFromDescription() - results = [] - if bugs: - if any(b.startswith('b/') for b in bugs): - results.append( - output_api.PresubmitNotifyResult( - 'Buganizer bugs should be prefixed with b:, not b/.')) - elif show_suggestions: - results.append( - output_api.PresubmitNotifyResult( - 'If this change has an associated bug, add Bug: [bug number] or ' - 'Fixed: [bug number].')) + bugs = change.BugsFromDescription() + results = [] + if bugs: + if any(b.startswith('b/') for b in bugs): + results.append( + output_api.PresubmitNotifyResult( + 'Buganizer bugs should be prefixed with b:, not b/.')) + elif show_suggestions: + results.append( + output_api.PresubmitNotifyResult( + 'If this change has an associated bug, add Bug: [bug number] ' + 'or Fixed: [bug number].')) - if 'Fixes' in change.GitFootersFromDescription(): - results.append( - output_api.PresubmitError( - 'Fixes: is the wrong footer tag, use Fixed: instead.')) - return results + if 'Fixes' in change.GitFootersFromDescription(): + results.append( + output_api.PresubmitError( + 'Fixes: is the wrong footer tag, use Fixed: instead.')) + return results def CheckChangeHasBugField(input_api, output_api): - return CheckChangeHasBugFieldFromChange(input_api.change, output_api) + return CheckChangeHasBugFieldFromChange(input_api.change, output_api) def CheckChangeHasNoUnwantedTagsFromChange(change, output_api): - UNWANTED_TAGS = { - 'FIXED': { - 'why': 'is not supported', - 'instead': 'Use "Fixed:" instead.' - }, - # TODO: BUG, ISSUE - } + UNWANTED_TAGS = { + 'FIXED': { + 'why': 'is not supported', + 'instead': 'Use "Fixed:" instead.' + }, + # TODO: BUG, ISSUE + } - errors = [] - for tag, desc in UNWANTED_TAGS.items(): - if tag in change.tags: - subs = tag, desc['why'], desc.get('instead', '') - errors.append(('%s= %s. %s' % subs).rstrip()) + errors = [] + for tag, desc in UNWANTED_TAGS.items(): + if tag in change.tags: + subs = tag, desc['why'], desc.get('instead', '') + errors.append(('%s= %s. %s' % subs).rstrip()) - return [output_api.PresubmitError('\n'.join(errors))] if errors else [] + return [output_api.PresubmitError('\n'.join(errors))] if errors else [] def CheckChangeHasNoUnwantedTags(input_api, output_api): - return CheckChangeHasNoUnwantedTagsFromChange(input_api.change, output_api) + return CheckChangeHasNoUnwantedTagsFromChange(input_api.change, output_api) def CheckDoNotSubmitInDescription(input_api, output_api): - """Checks that the user didn't add 'DO NOT ''SUBMIT' to the CL description. + """Checks that the user didn't add 'DO NOT ''SUBMIT' to the CL description. """ - # Keyword is concatenated to avoid presubmit check rejecting the CL. - keyword = 'DO NOT ' + 'SUBMIT' - if keyword in input_api.change.DescriptionText(): - return [output_api.PresubmitError( - keyword + ' is present in the changelist description.')] + # Keyword is concatenated to avoid presubmit check rejecting the CL. + keyword = 'DO NOT ' + 'SUBMIT' + if keyword in input_api.change.DescriptionText(): + return [ + output_api.PresubmitError( + keyword + ' is present in the changelist description.') + ] - return [] + return [] def CheckCorpLinksInDescription(input_api, output_api): - """Checks that the description doesn't contain corp links.""" - if _CORP_LINK_KEYWORD in input_api.change.DescriptionText(): - return [ - output_api.PresubmitPromptWarning( - 'Corp link is present in the changelist description.') - ] + """Checks that the description doesn't contain corp links.""" + if _CORP_LINK_KEYWORD in input_api.change.DescriptionText(): + return [ + output_api.PresubmitPromptWarning( + 'Corp link is present in the changelist description.') + ] - return [] + return [] def CheckChangeHasDescription(input_api, output_api): - """Checks the CL description is not empty.""" - text = input_api.change.DescriptionText() - if text.strip() == '': - if input_api.is_committing and not input_api.no_diffs: - return [output_api.PresubmitError('Add a description to the CL.')] + """Checks the CL description is not empty.""" + text = input_api.change.DescriptionText() + if text.strip() == '': + if input_api.is_committing and not input_api.no_diffs: + return [output_api.PresubmitError('Add a description to the CL.')] - return [output_api.PresubmitNotifyResult('Add a description to the CL.')] - return [] + return [ + output_api.PresubmitNotifyResult('Add a description to the CL.') + ] + return [] def CheckChangeWasUploaded(input_api, output_api): - """Checks that the issue was uploaded before committing.""" - if input_api.is_committing and not input_api.change.issue: - message = 'Issue wasn\'t uploaded. Please upload first.' - if input_api.no_diffs: - # Make this just a message with presubmit --all and --files - return [output_api.PresubmitNotifyResult(message)] - return [output_api.PresubmitError(message)] - return [] + """Checks that the issue was uploaded before committing.""" + if input_api.is_committing and not input_api.change.issue: + message = 'Issue wasn\'t uploaded. Please upload first.' + if input_api.no_diffs: + # Make this just a message with presubmit --all and --files + return [output_api.PresubmitNotifyResult(message)] + return [output_api.PresubmitError(message)] + return [] def CheckDescriptionUsesColonInsteadOfEquals(input_api, output_api): - """Checks that the CL description uses a colon after 'Bug' and 'Fixed' tags + """Checks that the CL description uses a colon after 'Bug' and 'Fixed' tags instead of equals. crbug.com only interprets the lines "Bug: xyz" and "Fixed: xyz" but not "Bug=xyz" or "Fixed=xyz". """ - text = input_api.change.DescriptionText() - if input_api.re.search(r'^(Bug|Fixed)=', - text, - flags=input_api.re.IGNORECASE - | input_api.re.MULTILINE): - return [output_api.PresubmitError('Use Bug:/Fixed: instead of Bug=/Fixed=')] - return [] + text = input_api.change.DescriptionText() + if input_api.re.search(r'^(Bug|Fixed)=', + text, + flags=input_api.re.IGNORECASE + | input_api.re.MULTILINE): + return [ + output_api.PresubmitError('Use Bug:/Fixed: instead of Bug=/Fixed=') + ] + return [] ### Content checks def CheckAuthorizedAuthor(input_api, output_api, bot_allowlist=None): - """For non-googler/chromites committers, verify the author's email address is + """For non-googler/chromites committers, verify the author's email address is in AUTHORS. """ - if input_api.is_committing or input_api.no_diffs: - error_type = output_api.PresubmitError - else: - error_type = output_api.PresubmitPromptWarning + if input_api.is_committing or input_api.no_diffs: + error_type = output_api.PresubmitError + else: + error_type = output_api.PresubmitPromptWarning - author = input_api.change.author_email - if not author: - input_api.logging.info('No author, skipping AUTHOR check') + author = input_api.change.author_email + if not author: + input_api.logging.info('No author, skipping AUTHOR check') + return [] + + # This is used for CLs created by trusted robot accounts. + if bot_allowlist and author in bot_allowlist: + return [] + + authors_path = input_api.os_path.join(input_api.PresubmitLocalPath(), + 'AUTHORS') + author_re = input_api.re.compile(r'[^#]+\s+\<(.+?)\>\s*$') + valid_authors = [] + with _io.open(authors_path, encoding='utf-8') as fp: + for line in fp: + m = author_re.match(line) + if m: + valid_authors.append(m.group(1).lower()) + + if not any( + input_api.fnmatch.fnmatch(author.lower(), valid) + for valid in valid_authors): + input_api.logging.info('Valid authors are %s', ', '.join(valid_authors)) + return [ + error_type(( + # pylint: disable=line-too-long + '%s is not in AUTHORS file. If you are a new contributor, please visit\n' + 'https://chromium.googlesource.com/chromium/src/+/refs/heads/main/docs/contributing.md#Legal-stuff\n' + # pylint: enable=line-too-long + 'and read the "Legal stuff" section.\n' + 'If you are a chromite, verify that the contributor signed the ' + 'CLA.') % author) + ] return [] - # This is used for CLs created by trusted robot accounts. - if bot_allowlist and author in bot_allowlist: - return [] - - authors_path = input_api.os_path.join( - input_api.PresubmitLocalPath(), 'AUTHORS') - author_re = input_api.re.compile(r'[^#]+\s+\<(.+?)\>\s*$') - valid_authors = [] - with _io.open(authors_path, encoding='utf-8') as fp: - for line in fp: - m = author_re.match(line) - if m: - valid_authors.append(m.group(1).lower()) - - if not any(input_api.fnmatch.fnmatch(author.lower(), valid) - for valid in valid_authors): - input_api.logging.info('Valid authors are %s', ', '.join(valid_authors)) - return [ - error_type(( - # pylint: disable=line-too-long - '%s is not in AUTHORS file. If you are a new contributor, please visit\n' - 'https://chromium.googlesource.com/chromium/src/+/refs/heads/main/docs/contributing.md#Legal-stuff\n' - # pylint: enable=line-too-long - 'and read the "Legal stuff" section\n' - 'If you are a chromite, verify that the contributor signed the CLA.') % - author) - ] - return [] - def CheckDoNotSubmitInFiles(input_api, output_api): - """Checks that the user didn't add 'DO NOT ''SUBMIT' to any files.""" - # We want to check every text file, not just source files. - file_filter = lambda x : x + """Checks that the user didn't add 'DO NOT ''SUBMIT' to any files.""" + # We want to check every text file, not just source files. + file_filter = lambda x: x - # Keyword is concatenated to avoid presubmit check rejecting the CL. - keyword = 'DO NOT ' + 'SUBMIT' - def DoNotSubmitRule(extension, line): - try: - return keyword not in line - # Fallback to True for non-text content - except UnicodeDecodeError: - return True + # Keyword is concatenated to avoid presubmit check rejecting the CL. + keyword = 'DO NOT ' + 'SUBMIT' - errors = _FindNewViolationsOfRule(DoNotSubmitRule, input_api, file_filter) - text = '\n'.join('Found %s in %s' % (keyword, loc) for loc in errors) - if text: - return [output_api.PresubmitError(text)] - return [] + def DoNotSubmitRule(extension, line): + try: + return keyword not in line + # Fallback to True for non-text content + except UnicodeDecodeError: + return True + + errors = _FindNewViolationsOfRule(DoNotSubmitRule, input_api, file_filter) + text = '\n'.join('Found %s in %s' % (keyword, loc) for loc in errors) + if text: + return [output_api.PresubmitError(text)] + return [] def CheckCorpLinksInFiles(input_api, output_api, source_file_filter=None): - """Checks that files do not contain a corp link.""" - errors = _FindNewViolationsOfRule( - lambda _, line: _CORP_LINK_KEYWORD not in line, input_api, - source_file_filter) - text = '\n'.join('Found corp link in %s' % loc for loc in errors) - if text: - return [output_api.PresubmitPromptWarning(text)] - return [] + """Checks that files do not contain a corp link.""" + errors = _FindNewViolationsOfRule( + lambda _, line: _CORP_LINK_KEYWORD not in line, input_api, + source_file_filter) + text = '\n'.join('Found corp link in %s' % loc for loc in errors) + if text: + return [output_api.PresubmitPromptWarning(text)] + return [] def GetCppLintFilters(lint_filters=None): - filters = OFF_UNLESS_MANUALLY_ENABLED_LINT_FILTERS[:] - if lint_filters is None: - lint_filters = OFF_BY_DEFAULT_LINT_FILTERS - filters.extend(lint_filters) - return filters + filters = OFF_UNLESS_MANUALLY_ENABLED_LINT_FILTERS[:] + if lint_filters is None: + lint_filters = OFF_BY_DEFAULT_LINT_FILTERS + filters.extend(lint_filters) + return filters -def CheckChangeLintsClean(input_api, output_api, source_file_filter=None, - lint_filters=None, verbose_level=None): - """Checks that all '.cc' and '.h' files pass cpplint.py.""" - _RE_IS_TEST = input_api.re.compile(r'.*tests?.(cc|h)$') - result = [] +def CheckChangeLintsClean(input_api, + output_api, + source_file_filter=None, + lint_filters=None, + verbose_level=None): + """Checks that all '.cc' and '.h' files pass cpplint.py.""" + _RE_IS_TEST = input_api.re.compile(r'.*tests?.(cc|h)$') + result = [] - cpplint = input_api.cpplint - # Access to a protected member _XX of a client class - # pylint: disable=protected-access - cpplint._cpplint_state.ResetErrorCounts() + cpplint = input_api.cpplint + # Access to a protected member _XX of a client class + # pylint: disable=protected-access + cpplint._cpplint_state.ResetErrorCounts() - cpplint._SetFilters(','.join(GetCppLintFilters(lint_filters))) + cpplint._SetFilters(','.join(GetCppLintFilters(lint_filters))) - # Use VS error format on Windows to make it easier to step through the - # results. - if input_api.platform == 'win32': - cpplint._SetOutputFormat('vs7') + # Use VS error format on Windows to make it easier to step through the + # results. + if input_api.platform == 'win32': + cpplint._SetOutputFormat('vs7') - if source_file_filter == None: - # The only valid extensions for cpplint are .cc, .h, .cpp, .cu, and .ch. - # Only process those extensions which are used in Chromium. - INCLUDE_CPP_FILES_ONLY = (r'.*\.(cc|h|cpp)$', ) - source_file_filter = lambda x: input_api.FilterSourceFile( - x, - files_to_check=INCLUDE_CPP_FILES_ONLY, - files_to_skip=input_api.DEFAULT_FILES_TO_SKIP) + if source_file_filter == None: + # The only valid extensions for cpplint are .cc, .h, .cpp, .cu, and .ch. + # Only process those extensions which are used in Chromium. + INCLUDE_CPP_FILES_ONLY = (r'.*\.(cc|h|cpp)$', ) + source_file_filter = lambda x: input_api.FilterSourceFile( + x, + files_to_check=INCLUDE_CPP_FILES_ONLY, + files_to_skip=input_api.DEFAULT_FILES_TO_SKIP) - # We currently are more strict with normal code than unit tests; 4 and 5 are - # the verbosity level that would normally be passed to cpplint.py through - # --verbose=#. Hopefully, in the future, we can be more verbose. - files = [f.AbsoluteLocalPath() for f in - input_api.AffectedSourceFiles(source_file_filter)] - for file_name in files: - if _RE_IS_TEST.match(file_name): - level = 5 - else: - level = 4 - - verbose_level = verbose_level or level - cpplint.ProcessFile(file_name, verbose_level) - - if cpplint._cpplint_state.error_count > 0: - # cpplint errors currently cannot be counted as errors during upload - # presubmits because some directories only run cpplint during upload and - # therefore are far from cpplint clean. - if input_api.is_committing: - res_type = output_api.PresubmitError - else: - res_type = output_api.PresubmitPromptWarning - result = [ - res_type('Changelist failed cpplint.py check. ' - 'Search the output for "(cpplint)"') + # We currently are more strict with normal code than unit tests; 4 and 5 are + # the verbosity level that would normally be passed to cpplint.py through + # --verbose=#. Hopefully, in the future, we can be more verbose. + files = [ + f.AbsoluteLocalPath() + for f in input_api.AffectedSourceFiles(source_file_filter) ] + for file_name in files: + if _RE_IS_TEST.match(file_name): + level = 5 + else: + level = 4 - return result + verbose_level = verbose_level or level + cpplint.ProcessFile(file_name, verbose_level) + + if cpplint._cpplint_state.error_count > 0: + # cpplint errors currently cannot be counted as errors during upload + # presubmits because some directories only run cpplint during upload and + # therefore are far from cpplint clean. + if input_api.is_committing: + res_type = output_api.PresubmitError + else: + res_type = output_api.PresubmitPromptWarning + result = [ + res_type('Changelist failed cpplint.py check. ' + 'Search the output for "(cpplint)"') + ] + + return result def CheckChangeHasNoCR(input_api, output_api, source_file_filter=None): - """Checks no '\r' (CR) character is in any source files.""" - cr_files = [] - for f in input_api.AffectedSourceFiles(source_file_filter): - if '\r' in input_api.ReadFile(f, 'rb'): - cr_files.append(f.LocalPath()) - if cr_files: - return [output_api.PresubmitPromptWarning( - 'Found a CR character in these files:', items=cr_files)] - return [] + """Checks no '\r' (CR) character is in any source files.""" + cr_files = [] + for f in input_api.AffectedSourceFiles(source_file_filter): + if '\r' in input_api.ReadFile(f, 'rb'): + cr_files.append(f.LocalPath()) + if cr_files: + return [ + output_api.PresubmitPromptWarning( + 'Found a CR character in these files:', items=cr_files) + ] + return [] def CheckChangeHasOnlyOneEol(input_api, output_api, source_file_filter=None): - """Checks the files ends with one and only one \n (LF).""" - eof_files = [] - for f in input_api.AffectedSourceFiles(source_file_filter): - contents = input_api.ReadFile(f, 'rb') - # Check that the file ends in one and only one newline character. - if len(contents) > 1 and (contents[-1:] != '\n' or contents[-2:-1] == '\n'): - eof_files.append(f.LocalPath()) + """Checks the files ends with one and only one \n (LF).""" + eof_files = [] + for f in input_api.AffectedSourceFiles(source_file_filter): + contents = input_api.ReadFile(f, 'rb') + # Check that the file ends in one and only one newline character. + if len(contents) > 1 and (contents[-1:] != '\n' + or contents[-2:-1] == '\n'): + eof_files.append(f.LocalPath()) - if eof_files: - return [output_api.PresubmitPromptWarning( - 'These files should end in one (and only one) newline character:', - items=eof_files)] - return [] + if eof_files: + return [ + output_api.PresubmitPromptWarning( + 'These files should end in one (and only one) newline character:', + items=eof_files) + ] + return [] -def CheckChangeHasNoCrAndHasOnlyOneEol(input_api, output_api, +def CheckChangeHasNoCrAndHasOnlyOneEol(input_api, + output_api, source_file_filter=None): - """Runs both CheckChangeHasNoCR and CheckChangeHasOnlyOneEOL in one pass. + """Runs both CheckChangeHasNoCR and CheckChangeHasOnlyOneEOL in one pass. It is faster because it is reading the file only once. """ - cr_files = [] - eof_files = [] - for f in input_api.AffectedSourceFiles(source_file_filter): - contents = input_api.ReadFile(f, 'rb') - if '\r' in contents: - cr_files.append(f.LocalPath()) - # Check that the file ends in one and only one newline character. - if len(contents) > 1 and (contents[-1:] != '\n' or contents[-2:-1] == '\n'): - eof_files.append(f.LocalPath()) - outputs = [] - if cr_files: - outputs.append(output_api.PresubmitPromptWarning( - 'Found a CR character in these files:', items=cr_files)) - if eof_files: - outputs.append(output_api.PresubmitPromptWarning( - 'These files should end in one (and only one) newline character:', - items=eof_files)) - return outputs + cr_files = [] + eof_files = [] + for f in input_api.AffectedSourceFiles(source_file_filter): + contents = input_api.ReadFile(f, 'rb') + if '\r' in contents: + cr_files.append(f.LocalPath()) + # Check that the file ends in one and only one newline character. + if len(contents) > 1 and (contents[-1:] != '\n' + or contents[-2:-1] == '\n'): + eof_files.append(f.LocalPath()) + outputs = [] + if cr_files: + outputs.append( + output_api.PresubmitPromptWarning( + 'Found a CR character in these files:', items=cr_files)) + if eof_files: + outputs.append( + output_api.PresubmitPromptWarning( + 'These files should end in one (and only one) newline character:', + items=eof_files)) + return outputs def CheckGenderNeutral(input_api, output_api, source_file_filter=None): - """Checks that there are no gendered pronouns in any of the text files to be + """Checks that there are no gendered pronouns in any of the text files to be submitted. """ - if input_api.no_diffs: + if input_api.no_diffs: + return [] + + gendered_re = input_api.re.compile( + r'(^|\s|\(|\[)([Hh]e|[Hh]is|[Hh]ers?|[Hh]im|[Ss]he|[Gg]uys?)\\b') + + errors = [] + for f in input_api.AffectedFiles(include_deletes=False, + file_filter=source_file_filter): + for line_num, line in f.ChangedContents(): + if gendered_re.search(line): + errors.append('%s (%d): %s' % (f.LocalPath(), line_num, line)) + + if errors: + return [ + output_api.PresubmitPromptWarning('Found a gendered pronoun in:', + long_text='\n'.join(errors)) + ] return [] - gendered_re = input_api.re.compile( - r'(^|\s|\(|\[)([Hh]e|[Hh]is|[Hh]ers?|[Hh]im|[Ss]he|[Gg]uys?)\\b') - - errors = [] - for f in input_api.AffectedFiles(include_deletes=False, - file_filter=source_file_filter): - for line_num, line in f.ChangedContents(): - if gendered_re.search(line): - errors.append('%s (%d): %s' % (f.LocalPath(), line_num, line)) - - if errors: - return [output_api.PresubmitPromptWarning('Found a gendered pronoun in:', - long_text='\n'.join(errors))] - return [] - def _ReportErrorFileAndLine(filename, line_num, dummy_line): - """Default error formatter for _FindNewViolationsOfRule.""" - return '%s:%s' % (filename, line_num) + """Default error formatter for _FindNewViolationsOfRule.""" + return '%s:%s' % (filename, line_num) def _GenerateAffectedFileExtList(input_api, source_file_filter): - """Generate a list of (file, extension) tuples from affected files. + """Generate a list of (file, extension) tuples from affected files. The result can be fed to _FindNewViolationsOfRule() directly, or could be filtered before doing that. @@ -420,16 +446,16 @@ def _GenerateAffectedFileExtList(input_api, source_file_filter): A list of (file, extension) tuples, where |file| is an affected file, and |extension| its file path extension. """ - for f in input_api.AffectedFiles( - include_deletes=False, file_filter=source_file_filter): - extension = str(f.LocalPath()).rsplit('.', 1)[-1] - yield (f, extension) + for f in input_api.AffectedFiles(include_deletes=False, + file_filter=source_file_filter): + extension = str(f.LocalPath()).rsplit('.', 1)[-1] + yield (f, extension) def _FindNewViolationsOfRuleForList(callable_rule, file_ext_list, error_formatter=_ReportErrorFileAndLine): - """Find all newly introduced violations of a per-line rule (a callable). + """Find all newly introduced violations of a per-line rule (a callable). Prefer calling _FindNewViolationsOfRule() instead of this function, unless the list of affected files need to be filtered in a special way. @@ -445,27 +471,27 @@ def _FindNewViolationsOfRuleForList(callable_rule, Returns: A list of the newly-introduced violations reported by the rule. """ - errors = [] - for f, extension in file_ext_list: - # For speed, we do two passes, checking first the full file. Shelling out - # to the SCM to determine the changed region can be quite expensive on - # Win32. Assuming that most files will be kept problem-free, we can - # skip the SCM operations most of the time. - if all(callable_rule(extension, line) for line in f.NewContents()): - continue # No violation found in full text: can skip considering diff. + errors = [] + for f, extension in file_ext_list: + # For speed, we do two passes, checking first the full file. Shelling + # out to the SCM to determine the changed region can be quite expensive + # on Win32. Assuming that most files will be kept problem-free, we can + # skip the SCM operations most of the time. + if all(callable_rule(extension, line) for line in f.NewContents()): + continue # No violation found in full text: can skip considering diff. - for line_num, line in f.ChangedContents(): - if not callable_rule(extension, line): - errors.append(error_formatter(f.LocalPath(), line_num, line)) + for line_num, line in f.ChangedContents(): + if not callable_rule(extension, line): + errors.append(error_formatter(f.LocalPath(), line_num, line)) - return errors + return errors def _FindNewViolationsOfRule(callable_rule, input_api, source_file_filter=None, error_formatter=_ReportErrorFileAndLine): - """Find all newly introduced violations of a per-line rule (a callable). + """Find all newly introduced violations of a per-line rule (a callable). Arguments: callable_rule: a callable taking a file extension and line of input and @@ -478,367 +504,396 @@ def _FindNewViolationsOfRule(callable_rule, Returns: A list of the newly-introduced violations reported by the rule. """ - if input_api.no_diffs: - return [] - return _FindNewViolationsOfRuleForList( - callable_rule, _GenerateAffectedFileExtList( - input_api, source_file_filter), error_formatter) + if input_api.no_diffs: + return [] + return _FindNewViolationsOfRuleForList( + callable_rule, + _GenerateAffectedFileExtList(input_api, source_file_filter), + error_formatter) def CheckChangeHasNoTabs(input_api, output_api, source_file_filter=None): - """Checks that there are no tab characters in any of the text files to be + """Checks that there are no tab characters in any of the text files to be submitted. """ - # In addition to the filter, make sure that makefiles are skipped. - if not source_file_filter: - # It's the default filter. - source_file_filter = input_api.FilterSourceFile - def filter_more(affected_file): - basename = input_api.os_path.basename(affected_file.LocalPath()) - return (not (basename in ('Makefile', 'makefile') or - basename.endswith('.mk')) and - source_file_filter(affected_file)) + # In addition to the filter, make sure that makefiles are skipped. + if not source_file_filter: + # It's the default filter. + source_file_filter = input_api.FilterSourceFile - tabs = _FindNewViolationsOfRule(lambda _, line : '\t' not in line, - input_api, filter_more) + def filter_more(affected_file): + basename = input_api.os_path.basename(affected_file.LocalPath()) + return (not (basename in ('Makefile', 'makefile') + or basename.endswith('.mk')) + and source_file_filter(affected_file)) - if tabs: - return [output_api.PresubmitPromptWarning('Found a tab character in:', - long_text='\n'.join(tabs))] - return [] + tabs = _FindNewViolationsOfRule(lambda _, line: '\t' not in line, input_api, + filter_more) + + if tabs: + return [ + output_api.PresubmitPromptWarning('Found a tab character in:', + long_text='\n'.join(tabs)) + ] + return [] def CheckChangeTodoHasOwner(input_api, output_api, source_file_filter=None): - """Checks that the user didn't add TODO(name) without an owner.""" + """Checks that the user didn't add TODO(name) without an owner.""" - unowned_todo = input_api.re.compile('TO''DO[^(]') - errors = _FindNewViolationsOfRule(lambda _, x : not unowned_todo.search(x), - input_api, source_file_filter) - errors = ['Found TO''DO with no owner in ' + x for x in errors] - if errors: - return [output_api.PresubmitPromptWarning('\n'.join(errors))] - return [] + unowned_todo = input_api.re.compile('TO' 'DO[^(]') + errors = _FindNewViolationsOfRule(lambda _, x: not unowned_todo.search(x), + input_api, source_file_filter) + errors = ['Found TO' 'DO with no owner in ' + x for x in errors] + if errors: + return [output_api.PresubmitPromptWarning('\n'.join(errors))] + return [] -def CheckChangeHasNoStrayWhitespace(input_api, output_api, +def CheckChangeHasNoStrayWhitespace(input_api, + output_api, source_file_filter=None): - """Checks that there is no stray whitespace at source lines end.""" - errors = _FindNewViolationsOfRule(lambda _, line : line.rstrip() == line, - input_api, source_file_filter) - if errors: - return [output_api.PresubmitPromptWarning( - 'Found line ending with white spaces in:', - long_text='\n'.join(errors))] - return [] + """Checks that there is no stray whitespace at source lines end.""" + errors = _FindNewViolationsOfRule(lambda _, line: line.rstrip() == line, + input_api, source_file_filter) + if errors: + return [ + output_api.PresubmitPromptWarning( + 'Found line ending with white spaces in:', + long_text='\n'.join(errors)) + ] + return [] def CheckLongLines(input_api, output_api, maxlen, source_file_filter=None): - """Checks that there aren't any lines longer than maxlen characters in any of + """Checks that there aren't any lines longer than maxlen characters in any of the text files to be submitted. """ - if input_api.no_diffs: - return [] - maxlens = { - 'java': 100, - # This is specifically for Android's handwritten makefiles (Android.mk). - 'mk': 200, - 'rs': 100, - '': maxlen, - } + if input_api.no_diffs: + return [] + maxlens = { + 'java': 100, + # This is specifically for Android's handwritten makefiles (Android.mk). + 'mk': 200, + 'rs': 100, + '': maxlen, + } - # Language specific exceptions to max line length. - # '.h' is considered an obj-c file extension, since OBJC_EXCEPTIONS are a - # superset of CPP_EXCEPTIONS. - CPP_FILE_EXTS = ('c', 'cc') - CPP_EXCEPTIONS = ('#define', '#endif', '#if', '#include', '#pragma') - HTML_FILE_EXTS = ('html',) - HTML_EXCEPTIONS = (' extra_maxlen: - return False + if line_len > extra_maxlen: + return False - if 'url(' in line and file_extension == 'css': - return True + if 'url(' in line and file_extension == 'css': + return True - if '= 0: + if is_global_pylint_directive(line, pos): + global_check_enabled = False # Global disable + else: + continue # Local disable. + + do_check = global_check_enabled + + pos = line.find('pylint: enable=line-too-long') + if pos >= 0: + if is_global_pylint_directive(line, pos): + global_check_enabled = True # Global enable + do_check = True # Ensure it applies to current line as well. + else: + do_check = True # Local enable + + if do_check and not line_is_short: + errors.append(error_formatter(file_path, line_num, line)) + + return errors + + def format_error(filename, line_num, line): + return '%s, line %s, %s chars' % (filename, line_num, len(line)) + + file_ext_list = list( + _GenerateAffectedFileExtList(input_api, source_file_filter)) - def check_python_long_lines(affected_files, error_formatter): errors = [] - global_check_enabled = True - for f in affected_files: - file_path = f.LocalPath() - for idx, line in enumerate(f.NewContents()): - line_num = idx + 1 - line_is_short = no_long_lines(PY_FILE_EXTS[0], line) + # For non-Python files, a simple line-based rule check is enough. + non_py_file_ext_list = [ + x for x in file_ext_list if x[1] not in PY_FILE_EXTS + ] + if non_py_file_ext_list: + errors += _FindNewViolationsOfRuleForList(no_long_lines, + non_py_file_ext_list, + error_formatter=format_error) - pos = line.find('pylint: disable=line-too-long') - if pos >= 0: - if is_global_pylint_directive(line, pos): - global_check_enabled = False # Global disable - else: - continue # Local disable. + # However, Python files need more sophisticated checks that need parsing + # the whole source file. + py_file_list = [x[0] for x in file_ext_list if x[1] in PY_FILE_EXTS] + if py_file_list: + errors += check_python_long_lines(py_file_list, + error_formatter=format_error) + if errors: + msg = 'Found %d lines longer than %s characters (first 5 shown).' % ( + len(errors), maxlen) + return [output_api.PresubmitPromptWarning(msg, items=errors[:5])] - do_check = global_check_enabled - - pos = line.find('pylint: enable=line-too-long') - if pos >= 0: - if is_global_pylint_directive(line, pos): - global_check_enabled = True # Global enable - do_check = True # Ensure it applies to current line as well. - else: - do_check = True # Local enable - - if do_check and not line_is_short: - errors.append(error_formatter(file_path, line_num, line)) - - return errors - - def format_error(filename, line_num, line): - return '%s, line %s, %s chars' % (filename, line_num, len(line)) - - file_ext_list = list( - _GenerateAffectedFileExtList(input_api, source_file_filter)) - - errors = [] - - # For non-Python files, a simple line-based rule check is enough. - non_py_file_ext_list = [x for x in file_ext_list if x[1] not in PY_FILE_EXTS] - if non_py_file_ext_list: - errors += _FindNewViolationsOfRuleForList( - no_long_lines, non_py_file_ext_list, error_formatter=format_error) - - # However, Python files need more sophisticated checks that need parsing - # the whole source file. - py_file_list = [x[0] for x in file_ext_list if x[1] in PY_FILE_EXTS] - if py_file_list: - errors += check_python_long_lines( - py_file_list, error_formatter=format_error) - if errors: - msg = 'Found %d lines longer than %s characters (first 5 shown).' % ( - len(errors), maxlen) - return [output_api.PresubmitPromptWarning(msg, items=errors[:5])] - - return [] + return [] -def CheckLicense(input_api, output_api, license_re_param=None, - project_name=None, source_file_filter=None, accept_empty_files=True): - """Verifies the license header. +def CheckLicense(input_api, + output_api, + license_re_param=None, + project_name=None, + source_file_filter=None, + accept_empty_files=True): + """Verifies the license header. """ - # Early-out if the license_re is guaranteed to match everything. - if license_re_param and license_re_param == '.*': - return [] + # Early-out if the license_re is guaranteed to match everything. + if license_re_param and license_re_param == '.*': + return [] - current_year = int(input_api.time.strftime('%Y')) + current_year = int(input_api.time.strftime('%Y')) - if license_re_param: - new_license_re = license_re = license_re_param - else: - project_name = project_name or 'Chromium' + if license_re_param: + new_license_re = license_re = license_re_param + else: + project_name = project_name or 'Chromium' - # Accept any year number from 2006 to the current year, or the special - # 2006-20xx string used on the oldest files. 2006-20xx is deprecated, but - # tolerated on old files. On new files the current year must be specified. - allowed_years = (str(s) for s in reversed(range(2006, current_year + 1))) - years_re = '(' + '|'.join(allowed_years) + '|2006-2008|2006-2009|2006-2010)' + # Accept any year number from 2006 to the current year, or the special + # 2006-20xx string used on the oldest files. 2006-20xx is deprecated, + # but tolerated on old files. On new files the current year must be + # specified. + allowed_years = (str(s) + for s in reversed(range(2006, current_year + 1))) + years_re = '(' + '|'.join( + allowed_years) + '|2006-2008|2006-2009|2006-2010)' - # Reduce duplication between the two regex expressions. - key_line = ('Use of this source code is governed by a BSD-style license ' - 'that can be') - # The (c) is deprecated, but tolerate it until it's removed from all files. - # "All rights reserved" is also deprecated, but tolerate it until it's - # removed from all files. - license_re = (r'.*? Copyright (\(c\) )?%(year)s The %(project)s Authors' - r'(\. All rights reserved\.)?\n' - r'.*? %(key_line)s\n' - r'.*? found in the LICENSE file\.(?: \*/)?\n') % { - 'year': years_re, - 'project': project_name, - 'key_line': key_line, - } - # On new files don't tolerate any digression from the ideal. - new_license_re = (r'.*? Copyright %(year)s The %(project)s Authors\n' + # Reduce duplication between the two regex expressions. + key_line = ( + 'Use of this source code is governed by a BSD-style license ' + 'that can be') + # The (c) is deprecated, but tolerate it until it's removed from all + # files. "All rights reserved" is also deprecated, but tolerate it until + # it's removed from all files. + license_re = (r'.*? Copyright (\(c\) )?%(year)s The %(project)s Authors' + r'(\. All rights reserved\.)?\n' r'.*? %(key_line)s\n' r'.*? found in the LICENSE file\.(?: \*/)?\n') % { 'year': years_re, 'project': project_name, 'key_line': key_line, } - - license_re = input_api.re.compile(license_re, input_api.re.MULTILINE) - new_license_re = input_api.re.compile(new_license_re, input_api.re.MULTILINE) - bad_files = [] - wrong_year_new_files = [] - bad_new_files = [] - for f in input_api.AffectedSourceFiles(source_file_filter): - # Only examine the first 1,000 bytes of the file to avoid expensive and - # fruitless regex searches over long files with no license. - # re.match would also avoid this but can't be used because some files have - # a shebang line ahead of the license. - # The \r\n fixup is because it is possible on Windows to copy/paste the - # license in such a way that \r\n line endings are inserted. This leads to - # confusing license error messages - it's better to let the separate \r\n - # check handle those. - contents = input_api.ReadFile(f, 'r')[:1000].replace('\r\n', '\n') - if accept_empty_files and not contents: - continue - if f.Action() == 'A': - # Stricter checking for new files (but might actually be moved). - match = new_license_re.search(contents) - if not match: - # License is totally wrong. - bad_new_files.append(f.LocalPath()) - elif not license_re_param and match.groups()[0] != str(current_year): - # If we're using the built-in license_re on a new file then make sure - # the year is correct. - wrong_year_new_files.append(f.LocalPath()) - elif not license_re.search(contents): - bad_files.append(f.LocalPath()) - results = [] - if bad_new_files: - if license_re_param: - error_message = ('License on new files must match:\n\n%s\n' % - license_re_param) - else: - # Verbatim text that can be copy-pasted into new files (possibly adjusting - # the leading comment delimiter). - new_license_text = ('// Copyright %(year)s The %(project)s Authors\n' - '// %(key_line)s\n' - '// found in the LICENSE file.\n') % { - 'year': current_year, + # On new files don't tolerate any digression from the ideal. + new_license_re = (r'.*? Copyright %(year)s The %(project)s Authors\n' + r'.*? %(key_line)s\n' + r'.*? found in the LICENSE file\.(?: \*/)?\n') % { + 'year': years_re, 'project': project_name, 'key_line': key_line, } - error_message = ( - 'License on new files must be:\n\n%s\n' % new_license_text + - '(adjusting the comment delimiter accordingly).\n\n' + - 'If this is a moved file, then update the license but do not ' + - 'update the year.\n\n') - error_message += 'Found a bad license header in these new or moved files:' - results.append(output_api.PresubmitError(error_message, - items=bad_new_files)) - if wrong_year_new_files: - # We can't distinguish between new and moved files, so this has to be a - # warning rather than an error. - results.append( - output_api.PresubmitPromptWarning( - 'License doesn\'t list the current year. If this is a new file, ' - 'use the current year. If this is a moved file then ignore this ' - 'warning.', - items=wrong_year_new_files)) - if bad_files: - results.append( - output_api.PresubmitPromptWarning( - 'License must match:\n%s\n' % license_re.pattern + - 'Found a bad license header in these files:', - items=bad_files)) - return results + + license_re = input_api.re.compile(license_re, input_api.re.MULTILINE) + new_license_re = input_api.re.compile(new_license_re, + input_api.re.MULTILINE) + bad_files = [] + wrong_year_new_files = [] + bad_new_files = [] + for f in input_api.AffectedSourceFiles(source_file_filter): + # Only examine the first 1,000 bytes of the file to avoid expensive and + # fruitless regex searches over long files with no license. + # re.match would also avoid this but can't be used because some files + # have a shebang line ahead of the license. The \r\n fixup is because it + # is possible on Windows to copy/paste the license in such a way that + # \r\n line endings are inserted. This leads to confusing license error + # messages - it's better to let the separate \r\n check handle those. + contents = input_api.ReadFile(f, 'r')[:1000].replace('\r\n', '\n') + if accept_empty_files and not contents: + continue + if f.Action() == 'A': + # Stricter checking for new files (but might actually be moved). + match = new_license_re.search(contents) + if not match: + # License is totally wrong. + bad_new_files.append(f.LocalPath()) + elif not license_re_param and match.groups()[0] != str( + current_year): + # If we're using the built-in license_re on a new file then make + # sure the year is correct. + wrong_year_new_files.append(f.LocalPath()) + elif not license_re.search(contents): + bad_files.append(f.LocalPath()) + results = [] + if bad_new_files: + if license_re_param: + error_message = ('License on new files must match:\n\n%s\n' % + license_re_param) + else: + # Verbatim text that can be copy-pasted into new files (possibly + # adjusting the leading comment delimiter). + new_license_text = ( + '// Copyright %(year)s The %(project)s Authors\n' + '// %(key_line)s\n' + '// found in the LICENSE file.\n') % { + 'year': current_year, + 'project': project_name, + 'key_line': key_line, + } + error_message = ( + 'License on new files must be:\n\n%s\n' % new_license_text + + '(adjusting the comment delimiter accordingly).\n\n' + + 'If this is a moved file, then update the license but do not ' + + 'update the year.\n\n') + error_message += 'Found a bad license header in these new or moved files:' + results.append( + output_api.PresubmitError(error_message, items=bad_new_files)) + if wrong_year_new_files: + # We can't distinguish between new and moved files, so this has to be a + # warning rather than an error. + results.append( + output_api.PresubmitPromptWarning( + 'License doesn\'t list the current year. If this is a new file, ' + 'use the current year. If this is a moved file then ignore this ' + 'warning.', + items=wrong_year_new_files)) + if bad_files: + results.append( + output_api.PresubmitPromptWarning( + 'License must match:\n%s\n' % license_re.pattern + + 'Found a bad license header in these files:', + items=bad_files)) + return results def CheckChromiumDependencyMetadata(input_api, output_api, file_filter=None): - """Check files for Chromium third party dependency metadata have sufficient + """Check files for Chromium third party dependency metadata have sufficient information, and are correctly formatted. See the README.chromium.template at https://chromium.googlesource.com/chromium/src/+/main/third_party/README.chromium.template """ - # If the file filter is unspecified, filter to known Chromium metadata files. - if file_filter is None: - file_filter = lambda f: metadata.discover.is_metadata_file(f.LocalPath()) + # If the file filter is unspecified, filter to known Chromium metadata + # files. + if file_filter is None: + file_filter = lambda f: metadata.discover.is_metadata_file(f.LocalPath( + )) - # The repo's root directory is required to check license files. - repo_root_dir = input_api.change.RepositoryRoot() + # The repo's root directory is required to check license files. + repo_root_dir = input_api.change.RepositoryRoot() - outputs = [] - for f in input_api.AffectedFiles(file_filter=file_filter): - if f.Action() == 'D': - # No need to validate a deleted file. - continue + outputs = [] + for f in input_api.AffectedFiles(file_filter=file_filter): + if f.Action() == 'D': + # No need to validate a deleted file. + continue - errors, warnings = metadata.validate.check_file( - filepath=f.AbsoluteLocalPath(), - repo_root_dir=repo_root_dir, - reader=input_api.ReadFile, - ) + errors, warnings = metadata.validate.check_file( + filepath=f.AbsoluteLocalPath(), + repo_root_dir=repo_root_dir, + reader=input_api.ReadFile, + ) - for warning in warnings: - outputs.append(output_api.PresubmitPromptWarning(warning, [f])) + for warning in warnings: + outputs.append(output_api.PresubmitPromptWarning(warning, [f])) - for error in errors: - outputs.append(output_api.PresubmitError(error, [f])) + for error in errors: + outputs.append(output_api.PresubmitError(error, [f])) - return outputs + return outputs ### Other checks + def CheckDoNotSubmit(input_api, output_api): - return ( - CheckDoNotSubmitInDescription(input_api, output_api) + - CheckDoNotSubmitInFiles(input_api, output_api) - ) + return (CheckDoNotSubmitInDescription(input_api, output_api) + + CheckDoNotSubmitInFiles(input_api, output_api)) -def CheckTreeIsOpen(input_api, output_api, - url=None, closed=None, json_url=None): - """Check whether to allow commit without prompt. +def CheckTreeIsOpen(input_api, + output_api, + url=None, + closed=None, + json_url=None): + """Check whether to allow commit without prompt. Supports two styles: 1. Checks that an url's content doesn't match a regexp that would mean that @@ -851,35 +906,42 @@ def CheckTreeIsOpen(input_api, output_api, closed: regex to match for closed status. json_url: url to download json style status. """ - if not input_api.is_committing or \ - 'PRESUBMIT_SKIP_NETWORK' in _os.environ: + if not input_api.is_committing or \ + 'PRESUBMIT_SKIP_NETWORK' in _os.environ: + return [] + try: + if json_url: + connection = input_api.urllib_request.urlopen(json_url) + status = input_api.json.loads(connection.read()) + connection.close() + if not status['can_commit_freely']: + short_text = 'Tree state is: ' + status['general_state'] + long_text = status['message'] + '\n' + json_url + if input_api.no_diffs: + return [ + output_api.PresubmitPromptWarning(short_text, + long_text=long_text) + ] + return [ + output_api.PresubmitError(short_text, long_text=long_text) + ] + else: + # TODO(bradnelson): drop this once all users are gone. + connection = input_api.urllib_request.urlopen(url) + status = connection.read() + connection.close() + if input_api.re.match(closed, status): + long_text = status + '\n' + url + return [ + output_api.PresubmitError('The tree is closed.', + long_text=long_text) + ] + except IOError as e: + return [ + output_api.PresubmitError('Error fetching tree status.', + long_text=str(e)) + ] return [] - try: - if json_url: - connection = input_api.urllib_request.urlopen(json_url) - status = input_api.json.loads(connection.read()) - connection.close() - if not status['can_commit_freely']: - short_text = 'Tree state is: ' + status['general_state'] - long_text = status['message'] + '\n' + json_url - if input_api.no_diffs: - return [ - output_api.PresubmitPromptWarning(short_text, long_text=long_text) - ] - return [output_api.PresubmitError(short_text, long_text=long_text)] - else: - # TODO(bradnelson): drop this once all users are gone. - connection = input_api.urllib_request.urlopen(url) - status = connection.read() - connection.close() - if input_api.re.match(closed, status): - long_text = status + '\n' + url - return [output_api.PresubmitError('The tree is closed.', - long_text=long_text)] - except IOError as e: - return [output_api.PresubmitError('Error fetching tree status.', - long_text=str(e))] - return [] def GetUnitTestsInDirectory(input_api, @@ -893,45 +955,45 @@ def GetUnitTestsInDirectory(input_api, skip_shebang_check=True, allowlist=None, blocklist=None): - """Lists all files in a directory and runs them. Doesn't recurse. + """Lists all files in a directory and runs them. Doesn't recurse. It's mainly a wrapper for RunUnitTests. Use allowlist and blocklist to filter tests accordingly. run_on_python2, run_on_python3, and skip_shebang_check are no longer used but have to be retained because of the many callers in other repos that pass them in. """ - del run_on_python2 - del run_on_python3 - del skip_shebang_check + del run_on_python2 + del run_on_python3 + del skip_shebang_check - unit_tests = [] - test_path = input_api.os_path.abspath( - input_api.os_path.join(input_api.PresubmitLocalPath(), directory)) + unit_tests = [] + test_path = input_api.os_path.abspath( + input_api.os_path.join(input_api.PresubmitLocalPath(), directory)) - def check(filename, filters): - return any(True for i in filters if input_api.re.match(i, filename)) + def check(filename, filters): + return any(True for i in filters if input_api.re.match(i, filename)) - to_run = found = 0 - for filename in input_api.os_listdir(test_path): - found += 1 - fullpath = input_api.os_path.join(test_path, filename) - if not input_api.os_path.isfile(fullpath): - continue - if files_to_check and not check(filename, files_to_check): - continue - if files_to_skip and check(filename, files_to_skip): - continue - unit_tests.append(input_api.os_path.join(directory, filename)) - to_run += 1 - input_api.logging.debug('Found %d files, running %d unit tests' - % (found, to_run)) - if not to_run: - return [ - output_api.PresubmitPromptWarning( - 'Out of %d files, found none that matched c=%r, s=%r in directory %s' - % (found, files_to_check, files_to_skip, directory)) - ] - return GetUnitTests(input_api, output_api, unit_tests, env) + to_run = found = 0 + for filename in input_api.os_listdir(test_path): + found += 1 + fullpath = input_api.os_path.join(test_path, filename) + if not input_api.os_path.isfile(fullpath): + continue + if files_to_check and not check(filename, files_to_check): + continue + if files_to_skip and check(filename, files_to_skip): + continue + unit_tests.append(input_api.os_path.join(directory, filename)) + to_run += 1 + input_api.logging.debug('Found %d files, running %d unit tests' % + (found, to_run)) + if not to_run: + return [ + output_api.PresubmitPromptWarning( + 'Out of %d files, found none that matched c=%r, s=%r in directory %s' + % (found, files_to_check, files_to_skip, directory)) + ] + return GetUnitTests(input_api, output_api, unit_tests, env) def GetUnitTests(input_api, @@ -941,45 +1003,45 @@ def GetUnitTests(input_api, run_on_python2=False, run_on_python3=True, skip_shebang_check=True): - """Runs all unit tests in a directory. + """Runs all unit tests in a directory. On Windows, sys.executable is used for unit tests ending with ".py". run_on_python2, run_on_python3, and skip_shebang_check are no longer used but have to be retained because of the many callers in other repos that pass them in. """ - del run_on_python2 - del run_on_python3 - del skip_shebang_check + del run_on_python2 + del run_on_python3 + del skip_shebang_check - # We don't want to hinder users from uploading incomplete patches, but we do - # want to report errors as errors when doing presubmit --all testing. - if input_api.is_committing or input_api.no_diffs: - message_type = output_api.PresubmitError - else: - message_type = output_api.PresubmitPromptWarning - - results = [] - for unit_test in unit_tests: - cmd = [unit_test] - if input_api.verbose: - cmd.append('--verbose') - kwargs = {'cwd': input_api.PresubmitLocalPath()} - if env: - kwargs['env'] = env - if not unit_test.endswith('.py'): - results.append(input_api.Command( - name=unit_test, - cmd=cmd, - kwargs=kwargs, - message=message_type)) + # We don't want to hinder users from uploading incomplete patches, but we do + # want to report errors as errors when doing presubmit --all testing. + if input_api.is_committing or input_api.no_diffs: + message_type = output_api.PresubmitError else: - results.append( - input_api.Command(name=unit_test, - cmd=cmd, - kwargs=kwargs, - message=message_type)) - return results + message_type = output_api.PresubmitPromptWarning + + results = [] + for unit_test in unit_tests: + cmd = [unit_test] + if input_api.verbose: + cmd.append('--verbose') + kwargs = {'cwd': input_api.PresubmitLocalPath()} + if env: + kwargs['env'] = env + if not unit_test.endswith('.py'): + results.append( + input_api.Command(name=unit_test, + cmd=cmd, + kwargs=kwargs, + message=message_type)) + else: + results.append( + input_api.Command(name=unit_test, + cmd=cmd, + kwargs=kwargs, + message=message_type)) + return results def GetUnitTestsRecursively(input_api, @@ -990,145 +1052,151 @@ def GetUnitTestsRecursively(input_api, run_on_python2=False, run_on_python3=True, skip_shebang_check=True): - """Gets all files in the directory tree (git repo) that match files_to_check. + """Gets all files in the directory tree (git repo) that match files_to_check. Restricts itself to only find files within the Change's source repo, not dependencies. run_on_python2, run_on_python3, and skip_shebang_check are no longer used but have to be retained because of the many callers in other repos that pass them in. """ - del run_on_python2 - del run_on_python3 - del skip_shebang_check + del run_on_python2 + del run_on_python3 + del skip_shebang_check - def check(filename): - return (any(input_api.re.match(f, filename) for f in files_to_check) and - not any(input_api.re.match(f, filename) for f in files_to_skip)) + def check(filename): + return (any(input_api.re.match(f, filename) for f in files_to_check) and + not any(input_api.re.match(f, filename) for f in files_to_skip)) - tests = [] + tests = [] - to_run = found = 0 - for filepath in input_api.change.AllFiles(directory): - found += 1 - if check(filepath): - to_run += 1 - tests.append(filepath) - input_api.logging.debug('Found %d files, running %d' % (found, to_run)) - if not to_run: - return [ - output_api.PresubmitPromptWarning( - 'Out of %d files, found none that matched c=%r, s=%r in directory %s' - % (found, files_to_check, files_to_skip, directory)) - ] + to_run = found = 0 + for filepath in input_api.change.AllFiles(directory): + found += 1 + if check(filepath): + to_run += 1 + tests.append(filepath) + input_api.logging.debug('Found %d files, running %d' % (found, to_run)) + if not to_run: + return [ + output_api.PresubmitPromptWarning( + 'Out of %d files, found none that matched c=%r, s=%r in directory %s' + % (found, files_to_check, files_to_skip, directory)) + ] - return GetUnitTests(input_api, output_api, tests) + return GetUnitTests(input_api, output_api, tests) def GetPythonUnitTests(input_api, output_api, unit_tests, python3=False): - """Run the unit tests out of process, capture the output and use the result + """Run the unit tests out of process, capture the output and use the result code to determine success. DEPRECATED. """ - # We don't want to hinder users from uploading incomplete patches. - if input_api.is_committing or input_api.no_diffs: - message_type = output_api.PresubmitError - else: - message_type = output_api.PresubmitNotifyResult - results = [] - for unit_test in unit_tests: - # Run the unit tests out of process. This is because some unit tests - # stub out base libraries and don't clean up their mess. It's too easy to - # get subtle bugs. - cwd = None - env = None - unit_test_name = unit_test - # 'python -m test.unit_test' doesn't work. We need to change to the right - # directory instead. - if '.' in unit_test: - # Tests imported in submodules (subdirectories) assume that the current - # directory is in the PYTHONPATH. Manually fix that. - unit_test = unit_test.replace('.', '/') - cwd = input_api.os_path.dirname(unit_test) - unit_test = input_api.os_path.basename(unit_test) - env = input_api.environ.copy() - # At least on Windows, it seems '.' must explicitly be in PYTHONPATH - backpath = [ - '.', input_api.os_path.pathsep.join(['..'] * (cwd.count('/') + 1)) - ] - # We convert to str, since on Windows on Python 2 only strings are allowed - # as environment variables, but literals are unicode since we're importing - # unicode_literals from __future__. - if env.get('PYTHONPATH'): - backpath.append(env.get('PYTHONPATH')) - env['PYTHONPATH'] = input_api.os_path.pathsep.join((backpath)) - env.pop('VPYTHON_CLEAR_PYTHONPATH', None) - cmd = [input_api.python3_executable, '-m', '%s' % unit_test] - results.append(input_api.Command( - name=unit_test_name, - cmd=cmd, - kwargs={'env': env, 'cwd': cwd}, - message=message_type)) - return results + # We don't want to hinder users from uploading incomplete patches. + if input_api.is_committing or input_api.no_diffs: + message_type = output_api.PresubmitError + else: + message_type = output_api.PresubmitNotifyResult + results = [] + for unit_test in unit_tests: + # Run the unit tests out of process. This is because some unit tests + # stub out base libraries and don't clean up their mess. It's too easy + # to get subtle bugs. + cwd = None + env = None + unit_test_name = unit_test + # 'python -m test.unit_test' doesn't work. We need to change to the + # right directory instead. + if '.' in unit_test: + # Tests imported in submodules (subdirectories) assume that the + # current directory is in the PYTHONPATH. Manually fix that. + unit_test = unit_test.replace('.', '/') + cwd = input_api.os_path.dirname(unit_test) + unit_test = input_api.os_path.basename(unit_test) + env = input_api.environ.copy() + # At least on Windows, it seems '.' must explicitly be in PYTHONPATH + backpath = [ + '.', + input_api.os_path.pathsep.join(['..'] * (cwd.count('/') + 1)) + ] + # We convert to str, since on Windows on Python 2 only strings are + # allowed as environment variables, but literals are unicode since + # we're importing unicode_literals from __future__. + if env.get('PYTHONPATH'): + backpath.append(env.get('PYTHONPATH')) + env['PYTHONPATH'] = input_api.os_path.pathsep.join((backpath)) + env.pop('VPYTHON_CLEAR_PYTHONPATH', None) + cmd = [input_api.python3_executable, '-m', '%s' % unit_test] + results.append( + input_api.Command(name=unit_test_name, + cmd=cmd, + kwargs={ + 'env': env, + 'cwd': cwd + }, + message=message_type)) + return results def RunUnitTestsInDirectory(input_api, *args, **kwargs): - """Run tests in a directory serially. + """Run tests in a directory serially. For better performance, use GetUnitTestsInDirectory and then pass to input_api.RunTests. """ - return input_api.RunTests( - GetUnitTestsInDirectory(input_api, *args, **kwargs), False) + return input_api.RunTests( + GetUnitTestsInDirectory(input_api, *args, **kwargs), False) def RunUnitTests(input_api, *args, **kwargs): - """Run tests serially. + """Run tests serially. For better performance, use GetUnitTests and then pass to input_api.RunTests. """ - return input_api.RunTests(GetUnitTests(input_api, *args, **kwargs), False) + return input_api.RunTests(GetUnitTests(input_api, *args, **kwargs), False) def RunPythonUnitTests(input_api, *args, **kwargs): - """Run python tests in a directory serially. + """Run python tests in a directory serially. DEPRECATED """ - return input_api.RunTests( - GetPythonUnitTests(input_api, *args, **kwargs), False) + return input_api.RunTests(GetPythonUnitTests(input_api, *args, **kwargs), + False) def _FetchAllFiles(input_api, files_to_check, files_to_skip): - """Hack to fetch all files.""" - # We cannot use AffectedFiles here because we want to test every python - # file on each single python change. It's because a change in a python file - # can break another unmodified file. - # Use code similar to InputApi.FilterSourceFile() - def Find(filepath, filters): - if input_api.platform == 'win32': - filepath = filepath.replace('\\', '/') + """Hack to fetch all files.""" - for item in filters: - if input_api.re.match(item, filepath): - return True - return False + # We cannot use AffectedFiles here because we want to test every python + # file on each single python change. It's because a change in a python file + # can break another unmodified file. + # Use code similar to InputApi.FilterSourceFile() + def Find(filepath, filters): + if input_api.platform == 'win32': + filepath = filepath.replace('\\', '/') - files = [] - path_len = len(input_api.PresubmitLocalPath()) - for dirpath, dirnames, filenames in input_api.os_walk( - input_api.PresubmitLocalPath()): - # Passes dirnames in block list to speed up search. - for item in dirnames[:]: - filepath = input_api.os_path.join(dirpath, item)[path_len + 1:] - if Find(filepath, files_to_skip): - dirnames.remove(item) - for item in filenames: - filepath = input_api.os_path.join(dirpath, item)[path_len + 1:] - if Find(filepath, files_to_check) and not Find(filepath, files_to_skip): - files.append(filepath) - return files + for item in filters: + if input_api.re.match(item, filepath): + return True + return False + + files = [] + path_len = len(input_api.PresubmitLocalPath()) + for dirpath, dirnames, filenames in input_api.os_walk( + input_api.PresubmitLocalPath()): + # Passes dirnames in block list to speed up search. + for item in dirnames[:]: + filepath = input_api.os_path.join(dirpath, item)[path_len + 1:] + if Find(filepath, files_to_skip): + dirnames.remove(item) + for item in filenames: + filepath = input_api.os_path.join(dirpath, item)[path_len + 1:] + if Find(filepath, + files_to_check) and not Find(filepath, files_to_skip): + files.append(filepath) + return files def GetPylint(input_api, @@ -1139,325 +1207,342 @@ def GetPylint(input_api, extra_paths_list=None, pylintrc=None, version='2.7'): - """Run pylint on python files. + """Run pylint on python files. The default files_to_check enforces looking only at *.py files. Currently only pylint version '2.6' and '2.7' are supported. """ - files_to_check = tuple(files_to_check or (r'.*\.py$', )) - files_to_skip = tuple(files_to_skip or input_api.DEFAULT_FILES_TO_SKIP) - extra_paths_list = extra_paths_list or [] + files_to_check = tuple(files_to_check or (r'.*\.py$', )) + files_to_skip = tuple(files_to_skip or input_api.DEFAULT_FILES_TO_SKIP) + extra_paths_list = extra_paths_list or [] - assert version in ('2.6', '2.7'), 'Unsupported pylint version: %s' % version + assert version in ('2.6', '2.7'), 'Unsupported pylint version: %s' % version - if input_api.is_committing or input_api.no_diffs: - error_type = output_api.PresubmitError - else: - error_type = output_api.PresubmitPromptWarning - - # Only trigger if there is at least one python file affected. - def rel_path(regex): - """Modifies a regex for a subject to accept paths relative to root.""" - def samefile(a, b): - # Default implementation for platforms lacking os.path.samefile - # (like Windows). - return input_api.os_path.abspath(a) == input_api.os_path.abspath(b) - samefile = getattr(input_api.os_path, 'samefile', samefile) - if samefile(input_api.PresubmitLocalPath(), - input_api.change.RepositoryRoot()): - return regex - - prefix = input_api.os_path.join(input_api.os_path.relpath( - input_api.PresubmitLocalPath(), input_api.change.RepositoryRoot()), '') - return input_api.re.escape(prefix) + regex - src_filter = lambda x: input_api.FilterSourceFile( - x, map(rel_path, files_to_check), map(rel_path, files_to_skip)) - if not input_api.AffectedSourceFiles(src_filter): - input_api.logging.info('Skipping pylint: no matching changes.') - return [] - - if pylintrc is not None: - pylintrc = input_api.os_path.join(input_api.PresubmitLocalPath(), pylintrc) - else: - pylintrc = input_api.os_path.join(_HERE, 'pylintrc') - extra_args = ['--rcfile=%s' % pylintrc] - if disabled_warnings: - extra_args.extend(['-d', ','.join(disabled_warnings)]) - - files = _FetchAllFiles(input_api, files_to_check, files_to_skip) - if not files: - return [] - files.sort() - - input_api.logging.info('Running pylint %s on %d files', version, len(files)) - input_api.logging.debug('Running pylint on: %s', files) - env = input_api.environ.copy() - env['PYTHONPATH'] = input_api.os_path.pathsep.join(extra_paths_list) - env.pop('VPYTHON_CLEAR_PYTHONPATH', None) - input_api.logging.debug(' with extra PYTHONPATH: %r', extra_paths_list) - files_per_job = 10 - - def GetPylintCmd(flist, extra, parallel): - # Windows needs help running python files so we explicitly specify - # the interpreter to use. It also has limitations on the size of - # the command-line, so we pass arguments via a pipe. - tool = input_api.os_path.join(_HERE, 'pylint-' + version) - kwargs = {'env': env} - if input_api.platform == 'win32': - # On Windows, scripts on the current directory take precedence over PATH. - # When `pylint.bat` calls `vpython`, it will execute the `vpython` of the - # depot_tools under test instead of the one in the bot. - # As a workaround, we run the tests from the parent directory instead. - cwd = input_api.change.RepositoryRoot() - if input_api.os_path.basename(cwd) == 'depot_tools': - kwargs['cwd'] = input_api.os_path.dirname(cwd) - flist = [input_api.os_path.join('depot_tools', f) for f in flist] - tool += '.bat' - - cmd = [tool, '--args-on-stdin'] - if len(flist) == 1: - description = flist[0] + if input_api.is_committing or input_api.no_diffs: + error_type = output_api.PresubmitError else: - description = '%s files' % len(flist) + error_type = output_api.PresubmitPromptWarning - args = extra_args[:] - if extra: - args.extend(extra) - description += ' using %s' % (extra,) - if parallel: - # Make sure we don't request more parallelism than is justified for the - # number of files we have to process. PyLint child-process startup time is - # significant. - jobs = min(input_api.cpu_count, 1 + len(flist) // files_per_job) - if jobs > 1: - args.append('--jobs=%s' % jobs) - description += ' on %d processes' % jobs + # Only trigger if there is at least one python file affected. + def rel_path(regex): + """Modifies a regex for a subject to accept paths relative to root.""" + def samefile(a, b): + # Default implementation for platforms lacking os.path.samefile + # (like Windows). + return input_api.os_path.abspath(a) == input_api.os_path.abspath(b) - kwargs['stdin'] = '\n'.join(args + flist).encode('utf-8') + samefile = getattr(input_api.os_path, 'samefile', samefile) + if samefile(input_api.PresubmitLocalPath(), + input_api.change.RepositoryRoot()): + return regex - return input_api.Command(name='Pylint (%s)' % description, - cmd=cmd, - kwargs=kwargs, - message=error_type, - python3=True) + prefix = input_api.os_path.join( + input_api.os_path.relpath(input_api.PresubmitLocalPath(), + input_api.change.RepositoryRoot()), '') + return input_api.re.escape(prefix) + regex + + src_filter = lambda x: input_api.FilterSourceFile( + x, map(rel_path, files_to_check), map(rel_path, files_to_skip)) + if not input_api.AffectedSourceFiles(src_filter): + input_api.logging.info('Skipping pylint: no matching changes.') + return [] + + if pylintrc is not None: + pylintrc = input_api.os_path.join(input_api.PresubmitLocalPath(), + pylintrc) + else: + pylintrc = input_api.os_path.join(_HERE, 'pylintrc') + extra_args = ['--rcfile=%s' % pylintrc] + if disabled_warnings: + extra_args.extend(['-d', ','.join(disabled_warnings)]) + + files = _FetchAllFiles(input_api, files_to_check, files_to_skip) + if not files: + return [] + files.sort() + + input_api.logging.info('Running pylint %s on %d files', version, len(files)) + input_api.logging.debug('Running pylint on: %s', files) + env = input_api.environ.copy() + env['PYTHONPATH'] = input_api.os_path.pathsep.join(extra_paths_list) + env.pop('VPYTHON_CLEAR_PYTHONPATH', None) + input_api.logging.debug(' with extra PYTHONPATH: %r', extra_paths_list) + files_per_job = 10 + + def GetPylintCmd(flist, extra, parallel): + # Windows needs help running python files so we explicitly specify + # the interpreter to use. It also has limitations on the size of + # the command-line, so we pass arguments via a pipe. + tool = input_api.os_path.join(_HERE, 'pylint-' + version) + kwargs = {'env': env} + if input_api.platform == 'win32': + # On Windows, scripts on the current directory take precedence over + # PATH. When `pylint.bat` calls `vpython`, it will execute the + # `vpython` of the depot_tools under test instead of the one in the + # bot. As a workaround, we run the tests from the parent directory + # instead. + cwd = input_api.change.RepositoryRoot() + if input_api.os_path.basename(cwd) == 'depot_tools': + kwargs['cwd'] = input_api.os_path.dirname(cwd) + flist = [ + input_api.os_path.join('depot_tools', f) for f in flist + ] + tool += '.bat' + + cmd = [tool, '--args-on-stdin'] + if len(flist) == 1: + description = flist[0] + else: + description = '%s files' % len(flist) + + args = extra_args[:] + if extra: + args.extend(extra) + description += ' using %s' % (extra, ) + if parallel: + # Make sure we don't request more parallelism than is justified for + # the number of files we have to process. PyLint child-process + # startup time is significant. + jobs = min(input_api.cpu_count, 1 + len(flist) // files_per_job) + if jobs > 1: + args.append('--jobs=%s' % jobs) + description += ' on %d processes' % jobs + + kwargs['stdin'] = '\n'.join(args + flist).encode('utf-8') + + return input_api.Command(name='Pylint (%s)' % description, + cmd=cmd, + kwargs=kwargs, + message=error_type, + python3=True) + + # pylint's cycle detection doesn't work in parallel, so spawn a second, + # single-threaded job for just that check. However, only do this if there + # are actually enough files to process to justify parallelism in the first + # place. Some PRESUBMITs explicitly mention cycle detection. + if len(files) >= files_per_job and not any( + 'R0401' in a or 'cyclic-import' in a for a in extra_args): + return [ + GetPylintCmd(files, ["--disable=cyclic-import"], True), + GetPylintCmd(files, ["--disable=all", "--enable=cyclic-import"], + False), + ] - # pylint's cycle detection doesn't work in parallel, so spawn a second, - # single-threaded job for just that check. However, only do this if there are - # actually enough files to process to justify parallelism in the first place. - # Some PRESUBMITs explicitly mention cycle detection. - if len(files) >= files_per_job and not any( - 'R0401' in a or 'cyclic-import' in a for a in extra_args): return [ - GetPylintCmd(files, ["--disable=cyclic-import"], True), - GetPylintCmd(files, ["--disable=all", "--enable=cyclic-import"], False), + GetPylintCmd(files, [], True), ] - return [ - GetPylintCmd(files, [], True), - ] - def RunPylint(input_api, *args, **kwargs): - """Legacy presubmit function. + """Legacy presubmit function. For better performance, get all tests and then pass to input_api.RunTests. """ - return input_api.RunTests(GetPylint(input_api, *args, **kwargs), False) + return input_api.RunTests(GetPylint(input_api, *args, **kwargs), False) def CheckDirMetadataFormat(input_api, output_api, dirmd_bin=None): - # TODO(crbug.com/1102997): Remove OWNERS once DIR_METADATA migration is - # complete. - file_filter = lambda f: ( - input_api.basename(f.LocalPath()) in ('DIR_METADATA', 'OWNERS')) - affected_files = { - f.AbsoluteLocalPath() - for f in input_api.change.AffectedFiles( - include_deletes=False, file_filter=file_filter) - } - if not affected_files: - return [] + # TODO(crbug.com/1102997): Remove OWNERS once DIR_METADATA migration is + # complete. + file_filter = lambda f: (input_api.basename(f.LocalPath()) in + ('DIR_METADATA', 'OWNERS')) + affected_files = { + f.AbsoluteLocalPath() + for f in input_api.change.AffectedFiles(include_deletes=False, + file_filter=file_filter) + } + if not affected_files: + return [] - name = 'Validate metadata in OWNERS and DIR_METADATA files' + name = 'Validate metadata in OWNERS and DIR_METADATA files' - if dirmd_bin is None: - dirmd_bin = 'dirmd.bat' if input_api.is_windows else 'dirmd' + if dirmd_bin is None: + dirmd_bin = 'dirmd.bat' if input_api.is_windows else 'dirmd' - # When running git cl presubmit --all this presubmit may be asked to check - # ~7,500 files, leading to a command line that is about 500,000 characters. - # This goes past the Windows 8191 character cmd.exe limit and causes cryptic - # failures. To avoid these we break the command up into smaller pieces. The - # non-Windows limit is chosen so that the code that splits up commands will - # get some exercise on other platforms. - # Depending on how long the command is on Windows the error may be: - # The command line is too long. - # Or it may be: - # OSError: Execution failed with error: [WinError 206] The filename or - # extension is too long. - # I suspect that the latter error comes from CreateProcess hitting its 32768 - # character limit. - files_per_command = 50 if input_api.is_windows else 1000 - affected_files = sorted(affected_files) - results = [] - for i in range(0, len(affected_files), files_per_command): - kwargs = {} - cmd = [dirmd_bin, 'validate'] + affected_files[i : i + files_per_command] - results.extend([input_api.Command( - name, cmd, kwargs, output_api.PresubmitError)]) - return results + # When running git cl presubmit --all this presubmit may be asked to check + # ~7,500 files, leading to a command line that is about 500,000 characters. + # This goes past the Windows 8191 character cmd.exe limit and causes cryptic + # failures. To avoid these we break the command up into smaller pieces. The + # non-Windows limit is chosen so that the code that splits up commands will + # get some exercise on other platforms. + # Depending on how long the command is on Windows the error may be: + # The command line is too long. + # Or it may be: + # OSError: Execution failed with error: [WinError 206] The filename or + # extension is too long. + # I suspect that the latter error comes from CreateProcess hitting its 32768 + # character limit. + files_per_command = 50 if input_api.is_windows else 1000 + affected_files = sorted(affected_files) + results = [] + for i in range(0, len(affected_files), files_per_command): + kwargs = {} + cmd = [dirmd_bin, 'validate'] + affected_files[i:i + files_per_command] + results.extend( + [input_api.Command(name, cmd, kwargs, output_api.PresubmitError)]) + return results def CheckNoNewMetadataInOwners(input_api, output_api): - """Check that no metadata is added to OWNERS files.""" - if input_api.no_diffs: - return [] + """Check that no metadata is added to OWNERS files.""" + if input_api.no_diffs: + return [] - _METADATA_LINE_RE = input_api.re.compile( - r'^#\s*(TEAM|COMPONENT|OS|WPT-NOTIFY)+\s*:\s*\S+$', - input_api.re.MULTILINE | input_api.re.IGNORECASE) - affected_files = input_api.change.AffectedFiles( - include_deletes=False, - file_filter=lambda f: input_api.basename(f.LocalPath()) == 'OWNERS') + _METADATA_LINE_RE = input_api.re.compile( + r'^#\s*(TEAM|COMPONENT|OS|WPT-NOTIFY)+\s*:\s*\S+$', + input_api.re.MULTILINE | input_api.re.IGNORECASE) + affected_files = input_api.change.AffectedFiles( + include_deletes=False, + file_filter=lambda f: input_api.basename(f.LocalPath()) == 'OWNERS') - errors = [] - for f in affected_files: - for _, line in f.ChangedContents(): - if _METADATA_LINE_RE.search(line): - errors.append(f.AbsoluteLocalPath()) - break + errors = [] + for f in affected_files: + for _, line in f.ChangedContents(): + if _METADATA_LINE_RE.search(line): + errors.append(f.AbsoluteLocalPath()) + break - if not errors: - return [] + if not errors: + return [] - return [output_api.PresubmitError( - 'New metadata was added to the following OWNERS files, but should ' - 'have been added to DIR_METADATA files instead:\n' + - '\n'.join(errors) + '\n' + - 'See https://source.chromium.org/chromium/infra/infra/+/HEAD:' - 'go/src/infra/tools/dirmd/proto/dir_metadata.proto for details.')] + return [ + output_api.PresubmitError( + 'New metadata was added to the following OWNERS files, but should ' + 'have been added to DIR_METADATA files instead:\n' + + '\n'.join(errors) + '\n' + + 'See https://source.chromium.org/chromium/infra/infra/+/HEAD:' + 'go/src/infra/tools/dirmd/proto/dir_metadata.proto for details.') + ] def CheckOwnersDirMetadataExclusive(input_api, output_api): - """Check that metadata in OWNERS files and DIR_METADATA files are mutually + """Check that metadata in OWNERS files and DIR_METADATA files are mutually exclusive. """ - _METADATA_LINE_RE = input_api.re.compile( - r'^#\s*(TEAM|COMPONENT|OS|WPT-NOTIFY)+\s*:\s*\S+$', - input_api.re.MULTILINE) - file_filter = ( - lambda f: input_api.basename(f.LocalPath()) in ('OWNERS', 'DIR_METADATA')) - affected_dirs = { - input_api.os_path.dirname(f.AbsoluteLocalPath()) - for f in input_api.change.AffectedFiles( - include_deletes=False, file_filter=file_filter) - } + _METADATA_LINE_RE = input_api.re.compile( + r'^#\s*(TEAM|COMPONENT|OS|WPT-NOTIFY)+\s*:\s*\S+$', + input_api.re.MULTILINE) + file_filter = (lambda f: input_api.basename(f.LocalPath()) in + ('OWNERS', 'DIR_METADATA')) + affected_dirs = { + input_api.os_path.dirname(f.AbsoluteLocalPath()) + for f in input_api.change.AffectedFiles(include_deletes=False, + file_filter=file_filter) + } - errors = [] - for path in affected_dirs: - owners_path = input_api.os_path.join(path, 'OWNERS') - dir_metadata_path = input_api.os_path.join(path, 'DIR_METADATA') - if (not input_api.os_path.isfile(dir_metadata_path) - or not input_api.os_path.isfile(owners_path)): - continue - if _METADATA_LINE_RE.search(input_api.ReadFile(owners_path)): - errors.append(owners_path) + errors = [] + for path in affected_dirs: + owners_path = input_api.os_path.join(path, 'OWNERS') + dir_metadata_path = input_api.os_path.join(path, 'DIR_METADATA') + if (not input_api.os_path.isfile(dir_metadata_path) + or not input_api.os_path.isfile(owners_path)): + continue + if _METADATA_LINE_RE.search(input_api.ReadFile(owners_path)): + errors.append(owners_path) - if not errors: - return [] + if not errors: + return [] - return [output_api.PresubmitError( - 'The following OWNERS files should contain no metadata, as there is a ' - 'DIR_METADATA file present in the same directory:\n' - + '\n'.join(errors))] + return [ + output_api.PresubmitError( + 'The following OWNERS files should contain no metadata, as there is a ' + 'DIR_METADATA file present in the same directory:\n' + + '\n'.join(errors)) + ] def CheckOwnersFormat(input_api, output_api): - if input_api.gerrit and input_api.gerrit.IsCodeOwnersEnabledOnRepo(): - return [] + if input_api.gerrit and input_api.gerrit.IsCodeOwnersEnabledOnRepo(): + return [] - return [ - output_api.PresubmitError( - 'code-owners is not enabled. Ask your host enable it on your gerrit ' - 'host. Read more about code-owners at ' - 'https://chromium-review.googlesource.com/' - 'plugins/code-owners/Documentation/index.html.') - ] + return [ + output_api.PresubmitError( + 'code-owners is not enabled. Ask your host enable it on your gerrit ' + 'host. Read more about code-owners at ' + 'https://chromium-review.googlesource.com/' + 'plugins/code-owners/Documentation/index.html.') + ] -def CheckOwners( - input_api, output_api, source_file_filter=None, allow_tbr=True): - # Skip OWNERS check when Owners-Override label is approved. This is intended - # for global owners, trusted bots, and on-call sheriffs. Review is still - # required for these changes. - if (input_api.change.issue - and input_api.gerrit.IsOwnersOverrideApproved(input_api.change.issue)): - return [] +def CheckOwners(input_api, output_api, source_file_filter=None, allow_tbr=True): + # Skip OWNERS check when Owners-Override label is approved. This is intended + # for global owners, trusted bots, and on-call sheriffs. Review is still + # required for these changes. + if (input_api.change.issue and input_api.gerrit.IsOwnersOverrideApproved( + input_api.change.issue)): + return [] - if input_api.gerrit and input_api.gerrit.IsCodeOwnersEnabledOnRepo(): - return [] + if input_api.gerrit and input_api.gerrit.IsCodeOwnersEnabledOnRepo(): + return [] - return [ - output_api.PresubmitError( - 'code-owners is not enabled. Ask your host enable it on your gerrit ' - 'host. Read more about code-owners at ' - 'https://chromium-review.googlesource.com/' - 'plugins/code-owners/Documentation/index.html.') - ] + return [ + output_api.PresubmitError( + 'code-owners is not enabled. Ask your host enable it on your gerrit ' + 'host. Read more about code-owners at ' + 'https://chromium-review.googlesource.com/' + 'plugins/code-owners/Documentation/index.html.') + ] -def GetCodereviewOwnerAndReviewers( - input_api, _email_regexp=None, approval_needed=True): - """Return the owner and reviewers of a change, if any. +def GetCodereviewOwnerAndReviewers(input_api, + _email_regexp=None, + approval_needed=True): + """Return the owner and reviewers of a change, if any. If approval_needed is True, only reviewers who have approved the change will be returned. """ - # Recognizes 'X@Y' email addresses. Very simplistic. - EMAIL_REGEXP = input_api.re.compile(r'^[\w\-\+\%\.]+\@[\w\-\+\%\.]+$') - issue = input_api.change.issue - if not issue: - return None, (set() if approval_needed else - _ReviewersFromChange(input_api.change)) + # Recognizes 'X@Y' email addresses. Very simplistic. + EMAIL_REGEXP = input_api.re.compile(r'^[\w\-\+\%\.]+\@[\w\-\+\%\.]+$') + issue = input_api.change.issue + if not issue: + return None, (set() if approval_needed else _ReviewersFromChange( + input_api.change)) - owner_email = input_api.gerrit.GetChangeOwner(issue) - reviewers = set( - r for r in input_api.gerrit.GetChangeReviewers(issue, approval_needed) - if _match_reviewer_email(r, owner_email, EMAIL_REGEXP)) - input_api.logging.debug('owner: %s; approvals given by: %s', - owner_email, ', '.join(sorted(reviewers))) - return owner_email, reviewers + owner_email = input_api.gerrit.GetChangeOwner(issue) + reviewers = set( + r for r in input_api.gerrit.GetChangeReviewers(issue, approval_needed) + if _match_reviewer_email(r, owner_email, EMAIL_REGEXP)) + input_api.logging.debug('owner: %s; approvals given by: %s', owner_email, + ', '.join(sorted(reviewers))) + return owner_email, reviewers def _ReviewersFromChange(change): - """Return the reviewers specified in the |change|, if any.""" - reviewers = set() - reviewers.update(change.ReviewersFromDescription()) - reviewers.update(change.TBRsFromDescription()) + """Return the reviewers specified in the |change|, if any.""" + reviewers = set() + reviewers.update(change.ReviewersFromDescription()) + reviewers.update(change.TBRsFromDescription()) - # Drop reviewers that aren't specified in email address format. - return set(reviewer for reviewer in reviewers if '@' in reviewer) + # Drop reviewers that aren't specified in email address format. + return set(reviewer for reviewer in reviewers if '@' in reviewer) def _match_reviewer_email(r, owner_email, email_regexp): - return email_regexp.match(r) and r != owner_email + return email_regexp.match(r) and r != owner_email def CheckSingletonInHeaders(input_api, output_api, source_file_filter=None): - """Deprecated, must be removed.""" - return [ - output_api.PresubmitNotifyResult( - 'CheckSingletonInHeaders is deprecated, please remove it.') - ] + """Deprecated, must be removed.""" + return [ + output_api.PresubmitNotifyResult( + 'CheckSingletonInHeaders is deprecated, please remove it.') + ] -def PanProjectChecks(input_api, output_api, - excluded_paths=None, text_files=None, - license_header=None, project_name=None, - owners_check=True, maxlen=80, global_checks=True): - """Checks that ALL chromium orbit projects should use. +def PanProjectChecks(input_api, + output_api, + excluded_paths=None, + text_files=None, + license_header=None, + project_name=None, + owners_check=True, + maxlen=80, + global_checks=True): + """Checks that ALL chromium orbit projects should use. These are checks to be run on all Chromium orbit project, including: Chromium @@ -1479,94 +1564,111 @@ def PanProjectChecks(input_api, output_api, Returns: A list of warning or error objects. """ - excluded_paths = tuple(excluded_paths or []) - text_files = tuple(text_files or ( - r'.+\.txt$', - r'.+\.json$', - )) + excluded_paths = tuple(excluded_paths or []) + text_files = tuple(text_files or ( + r'.+\.txt$', + r'.+\.json$', + )) - results = [] - # This code loads the default skip list (e.g. third_party, experimental, etc) - # and add our skip list (breakpad, skia and v8 are still not following - # google style and are not really living this repository). - # See presubmit_support.py InputApi.FilterSourceFile for the (simple) usage. - files_to_skip = input_api.DEFAULT_FILES_TO_SKIP + excluded_paths - files_to_check = input_api.DEFAULT_FILES_TO_CHECK + text_files - sources = lambda x: input_api.FilterSourceFile(x, files_to_skip=files_to_skip) - text_files = lambda x: input_api.FilterSourceFile( - x, files_to_skip=files_to_skip, files_to_check=files_to_check) + results = [] + # This code loads the default skip list (e.g. third_party, experimental, + # etc) and add our skip list (breakpad, skia and v8 are still not following + # google style and are not really living this repository). See + # presubmit_support.py InputApi.FilterSourceFile for the (simple) usage. + files_to_skip = input_api.DEFAULT_FILES_TO_SKIP + excluded_paths + files_to_check = input_api.DEFAULT_FILES_TO_CHECK + text_files + sources = lambda x: input_api.FilterSourceFile(x, + files_to_skip=files_to_skip) + text_files = lambda x: input_api.FilterSourceFile( + x, files_to_skip=files_to_skip, files_to_check=files_to_check) - snapshot_memory = [] - def snapshot(msg): - """Measures & prints performance warning if a rule is running slow.""" - dt2 = input_api.time.time() - if snapshot_memory: - delta_s = dt2 - snapshot_memory[0] - if delta_s > 0.5: - print(" %s took a long time: %.1fs" % (snapshot_memory[1], delta_s)) - snapshot_memory[:] = (dt2, msg) + snapshot_memory = [] - snapshot("checking owners files format") - try: - if not 'PRESUBMIT_SKIP_NETWORK' in _os.environ and owners_check: - snapshot("checking owners") - results.extend( - input_api.canned_checks.CheckOwnersFormat(input_api, output_api)) - results.extend( - input_api.canned_checks.CheckOwners(input_api, - output_api, - source_file_filter=None)) - except Exception as e: - print('Failed to check owners - %s' % str(e)) + def snapshot(msg): + """Measures & prints performance warning if a rule is running slow.""" + dt2 = input_api.time.time() + if snapshot_memory: + delta_s = dt2 - snapshot_memory[0] + if delta_s > 0.5: + print(" %s took a long time: %.1fs" % + (snapshot_memory[1], delta_s)) + snapshot_memory[:] = (dt2, msg) - snapshot("checking long lines") - results.extend(input_api.canned_checks.CheckLongLines( - input_api, output_api, maxlen, source_file_filter=sources)) - snapshot( "checking tabs") - results.extend(input_api.canned_checks.CheckChangeHasNoTabs( - input_api, output_api, source_file_filter=sources)) - snapshot( "checking stray whitespace") - results.extend(input_api.canned_checks.CheckChangeHasNoStrayWhitespace( - input_api, output_api, source_file_filter=sources)) - snapshot("checking license") - results.extend(input_api.canned_checks.CheckLicense( - input_api, output_api, license_header, project_name, - source_file_filter=sources)) - snapshot("checking corp links in files") - results.extend( - input_api.canned_checks.CheckCorpLinksInFiles(input_api, + snapshot("checking owners files format") + try: + if not 'PRESUBMIT_SKIP_NETWORK' in _os.environ and owners_check: + snapshot("checking owners") + results.extend( + input_api.canned_checks.CheckOwnersFormat( + input_api, output_api)) + results.extend( + input_api.canned_checks.CheckOwners(input_api, output_api, - source_file_filter=sources)) + source_file_filter=None)) + except Exception as e: + print('Failed to check owners - %s' % str(e)) + + snapshot("checking long lines") + results.extend( + input_api.canned_checks.CheckLongLines(input_api, + output_api, + maxlen, + source_file_filter=sources)) + snapshot("checking tabs") + results.extend( + input_api.canned_checks.CheckChangeHasNoTabs( + input_api, output_api, source_file_filter=sources)) + snapshot("checking stray whitespace") + results.extend( + input_api.canned_checks.CheckChangeHasNoStrayWhitespace( + input_api, output_api, source_file_filter=sources)) + snapshot("checking license") + results.extend( + input_api.canned_checks.CheckLicense(input_api, + output_api, + license_header, + project_name, + source_file_filter=sources)) + snapshot("checking corp links in files") + results.extend( + input_api.canned_checks.CheckCorpLinksInFiles( + input_api, output_api, source_file_filter=sources)) + + if input_api.is_committing: + if global_checks: + # These changes verify state that is global to the tree and can + # therefore be skipped when run from PRESUBMIT.py scripts deeper in + # the tree. Skipping these saves a bit of time and avoids having + # redundant output. This was initially designed for use by + # third_party/blink/PRESUBMIT.py. + snapshot("checking was uploaded") + results.extend( + input_api.canned_checks.CheckChangeWasUploaded( + input_api, output_api)) + snapshot("checking description") + results.extend( + input_api.canned_checks.CheckChangeHasDescription( + input_api, output_api)) + results.extend( + input_api.canned_checks.CheckDoNotSubmitInDescription( + input_api, output_api)) + results.extend( + input_api.canned_checks.CheckCorpLinksInDescription( + input_api, output_api)) + snapshot("checking do not submit in files") + results.extend( + input_api.canned_checks.CheckDoNotSubmitInFiles( + input_api, output_api)) - if input_api.is_committing: if global_checks: - # These changes verify state that is global to the tree and can therefore - # be skipped when run from PRESUBMIT.py scripts deeper in the tree. - # Skipping these saves a bit of time and avoids having redundant output. - # This was initially designed for use by third_party/blink/PRESUBMIT.py. - snapshot("checking was uploaded") - results.extend(input_api.canned_checks.CheckChangeWasUploaded( - input_api, output_api)) - snapshot("checking description") - results.extend(input_api.canned_checks.CheckChangeHasDescription( - input_api, output_api)) - results.extend(input_api.canned_checks.CheckDoNotSubmitInDescription( - input_api, output_api)) - results.extend( - input_api.canned_checks.CheckCorpLinksInDescription( - input_api, output_api)) - snapshot("checking do not submit in files") - results.extend(input_api.canned_checks.CheckDoNotSubmitInFiles( - input_api, output_api)) + if input_api.change.scm == 'git': + snapshot("checking for commit objects in tree") + results.extend( + input_api.canned_checks.CheckForCommitObjects( + input_api, output_api)) - if global_checks: - if input_api.change.scm == 'git': - snapshot("checking for commit objects in tree") - results.extend( - input_api.canned_checks.CheckForCommitObjects(input_api, output_api)) - - snapshot("done") - return results + snapshot("done") + return results def CheckPatchFormatted(input_api, @@ -1576,81 +1678,85 @@ def CheckPatchFormatted(input_api, check_js=False, check_python=None, result_factory=None): - result_factory = result_factory or output_api.PresubmitPromptWarning - import git_cl + result_factory = result_factory or output_api.PresubmitPromptWarning + import git_cl - display_args = [] - if not check_clang_format: - display_args.append('--no-clang-format') + display_args = [] + if not check_clang_format: + display_args.append('--no-clang-format') - if check_js: - display_args.append('--js') + if check_js: + display_args.append('--js') - # Explicitly setting check_python to will enable/disable python formatting - # on all files. Leaving it as None will enable checking patch formatting - # on files that have a .style.yapf file in a parent directory. - if check_python is not None: - if check_python: - display_args.append('--python') - else: - display_args.append('--no-python') + # Explicitly setting check_python to will enable/disable python formatting + # on all files. Leaving it as None will enable checking patch formatting + # on files that have a .style.yapf file in a parent directory. + if check_python is not None: + if check_python: + display_args.append('--python') + else: + display_args.append('--no-python') - cmd = ['-C', input_api.change.RepositoryRoot(), - 'cl', 'format', '--dry-run', '--presubmit'] + display_args + cmd = [ + '-C', + input_api.change.RepositoryRoot(), 'cl', 'format', '--dry-run', + '--presubmit' + ] + display_args - # Make sure the passed --upstream branch is applied to a dry run. - if input_api.change.UpstreamBranch(): - cmd.extend(['--upstream', input_api.change.UpstreamBranch()]) + # Make sure the passed --upstream branch is applied to a dry run. + if input_api.change.UpstreamBranch(): + cmd.extend(['--upstream', input_api.change.UpstreamBranch()]) - presubmit_subdir = input_api.os_path.relpath( - input_api.PresubmitLocalPath(), input_api.change.RepositoryRoot()) - if presubmit_subdir.startswith('..') or presubmit_subdir == '.': - presubmit_subdir = '' - # If the PRESUBMIT.py is in a parent repository, then format the entire - # subrepository. Otherwise, format only the code in the directory that - # contains the PRESUBMIT.py. - if presubmit_subdir: - cmd.append(input_api.PresubmitLocalPath()) - - code, _ = git_cl.RunGitWithCode(cmd, suppress_stderr=bypass_warnings) - # bypass_warnings? Only fail with code 2. - # As this is just a warning, ignore all other errors if the user - # happens to have a broken clang-format, doesn't use git, etc etc. - if code == 2 or (code and not bypass_warnings): + presubmit_subdir = input_api.os_path.relpath( + input_api.PresubmitLocalPath(), input_api.change.RepositoryRoot()) + if presubmit_subdir.startswith('..') or presubmit_subdir == '.': + presubmit_subdir = '' + # If the PRESUBMIT.py is in a parent repository, then format the entire + # subrepository. Otherwise, format only the code in the directory that + # contains the PRESUBMIT.py. if presubmit_subdir: - short_path = presubmit_subdir - else: - short_path = input_api.basename(input_api.change.RepositoryRoot()) - display_args.append(presubmit_subdir) - return [result_factory( - 'The %s directory requires source formatting. ' - 'Please run: git cl format %s' % - (short_path, ' '.join(display_args)))] - return [] + cmd.append(input_api.PresubmitLocalPath()) + + code, _ = git_cl.RunGitWithCode(cmd, suppress_stderr=bypass_warnings) + # bypass_warnings? Only fail with code 2. + # As this is just a warning, ignore all other errors if the user + # happens to have a broken clang-format, doesn't use git, etc etc. + if code == 2 or (code and not bypass_warnings): + if presubmit_subdir: + short_path = presubmit_subdir + else: + short_path = input_api.basename(input_api.change.RepositoryRoot()) + display_args.append(presubmit_subdir) + return [ + result_factory('The %s directory requires source formatting. ' + 'Please run: git cl format %s' % + (short_path, ' '.join(display_args))) + ] + return [] def CheckGNFormatted(input_api, output_api): - import gn - affected_files = input_api.AffectedFiles( - include_deletes=False, - file_filter=lambda x: x.LocalPath().endswith('.gn') or - x.LocalPath().endswith('.gni') or - x.LocalPath().endswith('.typemap')) - warnings = [] - for f in affected_files: - cmd = ['gn', 'format', '--dry-run', f.AbsoluteLocalPath()] - rc = gn.main(cmd) - if rc == 2: - warnings.append(output_api.PresubmitPromptWarning( - '%s requires formatting. Please run:\n gn format %s' % ( - f.AbsoluteLocalPath(), f.LocalPath()))) - # It's just a warning, so ignore other types of failures assuming they'll be - # caught elsewhere. - return warnings + import gn + affected_files = input_api.AffectedFiles( + include_deletes=False, + file_filter=lambda x: x.LocalPath().endswith('.gn') or x.LocalPath( + ).endswith('.gni') or x.LocalPath().endswith('.typemap')) + warnings = [] + for f in affected_files: + cmd = ['gn', 'format', '--dry-run', f.AbsoluteLocalPath()] + rc = gn.main(cmd) + if rc == 2: + warnings.append( + output_api.PresubmitPromptWarning( + '%s requires formatting. Please run:\n gn format %s' % + (f.AbsoluteLocalPath(), f.LocalPath()))) + # It's just a warning, so ignore other types of failures assuming they'll be + # caught elsewhere. + return warnings def CheckCIPDManifest(input_api, output_api, path=None, content=None): - """Verifies that a CIPD ensure file manifest is valid against all platforms. + """Verifies that a CIPD ensure file manifest is valid against all platforms. Exactly one of "path" or "content" must be provided. An assertion will occur if neither or both are provided. @@ -1659,59 +1765,55 @@ def CheckCIPDManifest(input_api, output_api, path=None, content=None): path (str): If provided, the filesystem path to the manifest to verify. content (str): If provided, the raw content of the manifest to veirfy. """ - cipd_bin = 'cipd' if not input_api.is_windows else 'cipd.bat' - cmd = [cipd_bin, 'ensure-file-verify'] - kwargs = {} + cipd_bin = 'cipd' if not input_api.is_windows else 'cipd.bat' + cmd = [cipd_bin, 'ensure-file-verify'] + kwargs = {} - if input_api.is_windows: - # Needs to be able to resolve "cipd.bat". - kwargs['shell'] = True + if input_api.is_windows: + # Needs to be able to resolve "cipd.bat". + kwargs['shell'] = True - if input_api.verbose: - cmd += ['-log-level', 'debug'] + if input_api.verbose: + cmd += ['-log-level', 'debug'] - if path: - assert content is None, 'Cannot provide both "path" and "content".' - cmd += ['-ensure-file', path] - name = 'Check CIPD manifest %r' % path - elif content: - assert path is None, 'Cannot provide both "path" and "content".' - cmd += ['-ensure-file=-'] - kwargs['stdin'] = content.encode('utf-8') - # quick and dirty parser to extract checked packages. - packages = [ - l.split()[0] for l in (ll.strip() for ll in content.splitlines()) - if ' ' in l and not l.startswith('$') - ] - name = 'Check CIPD packages from string: %r' % (packages,) - else: - raise Exception('Exactly one of "path" or "content" must be provided.') + if path: + assert content is None, 'Cannot provide both "path" and "content".' + cmd += ['-ensure-file', path] + name = 'Check CIPD manifest %r' % path + elif content: + assert path is None, 'Cannot provide both "path" and "content".' + cmd += ['-ensure-file=-'] + kwargs['stdin'] = content.encode('utf-8') + # quick and dirty parser to extract checked packages. + packages = [ + l.split()[0] for l in (ll.strip() for ll in content.splitlines()) + if ' ' in l and not l.startswith('$') + ] + name = 'Check CIPD packages from string: %r' % (packages, ) + else: + raise Exception('Exactly one of "path" or "content" must be provided.') - return input_api.Command( - name, - cmd, - kwargs, - output_api.PresubmitError) + return input_api.Command(name, cmd, kwargs, output_api.PresubmitError) def CheckCIPDPackages(input_api, output_api, platforms, packages): - """Verifies that all named CIPD packages can be resolved against all supplied + """Verifies that all named CIPD packages can be resolved against all supplied platforms. Args: platforms (list): List of CIPD platforms to verify. packages (dict): Mapping of package name to version. """ - manifest = [] - for p in platforms: - manifest.append('$VerifiedPlatform %s' % (p,)) - for k, v in packages.items(): - manifest.append('%s %s' % (k, v)) - return CheckCIPDManifest(input_api, output_api, content='\n'.join(manifest)) + manifest = [] + for p in platforms: + manifest.append('$VerifiedPlatform %s' % (p, )) + for k, v in packages.items(): + manifest.append('%s %s' % (k, v)) + return CheckCIPDManifest(input_api, output_api, content='\n'.join(manifest)) def CheckCIPDClientDigests(input_api, output_api, client_version_file): - """Verifies that *.digests file was correctly regenerated. + """Verifies that *.digests file was correctly regenerated. .digests file contains pinned hashes of the CIPD client. It is consulted during CIPD client bootstrap and self-update. It should be @@ -1720,21 +1822,24 @@ def CheckCIPDClientDigests(input_api, output_api, client_version_file): Args: client_version_file (str): Path to a text file with CIPD client version. """ - cmd = [ - 'cipd' if not input_api.is_windows else 'cipd.bat', - 'selfupdate-roll', '-check', '-version-file', client_version_file, - ] - if input_api.verbose: - cmd += ['-log-level', 'debug'] - return input_api.Command( - 'Check CIPD client_version_file.digests file', - cmd, - {'shell': True} if input_api.is_windows else {}, # to resolve cipd.bat - output_api.PresubmitError) + cmd = [ + 'cipd' if not input_api.is_windows else 'cipd.bat', + 'selfupdate-roll', + '-check', + '-version-file', + client_version_file, + ] + if input_api.verbose: + cmd += ['-log-level', 'debug'] + return input_api.Command( + 'Check CIPD client_version_file.digests file', + cmd, + {'shell': True} if input_api.is_windows else {}, # to resolve cipd.bat + output_api.PresubmitError) def CheckForCommitObjects(input_api, output_api): - """Validates that commit objects match DEPS. + """Validates that commit objects match DEPS. Commit objects are put into the git tree typically by submodule tooling. Because we use gclient to handle external repository references instead, @@ -1747,27 +1852,27 @@ def CheckForCommitObjects(input_api, output_api): Returns: A presubmit error if a commit object is not expected. """ - # Get DEPS file. - deps_file = input_api.os_path.join(input_api.PresubmitLocalPath(), 'DEPS') - if not input_api.os_path.isfile(deps_file): - # No DEPS file, carry on! - return [] + # Get DEPS file. + deps_file = input_api.os_path.join(input_api.PresubmitLocalPath(), 'DEPS') + if not input_api.os_path.isfile(deps_file): + # No DEPS file, carry on! + return [] - with open(deps_file) as f: - deps_content = f.read() - deps = _ParseDeps(deps_content) - # set default - if 'deps' not in deps: - deps['deps'] = {} - if 'git_dependencies' not in deps: - deps['git_dependencies'] = 'DEPS' + with open(deps_file) as f: + deps_content = f.read() + deps = _ParseDeps(deps_content) + # set default + if 'deps' not in deps: + deps['deps'] = {} + if 'git_dependencies' not in deps: + deps['git_dependencies'] = 'DEPS' - if deps['git_dependencies'] == 'SUBMODULES': - # git submodule is source of truth, so no further action needed. - return [] + if deps['git_dependencies'] == 'SUBMODULES': + # git submodule is source of truth, so no further action needed. + return [] - def parse_tree_entry(ent): - """Splits a tree entry into components + def parse_tree_entry(ent): + """Splits a tree entry into components Args: ent: a tree entry in the form "filemode type hash\tname" @@ -1775,118 +1880,120 @@ def CheckForCommitObjects(input_api, output_api): Returns: The tree entry split into component parts """ - tabparts = ent.split('\t', 1) - spaceparts = tabparts[0].split(' ', 2) - return (spaceparts[0], spaceparts[1], spaceparts[2], tabparts[1]) + tabparts = ent.split('\t', 1) + spaceparts = tabparts[0].split(' ', 2) + return (spaceparts[0], spaceparts[1], spaceparts[2], tabparts[1]) - full_tree = input_api.subprocess.check_output( - ['git', 'ls-tree', '-r', '--full-tree', '-z', 'HEAD'], - cwd=input_api.PresubmitLocalPath()) + full_tree = input_api.subprocess.check_output( + ['git', 'ls-tree', '-r', '--full-tree', '-z', 'HEAD'], + cwd=input_api.PresubmitLocalPath()) - # commit_tree_entries holds all commit entries (ie gitlink, submodule record). - commit_tree_entries = [] - for entry in full_tree.strip().split(b'\00'): - if not entry.startswith(b'160000'): - # Remove entries that we don't care about. 160000 indicates a gitlink. - continue - tree_entry = parse_tree_entry(entry.decode('utf-8')) - if tree_entry[1] == 'commit': - commit_tree_entries.append(tree_entry) + # commit_tree_entries holds all commit entries (ie gitlink, submodule + # record). + commit_tree_entries = [] + for entry in full_tree.strip().split(b'\00'): + if not entry.startswith(b'160000'): + # Remove entries that we don't care about. 160000 indicates a + # gitlink. + continue + tree_entry = parse_tree_entry(entry.decode('utf-8')) + if tree_entry[1] == 'commit': + commit_tree_entries.append(tree_entry) + + # No gitlinks found, return early. + if len(commit_tree_entries) == 0: + return [] + + if deps['git_dependencies'] == 'DEPS': + commit_tree_entries = [x[3] for x in commit_tree_entries] + return [ + output_api.PresubmitError( + 'Commit objects present within tree.\n' + 'This may be due to submodule-related interactions;\n' + 'the presence of a commit object in the tree may lead to odd\n' + 'situations where files are inconsistently checked-out.\n' + 'Remove these commit entries and validate your changeset ' + 'again:\n', commit_tree_entries) + ] + + assert deps['git_dependencies'] == 'SYNC', 'unexpected git_dependencies.' + + # Create mapping HASH -> PATH + git_submodules = {} + for commit_tree_entry in commit_tree_entries: + git_submodules[commit_tree_entry[2]] = commit_tree_entry[3] + + mismatch_entries = [] + deps_msg = "" + for dep_path, dep in deps['deps'].items(): + if 'dep_type' in dep and dep['dep_type'] != 'git': + continue + + url = dep if isinstance(dep, str) else dep['url'] + commit_hash = url.split('@')[-1] + # Two exceptions were in made in two projects prior to this check + # enforcement. We need to address those exceptions, but in the meantime + # we can't fail this global presubmit check + # https://chromium.googlesource.com/infra/infra/+/refs/heads/main/DEPS#45 + if dep_path == 'recipes-py' and commit_hash == 'refs/heads/main': + continue + + # https://chromium.googlesource.com/angle/angle/+/refs/heads/main/DEPS#412 + if dep_path == 'third_party/dummy_chromium': + continue + + if commit_hash in git_submodules: + git_submodules.pop(commit_hash) + else: + mismatch_entries.append(dep_path) + deps_msg += f"\n [DEPS] {dep_path} -> {commit_hash}" + + for commit_hash, path in git_submodules.items(): + mismatch_entries.append(path) + deps_msg += f"\n [gitlink] {path} -> {commit_hash}" + + if mismatch_entries: + return [ + output_api.PresubmitError( + 'DEPS file indicates git submodule migration is in progress,\n' + 'but the commit objects do not match DEPS entries.\n\n' + 'To reset all git submodule git entries to match DEPS, run\n' + 'the following command in the root of this repository:\n' + ' gclient gitmodules' + '\n\n' + 'The following entries diverged: ' + deps_msg) + ] - # No gitlinks found, return early. - if len(commit_tree_entries) == 0: return [] - if deps['git_dependencies'] == 'DEPS': - commit_tree_entries = [x[3] for x in commit_tree_entries] - return [ - output_api.PresubmitError( - 'Commit objects present within tree.\n' - 'This may be due to submodule-related interactions;\n' - 'the presence of a commit object in the tree may lead to odd\n' - 'situations where files are inconsistently checked-out.\n' - 'Remove these commit entries and validate your changeset ' - 'again:\n', commit_tree_entries) - ] - - assert deps['git_dependencies'] == 'SYNC', 'unexpected git_dependencies.' - - # Create mapping HASH -> PATH - git_submodules = {} - for commit_tree_entry in commit_tree_entries: - git_submodules[commit_tree_entry[2]] = commit_tree_entry[3] - - mismatch_entries = [] - deps_msg = "" - for dep_path, dep in deps['deps'].items(): - if 'dep_type' in dep and dep['dep_type'] != 'git': - continue - - url = dep if isinstance(dep, str) else dep['url'] - commit_hash = url.split('@')[-1] - # Two exceptions were in made in two projects prior to this check - # enforcement. We need to address those exceptions, but in the meantime we - # can't fail this global presubmit check - # https://chromium.googlesource.com/infra/infra/+/refs/heads/main/DEPS#45 - if dep_path == 'recipes-py' and commit_hash == 'refs/heads/main': - continue - - # https://chromium.googlesource.com/angle/angle/+/refs/heads/main/DEPS#412 - if dep_path == 'third_party/dummy_chromium': - continue - - if commit_hash in git_submodules: - git_submodules.pop(commit_hash) - else: - mismatch_entries.append(dep_path) - deps_msg += f"\n [DEPS] {dep_path} -> {commit_hash}" - - for commit_hash, path in git_submodules.items(): - mismatch_entries.append(path) - deps_msg += f"\n [gitlink] {path} -> {commit_hash}" - - if mismatch_entries: - return [ - output_api.PresubmitError( - 'DEPS file indicates git submodule migration is in progress,\n' - 'but the commit objects do not match DEPS entries.\n\n' - 'To reset all git submodule git entries to match DEPS, run\n' - 'the following command in the root of this repository:\n' - ' gclient gitmodules' - '\n\n' - 'The following entries diverged: ' + deps_msg) - ] - - return [] - def _ParseDeps(contents): - """Simple helper for parsing DEPS files.""" + """Simple helper for parsing DEPS files.""" - # Stubs for handling special syntax in the root DEPS file. - class _VarImpl: - def __init__(self, local_scope): - self._local_scope = local_scope + # Stubs for handling special syntax in the root DEPS file. + class _VarImpl: + def __init__(self, local_scope): + self._local_scope = local_scope - def Lookup(self, var_name): - """Implements the Var syntax.""" - try: - return self._local_scope['vars'][var_name] - except KeyError: - raise Exception('Var is not defined: %s' % var_name) + def Lookup(self, var_name): + """Implements the Var syntax.""" + try: + return self._local_scope['vars'][var_name] + except KeyError: + raise Exception('Var is not defined: %s' % var_name) - local_scope = {} - global_scope = { - 'Var': _VarImpl(local_scope).Lookup, - 'Str': str, - } + local_scope = {} + global_scope = { + 'Var': _VarImpl(local_scope).Lookup, + 'Str': str, + } - exec(contents, global_scope, local_scope) - return local_scope + exec(contents, global_scope, local_scope) + return local_scope def CheckVPythonSpec(input_api, output_api, file_filter=None): - """Validates any changed .vpython and .vpython3 files with vpython + """Validates any changed .vpython and .vpython3 files with vpython verification tool. Args: @@ -1900,24 +2007,25 @@ def CheckVPythonSpec(input_api, output_api, file_filter=None): Returns: A list of input_api.Command objects containing verification commands. """ - file_filter = file_filter or (lambda f: f.LocalPath().endswith('.vpython') or - f.LocalPath().endswith('.vpython3')) - affected_files = input_api.AffectedTestableFiles(file_filter=file_filter) - affected_files = map(lambda f: f.AbsoluteLocalPath(), affected_files) + file_filter = file_filter or (lambda f: f.LocalPath().endswith('.vpython') + or f.LocalPath().endswith('.vpython3')) + affected_files = input_api.AffectedTestableFiles(file_filter=file_filter) + affected_files = map(lambda f: f.AbsoluteLocalPath(), affected_files) - commands = [] - for f in affected_files: - commands.append( - input_api.Command('Verify %s' % f, [ - input_api.python3_executable, '-vpython-spec', f, '-vpython-tool', - 'verify' - ], {'stderr': input_api.subprocess.STDOUT}, output_api.PresubmitError)) + commands = [] + for f in affected_files: + commands.append( + input_api.Command('Verify %s' % f, [ + input_api.python3_executable, '-vpython-spec', f, + '-vpython-tool', 'verify' + ], {'stderr': input_api.subprocess.STDOUT}, + output_api.PresubmitError)) - return commands + return commands def CheckChangedLUCIConfigs(input_api, output_api): - """Validates the changed config file against LUCI Config. + """Validates the changed config file against LUCI Config. Only return the warning and/or error for files in input_api.AffectedFiles(). @@ -1927,155 +2035,169 @@ def CheckChangedLUCIConfigs(input_api, output_api): A list presubmit errors and/or warnings from the validation result of files in input_api.AffectedFiles() """ - import json - import logging + import json + import logging - import auth - import git_cl + import auth + import git_cl - LUCI_CONFIG_HOST_NAME = 'luci-config.appspot.com' + LUCI_CONFIG_HOST_NAME = 'luci-config.appspot.com' - cl = git_cl.Changelist() - if input_api.change.issue and input_api.gerrit: - remote_branch = input_api.gerrit.GetDestRef(input_api.change.issue) - else: - remote, remote_branch = cl.GetRemoteBranch() - if remote_branch.startswith('refs/remotes/%s/' % remote): - remote_branch = remote_branch.replace( - 'refs/remotes/%s/' % remote, 'refs/heads/', 1) - if remote_branch.startswith('refs/remotes/branch-heads/'): - remote_branch = remote_branch.replace( - 'refs/remotes/branch-heads/', 'refs/branch-heads/', 1) + cl = git_cl.Changelist() + if input_api.change.issue and input_api.gerrit: + remote_branch = input_api.gerrit.GetDestRef(input_api.change.issue) + else: + remote, remote_branch = cl.GetRemoteBranch() + if remote_branch.startswith('refs/remotes/%s/' % remote): + remote_branch = remote_branch.replace('refs/remotes/%s/' % remote, + 'refs/heads/', 1) + if remote_branch.startswith('refs/remotes/branch-heads/'): + remote_branch = remote_branch.replace('refs/remotes/branch-heads/', + 'refs/branch-heads/', 1) - remote_host_url = cl.GetRemoteUrl() - if not remote_host_url: - return [output_api.PresubmitError( - 'Remote host url for git has not been defined')] - remote_host_url = remote_host_url.rstrip('/') - if remote_host_url.endswith('.git'): - remote_host_url = remote_host_url[:-len('.git')] - - # authentication - try: - acc_tkn = auth.Authenticator().get_access_token() - except auth.LoginRequiredError as e: - return [output_api.PresubmitError( - 'Error in authenticating user.', long_text=str(e))] - - def request(endpoint, body=None): - api_url = ('https://%s/_ah/api/config/v1/%s' - % (LUCI_CONFIG_HOST_NAME, endpoint)) - req = input_api.urllib_request.Request(api_url) - req.add_header('Authorization', 'Bearer %s' % acc_tkn.token) - if body is not None: - req.data = zlib.compress(json.dumps(body).encode('utf-8')) - req.add_header('Content-Type', 'application/json-zlib') - return json.load(input_api.urllib_request.urlopen(req)) - - try: - config_sets = request('config-sets').get('config_sets') - except input_api.urllib_error.HTTPError as e: - return [output_api.PresubmitError( - 'Config set request to luci-config failed', long_text=str(e))] - if not config_sets: - return [output_api.PresubmitPromptWarning('No config_sets were returned')] - loc_pref = '%s/+/%s/' % (remote_host_url, remote_branch) - logging.debug('Derived location prefix: %s', loc_pref) - dir_to_config_set = {} - for cs in config_sets: - if cs['location'].startswith(loc_pref) or ('%s/' % - cs['location']) == loc_pref: - path = cs['location'][len(loc_pref):].rstrip('/') - d = input_api.os_path.join(*path.split('/')) if path else '.' - dir_to_config_set[d] = cs['config_set'] - if not dir_to_config_set: - warning_long_text_lines = [ - 'No config_set found for %s.' % loc_pref, - 'Found the following:', - ] - for loc in sorted(cs['location'] for cs in config_sets): - warning_long_text_lines.append(' %s' % loc) - warning_long_text_lines.append('') - warning_long_text_lines.append( - 'If the requested location is internal,' - ' the requester may not have access.') - - return [output_api.PresubmitPromptWarning( - warning_long_text_lines[0], - long_text='\n'.join(warning_long_text_lines))] - - dir_to_fileSet = {} - for f in input_api.AffectedFiles(include_deletes=False): - for d in dir_to_config_set: - if d != '.' and not f.LocalPath().startswith(d): - continue # file doesn't belong to this config set - rel_path = f.LocalPath() if d == '.' else input_api.os_path.relpath( - f.LocalPath(), start=d) - fileSet = dir_to_fileSet.setdefault(d, set()) - fileSet.add(rel_path.replace(_os.sep, '/')) - dir_to_fileSet[d] = fileSet - - outputs = [] - lucicfg = 'lucicfg' if not input_api.is_windows else 'lucicfg.bat' - log_level = 'debug' if input_api.verbose else 'warning' - repo_root = input_api.change.RepositoryRoot() - for d, fileSet in dir_to_fileSet.items(): - config_set = dir_to_config_set[d] - with input_api.CreateTemporaryFile() as f: - cmd = [ - lucicfg, 'validate', d, '-config-set', config_set, '-log-level', - log_level, '-json-output', f.name - ] - # return code is not important as the validation failure will be retrieved - # from the output json file. - out, _ = input_api.subprocess.communicate( - cmd, - stderr=input_api.subprocess.PIPE, - shell=input_api.is_windows, # to resolve *.bat - cwd=repo_root, - ) - logging.debug('running %s\nSTDOUT:\n%s\nSTDERR:\n%s', cmd, out[0], out[1]) - try: - result = json.load(f) - except json.JSONDecodeError as e: - outputs.append( + remote_host_url = cl.GetRemoteUrl() + if not remote_host_url: + return [ output_api.PresubmitError( - 'Error when parsing lucicfg validate output', long_text=str(e))) - else: - result = result.get('result', None) - if result: - non_affected_file_msg_count = 0 - for validation_result in (result.get('validation', None) or []): - for msg in (validation_result.get('messages', None) or []): - if d != '.' and msg['path'] not in fileSet: - non_affected_file_msg_count += 1 - continue - sev = msg['severity'] - if sev == 'WARNING': - out_f = output_api.PresubmitPromptWarning - elif sev in ('ERROR', 'CRITICAL'): - out_f = output_api.PresubmitError - else: - out_f = output_api.PresubmitNotifyResult - outputs.append( - out_f('Config validation for file(%s): %s' % - (msg['path'], msg['text']))) - if non_affected_file_msg_count: - reproduce_cmd = [ - lucicfg, 'validate', - repo_root if d == '.' else input_api.os_path.join(repo_root, d), - '-config-set', config_set + 'Remote host url for git has not been defined') + ] + remote_host_url = remote_host_url.rstrip('/') + if remote_host_url.endswith('.git'): + remote_host_url = remote_host_url[:-len('.git')] + + # authentication + try: + acc_tkn = auth.Authenticator().get_access_token() + except auth.LoginRequiredError as e: + return [ + output_api.PresubmitError('Error in authenticating user.', + long_text=str(e)) + ] + + def request(endpoint, body=None): + api_url = ('https://%s/_ah/api/config/v1/%s' % + (LUCI_CONFIG_HOST_NAME, endpoint)) + req = input_api.urllib_request.Request(api_url) + req.add_header('Authorization', 'Bearer %s' % acc_tkn.token) + if body is not None: + req.data = zlib.compress(json.dumps(body).encode('utf-8')) + req.add_header('Content-Type', 'application/json-zlib') + return json.load(input_api.urllib_request.urlopen(req)) + + try: + config_sets = request('config-sets').get('config_sets') + except input_api.urllib_error.HTTPError as e: + return [ + output_api.PresubmitError( + 'Config set request to luci-config failed', long_text=str(e)) + ] + if not config_sets: + return [ + output_api.PresubmitPromptWarning('No config_sets were returned') + ] + loc_pref = '%s/+/%s/' % (remote_host_url, remote_branch) + logging.debug('Derived location prefix: %s', loc_pref) + dir_to_config_set = {} + for cs in config_sets: + if cs['location'].startswith(loc_pref) or ('%s/' % + cs['location']) == loc_pref: + path = cs['location'][len(loc_pref):].rstrip('/') + d = input_api.os_path.join(*path.split('/')) if path else '.' + dir_to_config_set[d] = cs['config_set'] + if not dir_to_config_set: + warning_long_text_lines = [ + 'No config_set found for %s.' % loc_pref, + 'Found the following:', + ] + for loc in sorted(cs['location'] for cs in config_sets): + warning_long_text_lines.append(' %s' % loc) + warning_long_text_lines.append('') + warning_long_text_lines.append('If the requested location is internal,' + ' the requester may not have access.') + + return [ + output_api.PresubmitPromptWarning( + warning_long_text_lines[0], + long_text='\n'.join(warning_long_text_lines)) + ] + + dir_to_fileSet = {} + for f in input_api.AffectedFiles(include_deletes=False): + for d in dir_to_config_set: + if d != '.' and not f.LocalPath().startswith(d): + continue # file doesn't belong to this config set + rel_path = f.LocalPath() if d == '.' else input_api.os_path.relpath( + f.LocalPath(), start=d) + fileSet = dir_to_fileSet.setdefault(d, set()) + fileSet.add(rel_path.replace(_os.sep, '/')) + dir_to_fileSet[d] = fileSet + + outputs = [] + lucicfg = 'lucicfg' if not input_api.is_windows else 'lucicfg.bat' + log_level = 'debug' if input_api.verbose else 'warning' + repo_root = input_api.change.RepositoryRoot() + for d, fileSet in dir_to_fileSet.items(): + config_set = dir_to_config_set[d] + with input_api.CreateTemporaryFile() as f: + cmd = [ + lucicfg, 'validate', d, '-config-set', config_set, '-log-level', + log_level, '-json-output', f.name ] - outputs.append( - output_api.PresubmitPromptWarning( - 'Found %d additional errors/warnings in files that are not ' - 'modified, run `%s` to reveal them' % - (non_affected_file_msg_count, ' '.join(reproduce_cmd)))) - return outputs + # return code is not important as the validation failure will be + # retrieved from the output json file. + out, _ = input_api.subprocess.communicate( + cmd, + stderr=input_api.subprocess.PIPE, + shell=input_api.is_windows, # to resolve *.bat + cwd=repo_root, + ) + logging.debug('running %s\nSTDOUT:\n%s\nSTDERR:\n%s', cmd, out[0], + out[1]) + try: + result = json.load(f) + except json.JSONDecodeError as e: + outputs.append( + output_api.PresubmitError( + 'Error when parsing lucicfg validate output', + long_text=str(e))) + else: + result = result.get('result', None) + if result: + non_affected_file_msg_count = 0 + for validation_result in (result.get('validation', None) + or []): + for msg in (validation_result.get('messages', None) + or []): + if d != '.' and msg['path'] not in fileSet: + non_affected_file_msg_count += 1 + continue + sev = msg['severity'] + if sev == 'WARNING': + out_f = output_api.PresubmitPromptWarning + elif sev in ('ERROR', 'CRITICAL'): + out_f = output_api.PresubmitError + else: + out_f = output_api.PresubmitNotifyResult + outputs.append( + out_f('Config validation for file(%s): %s' % + (msg['path'], msg['text']))) + if non_affected_file_msg_count: + reproduce_cmd = [ + lucicfg, 'validate', + repo_root if d == '.' else input_api.os_path.join( + repo_root, d), '-config-set', config_set + ] + outputs.append( + output_api.PresubmitPromptWarning( + 'Found %d additional errors/warnings in files that are not ' + 'modified, run `%s` to reveal them' % + (non_affected_file_msg_count, + ' '.join(reproduce_cmd)))) + return outputs def CheckLucicfgGenOutput(input_api, output_api, entry_script): - """Verifies configs produced by `lucicfg` are up-to-date and pass validation. + """Verifies configs produced by `lucicfg` are up-to-date and pass validation. Runs the check unconditionally, regardless of what files are modified. Examine input_api.AffectedFiles() yourself before using CheckLucicfgGenOutput if this @@ -2091,41 +2213,46 @@ def CheckLucicfgGenOutput(input_api, output_api, entry_script): Returns: A list of input_api.Command objects containing verification commands. """ - return [ - input_api.Command( - 'lucicfg validate "%s"' % entry_script, - [ - 'lucicfg' if not input_api.is_windows else 'lucicfg.bat', - 'validate', entry_script, - '-log-level', 'debug' if input_api.verbose else 'warning', - ], - { - 'stderr': input_api.subprocess.STDOUT, - 'shell': input_api.is_windows, # to resolve *.bat - 'cwd': input_api.PresubmitLocalPath(), - }, - output_api.PresubmitError) - ] + return [ + input_api.Command( + 'lucicfg validate "%s"' % entry_script, + [ + 'lucicfg' if not input_api.is_windows else 'lucicfg.bat', + 'validate', + entry_script, + '-log-level', + 'debug' if input_api.verbose else 'warning', + ], + { + 'stderr': input_api.subprocess.STDOUT, + 'shell': input_api.is_windows, # to resolve *.bat + 'cwd': input_api.PresubmitLocalPath(), + }, + output_api.PresubmitError) + ] + def CheckJsonParses(input_api, output_api, file_filter=None): - """Verifies that all JSON files at least parse as valid JSON. By default, + """Verifies that all JSON files at least parse as valid JSON. By default, file_filter will look for all files that end with .json""" - import json - if file_filter is None: - file_filter = lambda x: x.LocalPath().endswith('.json') - affected_files = input_api.AffectedFiles( - include_deletes=False, - file_filter=file_filter) - warnings = [] - for f in affected_files: - with _io.open(f.AbsoluteLocalPath(), encoding='utf-8') as j: - try: - json.load(j) - except ValueError: - # Just a warning for now, in case people are using JSON5 somewhere. - warnings.append(output_api.PresubmitPromptWarning( - '%s does not appear to be valid JSON.' % f.LocalPath())) - return warnings + import json + if file_filter is None: + file_filter = lambda x: x.LocalPath().endswith('.json') + affected_files = input_api.AffectedFiles(include_deletes=False, + file_filter=file_filter) + warnings = [] + for f in affected_files: + with _io.open(f.AbsoluteLocalPath(), encoding='utf-8') as j: + try: + json.load(j) + except ValueError: + # Just a warning for now, in case people are using JSON5 + # somewhere. + warnings.append( + output_api.PresubmitPromptWarning( + '%s does not appear to be valid JSON.' % f.LocalPath())) + return warnings + # string pattern, sequence of strings to show when pattern matches, # error flag. True if match is a presubmit error, otherwise it's a warning. @@ -2150,135 +2277,137 @@ _NON_INCLUSIVE_TERMS = ( def _GetMessageForMatchingTerm(input_api, affected_file, line_number, line, term, message): - """Helper method for CheckInclusiveLanguage. + """Helper method for CheckInclusiveLanguage. Returns an string composed of the name of the file, the line number where the match has been found and the additional text passed as |message| in case the target type name matches the text inside the line passed as parameter. """ - result = [] + result = [] + + # A // nocheck comment will bypass this error. + if line.endswith(" nocheck") or line.endswith(""): + return result + + # Ignore C-style single-line comments about banned terms. + if input_api.re.search(r"//.*$", line): + line = input_api.re.sub(r"//.*$", "", line) + + # Ignore lines from C-style multi-line comments. + if input_api.re.search(r"^\s*\*", line): + return result + + # Ignore Python-style comments about banned terms. + # This actually removes comment text from the first # on. + if input_api.re.search(r"#.*$", line): + line = input_api.re.sub(r"#.*$", "", line) + + matched = False + if term[0:1] == '/': + regex = term[1:] + if input_api.re.search(regex, line): + matched = True + elif term in line: + matched = True + + if matched: + result.append(' %s:%d:' % (affected_file.LocalPath(), line_number)) + for message_line in message: + result.append(' %s' % message_line) - # A // nocheck comment will bypass this error. - if line.endswith(" nocheck") or line.endswith(""): return result - # Ignore C-style single-line comments about banned terms. - if input_api.re.search(r"//.*$", line): - line = input_api.re.sub(r"//.*$", "", line) - # Ignore lines from C-style multi-line comments. - if input_api.re.search(r"^\s*\*", line): - return result - - # Ignore Python-style comments about banned terms. - # This actually removes comment text from the first # on. - if input_api.re.search(r"#.*$", line): - line = input_api.re.sub(r"#.*$", "", line) - - matched = False - if term[0:1] == '/': - regex = term[1:] - if input_api.re.search(regex, line): - matched = True - elif term in line: - matched = True - - if matched: - result.append(' %s:%d:' % (affected_file.LocalPath(), line_number)) - for message_line in message: - result.append(' %s' % message_line) - - return result - - -def CheckInclusiveLanguage(input_api, output_api, +def CheckInclusiveLanguage(input_api, + output_api, excluded_directories_relative_path=None, non_inclusive_terms=_NON_INCLUSIVE_TERMS): - """Make sure that banned non-inclusive terms are not used.""" + """Make sure that banned non-inclusive terms are not used.""" - # Presubmit checks may run on a bot where the changes are actually - # in a repo that isn't chromium/src (e.g., when testing src + tip-of-tree - # ANGLE), but this particular check only makes sense for changes to - # chromium/src. - if input_api.change.RepositoryRoot() != input_api.PresubmitLocalPath(): - return [] - if input_api.no_diffs: - return [] + # Presubmit checks may run on a bot where the changes are actually + # in a repo that isn't chromium/src (e.g., when testing src + tip-of-tree + # ANGLE), but this particular check only makes sense for changes to + # chromium/src. + if input_api.change.RepositoryRoot() != input_api.PresubmitLocalPath(): + return [] + if input_api.no_diffs: + return [] - warnings = [] - errors = [] + warnings = [] + errors = [] - if excluded_directories_relative_path is None: - excluded_directories_relative_path = [ - 'infra', - 'inclusive_language_presubmit_exempt_dirs.txt' - ] + if excluded_directories_relative_path is None: + excluded_directories_relative_path = [ + 'infra', 'inclusive_language_presubmit_exempt_dirs.txt' + ] - # Note that this matches exact path prefixes, and does not match - # subdirectories. Only files directly in an excluded path will - # match. - def IsExcludedFile(affected_file, excluded_paths): - local_dir = input_api.os_path.dirname(affected_file.LocalPath()) + # Note that this matches exact path prefixes, and does not match + # subdirectories. Only files directly in an excluded path will + # match. + def IsExcludedFile(affected_file, excluded_paths): + local_dir = input_api.os_path.dirname(affected_file.LocalPath()) - # Excluded paths use forward slashes. - if input_api.platform == 'win32': - local_dir = local_dir.replace('\\', '/') + # Excluded paths use forward slashes. + if input_api.platform == 'win32': + local_dir = local_dir.replace('\\', '/') - return local_dir in excluded_paths + return local_dir in excluded_paths - def CheckForMatch(affected_file, line_num, line, term, message, error): - problems = _GetMessageForMatchingTerm(input_api, affected_file, line_num, - line, term, message) + def CheckForMatch(affected_file, line_num, line, term, message, error): + problems = _GetMessageForMatchingTerm(input_api, affected_file, + line_num, line, term, message) - if problems: - if error: - errors.extend(problems) - else: - warnings.extend(problems) + if problems: + if error: + errors.extend(problems) + else: + warnings.extend(problems) - excluded_paths = [] - dirs_file_path = input_api.os_path.join(input_api.change.RepositoryRoot(), - *excluded_directories_relative_path) - f = input_api.ReadFile(dirs_file_path) + excluded_paths = [] + dirs_file_path = input_api.os_path.join(input_api.change.RepositoryRoot(), + *excluded_directories_relative_path) + f = input_api.ReadFile(dirs_file_path) - for line in f.splitlines(): - path = line.split()[0] - if len(path) > 0: - excluded_paths.append(path) + for line in f.splitlines(): + path = line.split()[0] + if len(path) > 0: + excluded_paths.append(path) - excluded_paths = set(excluded_paths) - for f in input_api.AffectedFiles(): - for line_num, line in f.ChangedContents(): - for term, message, error in non_inclusive_terms: - if IsExcludedFile(f, excluded_paths): - continue - CheckForMatch(f, line_num, line, term, message, error) + excluded_paths = set(excluded_paths) + for f in input_api.AffectedFiles(): + for line_num, line in f.ChangedContents(): + for term, message, error in non_inclusive_terms: + if IsExcludedFile(f, excluded_paths): + continue + CheckForMatch(f, line_num, line, term, message, error) - result = [] - if (warnings): - result.append( - output_api.PresubmitPromptWarning( - 'Banned non-inclusive language was used.\n' + '\n'.join(warnings))) - if (errors): - result.append( - output_api.PresubmitError('Banned non-inclusive language was used.\n' + - '\n'.join(errors))) - return result + result = [] + if (warnings): + result.append( + output_api.PresubmitPromptWarning( + 'Banned non-inclusive language was used.\n' + + '\n'.join(warnings))) + if (errors): + result.append( + output_api.PresubmitError( + 'Banned non-inclusive language was used.\n' + + '\n'.join(errors))) + return result def CheckUpdateOwnersFileReferences(input_api, output_api): - """Checks whether an OWNERS file is being (re)moved and if so asks the + """Checks whether an OWNERS file is being (re)moved and if so asks the contributor to update any file:// references to it.""" - files = [] - # AffectedFiles() includes owner files, not AffectedSourceFiles(). - for f in input_api.AffectedFiles(): - # Moved files appear here as one deletion and one addition. - if f.LocalPath().endswith('OWNERS') and f.Action() == 'D': - files.append(f.LocalPath()) - if not files: - return [] - return [ - output_api.PresubmitPromptWarning( - 'OWNERS files being moved/removed, please update any file:// ' + - 'references to them in other OWNERS files', files) - ] + files = [] + # AffectedFiles() includes owner files, not AffectedSourceFiles(). + for f in input_api.AffectedFiles(): + # Moved files appear here as one deletion and one addition. + if f.LocalPath().endswith('OWNERS') and f.Action() == 'D': + files.append(f.LocalPath()) + if not files: + return [] + return [ + output_api.PresubmitPromptWarning( + 'OWNERS files being moved/removed, please update any file:// ' + + 'references to them in other OWNERS files', files) + ] diff --git a/presubmit_support.py b/presubmit_support.py index de8ce6ade5..b986b7bc77 100755 --- a/presubmit_support.py +++ b/presubmit_support.py @@ -2,7 +2,6 @@ # Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Enables directory-specific presubmit checks to run at upload and/or commit. """ @@ -53,6 +52,8 @@ import rdb_wrapper import scm import subprocess2 as subprocess # Exposed through the API. +# TODO: Should fix these warnings. +# pylint: disable=line-too-long # Ask for feedback only once in program lifetime. _ASKED_FOR_FEEDBACK = False @@ -63,30 +64,29 @@ _SHOW_CALLSTACKS = False def time_time(): - # Use this so that it can be mocked in tests without interfering with python - # system machinery. - return time.time() + # Use this so that it can be mocked in tests without interfering with python + # system machinery. + return time.time() class PresubmitFailure(Exception): - pass + pass class CommandData(object): - def __init__(self, name, cmd, kwargs, message, python3=True): - # The python3 argument is ignored but has to be retained because of the many - # callers in other repos that pass it in. - del python3 - self.name = name - self.cmd = cmd - self.stdin = kwargs.get('stdin', None) - self.kwargs = kwargs.copy() - self.kwargs['stdout'] = subprocess.PIPE - self.kwargs['stderr'] = subprocess.STDOUT - self.kwargs['stdin'] = subprocess.PIPE - self.message = message - self.info = None - + def __init__(self, name, cmd, kwargs, message, python3=True): + # The python3 argument is ignored but has to be retained because of the + # many callers in other repos that pass it in. + del python3 + self.name = name + self.cmd = cmd + self.stdin = kwargs.get('stdin', None) + self.kwargs = kwargs.copy() + self.kwargs['stdout'] = subprocess.PIPE + self.kwargs['stderr'] = subprocess.STDOUT + self.kwargs['stdin'] = subprocess.PIPE + self.message = message + self.info = None # Adapted from @@ -102,476 +102,514 @@ class CommandData(object): # or a subprocess, including the one the current call is waiting for), # wait(p) will call p.terminate(). class SigintHandler(object): - sigint_returncodes = {-signal.SIGINT, # Unix - -1073741510, # Windows - } - def __init__(self): - self.__lock = threading.Lock() - self.__processes = set() - self.__got_sigint = False - self.__previous_signal = signal.signal(signal.SIGINT, self.interrupt) + sigint_returncodes = { + -signal.SIGINT, # Unix + -1073741510, # Windows + } - def __on_sigint(self): - self.__got_sigint = True - while self.__processes: - try: - self.__processes.pop().terminate() - except OSError: - pass + def __init__(self): + self.__lock = threading.Lock() + self.__processes = set() + self.__got_sigint = False + self.__previous_signal = signal.signal(signal.SIGINT, self.interrupt) - def interrupt(self, signal_num, frame): - with self.__lock: - self.__on_sigint() - self.__previous_signal(signal_num, frame) + def __on_sigint(self): + self.__got_sigint = True + while self.__processes: + try: + self.__processes.pop().terminate() + except OSError: + pass - def got_sigint(self): - with self.__lock: - return self.__got_sigint + def interrupt(self, signal_num, frame): + with self.__lock: + self.__on_sigint() + self.__previous_signal(signal_num, frame) + + def got_sigint(self): + with self.__lock: + return self.__got_sigint + + def wait(self, p, stdin): + with self.__lock: + if self.__got_sigint: + p.terminate() + self.__processes.add(p) + stdout, stderr = p.communicate(stdin) + code = p.returncode + with self.__lock: + self.__processes.discard(p) + if code in self.sigint_returncodes: + self.__on_sigint() + return stdout, stderr - def wait(self, p, stdin): - with self.__lock: - if self.__got_sigint: - p.terminate() - self.__processes.add(p) - stdout, stderr = p.communicate(stdin) - code = p.returncode - with self.__lock: - self.__processes.discard(p) - if code in self.sigint_returncodes: - self.__on_sigint() - return stdout, stderr sigint_handler = SigintHandler() class Timer(object): - def __init__(self, timeout, fn): - self.completed = False - self._fn = fn - self._timer = threading.Timer(timeout, self._onTimer) if timeout else None + def __init__(self, timeout, fn): + self.completed = False + self._fn = fn + self._timer = threading.Timer(timeout, + self._onTimer) if timeout else None - def __enter__(self): - if self._timer: - self._timer.start() - return self + def __enter__(self): + if self._timer: + self._timer.start() + return self - def __exit__(self, _type, _value, _traceback): - if self._timer: - self._timer.cancel() + def __exit__(self, _type, _value, _traceback): + if self._timer: + self._timer.cancel() - def _onTimer(self): - self._fn() - self.completed = True + def _onTimer(self): + self._fn() + self.completed = True class ThreadPool(object): - def __init__(self, pool_size=None, timeout=None): - self.timeout = timeout - self._pool_size = pool_size or multiprocessing.cpu_count() - if sys.platform == 'win32': - # TODO(crbug.com/1190269) - we can't use more than 56 child processes on - # Windows or Python3 may hang. - self._pool_size = min(self._pool_size, 56) - self._messages = [] - self._messages_lock = threading.Lock() - self._tests = [] - self._tests_lock = threading.Lock() - self._nonparallel_tests = [] + def __init__(self, pool_size=None, timeout=None): + self.timeout = timeout + self._pool_size = pool_size or multiprocessing.cpu_count() + if sys.platform == 'win32': + # TODO(crbug.com/1190269) - we can't use more than 56 child + # processes on Windows or Python3 may hang. + self._pool_size = min(self._pool_size, 56) + self._messages = [] + self._messages_lock = threading.Lock() + self._tests = [] + self._tests_lock = threading.Lock() + self._nonparallel_tests = [] - def _GetCommand(self, test): - vpython = 'vpython3' - if sys.platform == 'win32': - vpython += '.bat' + def _GetCommand(self, test): + vpython = 'vpython3' + if sys.platform == 'win32': + vpython += '.bat' - cmd = test.cmd - if cmd[0] == 'python': - cmd = list(cmd) - cmd[0] = vpython - elif cmd[0].endswith('.py'): - cmd = [vpython] + cmd + cmd = test.cmd + if cmd[0] == 'python': + cmd = list(cmd) + cmd[0] = vpython + elif cmd[0].endswith('.py'): + cmd = [vpython] + cmd - # On Windows, scripts on the current directory take precedence over PATH, so - # that when testing depot_tools on Windows, calling `vpython.bat` will - # execute the copy of vpython of the depot_tools under test instead of the - # one in the bot. - # As a workaround, we run the tests from the parent directory instead. - if (cmd[0] == vpython and - 'cwd' in test.kwargs and - os.path.basename(test.kwargs['cwd']) == 'depot_tools'): - test.kwargs['cwd'] = os.path.dirname(test.kwargs['cwd']) - cmd[1] = os.path.join('depot_tools', cmd[1]) + # On Windows, scripts on the current directory take precedence over + # PATH, so that when testing depot_tools on Windows, calling + # `vpython.bat` will execute the copy of vpython of the depot_tools + # under test instead of the one in the bot. As a workaround, we run the + # tests from the parent directory instead. + if (cmd[0] == vpython and 'cwd' in test.kwargs + and os.path.basename(test.kwargs['cwd']) == 'depot_tools'): + test.kwargs['cwd'] = os.path.dirname(test.kwargs['cwd']) + cmd[1] = os.path.join('depot_tools', cmd[1]) - return cmd + return cmd - def _RunWithTimeout(self, cmd, stdin, kwargs): - p = subprocess.Popen(cmd, **kwargs) - with Timer(self.timeout, p.terminate) as timer: - stdout, _ = sigint_handler.wait(p, stdin) - stdout = stdout.decode('utf-8', 'ignore') - if timer.completed: - stdout = 'Process timed out after %ss\n%s' % (self.timeout, stdout) - return p.returncode, stdout + def _RunWithTimeout(self, cmd, stdin, kwargs): + p = subprocess.Popen(cmd, **kwargs) + with Timer(self.timeout, p.terminate) as timer: + stdout, _ = sigint_handler.wait(p, stdin) + stdout = stdout.decode('utf-8', 'ignore') + if timer.completed: + stdout = 'Process timed out after %ss\n%s' % (self.timeout, + stdout) + return p.returncode, stdout - def CallCommand(self, test, show_callstack=None): - """Runs an external program. + def CallCommand(self, test, show_callstack=None): + """Runs an external program. This function converts invocation of .py files and invocations of 'python' to vpython invocations. """ - cmd = self._GetCommand(test) - try: - start = time_time() - returncode, stdout = self._RunWithTimeout(cmd, test.stdin, test.kwargs) - duration = time_time() - start - except Exception: - duration = time_time() - start - return test.message( - '%s\n%s exec failure (%4.2fs)\n%s' % - (test.name, ' '.join(cmd), duration, traceback.format_exc()), - show_callstack=show_callstack) + cmd = self._GetCommand(test) + try: + start = time_time() + returncode, stdout = self._RunWithTimeout(cmd, test.stdin, + test.kwargs) + duration = time_time() - start + except Exception: + duration = time_time() - start + return test.message( + '%s\n%s exec failure (%4.2fs)\n%s' % + (test.name, ' '.join(cmd), duration, traceback.format_exc()), + show_callstack=show_callstack) - if returncode != 0: - return test.message('%s\n%s (%4.2fs) failed\n%s' % - (test.name, ' '.join(cmd), duration, stdout), - show_callstack=show_callstack) + if returncode != 0: + return test.message('%s\n%s (%4.2fs) failed\n%s' % + (test.name, ' '.join(cmd), duration, stdout), + show_callstack=show_callstack) - if test.info: - return test.info('%s\n%s (%4.2fs)' % (test.name, ' '.join(cmd), duration), - show_callstack=show_callstack) + if test.info: + return test.info('%s\n%s (%4.2fs)' % + (test.name, ' '.join(cmd), duration), + show_callstack=show_callstack) - def AddTests(self, tests, parallel=True): - if parallel: - self._tests.extend(tests) - else: - self._nonparallel_tests.extend(tests) + def AddTests(self, tests, parallel=True): + if parallel: + self._tests.extend(tests) + else: + self._nonparallel_tests.extend(tests) - def RunAsync(self): - self._messages = [] + def RunAsync(self): + self._messages = [] - def _WorkerFn(): - while True: - test = None - with self._tests_lock: - if not self._tests: - break - test = self._tests.pop() - result = self.CallCommand(test, show_callstack=False) - if result: - with self._messages_lock: - self._messages.append(result) + def _WorkerFn(): + while True: + test = None + with self._tests_lock: + if not self._tests: + break + test = self._tests.pop() + result = self.CallCommand(test, show_callstack=False) + if result: + with self._messages_lock: + self._messages.append(result) - def _StartDaemon(): - t = threading.Thread(target=_WorkerFn) - t.daemon = True - t.start() - return t + def _StartDaemon(): + t = threading.Thread(target=_WorkerFn) + t.daemon = True + t.start() + return t - while self._nonparallel_tests: - test = self._nonparallel_tests.pop() - result = self.CallCommand(test) - if result: - self._messages.append(result) + while self._nonparallel_tests: + test = self._nonparallel_tests.pop() + result = self.CallCommand(test) + if result: + self._messages.append(result) - if self._tests: - threads = [_StartDaemon() for _ in range(self._pool_size)] - for worker in threads: - worker.join() + if self._tests: + threads = [_StartDaemon() for _ in range(self._pool_size)] + for worker in threads: + worker.join() - return self._messages + return self._messages def normpath(path): - '''Version of os.path.normpath that also changes backward slashes to + '''Version of os.path.normpath that also changes backward slashes to forward slashes when not running on Windows. ''' - # This is safe to always do because the Windows version of os.path.normpath - # will replace forward slashes with backward slashes. - path = path.replace(os.sep, '/') - return os.path.normpath(path) + # This is safe to always do because the Windows version of os.path.normpath + # will replace forward slashes with backward slashes. + path = path.replace(os.sep, '/') + return os.path.normpath(path) def _RightHandSideLinesImpl(affected_files): - """Implements RightHandSideLines for InputApi and GclChange.""" - for af in affected_files: - lines = af.ChangedContents() - for line in lines: - yield (af, line[0], line[1]) + """Implements RightHandSideLines for InputApi and GclChange.""" + for af in affected_files: + lines = af.ChangedContents() + for line in lines: + yield (af, line[0], line[1]) def prompt_should_continue(prompt_string): - sys.stdout.write(prompt_string) - sys.stdout.flush() - response = sys.stdin.readline().strip().lower() - return response in ('y', 'yes') + sys.stdout.write(prompt_string) + sys.stdout.flush() + response = sys.stdin.readline().strip().lower() + return response in ('y', 'yes') # Top level object so multiprocessing can pickle # Public access through OutputApi object. class _PresubmitResult(object): - """Base class for result objects.""" - fatal = False - should_prompt = False + """Base class for result objects.""" + fatal = False + should_prompt = False - def __init__(self, message, items=None, long_text='', show_callstack=None): - """ + def __init__(self, message, items=None, long_text='', show_callstack=None): + """ message: A short one-line message to indicate errors. items: A list of short strings to indicate where errors occurred. long_text: multi-line text output, e.g. from another tool """ - self._message = _PresubmitResult._ensure_str(message) - self._items = items or [] - self._long_text = _PresubmitResult._ensure_str(long_text.rstrip()) - if show_callstack is None: - show_callstack = _SHOW_CALLSTACKS - if show_callstack: - self._long_text += 'Presubmit result call stack is:\n' - self._long_text += ''.join(traceback.format_stack(None, 8)) + self._message = _PresubmitResult._ensure_str(message) + self._items = items or [] + self._long_text = _PresubmitResult._ensure_str(long_text.rstrip()) + if show_callstack is None: + show_callstack = _SHOW_CALLSTACKS + if show_callstack: + self._long_text += 'Presubmit result call stack is:\n' + self._long_text += ''.join(traceback.format_stack(None, 8)) - @staticmethod - def _ensure_str(val): - """ + @staticmethod + def _ensure_str(val): + """ val: A "stringish" value. Can be any of str or bytes. returns: A str after applying encoding/decoding as needed. Assumes/uses UTF-8 for relevant inputs/outputs. """ - if isinstance(val, str): - return val - if isinstance(val, bytes): - return val.decode() - raise ValueError("Unknown string type %s" % type(val)) + if isinstance(val, str): + return val + if isinstance(val, bytes): + return val.decode() + raise ValueError("Unknown string type %s" % type(val)) - def handle(self): - sys.stdout.write(self._message) - sys.stdout.write('\n') - for item in self._items: - sys.stdout.write(' ') - # Write separately in case it's unicode. - sys.stdout.write(str(item)) - sys.stdout.write('\n') - if self._long_text: - sys.stdout.write('\n***************\n') - # Write separately in case it's unicode. - sys.stdout.write(self._long_text) - sys.stdout.write('\n***************\n') + def handle(self): + sys.stdout.write(self._message) + sys.stdout.write('\n') + for item in self._items: + sys.stdout.write(' ') + # Write separately in case it's unicode. + sys.stdout.write(str(item)) + sys.stdout.write('\n') + if self._long_text: + sys.stdout.write('\n***************\n') + # Write separately in case it's unicode. + sys.stdout.write(self._long_text) + sys.stdout.write('\n***************\n') - def json_format(self): - return { - 'message': self._message, - 'items': [str(item) for item in self._items], - 'long_text': self._long_text, - 'fatal': self.fatal - } + def json_format(self): + return { + 'message': self._message, + 'items': [str(item) for item in self._items], + 'long_text': self._long_text, + 'fatal': self.fatal + } # Top level object so multiprocessing can pickle # Public access through OutputApi object. class _PresubmitError(_PresubmitResult): - """A hard presubmit error.""" - fatal = True + """A hard presubmit error.""" + fatal = True # Top level object so multiprocessing can pickle # Public access through OutputApi object. class _PresubmitPromptWarning(_PresubmitResult): - """An warning that prompts the user if they want to continue.""" - should_prompt = True + """An warning that prompts the user if they want to continue.""" + should_prompt = True # Top level object so multiprocessing can pickle # Public access through OutputApi object. class _PresubmitNotifyResult(_PresubmitResult): - """Just print something to the screen -- but it's not even a warning.""" + """Just print something to the screen -- but it's not even a warning.""" # Top level object so multiprocessing can pickle # Public access through OutputApi object. class _MailTextResult(_PresubmitResult): - """A warning that should be included in the review request email.""" - def __init__(self, *args, **kwargs): - super(_MailTextResult, self).__init__() - raise NotImplementedError() + """A warning that should be included in the review request email.""" + def __init__(self, *args, **kwargs): + super(_MailTextResult, self).__init__() + raise NotImplementedError() + class GerritAccessor(object): - """Limited Gerrit functionality for canned presubmit checks to work. + """Limited Gerrit functionality for canned presubmit checks to work. To avoid excessive Gerrit calls, caches the results. """ + def __init__(self, url=None, project=None, branch=None): + self.host = urlparse.urlparse(url).netloc if url else None + self.project = project + self.branch = branch + self.cache = {} + self.code_owners_enabled = None - def __init__(self, url=None, project=None, branch=None): - self.host = urlparse.urlparse(url).netloc if url else None - self.project = project - self.branch = branch - self.cache = {} - self.code_owners_enabled = None + def _FetchChangeDetail(self, issue): + # Separate function to be easily mocked in tests. + try: + return gerrit_util.GetChangeDetail( + self.host, str(issue), + ['ALL_REVISIONS', 'DETAILED_LABELS', 'ALL_COMMITS']) + except gerrit_util.GerritError as e: + if e.http_status == 404: + raise Exception('Either Gerrit issue %s doesn\'t exist, or ' + 'no credentials to fetch issue details' % issue) + raise - def _FetchChangeDetail(self, issue): - # Separate function to be easily mocked in tests. - try: - return gerrit_util.GetChangeDetail( - self.host, str(issue), - ['ALL_REVISIONS', 'DETAILED_LABELS', 'ALL_COMMITS']) - except gerrit_util.GerritError as e: - if e.http_status == 404: - raise Exception('Either Gerrit issue %s doesn\'t exist, or ' - 'no credentials to fetch issue details' % issue) - raise - - def GetChangeInfo(self, issue): - """Returns labels and all revisions (patchsets) for this issue. + def GetChangeInfo(self, issue): + """Returns labels and all revisions (patchsets) for this issue. The result is a dictionary according to Gerrit REST Api. https://gerrit-review.googlesource.com/Documentation/rest-api.html However, API isn't very clear what's inside, so see tests for example. """ - assert issue - cache_key = int(issue) - if cache_key not in self.cache: - self.cache[cache_key] = self._FetchChangeDetail(issue) - return self.cache[cache_key] + assert issue + cache_key = int(issue) + if cache_key not in self.cache: + self.cache[cache_key] = self._FetchChangeDetail(issue) + return self.cache[cache_key] - def GetChangeDescription(self, issue, patchset=None): - """If patchset is none, fetches current patchset.""" - info = self.GetChangeInfo(issue) - # info is a reference to cache. We'll modify it here adding description to - # it to the right patchset, if it is not yet there. + def GetChangeDescription(self, issue, patchset=None): + """If patchset is none, fetches current patchset.""" + info = self.GetChangeInfo(issue) + # info is a reference to cache. We'll modify it here adding description + # to it to the right patchset, if it is not yet there. - # Find revision info for the patchset we want. - if patchset is not None: - for rev, rev_info in info['revisions'].items(): - if str(rev_info['_number']) == str(patchset): - break - else: - raise Exception('patchset %s doesn\'t exist in issue %s' % ( - patchset, issue)) - else: - rev = info['current_revision'] - rev_info = info['revisions'][rev] + # Find revision info for the patchset we want. + if patchset is not None: + for rev, rev_info in info['revisions'].items(): + if str(rev_info['_number']) == str(patchset): + break + else: + raise Exception('patchset %s doesn\'t exist in issue %s' % + (patchset, issue)) + else: + rev = info['current_revision'] + rev_info = info['revisions'][rev] - return rev_info['commit']['message'] + return rev_info['commit']['message'] - def GetDestRef(self, issue): - ref = self.GetChangeInfo(issue)['branch'] - if not ref.startswith('refs/'): - # NOTE: it is possible to create 'refs/x' branch, - # aka 'refs/heads/refs/x'. However, this is ill-advised. - ref = 'refs/heads/%s' % ref - return ref + def GetDestRef(self, issue): + ref = self.GetChangeInfo(issue)['branch'] + if not ref.startswith('refs/'): + # NOTE: it is possible to create 'refs/x' branch, + # aka 'refs/heads/refs/x'. However, this is ill-advised. + ref = 'refs/heads/%s' % ref + return ref - def _GetApproversForLabel(self, issue, label): - change_info = self.GetChangeInfo(issue) - label_info = change_info.get('labels', {}).get(label, {}) - values = label_info.get('values', {}).keys() - if not values: - return [] - max_value = max(int(v) for v in values) - return [v for v in label_info.get('all', []) - if v.get('value', 0) == max_value] + def _GetApproversForLabel(self, issue, label): + change_info = self.GetChangeInfo(issue) + label_info = change_info.get('labels', {}).get(label, {}) + values = label_info.get('values', {}).keys() + if not values: + return [] + max_value = max(int(v) for v in values) + return [ + v for v in label_info.get('all', []) + if v.get('value', 0) == max_value + ] - def IsBotCommitApproved(self, issue): - return bool(self._GetApproversForLabel(issue, 'Bot-Commit')) + def IsBotCommitApproved(self, issue): + return bool(self._GetApproversForLabel(issue, 'Bot-Commit')) - def IsOwnersOverrideApproved(self, issue): - return bool(self._GetApproversForLabel(issue, 'Owners-Override')) + def IsOwnersOverrideApproved(self, issue): + return bool(self._GetApproversForLabel(issue, 'Owners-Override')) - def GetChangeOwner(self, issue): - return self.GetChangeInfo(issue)['owner']['email'] + def GetChangeOwner(self, issue): + return self.GetChangeInfo(issue)['owner']['email'] - def GetChangeReviewers(self, issue, approving_only=True): - changeinfo = self.GetChangeInfo(issue) - if approving_only: - reviewers = self._GetApproversForLabel(issue, 'Code-Review') - else: - reviewers = changeinfo.get('reviewers', {}).get('REVIEWER', []) - return [r.get('email') for r in reviewers] + def GetChangeReviewers(self, issue, approving_only=True): + changeinfo = self.GetChangeInfo(issue) + if approving_only: + reviewers = self._GetApproversForLabel(issue, 'Code-Review') + else: + reviewers = changeinfo.get('reviewers', {}).get('REVIEWER', []) + return [r.get('email') for r in reviewers] - def UpdateDescription(self, description, issue): - gerrit_util.SetCommitMessage(self.host, issue, description, notify='NONE') + def UpdateDescription(self, description, issue): + gerrit_util.SetCommitMessage(self.host, + issue, + description, + notify='NONE') - def IsCodeOwnersEnabledOnRepo(self): - if self.code_owners_enabled is None: - self.code_owners_enabled = gerrit_util.IsCodeOwnersEnabledOnRepo( - self.host, self.project) - return self.code_owners_enabled + def IsCodeOwnersEnabledOnRepo(self): + if self.code_owners_enabled is None: + self.code_owners_enabled = gerrit_util.IsCodeOwnersEnabledOnRepo( + self.host, self.project) + return self.code_owners_enabled class OutputApi(object): - """An instance of OutputApi gets passed to presubmit scripts so that they + """An instance of OutputApi gets passed to presubmit scripts so that they can output various types of results. """ - PresubmitResult = _PresubmitResult - PresubmitError = _PresubmitError - PresubmitPromptWarning = _PresubmitPromptWarning - PresubmitNotifyResult = _PresubmitNotifyResult - MailTextResult = _MailTextResult + PresubmitResult = _PresubmitResult + PresubmitError = _PresubmitError + PresubmitPromptWarning = _PresubmitPromptWarning + PresubmitNotifyResult = _PresubmitNotifyResult + MailTextResult = _MailTextResult - def __init__(self, is_committing): - self.is_committing = is_committing - self.more_cc = [] + def __init__(self, is_committing): + self.is_committing = is_committing + self.more_cc = [] - def AppendCC(self, cc): - """Appends a user to cc for this change.""" - self.more_cc.append(cc) + def AppendCC(self, cc): + """Appends a user to cc for this change.""" + self.more_cc.append(cc) - def PresubmitPromptOrNotify(self, *args, **kwargs): - """Warn the user when uploading, but only notify if committing.""" - if self.is_committing: - return self.PresubmitNotifyResult(*args, **kwargs) - return self.PresubmitPromptWarning(*args, **kwargs) + def PresubmitPromptOrNotify(self, *args, **kwargs): + """Warn the user when uploading, but only notify if committing.""" + if self.is_committing: + return self.PresubmitNotifyResult(*args, **kwargs) + return self.PresubmitPromptWarning(*args, **kwargs) class InputApi(object): - """An instance of this object is passed to presubmit scripts so they can + """An instance of this object is passed to presubmit scripts so they can know stuff about the change they're looking at. """ - # Method could be a function - # pylint: disable=no-self-use + # Method could be a function + # pylint: disable=no-self-use - # File extensions that are considered source files from a style guide - # perspective. Don't modify this list from a presubmit script! - # - # Files without an extension aren't included in the list. If you want to - # filter them as source files, add r'(^|.*?[\\\/])[^.]+$' to the allow list. - # Note that ALL CAPS files are skipped in DEFAULT_FILES_TO_SKIP below. - DEFAULT_FILES_TO_CHECK = ( - # C++ and friends - r'.+\.c$', r'.+\.cc$', r'.+\.cpp$', r'.+\.h$', r'.+\.m$', r'.+\.mm$', - r'.+\.inl$', r'.+\.asm$', r'.+\.hxx$', r'.+\.hpp$', r'.+\.s$', r'.+\.S$', - # Scripts - r'.+\.js$', r'.+\.ts$', r'.+\.py$', r'.+\.sh$', r'.+\.rb$', r'.+\.pl$', - r'.+\.pm$', - # Other - r'.+\.java$', r'.+\.mk$', r'.+\.am$', r'.+\.css$', r'.+\.mojom$', - r'.+\.fidl$', r'.+\.rs$', - ) + # File extensions that are considered source files from a style guide + # perspective. Don't modify this list from a presubmit script! + # + # Files without an extension aren't included in the list. If you want to + # filter them as source files, add r'(^|.*?[\\\/])[^.]+$' to the allow list. + # Note that ALL CAPS files are skipped in DEFAULT_FILES_TO_SKIP below. + DEFAULT_FILES_TO_CHECK = ( + # C++ and friends + r'.+\.c$', + r'.+\.cc$', + r'.+\.cpp$', + r'.+\.h$', + r'.+\.m$', + r'.+\.mm$', + r'.+\.inl$', + r'.+\.asm$', + r'.+\.hxx$', + r'.+\.hpp$', + r'.+\.s$', + r'.+\.S$', + # Scripts + r'.+\.js$', + r'.+\.ts$', + r'.+\.py$', + r'.+\.sh$', + r'.+\.rb$', + r'.+\.pl$', + r'.+\.pm$', + # Other + r'.+\.java$', + r'.+\.mk$', + r'.+\.am$', + r'.+\.css$', + r'.+\.mojom$', + r'.+\.fidl$', + r'.+\.rs$', + ) - # Path regexp that should be excluded from being considered containing source - # files. Don't modify this list from a presubmit script! - DEFAULT_FILES_TO_SKIP = ( - r'testing_support[\\\/]google_appengine[\\\/].*', - r'.*\bexperimental[\\\/].*', - # Exclude third_party/.* but NOT third_party/{WebKit,blink} - # (crbug.com/539768 and crbug.com/836555). - r'.*\bthird_party[\\\/](?!(WebKit|blink)[\\\/]).*', - # Output directories (just in case) - r'.*\bDebug[\\\/].*', - r'.*\bRelease[\\\/].*', - r'.*\bxcodebuild[\\\/].*', - r'.*\bout[\\\/].*', - # All caps files like README and LICENCE. - r'.*\b[A-Z0-9_]{2,}$', - # SCM (can happen in dual SCM configuration). (Slightly over aggressive) - r'(|.*[\\\/])\.git[\\\/].*', - r'(|.*[\\\/])\.svn[\\\/].*', - # There is no point in processing a patch file. - r'.+\.diff$', - r'.+\.patch$', - ) + # Path regexp that should be excluded from being considered containing + # source files. Don't modify this list from a presubmit script! + DEFAULT_FILES_TO_SKIP = ( + r'testing_support[\\\/]google_appengine[\\\/].*', + r'.*\bexperimental[\\\/].*', + # Exclude third_party/.* but NOT third_party/{WebKit,blink} + # (crbug.com/539768 and crbug.com/836555). + r'.*\bthird_party[\\\/](?!(WebKit|blink)[\\\/]).*', + # Output directories (just in case) + r'.*\bDebug[\\\/].*', + r'.*\bRelease[\\\/].*', + r'.*\bxcodebuild[\\\/].*', + r'.*\bout[\\\/].*', + # All caps files like README and LICENCE. + r'.*\b[A-Z0-9_]{2,}$', + # SCM (can happen in dual SCM configuration). (Slightly over aggressive) + r'(|.*[\\\/])\.git[\\\/].*', + r'(|.*[\\\/])\.svn[\\\/].*', + # There is no point in processing a patch file. + r'.+\.diff$', + r'.+\.patch$', + ) - def __init__(self, change, presubmit_path, is_committing, - verbose, gerrit_obj, dry_run=None, thread_pool=None, parallel=False, - no_diffs=False): - """Builds an InputApi object. + def __init__(self, + change, + presubmit_path, + is_committing, + verbose, + gerrit_obj, + dry_run=None, + thread_pool=None, + parallel=False, + no_diffs=False): + """Builds an InputApi object. Args: change: A presubmit.Change object. @@ -584,157 +622,161 @@ class InputApi(object): no_diffs: if true, implies that --files or --all was specified so some checks can be skipped, and some errors will be messages. """ - # Version number of the presubmit_support script. - self.version = [int(x) for x in __version__.split('.')] - self.change = change - self.is_committing = is_committing - self.gerrit = gerrit_obj - self.dry_run = dry_run - self.no_diffs = no_diffs + # Version number of the presubmit_support script. + self.version = [int(x) for x in __version__.split('.')] + self.change = change + self.is_committing = is_committing + self.gerrit = gerrit_obj + self.dry_run = dry_run + self.no_diffs = no_diffs - self.parallel = parallel - self.thread_pool = thread_pool or ThreadPool() + self.parallel = parallel + self.thread_pool = thread_pool or ThreadPool() - # We expose various modules and functions as attributes of the input_api - # so that presubmit scripts don't have to import them. - self.ast = ast - self.basename = os.path.basename - self.cpplint = cpplint - self.fnmatch = fnmatch - self.gclient_paths = gclient_paths - self.glob = glob.glob - self.json = json - self.logging = logging.getLogger('PRESUBMIT') - self.os_listdir = os.listdir - self.os_path = os.path - self.os_stat = os.stat - self.os_walk = os.walk - self.re = re - self.subprocess = subprocess - self.sys = sys - self.tempfile = tempfile - self.time = time - self.unittest = unittest - self.urllib_request = urllib_request - self.urllib_error = urllib_error + # We expose various modules and functions as attributes of the input_api + # so that presubmit scripts don't have to import them. + self.ast = ast + self.basename = os.path.basename + self.cpplint = cpplint + self.fnmatch = fnmatch + self.gclient_paths = gclient_paths + self.glob = glob.glob + self.json = json + self.logging = logging.getLogger('PRESUBMIT') + self.os_listdir = os.listdir + self.os_path = os.path + self.os_stat = os.stat + self.os_walk = os.walk + self.re = re + self.subprocess = subprocess + self.sys = sys + self.tempfile = tempfile + self.time = time + self.unittest = unittest + self.urllib_request = urllib_request + self.urllib_error = urllib_error - self.is_windows = sys.platform == 'win32' + self.is_windows = sys.platform == 'win32' - # Set python_executable to 'vpython3' in order to allow scripts in other - # repos (e.g. src.git) to automatically pick up that repo's .vpython file, - # instead of inheriting the one in depot_tools. - self.python_executable = 'vpython3' - # Offer a python 3 executable for use during the migration off of python 2. - self.python3_executable = 'vpython3' - self.environ = os.environ + # Set python_executable to 'vpython3' in order to allow scripts in other + # repos (e.g. src.git) to automatically pick up that repo's .vpython + # file, instead of inheriting the one in depot_tools. + self.python_executable = 'vpython3' + # Offer a python 3 executable for use during the migration off of python + # 2. + self.python3_executable = 'vpython3' + self.environ = os.environ - # InputApi.platform is the platform you're currently running on. - self.platform = sys.platform + # InputApi.platform is the platform you're currently running on. + self.platform = sys.platform - self.cpu_count = multiprocessing.cpu_count() - if self.is_windows: - # TODO(crbug.com/1190269) - we can't use more than 56 child processes on - # Windows or Python3 may hang. - self.cpu_count = min(self.cpu_count, 56) + self.cpu_count = multiprocessing.cpu_count() + if self.is_windows: + # TODO(crbug.com/1190269) - we can't use more than 56 child + # processes on Windows or Python3 may hang. + self.cpu_count = min(self.cpu_count, 56) - # The local path of the currently-being-processed presubmit script. - self._current_presubmit_path = os.path.dirname(presubmit_path) + # The local path of the currently-being-processed presubmit script. + self._current_presubmit_path = os.path.dirname(presubmit_path) - # We carry the canned checks so presubmit scripts can easily use them. - self.canned_checks = presubmit_canned_checks + # We carry the canned checks so presubmit scripts can easily use them. + self.canned_checks = presubmit_canned_checks - # Temporary files we must manually remove at the end of a run. - self._named_temporary_files = [] + # Temporary files we must manually remove at the end of a run. + self._named_temporary_files = [] - self.owners_client = None - if self.gerrit and not 'PRESUBMIT_SKIP_NETWORK' in self.environ: - try: - self.owners_client = owners_client.GetCodeOwnersClient( - host=self.gerrit.host, - project=self.gerrit.project, - branch=self.gerrit.branch) - except Exception as e: - print('Failed to set owners_client - %s' % str(e)) - self.owners_finder = owners_finder.OwnersFinder - self.verbose = verbose - self.Command = CommandData + self.owners_client = None + if self.gerrit and not 'PRESUBMIT_SKIP_NETWORK' in self.environ: + try: + self.owners_client = owners_client.GetCodeOwnersClient( + host=self.gerrit.host, + project=self.gerrit.project, + branch=self.gerrit.branch) + except Exception as e: + print('Failed to set owners_client - %s' % str(e)) + self.owners_finder = owners_finder.OwnersFinder + self.verbose = verbose + self.Command = CommandData - # Replace and as headers that need to be included - # with 'base/containers/hash_tables.h' instead. - # Access to a protected member _XX of a client class - # pylint: disable=protected-access - self.cpplint._re_pattern_templates = [ - (a, b, 'base/containers/hash_tables.h') - if header in ('', '') else (a, b, header) - for (a, b, header) in cpplint._re_pattern_templates - ] + # Replace and as headers that need to be included + # with 'base/containers/hash_tables.h' instead. + # Access to a protected member _XX of a client class + # pylint: disable=protected-access + self.cpplint._re_pattern_templates = [ + (a, b, + 'base/containers/hash_tables.h') if header in ('', + '') else + (a, b, header) for (a, b, header) in cpplint._re_pattern_templates + ] - def SetTimeout(self, timeout): - self.thread_pool.timeout = timeout + def SetTimeout(self, timeout): + self.thread_pool.timeout = timeout - def PresubmitLocalPath(self): - """Returns the local path of the presubmit script currently being run. + def PresubmitLocalPath(self): + """Returns the local path of the presubmit script currently being run. This is useful if you don't want to hard-code absolute paths in the presubmit script. For example, It can be used to find another file relative to the PRESUBMIT.py script, so the whole tree can be branched and the presubmit script still works, without editing its content. """ - return self._current_presubmit_path + return self._current_presubmit_path - def AffectedFiles(self, include_deletes=True, file_filter=None): - """Same as input_api.change.AffectedFiles() except only lists files + def AffectedFiles(self, include_deletes=True, file_filter=None): + """Same as input_api.change.AffectedFiles() except only lists files (and optionally directories) in the same directory as the current presubmit script, or subdirectories thereof. Note that files are listed using the OS path separator, so backslashes are used as separators on Windows. """ - dir_with_slash = normpath(self.PresubmitLocalPath()) - # normpath strips trailing path separators, so the trailing separator has to - # be added after the normpath call. - if len(dir_with_slash) > 0: - dir_with_slash += os.path.sep + dir_with_slash = normpath(self.PresubmitLocalPath()) + # normpath strips trailing path separators, so the trailing separator + # has to be added after the normpath call. + if len(dir_with_slash) > 0: + dir_with_slash += os.path.sep - return list(filter( - lambda x: normpath(x.AbsoluteLocalPath()).startswith(dir_with_slash), - self.change.AffectedFiles(include_deletes, file_filter))) + return list( + filter( + lambda x: normpath(x.AbsoluteLocalPath()).startswith( + dir_with_slash), + self.change.AffectedFiles(include_deletes, file_filter))) - def LocalPaths(self): - """Returns local paths of input_api.AffectedFiles().""" - paths = [af.LocalPath() for af in self.AffectedFiles()] - logging.debug('LocalPaths: %s', paths) - return paths + def LocalPaths(self): + """Returns local paths of input_api.AffectedFiles().""" + paths = [af.LocalPath() for af in self.AffectedFiles()] + logging.debug('LocalPaths: %s', paths) + return paths - def AbsoluteLocalPaths(self): - """Returns absolute local paths of input_api.AffectedFiles().""" - return [af.AbsoluteLocalPath() for af in self.AffectedFiles()] + def AbsoluteLocalPaths(self): + """Returns absolute local paths of input_api.AffectedFiles().""" + return [af.AbsoluteLocalPath() for af in self.AffectedFiles()] - def AffectedTestableFiles(self, include_deletes=None, **kwargs): - """Same as input_api.change.AffectedTestableFiles() except only lists files + def AffectedTestableFiles(self, include_deletes=None, **kwargs): + """Same as input_api.change.AffectedTestableFiles() except only lists files in the same directory as the current presubmit script, or subdirectories thereof. """ - if include_deletes is not None: - warn('AffectedTestableFiles(include_deletes=%s)' - ' is deprecated and ignored' % str(include_deletes), - category=DeprecationWarning, - stacklevel=2) - # pylint: disable=consider-using-generator - return [ - x for x in self.AffectedFiles(include_deletes=False, **kwargs) - if x.IsTestableFile() - ] + if include_deletes is not None: + warn('AffectedTestableFiles(include_deletes=%s)' + ' is deprecated and ignored' % str(include_deletes), + category=DeprecationWarning, + stacklevel=2) + # pylint: disable=consider-using-generator + return [ + x for x in self.AffectedFiles(include_deletes=False, **kwargs) + if x.IsTestableFile() + ] - def AffectedTextFiles(self, include_deletes=None): - """An alias to AffectedTestableFiles for backwards compatibility.""" - return self.AffectedTestableFiles(include_deletes=include_deletes) + def AffectedTextFiles(self, include_deletes=None): + """An alias to AffectedTestableFiles for backwards compatibility.""" + return self.AffectedTestableFiles(include_deletes=include_deletes) - def FilterSourceFile(self, - affected_file, - files_to_check=None, - files_to_skip=None, - allow_list=None, - block_list=None): - """Filters out files that aren't considered 'source file'. + def FilterSourceFile(self, + affected_file, + files_to_check=None, + files_to_skip=None, + allow_list=None, + block_list=None): + """Filters out files that aren't considered 'source file'. If files_to_check or files_to_skip is None, InputApi.DEFAULT_FILES_TO_CHECK and InputApi.DEFAULT_FILES_TO_SKIP is used respectively. @@ -746,36 +788,37 @@ class InputApi(object): Note: Copy-paste this function to suit your needs or use a lambda function. """ - if files_to_check is None: - files_to_check = self.DEFAULT_FILES_TO_CHECK - if files_to_skip is None: - files_to_skip = self.DEFAULT_FILES_TO_SKIP + if files_to_check is None: + files_to_check = self.DEFAULT_FILES_TO_CHECK + if files_to_skip is None: + files_to_skip = self.DEFAULT_FILES_TO_SKIP - def Find(affected_file, items): - local_path = affected_file.LocalPath() - for item in items: - if self.re.match(item, local_path): - return True - # Handle the cases where the files regex only handles /, but the local - # path uses \. - if self.is_windows and self.re.match(item, local_path.replace( - '\\', '/')): - return True - return False - return (Find(affected_file, files_to_check) and - not Find(affected_file, files_to_skip)) + def Find(affected_file, items): + local_path = affected_file.LocalPath() + for item in items: + if self.re.match(item, local_path): + return True + # Handle the cases where the files regex only handles /, but the + # local path uses \. + if self.is_windows and self.re.match( + item, local_path.replace('\\', '/')): + return True + return False - def AffectedSourceFiles(self, source_file): - """Filter the list of AffectedTestableFiles by the function source_file. + return (Find(affected_file, files_to_check) + and not Find(affected_file, files_to_skip)) + + def AffectedSourceFiles(self, source_file): + """Filter the list of AffectedTestableFiles by the function source_file. If source_file is None, InputApi.FilterSourceFile() is used. """ - if not source_file: - source_file = self.FilterSourceFile - return list(filter(source_file, self.AffectedTestableFiles())) + if not source_file: + source_file = self.FilterSourceFile + return list(filter(source_file, self.AffectedTestableFiles())) - def RightHandSideLines(self, source_file_filter=None): - """An iterator over all text lines in 'new' version of changed files. + def RightHandSideLines(self, source_file_filter=None): + """An iterator over all text lines in 'new' version of changed files. Only lists lines from new or modified text files in the change that are contained by the directory of the currently executing presubmit script. @@ -791,22 +834,22 @@ class InputApi(object): Note: The carriage return (LF or CR) is stripped off. """ - files = self.AffectedSourceFiles(source_file_filter) - return _RightHandSideLinesImpl(files) + files = self.AffectedSourceFiles(source_file_filter) + return _RightHandSideLinesImpl(files) - def ReadFile(self, file_item, mode='r'): - """Reads an arbitrary file. + def ReadFile(self, file_item, mode='r'): + """Reads an arbitrary file. Deny reading anything outside the repository. """ - if isinstance(file_item, AffectedFile): - file_item = file_item.AbsoluteLocalPath() - if not file_item.startswith(self.change.RepositoryRoot()): - raise IOError('Access outside the repository root is denied.') - return gclient_utils.FileRead(file_item, mode) + if isinstance(file_item, AffectedFile): + file_item = file_item.AbsoluteLocalPath() + if not file_item.startswith(self.change.RepositoryRoot()): + raise IOError('Access outside the repository root is denied.') + return gclient_utils.FileRead(file_item, mode) - def CreateTemporaryFile(self, **kwargs): - """Returns a named temporary file that must be removed with a call to + def CreateTemporaryFile(self, **kwargs): + """Returns a named temporary file that must be removed with a call to RemoveTemporaryFiles(). All keyword arguments are forwarded to tempfile.NamedTemporaryFile(), @@ -826,158 +869,163 @@ class InputApi(object): any temporary file; this is done transparently by the presubmit handling code. """ - if 'delete' in kwargs: - # Prevent users from passing |delete|; we take care of file deletion - # ourselves and this prevents unintuitive error messages when we pass - # delete=False and 'delete' is also in kwargs. - raise TypeError('CreateTemporaryFile() does not take a "delete" ' - 'argument, file deletion is handled automatically by ' - 'the same presubmit_support code that creates InputApi ' - 'objects.') - temp_file = self.tempfile.NamedTemporaryFile(delete=False, **kwargs) - self._named_temporary_files.append(temp_file.name) - return temp_file + if 'delete' in kwargs: + # Prevent users from passing |delete|; we take care of file deletion + # ourselves and this prevents unintuitive error messages when we + # pass delete=False and 'delete' is also in kwargs. + raise TypeError( + 'CreateTemporaryFile() does not take a "delete" ' + 'argument, file deletion is handled automatically by ' + 'the same presubmit_support code that creates InputApi ' + 'objects.') + temp_file = self.tempfile.NamedTemporaryFile(delete=False, **kwargs) + self._named_temporary_files.append(temp_file.name) + return temp_file - @property - def tbr(self): - """Returns if a change is TBR'ed.""" - return 'TBR' in self.change.tags or self.change.TBRsFromDescription() + @property + def tbr(self): + """Returns if a change is TBR'ed.""" + return 'TBR' in self.change.tags or self.change.TBRsFromDescription() - def RunTests(self, tests_mix, parallel=True): - tests = [] - msgs = [] - for t in tests_mix: - if isinstance(t, OutputApi.PresubmitResult) and t: - msgs.append(t) - else: - assert issubclass(t.message, _PresubmitResult) - tests.append(t) - if self.verbose: - t.info = _PresubmitNotifyResult - if not t.kwargs.get('cwd'): - t.kwargs['cwd'] = self.PresubmitLocalPath() - self.thread_pool.AddTests(tests, parallel) - # When self.parallel is True (i.e. --parallel is passed as an option) - # RunTests doesn't actually run tests. It adds them to a ThreadPool that - # will run all tests once all PRESUBMIT files are processed. - # Otherwise, it will run them and return the results. - if not self.parallel: - msgs.extend(self.thread_pool.RunAsync()) - return msgs + def RunTests(self, tests_mix, parallel=True): + tests = [] + msgs = [] + for t in tests_mix: + if isinstance(t, OutputApi.PresubmitResult) and t: + msgs.append(t) + else: + assert issubclass(t.message, _PresubmitResult) + tests.append(t) + if self.verbose: + t.info = _PresubmitNotifyResult + if not t.kwargs.get('cwd'): + t.kwargs['cwd'] = self.PresubmitLocalPath() + self.thread_pool.AddTests(tests, parallel) + # When self.parallel is True (i.e. --parallel is passed as an option) + # RunTests doesn't actually run tests. It adds them to a ThreadPool that + # will run all tests once all PRESUBMIT files are processed. + # Otherwise, it will run them and return the results. + if not self.parallel: + msgs.extend(self.thread_pool.RunAsync()) + return msgs class _DiffCache(object): - """Caches diffs retrieved from a particular SCM.""" - def __init__(self, upstream=None): - """Stores the upstream revision against which all diffs will be computed.""" - self._upstream = upstream + """Caches diffs retrieved from a particular SCM.""" + def __init__(self, upstream=None): + """Stores the upstream revision against which all diffs will be computed.""" + self._upstream = upstream - def GetDiff(self, path, local_root): - """Get the diff for a particular path.""" - raise NotImplementedError() + def GetDiff(self, path, local_root): + """Get the diff for a particular path.""" + raise NotImplementedError() - def GetOldContents(self, path, local_root): - """Get the old version for a particular path.""" - raise NotImplementedError() + def GetOldContents(self, path, local_root): + """Get the old version for a particular path.""" + raise NotImplementedError() class _GitDiffCache(_DiffCache): - """DiffCache implementation for git; gets all file diffs at once.""" - def __init__(self, upstream): - super(_GitDiffCache, self).__init__(upstream=upstream) - self._diffs_by_file = None + """DiffCache implementation for git; gets all file diffs at once.""" + def __init__(self, upstream): + super(_GitDiffCache, self).__init__(upstream=upstream) + self._diffs_by_file = None - def GetDiff(self, path, local_root): - # Compare against None to distinguish between None and an initialized but - # empty dictionary. - if self._diffs_by_file == None: - # Compute a single diff for all files and parse the output; should - # with git this is much faster than computing one diff for each file. - diffs = {} + def GetDiff(self, path, local_root): + # Compare against None to distinguish between None and an initialized + # but empty dictionary. + if self._diffs_by_file == None: + # Compute a single diff for all files and parse the output; should + # with git this is much faster than computing one diff for each + # file. + diffs = {} - # Don't specify any filenames below, because there are command line length - # limits on some platforms and GenerateDiff would fail. - unified_diff = scm.GIT.GenerateDiff(local_root, files=[], full_move=True, - branch=self._upstream) + # Don't specify any filenames below, because there are command line + # length limits on some platforms and GenerateDiff would fail. + unified_diff = scm.GIT.GenerateDiff(local_root, + files=[], + full_move=True, + branch=self._upstream) - # This regex matches the path twice, separated by a space. Note that - # filename itself may contain spaces. - file_marker = re.compile('^diff --git (?P.*) (?P=filename)$') - current_diff = [] - keep_line_endings = True - for x in unified_diff.splitlines(keep_line_endings): - match = file_marker.match(x) - if match: - # Marks the start of a new per-file section. - diffs[match.group('filename')] = current_diff = [x] - elif x.startswith('diff --git'): - raise PresubmitFailure('Unexpected diff line: %s' % x) - else: - current_diff.append(x) + # This regex matches the path twice, separated by a space. Note that + # filename itself may contain spaces. + file_marker = re.compile( + '^diff --git (?P.*) (?P=filename)$') + current_diff = [] + keep_line_endings = True + for x in unified_diff.splitlines(keep_line_endings): + match = file_marker.match(x) + if match: + # Marks the start of a new per-file section. + diffs[match.group('filename')] = current_diff = [x] + elif x.startswith('diff --git'): + raise PresubmitFailure('Unexpected diff line: %s' % x) + else: + current_diff.append(x) - self._diffs_by_file = dict( - (normpath(path), ''.join(diff)) for path, diff in diffs.items()) + self._diffs_by_file = dict( + (normpath(path), ''.join(diff)) for path, diff in diffs.items()) - if path not in self._diffs_by_file: - # SCM didn't have any diff on this file. It could be that the file was not - # modified at all (e.g. user used --all flag in git cl presubmit). - # Intead of failing, return empty string. - # See: https://crbug.com/808346. - return '' + if path not in self._diffs_by_file: + # SCM didn't have any diff on this file. It could be that the file + # was not modified at all (e.g. user used --all flag in git cl + # presubmit). Intead of failing, return empty string. See: + # https://crbug.com/808346. + return '' - return self._diffs_by_file[path] + return self._diffs_by_file[path] - def GetOldContents(self, path, local_root): - return scm.GIT.GetOldContents(local_root, path, branch=self._upstream) + def GetOldContents(self, path, local_root): + return scm.GIT.GetOldContents(local_root, path, branch=self._upstream) class AffectedFile(object): - """Representation of a file in a change.""" + """Representation of a file in a change.""" - DIFF_CACHE = _DiffCache + DIFF_CACHE = _DiffCache - # Method could be a function - # pylint: disable=no-self-use - def __init__(self, path, action, repository_root, diff_cache): - self._path = path - self._action = action - self._local_root = repository_root - self._is_directory = None - self._cached_changed_contents = None - self._cached_new_contents = None - self._diff_cache = diff_cache - logging.debug('%s(%s)', self.__class__.__name__, self._path) + # Method could be a function + # pylint: disable=no-self-use + def __init__(self, path, action, repository_root, diff_cache): + self._path = path + self._action = action + self._local_root = repository_root + self._is_directory = None + self._cached_changed_contents = None + self._cached_new_contents = None + self._diff_cache = diff_cache + logging.debug('%s(%s)', self.__class__.__name__, self._path) - def LocalPath(self): - """Returns the path of this file on the local disk relative to client root. + def LocalPath(self): + """Returns the path of this file on the local disk relative to client root. This should be used for error messages but not for accessing files, because presubmit checks are run with CWD=PresubmitLocalPath() (which is often != client root). """ - return normpath(self._path) + return normpath(self._path) - def AbsoluteLocalPath(self): - """Returns the absolute path of this file on the local disk. + def AbsoluteLocalPath(self): + """Returns the absolute path of this file on the local disk. """ - return os.path.abspath(os.path.join(self._local_root, self.LocalPath())) + return os.path.abspath(os.path.join(self._local_root, self.LocalPath())) - def Action(self): - """Returns the action on this opened file, e.g. A, M, D, etc.""" - return self._action + def Action(self): + """Returns the action on this opened file, e.g. A, M, D, etc.""" + return self._action - def IsTestableFile(self): - """Returns True if the file is a text file and not a binary file. + def IsTestableFile(self): + """Returns True if the file is a text file and not a binary file. Deleted files are not text file.""" - raise NotImplementedError() # Implement when needed + raise NotImplementedError() # Implement when needed - def IsTextFile(self): - """An alias to IsTestableFile for backwards compatibility.""" - return self.IsTestableFile() + def IsTextFile(self): + """An alias to IsTestableFile for backwards compatibility.""" + return self.IsTestableFile() - def OldContents(self): - """Returns an iterator over the lines in the old version of file. + def OldContents(self): + """Returns an iterator over the lines in the old version of file. The old version is the file before any modifications in the user's workspace, i.e. the 'left hand side'. @@ -985,11 +1033,11 @@ class AffectedFile(object): Contents will be empty if the file is a directory or does not exist. Note: The carriage returns (LF or CR) are stripped off. """ - return self._diff_cache.GetOldContents(self.LocalPath(), - self._local_root).splitlines() + return self._diff_cache.GetOldContents(self.LocalPath(), + self._local_root).splitlines() - def NewContents(self): - """Returns an iterator over the lines in the new version of file. + def NewContents(self): + """Returns an iterator over the lines in the new version of file. The new version is the file in the user's workspace, i.e. the 'right hand side'. @@ -997,83 +1045,84 @@ class AffectedFile(object): Contents will be empty if the file is a directory or does not exist. Note: The carriage returns (LF or CR) are stripped off. """ - if self._cached_new_contents is None: - self._cached_new_contents = [] - try: - self._cached_new_contents = gclient_utils.FileRead( - self.AbsoluteLocalPath(), 'rU').splitlines() - except IOError: - pass # File not found? That's fine; maybe it was deleted. - except UnicodeDecodeError as e: - # log the filename since we're probably trying to read a binary - # file, and shouldn't be. - print('Error reading %s: %s' % (self.AbsoluteLocalPath(), e)) - raise + if self._cached_new_contents is None: + self._cached_new_contents = [] + try: + self._cached_new_contents = gclient_utils.FileRead( + self.AbsoluteLocalPath(), 'rU').splitlines() + except IOError: + pass # File not found? That's fine; maybe it was deleted. + except UnicodeDecodeError as e: + # log the filename since we're probably trying to read a binary + # file, and shouldn't be. + print('Error reading %s: %s' % (self.AbsoluteLocalPath(), e)) + raise - return self._cached_new_contents[:] + return self._cached_new_contents[:] - def ChangedContents(self, keeplinebreaks=False): - """Returns a list of tuples (line number, line text) of all new lines. + def ChangedContents(self, keeplinebreaks=False): + """Returns a list of tuples (line number, line text) of all new lines. This relies on the scm diff output describing each changed code section with a line of the form ^@@ , , @@$ """ - # Don't return cached results when line breaks are requested. - if not keeplinebreaks and self._cached_changed_contents is not None: - return self._cached_changed_contents[:] - result = [] - line_num = 0 + # Don't return cached results when line breaks are requested. + if not keeplinebreaks and self._cached_changed_contents is not None: + return self._cached_changed_contents[:] + result = [] + line_num = 0 - # The keeplinebreaks parameter to splitlines must be True or else the - # CheckForWindowsLineEndings presubmit will be a NOP. - for line in self.GenerateScmDiff().splitlines(keeplinebreaks): - m = re.match(r'^@@ [0-9\,\+\-]+ \+([0-9]+)\,[0-9]+ @@', line) - if m: - line_num = int(m.groups(1)[0]) - continue - if line.startswith('+') and not line.startswith('++'): - result.append((line_num, line[1:])) - if not line.startswith('-'): - line_num += 1 - # Don't cache results with line breaks. - if keeplinebreaks: - return result; - self._cached_changed_contents = result - return self._cached_changed_contents[:] + # The keeplinebreaks parameter to splitlines must be True or else the + # CheckForWindowsLineEndings presubmit will be a NOP. + for line in self.GenerateScmDiff().splitlines(keeplinebreaks): + m = re.match(r'^@@ [0-9\,\+\-]+ \+([0-9]+)\,[0-9]+ @@', line) + if m: + line_num = int(m.groups(1)[0]) + continue + if line.startswith('+') and not line.startswith('++'): + result.append((line_num, line[1:])) + if not line.startswith('-'): + line_num += 1 + # Don't cache results with line breaks. + if keeplinebreaks: + return result + self._cached_changed_contents = result + return self._cached_changed_contents[:] - def __str__(self): - return self.LocalPath() + def __str__(self): + return self.LocalPath() - def GenerateScmDiff(self): - return self._diff_cache.GetDiff(self.LocalPath(), self._local_root) + def GenerateScmDiff(self): + return self._diff_cache.GetDiff(self.LocalPath(), self._local_root) class GitAffectedFile(AffectedFile): - """Representation of a file in a change out of a git checkout.""" - # Method 'NNN' is abstract in class 'NNN' but is not overridden - # pylint: disable=abstract-method + """Representation of a file in a change out of a git checkout.""" + # Method 'NNN' is abstract in class 'NNN' but is not overridden + # pylint: disable=abstract-method - DIFF_CACHE = _GitDiffCache + DIFF_CACHE = _GitDiffCache - def __init__(self, *args, **kwargs): - AffectedFile.__init__(self, *args, **kwargs) - self._server_path = None - self._is_testable_file = None + def __init__(self, *args, **kwargs): + AffectedFile.__init__(self, *args, **kwargs) + self._server_path = None + self._is_testable_file = None - def IsTestableFile(self): - if self._is_testable_file is None: - if self.Action() == 'D': - # A deleted file is not testable. - self._is_testable_file = False - else: - self._is_testable_file = os.path.isfile(self.AbsoluteLocalPath()) - return self._is_testable_file + def IsTestableFile(self): + if self._is_testable_file is None: + if self.Action() == 'D': + # A deleted file is not testable. + self._is_testable_file = False + else: + self._is_testable_file = os.path.isfile( + self.AbsoluteLocalPath()) + return self._is_testable_file class Change(object): - """Describe a change. + """Describe a change. Used directly by the presubmit scripts to query the current change being tested. @@ -1083,84 +1132,90 @@ class Change(object): self.KEY: equivalent to tags['KEY'] """ - _AFFECTED_FILES = AffectedFile + _AFFECTED_FILES = AffectedFile - # Matches key/value (or 'tag') lines in changelist descriptions. - TAG_LINE_RE = re.compile( - '^[ \t]*(?P[A-Z][A-Z_0-9]*)[ \t]*=[ \t]*(?P.*?)[ \t]*$') - scm = '' + # Matches key/value (or 'tag') lines in changelist descriptions. + TAG_LINE_RE = re.compile( + '^[ \t]*(?P[A-Z][A-Z_0-9]*)[ \t]*=[ \t]*(?P.*?)[ \t]*$') + scm = '' - def __init__( - self, name, description, local_root, files, issue, patchset, author, - upstream=None): - if files is None: - files = [] - self._name = name - # Convert root into an absolute path. - self._local_root = os.path.abspath(local_root) - self._upstream = upstream - self.issue = issue - self.patchset = patchset - self.author_email = author + def __init__(self, + name, + description, + local_root, + files, + issue, + patchset, + author, + upstream=None): + if files is None: + files = [] + self._name = name + # Convert root into an absolute path. + self._local_root = os.path.abspath(local_root) + self._upstream = upstream + self.issue = issue + self.patchset = patchset + self.author_email = author - self._full_description = '' - self.tags = {} - self._description_without_tags = '' - self.SetDescriptionText(description) + self._full_description = '' + self.tags = {} + self._description_without_tags = '' + self.SetDescriptionText(description) - assert all( - (isinstance(f, (list, tuple)) and len(f) == 2) for f in files), files + assert all((isinstance(f, (list, tuple)) and len(f) == 2) + for f in files), files - diff_cache = self._AFFECTED_FILES.DIFF_CACHE(self._upstream) - self._affected_files = [ - self._AFFECTED_FILES(path, action.strip(), self._local_root, diff_cache) - for action, path in files - ] + diff_cache = self._AFFECTED_FILES.DIFF_CACHE(self._upstream) + self._affected_files = [ + self._AFFECTED_FILES(path, action.strip(), self._local_root, + diff_cache) for action, path in files + ] - def UpstreamBranch(self): - """Returns the upstream branch for the change.""" - return self._upstream + def UpstreamBranch(self): + """Returns the upstream branch for the change.""" + return self._upstream - def Name(self): - """Returns the change name.""" - return self._name + def Name(self): + """Returns the change name.""" + return self._name - def DescriptionText(self): - """Returns the user-entered changelist description, minus tags. + def DescriptionText(self): + """Returns the user-entered changelist description, minus tags. Any line in the user-provided description starting with e.g. 'FOO=' (whitespace permitted before and around) is considered a tag line. Such lines are stripped out of the description this function returns. """ - return self._description_without_tags + return self._description_without_tags - def FullDescriptionText(self): - """Returns the complete changelist description including tags.""" - return self._full_description + def FullDescriptionText(self): + """Returns the complete changelist description including tags.""" + return self._full_description - def SetDescriptionText(self, description): - """Sets the full description text (including tags) to |description|. + def SetDescriptionText(self, description): + """Sets the full description text (including tags) to |description|. Also updates the list of tags.""" - self._full_description = description + self._full_description = description - # From the description text, build up a dictionary of key/value pairs - # plus the description minus all key/value or 'tag' lines. - description_without_tags = [] - self.tags = {} - for line in self._full_description.splitlines(): - m = self.TAG_LINE_RE.match(line) - if m: - self.tags[m.group('key')] = m.group('value') - else: - description_without_tags.append(line) + # From the description text, build up a dictionary of key/value pairs + # plus the description minus all key/value or 'tag' lines. + description_without_tags = [] + self.tags = {} + for line in self._full_description.splitlines(): + m = self.TAG_LINE_RE.match(line) + if m: + self.tags[m.group('key')] = m.group('value') + else: + description_without_tags.append(line) - # Change back to text and remove whitespace at end. - self._description_without_tags = ( - '\n'.join(description_without_tags).rstrip()) + # Change back to text and remove whitespace at end. + self._description_without_tags = ( + '\n'.join(description_without_tags).rstrip()) - def AddDescriptionFooter(self, key, value): - """Adds the given footer to the change description. + def AddDescriptionFooter(self, key, value): + """Adds the given footer to the change description. Args: key: A string with the key for the git footer. It must conform to @@ -1168,79 +1223,86 @@ class Change(object): normalized so that each token is title-cased. value: A string with the value for the git footer. """ - description = git_footers.add_footer( - self.FullDescriptionText(), git_footers.normalize_name(key), value) - self.SetDescriptionText(description) + description = git_footers.add_footer(self.FullDescriptionText(), + git_footers.normalize_name(key), + value) + self.SetDescriptionText(description) - def RepositoryRoot(self): - """Returns the repository (checkout) root directory for this change, + def RepositoryRoot(self): + """Returns the repository (checkout) root directory for this change, as an absolute path. """ - return self._local_root + return self._local_root - def __getattr__(self, attr): - """Return tags directly as attributes on the object.""" - if not re.match(r'^[A-Z_]*$', attr): - raise AttributeError(self, attr) - return self.tags.get(attr) + def __getattr__(self, attr): + """Return tags directly as attributes on the object.""" + if not re.match(r'^[A-Z_]*$', attr): + raise AttributeError(self, attr) + return self.tags.get(attr) - def GitFootersFromDescription(self): - """Return the git footers present in the description. + def GitFootersFromDescription(self): + """Return the git footers present in the description. Returns: footers: A dict of {footer: [values]} containing a multimap of the footers in the change description. """ - return git_footers.parse_footers(self.FullDescriptionText()) + return git_footers.parse_footers(self.FullDescriptionText()) - def BugsFromDescription(self): - """Returns all bugs referenced in the commit description.""" - bug_tags = ['BUG', 'FIXED'] + def BugsFromDescription(self): + """Returns all bugs referenced in the commit description.""" + bug_tags = ['BUG', 'FIXED'] - tags = [] - for tag in bug_tags: - values = self.tags.get(tag) - if values: - tags += [value.strip() for value in values.split(',')] + tags = [] + for tag in bug_tags: + values = self.tags.get(tag) + if values: + tags += [value.strip() for value in values.split(',')] - footers = [] - parsed = self.GitFootersFromDescription() - unsplit_footers = parsed.get('Bug', []) + parsed.get('Fixed', []) - for unsplit_footer in unsplit_footers: - footers += [b.strip() for b in unsplit_footer.split(',')] - return sorted(set(tags + footers)) + footers = [] + parsed = self.GitFootersFromDescription() + unsplit_footers = parsed.get('Bug', []) + parsed.get('Fixed', []) + for unsplit_footer in unsplit_footers: + footers += [b.strip() for b in unsplit_footer.split(',')] + return sorted(set(tags + footers)) - def ReviewersFromDescription(self): - """Returns all reviewers listed in the commit description.""" - # We don't support a 'R:' git-footer for reviewers; that is in metadata. - tags = [r.strip() for r in self.tags.get('R', '').split(',') if r.strip()] - return sorted(set(tags)) + def ReviewersFromDescription(self): + """Returns all reviewers listed in the commit description.""" + # We don't support a 'R:' git-footer for reviewers; that is in metadata. + tags = [ + r.strip() for r in self.tags.get('R', '').split(',') if r.strip() + ] + return sorted(set(tags)) - def TBRsFromDescription(self): - """Returns all TBR reviewers listed in the commit description.""" - tags = [r.strip() for r in self.tags.get('TBR', '').split(',') if r.strip()] - # TODO(crbug.com/839208): Remove support for 'Tbr:' when TBRs are - # programmatically determined by self-CR+1s. - footers = self.GitFootersFromDescription().get('Tbr', []) - return sorted(set(tags + footers)) + def TBRsFromDescription(self): + """Returns all TBR reviewers listed in the commit description.""" + tags = [ + r.strip() for r in self.tags.get('TBR', '').split(',') if r.strip() + ] + # TODO(crbug.com/839208): Remove support for 'Tbr:' when TBRs are + # programmatically determined by self-CR+1s. + footers = self.GitFootersFromDescription().get('Tbr', []) + return sorted(set(tags + footers)) - # TODO(crbug.com/753425): Delete these once we're sure they're unused. - @property - def BUG(self): - return ','.join(self.BugsFromDescription()) - @property - def R(self): - return ','.join(self.ReviewersFromDescription()) - @property - def TBR(self): - return ','.join(self.TBRsFromDescription()) + # TODO(crbug.com/753425): Delete these once we're sure they're unused. + @property + def BUG(self): + return ','.join(self.BugsFromDescription()) - def AllFiles(self, root=None): - """List all files under source control in the repo.""" - raise NotImplementedError() + @property + def R(self): + return ','.join(self.ReviewersFromDescription()) - def AffectedFiles(self, include_deletes=True, file_filter=None): - """Returns a list of AffectedFile instances for all files in the change. + @property + def TBR(self): + return ','.join(self.TBRsFromDescription()) + + def AllFiles(self, root=None): + """List all files under source control in the repo.""" + raise NotImplementedError() + + def AffectedFiles(self, include_deletes=True, file_filter=None): + """Returns a list of AffectedFile instances for all files in the change. Args: include_deletes: If false, deleted files will be filtered out. @@ -1249,37 +1311,37 @@ class Change(object): Returns: [AffectedFile(path, action), AffectedFile(path, action)] """ - affected = list(filter(file_filter, self._affected_files)) + affected = list(filter(file_filter, self._affected_files)) - if include_deletes: - return affected - return list(filter(lambda x: x.Action() != 'D', affected)) + if include_deletes: + return affected + return list(filter(lambda x: x.Action() != 'D', affected)) - def AffectedTestableFiles(self, include_deletes=None, **kwargs): - """Return a list of the existing text files in a change.""" - if include_deletes is not None: - warn('AffectedTeestableFiles(include_deletes=%s)' - ' is deprecated and ignored' % str(include_deletes), - category=DeprecationWarning, - stacklevel=2) - return list(filter( - lambda x: x.IsTestableFile(), - self.AffectedFiles(include_deletes=False, **kwargs))) + def AffectedTestableFiles(self, include_deletes=None, **kwargs): + """Return a list of the existing text files in a change.""" + if include_deletes is not None: + warn('AffectedTeestableFiles(include_deletes=%s)' + ' is deprecated and ignored' % str(include_deletes), + category=DeprecationWarning, + stacklevel=2) + return list( + filter(lambda x: x.IsTestableFile(), + self.AffectedFiles(include_deletes=False, **kwargs))) - def AffectedTextFiles(self, include_deletes=None): - """An alias to AffectedTestableFiles for backwards compatibility.""" - return self.AffectedTestableFiles(include_deletes=include_deletes) + def AffectedTextFiles(self, include_deletes=None): + """An alias to AffectedTestableFiles for backwards compatibility.""" + return self.AffectedTestableFiles(include_deletes=include_deletes) - def LocalPaths(self): - """Convenience function.""" - return [af.LocalPath() for af in self.AffectedFiles()] + def LocalPaths(self): + """Convenience function.""" + return [af.LocalPath() for af in self.AffectedFiles()] - def AbsoluteLocalPaths(self): - """Convenience function.""" - return [af.AbsoluteLocalPath() for af in self.AffectedFiles()] + def AbsoluteLocalPaths(self): + """Convenience function.""" + return [af.AbsoluteLocalPath() for af in self.AffectedFiles()] - def RightHandSideLines(self): - """An iterator over all text lines in 'new' version of changed files. + def RightHandSideLines(self): + """An iterator over all text lines in 'new' version of changed files. Lists lines from new or modified text files in the change. @@ -1292,32 +1354,33 @@ class Change(object): integer line number (1-based); and the contents of the line as a string. """ - return _RightHandSideLinesImpl( - x for x in self.AffectedFiles(include_deletes=False) - if x.IsTestableFile()) + return _RightHandSideLinesImpl( + x for x in self.AffectedFiles(include_deletes=False) + if x.IsTestableFile()) - def OriginalOwnersFiles(self): - """A map from path names of affected OWNERS files to their old content.""" - def owners_file_filter(f): - return 'OWNERS' in os.path.split(f.LocalPath())[1] - files = self.AffectedFiles(file_filter=owners_file_filter) - return {f.LocalPath(): f.OldContents() for f in files} + def OriginalOwnersFiles(self): + """A map from path names of affected OWNERS files to their old content.""" + def owners_file_filter(f): + return 'OWNERS' in os.path.split(f.LocalPath())[1] + + files = self.AffectedFiles(file_filter=owners_file_filter) + return {f.LocalPath(): f.OldContents() for f in files} class GitChange(Change): - _AFFECTED_FILES = GitAffectedFile - scm = 'git' + _AFFECTED_FILES = GitAffectedFile + scm = 'git' - def AllFiles(self, root=None): - """List all files under source control in the repo.""" - root = root or self.RepositoryRoot() - return subprocess.check_output( - ['git', '-c', 'core.quotePath=false', 'ls-files', '--', '.'], - cwd=root).decode('utf-8', 'ignore').splitlines() + def AllFiles(self, root=None): + """List all files under source control in the repo.""" + root = root or self.RepositoryRoot() + return subprocess.check_output( + ['git', '-c', 'core.quotePath=false', 'ls-files', '--', '.'], + cwd=root).decode('utf-8', 'ignore').splitlines() def ListRelevantPresubmitFiles(files, root): - """Finds all presubmit files that apply to a given set of source files. + """Finds all presubmit files that apply to a given set of source files. If inherit-review-settings-ok is present right under root, looks for PRESUBMIT.py in directories enclosing root. @@ -1329,58 +1392,59 @@ def ListRelevantPresubmitFiles(files, root): Return: List of absolute paths of the existing PRESUBMIT.py scripts. """ - files = [normpath(os.path.join(root, f)) for f in files] + files = [normpath(os.path.join(root, f)) for f in files] - # List all the individual directories containing files. - directories = {os.path.dirname(f) for f in files} + # List all the individual directories containing files. + directories = {os.path.dirname(f) for f in files} - # Ignore root if inherit-review-settings-ok is present. - if os.path.isfile(os.path.join(root, 'inherit-review-settings-ok')): - root = None + # Ignore root if inherit-review-settings-ok is present. + if os.path.isfile(os.path.join(root, 'inherit-review-settings-ok')): + root = None - # Collect all unique directories that may contain PRESUBMIT.py. - candidates = set() - for directory in directories: - while True: - if directory in candidates: - break - candidates.add(directory) - if directory == root: - break - parent_dir = os.path.dirname(directory) - if parent_dir == directory: - # We hit the system root directory. - break - directory = parent_dir + # Collect all unique directories that may contain PRESUBMIT.py. + candidates = set() + for directory in directories: + while True: + if directory in candidates: + break + candidates.add(directory) + if directory == root: + break + parent_dir = os.path.dirname(directory) + if parent_dir == directory: + # We hit the system root directory. + break + directory = parent_dir - # Look for PRESUBMIT.py in all candidate directories. - results = [] - for directory in sorted(list(candidates)): - try: - for f in os.listdir(directory): - p = os.path.join(directory, f) - if os.path.isfile(p) and re.match( - r'PRESUBMIT.*\.py$', f) and not f.startswith('PRESUBMIT_test'): - results.append(p) - except OSError: - pass + # Look for PRESUBMIT.py in all candidate directories. + results = [] + for directory in sorted(list(candidates)): + try: + for f in os.listdir(directory): + p = os.path.join(directory, f) + if os.path.isfile(p) and re.match( + r'PRESUBMIT.*\.py$', + f) and not f.startswith('PRESUBMIT_test'): + results.append(p) + except OSError: + pass - logging.debug('Presubmit files: %s', ','.join(results)) - return results + logging.debug('Presubmit files: %s', ','.join(results)) + return results class GetPostUploadExecuter(object): - def __init__(self, change, gerrit_obj): - """ + def __init__(self, change, gerrit_obj): + """ Args: change: The Change object. gerrit_obj: provides basic Gerrit codereview functionality. """ - self.change = change - self.gerrit = gerrit_obj + self.change = change + self.gerrit = gerrit_obj - def ExecPresubmitScript(self, script_text, presubmit_path): - """Executes PostUploadHook() from a single presubmit script. + def ExecPresubmitScript(self, script_text, presubmit_path): + """Executes PostUploadHook() from a single presubmit script. Caller is responsible for validating whether the hook should be executed and should only call this function if it should be. @@ -1391,97 +1455,107 @@ class GetPostUploadExecuter(object): Return: A list of results objects. """ - # Change to the presubmit file's directory to support local imports. - presubmit_dir = os.path.dirname(presubmit_path) - main_path = os.getcwd() - try: - os.chdir(presubmit_dir) - return self._execute_with_local_working_directory(script_text, - presubmit_dir, - presubmit_path) - finally: - # Return the process to the original working directory. - os.chdir(main_path) + # Change to the presubmit file's directory to support local imports. + presubmit_dir = os.path.dirname(presubmit_path) + main_path = os.getcwd() + try: + os.chdir(presubmit_dir) + return self._execute_with_local_working_directory( + script_text, presubmit_dir, presubmit_path) + finally: + # Return the process to the original working directory. + os.chdir(main_path) - def _execute_with_local_working_directory(self, script_text, presubmit_dir, - presubmit_path): - context = {} - try: - exec(compile(script_text, presubmit_path, 'exec', dont_inherit=True), - context) - except Exception as e: - raise PresubmitFailure('"%s" had an exception.\n%s' - % (presubmit_path, e)) + def _execute_with_local_working_directory(self, script_text, presubmit_dir, + presubmit_path): + context = {} + try: + exec( + compile(script_text, presubmit_path, 'exec', dont_inherit=True), + context) + except Exception as e: + raise PresubmitFailure('"%s" had an exception.\n%s' % + (presubmit_path, e)) - function_name = 'PostUploadHook' - if function_name not in context: - return {} - post_upload_hook = context[function_name] - if not len(inspect.getfullargspec(post_upload_hook)[0]) == 3: - raise PresubmitFailure( - 'Expected function "PostUploadHook" to take three arguments.') - return post_upload_hook(self.gerrit, self.change, OutputApi(False)) + function_name = 'PostUploadHook' + if function_name not in context: + return {} + post_upload_hook = context[function_name] + if not len(inspect.getfullargspec(post_upload_hook)[0]) == 3: + raise PresubmitFailure( + 'Expected function "PostUploadHook" to take three arguments.') + return post_upload_hook(self.gerrit, self.change, OutputApi(False)) def _MergeMasters(masters1, masters2): - """Merges two master maps. Merges also the tests of each builder.""" - result = {} - for (master, builders) in itertools.chain(masters1.items(), - masters2.items()): - new_builders = result.setdefault(master, {}) - for (builder, tests) in builders.items(): - new_builders.setdefault(builder, set([])).update(tests) - return result + """Merges two master maps. Merges also the tests of each builder.""" + result = {} + for (master, builders) in itertools.chain(masters1.items(), + masters2.items()): + new_builders = result.setdefault(master, {}) + for (builder, tests) in builders.items(): + new_builders.setdefault(builder, set([])).update(tests) + return result def DoPostUploadExecuter(change, gerrit_obj, verbose): - """Execute the post upload hook. + """Execute the post upload hook. Args: change: The Change object. gerrit_obj: The GerritAccessor object. verbose: Prints debug info. """ - python_version = 'Python %s' % sys.version_info.major - sys.stdout.write('Running %s post upload checks ...\n' % python_version) - presubmit_files = ListRelevantPresubmitFiles( - change.LocalPaths(), change.RepositoryRoot()) - if not presubmit_files and verbose: - sys.stdout.write('Warning, no PRESUBMIT.py found.\n') - results = [] - executer = GetPostUploadExecuter(change, gerrit_obj) - # The root presubmit file should be executed after the ones in subdirectories. - # i.e. the specific post upload hooks should run before the general ones. - # Thus, reverse the order provided by ListRelevantPresubmitFiles. - presubmit_files.reverse() + python_version = 'Python %s' % sys.version_info.major + sys.stdout.write('Running %s post upload checks ...\n' % python_version) + presubmit_files = ListRelevantPresubmitFiles(change.LocalPaths(), + change.RepositoryRoot()) + if not presubmit_files and verbose: + sys.stdout.write('Warning, no PRESUBMIT.py found.\n') + results = [] + executer = GetPostUploadExecuter(change, gerrit_obj) + # The root presubmit file should be executed after the ones in + # subdirectories. i.e. the specific post upload hooks should run before the + # general ones. Thus, reverse the order provided by + # ListRelevantPresubmitFiles. + presubmit_files.reverse() - for filename in presubmit_files: - filename = os.path.abspath(filename) - # Accept CRLF presubmit script. - presubmit_script = gclient_utils.FileRead(filename).replace('\r\n', '\n') - if verbose: - sys.stdout.write('Running %s\n' % filename) - results.extend(executer.ExecPresubmitScript(presubmit_script, filename)) + for filename in presubmit_files: + filename = os.path.abspath(filename) + # Accept CRLF presubmit script. + presubmit_script = gclient_utils.FileRead(filename).replace( + '\r\n', '\n') + if verbose: + sys.stdout.write('Running %s\n' % filename) + results.extend(executer.ExecPresubmitScript(presubmit_script, filename)) - if not results: - return 0 + if not results: + return 0 - sys.stdout.write('\n') - sys.stdout.write('** Post Upload Hook Messages **\n') - - exit_code = 0 - for result in results: - if result.fatal: - exit_code = 1 - result.handle() sys.stdout.write('\n') + sys.stdout.write('** Post Upload Hook Messages **\n') + + exit_code = 0 + for result in results: + if result.fatal: + exit_code = 1 + result.handle() + sys.stdout.write('\n') + + return exit_code - return exit_code class PresubmitExecuter(object): - def __init__(self, change, committing, verbose, gerrit_obj, dry_run=None, - thread_pool=None, parallel=False, no_diffs=False): - """ + def __init__(self, + change, + committing, + verbose, + gerrit_obj, + dry_run=None, + thread_pool=None, + parallel=False, + no_diffs=False): + """ Args: change: The Change object. committing: True if 'git cl land' is running, False if 'git cl upload' is. @@ -1492,18 +1566,18 @@ class PresubmitExecuter(object): no_diffs: if true, implies that --files or --all was specified so some checks can be skipped, and some errors will be messages. """ - self.change = change - self.committing = committing - self.gerrit = gerrit_obj - self.verbose = verbose - self.dry_run = dry_run - self.more_cc = [] - self.thread_pool = thread_pool - self.parallel = parallel - self.no_diffs = no_diffs + self.change = change + self.committing = committing + self.gerrit = gerrit_obj + self.verbose = verbose + self.dry_run = dry_run + self.more_cc = [] + self.thread_pool = thread_pool + self.parallel = parallel + self.no_diffs = no_diffs - def ExecPresubmitScript(self, script_text, presubmit_path): - """Executes a single presubmit script. + def ExecPresubmitScript(self, script_text, presubmit_path): + """Executes a single presubmit script. Caller is responsible for validating whether the hook should be executed and should only call this function if it should be. @@ -1515,107 +1589,119 @@ class PresubmitExecuter(object): Return: A list of result objects, empty if no problems. """ - # Change to the presubmit file's directory to support local imports. - presubmit_dir = os.path.dirname(presubmit_path) - main_path = os.getcwd() - try: - os.chdir(presubmit_dir) - return self._execute_with_local_working_directory(script_text, - presubmit_dir, - presubmit_path) - finally: - # Return the process to the original working directory. - os.chdir(main_path) + # Change to the presubmit file's directory to support local imports. + presubmit_dir = os.path.dirname(presubmit_path) + main_path = os.getcwd() + try: + os.chdir(presubmit_dir) + return self._execute_with_local_working_directory( + script_text, presubmit_dir, presubmit_path) + finally: + # Return the process to the original working directory. + os.chdir(main_path) - def _execute_with_local_working_directory(self, script_text, presubmit_dir, - presubmit_path): - # Load the presubmit script into context. - input_api = InputApi(self.change, presubmit_path, self.committing, - self.verbose, gerrit_obj=self.gerrit, - dry_run=self.dry_run, thread_pool=self.thread_pool, - parallel=self.parallel, no_diffs=self.no_diffs) - output_api = OutputApi(self.committing) - context = {} + def _execute_with_local_working_directory(self, script_text, presubmit_dir, + presubmit_path): + # Load the presubmit script into context. + input_api = InputApi(self.change, + presubmit_path, + self.committing, + self.verbose, + gerrit_obj=self.gerrit, + dry_run=self.dry_run, + thread_pool=self.thread_pool, + parallel=self.parallel, + no_diffs=self.no_diffs) + output_api = OutputApi(self.committing) + context = {} - try: - exec(compile(script_text, presubmit_path, 'exec', dont_inherit=True), - context) - except Exception as e: - raise PresubmitFailure('"%s" had an exception.\n%s' % (presubmit_path, e)) + try: + exec( + compile(script_text, presubmit_path, 'exec', dont_inherit=True), + context) + except Exception as e: + raise PresubmitFailure('"%s" had an exception.\n%s' % + (presubmit_path, e)) - context['__args'] = (input_api, output_api) + context['__args'] = (input_api, output_api) - # Get path of presubmit directory relative to repository root. - # Always use forward slashes, so that path is same in *nix and Windows - root = input_api.change.RepositoryRoot() - rel_path = os.path.relpath(presubmit_dir, root) - rel_path = rel_path.replace(os.path.sep, '/') + # Get path of presubmit directory relative to repository root. + # Always use forward slashes, so that path is same in *nix and Windows + root = input_api.change.RepositoryRoot() + rel_path = os.path.relpath(presubmit_dir, root) + rel_path = rel_path.replace(os.path.sep, '/') - # Get the URL of git remote origin and use it to identify host and project - host = project = '' - if self.gerrit: - host = self.gerrit.host or '' - project = self.gerrit.project or '' + # Get the URL of git remote origin and use it to identify host and + # project + host = project = '' + if self.gerrit: + host = self.gerrit.host or '' + project = self.gerrit.project or '' - # Prefix for test names - prefix = 'presubmit:%s/%s:%s/' % (host, project, rel_path) + # Prefix for test names + prefix = 'presubmit:%s/%s:%s/' % (host, project, rel_path) - # Perform all the desired presubmit checks. - results = [] + # Perform all the desired presubmit checks. + results = [] - try: - version = [ - int(x) for x in context.get('PRESUBMIT_VERSION', '0.0.0').split('.') - ] + try: + version = [ + int(x) + for x in context.get('PRESUBMIT_VERSION', '0.0.0').split('.') + ] - with rdb_wrapper.client(prefix) as sink: - if version >= [2, 0, 0]: - # Copy the keys to prevent "dictionary changed size during iteration" - # exception if checks add globals to context. E.g. sometimes the - # Python runtime will add __warningregistry__. - for function_name in list(context.keys()): - if not function_name.startswith('Check'): - continue - if function_name.endswith('Commit') and not self.committing: - continue - if function_name.endswith('Upload') and self.committing: - continue - logging.debug('Running %s in %s', function_name, presubmit_path) - results.extend( - self._run_check_function(function_name, context, sink, - presubmit_path)) - logging.debug('Running %s done.', function_name) - self.more_cc.extend(output_api.more_cc) - # Clear the CC list between running each presubmit check to prevent - # CCs from being repeatedly appended. - output_api.more_cc = [] + with rdb_wrapper.client(prefix) as sink: + if version >= [2, 0, 0]: + # Copy the keys to prevent "dictionary changed size during + # iteration" exception if checks add globals to context. + # E.g. sometimes the Python runtime will add + # __warningregistry__. + for function_name in list(context.keys()): + if not function_name.startswith('Check'): + continue + if function_name.endswith( + 'Commit') and not self.committing: + continue + if function_name.endswith('Upload') and self.committing: + continue + logging.debug('Running %s in %s', function_name, + presubmit_path) + results.extend( + self._run_check_function(function_name, context, + sink, presubmit_path)) + logging.debug('Running %s done.', function_name) + self.more_cc.extend(output_api.more_cc) + # Clear the CC list between running each presubmit check + # to prevent CCs from being repeatedly appended. + output_api.more_cc = [] - else: # Old format - if self.committing: - function_name = 'CheckChangeOnCommit' - else: - function_name = 'CheckChangeOnUpload' - if function_name in list(context.keys()): - logging.debug('Running %s in %s', function_name, presubmit_path) - results.extend( - self._run_check_function(function_name, context, sink, - presubmit_path)) - logging.debug('Running %s done.', function_name) - self.more_cc.extend(output_api.more_cc) - # Clear the CC list between running each presubmit check to prevent - # CCs from being repeatedly appended. - output_api.more_cc = [] + else: # Old format + if self.committing: + function_name = 'CheckChangeOnCommit' + else: + function_name = 'CheckChangeOnUpload' + if function_name in list(context.keys()): + logging.debug('Running %s in %s', function_name, + presubmit_path) + results.extend( + self._run_check_function(function_name, context, + sink, presubmit_path)) + logging.debug('Running %s done.', function_name) + self.more_cc.extend(output_api.more_cc) + # Clear the CC list between running each presubmit check + # to prevent CCs from being repeatedly appended. + output_api.more_cc = [] - finally: - for f in input_api._named_temporary_files: - os.remove(f) + finally: + for f in input_api._named_temporary_files: + os.remove(f) - self.more_cc = sorted(set(self.more_cc)) + self.more_cc = sorted(set(self.more_cc)) - return results + return results - def _run_check_function(self, function_name, context, sink, presubmit_path): - """Evaluates and returns the result of a given presubmit function. + def _run_check_function(self, function_name, context, sink, presubmit_path): + """Evaluates and returns the result of a given presubmit function. If sink is given, the result of the presubmit function will be reported to the ResultSink. @@ -1627,48 +1713,50 @@ class PresubmitExecuter(object): Returns: the result of the presubmit function call. """ - start_time = time_time() - try: - result = eval(function_name + '(*__args)', context) - self._check_result_type(result) - except Exception: - _, e_value, _ = sys.exc_info() - result = [ - OutputApi.PresubmitError( - 'Evaluation of %s failed: %s, %s' % - (function_name, e_value, traceback.format_exc())) - ] + start_time = time_time() + try: + result = eval(function_name + '(*__args)', context) + self._check_result_type(result) + except Exception: + _, e_value, _ = sys.exc_info() + result = [ + OutputApi.PresubmitError( + 'Evaluation of %s failed: %s, %s' % + (function_name, e_value, traceback.format_exc())) + ] - elapsed_time = time_time() - start_time - if elapsed_time > 10.0: - sys.stdout.write('%6.1fs to run %s from %s.\n' % - (elapsed_time, function_name, presubmit_path)) - if sink: - failure_reason = None - status = rdb_wrapper.STATUS_PASS - if any(r.fatal for r in result): - status = rdb_wrapper.STATUS_FAIL - failure_reasons = [] - for r in result: - fields = r.json_format() - message = fields['message'] - items = '\n'.join(' %s' % item for item in fields['items']) - failure_reasons.append('\n'.join([message, items])) - if failure_reasons: - failure_reason = '\n'.join(failure_reasons) - sink.report(function_name, status, elapsed_time, failure_reason) + elapsed_time = time_time() - start_time + if elapsed_time > 10.0: + sys.stdout.write('%6.1fs to run %s from %s.\n' % + (elapsed_time, function_name, presubmit_path)) + if sink: + failure_reason = None + status = rdb_wrapper.STATUS_PASS + if any(r.fatal for r in result): + status = rdb_wrapper.STATUS_FAIL + failure_reasons = [] + for r in result: + fields = r.json_format() + message = fields['message'] + items = '\n'.join(' %s' % item for item in fields['items']) + failure_reasons.append('\n'.join([message, items])) + if failure_reasons: + failure_reason = '\n'.join(failure_reasons) + sink.report(function_name, status, elapsed_time, failure_reason) - return result + return result - def _check_result_type(self, result): - """Helper function which ensures result is a list, and all elements are + def _check_result_type(self, result): + """Helper function which ensures result is a list, and all elements are instances of OutputApi.PresubmitResult""" - if not isinstance(result, (tuple, list)): - raise PresubmitFailure('Presubmit functions must return a tuple or list') - if not all(isinstance(res, OutputApi.PresubmitResult) for res in result): - raise PresubmitFailure( - 'All presubmit results must be of types derived from ' - 'output_api.PresubmitResult') + if not isinstance(result, (tuple, list)): + raise PresubmitFailure( + 'Presubmit functions must return a tuple or list') + if not all( + isinstance(res, OutputApi.PresubmitResult) for res in result): + raise PresubmitFailure( + 'All presubmit results must be of types derived from ' + 'output_api.PresubmitResult') def DoPresubmitChecks(change, @@ -1681,7 +1769,7 @@ def DoPresubmitChecks(change, parallel=False, json_output=None, no_diffs=False): - """Runs all presubmit checks that apply to the files in the change. + """Runs all presubmit checks that apply to the files in the change. This finds all PRESUBMIT.py files in directories enclosing the files in the change (up to the repository root) and calls the relevant entrypoint function @@ -1706,144 +1794,148 @@ def DoPresubmitChecks(change, Return: 1 if presubmit checks failed or 0 otherwise. """ - old_environ = os.environ - try: - # Make sure python subprocesses won't generate .pyc files. - os.environ = os.environ.copy() - os.environ['PYTHONDONTWRITEBYTECODE'] = '1' + old_environ = os.environ + try: + # Make sure python subprocesses won't generate .pyc files. + os.environ = os.environ.copy() + os.environ['PYTHONDONTWRITEBYTECODE'] = '1' - python_version = 'Python %s' % sys.version_info.major - if committing: - sys.stdout.write('Running %s presubmit commit checks ...\n' % - python_version) - else: - sys.stdout.write('Running %s presubmit upload checks ...\n' % - python_version) - start_time = time_time() - presubmit_files = ListRelevantPresubmitFiles( - change.AbsoluteLocalPaths(), change.RepositoryRoot()) - if not presubmit_files and verbose: - sys.stdout.write('Warning, no PRESUBMIT.py found.\n') - results = [] - thread_pool = ThreadPool() - executer = PresubmitExecuter(change, committing, verbose, gerrit_obj, - dry_run, thread_pool, parallel, no_diffs) - if default_presubmit: - if verbose: - sys.stdout.write('Running default presubmit script.\n') - fake_path = os.path.join(change.RepositoryRoot(), 'PRESUBMIT.py') - results += executer.ExecPresubmitScript(default_presubmit, fake_path) - for filename in presubmit_files: - filename = os.path.abspath(filename) - # Accept CRLF presubmit script. - presubmit_script = gclient_utils.FileRead(filename).replace('\r\n', '\n') - if verbose: - sys.stdout.write('Running %s\n' % filename) - results += executer.ExecPresubmitScript(presubmit_script, filename) + python_version = 'Python %s' % sys.version_info.major + if committing: + sys.stdout.write('Running %s presubmit commit checks ...\n' % + python_version) + else: + sys.stdout.write('Running %s presubmit upload checks ...\n' % + python_version) + start_time = time_time() + presubmit_files = ListRelevantPresubmitFiles( + change.AbsoluteLocalPaths(), change.RepositoryRoot()) + if not presubmit_files and verbose: + sys.stdout.write('Warning, no PRESUBMIT.py found.\n') + results = [] + thread_pool = ThreadPool() + executer = PresubmitExecuter(change, committing, verbose, gerrit_obj, + dry_run, thread_pool, parallel, no_diffs) + if default_presubmit: + if verbose: + sys.stdout.write('Running default presubmit script.\n') + fake_path = os.path.join(change.RepositoryRoot(), 'PRESUBMIT.py') + results += executer.ExecPresubmitScript(default_presubmit, + fake_path) + for filename in presubmit_files: + filename = os.path.abspath(filename) + # Accept CRLF presubmit script. + presubmit_script = gclient_utils.FileRead(filename).replace( + '\r\n', '\n') + if verbose: + sys.stdout.write('Running %s\n' % filename) + results += executer.ExecPresubmitScript(presubmit_script, filename) - results += thread_pool.RunAsync() + results += thread_pool.RunAsync() - messages = {} - should_prompt = False - presubmits_failed = False - for result in results: - if result.fatal: - presubmits_failed = True - messages.setdefault('ERRORS', []).append(result) - elif result.should_prompt: - should_prompt = True - messages.setdefault('Warnings', []).append(result) - else: - messages.setdefault('Messages', []).append(result) + messages = {} + should_prompt = False + presubmits_failed = False + for result in results: + if result.fatal: + presubmits_failed = True + messages.setdefault('ERRORS', []).append(result) + elif result.should_prompt: + should_prompt = True + messages.setdefault('Warnings', []).append(result) + else: + messages.setdefault('Messages', []).append(result) - # Print the different message types in a consistent order. ERRORS go last - # so that they will be most visible in the local-presubmit output. - for name in ['Messages', 'Warnings', 'ERRORS']: - if name in messages: - items = messages[name] - sys.stdout.write('** Presubmit %s: %d **\n' % (name, len(items))) - for item in items: - item.handle() - sys.stdout.write('\n') + # Print the different message types in a consistent order. ERRORS go + # last so that they will be most visible in the local-presubmit output. + for name in ['Messages', 'Warnings', 'ERRORS']: + if name in messages: + items = messages[name] + sys.stdout.write('** Presubmit %s: %d **\n' % + (name, len(items))) + for item in items: + item.handle() + sys.stdout.write('\n') - total_time = time_time() - start_time - if total_time > 1.0: - sys.stdout.write( - 'Presubmit checks took %.1fs to calculate.\n' % total_time) + total_time = time_time() - start_time + if total_time > 1.0: + sys.stdout.write('Presubmit checks took %.1fs to calculate.\n' % + total_time) - if not should_prompt and not presubmits_failed: - sys.stdout.write('%s presubmit checks passed.\n\n' % python_version) - elif should_prompt and not presubmits_failed: - sys.stdout.write('There were %s presubmit warnings. ' % python_version) - if may_prompt: - presubmits_failed = not prompt_should_continue( - 'Are you sure you wish to continue? (y/N): ') - else: - sys.stdout.write('\n') - else: - sys.stdout.write('There were %s presubmit errors.\n' % python_version) + if not should_prompt and not presubmits_failed: + sys.stdout.write('%s presubmit checks passed.\n\n' % python_version) + elif should_prompt and not presubmits_failed: + sys.stdout.write('There were %s presubmit warnings. ' % + python_version) + if may_prompt: + presubmits_failed = not prompt_should_continue( + 'Are you sure you wish to continue? (y/N): ') + else: + sys.stdout.write('\n') + else: + sys.stdout.write('There were %s presubmit errors.\n' % + python_version) - if json_output: - # Write the presubmit results to json output - presubmit_results = { - 'errors': [ - error.json_format() - for error in messages.get('ERRORS', []) - ], - 'notifications': [ - notification.json_format() - for notification in messages.get('Messages', []) - ], - 'warnings': [ - warning.json_format() - for warning in messages.get('Warnings', []) - ], - 'more_cc': executer.more_cc, - } + if json_output: + # Write the presubmit results to json output + presubmit_results = { + 'errors': + [error.json_format() for error in messages.get('ERRORS', [])], + 'notifications': [ + notification.json_format() + for notification in messages.get('Messages', []) + ], + 'warnings': [ + warning.json_format() + for warning in messages.get('Warnings', []) + ], + 'more_cc': + executer.more_cc, + } - gclient_utils.FileWrite( - json_output, json.dumps(presubmit_results, sort_keys=True)) + gclient_utils.FileWrite( + json_output, json.dumps(presubmit_results, sort_keys=True)) - global _ASKED_FOR_FEEDBACK - # Ask for feedback one time out of 5. - if (results and random.randint(0, 4) == 0 and not _ASKED_FOR_FEEDBACK): - sys.stdout.write( - 'Was the presubmit check useful? If not, run "git cl presubmit -v"\n' - 'to figure out which PRESUBMIT.py was run, then run git blame\n' - 'on the file to figure out who to ask for help.\n') - _ASKED_FOR_FEEDBACK = True + global _ASKED_FOR_FEEDBACK + # Ask for feedback one time out of 5. + if (results and random.randint(0, 4) == 0 and not _ASKED_FOR_FEEDBACK): + sys.stdout.write( + 'Was the presubmit check useful? If not, run "git cl presubmit -v"\n' + 'to figure out which PRESUBMIT.py was run, then run git blame\n' + 'on the file to figure out who to ask for help.\n') + _ASKED_FOR_FEEDBACK = True - return 1 if presubmits_failed else 0 - finally: - os.environ = old_environ + return 1 if presubmits_failed else 0 + finally: + os.environ = old_environ def _scan_sub_dirs(mask, recursive): - if not recursive: - return [x for x in glob.glob(mask) if x not in ('.svn', '.git')] + if not recursive: + return [x for x in glob.glob(mask) if x not in ('.svn', '.git')] - results = [] - for root, dirs, files in os.walk('.'): - if '.svn' in dirs: - dirs.remove('.svn') - if '.git' in dirs: - dirs.remove('.git') - for name in files: - if fnmatch.fnmatch(name, mask): - results.append(os.path.join(root, name)) - return results + results = [] + for root, dirs, files in os.walk('.'): + if '.svn' in dirs: + dirs.remove('.svn') + if '.git' in dirs: + dirs.remove('.git') + for name in files: + if fnmatch.fnmatch(name, mask): + results.append(os.path.join(root, name)) + return results def _parse_files(args, recursive): - logging.debug('Searching for %s', args) - files = [] - for arg in args: - files.extend([('M', f) for f in _scan_sub_dirs(arg, recursive)]) - return files + logging.debug('Searching for %s', args) + files = [] + for arg in args: + files.extend([('M', f) for f in _scan_sub_dirs(arg, recursive)]) + return files def _parse_change(parser, options): - """Process change options. + """Process change options. Args: parser: The parser used to parse the arguments from command line. @@ -1851,47 +1943,46 @@ def _parse_change(parser, options): Returns: A GitChange if the change root is a git repository, or a Change otherwise. """ - if options.files and options.all_files: - parser.error(' cannot be specified when --all-files is set.') + if options.files and options.all_files: + parser.error(' cannot be specified when --all-files is set.') - change_scm = scm.determine_scm(options.root) - if change_scm != 'git' and not options.files: - parser.error(' is not optional for unversioned directories.') + change_scm = scm.determine_scm(options.root) + if change_scm != 'git' and not options.files: + parser.error(' is not optional for unversioned directories.') - if options.files: - if options.source_controlled_only: - # Get the filtered set of files from SCM. - change_files = [] - for name in scm.GIT.GetAllFiles(options.root): - for mask in options.files: - if fnmatch.fnmatch(name, mask): - change_files.append(('M', name)) - break + if options.files: + if options.source_controlled_only: + # Get the filtered set of files from SCM. + change_files = [] + for name in scm.GIT.GetAllFiles(options.root): + for mask in options.files: + if fnmatch.fnmatch(name, mask): + change_files.append(('M', name)) + break + else: + # Get the filtered set of files from a directory scan. + change_files = _parse_files(options.files, options.recursive) + elif options.all_files: + change_files = [('M', f) for f in scm.GIT.GetAllFiles(options.root)] else: - # Get the filtered set of files from a directory scan. - change_files = _parse_files(options.files, options.recursive) - elif options.all_files: - change_files = [('M', f) for f in scm.GIT.GetAllFiles(options.root)] - else: - change_files = scm.GIT.CaptureStatus( - options.root, options.upstream or None) + change_files = scm.GIT.CaptureStatus(options.root, options.upstream + or None) - logging.info('Found %d file(s).', len(change_files)) + logging.info('Found %d file(s).', len(change_files)) - change_class = GitChange if change_scm == 'git' else Change - return change_class( - options.name, - options.description, - options.root, - change_files, - options.issue, - options.patchset, - options.author, - upstream=options.upstream) + change_class = GitChange if change_scm == 'git' else Change + return change_class(options.name, + options.description, + options.root, + change_files, + options.issue, + options.patchset, + options.author, + upstream=options.upstream) def _parse_gerrit_options(parser, options): - """Process gerrit options. + """Process gerrit options. SIDE EFFECTS: Modifies options.author and options.description from Gerrit if options.gerrit_fetch is set. @@ -1902,153 +1993,174 @@ def _parse_gerrit_options(parser, options): Returns: A GerritAccessor object if options.gerrit_url is set, or None otherwise. """ - gerrit_obj = None - if options.gerrit_url: - gerrit_obj = GerritAccessor( - url=options.gerrit_url, - project=options.gerrit_project, - branch=options.gerrit_branch) + gerrit_obj = None + if options.gerrit_url: + gerrit_obj = GerritAccessor(url=options.gerrit_url, + project=options.gerrit_project, + branch=options.gerrit_branch) + + if not options.gerrit_fetch: + return gerrit_obj + + if not options.gerrit_url or not options.issue or not options.patchset: + parser.error( + '--gerrit_fetch requires --gerrit_url, --issue and --patchset.') + + options.author = gerrit_obj.GetChangeOwner(options.issue) + options.description = gerrit_obj.GetChangeDescription( + options.issue, options.patchset) + + logging.info('Got author: "%s"', options.author) + logging.info('Got description: """\n%s\n"""', options.description) - if not options.gerrit_fetch: return gerrit_obj - if not options.gerrit_url or not options.issue or not options.patchset: - parser.error( - '--gerrit_fetch requires --gerrit_url, --issue and --patchset.') - - options.author = gerrit_obj.GetChangeOwner(options.issue) - options.description = gerrit_obj.GetChangeDescription( - options.issue, options.patchset) - - logging.info('Got author: "%s"', options.author) - logging.info('Got description: """\n%s\n"""', options.description) - - return gerrit_obj - @contextlib.contextmanager def canned_check_filter(method_names): - filtered = {} - try: - for method_name in method_names: - if not hasattr(presubmit_canned_checks, method_name): - logging.warning('Skipping unknown "canned" check %s' % method_name) - continue - filtered[method_name] = getattr(presubmit_canned_checks, method_name) - setattr(presubmit_canned_checks, method_name, lambda *_a, **_kw: []) - yield - finally: - for name, method in filtered.items(): - setattr(presubmit_canned_checks, name, method) + filtered = {} + try: + for method_name in method_names: + if not hasattr(presubmit_canned_checks, method_name): + logging.warning('Skipping unknown "canned" check %s' % + method_name) + continue + filtered[method_name] = getattr(presubmit_canned_checks, + method_name) + setattr(presubmit_canned_checks, method_name, lambda *_a, **_kw: []) + yield + finally: + for name, method in filtered.items(): + setattr(presubmit_canned_checks, name, method) def main(argv=None): - parser = argparse.ArgumentParser(usage='%(prog)s [options] ') - hooks = parser.add_mutually_exclusive_group() - hooks.add_argument('-c', '--commit', action='store_true', - help='Use commit instead of upload checks.') - hooks.add_argument('-u', '--upload', action='store_false', dest='commit', - help='Use upload instead of commit checks.') - hooks.add_argument('--post_upload', action='store_true', - help='Run post-upload commit hooks.') - parser.add_argument('-r', '--recursive', action='store_true', - help='Act recursively.') - parser.add_argument('-v', '--verbose', action='count', default=0, - help='Use 2 times for more debug info.') - parser.add_argument('--name', default='no name') - parser.add_argument('--author') - desc = parser.add_mutually_exclusive_group() - desc.add_argument('--description', default='', help='The change description.') - desc.add_argument('--description_file', - help='File to read change description from.') - parser.add_argument('--issue', type=int, default=0) - parser.add_argument('--patchset', type=int, default=0) - parser.add_argument('--root', default=os.getcwd(), - help='Search for PRESUBMIT.py up to this directory. ' - 'If inherit-review-settings-ok is present in this ' - 'directory, parent directories up to the root file ' - 'system directories will also be searched.') - parser.add_argument('--upstream', - help='Git only: the base ref or upstream branch against ' - 'which the diff should be computed.') - parser.add_argument('--default_presubmit') - parser.add_argument('--may_prompt', action='store_true', default=False) - parser.add_argument('--skip_canned', action='append', default=[], - help='A list of checks to skip which appear in ' - 'presubmit_canned_checks. Can be provided multiple times ' - 'to skip multiple canned checks.') - parser.add_argument('--dry_run', action='store_true', help=argparse.SUPPRESS) - parser.add_argument('--gerrit_url', help=argparse.SUPPRESS) - parser.add_argument('--gerrit_project', help=argparse.SUPPRESS) - parser.add_argument('--gerrit_branch', help=argparse.SUPPRESS) - parser.add_argument('--gerrit_fetch', action='store_true', - help=argparse.SUPPRESS) - parser.add_argument('--parallel', action='store_true', - help='Run all tests specified by input_api.RunTests in ' - 'all PRESUBMIT files in parallel.') - parser.add_argument('--json_output', - help='Write presubmit errors to json output.') - parser.add_argument('--all_files', action='store_true', - help='Mark all files under source control as modified.') + parser = argparse.ArgumentParser(usage='%(prog)s [options] ') + hooks = parser.add_mutually_exclusive_group() + hooks.add_argument('-c', + '--commit', + action='store_true', + help='Use commit instead of upload checks.') + hooks.add_argument('-u', + '--upload', + action='store_false', + dest='commit', + help='Use upload instead of commit checks.') + hooks.add_argument('--post_upload', + action='store_true', + help='Run post-upload commit hooks.') + parser.add_argument('-r', + '--recursive', + action='store_true', + help='Act recursively.') + parser.add_argument('-v', + '--verbose', + action='count', + default=0, + help='Use 2 times for more debug info.') + parser.add_argument('--name', default='no name') + parser.add_argument('--author') + desc = parser.add_mutually_exclusive_group() + desc.add_argument('--description', + default='', + help='The change description.') + desc.add_argument('--description_file', + help='File to read change description from.') + parser.add_argument('--issue', type=int, default=0) + parser.add_argument('--patchset', type=int, default=0) + parser.add_argument('--root', + default=os.getcwd(), + help='Search for PRESUBMIT.py up to this directory. ' + 'If inherit-review-settings-ok is present in this ' + 'directory, parent directories up to the root file ' + 'system directories will also be searched.') + parser.add_argument( + '--upstream', + help='Git only: the base ref or upstream branch against ' + 'which the diff should be computed.') + parser.add_argument('--default_presubmit') + parser.add_argument('--may_prompt', action='store_true', default=False) + parser.add_argument( + '--skip_canned', + action='append', + default=[], + help='A list of checks to skip which appear in ' + 'presubmit_canned_checks. Can be provided multiple times ' + 'to skip multiple canned checks.') + parser.add_argument('--dry_run', + action='store_true', + help=argparse.SUPPRESS) + parser.add_argument('--gerrit_url', help=argparse.SUPPRESS) + parser.add_argument('--gerrit_project', help=argparse.SUPPRESS) + parser.add_argument('--gerrit_branch', help=argparse.SUPPRESS) + parser.add_argument('--gerrit_fetch', + action='store_true', + help=argparse.SUPPRESS) + parser.add_argument('--parallel', + action='store_true', + help='Run all tests specified by input_api.RunTests in ' + 'all PRESUBMIT files in parallel.') + parser.add_argument('--json_output', + help='Write presubmit errors to json output.') + parser.add_argument('--all_files', + action='store_true', + help='Mark all files under source control as modified.') - parser.add_argument('files', nargs='*', - help='List of files to be marked as modified when ' - 'executing presubmit or post-upload hooks. fnmatch ' - 'wildcards can also be used.') - parser.add_argument('--source_controlled_only', action='store_true', - help='Constrain \'files\' to those in source control.') - parser.add_argument('--no_diffs', action='store_true', - help='Assume that all "modified" files have no diffs.') - options = parser.parse_args(argv) + parser.add_argument('files', + nargs='*', + help='List of files to be marked as modified when ' + 'executing presubmit or post-upload hooks. fnmatch ' + 'wildcards can also be used.') + parser.add_argument('--source_controlled_only', + action='store_true', + help='Constrain \'files\' to those in source control.') + parser.add_argument('--no_diffs', + action='store_true', + help='Assume that all "modified" files have no diffs.') + options = parser.parse_args(argv) - log_level = logging.ERROR - if options.verbose >= 2: - log_level = logging.DEBUG - elif options.verbose: - log_level = logging.INFO - log_format = ('[%(levelname).1s%(asctime)s %(process)d %(thread)d ' - '%(filename)s] %(message)s') - logging.basicConfig(format=log_format, level=log_level) + log_level = logging.ERROR + if options.verbose >= 2: + log_level = logging.DEBUG + elif options.verbose: + log_level = logging.INFO + log_format = ('[%(levelname).1s%(asctime)s %(process)d %(thread)d ' + '%(filename)s] %(message)s') + logging.basicConfig(format=log_format, level=log_level) - # Print call stacks when _PresubmitResult objects are created with -v -v is - # specified. This helps track down where presubmit messages are coming from. - if options.verbose >= 2: - global _SHOW_CALLSTACKS - _SHOW_CALLSTACKS = True + # Print call stacks when _PresubmitResult objects are created with -v -v is + # specified. This helps track down where presubmit messages are coming from. + if options.verbose >= 2: + global _SHOW_CALLSTACKS + _SHOW_CALLSTACKS = True - if options.description_file: - options.description = gclient_utils.FileRead(options.description_file) - gerrit_obj = _parse_gerrit_options(parser, options) - change = _parse_change(parser, options) + if options.description_file: + options.description = gclient_utils.FileRead(options.description_file) + gerrit_obj = _parse_gerrit_options(parser, options) + change = _parse_change(parser, options) - try: - if options.post_upload: - return DoPostUploadExecuter(change, gerrit_obj, options.verbose) - with canned_check_filter(options.skip_canned): - return DoPresubmitChecks( - change, - options.commit, - options.verbose, - options.default_presubmit, - options.may_prompt, - gerrit_obj, - options.dry_run, - options.parallel, - options.json_output, - options.no_diffs) - except PresubmitFailure as e: - import utils - print(e, file=sys.stderr) - print('Maybe your depot_tools is out of date?', file=sys.stderr) - print('depot_tools version: %s' % utils.depot_tools_version(), - file=sys.stderr) - return 2 + try: + if options.post_upload: + return DoPostUploadExecuter(change, gerrit_obj, options.verbose) + with canned_check_filter(options.skip_canned): + return DoPresubmitChecks(change, options.commit, options.verbose, + options.default_presubmit, + options.may_prompt, gerrit_obj, + options.dry_run, options.parallel, + options.json_output, options.no_diffs) + except PresubmitFailure as e: + import utils + print(e, file=sys.stderr) + print('Maybe your depot_tools is out of date?', file=sys.stderr) + print('depot_tools version: %s' % utils.depot_tools_version(), + file=sys.stderr) + return 2 if __name__ == '__main__': - fix_encoding.fix_encoding() - try: - sys.exit(main()) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(2) + fix_encoding.fix_encoding() + try: + sys.exit(main()) + except KeyboardInterrupt: + sys.stderr.write('interrupted\n') + sys.exit(2) diff --git a/pylint-2.6 b/pylint-2.6 index e65fd5d9f1..a7a541b690 100755 --- a/pylint-2.6 +++ b/pylint-2.6 @@ -69,4 +69,4 @@ import sys import pylint_main if __name__ == '__main__': - sys.exit(pylint_main.main(sys.argv[1:])) + sys.exit(pylint_main.main(sys.argv[1:])) diff --git a/pylint-2.7 b/pylint-2.7 index d8c52ac8d4..ae30b899b3 100755 --- a/pylint-2.7 +++ b/pylint-2.7 @@ -69,4 +69,4 @@ import sys import pylint_main if __name__ == '__main__': - sys.exit(pylint_main.main(sys.argv[1:])) + sys.exit(pylint_main.main(sys.argv[1:])) diff --git a/pylint_main.py b/pylint_main.py index 25cb40b6a3..e9717ebb07 100755 --- a/pylint_main.py +++ b/pylint_main.py @@ -2,7 +2,6 @@ # Copyright 2019 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Chromium wrapper for pylint for passing args via stdin. This will be executed by vpython with the right pylint versions. @@ -21,36 +20,37 @@ ARGS_ON_STDIN = '--args-on-stdin' def main(argv): - """Our main wrapper.""" - # Add support for a custom mode where arguments are fed line by line on - # stdin. This allows us to get around command line length limitations. - if ARGS_ON_STDIN in argv: - argv = [x for x in argv if x != ARGS_ON_STDIN] - argv.extend(x.strip() for x in sys.stdin) + """Our main wrapper.""" + # Add support for a custom mode where arguments are fed line by line on + # stdin. This allows us to get around command line length limitations. + if ARGS_ON_STDIN in argv: + argv = [x for x in argv if x != ARGS_ON_STDIN] + argv.extend(x.strip() for x in sys.stdin) - # Set default config options with the PYLINTRC environment variable. This will - # allow overriding with "more local" config file options, such as a local - # "pylintrc" file, the "--rcfile" command-line flag, or an existing PYLINTRC. - # - # Note that this is not quite the same thing as replacing pylint's built-in - # defaults, since, based on config file precedence, it will not be overridden - # by "more global" config file options, such as ~/.pylintrc, - # ~/.config/pylintrc, or /etc/pylintrc. This is generally the desired - # behavior, since we want to enforce these defaults in most cases, but allow - # them to be overridden for specific code or repos. - # - # If someone really doesn't ever want the depot_tools pylintrc, they can set - # their own PYLINTRC, or set an empty PYLINTRC to use pylint's normal config - # file resolution, which would include the "more global" options that are - # normally overridden by the depot_tools config. - if os.path.isfile(RC_FILE) and 'PYLINTRC' not in os.environ: - os.environ['PYLINTRC'] = RC_FILE + # Set default config options with the PYLINTRC environment variable. This + # will allow overriding with "more local" config file options, such as a + # local "pylintrc" file, the "--rcfile" command-line flag, or an existing + # PYLINTRC. + # + # Note that this is not quite the same thing as replacing pylint's built-in + # defaults, since, based on config file precedence, it will not be + # overridden by "more global" config file options, such as ~/.pylintrc, + # ~/.config/pylintrc, or /etc/pylintrc. This is generally the desired + # behavior, since we want to enforce these defaults in most cases, but allow + # them to be overridden for specific code or repos. + # + # If someone really doesn't ever want the depot_tools pylintrc, they can set + # their own PYLINTRC, or set an empty PYLINTRC to use pylint's normal config + # file resolution, which would include the "more global" options that are + # normally overridden by the depot_tools config. + if os.path.isfile(RC_FILE) and 'PYLINTRC' not in os.environ: + os.environ['PYLINTRC'] = RC_FILE - # This import has to happen after PYLINTRC is set because the module tries to - # resolve the config file location on load. - from pylint import lint # pylint: disable=bad-option-value,import-outside-toplevel - lint.Run(argv) + # This import has to happen after PYLINTRC is set because the module tries + # to resolve the config file location on load. + from pylint import lint # pylint: disable=bad-option-value,import-outside-toplevel + lint.Run(argv) if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) + sys.exit(main(sys.argv[1:])) diff --git a/rdb_wrapper.py b/rdb_wrapper.py index b6021e627a..2e0843a92c 100644 --- a/rdb_wrapper.py +++ b/rdb_wrapper.py @@ -15,23 +15,21 @@ STATUS_CRASH = 'CRASH' STATUS_ABORT = 'ABORT' STATUS_SKIP = 'SKIP' - # ResultDB limits failure reasons to 1024 characters. _FAILURE_REASON_LENGTH_LIMIT = 1024 - # Message to use at the end of a truncated failure reason. _FAILURE_REASON_TRUNCATE_TEXT = '\n...\nFailure reason was truncated.' class ResultSink(object): - def __init__(self, session, url, prefix): - self._session = session - self._url = url - self._prefix = prefix + def __init__(self, session, url, prefix): + self._session = session + self._url = url + self._prefix = prefix - def report(self, function_name, status, elapsed_time, failure_reason=None): - """Reports the result and elapsed time of a presubmit function call. + def report(self, function_name, status, elapsed_time, failure_reason=None): + """Reports the result and elapsed time of a presubmit function call. Args: function_name (str): The name of the presubmit function @@ -39,24 +37,24 @@ class ResultSink(object): elapsed_time: the time taken to invoke the presubmit function failure_reason (str or None): if set, the failure reason """ - tr = { - 'testId': self._prefix + function_name, - 'status': status, - 'expected': status == STATUS_PASS, - 'duration': '{:.9f}s'.format(elapsed_time) - } - if failure_reason: - if len(failure_reason) > _FAILURE_REASON_LENGTH_LIMIT: - failure_reason = failure_reason[ - :-len(_FAILURE_REASON_TRUNCATE_TEXT) - 1] - failure_reason += _FAILURE_REASON_TRUNCATE_TEXT - tr['failureReason'] = {'primaryErrorMessage': failure_reason} - self._session.post(self._url, json={'testResults': [tr]}) + tr = { + 'testId': self._prefix + function_name, + 'status': status, + 'expected': status == STATUS_PASS, + 'duration': '{:.9f}s'.format(elapsed_time) + } + if failure_reason: + if len(failure_reason) > _FAILURE_REASON_LENGTH_LIMIT: + failure_reason = failure_reason[:-len( + _FAILURE_REASON_TRUNCATE_TEXT) - 1] + failure_reason += _FAILURE_REASON_TRUNCATE_TEXT + tr['failureReason'] = {'primaryErrorMessage': failure_reason} + self._session.post(self._url, json={'testResults': [tr]}) @contextlib.contextmanager def client(prefix): - """Returns a client for ResultSink. + """Returns a client for ResultSink. This is a context manager that returns a client for ResultSink, if LUCI_CONTEXT with a section of result_sink is present. When the context @@ -71,24 +69,24 @@ def client(prefix): Returns: An instance of ResultSink() if the luci context is present. None, otherwise. """ - luci_ctx = os.environ.get('LUCI_CONTEXT') - if not luci_ctx: - yield None - return + luci_ctx = os.environ.get('LUCI_CONTEXT') + if not luci_ctx: + yield None + return - sink_ctx = None - with open(luci_ctx) as f: - sink_ctx = json.load(f).get('result_sink') - if not sink_ctx: - yield None - return + sink_ctx = None + with open(luci_ctx) as f: + sink_ctx = json.load(f).get('result_sink') + if not sink_ctx: + yield None + return - url = 'http://{0}/prpc/luci.resultsink.v1.Sink/ReportTestResults'.format( - sink_ctx['address']) - with requests.Session() as s: - s.headers = { - 'Content-Type': 'application/json', - 'Accept': 'application/json', - 'Authorization': 'ResultSink {0}'.format(sink_ctx['auth_token']) - } - yield ResultSink(s, url, prefix) + url = 'http://{0}/prpc/luci.resultsink.v1.Sink/ReportTestResults'.format( + sink_ctx['address']) + with requests.Session() as s: + s.headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + 'Authorization': 'ResultSink {0}'.format(sink_ctx['auth_token']) + } + yield ResultSink(s, url, prefix) diff --git a/fetch_configs/.style.yapf b/recipes/.style.yapf similarity index 73% rename from fetch_configs/.style.yapf rename to recipes/.style.yapf index 4741fb4f3b..24681e21f7 100644 --- a/fetch_configs/.style.yapf +++ b/recipes/.style.yapf @@ -1,3 +1,4 @@ [style] based_on_style = pep8 +indent_width = 2 column_limit = 80 diff --git a/reclient_helper.py b/reclient_helper.py index 15723d72f3..05fdc3b03a 100644 --- a/reclient_helper.py +++ b/reclient_helper.py @@ -19,65 +19,66 @@ import reclient_metrics def find_reclient_bin_dir(): - tools_path = gclient_paths.GetBuildtoolsPath() - if not tools_path: - return None + tools_path = gclient_paths.GetBuildtoolsPath() + if not tools_path: + return None - reclient_bin_dir = os.path.join(tools_path, 'reclient') - if os.path.isdir(reclient_bin_dir): - return reclient_bin_dir - return None + reclient_bin_dir = os.path.join(tools_path, 'reclient') + if os.path.isdir(reclient_bin_dir): + return reclient_bin_dir + return None def find_reclient_cfg(): - tools_path = gclient_paths.GetBuildtoolsPath() - if not tools_path: - return None + tools_path = gclient_paths.GetBuildtoolsPath() + if not tools_path: + return None - reclient_cfg = os.path.join(tools_path, 'reclient_cfgs', 'reproxy.cfg') - if os.path.isfile(reclient_cfg): - return reclient_cfg - return None + reclient_cfg = os.path.join(tools_path, 'reclient_cfgs', 'reproxy.cfg') + if os.path.isfile(reclient_cfg): + return reclient_cfg + return None def run(cmd_args): - if os.environ.get('NINJA_SUMMARIZE_BUILD') == '1': - print(' '.join(cmd_args)) - return subprocess.call(cmd_args) + if os.environ.get('NINJA_SUMMARIZE_BUILD') == '1': + print(' '.join(cmd_args)) + return subprocess.call(cmd_args) def start_reproxy(reclient_cfg, reclient_bin_dir): - return run([ - os.path.join(reclient_bin_dir, - 'bootstrap' + gclient_paths.GetExeSuffix()), '--re_proxy=' + - os.path.join(reclient_bin_dir, 'reproxy' + gclient_paths.GetExeSuffix()), - '--cfg=' + reclient_cfg - ]) + return run([ + os.path.join(reclient_bin_dir, + 'bootstrap' + gclient_paths.GetExeSuffix()), + '--re_proxy=' + os.path.join(reclient_bin_dir, + 'reproxy' + gclient_paths.GetExeSuffix()), + '--cfg=' + reclient_cfg + ]) def stop_reproxy(reclient_cfg, reclient_bin_dir): - return run([ - os.path.join(reclient_bin_dir, - 'bootstrap' + gclient_paths.GetExeSuffix()), '--shutdown', - '--cfg=' + reclient_cfg - ]) + return run([ + os.path.join(reclient_bin_dir, + 'bootstrap' + gclient_paths.GetExeSuffix()), '--shutdown', + '--cfg=' + reclient_cfg + ]) def find_ninja_out_dir(args): - # Ninja uses getopt_long, which allows to intermix non-option arguments. - # To leave non supported parameters untouched, we do not use getopt. - for index, arg in enumerate(args[1:]): - if arg == '-C': - # + 1 to get the next argument and +1 because we trimmed off args[0] - return args[index + 2] - if arg.startswith('-C'): - # Support -Cout/Default - return arg[2:] - return '.' + # Ninja uses getopt_long, which allows to intermix non-option arguments. + # To leave non supported parameters untouched, we do not use getopt. + for index, arg in enumerate(args[1:]): + if arg == '-C': + # + 1 to get the next argument and +1 because we trimmed off args[0] + return args[index + 2] + if arg.startswith('-C'): + # Support -Cout/Default + return arg[2:] + return '.' def find_cache_dir(tmp_dir): - """Helper to find the correct cache directory for a build. + """Helper to find the correct cache directory for a build. tmp_dir should be a build specific temp directory within the out directory. @@ -86,15 +87,15 @@ def find_cache_dir(tmp_dir): If this is not called from within a gclient checkout, the cache dir will be: tmp_dir/cache """ - gclient_root = gclient_paths.FindGclientRoot(os.getcwd()) - if gclient_root: - return os.path.join(gclient_root, '.reproxy_cache', - hashlib.md5(tmp_dir.encode()).hexdigest()) - return os.path.join(tmp_dir, 'cache') + gclient_root = gclient_paths.FindGclientRoot(os.getcwd()) + if gclient_root: + return os.path.join(gclient_root, '.reproxy_cache', + hashlib.md5(tmp_dir.encode()).hexdigest()) + return os.path.join(tmp_dir, 'cache') def set_reproxy_metrics_flags(tool): - """Helper to setup metrics collection flags for reproxy. + """Helper to setup metrics collection flags for reproxy. The following env vars are set if not already set: RBE_metrics_project=chromium-reclient-metrics @@ -103,23 +104,23 @@ def set_reproxy_metrics_flags(tool): RBE_metrics_labels=source=developer,tool={tool} RBE_metrics_prefix=go.chromium.org """ - autoninja_id = os.environ.get("AUTONINJA_BUILD_ID") - if autoninja_id is not None: - os.environ.setdefault("RBE_invocation_id", autoninja_id) - os.environ.setdefault("RBE_metrics_project", "chromium-reclient-metrics") - os.environ.setdefault("RBE_metrics_table", "rbe_metrics.builds") - os.environ.setdefault("RBE_metrics_labels", "source=developer,tool=" + tool) - os.environ.setdefault("RBE_metrics_prefix", "go.chromium.org") + autoninja_id = os.environ.get("AUTONINJA_BUILD_ID") + if autoninja_id is not None: + os.environ.setdefault("RBE_invocation_id", autoninja_id) + os.environ.setdefault("RBE_metrics_project", "chromium-reclient-metrics") + os.environ.setdefault("RBE_metrics_table", "rbe_metrics.builds") + os.environ.setdefault("RBE_metrics_labels", "source=developer,tool=" + tool) + os.environ.setdefault("RBE_metrics_prefix", "go.chromium.org") def remove_mdproxy_from_path(): - os.environ["PATH"] = os.pathsep.join( - d for d in os.environ.get("PATH", "").split(os.pathsep) - if "mdproxy" not in d) + os.environ["PATH"] = os.pathsep.join( + d for d in os.environ.get("PATH", "").split(os.pathsep) + if "mdproxy" not in d) def set_reproxy_path_flags(out_dir, make_dirs=True): - """Helper to setup the logs and cache directories for reclient. + """Helper to setup the logs and cache directories for reclient. Creates the following directory structure if make_dirs is true: If in a gclient checkout @@ -146,98 +147,100 @@ def set_reproxy_path_flags(out_dir, make_dirs=True): Windows Only: RBE_server_address=pipe://md5(out_dir/.reproxy_tmp)/reproxy.pipe """ - tmp_dir = os.path.abspath(os.path.join(out_dir, '.reproxy_tmp')) - log_dir = os.path.join(tmp_dir, 'logs') - racing_dir = os.path.join(tmp_dir, 'racing') - cache_dir = find_cache_dir(tmp_dir) - if make_dirs: - if os.path.exists(log_dir): - try: - # Clear log dir before each build to ensure correct metric aggregation. - shutil.rmtree(log_dir) - except OSError: - print( - "Couldn't clear logs because reproxy did " - "not shutdown after the last build", - file=sys.stderr) - os.makedirs(tmp_dir, exist_ok=True) - os.makedirs(log_dir, exist_ok=True) - os.makedirs(cache_dir, exist_ok=True) - os.makedirs(racing_dir, exist_ok=True) - os.environ.setdefault("RBE_output_dir", log_dir) - os.environ.setdefault("RBE_proxy_log_dir", log_dir) - os.environ.setdefault("RBE_log_dir", log_dir) - os.environ.setdefault("RBE_cache_dir", cache_dir) - os.environ.setdefault("RBE_racing_tmp_dir", racing_dir) - if sys.platform.startswith('win'): - pipe_dir = hashlib.md5(tmp_dir.encode()).hexdigest() - os.environ.setdefault("RBE_server_address", - "pipe://%s/reproxy.pipe" % pipe_dir) - else: - # unix domain socket has path length limit, so use fixed size path here. - # ref: https://www.man7.org/linux/man-pages/man7/unix.7.html - os.environ.setdefault( - "RBE_server_address", "unix:///tmp/reproxy_%s.sock" % - hashlib.sha256(tmp_dir.encode()).hexdigest()) + tmp_dir = os.path.abspath(os.path.join(out_dir, '.reproxy_tmp')) + log_dir = os.path.join(tmp_dir, 'logs') + racing_dir = os.path.join(tmp_dir, 'racing') + cache_dir = find_cache_dir(tmp_dir) + if make_dirs: + if os.path.exists(log_dir): + try: + # Clear log dir before each build to ensure correct metric + # aggregation. + shutil.rmtree(log_dir) + except OSError: + print( + "Couldn't clear logs because reproxy did " + "not shutdown after the last build", + file=sys.stderr) + os.makedirs(tmp_dir, exist_ok=True) + os.makedirs(log_dir, exist_ok=True) + os.makedirs(cache_dir, exist_ok=True) + os.makedirs(racing_dir, exist_ok=True) + os.environ.setdefault("RBE_output_dir", log_dir) + os.environ.setdefault("RBE_proxy_log_dir", log_dir) + os.environ.setdefault("RBE_log_dir", log_dir) + os.environ.setdefault("RBE_cache_dir", cache_dir) + os.environ.setdefault("RBE_racing_tmp_dir", racing_dir) + if sys.platform.startswith('win'): + pipe_dir = hashlib.md5(tmp_dir.encode()).hexdigest() + os.environ.setdefault("RBE_server_address", + "pipe://%s/reproxy.pipe" % pipe_dir) + else: + # unix domain socket has path length limit, so use fixed size path here. + # ref: https://www.man7.org/linux/man-pages/man7/unix.7.html + os.environ.setdefault( + "RBE_server_address", "unix:///tmp/reproxy_%s.sock" % + hashlib.sha256(tmp_dir.encode()).hexdigest()) def set_racing_defaults(): - os.environ.setdefault("RBE_local_resource_fraction", "0.2") - os.environ.setdefault("RBE_racing_bias", "0.95") + os.environ.setdefault("RBE_local_resource_fraction", "0.2") + os.environ.setdefault("RBE_racing_bias", "0.95") @contextlib.contextmanager def build_context(argv, tool): - # If use_remoteexec is set, but the reclient binaries or configs don't - # exist, display an error message and stop. Otherwise, the build will - # attempt to run with rewrapper wrapping actions, but will fail with - # possible non-obvious problems. - reclient_bin_dir = find_reclient_bin_dir() - reclient_cfg = find_reclient_cfg() - if reclient_bin_dir is None or reclient_cfg is None: - print(('Build is configured to use reclient but necessary binaries ' - "or config files can't be found.\n" - 'Please check if `"download_remoteexec_cfg": True` custom var is set' - ' in `.gclient`, and run `gclient sync`.'), - file=sys.stderr) - yield 1 - return + # If use_remoteexec is set, but the reclient binaries or configs don't + # exist, display an error message and stop. Otherwise, the build will + # attempt to run with rewrapper wrapping actions, but will fail with + # possible non-obvious problems. + reclient_bin_dir = find_reclient_bin_dir() + reclient_cfg = find_reclient_cfg() + if reclient_bin_dir is None or reclient_cfg is None: + print( + 'Build is configured to use reclient but necessary binaries ' + "or config files can't be found.\n" + 'Please check if `"download_remoteexec_cfg": True` custom var is ' + 'set in `.gclient`, and run `gclient sync`.', + file=sys.stderr) + yield 1 + return - ninja_out = find_ninja_out_dir(argv) + ninja_out = find_ninja_out_dir(argv) - try: - set_reproxy_path_flags(ninja_out) - except OSError: - print("Error creating reproxy_tmp in output dir", file=sys.stderr) - yield 1 - return + try: + set_reproxy_path_flags(ninja_out) + except OSError: + print("Error creating reproxy_tmp in output dir", file=sys.stderr) + yield 1 + return - if reclient_metrics.check_status(ninja_out): - set_reproxy_metrics_flags(tool) + if reclient_metrics.check_status(ninja_out): + set_reproxy_metrics_flags(tool) - if os.environ.get('RBE_instance', None): - print('WARNING: Using RBE_instance=%s\n' % - os.environ.get('RBE_instance', '')) + if os.environ.get('RBE_instance', None): + print('WARNING: Using RBE_instance=%s\n' % + os.environ.get('RBE_instance', '')) - remote_disabled = os.environ.get('RBE_remote_disabled') - if remote_disabled not in ('1', 't', 'T', 'true', 'TRUE', 'True'): - set_racing_defaults() + remote_disabled = os.environ.get('RBE_remote_disabled') + if remote_disabled not in ('1', 't', 'T', 'true', 'TRUE', 'True'): + set_racing_defaults() - # TODO(b/292523514) remove this once a fix is landed in reproxy - remove_mdproxy_from_path() + # TODO(b/292523514) remove this once a fix is landed in reproxy + remove_mdproxy_from_path() - start = time.time() - reproxy_ret_code = start_reproxy(reclient_cfg, reclient_bin_dir) - elapsed = time.time() - start - print('%1.3f s to start reproxy' % elapsed) - if reproxy_ret_code != 0: - yield reproxy_ret_code - return - try: - yield - finally: - print("Shutting down reproxy...", file=sys.stderr) start = time.time() - stop_reproxy(reclient_cfg, reclient_bin_dir) + reproxy_ret_code = start_reproxy(reclient_cfg, reclient_bin_dir) elapsed = time.time() - start - print('%1.3f s to stop reproxy' % elapsed) + print('%1.3f s to start reproxy' % elapsed) + if reproxy_ret_code != 0: + yield reproxy_ret_code + return + try: + yield + finally: + print("Shutting down reproxy...", file=sys.stderr) + start = time.time() + stop_reproxy(reclient_cfg, reclient_bin_dir) + elapsed = time.time() - start + print('%1.3f s to stop reproxy' % elapsed) diff --git a/reclient_metrics.py b/reclient_metrics.py index 51164b66a0..c6158d69ee 100755 --- a/reclient_metrics.py +++ b/reclient_metrics.py @@ -16,36 +16,36 @@ VERSION = 1 def default_config(): - return { - 'is-googler': is_googler(), - 'countdown': 10, - 'version': VERSION, - } + return { + 'is-googler': is_googler(), + 'countdown': 10, + 'version': VERSION, + } def load_config(): - config = None - try: - with open(CONFIG) as f: - raw_config = json.load(f) - if raw_config['version'] == VERSION: - raw_config['countdown'] = max(0, raw_config['countdown'] - 1) - config = raw_config - except Exception: - pass - if not config: - config = default_config() - save_config(config) - return config + config = None + try: + with open(CONFIG) as f: + raw_config = json.load(f) + if raw_config['version'] == VERSION: + raw_config['countdown'] = max(0, raw_config['countdown'] - 1) + config = raw_config + except Exception: + pass + if not config: + config = default_config() + save_config(config) + return config def save_config(config): - with open(CONFIG, 'w') as f: - json.dump(config, f) + with open(CONFIG, 'w') as f: + json.dump(config, f) def show_message(config, ninja_out): - print(""" + print(""" Your reclient metrics will be uploaded to the chromium build metrics database. The uploaded metrics will be used to analyze user side build performance. We upload the contents of {ninja_out_abs}. @@ -73,71 +73,71 @@ You can find a more detailed explanation in or https://chromium.googlesource.com/chromium/tools/depot_tools/+/main/reclient_metrics.README.md """.format( - ninja_out_abs=os.path.abspath( - os.path.join(ninja_out, ".reproxy_tmp", "logs", "rbe_metrics.txt")), - config_count=config.get("countdown", 0), - file_path=__file__, - metrics_readme_path=os.path.abspath( - os.path.join(THIS_DIR, "reclient_metrics.README.md")), - )) + ninja_out_abs=os.path.abspath( + os.path.join(ninja_out, ".reproxy_tmp", "logs", "rbe_metrics.txt")), + config_count=config.get("countdown", 0), + file_path=__file__, + metrics_readme_path=os.path.abspath( + os.path.join(THIS_DIR, "reclient_metrics.README.md")), + )) def is_googler(config=None): - """Check whether this user is Googler or not.""" - if config is not None and 'is-googler' in config: - return config['is-googler'] - # Use cipd auth-info to check for googler status as - # downloading rewrapper configs already requires cipd to be logged in - p = subprocess.run('cipd auth-info', - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - shell=True) - if p.returncode != 0: - return False - lines = p.stdout.splitlines() - if len(lines) == 0: - return False - l = lines[0] - # |l| will be like 'Logged in as @google.com.' for googlers. - return l.startswith('Logged in as ') and l.endswith('@google.com.') + """Check whether this user is Googler or not.""" + if config is not None and 'is-googler' in config: + return config['is-googler'] + # Use cipd auth-info to check for googler status as + # downloading rewrapper configs already requires cipd to be logged in + p = subprocess.run('cipd auth-info', + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + shell=True) + if p.returncode != 0: + return False + lines = p.stdout.splitlines() + if len(lines) == 0: + return False + l = lines[0] + # |l| will be like 'Logged in as @google.com.' for googlers. + return l.startswith('Logged in as ') and l.endswith('@google.com.') def check_status(ninja_out): - """Checks metrics collections status and shows notice to user if needed. + """Checks metrics collections status and shows notice to user if needed. Returns True if metrics should be collected.""" - config = load_config() - if not is_googler(config): - return False - if 'opt-in' in config: - return config['opt-in'] - if config.get("countdown", 0) > 0: - show_message(config, ninja_out) - return False - return True + config = load_config() + if not is_googler(config): + return False + if 'opt-in' in config: + return config['opt-in'] + if config.get("countdown", 0) > 0: + show_message(config, ninja_out) + return False + return True def main(argv): - cfg = load_config() + cfg = load_config() - if not is_googler(cfg): - save_config(cfg) - return 0 + if not is_googler(cfg): + save_config(cfg) + return 0 - if len(argv) == 2 and argv[1] == 'opt-in': - cfg['opt-in'] = True - cfg['countdown'] = 0 - save_config(cfg) - print('reclient metrics upload is opted in.') - return 0 + if len(argv) == 2 and argv[1] == 'opt-in': + cfg['opt-in'] = True + cfg['countdown'] = 0 + save_config(cfg) + print('reclient metrics upload is opted in.') + return 0 - if len(argv) == 2 and argv[1] == 'opt-out': - cfg['opt-in'] = False - save_config(cfg) - print('reclient metrics upload is opted out.') - return 0 + if len(argv) == 2 and argv[1] == 'opt-out': + cfg['opt-in'] = False + save_config(cfg) + print('reclient metrics upload is opted out.') + return 0 if __name__ == '__main__': - sys.exit(main(sys.argv)) + sys.exit(main(sys.argv)) diff --git a/reclientreport.py b/reclientreport.py index 8f514520a8..69eaca9f4b 100644 --- a/reclientreport.py +++ b/reclientreport.py @@ -21,43 +21,43 @@ import reclient_helper # TODO(b/296402157): Remove once reclientreport binary saves all logs on windows def temp_win_impl__b_296402157(out_dir): - '''Temporary implementation until b/296402157 is fixed''' - log_dir = os.path.abspath(os.path.join(out_dir, '.reproxy_tmp', 'logs')) - with tempfile.NamedTemporaryFile(prefix='reclientreport', - suffix='.tar.gz', - delete=False) as f: - with tarfile.open(fileobj=f, mode='w:gz') as tar: - tar.add(log_dir, arcname=os.path.basename(log_dir)) - print( - f'Created log file at {f.name}. Please attach this to your bug report!') + '''Temporary implementation until b/296402157 is fixed''' + log_dir = os.path.abspath(os.path.join(out_dir, '.reproxy_tmp', 'logs')) + with tempfile.NamedTemporaryFile(prefix='reclientreport', + suffix='.tar.gz', + delete=False) as f: + with tarfile.open(fileobj=f, mode='w:gz') as tar: + tar.add(log_dir, arcname=os.path.basename(log_dir)) + print(f'Created log file at {f.name}. Please attach this to your bug ' + 'report!') def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--ninja_out", - "-C", - required=True, - help="ninja out directory used for the autoninja build") - parser.add_argument('args', nargs=argparse.REMAINDER) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--ninja_out", + "-C", + required=True, + help="ninja out directory used for the autoninja build") + parser.add_argument('args', nargs=argparse.REMAINDER) - args, extras = parser.parse_known_args() - if sys.platform.startswith('win'): - temp_win_impl__b_296402157(args.ninja_out) - return - if args.args and args.args[0] == '--': - args.args.pop(0) - if extras: - args.args = extras + args.args + args, extras = parser.parse_known_args() + if sys.platform.startswith('win'): + temp_win_impl__b_296402157(args.ninja_out) + return + if args.args and args.args[0] == '--': + args.args.pop(0) + if extras: + args.args = extras + args.args - reclient_helper.set_reproxy_path_flags(args.ninja_out, make_dirs=False) - reclient_bin_dir = reclient_helper.find_reclient_bin_dir() - code = subprocess.call([os.path.join(reclient_bin_dir, 'reclientreport')] + - args.args) - if code != 0: - print("Failed to collect logs, make sure that %s/.reproxy_tmp exists" % - args.ninja_out, - file=sys.stderr) + reclient_helper.set_reproxy_path_flags(args.ninja_out, make_dirs=False) + reclient_bin_dir = reclient_helper.find_reclient_bin_dir() + code = subprocess.call([os.path.join(reclient_bin_dir, 'reclientreport')] + + args.args) + if code != 0: + print("Failed to collect logs, make sure that %s/.reproxy_tmp exists" % + args.ninja_out, + file=sys.stderr) if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/repo b/repo index 9dc4dc5e76..9f8357d87d 100755 --- a/repo +++ b/repo @@ -2,7 +2,6 @@ # Copyright 2020 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Wrapper around repo to auto-update depot_tools during sync. gclient keeps depot_tools up-to-date automatically for Chromium developers. @@ -16,7 +15,6 @@ from pathlib import Path import subprocess import sys - # Some useful paths. DEPOT_TOOLS_DIR = Path(__file__).resolve().parent UPDATE_DEPOT_TOOLS = DEPOT_TOOLS_DIR / 'update_depot_tools' @@ -24,35 +22,37 @@ REPO = DEPOT_TOOLS_DIR / 'repo_launcher' def _UpdateDepotTools(): - """Help CrOS users keep their depot_tools checkouts up-to-date.""" - if os.getenv('DEPOT_TOOLS_UPDATE') == '0': - return + """Help CrOS users keep their depot_tools checkouts up-to-date.""" + if os.getenv('DEPOT_TOOLS_UPDATE') == '0': + return - # We don't update the copy that's part of the CrOS repo client checkout. - path = DEPOT_TOOLS_DIR - while path != path.parent: - if (path / '.repo').is_dir() and (path / 'chromite').is_dir(): - return - path = path.parent + # We don't update the copy that's part of the CrOS repo client checkout. + path = DEPOT_TOOLS_DIR + while path != path.parent: + if (path / '.repo').is_dir() and (path / 'chromite').is_dir(): + return + path = path.parent - if UPDATE_DEPOT_TOOLS.exists(): - subprocess.run([UPDATE_DEPOT_TOOLS], check=True) - else: - print(f'warning: {UPDATE_DEPOT_TOOLS} does not exist; export ' - 'DEPOT_TOOLS_UPDATE=0 to disable.', file=sys.stderr) + if UPDATE_DEPOT_TOOLS.exists(): + subprocess.run([UPDATE_DEPOT_TOOLS], check=True) + else: + print( + f'warning: {UPDATE_DEPOT_TOOLS} does not exist; export ' + 'DEPOT_TOOLS_UPDATE=0 to disable.', + file=sys.stderr) def main(argv): - # This is a bit hacky, but should be "good enough". If repo itself gains - # support for sync hooks, we could switch to that. - if argv and argv[0] == 'sync': - _UpdateDepotTools() + # This is a bit hacky, but should be "good enough". If repo itself gains + # support for sync hooks, we could switch to that. + if argv and argv[0] == 'sync': + _UpdateDepotTools() - # Set the default to our fork. - os.environ["REPO_URL"] = "https://chromium.googlesource.com/external/repo" + # Set the default to our fork. + os.environ["REPO_URL"] = "https://chromium.googlesource.com/external/repo" - os.execv(sys.executable, [sys.executable, str(REPO)] + argv) + os.execv(sys.executable, [sys.executable, str(REPO)] + argv) if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) + sys.exit(main(sys.argv[1:])) diff --git a/roll_dep.py b/roll_dep.py index 266bca4856..fc15c6a4dc 100755 --- a/roll_dep.py +++ b/roll_dep.py @@ -2,7 +2,6 @@ # Copyright 2015 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Rolls DEPS controlled dependency. Works only with git checkout and git dependencies. Currently this script will @@ -20,9 +19,8 @@ import sys import tempfile NEED_SHELL = sys.platform.startswith('win') -GCLIENT_PATH = os.path.join( - os.path.dirname(os.path.abspath(__file__)), 'gclient.py') - +GCLIENT_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'gclient.py') # Commit subject that will be considered a roll. In the format generated by the # git log used, so it's "-- " @@ -33,309 +31,318 @@ _ROLL_SUBJECT = re.compile( r'[^ ]+ ' # Subject r'(' - # Generated by - # https://skia.googlesource.com/buildbot/+/HEAdA/autoroll/go/repo_manager/deps_repo_manager.go - r'Roll [^ ]+ [a-f0-9]+\.\.[a-f0-9]+ \(\d+ commits\)' - r'|' - # Generated by - # https://chromium.googlesource.com/infra/infra/+/HEAD/recipes/recipe_modules/recipe_autoroller/api.py - r'Roll recipe dependencies \(trivial\)\.' + # Generated by + # https://skia.googlesource.com/buildbot/+/HEAdA/autoroll/go/repo_manager/deps_repo_manager.go + r'Roll [^ ]+ [a-f0-9]+\.\.[a-f0-9]+ \(\d+ commits\)' + r'|' + # Generated by + # https://chromium.googlesource.com/infra/infra/+/HEAD/recipes/recipe_modules/recipe_autoroller/api.py + r'Roll recipe dependencies \(trivial\)\.' r')$') class Error(Exception): - pass + pass class AlreadyRolledError(Error): - pass + pass def check_output(*args, **kwargs): - """subprocess2.check_output() passing shell=True on Windows for git.""" - kwargs.setdefault('shell', NEED_SHELL) - return subprocess2.check_output(*args, **kwargs).decode('utf-8') + """subprocess2.check_output() passing shell=True on Windows for git.""" + kwargs.setdefault('shell', NEED_SHELL) + return subprocess2.check_output(*args, **kwargs).decode('utf-8') def check_call(*args, **kwargs): - """subprocess2.check_call() passing shell=True on Windows for git.""" - kwargs.setdefault('shell', NEED_SHELL) - subprocess2.check_call(*args, **kwargs) + """subprocess2.check_call() passing shell=True on Windows for git.""" + kwargs.setdefault('shell', NEED_SHELL) + subprocess2.check_call(*args, **kwargs) def return_code(*args, **kwargs): - """subprocess2.call() passing shell=True on Windows for git and + """subprocess2.call() passing shell=True on Windows for git and subprocess2.DEVNULL for stdout and stderr.""" - kwargs.setdefault('shell', NEED_SHELL) - kwargs.setdefault('stdout', subprocess2.DEVNULL) - kwargs.setdefault('stderr', subprocess2.DEVNULL) - return subprocess2.call(*args, **kwargs) + kwargs.setdefault('shell', NEED_SHELL) + kwargs.setdefault('stdout', subprocess2.DEVNULL) + kwargs.setdefault('stderr', subprocess2.DEVNULL) + return subprocess2.call(*args, **kwargs) def is_pristine(root): - """Returns True if a git checkout is pristine.""" - # `git rev-parse --verify` has a non-zero return code if the revision - # doesn't exist. - diff_cmd = ['git', 'diff', '--ignore-submodules', 'origin/main'] - return (not check_output(diff_cmd, cwd=root).strip() and - not check_output(diff_cmd + ['--cached'], cwd=root).strip()) - + """Returns True if a git checkout is pristine.""" + # `git rev-parse --verify` has a non-zero return code if the revision + # doesn't exist. + diff_cmd = ['git', 'diff', '--ignore-submodules', 'origin/main'] + return (not check_output(diff_cmd, cwd=root).strip() + and not check_output(diff_cmd + ['--cached'], cwd=root).strip()) def get_log_url(upstream_url, head, tot): - """Returns an URL to read logs via a Web UI if applicable.""" - if re.match(r'https://[^/]*\.googlesource\.com/', upstream_url): - # gitiles - return '%s/+log/%s..%s' % (upstream_url, head[:12], tot[:12]) - if upstream_url.startswith('https://github.com/'): - upstream_url = upstream_url.rstrip('/') - if upstream_url.endswith('.git'): - upstream_url = upstream_url[:-len('.git')] - return '%s/compare/%s...%s' % (upstream_url, head[:12], tot[:12]) - return None + """Returns an URL to read logs via a Web UI if applicable.""" + if re.match(r'https://[^/]*\.googlesource\.com/', upstream_url): + # gitiles + return '%s/+log/%s..%s' % (upstream_url, head[:12], tot[:12]) + if upstream_url.startswith('https://github.com/'): + upstream_url = upstream_url.rstrip('/') + if upstream_url.endswith('.git'): + upstream_url = upstream_url[:-len('.git')] + return '%s/compare/%s...%s' % (upstream_url, head[:12], tot[:12]) + return None def should_show_log(upstream_url): - """Returns True if a short log should be included in the tree.""" - # Skip logs for very active projects. - if upstream_url.endswith('/v8/v8.git'): - return False - if 'webrtc' in upstream_url: - return False - return True + """Returns True if a short log should be included in the tree.""" + # Skip logs for very active projects. + if upstream_url.endswith('/v8/v8.git'): + return False + if 'webrtc' in upstream_url: + return False + return True def gclient(args): - """Executes gclient with the given args and returns the stdout.""" - return check_output([sys.executable, GCLIENT_PATH] + args).strip() + """Executes gclient with the given args and returns the stdout.""" + return check_output([sys.executable, GCLIENT_PATH] + args).strip() -def generate_commit_message( - full_dir, dependency, head, roll_to, no_log, log_limit): - """Creates the commit message for this specific roll.""" - commit_range = '%s..%s' % (head, roll_to) - commit_range_for_header = '%s..%s' % (head[:9], roll_to[:9]) - upstream_url = check_output( - ['git', 'config', 'remote.origin.url'], cwd=full_dir).strip() - log_url = get_log_url(upstream_url, head, roll_to) - cmd = ['git', 'log', commit_range, '--date=short', '--no-merges'] - logs = check_output( - # Args with '=' are automatically quoted. - cmd + ['--format=%ad %ae %s', '--'], - cwd=full_dir).rstrip() - logs = re.sub(r'(?m)^(\d\d\d\d-\d\d-\d\d [^@]+)@[^ ]+( .*)$', r'\1\2', logs) - lines = logs.splitlines() - cleaned_lines = [l for l in lines if not _ROLL_SUBJECT.match(l)] - logs = '\n'.join(cleaned_lines) + '\n' +def generate_commit_message(full_dir, dependency, head, roll_to, no_log, + log_limit): + """Creates the commit message for this specific roll.""" + commit_range = '%s..%s' % (head, roll_to) + commit_range_for_header = '%s..%s' % (head[:9], roll_to[:9]) + upstream_url = check_output(['git', 'config', 'remote.origin.url'], + cwd=full_dir).strip() + log_url = get_log_url(upstream_url, head, roll_to) + cmd = ['git', 'log', commit_range, '--date=short', '--no-merges'] + logs = check_output( + # Args with '=' are automatically quoted. + cmd + ['--format=%ad %ae %s', '--'], + cwd=full_dir).rstrip() + logs = re.sub(r'(?m)^(\d\d\d\d-\d\d-\d\d [^@]+)@[^ ]+( .*)$', r'\1\2', logs) + lines = logs.splitlines() + cleaned_lines = [l for l in lines if not _ROLL_SUBJECT.match(l)] + logs = '\n'.join(cleaned_lines) + '\n' - nb_commits = len(lines) - rolls = nb_commits - len(cleaned_lines) - header = 'Roll %s/ %s (%d commit%s%s)\n\n' % ( - dependency, - commit_range_for_header, - nb_commits, - 's' if nb_commits > 1 else '', - ('; %s trivial rolls' % rolls) if rolls else '') - log_section = '' - if log_url: - log_section = log_url + '\n\n' - log_section += '$ %s ' % ' '.join(cmd) - log_section += '--format=\'%ad %ae %s\'\n' - log_section = log_section.replace(commit_range, commit_range_for_header) - # It is important that --no-log continues to work, as it is used by - # internal -> external rollers. Please do not remove or break it. - if not no_log and should_show_log(upstream_url): - if len(cleaned_lines) > log_limit: - # Keep the first N/2 log entries and last N/2 entries. - lines = logs.splitlines(True) - lines = lines[:log_limit//2] + ['(...)\n'] + lines[-log_limit//2:] - logs = ''.join(lines) - log_section += logs - return header + log_section + nb_commits = len(lines) + rolls = nb_commits - len(cleaned_lines) + header = 'Roll %s/ %s (%d commit%s%s)\n\n' % ( + dependency, commit_range_for_header, nb_commits, + 's' if nb_commits > 1 else '', + ('; %s trivial rolls' % rolls) if rolls else '') + log_section = '' + if log_url: + log_section = log_url + '\n\n' + log_section += '$ %s ' % ' '.join(cmd) + log_section += '--format=\'%ad %ae %s\'\n' + log_section = log_section.replace(commit_range, commit_range_for_header) + # It is important that --no-log continues to work, as it is used by + # internal -> external rollers. Please do not remove or break it. + if not no_log and should_show_log(upstream_url): + if len(cleaned_lines) > log_limit: + # Keep the first N/2 log entries and last N/2 entries. + lines = logs.splitlines(True) + lines = lines[:log_limit // 2] + ['(...)\n' + ] + lines[-log_limit // 2:] + logs = ''.join(lines) + log_section += logs + return header + log_section def is_submoduled(): - """Returns true if gclient root has submodules""" - return os.path.isfile(os.path.join(gclient(['root']), ".gitmodules")) + """Returns true if gclient root has submodules""" + return os.path.isfile(os.path.join(gclient(['root']), ".gitmodules")) def get_submodule_rev(submodule): - """Returns revision of the given submodule path""" - rev_output = check_output(['git', 'submodule', 'status', submodule], - cwd=gclient(['root'])).strip() + """Returns revision of the given submodule path""" + rev_output = check_output(['git', 'submodule', 'status', submodule], + cwd=gclient(['root'])).strip() - # git submodule status returns all submodules with its rev in the - # pattern: `(+|-| )() (submodule.path)` - revision = rev_output.split(' ')[0] - return revision[1:] if revision[0] in ('+', '-') else revision + # git submodule status returns all submodules with its rev in the + # pattern: `(+|-| )() (submodule.path)` + revision = rev_output.split(' ')[0] + return revision[1:] if revision[0] in ('+', '-') else revision def calculate_roll(full_dir, dependency, roll_to): - """Calculates the roll for a dependency by processing gclient_dict, and + """Calculates the roll for a dependency by processing gclient_dict, and fetching the dependency via git. """ - # if the super-project uses submodules, get rev directly using git. - if is_submoduled(): - head = get_submodule_rev(dependency) - else: - head = gclient(['getdep', '-r', dependency]) - if not head: - raise Error('%s is unpinned.' % dependency) - check_call(['git', 'fetch', 'origin', '--quiet'], cwd=full_dir) - if roll_to == 'origin/HEAD': - check_output(['git', 'remote', 'set-head', 'origin', '-a'], cwd=full_dir) - - roll_to = check_output(['git', 'rev-parse', roll_to], cwd=full_dir).strip() - return head, roll_to + # if the super-project uses submodules, get rev directly using git. + if is_submoduled(): + head = get_submodule_rev(dependency) + else: + head = gclient(['getdep', '-r', dependency]) + if not head: + raise Error('%s is unpinned.' % dependency) + check_call(['git', 'fetch', 'origin', '--quiet'], cwd=full_dir) + if roll_to == 'origin/HEAD': + check_output(['git', 'remote', 'set-head', 'origin', '-a'], + cwd=full_dir) + roll_to = check_output(['git', 'rev-parse', roll_to], cwd=full_dir).strip() + return head, roll_to def gen_commit_msg(logs, cmdline, reviewers, bug): - """Returns the final commit message.""" - commit_msg = '' - if len(logs) > 1: - commit_msg = 'Rolling %d dependencies\n\n' % len(logs) - commit_msg += '\n\n'.join(logs) - commit_msg += '\nCreated with:\n ' + cmdline + '\n' - commit_msg += 'R=%s\n' % ','.join(reviewers) if reviewers else '' - commit_msg += '\nBug: %s\n' % bug if bug else '' - return commit_msg + """Returns the final commit message.""" + commit_msg = '' + if len(logs) > 1: + commit_msg = 'Rolling %d dependencies\n\n' % len(logs) + commit_msg += '\n\n'.join(logs) + commit_msg += '\nCreated with:\n ' + cmdline + '\n' + commit_msg += 'R=%s\n' % ','.join(reviewers) if reviewers else '' + commit_msg += '\nBug: %s\n' % bug if bug else '' + return commit_msg def finalize(commit_msg, current_dir, rolls): - """Commits changes to the DEPS file, then uploads a CL.""" - print('Commit message:') - print('\n'.join(' ' + i for i in commit_msg.splitlines())) + """Commits changes to the DEPS file, then uploads a CL.""" + print('Commit message:') + print('\n'.join(' ' + i for i in commit_msg.splitlines())) - # Pull the dependency to the right revision. This is surprising to users - # otherwise. The revision update is done before commiting to update - # submodule revision if present. - for dependency, (_head, roll_to, full_dir) in sorted(rolls.items()): - check_call(['git', 'checkout', '--quiet', roll_to], cwd=full_dir) + # Pull the dependency to the right revision. This is surprising to users + # otherwise. The revision update is done before commiting to update + # submodule revision if present. + for dependency, (_head, roll_to, full_dir) in sorted(rolls.items()): + check_call(['git', 'checkout', '--quiet', roll_to], cwd=full_dir) - # This adds the submodule revision update to the commit. - if is_submoduled(): - check_call([ - 'git', 'update-index', '--add', '--cacheinfo', '160000,{},{}'.format( - roll_to, dependency) - ], - cwd=current_dir) + # This adds the submodule revision update to the commit. + if is_submoduled(): + check_call([ + 'git', 'update-index', '--add', '--cacheinfo', + '160000,{},{}'.format(roll_to, dependency) + ], + cwd=current_dir) - check_call(['git', 'add', 'DEPS'], cwd=current_dir) - # We have to set delete=False and then let the object go out of scope so - # that the file can be opened by name on Windows. - with tempfile.NamedTemporaryFile('w+', newline='', delete=False) as f: - commit_filename = f.name - f.write(commit_msg) - check_call(['git', 'commit', '--quiet', '--file', commit_filename], - cwd=current_dir) - os.remove(commit_filename) + check_call(['git', 'add', 'DEPS'], cwd=current_dir) + # We have to set delete=False and then let the object go out of scope so + # that the file can be opened by name on Windows. + with tempfile.NamedTemporaryFile('w+', newline='', delete=False) as f: + commit_filename = f.name + f.write(commit_msg) + check_call(['git', 'commit', '--quiet', '--file', commit_filename], + cwd=current_dir) + os.remove(commit_filename) def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - '--ignore-dirty-tree', action='store_true', - help='Roll anyways, even if there is a diff.') - parser.add_argument( - '-r', - '--reviewer', - action='append', - help= - 'To specify multiple reviewers, either use a comma separated list, e.g. ' - '-r joe,jane,john or provide the flag multiple times, e.g. ' - '-r joe -r jane. Defaults to @chromium.org') - parser.add_argument('-b', '--bug', help='Associate a bug number to the roll') - # It is important that --no-log continues to work, as it is used by - # internal -> external rollers. Please do not remove or break it. - parser.add_argument( - '--no-log', action='store_true', - help='Do not include the short log in the commit message') - parser.add_argument( - '--log-limit', type=int, default=100, - help='Trim log after N commits (default: %(default)s)') - parser.add_argument( - '--roll-to', default='origin/HEAD', - help='Specify the new commit to roll to (default: %(default)s)') - parser.add_argument( - '--key', action='append', default=[], - help='Regex(es) for dependency in DEPS file') - parser.add_argument('dep_path', nargs='+', help='Path(s) to dependency') - args = parser.parse_args() + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--ignore-dirty-tree', + action='store_true', + help='Roll anyways, even if there is a diff.') + parser.add_argument( + '-r', + '--reviewer', + action='append', + help='To specify multiple reviewers, either use a comma separated ' + 'list, e.g. -r joe,jane,john or provide the flag multiple times, e.g. ' + '-r joe -r jane. Defaults to @chromium.org') + parser.add_argument('-b', + '--bug', + help='Associate a bug number to the roll') + # It is important that --no-log continues to work, as it is used by + # internal -> external rollers. Please do not remove or break it. + parser.add_argument( + '--no-log', + action='store_true', + help='Do not include the short log in the commit message') + parser.add_argument('--log-limit', + type=int, + default=100, + help='Trim log after N commits (default: %(default)s)') + parser.add_argument( + '--roll-to', + default='origin/HEAD', + help='Specify the new commit to roll to (default: %(default)s)') + parser.add_argument('--key', + action='append', + default=[], + help='Regex(es) for dependency in DEPS file') + parser.add_argument('dep_path', nargs='+', help='Path(s) to dependency') + args = parser.parse_args() - if len(args.dep_path) > 1: - if args.roll_to != 'origin/HEAD': - parser.error( - 'Can\'t use multiple paths to roll simultaneously and --roll-to') - if args.key: - parser.error( - 'Can\'t use multiple paths to roll simultaneously and --key') - reviewers = None - if args.reviewer: - reviewers = list(itertools.chain(*[r.split(',') for r in args.reviewer])) - for i, r in enumerate(reviewers): - if not '@' in r: - reviewers[i] = r + '@chromium.org' + if len(args.dep_path) > 1: + if args.roll_to != 'origin/HEAD': + parser.error( + 'Can\'t use multiple paths to roll simultaneously and --roll-to' + ) + if args.key: + parser.error( + 'Can\'t use multiple paths to roll simultaneously and --key') + reviewers = None + if args.reviewer: + reviewers = list(itertools.chain(*[r.split(',') + for r in args.reviewer])) + for i, r in enumerate(reviewers): + if not '@' in r: + reviewers[i] = r + '@chromium.org' - gclient_root = gclient(['root']) - current_dir = os.getcwd() - dependencies = sorted(d.replace('\\', '/').rstrip('/') for d in args.dep_path) - cmdline = 'roll-dep ' + ' '.join(dependencies) + ''.join( - ' --key ' + k for k in args.key) - try: - if not args.ignore_dirty_tree and not is_pristine(current_dir): - raise Error( - 'Ensure %s is clean first (no non-merged commits).' % current_dir) - # First gather all the information without modifying anything, except for a - # git fetch. - rolls = {} - for dependency in dependencies: - full_dir = os.path.normpath(os.path.join(gclient_root, dependency)) - if not os.path.isdir(full_dir): - print('Dependency %s not found at %s' % (dependency, full_dir)) - full_dir = os.path.normpath(os.path.join(current_dir, dependency)) - print('Will look for relative dependency at %s' % full_dir) - if not os.path.isdir(full_dir): - raise Error('Directory not found: %s (%s)' % (dependency, full_dir)) + gclient_root = gclient(['root']) + current_dir = os.getcwd() + dependencies = sorted( + d.replace('\\', '/').rstrip('/') for d in args.dep_path) + cmdline = 'roll-dep ' + ' '.join(dependencies) + ''.join(' --key ' + k + for k in args.key) + try: + if not args.ignore_dirty_tree and not is_pristine(current_dir): + raise Error('Ensure %s is clean first (no non-merged commits).' % + current_dir) + # First gather all the information without modifying anything, except + # for a git fetch. + rolls = {} + for dependency in dependencies: + full_dir = os.path.normpath(os.path.join(gclient_root, dependency)) + if not os.path.isdir(full_dir): + print('Dependency %s not found at %s' % (dependency, full_dir)) + full_dir = os.path.normpath( + os.path.join(current_dir, dependency)) + print('Will look for relative dependency at %s' % full_dir) + if not os.path.isdir(full_dir): + raise Error('Directory not found: %s (%s)' % + (dependency, full_dir)) - head, roll_to = calculate_roll(full_dir, dependency, args.roll_to) - if roll_to == head: - if len(dependencies) == 1: - raise AlreadyRolledError('No revision to roll!') - print('%s: Already at latest commit %s' % (dependency, roll_to)) - else: - print( - '%s: Rolling from %s to %s' % (dependency, head[:10], roll_to[:10])) - rolls[dependency] = (head, roll_to, full_dir) + head, roll_to = calculate_roll(full_dir, dependency, args.roll_to) + if roll_to == head: + if len(dependencies) == 1: + raise AlreadyRolledError('No revision to roll!') + print('%s: Already at latest commit %s' % (dependency, roll_to)) + else: + print('%s: Rolling from %s to %s' % + (dependency, head[:10], roll_to[:10])) + rolls[dependency] = (head, roll_to, full_dir) - logs = [] - setdep_args = [] - for dependency, (head, roll_to, full_dir) in sorted(rolls.items()): - log = generate_commit_message( - full_dir, dependency, head, roll_to, args.no_log, args.log_limit) - logs.append(log) - setdep_args.extend(['-r', '{}@{}'.format(dependency, roll_to)]) + logs = [] + setdep_args = [] + for dependency, (head, roll_to, full_dir) in sorted(rolls.items()): + log = generate_commit_message(full_dir, dependency, head, roll_to, + args.no_log, args.log_limit) + logs.append(log) + setdep_args.extend(['-r', '{}@{}'.format(dependency, roll_to)]) - # DEPS is updated even if the repository uses submodules. - gclient(['setdep'] + setdep_args) + # DEPS is updated even if the repository uses submodules. + gclient(['setdep'] + setdep_args) - commit_msg = gen_commit_msg(logs, cmdline, reviewers, args.bug) - finalize(commit_msg, current_dir, rolls) - except Error as e: - sys.stderr.write('error: %s\n' % e) - return 2 if isinstance(e, AlreadyRolledError) else 1 - except subprocess2.CalledProcessError: - return 1 + commit_msg = gen_commit_msg(logs, cmdline, reviewers, args.bug) + finalize(commit_msg, current_dir, rolls) + except Error as e: + sys.stderr.write('error: %s\n' % e) + return 2 if isinstance(e, AlreadyRolledError) else 1 + except subprocess2.CalledProcessError: + return 1 - print('') - if not reviewers: - print('You forgot to pass -r, make sure to insert a R=foo@example.com line') - print('to the commit description before emailing.') print('') - print('Run:') - print(' git cl upload --send-mail') - return 0 + if not reviewers: + print('You forgot to pass -r, make sure to insert a R=foo@example.com ' + 'line') + print('to the commit description before emailing.') + print('') + print('Run:') + print(' git cl upload --send-mail') + return 0 if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/rustfmt.py b/rustfmt.py index aae33a12d8..65246aef57 100755 --- a/rustfmt.py +++ b/rustfmt.py @@ -15,56 +15,56 @@ import sys class NotFoundError(Exception): - """A file could not be found.""" - - def __init__(self, e): - Exception.__init__( - self, 'Problem while looking for rustfmt in Chromium source tree:\n' - '%s' % e) + """A file could not be found.""" + def __init__(self, e): + Exception.__init__( + self, 'Problem while looking for rustfmt in Chromium source tree:\n' + '%s' % e) def FindRustfmtToolInChromiumTree(): - """Return a path to the rustfmt executable, or die trying.""" - chromium_src_path = gclient_paths.GetPrimarySolutionPath() - if not chromium_src_path: - raise NotFoundError( - 'Could not find checkout in any parent of the current path.\n' - 'Set CHROMIUM_BUILDTOOLS_PATH to use outside of a chromium checkout.') + """Return a path to the rustfmt executable, or die trying.""" + chromium_src_path = gclient_paths.GetPrimarySolutionPath() + if not chromium_src_path: + raise NotFoundError( + 'Could not find checkout in any parent of the current path.\n' + 'Set CHROMIUM_BUILDTOOLS_PATH to use outside of a chromium ' + 'checkout.') - tool_path = os.path.join(chromium_src_path, 'third_party', 'rust-toolchain', - 'bin', 'rustfmt' + gclient_paths.GetExeSuffix()) - if not os.path.exists(tool_path): - raise NotFoundError('File does not exist: %s' % tool_path) - return tool_path + tool_path = os.path.join(chromium_src_path, 'third_party', 'rust-toolchain', + 'bin', 'rustfmt' + gclient_paths.GetExeSuffix()) + if not os.path.exists(tool_path): + raise NotFoundError('File does not exist: %s' % tool_path) + return tool_path def IsRustfmtSupported(): - try: - FindRustfmtToolInChromiumTree() - return True - except NotFoundError: - return False + try: + FindRustfmtToolInChromiumTree() + return True + except NotFoundError: + return False def main(args): - try: - tool = FindRustfmtToolInChromiumTree() - except NotFoundError as e: - sys.stderr.write("%s\n" % str(e)) - return 1 + try: + tool = FindRustfmtToolInChromiumTree() + except NotFoundError as e: + sys.stderr.write("%s\n" % str(e)) + return 1 - # Add some visibility to --help showing where the tool lives, since this - # redirection can be a little opaque. - help_syntax = ('-h', '--help', '-help', '-help-list', '--help-list') - if any(match in args for match in help_syntax): - print('\nDepot tools redirects you to the rustfmt at:\n %s\n' % tool) + # Add some visibility to --help showing where the tool lives, since this + # redirection can be a little opaque. + help_syntax = ('-h', '--help', '-help', '-help-list', '--help-list') + if any(match in args for match in help_syntax): + print('\nDepot tools redirects you to the rustfmt at:\n %s\n' % tool) - return subprocess.call([tool] + args) + return subprocess.call([tool] + args) if __name__ == '__main__': - try: - sys.exit(main(sys.argv[1:])) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(1) + try: + sys.exit(main(sys.argv[1:])) + except KeyboardInterrupt: + sys.stderr.write('interrupted\n') + sys.exit(1) diff --git a/scm.py b/scm.py index 6921824ffb..904f67c887 100644 --- a/scm.py +++ b/scm.py @@ -1,7 +1,6 @@ # Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """SCM-specific utility classes.""" import distutils.version @@ -15,463 +14,478 @@ import sys import gclient_utils import subprocess2 +# TODO: Should fix these warnings. +# pylint: disable=line-too-long + def ValidateEmail(email): - return ( - re.match(r"^[a-zA-Z0-9._%\-+]+@[a-zA-Z0-9._%-]+.[a-zA-Z]{2,6}$", email) - is not None) + return (re.match(r"^[a-zA-Z0-9._%\-+]+@[a-zA-Z0-9._%-]+.[a-zA-Z]{2,6}$", + email) is not None) def GetCasedPath(path): - """Elcheapos way to get the real path case on Windows.""" - if sys.platform.startswith('win') and os.path.exists(path): - # Reconstruct the path. - path = os.path.abspath(path) - paths = path.split('\\') - for i in range(len(paths)): - if i == 0: - # Skip drive letter. - continue - subpath = '\\'.join(paths[:i+1]) - prev = len('\\'.join(paths[:i])) - # glob.glob will return the cased path for the last item only. This is why - # we are calling it in a loop. Extract the data we want and put it back - # into the list. - paths[i] = glob.glob(subpath + '*')[0][prev+1:len(subpath)] - path = '\\'.join(paths) - return path + """Elcheapos way to get the real path case on Windows.""" + if sys.platform.startswith('win') and os.path.exists(path): + # Reconstruct the path. + path = os.path.abspath(path) + paths = path.split('\\') + for i in range(len(paths)): + if i == 0: + # Skip drive letter. + continue + subpath = '\\'.join(paths[:i + 1]) + prev = len('\\'.join(paths[:i])) + # glob.glob will return the cased path for the last item only. This + # is why we are calling it in a loop. Extract the data we want and + # put it back into the list. + paths[i] = glob.glob(subpath + '*')[0][prev + 1:len(subpath)] + path = '\\'.join(paths) + return path def GenFakeDiff(filename): - """Generates a fake diff from a file.""" - file_content = gclient_utils.FileRead(filename, 'rb').splitlines(True) - filename = filename.replace(os.sep, '/') - nb_lines = len(file_content) - # We need to use / since patch on unix will fail otherwise. - data = io.StringIO() - data.write("Index: %s\n" % filename) - data.write('=' * 67 + '\n') - # Note: Should we use /dev/null instead? - data.write("--- %s\n" % filename) - data.write("+++ %s\n" % filename) - data.write("@@ -0,0 +1,%d @@\n" % nb_lines) - # Prepend '+' to every lines. - for line in file_content: - data.write('+') - data.write(line) - result = data.getvalue() - data.close() - return result + """Generates a fake diff from a file.""" + file_content = gclient_utils.FileRead(filename, 'rb').splitlines(True) + filename = filename.replace(os.sep, '/') + nb_lines = len(file_content) + # We need to use / since patch on unix will fail otherwise. + data = io.StringIO() + data.write("Index: %s\n" % filename) + data.write('=' * 67 + '\n') + # Note: Should we use /dev/null instead? + data.write("--- %s\n" % filename) + data.write("+++ %s\n" % filename) + data.write("@@ -0,0 +1,%d @@\n" % nb_lines) + # Prepend '+' to every lines. + for line in file_content: + data.write('+') + data.write(line) + result = data.getvalue() + data.close() + return result def determine_scm(root): - """Similar to upload.py's version but much simpler. + """Similar to upload.py's version but much simpler. Returns 'git' or None. """ - if os.path.isdir(os.path.join(root, '.git')): - return 'git' + if os.path.isdir(os.path.join(root, '.git')): + return 'git' - try: - subprocess2.check_call( - ['git', 'rev-parse', '--show-cdup'], - stdout=subprocess2.DEVNULL, - stderr=subprocess2.DEVNULL, - cwd=root) - return 'git' - except (OSError, subprocess2.CalledProcessError): - return None + try: + subprocess2.check_call(['git', 'rev-parse', '--show-cdup'], + stdout=subprocess2.DEVNULL, + stderr=subprocess2.DEVNULL, + cwd=root) + return 'git' + except (OSError, subprocess2.CalledProcessError): + return None def only_int(val): - if val.isdigit(): - return int(val) + if val.isdigit(): + return int(val) - return 0 + return 0 class GIT(object): - current_version = None + current_version = None - @staticmethod - def ApplyEnvVars(kwargs): - env = kwargs.pop('env', None) or os.environ.copy() - # Don't prompt for passwords; just fail quickly and noisily. - # By default, git will use an interactive terminal prompt when a username/ - # password is needed. That shouldn't happen in the chromium workflow, - # and if it does, then gclient may hide the prompt in the midst of a flood - # of terminal spew. The only indication that something has gone wrong - # will be when gclient hangs unresponsively. Instead, we disable the - # password prompt and simply allow git to fail noisily. The error - # message produced by git will be copied to gclient's output. - env.setdefault('GIT_ASKPASS', 'true') - env.setdefault('SSH_ASKPASS', 'true') - # 'cat' is a magical git string that disables pagers on all platforms. - env.setdefault('GIT_PAGER', 'cat') - return env + @staticmethod + def ApplyEnvVars(kwargs): + env = kwargs.pop('env', None) or os.environ.copy() + # Don't prompt for passwords; just fail quickly and noisily. + # By default, git will use an interactive terminal prompt when a + # username/ password is needed. That shouldn't happen in the chromium + # workflow, and if it does, then gclient may hide the prompt in the + # midst of a flood of terminal spew. The only indication that something + # has gone wrong will be when gclient hangs unresponsively. Instead, we + # disable the password prompt and simply allow git to fail noisily. The + # error message produced by git will be copied to gclient's output. + env.setdefault('GIT_ASKPASS', 'true') + env.setdefault('SSH_ASKPASS', 'true') + # 'cat' is a magical git string that disables pagers on all platforms. + env.setdefault('GIT_PAGER', 'cat') + return env - @staticmethod - def Capture(args, cwd=None, strip_out=True, **kwargs): - env = GIT.ApplyEnvVars(kwargs) - output = subprocess2.check_output( - ['git'] + args, cwd=cwd, stderr=subprocess2.PIPE, env=env, **kwargs) - output = output.decode('utf-8', 'replace') - return output.strip() if strip_out else output + @staticmethod + def Capture(args, cwd=None, strip_out=True, **kwargs): + env = GIT.ApplyEnvVars(kwargs) + output = subprocess2.check_output(['git'] + args, + cwd=cwd, + stderr=subprocess2.PIPE, + env=env, + **kwargs) + output = output.decode('utf-8', 'replace') + return output.strip() if strip_out else output - @staticmethod - def CaptureStatus(cwd, upstream_branch, end_commit=None): - # type: (str, str, Optional[str]) -> Sequence[Tuple[str, str]] - """Returns git status. + @staticmethod + def CaptureStatus(cwd, upstream_branch, end_commit=None): + # type: (str, str, Optional[str]) -> Sequence[Tuple[str, str]] + """Returns git status. Returns an array of (status, file) tuples.""" - if end_commit is None: - end_commit = '' - if upstream_branch is None: - upstream_branch = GIT.GetUpstreamBranch(cwd) - if upstream_branch is None: - raise gclient_utils.Error('Cannot determine upstream branch') - command = [ - '-c', 'core.quotePath=false', 'diff', '--name-status', '--no-renames', - '--ignore-submodules=all', '-r', - '%s...%s' % (upstream_branch, end_commit) - ] - status = GIT.Capture(command, cwd) - results = [] - if status: - for statusline in status.splitlines(): - # 3-way merges can cause the status can be 'MMM' instead of 'M'. This - # can happen when the user has 2 local branches and he diffs between - # these 2 branches instead diffing to upstream. - m = re.match(r'^(\w)+\t(.+)$', statusline) - if not m: - raise gclient_utils.Error( - 'status currently unsupported: %s' % statusline) - # Only grab the first letter. - results.append(('%s ' % m.group(1)[0], m.group(2))) - return results + if end_commit is None: + end_commit = '' + if upstream_branch is None: + upstream_branch = GIT.GetUpstreamBranch(cwd) + if upstream_branch is None: + raise gclient_utils.Error('Cannot determine upstream branch') + command = [ + '-c', 'core.quotePath=false', 'diff', '--name-status', + '--no-renames', '--ignore-submodules=all', '-r', + '%s...%s' % (upstream_branch, end_commit) + ] + status = GIT.Capture(command, cwd) + results = [] + if status: + for statusline in status.splitlines(): + # 3-way merges can cause the status can be 'MMM' instead of 'M'. + # This can happen when the user has 2 local branches and he + # diffs between these 2 branches instead diffing to upstream. + m = re.match(r'^(\w)+\t(.+)$', statusline) + if not m: + raise gclient_utils.Error( + 'status currently unsupported: %s' % statusline) + # Only grab the first letter. + results.append(('%s ' % m.group(1)[0], m.group(2))) + return results - @staticmethod - def GetConfig(cwd, key, default=None): - try: - return GIT.Capture(['config', key], cwd=cwd) - except subprocess2.CalledProcessError: - return default + @staticmethod + def GetConfig(cwd, key, default=None): + try: + return GIT.Capture(['config', key], cwd=cwd) + except subprocess2.CalledProcessError: + return default - @staticmethod - def GetBranchConfig(cwd, branch, key, default=None): - assert branch, 'A branch must be given' - key = 'branch.%s.%s' % (branch, key) - return GIT.GetConfig(cwd, key, default) + @staticmethod + def GetBranchConfig(cwd, branch, key, default=None): + assert branch, 'A branch must be given' + key = 'branch.%s.%s' % (branch, key) + return GIT.GetConfig(cwd, key, default) - @staticmethod - def SetConfig(cwd, key, value=None): - if value is None: - args = ['config', '--unset', key] - else: - args = ['config', key, value] - GIT.Capture(args, cwd=cwd) + @staticmethod + def SetConfig(cwd, key, value=None): + if value is None: + args = ['config', '--unset', key] + else: + args = ['config', key, value] + GIT.Capture(args, cwd=cwd) - @staticmethod - def SetBranchConfig(cwd, branch, key, value=None): - assert branch, 'A branch must be given' - key = 'branch.%s.%s' % (branch, key) - GIT.SetConfig(cwd, key, value) + @staticmethod + def SetBranchConfig(cwd, branch, key, value=None): + assert branch, 'A branch must be given' + key = 'branch.%s.%s' % (branch, key) + GIT.SetConfig(cwd, key, value) - @staticmethod - def IsWorkTreeDirty(cwd): - return GIT.Capture(['status', '-s'], cwd=cwd) != '' + @staticmethod + def IsWorkTreeDirty(cwd): + return GIT.Capture(['status', '-s'], cwd=cwd) != '' - @staticmethod - def GetEmail(cwd): - """Retrieves the user email address if known.""" - return GIT.GetConfig(cwd, 'user.email', '') + @staticmethod + def GetEmail(cwd): + """Retrieves the user email address if known.""" + return GIT.GetConfig(cwd, 'user.email', '') - @staticmethod - def ShortBranchName(branch): - """Converts a name like 'refs/heads/foo' to just 'foo'.""" - return branch.replace('refs/heads/', '') + @staticmethod + def ShortBranchName(branch): + """Converts a name like 'refs/heads/foo' to just 'foo'.""" + return branch.replace('refs/heads/', '') - @staticmethod - def GetBranchRef(cwd): - """Returns the full branch reference, e.g. 'refs/heads/main'.""" - try: - return GIT.Capture(['symbolic-ref', 'HEAD'], cwd=cwd) - except subprocess2.CalledProcessError: - return None + @staticmethod + def GetBranchRef(cwd): + """Returns the full branch reference, e.g. 'refs/heads/main'.""" + try: + return GIT.Capture(['symbolic-ref', 'HEAD'], cwd=cwd) + except subprocess2.CalledProcessError: + return None - @staticmethod - def GetRemoteHeadRef(cwd, url, remote): - """Returns the full default remote branch reference, e.g. + @staticmethod + def GetRemoteHeadRef(cwd, url, remote): + """Returns the full default remote branch reference, e.g. 'refs/remotes/origin/main'.""" - if os.path.exists(cwd): - try: - # Try using local git copy first - ref = 'refs/remotes/%s/HEAD' % remote - ref = GIT.Capture(['symbolic-ref', ref], cwd=cwd) - if not ref.endswith('master'): - return ref - # Check if there are changes in the default branch for this particular - # repository. - GIT.Capture(['remote', 'set-head', '-a', remote], cwd=cwd) - return GIT.Capture(['symbolic-ref', ref], cwd=cwd) - except subprocess2.CalledProcessError: - pass + if os.path.exists(cwd): + try: + # Try using local git copy first + ref = 'refs/remotes/%s/HEAD' % remote + ref = GIT.Capture(['symbolic-ref', ref], cwd=cwd) + if not ref.endswith('master'): + return ref + # Check if there are changes in the default branch for this + # particular repository. + GIT.Capture(['remote', 'set-head', '-a', remote], cwd=cwd) + return GIT.Capture(['symbolic-ref', ref], cwd=cwd) + except subprocess2.CalledProcessError: + pass - try: - # Fetch information from git server - resp = GIT.Capture(['ls-remote', '--symref', url, 'HEAD']) - regex = r'^ref: (.*)\tHEAD$' - for line in resp.split('\n'): - m = re.match(regex, line) - if m: - return ''.join(GIT.RefToRemoteRef(m.group(1), remote)) - except subprocess2.CalledProcessError: - pass - # Return default branch - return 'refs/remotes/%s/main' % remote + try: + # Fetch information from git server + resp = GIT.Capture(['ls-remote', '--symref', url, 'HEAD']) + regex = r'^ref: (.*)\tHEAD$' + for line in resp.split('\n'): + m = re.match(regex, line) + if m: + return ''.join(GIT.RefToRemoteRef(m.group(1), remote)) + except subprocess2.CalledProcessError: + pass + # Return default branch + return 'refs/remotes/%s/main' % remote - @staticmethod - def GetBranch(cwd): - """Returns the short branch name, e.g. 'main'.""" - branchref = GIT.GetBranchRef(cwd) - if branchref: - return GIT.ShortBranchName(branchref) - return None + @staticmethod + def GetBranch(cwd): + """Returns the short branch name, e.g. 'main'.""" + branchref = GIT.GetBranchRef(cwd) + if branchref: + return GIT.ShortBranchName(branchref) + return None - @staticmethod - def GetRemoteBranches(cwd): - return GIT.Capture(['branch', '-r'], cwd=cwd).split() + @staticmethod + def GetRemoteBranches(cwd): + return GIT.Capture(['branch', '-r'], cwd=cwd).split() - @staticmethod - def FetchUpstreamTuple(cwd, branch=None): - """Returns a tuple containing remote and remote ref, + @staticmethod + def FetchUpstreamTuple(cwd, branch=None): + """Returns a tuple containing remote and remote ref, e.g. 'origin', 'refs/heads/main' """ - try: - branch = branch or GIT.GetBranch(cwd) - except subprocess2.CalledProcessError: - pass - if branch: - upstream_branch = GIT.GetBranchConfig(cwd, branch, 'merge') - if upstream_branch: - remote = GIT.GetBranchConfig(cwd, branch, 'remote', '.') - return remote, upstream_branch + try: + branch = branch or GIT.GetBranch(cwd) + except subprocess2.CalledProcessError: + pass + if branch: + upstream_branch = GIT.GetBranchConfig(cwd, branch, 'merge') + if upstream_branch: + remote = GIT.GetBranchConfig(cwd, branch, 'remote', '.') + return remote, upstream_branch - upstream_branch = GIT.GetConfig(cwd, 'rietveld.upstream-branch') - if upstream_branch: - remote = GIT.GetConfig(cwd, 'rietveld.upstream-remote', '.') - return remote, upstream_branch + upstream_branch = GIT.GetConfig(cwd, 'rietveld.upstream-branch') + if upstream_branch: + remote = GIT.GetConfig(cwd, 'rietveld.upstream-remote', '.') + return remote, upstream_branch - # Else, try to guess the origin remote. - remote_branches = GIT.GetRemoteBranches(cwd) - if 'origin/main' in remote_branches: - # Fall back on origin/main if it exits. - return 'origin', 'refs/heads/main' + # Else, try to guess the origin remote. + remote_branches = GIT.GetRemoteBranches(cwd) + if 'origin/main' in remote_branches: + # Fall back on origin/main if it exits. + return 'origin', 'refs/heads/main' - if 'origin/master' in remote_branches: - # Fall back on origin/master if it exits. - return 'origin', 'refs/heads/master' + if 'origin/master' in remote_branches: + # Fall back on origin/master if it exits. + return 'origin', 'refs/heads/master' - return None, None + return None, None - @staticmethod - def RefToRemoteRef(ref, remote): - """Convert a checkout ref to the equivalent remote ref. + @staticmethod + def RefToRemoteRef(ref, remote): + """Convert a checkout ref to the equivalent remote ref. Returns: A tuple of the remote ref's (common prefix, unique suffix), or None if it doesn't appear to refer to a remote ref (e.g. it's a commit hash). """ - # TODO(mmoss): This is just a brute-force mapping based of the expected git - # config. It's a bit better than the even more brute-force replace('heads', - # ...), but could still be smarter (like maybe actually using values gleaned - # from the git config). - m = re.match('^(refs/(remotes/)?)?branch-heads/', ref or '') - if m: - return ('refs/remotes/branch-heads/', ref.replace(m.group(0), '')) + # TODO(mmoss): This is just a brute-force mapping based of the expected + # git config. It's a bit better than the even more brute-force + # replace('heads', ...), but could still be smarter (like maybe actually + # using values gleaned from the git config). + m = re.match('^(refs/(remotes/)?)?branch-heads/', ref or '') + if m: + return ('refs/remotes/branch-heads/', ref.replace(m.group(0), '')) - m = re.match('^((refs/)?remotes/)?%s/|(refs/)?heads/' % remote, ref or '') - if m: - return ('refs/remotes/%s/' % remote, ref.replace(m.group(0), '')) + m = re.match('^((refs/)?remotes/)?%s/|(refs/)?heads/' % remote, ref + or '') + if m: + return ('refs/remotes/%s/' % remote, ref.replace(m.group(0), '')) - return None + return None - @staticmethod - def RemoteRefToRef(ref, remote): - assert remote, 'A remote must be given' - if not ref or not ref.startswith('refs/'): - return None - if not ref.startswith('refs/remotes/'): - return ref - if ref.startswith('refs/remotes/branch-heads/'): - return 'refs' + ref[len('refs/remotes'):] - if ref.startswith('refs/remotes/%s/' % remote): - return 'refs/heads' + ref[len('refs/remotes/%s' % remote):] - return None + @staticmethod + def RemoteRefToRef(ref, remote): + assert remote, 'A remote must be given' + if not ref or not ref.startswith('refs/'): + return None + if not ref.startswith('refs/remotes/'): + return ref + if ref.startswith('refs/remotes/branch-heads/'): + return 'refs' + ref[len('refs/remotes'):] + if ref.startswith('refs/remotes/%s/' % remote): + return 'refs/heads' + ref[len('refs/remotes/%s' % remote):] + return None - @staticmethod - def GetUpstreamBranch(cwd): - """Gets the current branch's upstream branch.""" - remote, upstream_branch = GIT.FetchUpstreamTuple(cwd) - if remote != '.' and upstream_branch: - remote_ref = GIT.RefToRemoteRef(upstream_branch, remote) - if remote_ref: - upstream_branch = ''.join(remote_ref) - return upstream_branch + @staticmethod + def GetUpstreamBranch(cwd): + """Gets the current branch's upstream branch.""" + remote, upstream_branch = GIT.FetchUpstreamTuple(cwd) + if remote != '.' and upstream_branch: + remote_ref = GIT.RefToRemoteRef(upstream_branch, remote) + if remote_ref: + upstream_branch = ''.join(remote_ref) + return upstream_branch - @staticmethod - def IsAncestor(maybe_ancestor, ref, cwd=None): - # type: (string, string, Optional[string]) -> bool - """Verifies if |maybe_ancestor| is an ancestor of |ref|.""" - try: - GIT.Capture(['merge-base', '--is-ancestor', maybe_ancestor, ref], cwd=cwd) - return True - except subprocess2.CalledProcessError: - return False + @staticmethod + def IsAncestor(maybe_ancestor, ref, cwd=None): + # type: (string, string, Optional[string]) -> bool + """Verifies if |maybe_ancestor| is an ancestor of |ref|.""" + try: + GIT.Capture(['merge-base', '--is-ancestor', maybe_ancestor, ref], + cwd=cwd) + return True + except subprocess2.CalledProcessError: + return False - @staticmethod - def GetOldContents(cwd, filename, branch=None): - if not branch: - branch = GIT.GetUpstreamBranch(cwd) - if platform.system() == 'Windows': - # git show : wants a posix path. - filename = filename.replace('\\', '/') - command = ['show', '%s:%s' % (branch, filename)] - try: - return GIT.Capture(command, cwd=cwd, strip_out=False) - except subprocess2.CalledProcessError: - return '' + @staticmethod + def GetOldContents(cwd, filename, branch=None): + if not branch: + branch = GIT.GetUpstreamBranch(cwd) + if platform.system() == 'Windows': + # git show : wants a posix path. + filename = filename.replace('\\', '/') + command = ['show', '%s:%s' % (branch, filename)] + try: + return GIT.Capture(command, cwd=cwd, strip_out=False) + except subprocess2.CalledProcessError: + return '' - @staticmethod - def GenerateDiff(cwd, branch=None, branch_head='HEAD', full_move=False, - files=None): - """Diffs against the upstream branch or optionally another branch. + @staticmethod + def GenerateDiff(cwd, + branch=None, + branch_head='HEAD', + full_move=False, + files=None): + """Diffs against the upstream branch or optionally another branch. full_move means that move or copy operations should completely recreate the files, usually in the prospect to apply the patch for a try job.""" - if not branch: - branch = GIT.GetUpstreamBranch(cwd) - command = ['-c', 'core.quotePath=false', 'diff', - '-p', '--no-color', '--no-prefix', '--no-ext-diff', - branch + "..." + branch_head] - if full_move: - command.append('--no-renames') - else: - command.append('-C') - # TODO(maruel): --binary support. - if files: - command.append('--') - command.extend(files) - diff = GIT.Capture(command, cwd=cwd, strip_out=False).splitlines(True) - for i in range(len(diff)): - # In the case of added files, replace /dev/null with the path to the - # file being added. - if diff[i].startswith('--- /dev/null'): - diff[i] = '--- %s' % diff[i+1][4:] - return ''.join(diff) + if not branch: + branch = GIT.GetUpstreamBranch(cwd) + command = [ + '-c', 'core.quotePath=false', 'diff', '-p', '--no-color', + '--no-prefix', '--no-ext-diff', branch + "..." + branch_head + ] + if full_move: + command.append('--no-renames') + else: + command.append('-C') + # TODO(maruel): --binary support. + if files: + command.append('--') + command.extend(files) + diff = GIT.Capture(command, cwd=cwd, strip_out=False).splitlines(True) + for i in range(len(diff)): + # In the case of added files, replace /dev/null with the path to the + # file being added. + if diff[i].startswith('--- /dev/null'): + diff[i] = '--- %s' % diff[i + 1][4:] + return ''.join(diff) - @staticmethod - def GetDifferentFiles(cwd, branch=None, branch_head='HEAD'): - """Returns the list of modified files between two branches.""" - if not branch: - branch = GIT.GetUpstreamBranch(cwd) - command = ['-c', 'core.quotePath=false', 'diff', - '--name-only', branch + "..." + branch_head] - return GIT.Capture(command, cwd=cwd).splitlines(False) + @staticmethod + def GetDifferentFiles(cwd, branch=None, branch_head='HEAD'): + """Returns the list of modified files between two branches.""" + if not branch: + branch = GIT.GetUpstreamBranch(cwd) + command = [ + '-c', 'core.quotePath=false', 'diff', '--name-only', + branch + "..." + branch_head + ] + return GIT.Capture(command, cwd=cwd).splitlines(False) - @staticmethod - def GetAllFiles(cwd): - """Returns the list of all files under revision control.""" - command = ['-c', 'core.quotePath=false', 'ls-files', '-s', '--', '.'] - files = GIT.Capture(command, cwd=cwd).splitlines(False) - # return only files - return [f.split(maxsplit=3)[-1] for f in files if f.startswith('100')] + @staticmethod + def GetAllFiles(cwd): + """Returns the list of all files under revision control.""" + command = ['-c', 'core.quotePath=false', 'ls-files', '-s', '--', '.'] + files = GIT.Capture(command, cwd=cwd).splitlines(False) + # return only files + return [f.split(maxsplit=3)[-1] for f in files if f.startswith('100')] - @staticmethod - def GetSubmoduleCommits(cwd, submodules): - # type: (string, List[string]) => Mapping[string][string] - """Returns a mapping of staged or committed new commits for submodules.""" - if not submodules: - return {} - result = subprocess2.check_output(['git', 'ls-files', '-s', '--'] + - submodules, - cwd=cwd).decode('utf-8') - commit_hashes = {} - for r in result.splitlines(): - # ['', '', '', '']. - record = r.strip().split(maxsplit=3) # path can contain spaces. - assert record[0] == '160000', 'file is not a gitlink: %s' % record - commit_hashes[record[3]] = record[1] - return commit_hashes + @staticmethod + def GetSubmoduleCommits(cwd, submodules): + # type: (string, List[string]) => Mapping[string][string] + """Returns a mapping of staged or committed new commits for submodules.""" + if not submodules: + return {} + result = subprocess2.check_output(['git', 'ls-files', '-s', '--'] + + submodules, + cwd=cwd).decode('utf-8') + commit_hashes = {} + for r in result.splitlines(): + # ['', '', '', '']. + record = r.strip().split(maxsplit=3) # path can contain spaces. + assert record[0] == '160000', 'file is not a gitlink: %s' % record + commit_hashes[record[3]] = record[1] + return commit_hashes - @staticmethod - def GetPatchName(cwd): - """Constructs a name for this patch.""" - short_sha = GIT.Capture(['rev-parse', '--short=4', 'HEAD'], cwd=cwd) - return "%s#%s" % (GIT.GetBranch(cwd), short_sha) + @staticmethod + def GetPatchName(cwd): + """Constructs a name for this patch.""" + short_sha = GIT.Capture(['rev-parse', '--short=4', 'HEAD'], cwd=cwd) + return "%s#%s" % (GIT.GetBranch(cwd), short_sha) - @staticmethod - def GetCheckoutRoot(cwd): - """Returns the top level directory of a git checkout as an absolute path. + @staticmethod + def GetCheckoutRoot(cwd): + """Returns the top level directory of a git checkout as an absolute path. """ - root = GIT.Capture(['rev-parse', '--show-cdup'], cwd=cwd) - return os.path.abspath(os.path.join(cwd, root)) + root = GIT.Capture(['rev-parse', '--show-cdup'], cwd=cwd) + return os.path.abspath(os.path.join(cwd, root)) - @staticmethod - def GetGitDir(cwd): - return os.path.abspath(GIT.Capture(['rev-parse', '--git-dir'], cwd=cwd)) + @staticmethod + def GetGitDir(cwd): + return os.path.abspath(GIT.Capture(['rev-parse', '--git-dir'], cwd=cwd)) - @staticmethod - def IsInsideWorkTree(cwd): - try: - return GIT.Capture(['rev-parse', '--is-inside-work-tree'], cwd=cwd) - except (OSError, subprocess2.CalledProcessError): - return False + @staticmethod + def IsInsideWorkTree(cwd): + try: + return GIT.Capture(['rev-parse', '--is-inside-work-tree'], cwd=cwd) + except (OSError, subprocess2.CalledProcessError): + return False - @staticmethod - def IsDirectoryVersioned(cwd, relative_dir): - """Checks whether the given |relative_dir| is part of cwd's repo.""" - return bool(GIT.Capture(['ls-tree', 'HEAD', relative_dir], cwd=cwd)) + @staticmethod + def IsDirectoryVersioned(cwd, relative_dir): + """Checks whether the given |relative_dir| is part of cwd's repo.""" + return bool(GIT.Capture(['ls-tree', 'HEAD', relative_dir], cwd=cwd)) - @staticmethod - def CleanupDir(cwd, relative_dir): - """Cleans up untracked file inside |relative_dir|.""" - return bool(GIT.Capture(['clean', '-df', relative_dir], cwd=cwd)) + @staticmethod + def CleanupDir(cwd, relative_dir): + """Cleans up untracked file inside |relative_dir|.""" + return bool(GIT.Capture(['clean', '-df', relative_dir], cwd=cwd)) - @staticmethod - def ResolveCommit(cwd, rev): - # We do this instead of rev-parse --verify rev^{commit}, since on Windows - # git can be either an executable or batch script, each of which requires - # escaping the caret (^) a different way. - if gclient_utils.IsFullGitSha(rev): - # git-rev parse --verify FULL_GIT_SHA always succeeds, even if we don't - # have FULL_GIT_SHA locally. Removing the last character forces git to - # check if FULL_GIT_SHA refers to an object in the local database. - rev = rev[:-1] - try: - return GIT.Capture(['rev-parse', '--quiet', '--verify', rev], cwd=cwd) - except subprocess2.CalledProcessError: - return None + @staticmethod + def ResolveCommit(cwd, rev): + # We do this instead of rev-parse --verify rev^{commit}, since on + # Windows git can be either an executable or batch script, each of which + # requires escaping the caret (^) a different way. + if gclient_utils.IsFullGitSha(rev): + # git-rev parse --verify FULL_GIT_SHA always succeeds, even if we + # don't have FULL_GIT_SHA locally. Removing the last character + # forces git to check if FULL_GIT_SHA refers to an object in the + # local database. + rev = rev[:-1] + try: + return GIT.Capture(['rev-parse', '--quiet', '--verify', rev], + cwd=cwd) + except subprocess2.CalledProcessError: + return None - @staticmethod - def IsValidRevision(cwd, rev, sha_only=False): - """Verifies the revision is a proper git revision. + @staticmethod + def IsValidRevision(cwd, rev, sha_only=False): + """Verifies the revision is a proper git revision. sha_only: Fail unless rev is a sha hash. """ - sha = GIT.ResolveCommit(cwd, rev) - if sha is None: - return False - if sha_only: - return sha == rev.lower() - return True + sha = GIT.ResolveCommit(cwd, rev) + if sha is None: + return False + if sha_only: + return sha == rev.lower() + return True - @classmethod - def AssertVersion(cls, min_version): - """Asserts git's version is at least min_version.""" - if cls.current_version is None: - current_version = cls.Capture(['--version'], '.') - matched = re.search(r'git version (.+)', current_version) - cls.current_version = distutils.version.LooseVersion(matched.group(1)) - min_version = distutils.version.LooseVersion(min_version) - return (min_version <= cls.current_version, cls.current_version) + @classmethod + def AssertVersion(cls, min_version): + """Asserts git's version is at least min_version.""" + if cls.current_version is None: + current_version = cls.Capture(['--version'], '.') + matched = re.search(r'git version (.+)', current_version) + cls.current_version = distutils.version.LooseVersion( + matched.group(1)) + min_version = distutils.version.LooseVersion(min_version) + return (min_version <= cls.current_version, cls.current_version) diff --git a/setup_color.py b/setup_color.py index 40e96b5f6e..af039c82ad 100644 --- a/setup_color.py +++ b/setup_color.py @@ -16,115 +16,120 @@ OUT_TYPE = 'unknown' def enable_native_ansi(): - """Enables native ANSI sequences in console. Windows 10 only. + """Enables native ANSI sequences in console. Windows 10 only. Returns whether successful. """ - kernel32 = ctypes.windll.kernel32 - ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x04 + kernel32 = ctypes.windll.kernel32 + ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x04 - out_handle = kernel32.GetStdHandle(subprocess.STD_OUTPUT_HANDLE) + out_handle = kernel32.GetStdHandle(subprocess.STD_OUTPUT_HANDLE) - # GetConsoleMode fails if the terminal isn't native. - mode = ctypes.wintypes.DWORD() - if kernel32.GetConsoleMode(out_handle, ctypes.byref(mode)) == 0: - return False + # GetConsoleMode fails if the terminal isn't native. + mode = ctypes.wintypes.DWORD() + if kernel32.GetConsoleMode(out_handle, ctypes.byref(mode)) == 0: + return False - if not (mode.value & ENABLE_VIRTUAL_TERMINAL_PROCESSING): - if kernel32.SetConsoleMode( - out_handle, mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING) == 0: - print( - 'kernel32.SetConsoleMode to enable ANSI sequences failed', - file=sys.stderr) - return False + if not (mode.value & ENABLE_VIRTUAL_TERMINAL_PROCESSING): + if kernel32.SetConsoleMode( + out_handle, + mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING) == 0: + print('kernel32.SetConsoleMode to enable ANSI sequences failed', + file=sys.stderr) + return False - return True + return True def init(): - # should_wrap instructs colorama to wrap stdout/stderr with an ANSI colorcode - # interpreter that converts them to SetConsoleTextAttribute calls. This only - # should be True in cases where we're connected to cmd.exe's console. Setting - # this to True on non-windows systems has no effect. - should_wrap = False - global IS_TTY, OUT_TYPE - IS_TTY = sys.stdout.isatty() - is_windows = sys.platform.startswith('win') - if IS_TTY: - # Yay! We detected a console in the normal way. It doesn't really matter - # if it's windows or not, we win. - OUT_TYPE = 'console' - should_wrap = True - elif is_windows: - # assume this is some sort of file - OUT_TYPE = 'file (win)' + # should_wrap instructs colorama to wrap stdout/stderr with an ANSI + # colorcode interpreter that converts them to SetConsoleTextAttribute calls. + # This only should be True in cases where we're connected to cmd.exe's + # console. Setting this to True on non-windows systems has no effect. + should_wrap = False + global IS_TTY, OUT_TYPE + IS_TTY = sys.stdout.isatty() + is_windows = sys.platform.startswith('win') + if IS_TTY: + # Yay! We detected a console in the normal way. It doesn't really matter + # if it's windows or not, we win. + OUT_TYPE = 'console' + should_wrap = True + elif is_windows: + # assume this is some sort of file + OUT_TYPE = 'file (win)' - import msvcrt - h = msvcrt.get_osfhandle(sys.stdout.fileno()) - # h is the win32 HANDLE for stdout. - ftype = ctypes.windll.kernel32.GetFileType(h) - if ftype == 2: # FILE_TYPE_CHAR - # This is a normal cmd console, but we'll only get here if we're running - # inside a `git command` which is actually git->bash->command. Not sure - # why isatty doesn't detect this case. - OUT_TYPE = 'console (cmd via msys)' - IS_TTY = True - should_wrap = True - elif ftype == 3: # FILE_TYPE_PIPE - OUT_TYPE = 'pipe (win)' - # This is some kind of pipe on windows. This could either be a real pipe - # or this could be msys using a pipe to emulate a pty. We use the same - # algorithm that msys-git uses to determine if it's connected to a pty or - # not. + import msvcrt + h = msvcrt.get_osfhandle(sys.stdout.fileno()) + # h is the win32 HANDLE for stdout. + ftype = ctypes.windll.kernel32.GetFileType(h) + if ftype == 2: # FILE_TYPE_CHAR + # This is a normal cmd console, but we'll only get here if we're + # running inside a `git command` which is actually + # git->bash->command. Not sure why isatty doesn't detect this case. + OUT_TYPE = 'console (cmd via msys)' + IS_TTY = True + should_wrap = True + elif ftype == 3: # FILE_TYPE_PIPE + OUT_TYPE = 'pipe (win)' - # This function and the structures are defined in the MSDN documentation - # using the same names. - def NT_SUCCESS(status): - # The first two bits of status are the severity. The success - # severities are 0 and 1, and the !success severities are 2 and 3. - # Therefore since ctypes interprets the default restype of the call - # to be an 'C int' (which is guaranteed to be signed 32 bits), All - # success codes are positive, and all !success codes are negative. - return status >= 0 + # This is some kind of pipe on windows. This could either be a real + # pipe or this could be msys using a pipe to emulate a pty. We use + # the same algorithm that msys-git uses to determine if it's + # connected to a pty or not. - class UNICODE_STRING(ctypes.Structure): - _fields_ = [('Length', ctypes.c_ushort), - ('MaximumLength', ctypes.c_ushort), - ('Buffer', ctypes.c_wchar_p)] + # This function and the structures are defined in the MSDN + # documentation using the same names. + def NT_SUCCESS(status): + # The first two bits of status are the severity. The success + # severities are 0 and 1, and the !success severities are 2 and + # 3. Therefore since ctypes interprets the default restype of + # the call to be an 'C int' (which is guaranteed to be signed 32 + # bits), All success codes are positive, and all !success codes + # are negative. + return status >= 0 - class OBJECT_NAME_INFORMATION(ctypes.Structure): - _fields_ = [('Name', UNICODE_STRING), - ('NameBuffer', ctypes.c_wchar_p)] + class UNICODE_STRING(ctypes.Structure): + _fields_ = [('Length', ctypes.c_ushort), + ('MaximumLength', ctypes.c_ushort), + ('Buffer', ctypes.c_wchar_p)] - buf = ctypes.create_string_buffer(1024) - # Ask NT what the name of the object our stdout HANDLE is. It would be - # possible to use GetFileInformationByHandleEx, but it's only available - # on Vista+. If you're reading this in 2017 or later, feel free to - # refactor this out. - # - # The '1' here is ObjectNameInformation - if NT_SUCCESS(ctypes.windll.ntdll.NtQueryObject(h, 1, buf, len(buf)-2, - None)): - out = OBJECT_NAME_INFORMATION.from_buffer(buf) - name = out.Name.Buffer.split('\\')[-1] - IS_TTY = name.startswith('msys-') and '-pty' in name - if IS_TTY: - OUT_TYPE = 'bash (msys)' + class OBJECT_NAME_INFORMATION(ctypes.Structure): + _fields_ = [('Name', UNICODE_STRING), + ('NameBuffer', ctypes.c_wchar_p)] + + buf = ctypes.create_string_buffer(1024) + # Ask NT what the name of the object our stdout HANDLE is. It would + # be possible to use GetFileInformationByHandleEx, but it's only + # available on Vista+. If you're reading this in 2017 or later, feel + # free to refactor this out. + # + # The '1' here is ObjectNameInformation + if NT_SUCCESS( + ctypes.windll.ntdll.NtQueryObject(h, 1, buf, + len(buf) - 2, None)): + out = OBJECT_NAME_INFORMATION.from_buffer(buf) + name = out.Name.Buffer.split('\\')[-1] + IS_TTY = name.startswith('msys-') and '-pty' in name + if IS_TTY: + OUT_TYPE = 'bash (msys)' + else: + # A normal file, or an unknown file type. + pass else: - # A normal file, or an unknown file type. - pass - else: - # This is non-windows, so we trust isatty. - OUT_TYPE = 'pipe or file' + # This is non-windows, so we trust isatty. + OUT_TYPE = 'pipe or file' - if IS_TTY and is_windows: - # Wrapping may cause errors on some Windows versions (crbug.com/1114548). - if platform.release() != '10' or enable_native_ansi(): - should_wrap = False + if IS_TTY and is_windows: + # Wrapping may cause errors on some Windows versions + # (crbug.com/1114548). + if platform.release() != '10' or enable_native_ansi(): + should_wrap = False + + colorama.init(wrap=should_wrap) - colorama.init(wrap=should_wrap) if __name__ == '__main__': - init() - print('IS_TTY:', IS_TTY) - print('OUT_TYPE:', OUT_TYPE) + init() + print('IS_TTY:', IS_TTY) + print('OUT_TYPE:', OUT_TYPE) diff --git a/siso.py b/siso.py index eaa660a8ab..fb73451f92 100644 --- a/siso.py +++ b/siso.py @@ -15,68 +15,72 @@ import gclient_paths def main(args): - # On Windows the siso.bat script passes along the arguments enclosed in - # double quotes. This prevents multiple levels of parsing of the special '^' - # characters needed when compiling a single file. When this case is detected, - # we need to split the argument. This means that arguments containing actual - # spaces are not supported by siso.bat, but that is not a real limitation. - if sys.platform.startswith('win') and len(args) == 2: - args = args[:1] + args[1].split() + # On Windows the siso.bat script passes along the arguments enclosed in + # double quotes. This prevents multiple levels of parsing of the special '^' + # characters needed when compiling a single file. When this case is + # detected, we need to split the argument. This means that arguments + # containing actual spaces are not supported by siso.bat, but that is not a + # real limitation. + if sys.platform.startswith('win') and len(args) == 2: + args = args[:1] + args[1].split() - # macOS's python sets CPATH, LIBRARY_PATH, SDKROOT implicitly. - # https://openradar.appspot.com/radar?id=5608755232243712 - # - # Removing those environment variables to avoid affecting clang's behaviors. - if sys.platform == 'darwin': - os.environ.pop("CPATH", None) - os.environ.pop("LIBRARY_PATH", None) - os.environ.pop("SDKROOT", None) + # macOS's python sets CPATH, LIBRARY_PATH, SDKROOT implicitly. + # https://openradar.appspot.com/radar?id=5608755232243712 + # + # Removing those environment variables to avoid affecting clang's behaviors. + if sys.platform == 'darwin': + os.environ.pop("CPATH", None) + os.environ.pop("LIBRARY_PATH", None) + os.environ.pop("SDKROOT", None) - environ = os.environ.copy() + environ = os.environ.copy() - # Get gclient root + src. - primary_solution_path = gclient_paths.GetPrimarySolutionPath() - gclient_root_path = gclient_paths.FindGclientRoot(os.getcwd()) - gclient_src_root_path = None - if gclient_root_path: - gclient_src_root_path = os.path.join(gclient_root_path, 'src') + # Get gclient root + src. + primary_solution_path = gclient_paths.GetPrimarySolutionPath() + gclient_root_path = gclient_paths.FindGclientRoot(os.getcwd()) + gclient_src_root_path = None + if gclient_root_path: + gclient_src_root_path = os.path.join(gclient_root_path, 'src') - siso_override_path = os.environ.get('SISO_PATH') - if siso_override_path: - print('depot_tools/siso.py: Using Siso binary from SISO_PATH: %s.' % - siso_override_path) - if not os.path.isfile(siso_override_path): - print('depot_tools/siso.py: Could not find Siso at provided SISO_PATH.', - file=sys.stderr) - return 1 + siso_override_path = os.environ.get('SISO_PATH') + if siso_override_path: + print('depot_tools/siso.py: Using Siso binary from SISO_PATH: %s.' % + siso_override_path) + if not os.path.isfile(siso_override_path): + print( + 'depot_tools/siso.py: Could not find Siso at provided ' + 'SISO_PATH.', + file=sys.stderr) + return 1 - for base_path in set( - [primary_solution_path, gclient_root_path, gclient_src_root_path]): - if not base_path: - continue - env = environ.copy() - sisoenv_path = os.path.join(base_path, 'build', 'config', 'siso', - '.sisoenv') - if not os.path.exists(sisoenv_path): - continue - with open(sisoenv_path) as f: - for line in f.readlines(): - k, v = line.rstrip().split('=', 1) - env[k] = v - siso_path = siso_override_path or os.path.join( - base_path, 'third_party', 'siso', 'siso' + gclient_paths.GetExeSuffix()) - if os.path.isfile(siso_path): - return subprocess.call([siso_path] + args[1:], env=env) + for base_path in set( + [primary_solution_path, gclient_root_path, gclient_src_root_path]): + if not base_path: + continue + env = environ.copy() + sisoenv_path = os.path.join(base_path, 'build', 'config', 'siso', + '.sisoenv') + if not os.path.exists(sisoenv_path): + continue + with open(sisoenv_path) as f: + for line in f.readlines(): + k, v = line.rstrip().split('=', 1) + env[k] = v + siso_path = siso_override_path or os.path.join( + base_path, 'third_party', 'siso', + 'siso' + gclient_paths.GetExeSuffix()) + if os.path.isfile(siso_path): + return subprocess.call([siso_path] + args[1:], env=env) - print( - 'depot_tools/siso.py: Could not find .sisoenv under build/config/siso of ' - 'the current project. Did you run gclient sync?', - file=sys.stderr) - return 1 + print( + 'depot_tools/siso.py: Could not find .sisoenv under build/config/siso ' + 'of the current project. Did you run gclient sync?', + file=sys.stderr) + return 1 if __name__ == '__main__': - try: - sys.exit(main(sys.argv)) - except KeyboardInterrupt: - sys.exit(1) + try: + sys.exit(main(sys.argv)) + except KeyboardInterrupt: + sys.exit(1) diff --git a/split_cl.py b/split_cl.py index 16cb8f25de..70768755c6 100644 --- a/split_cl.py +++ b/split_cl.py @@ -2,7 +2,6 @@ # Copyright 2017 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Splits a branch into smaller branches and uploads CLs.""" from __future__ import print_function @@ -19,7 +18,6 @@ import scm import git_common as git - # If a call to `git cl split` will generate more than this number of CLs, the # command will prompt the user to make sure they know what they're doing. Large # numbers of CLs generated by `git cl split` have caused infrastructure issues @@ -36,62 +34,62 @@ FilesAndOwnersDirectory = collections.namedtuple("FilesAndOwnersDirectory", def EnsureInGitRepository(): - """Throws an exception if the current directory is not a git repository.""" - git.run('rev-parse') + """Throws an exception if the current directory is not a git repository.""" + git.run('rev-parse') def CreateBranchForDirectories(prefix, directories, upstream): - """Creates a branch named |prefix| + "_" + |directories[0]| + "_split". + """Creates a branch named |prefix| + "_" + |directories[0]| + "_split". Return false if the branch already exists. |upstream| is used as upstream for the created branch. """ - existing_branches = set(git.branches(use_limit = False)) - branch_name = prefix + '_' + directories[0] + '_split' - if branch_name in existing_branches: - return False - git.run('checkout', '-t', upstream, '-b', branch_name) - return True + existing_branches = set(git.branches(use_limit=False)) + branch_name = prefix + '_' + directories[0] + '_split' + if branch_name in existing_branches: + return False + git.run('checkout', '-t', upstream, '-b', branch_name) + return True def FormatDirectoriesForPrinting(directories, prefix=None): - """Formats directory list for printing + """Formats directory list for printing Uses dedicated format for single-item list.""" - prefixed = directories - if prefix: - prefixed = [(prefix + d) for d in directories] + prefixed = directories + if prefix: + prefixed = [(prefix + d) for d in directories] - return str(prefixed) if len(prefixed) > 1 else str(prefixed[0]) + return str(prefixed) if len(prefixed) > 1 else str(prefixed[0]) def FormatDescriptionOrComment(txt, directories): - """Replaces $directory with |directories| in |txt|.""" - to_insert = FormatDirectoriesForPrinting(directories, prefix='/') - return txt.replace('$directory', to_insert) + """Replaces $directory with |directories| in |txt|.""" + to_insert = FormatDirectoriesForPrinting(directories, prefix='/') + return txt.replace('$directory', to_insert) def AddUploadedByGitClSplitToDescription(description): - """Adds a 'This CL was uploaded by git cl split.' line to |description|. + """Adds a 'This CL was uploaded by git cl split.' line to |description|. The line is added before footers, or at the end of |description| if it has no footers. """ - split_footers = git_footers.split_footers(description) - lines = split_footers[0] - if lines[-1] and not lines[-1].isspace(): - lines = lines + [''] - lines = lines + ['This CL was uploaded by git cl split.'] - if split_footers[1]: - lines += [''] + split_footers[1] - return '\n'.join(lines) + split_footers = git_footers.split_footers(description) + lines = split_footers[0] + if lines[-1] and not lines[-1].isspace(): + lines = lines + [''] + lines = lines + ['This CL was uploaded by git cl split.'] + if split_footers[1]: + lines += [''] + split_footers[1] + return '\n'.join(lines) def UploadCl(refactor_branch, refactor_branch_upstream, directories, files, description, comment, reviewers, changelist, cmd_upload, cq_dry_run, enable_auto_submit, topic, repository_root): - """Uploads a CL with all changes to |files| in |refactor_branch|. + """Uploads a CL with all changes to |files| in |refactor_branch|. Args: refactor_branch: Name of the branch that contains the changes to upload. @@ -108,89 +106,92 @@ def UploadCl(refactor_branch, refactor_branch_upstream, directories, files, enable_auto_submit: If CL uploads should also enable auto submit. topic: Topic to associate with uploaded CLs. """ - # Create a branch. - if not CreateBranchForDirectories(refactor_branch, directories, - refactor_branch_upstream): - print('Skipping ' + FormatDirectoriesForPrinting(directories) + - ' for which a branch already exists.') - return + # Create a branch. + if not CreateBranchForDirectories(refactor_branch, directories, + refactor_branch_upstream): + print('Skipping ' + FormatDirectoriesForPrinting(directories) + + ' for which a branch already exists.') + return - # Checkout all changes to files in |files|. - deleted_files = [] - modified_files = [] - for action, f in files: - abspath = os.path.abspath(os.path.join(repository_root, f)) - if action == 'D': - deleted_files.append(abspath) - else: - modified_files.append(abspath) + # Checkout all changes to files in |files|. + deleted_files = [] + modified_files = [] + for action, f in files: + abspath = os.path.abspath(os.path.join(repository_root, f)) + if action == 'D': + deleted_files.append(abspath) + else: + modified_files.append(abspath) - if deleted_files: - git.run(*['rm'] + deleted_files) - if modified_files: - git.run(*['checkout', refactor_branch, '--'] + modified_files) + if deleted_files: + git.run(*['rm'] + deleted_files) + if modified_files: + git.run(*['checkout', refactor_branch, '--'] + modified_files) - # Commit changes. The temporary file is created with delete=False so that it - # can be deleted manually after git has read it rather than automatically - # when it is closed. - with gclient_utils.temporary_file() as tmp_file: - gclient_utils.FileWrite( - tmp_file, FormatDescriptionOrComment(description, directories)) - git.run('commit', '-F', tmp_file) + # Commit changes. The temporary file is created with delete=False so that it + # can be deleted manually after git has read it rather than automatically + # when it is closed. + with gclient_utils.temporary_file() as tmp_file: + gclient_utils.FileWrite( + tmp_file, FormatDescriptionOrComment(description, directories)) + git.run('commit', '-F', tmp_file) - # Upload a CL. - upload_args = ['-f'] - if reviewers: - upload_args.extend(['-r', ','.join(sorted(reviewers))]) - if cq_dry_run: - upload_args.append('--cq-dry-run') - if not comment: - upload_args.append('--send-mail') - if enable_auto_submit: - upload_args.append('--enable-auto-submit') - if topic: - upload_args.append('--topic={}'.format(topic)) - print('Uploading CL for ' + FormatDirectoriesForPrinting(directories) + '...') + # Upload a CL. + upload_args = ['-f'] + if reviewers: + upload_args.extend(['-r', ','.join(sorted(reviewers))]) + if cq_dry_run: + upload_args.append('--cq-dry-run') + if not comment: + upload_args.append('--send-mail') + if enable_auto_submit: + upload_args.append('--enable-auto-submit') + if topic: + upload_args.append('--topic={}'.format(topic)) + print('Uploading CL for ' + FormatDirectoriesForPrinting(directories) + + '...') - ret = cmd_upload(upload_args) - if ret != 0: - print('Uploading failed.') - print('Note: git cl split has built-in resume capabilities.') - print('Delete ' + git.current_branch() + - ' then run git cl split again to resume uploading.') + ret = cmd_upload(upload_args) + if ret != 0: + print('Uploading failed.') + print('Note: git cl split has built-in resume capabilities.') + print('Delete ' + git.current_branch() + + ' then run git cl split again to resume uploading.') - if comment: - changelist().AddComment(FormatDescriptionOrComment(comment, directories), - publish=True) + if comment: + changelist().AddComment(FormatDescriptionOrComment( + comment, directories), + publish=True) def GetFilesSplitByOwners(files, max_depth): - """Returns a map of files split by OWNERS file. + """Returns a map of files split by OWNERS file. Returns: A map where keys are paths to directories containing an OWNERS file and values are lists of files sharing an OWNERS file. """ - files_split_by_owners = {} - for action, path in files: - # normpath() is important to normalize separators here, in prepration for - # str.split() before. It would be nicer to use something like pathlib here - # but alas... - dir_with_owners = os.path.normpath(os.path.dirname(path)) - if max_depth >= 1: - dir_with_owners = os.path.join( - *dir_with_owners.split(os.path.sep)[:max_depth]) - # Find the closest parent directory with an OWNERS file. - while (dir_with_owners not in files_split_by_owners - and not os.path.isfile(os.path.join(dir_with_owners, 'OWNERS'))): - dir_with_owners = os.path.dirname(dir_with_owners) - files_split_by_owners.setdefault(dir_with_owners, []).append((action, path)) - return files_split_by_owners + files_split_by_owners = {} + for action, path in files: + # normpath() is important to normalize separators here, in prepration + # for str.split() before. It would be nicer to use something like + # pathlib here but alas... + dir_with_owners = os.path.normpath(os.path.dirname(path)) + if max_depth >= 1: + dir_with_owners = os.path.join( + *dir_with_owners.split(os.path.sep)[:max_depth]) + # Find the closest parent directory with an OWNERS file. + while (dir_with_owners not in files_split_by_owners + and not os.path.isfile(os.path.join(dir_with_owners, 'OWNERS'))): + dir_with_owners = os.path.dirname(dir_with_owners) + files_split_by_owners.setdefault(dir_with_owners, []).append( + (action, path)) + return files_split_by_owners def PrintClInfo(cl_index, num_cls, directories, file_paths, description, reviewers, enable_auto_submit, topic): - """Prints info about a CL. + """Prints info about a CL. Args: cl_index: The index of this CL in the list of CLs to upload. @@ -203,23 +204,23 @@ def PrintClInfo(cl_index, num_cls, directories, file_paths, description, enable_auto_submit: If the CL should also have auto submit enabled. topic: Topic to set for this CL. """ - description_lines = FormatDescriptionOrComment(description, - directories).splitlines() - indented_description = '\n'.join([' ' + l for l in description_lines]) + description_lines = FormatDescriptionOrComment(description, + directories).splitlines() + indented_description = '\n'.join([' ' + l for l in description_lines]) - print('CL {}/{}'.format(cl_index, num_cls)) - print('Paths: {}'.format(FormatDirectoriesForPrinting(directories))) - print('Reviewers: {}'.format(', '.join(reviewers))) - print('Auto-Submit: {}'.format(enable_auto_submit)) - print('Topic: {}'.format(topic)) - print('\n' + indented_description + '\n') - print('\n'.join(file_paths)) - print() + print('CL {}/{}'.format(cl_index, num_cls)) + print('Paths: {}'.format(FormatDirectoriesForPrinting(directories))) + print('Reviewers: {}'.format(', '.join(reviewers))) + print('Auto-Submit: {}'.format(enable_auto_submit)) + print('Topic: {}'.format(topic)) + print('\n' + indented_description + '\n') + print('\n'.join(file_paths)) + print() def SplitCl(description_file, comment_file, changelist, cmd_upload, dry_run, cq_dry_run, enable_auto_submit, max_depth, topic, repository_root): - """"Splits a branch into smaller branches and uploads CLs. + """"Splits a branch into smaller branches and uploads CLs. Args: description_file: File containing the description of uploaded CLs. @@ -236,89 +237,90 @@ def SplitCl(description_file, comment_file, changelist, cmd_upload, dry_run, Returns: 0 in case of success. 1 in case of error. """ - description = AddUploadedByGitClSplitToDescription( - gclient_utils.FileRead(description_file)) - comment = gclient_utils.FileRead(comment_file) if comment_file else None + description = AddUploadedByGitClSplitToDescription( + gclient_utils.FileRead(description_file)) + comment = gclient_utils.FileRead(comment_file) if comment_file else None - try: - EnsureInGitRepository() + try: + EnsureInGitRepository() - cl = changelist() - upstream = cl.GetCommonAncestorWithUpstream() - files = [ - (action.strip(), f) - for action, f in scm.GIT.CaptureStatus(repository_root, upstream) - ] + cl = changelist() + upstream = cl.GetCommonAncestorWithUpstream() + files = [ + (action.strip(), f) + for action, f in scm.GIT.CaptureStatus(repository_root, upstream) + ] - if not files: - print('Cannot split an empty CL.') - return 1 + if not files: + print('Cannot split an empty CL.') + return 1 - author = git.run('config', 'user.email').strip() or None - refactor_branch = git.current_branch() - assert refactor_branch, "Can't run from detached branch." - refactor_branch_upstream = git.upstream(refactor_branch) - assert refactor_branch_upstream, \ - "Branch %s must have an upstream." % refactor_branch + author = git.run('config', 'user.email').strip() or None + refactor_branch = git.current_branch() + assert refactor_branch, "Can't run from detached branch." + refactor_branch_upstream = git.upstream(refactor_branch) + assert refactor_branch_upstream, \ + "Branch %s must have an upstream." % refactor_branch - if not CheckDescriptionBugLink(description): - return 0 + if not CheckDescriptionBugLink(description): + return 0 - files_split_by_reviewers = SelectReviewersForFiles(cl, author, files, - max_depth) + files_split_by_reviewers = SelectReviewersForFiles( + cl, author, files, max_depth) - num_cls = len(files_split_by_reviewers) - print('Will split current branch (' + refactor_branch + ') into ' + - str(num_cls) + ' CLs.\n') - if cq_dry_run and num_cls > CL_SPLIT_FORCE_LIMIT: - print( - 'This will generate "%r" CLs. This many CLs can potentially generate' - ' too much load on the build infrastructure. Please email' - ' infra-dev@chromium.org to ensure that this won\'t break anything.' - ' The infra team reserves the right to cancel your jobs if they are' - ' overloading the CQ.' % num_cls) - answer = gclient_utils.AskForData('Proceed? (y/n):') - if answer.lower() != 'y': - return 0 + num_cls = len(files_split_by_reviewers) + print('Will split current branch (' + refactor_branch + ') into ' + + str(num_cls) + ' CLs.\n') + if cq_dry_run and num_cls > CL_SPLIT_FORCE_LIMIT: + print( + 'This will generate "%r" CLs. This many CLs can potentially' + ' generate too much load on the build infrastructure. Please' + ' email infra-dev@chromium.org to ensure that this won\'t break' + ' anything. The infra team reserves the right to cancel your' + ' jobs if they are overloading the CQ.' % num_cls) + answer = gclient_utils.AskForData('Proceed? (y/n):') + if answer.lower() != 'y': + return 0 - cls_per_reviewer = collections.defaultdict(int) - for cl_index, (reviewers, cl_info) in \ - enumerate(files_split_by_reviewers.items(), 1): - # Convert reviewers from tuple to set. - reviewer_set = set(reviewers) - if dry_run: - file_paths = [f for _, f in cl_info.files] - PrintClInfo(cl_index, num_cls, cl_info.owners_directories, file_paths, - description, reviewer_set, enable_auto_submit, topic) - else: - UploadCl(refactor_branch, refactor_branch_upstream, - cl_info.owners_directories, cl_info.files, description, - comment, reviewer_set, changelist, cmd_upload, cq_dry_run, - enable_auto_submit, topic, repository_root) + cls_per_reviewer = collections.defaultdict(int) + for cl_index, (reviewers, cl_info) in \ + enumerate(files_split_by_reviewers.items(), 1): + # Convert reviewers from tuple to set. + reviewer_set = set(reviewers) + if dry_run: + file_paths = [f for _, f in cl_info.files] + PrintClInfo(cl_index, num_cls, cl_info.owners_directories, + file_paths, description, reviewer_set, + enable_auto_submit, topic) + else: + UploadCl(refactor_branch, refactor_branch_upstream, + cl_info.owners_directories, cl_info.files, description, + comment, reviewer_set, changelist, cmd_upload, + cq_dry_run, enable_auto_submit, topic, repository_root) - for reviewer in reviewers: - cls_per_reviewer[reviewer] += 1 + for reviewer in reviewers: + cls_per_reviewer[reviewer] += 1 - # List the top reviewers that will be sent the most CLs as a result of the - # split. - reviewer_rankings = sorted(cls_per_reviewer.items(), - key=lambda item: item[1], - reverse=True) - print('The top reviewers are:') - for reviewer, count in reviewer_rankings[:CL_SPLIT_TOP_REVIEWERS]: - print(f' {reviewer}: {count} CLs') + # List the top reviewers that will be sent the most CLs as a result of + # the split. + reviewer_rankings = sorted(cls_per_reviewer.items(), + key=lambda item: item[1], + reverse=True) + print('The top reviewers are:') + for reviewer, count in reviewer_rankings[:CL_SPLIT_TOP_REVIEWERS]: + print(f' {reviewer}: {count} CLs') - # Go back to the original branch. - git.run('checkout', refactor_branch) + # Go back to the original branch. + git.run('checkout', refactor_branch) - except subprocess2.CalledProcessError as cpe: - sys.stderr.write(cpe.stderr) - return 1 - return 0 + except subprocess2.CalledProcessError as cpe: + sys.stderr.write(cpe.stderr) + return 1 + return 0 def CheckDescriptionBugLink(description): - """Verifies that the description contains a bug link. + """Verifies that the description contains a bug link. Examples: Bug: 123 @@ -326,17 +328,17 @@ def CheckDescriptionBugLink(description): Prompts user if the description does not contain a bug link. """ - bug_pattern = re.compile(r"^Bug:\s*(?:[a-zA-Z]+:)?[0-9]+", re.MULTILINE) - matches = re.findall(bug_pattern, description) - answer = 'y' - if not matches: - answer = gclient_utils.AskForData( - 'Description does not include a bug link. Proceed? (y/n):') - return answer.lower() == 'y' + bug_pattern = re.compile(r"^Bug:\s*(?:[a-zA-Z]+:)?[0-9]+", re.MULTILINE) + matches = re.findall(bug_pattern, description) + answer = 'y' + if not matches: + answer = gclient_utils.AskForData( + 'Description does not include a bug link. Proceed? (y/n):') + return answer.lower() == 'y' def SelectReviewersForFiles(cl, author, files, max_depth): - """Selects reviewers for passed-in files + """Selects reviewers for passed-in files Args: cl: Changelist class instance @@ -345,24 +347,24 @@ def SelectReviewersForFiles(cl, author, files, max_depth): max_depth: The maximum directory depth to search for OWNERS files. A value less than 1 means no limit. """ - info_split_by_owners = GetFilesSplitByOwners(files, max_depth) + info_split_by_owners = GetFilesSplitByOwners(files, max_depth) - info_split_by_reviewers = {} + info_split_by_reviewers = {} - for (directory, split_files) in info_split_by_owners.items(): - # Use '/' as a path separator in the branch name and the CL description - # and comment. - directory = directory.replace(os.path.sep, '/') - file_paths = [f for _, f in split_files] - # Convert reviewers list to tuple in order to use reviewers as key to - # dictionary. - reviewers = tuple( - cl.owners_client.SuggestOwners( - file_paths, exclude=[author, cl.owners_client.EVERYONE])) + for (directory, split_files) in info_split_by_owners.items(): + # Use '/' as a path separator in the branch name and the CL description + # and comment. + directory = directory.replace(os.path.sep, '/') + file_paths = [f for _, f in split_files] + # Convert reviewers list to tuple in order to use reviewers as key to + # dictionary. + reviewers = tuple( + cl.owners_client.SuggestOwners( + file_paths, exclude=[author, cl.owners_client.EVERYONE])) - if not reviewers in info_split_by_reviewers: - info_split_by_reviewers[reviewers] = FilesAndOwnersDirectory([], []) - info_split_by_reviewers[reviewers].files.extend(split_files) - info_split_by_reviewers[reviewers].owners_directories.append(directory) + if not reviewers in info_split_by_reviewers: + info_split_by_reviewers[reviewers] = FilesAndOwnersDirectory([], []) + info_split_by_reviewers[reviewers].files.extend(split_files) + info_split_by_reviewers[reviewers].owners_directories.append(directory) - return info_split_by_reviewers + return info_split_by_reviewers diff --git a/subcommand.py b/subcommand.py index 9d468d60c8..0d5a8aec76 100644 --- a/subcommand.py +++ b/subcommand.py @@ -1,7 +1,6 @@ # Copyright 2013 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Manages subcommands in a script. Each subcommand should look like this: @@ -46,51 +45,55 @@ import textwrap def usage(more): - """Adds a 'usage_more' property to a CMD function.""" - def hook(fn): - fn.usage_more = more - return fn - return hook + """Adds a 'usage_more' property to a CMD function.""" + def hook(fn): + fn.usage_more = more + return fn + + return hook def epilog(text): - """Adds an 'epilog' property to a CMD function. + """Adds an 'epilog' property to a CMD function. It will be shown in the epilog. Usually useful for examples. """ - def hook(fn): - fn.epilog = text - return fn - return hook + def hook(fn): + fn.epilog = text + return fn + + return hook def CMDhelp(parser, args): - """Prints list of commands or help for a specific command.""" - # This is the default help implementation. It can be disabled or overridden if - # wanted. - if not any(i in ('-h', '--help') for i in args): - args = args + ['--help'] - parser.parse_args(args) - # Never gets there. - assert False + """Prints list of commands or help for a specific command.""" + # This is the default help implementation. It can be disabled or overridden + # if wanted. + if not any(i in ('-h', '--help') for i in args): + args = args + ['--help'] + parser.parse_args(args) + # Never gets there. + assert False def _get_color_module(): - """Returns the colorama module if available. + """Returns the colorama module if available. If so, assumes colors are supported and return the module handle. """ - return sys.modules.get('colorama') or sys.modules.get('third_party.colorama') + return sys.modules.get('colorama') or sys.modules.get( + 'third_party.colorama') def _function_to_name(name): - """Returns the name of a CMD function.""" - return name[3:].replace('_', '-') + """Returns the name of a CMD function.""" + return name[3:].replace('_', '-') class CommandDispatcher(object): - def __init__(self, module): - """module is the name of the main python module where to look for commands. + def __init__(self, module): + """module is the name of the main python module where to look for + commands. The python builtin variable __name__ MUST be used for |module|. If the script is executed in the form 'python script.py', __name__ == '__main__' @@ -98,10 +101,10 @@ class CommandDispatcher(object): tested, __main__ will be the unit test's module so it has to reference to itself with 'script'. __name__ always match the right value. """ - self.module = sys.modules[module] + self.module = sys.modules[module] - def enumerate_commands(self): - """Returns a dict of command and their handling function. + def enumerate_commands(self): + """Returns a dict of command and their handling function. The commands must be in the '__main__' modules. To import a command from a submodule, use: @@ -115,149 +118,147 @@ class CommandDispatcher(object): e.g.: CMDhelp = None """ - cmds = dict( - (_function_to_name(name), getattr(self.module, name)) - for name in dir(self.module) if name.startswith('CMD')) - cmds.setdefault('help', CMDhelp) - return cmds + cmds = dict((_function_to_name(name), getattr(self.module, name)) + for name in dir(self.module) if name.startswith('CMD')) + cmds.setdefault('help', CMDhelp) + return cmds - def find_nearest_command(self, name_asked): - """Retrieves the function to handle a command as supplied by the user. + def find_nearest_command(self, name_asked): + """Retrieves the function to handle a command as supplied by the user. It automatically tries to guess the _intended command_ by handling typos and/or incomplete names. """ - commands = self.enumerate_commands() - name_to_dash = name_asked.replace('_', '-') - if name_to_dash in commands: - return commands[name_to_dash] + commands = self.enumerate_commands() + name_to_dash = name_asked.replace('_', '-') + if name_to_dash in commands: + return commands[name_to_dash] - # An exact match was not found. Try to be smart and look if there's - # something similar. - commands_with_prefix = [c for c in commands if c.startswith(name_asked)] - if len(commands_with_prefix) == 1: - return commands[commands_with_prefix[0]] + # An exact match was not found. Try to be smart and look if there's + # something similar. + commands_with_prefix = [c for c in commands if c.startswith(name_asked)] + if len(commands_with_prefix) == 1: + return commands[commands_with_prefix[0]] - # A #closeenough approximation of levenshtein distance. - def close_enough(a, b): - return difflib.SequenceMatcher(a=a, b=b).ratio() + # A #closeenough approximation of levenshtein distance. + def close_enough(a, b): + return difflib.SequenceMatcher(a=a, b=b).ratio() - hamming_commands = sorted( - ((close_enough(c, name_asked), c) for c in commands), - reverse=True) - if (hamming_commands[0][0] - hamming_commands[1][0]) < 0.3: - # Too ambiguous. - return None + hamming_commands = sorted( + ((close_enough(c, name_asked), c) for c in commands), reverse=True) + if (hamming_commands[0][0] - hamming_commands[1][0]) < 0.3: + # Too ambiguous. + return None - if hamming_commands[0][0] < 0.8: - # Not similar enough. Don't be a fool and run a random command. - return None + if hamming_commands[0][0] < 0.8: + # Not similar enough. Don't be a fool and run a random command. + return None - return commands[hamming_commands[0][1]] + return commands[hamming_commands[0][1]] - def _gen_commands_list(self): - """Generates the short list of supported commands.""" - commands = self.enumerate_commands() - docs = sorted( - (cmd_name, self._create_command_summary(cmd_name, handler)) - for cmd_name, handler in commands.items()) - # Skip commands without a docstring. - docs = [i for i in docs if i[1]] - # Then calculate maximum length for alignment: - length = max(len(c) for c in commands) + def _gen_commands_list(self): + """Generates the short list of supported commands.""" + commands = self.enumerate_commands() + docs = sorted( + (cmd_name, self._create_command_summary(cmd_name, handler)) + for cmd_name, handler in commands.items()) + # Skip commands without a docstring. + docs = [i for i in docs if i[1]] + # Then calculate maximum length for alignment: + length = max(len(c) for c in commands) - # Look if color is supported. - colors = _get_color_module() - green = reset = '' - if colors: - green = colors.Fore.GREEN - reset = colors.Fore.RESET - return ( - 'Commands are:\n' + - ''.join( - ' %s%-*s%s %s\n' % (green, length, cmd_name, reset, doc) - for cmd_name, doc in docs)) + # Look if color is supported. + colors = _get_color_module() + green = reset = '' + if colors: + green = colors.Fore.GREEN + reset = colors.Fore.RESET + return ('Commands are:\n' + + ''.join(' %s%-*s%s %s\n' % + (green, length, cmd_name, reset, doc) + for cmd_name, doc in docs)) - def _add_command_usage(self, parser, command): - """Modifies an OptionParser object with the function's documentation.""" - cmd_name = _function_to_name(command.__name__) - if cmd_name == 'help': - cmd_name = '' - # Use the module's docstring as the description for the 'help' command if - # available. - parser.description = (self.module.__doc__ or '').rstrip() - if parser.description: - parser.description += '\n\n' - parser.description += self._gen_commands_list() - # Do not touch epilog. - else: - # Use the command's docstring if available. For commands, unlike module - # docstring, realign. - lines = (command.__doc__ or '').rstrip().splitlines() - if lines[:1]: - rest = textwrap.dedent('\n'.join(lines[1:])) - parser.description = '\n'.join((lines[0], rest)) - else: - parser.description = lines[0] if lines else '' - if parser.description: - parser.description += '\n' - parser.epilog = getattr(command, 'epilog', None) - if parser.epilog: - parser.epilog = '\n' + parser.epilog.strip() + '\n' + def _add_command_usage(self, parser, command): + """Modifies an OptionParser object with the function's documentation.""" + cmd_name = _function_to_name(command.__name__) + if cmd_name == 'help': + cmd_name = '' + # Use the module's docstring as the description for the 'help' + # command if available. + parser.description = (self.module.__doc__ or '').rstrip() + if parser.description: + parser.description += '\n\n' + parser.description += self._gen_commands_list() + # Do not touch epilog. + else: + # Use the command's docstring if available. For commands, unlike + # module docstring, realign. + lines = (command.__doc__ or '').rstrip().splitlines() + if lines[:1]: + rest = textwrap.dedent('\n'.join(lines[1:])) + parser.description = '\n'.join((lines[0], rest)) + else: + parser.description = lines[0] if lines else '' + if parser.description: + parser.description += '\n' + parser.epilog = getattr(command, 'epilog', None) + if parser.epilog: + parser.epilog = '\n' + parser.epilog.strip() + '\n' - more = getattr(command, 'usage_more', '') - extra = '' if not more else ' ' + more - parser.set_usage('usage: %%prog %s [options]%s' % (cmd_name, extra)) + more = getattr(command, 'usage_more', '') + extra = '' if not more else ' ' + more + parser.set_usage('usage: %%prog %s [options]%s' % (cmd_name, extra)) - @staticmethod - def _create_command_summary(cmd_name, command): - """Creates a oneliner summary from the command's docstring.""" - if cmd_name != _function_to_name(command.__name__): - # Skip aliases. For example using at module level: - # CMDfoo = CMDbar - return '' - doc = command.__doc__ or '' - line = doc.split('\n', 1)[0].rstrip('.') - if not line: - return line - return (line[0].lower() + line[1:]).strip() + @staticmethod + def _create_command_summary(cmd_name, command): + """Creates a oneliner summary from the command's docstring.""" + if cmd_name != _function_to_name(command.__name__): + # Skip aliases. For example using at module level: + # CMDfoo = CMDbar + return '' + doc = command.__doc__ or '' + line = doc.split('\n', 1)[0].rstrip('.') + if not line: + return line + return (line[0].lower() + line[1:]).strip() - def execute(self, parser, args): - """Dispatches execution to the right command. + def execute(self, parser, args): + """Dispatches execution to the right command. Fallbacks to 'help' if not disabled. """ - # Unconditionally disable format_description() and format_epilog(). - # Technically, a formatter should be used but it's not worth (yet) the - # trouble. - parser.format_description = lambda _: parser.description or '' - parser.format_epilog = lambda _: parser.epilog or '' + # Unconditionally disable format_description() and format_epilog(). + # Technically, a formatter should be used but it's not worth (yet) the + # trouble. + parser.format_description = lambda _: parser.description or '' + parser.format_epilog = lambda _: parser.epilog or '' - if args: - if args[0] in ('-h', '--help') and len(args) > 1: - # Reverse the argument order so 'tool --help cmd' is rewritten to - # 'tool cmd --help'. - args = [args[1], args[0]] + args[2:] - command = self.find_nearest_command(args[0]) - if command: - if command.__name__ == 'CMDhelp' and len(args) > 1: - # Reverse the argument order so 'tool help cmd' is rewritten to - # 'tool cmd --help'. Do it here since we want 'tool help cmd' to work - # too. - args = [args[1], '--help'] + args[2:] - command = self.find_nearest_command(args[0]) or command + if args: + if args[0] in ('-h', '--help') and len(args) > 1: + # Reverse the argument order so 'tool --help cmd' is rewritten + # to 'tool cmd --help'. + args = [args[1], args[0]] + args[2:] + command = self.find_nearest_command(args[0]) + if command: + if command.__name__ == 'CMDhelp' and len(args) > 1: + # Reverse the argument order so 'tool help cmd' is rewritten + # to 'tool cmd --help'. Do it here since we want 'tool help + # cmd' to work too. + args = [args[1], '--help'] + args[2:] + command = self.find_nearest_command(args[0]) or command - # "fix" the usage and the description now that we know the subcommand. - self._add_command_usage(parser, command) - return command(parser, args[1:]) + # "fix" the usage and the description now that we know the + # subcommand. + self._add_command_usage(parser, command) + return command(parser, args[1:]) - cmdhelp = self.enumerate_commands().get('help') - if cmdhelp: - # Not a known command. Default to help. - self._add_command_usage(parser, cmdhelp) - # Don't pass list of arguments as those may not be supported by cmdhelp. - # See: https://crbug.com/1352093 - return cmdhelp(parser, []) + cmdhelp = self.enumerate_commands().get('help') + if cmdhelp: + # Not a known command. Default to help. + self._add_command_usage(parser, cmdhelp) + # Don't pass list of arguments as those may not be supported by + # cmdhelp. See: https://crbug.com/1352093 + return cmdhelp(parser, []) - # Nothing can be done. - return 2 + # Nothing can be done. + return 2 diff --git a/subprocess2.py b/subprocess2.py index 481917a383..bcee43138c 100644 --- a/subprocess2.py +++ b/subprocess2.py @@ -15,7 +15,6 @@ import subprocess import sys import threading - # Constants forwarded from subprocess. PIPE = subprocess.PIPE STDOUT = subprocess.STDOUT @@ -23,72 +22,74 @@ DEVNULL = subprocess.DEVNULL class CalledProcessError(subprocess.CalledProcessError): - """Augment the standard exception with more data.""" - def __init__(self, returncode, cmd, cwd, stdout, stderr): - super(CalledProcessError, self).__init__(returncode, cmd, output=stdout) - self.stdout = self.output # for backward compatibility. - self.stderr = stderr - self.cwd = cwd + """Augment the standard exception with more data.""" + def __init__(self, returncode, cmd, cwd, stdout, stderr): + super(CalledProcessError, self).__init__(returncode, cmd, output=stdout) + self.stdout = self.output # for backward compatibility. + self.stderr = stderr + self.cwd = cwd - def __str__(self): - out = 'Command %r returned non-zero exit status %s' % ( - ' '.join(self.cmd), self.returncode) - if self.cwd: - out += ' in ' + self.cwd - if self.stdout: - out += '\n' + self.stdout.decode('utf-8', 'ignore') - if self.stderr: - out += '\n' + self.stderr.decode('utf-8', 'ignore') - return out + def __str__(self): + out = 'Command %r returned non-zero exit status %s' % (' '.join( + self.cmd), self.returncode) + if self.cwd: + out += ' in ' + self.cwd + if self.stdout: + out += '\n' + self.stdout.decode('utf-8', 'ignore') + if self.stderr: + out += '\n' + self.stderr.decode('utf-8', 'ignore') + return out class CygwinRebaseError(CalledProcessError): - """Occurs when cygwin's fork() emulation fails due to rebased dll.""" + """Occurs when cygwin's fork() emulation fails due to rebased dll.""" ## Utility functions def kill_pid(pid): - """Kills a process by its process id.""" - try: - # Unable to import 'module' - # pylint: disable=no-member,F0401 - import signal - return os.kill(pid, signal.SIGTERM) - except ImportError: - pass + """Kills a process by its process id.""" + try: + # Unable to import 'module' + # pylint: disable=no-member,F0401 + import signal + return os.kill(pid, signal.SIGTERM) + except ImportError: + pass def get_english_env(env): - """Forces LANG and/or LANGUAGE to be English. + """Forces LANG and/or LANGUAGE to be English. Forces encoding to utf-8 for subprocesses. Returns None if it is unnecessary. """ - if sys.platform == 'win32': - return None - env = env or os.environ + if sys.platform == 'win32': + return None + env = env or os.environ - # Test if it is necessary at all. - is_english = lambda name: env.get(name, 'en').startswith('en') + # Test if it is necessary at all. + is_english = lambda name: env.get(name, 'en').startswith('en') - if is_english('LANG') and is_english('LANGUAGE'): - return None + if is_english('LANG') and is_english('LANGUAGE'): + return None - # Requires modifications. - env = env.copy() - def fix_lang(name): - if not is_english(name): - env[name] = 'en_US.UTF-8' - fix_lang('LANG') - fix_lang('LANGUAGE') - return env + # Requires modifications. + env = env.copy() + + def fix_lang(name): + if not is_english(name): + env[name] = 'en_US.UTF-8' + + fix_lang('LANG') + fix_lang('LANGUAGE') + return env class Popen(subprocess.Popen): - """Wraps subprocess.Popen() with various workarounds. + """Wraps subprocess.Popen() with various workarounds. - Forces English output since it's easier to parse the stdout if it is always in English. @@ -100,69 +101,68 @@ class Popen(subprocess.Popen): Note: Popen() can throw OSError when cwd or args[0] doesn't exist. Translate exceptions generated by cygwin when it fails trying to emulate fork(). """ - # subprocess.Popen.__init__() is not threadsafe; there is a race between - # creating the exec-error pipe for the child and setting it to CLOEXEC during - # which another thread can fork and cause the pipe to be inherited by its - # descendents, which will cause the current Popen to hang until all those - # descendents exit. Protect this with a lock so that only one fork/exec can - # happen at a time. - popen_lock = threading.Lock() + # subprocess.Popen.__init__() is not threadsafe; there is a race between + # creating the exec-error pipe for the child and setting it to CLOEXEC + # during which another thread can fork and cause the pipe to be inherited by + # its descendents, which will cause the current Popen to hang until all + # those descendents exit. Protect this with a lock so that only one + # fork/exec can happen at a time. + popen_lock = threading.Lock() - def __init__(self, args, **kwargs): - env = get_english_env(kwargs.get('env')) - if env: - kwargs['env'] = env - if kwargs.get('env') is not None: - # Subprocess expects environment variables to be strings in Python 3. - def ensure_str(value): - if isinstance(value, bytes): - return value.decode() - return value + def __init__(self, args, **kwargs): + env = get_english_env(kwargs.get('env')) + if env: + kwargs['env'] = env + if kwargs.get('env') is not None: + # Subprocess expects environment variables to be strings in Python + # 3. + def ensure_str(value): + if isinstance(value, bytes): + return value.decode() + return value - kwargs['env'] = { - ensure_str(k): ensure_str(v) - for k, v in kwargs['env'].items() - } - if kwargs.get('shell') is None: - # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for - # the executable, but shell=True makes subprocess on Linux fail when it's - # called with a list because it only tries to execute the first item in - # the list. - kwargs['shell'] = bool(sys.platform=='win32') + kwargs['env'] = { + ensure_str(k): ensure_str(v) + for k, v in kwargs['env'].items() + } + if kwargs.get('shell') is None: + # *Sigh*: Windows needs shell=True, or else it won't search %PATH% + # for the executable, but shell=True makes subprocess on Linux fail + # when it's called with a list because it only tries to execute the + # first item in the list. + kwargs['shell'] = bool(sys.platform == 'win32') - if isinstance(args, (str, bytes)): - tmp_str = args - elif isinstance(args, (list, tuple)): - tmp_str = ' '.join(args) - else: - raise CalledProcessError(None, args, kwargs.get('cwd'), None, None) - if kwargs.get('cwd', None): - tmp_str += '; cwd=%s' % kwargs['cwd'] - logging.debug(tmp_str) + if isinstance(args, (str, bytes)): + tmp_str = args + elif isinstance(args, (list, tuple)): + tmp_str = ' '.join(args) + else: + raise CalledProcessError(None, args, kwargs.get('cwd'), None, None) + if kwargs.get('cwd', None): + tmp_str += '; cwd=%s' % kwargs['cwd'] + logging.debug(tmp_str) - try: - with self.popen_lock: - super(Popen, self).__init__(args, **kwargs) - except OSError as e: - if e.errno == errno.EAGAIN and sys.platform == 'cygwin': - # Convert fork() emulation failure into a CygwinRebaseError(). - raise CygwinRebaseError( - e.errno, - args, - kwargs.get('cwd'), - None, - 'Visit ' - 'http://code.google.com/p/chromium/wiki/CygwinDllRemappingFailure ' - 'to learn how to fix this error; you need to rebase your cygwin ' - 'dlls') - # Popen() can throw OSError when cwd or args[0] doesn't exist. - raise OSError('Execution failed with error: %s.\n' - 'Check that %s or %s exist and have execution permission.' - % (str(e), kwargs.get('cwd'), args[0])) + try: + with self.popen_lock: + super(Popen, self).__init__(args, **kwargs) + except OSError as e: + if e.errno == errno.EAGAIN and sys.platform == 'cygwin': + # Convert fork() emulation failure into a CygwinRebaseError(). + raise CygwinRebaseError( + e.errno, args, kwargs.get('cwd'), None, 'Visit ' + 'http://code.google.com/p/chromium/wiki/' + 'CygwinDllRemappingFailure ' + 'to learn how to fix this error; you need to rebase your ' + 'cygwin dlls') + # Popen() can throw OSError when cwd or args[0] doesn't exist. + raise OSError( + 'Execution failed with error: %s.\n' + 'Check that %s or %s exist and have execution permission.' % + (str(e), kwargs.get('cwd'), args[0])) def communicate(args, **kwargs): - """Wraps subprocess.Popen().communicate(). + """Wraps subprocess.Popen().communicate(). Returns ((stdout, stderr), returncode). @@ -170,19 +170,19 @@ def communicate(args, **kwargs): output, print a warning to stderr. - Automatically passes stdin content as input so do not specify stdin=PIPE. """ - stdin = None - # When stdin is passed as an argument, use it as the actual input data and - # set the Popen() parameter accordingly. - if 'stdin' in kwargs and isinstance(kwargs['stdin'], (str, bytes)): - stdin = kwargs['stdin'] - kwargs['stdin'] = PIPE + stdin = None + # When stdin is passed as an argument, use it as the actual input data and + # set the Popen() parameter accordingly. + if 'stdin' in kwargs and isinstance(kwargs['stdin'], (str, bytes)): + stdin = kwargs['stdin'] + kwargs['stdin'] = PIPE - proc = Popen(args, **kwargs) - return proc.communicate(stdin), proc.returncode + proc = Popen(args, **kwargs) + return proc.communicate(stdin), proc.returncode def call(args, **kwargs): - """Emulates subprocess.call(). + """Emulates subprocess.call(). Automatically convert stdout=PIPE or stderr=PIPE to DEVNULL. In no case they can be returned since no code path raises @@ -190,47 +190,47 @@ def call(args, **kwargs): Returns exit code. """ - if kwargs.get('stdout') == PIPE: - kwargs['stdout'] = DEVNULL - if kwargs.get('stderr') == PIPE: - kwargs['stderr'] = DEVNULL - return communicate(args, **kwargs)[1] + if kwargs.get('stdout') == PIPE: + kwargs['stdout'] = DEVNULL + if kwargs.get('stderr') == PIPE: + kwargs['stderr'] = DEVNULL + return communicate(args, **kwargs)[1] def check_call_out(args, **kwargs): - """Improved version of subprocess.check_call(). + """Improved version of subprocess.check_call(). Returns (stdout, stderr), unlike subprocess.check_call(). """ - out, returncode = communicate(args, **kwargs) - if returncode: - raise CalledProcessError( - returncode, args, kwargs.get('cwd'), out[0], out[1]) - return out + out, returncode = communicate(args, **kwargs) + if returncode: + raise CalledProcessError(returncode, args, kwargs.get('cwd'), out[0], + out[1]) + return out def check_call(args, **kwargs): - """Emulate subprocess.check_call().""" - check_call_out(args, **kwargs) - return 0 + """Emulate subprocess.check_call().""" + check_call_out(args, **kwargs) + return 0 def capture(args, **kwargs): - """Captures stdout of a process call and returns it. + """Captures stdout of a process call and returns it. Returns stdout. - Discards returncode. - Blocks stdin by default if not specified since no output will be visible. """ - kwargs.setdefault('stdin', DEVNULL) + kwargs.setdefault('stdin', DEVNULL) - # Like check_output, deny the caller from using stdout arg. - return communicate(args, stdout=PIPE, **kwargs)[0][0] + # Like check_output, deny the caller from using stdout arg. + return communicate(args, stdout=PIPE, **kwargs)[0][0] def check_output(args, **kwargs): - """Emulates subprocess.check_output(). + """Emulates subprocess.check_output(). Captures stdout of a process call and returns stdout only. @@ -238,7 +238,7 @@ def check_output(args, **kwargs): - Blocks stdin by default if not specified since no output will be visible. - As per doc, "The stdout argument is not allowed as it is used internally." """ - kwargs.setdefault('stdin', DEVNULL) - if 'stdout' in kwargs: - raise ValueError('stdout argument not allowed, it would be overridden.') - return check_call_out(args, stdout=PIPE, **kwargs)[0] + kwargs.setdefault('stdin', DEVNULL) + if 'stdout' in kwargs: + raise ValueError('stdout argument not allowed, it would be overridden.') + return check_call_out(args, stdout=PIPE, **kwargs)[0] diff --git a/swift_format.py b/swift_format.py index 219211da7c..df31e7cc48 100644 --- a/swift_format.py +++ b/swift_format.py @@ -15,60 +15,60 @@ import sys class NotFoundError(Exception): - """A file could not be found.""" - - def __init__(self, e): - Exception.__init__( - self, - 'Problem while looking for swift-format in Chromium source tree:\n' - '%s' % e) + """A file could not be found.""" + def __init__(self, e): + Exception.__init__( + self, + 'Problem while looking for swift-format in Chromium source tree:\n' + '%s' % e) def FindSwiftFormatToolInChromiumTree(): - """Return a path to the rustfmt executable, or die trying.""" - chromium_src_path = gclient_paths.GetPrimarySolutionPath() - if not chromium_src_path: - raise NotFoundError( - 'Could not find checkout in any parent of the current path.\n' - 'Set CHROMIUM_BUILDTOOLS_PATH to use outside of a chromium checkout.') + """Return a path to the rustfmt executable, or die trying.""" + chromium_src_path = gclient_paths.GetPrimarySolutionPath() + if not chromium_src_path: + raise NotFoundError( + 'Could not find checkout in any parent of the current path.\n' + 'Set CHROMIUM_BUILDTOOLS_PATH to use outside of a chromium ' + 'checkout.') - tool_path = os.path.join(chromium_src_path, 'third_party', 'swift-format', - 'swift-format') - if not os.path.exists(tool_path): - raise NotFoundError('File does not exist: %s' % tool_path) - return tool_path + tool_path = os.path.join(chromium_src_path, 'third_party', 'swift-format', + 'swift-format') + if not os.path.exists(tool_path): + raise NotFoundError('File does not exist: %s' % tool_path) + return tool_path def IsSwiftFormatSupported(): - if sys.platform != 'darwin': - return False - try: - FindSwiftFormatToolInChromiumTree() - return True - except NotFoundError: - return False + if sys.platform != 'darwin': + return False + try: + FindSwiftFormatToolInChromiumTree() + return True + except NotFoundError: + return False def main(args): - try: - tool = FindSwiftFormatToolInChromiumTree() - except NotFoundError as e: - sys.stderr.write("%s\n" % str(e)) - return 1 + try: + tool = FindSwiftFormatToolInChromiumTree() + except NotFoundError as e: + sys.stderr.write("%s\n" % str(e)) + return 1 - # Add some visibility to --help showing where the tool lives, since this - # redirection can be a little opaque. - help_syntax = ('-h', '--help', '-help', '-help-list', '--help-list') - if any(match in args for match in help_syntax): - print('\nDepot tools redirects you to the swift-format at:\n %s\n' % - tool) + # Add some visibility to --help showing where the tool lives, since this + # redirection can be a little opaque. + help_syntax = ('-h', '--help', '-help', '-help-list', '--help-list') + if any(match in args for match in help_syntax): + print('\nDepot tools redirects you to the swift-format at:\n %s\n' % + tool) - return subprocess.call([tool] + args) + return subprocess.call([tool] + args) if __name__ == '__main__': - try: - sys.exit(main(sys.argv[1:])) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(1) + try: + sys.exit(main(sys.argv[1:])) + except KeyboardInterrupt: + sys.stderr.write('interrupted\n') + sys.exit(1) diff --git a/testing_support/.style.yapf b/testing_support/.style.yapf deleted file mode 100644 index 4741fb4f3b..0000000000 --- a/testing_support/.style.yapf +++ /dev/null @@ -1,3 +0,0 @@ -[style] -based_on_style = pep8 -column_limit = 80 diff --git a/testing_support/coverage_utils.py b/testing_support/coverage_utils.py index c3f9b18b4d..0fefab3e52 100644 --- a/testing_support/coverage_utils.py +++ b/testing_support/coverage_utils.py @@ -63,9 +63,8 @@ def covered_main(includes, sys.path.insert(0, os.path.join(ROOT_PATH, 'third_party')) import coverage else: - print( - "ERROR: python-coverage (%s) is required to be installed on " - "your PYTHONPATH to run this test." % require_native) + print("ERROR: python-coverage (%s) is required to be installed on " + "your PYTHONPATH to run this test." % require_native) sys.exit(1) COVERAGE = coverage.coverage(include=includes) diff --git a/tests/.style.yapf b/tests/.style.yapf deleted file mode 100644 index 4741fb4f3b..0000000000 --- a/tests/.style.yapf +++ /dev/null @@ -1,3 +0,0 @@ -[style] -based_on_style = pep8 -column_limit = 80 diff --git a/update_depot_tools_toggle.py b/update_depot_tools_toggle.py index 46fb109814..452ddd5308 100755 --- a/update_depot_tools_toggle.py +++ b/update_depot_tools_toggle.py @@ -2,7 +2,6 @@ # Copyright (c) 2017 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Small utility script to enable/disable `depot_tools` automatic updating.""" import argparse @@ -10,29 +9,31 @@ import datetime import os import sys - DEPOT_TOOLS_ROOT = os.path.abspath(os.path.dirname(__file__)) SENTINEL_PATH = os.path.join(DEPOT_TOOLS_ROOT, '.disable_auto_update') def main(): - parser = argparse.ArgumentParser() - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('--enable', action='store_true', - help='Enable auto-updating.') - group.add_argument('--disable', action='store_true', - help='Disable auto-updating.') - args = parser.parse_args() + parser = argparse.ArgumentParser() + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('--enable', + action='store_true', + help='Enable auto-updating.') + group.add_argument('--disable', + action='store_true', + help='Disable auto-updating.') + args = parser.parse_args() - if args.enable: - if os.path.exists(SENTINEL_PATH): - os.unlink(SENTINEL_PATH) - if args.disable: - if not os.path.exists(SENTINEL_PATH): - with open(SENTINEL_PATH, 'w') as fd: - fd.write('Disabled by %s at %s\n' % (__file__, datetime.datetime.now())) - return 0 + if args.enable: + if os.path.exists(SENTINEL_PATH): + os.unlink(SENTINEL_PATH) + if args.disable: + if not os.path.exists(SENTINEL_PATH): + with open(SENTINEL_PATH, 'w') as fd: + fd.write('Disabled by %s at %s\n' % + (__file__, datetime.datetime.now())) + return 0 if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/upload_metrics.py b/upload_metrics.py index 2174b0c1b1..0502790601 100644 --- a/upload_metrics.py +++ b/upload_metrics.py @@ -11,23 +11,24 @@ import urllib.request import auth import metrics_utils -def main(): - metrics = input() - try: - headers = {} - if 'bot_metrics' in metrics: - token = auth.Authenticator().get_access_token().token - headers = {'Authorization': 'Bearer ' + token} - urllib.request.urlopen(urllib.request.Request( - url=metrics_utils.APP_URL + '/upload', - data=metrics.encode('utf-8'), - headers=headers)) - except (urllib.error.HTTPError, urllib.error.URLError, - http.client.RemoteDisconnected): - pass - return 0 +def main(): + metrics = input() + try: + headers = {} + if 'bot_metrics' in metrics: + token = auth.Authenticator().get_access_token().token + headers = {'Authorization': 'Bearer ' + token} + urllib.request.urlopen( + urllib.request.Request(url=metrics_utils.APP_URL + '/upload', + data=metrics.encode('utf-8'), + headers=headers)) + except (urllib.error.HTTPError, urllib.error.URLError, + http.client.RemoteDisconnected): + pass + + return 0 if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/upload_to_google_storage.py b/upload_to_google_storage.py index 3a590abbd0..bf6d392b46 100755 --- a/upload_to_google_storage.py +++ b/upload_to_google_storage.py @@ -2,7 +2,6 @@ # Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Uploads files to Google Storage content addressed.""" from __future__ import print_function @@ -44,270 +43,292 @@ find . -name .svn -prune -o -size +1000k -type f -print0 | %prog -0 -b bkt - def get_md5(filename): - md5_calculator = hashlib.md5() - with open(filename, 'rb') as f: - while True: - chunk = f.read(1024*1024) - if not chunk: - break - md5_calculator.update(chunk) - return md5_calculator.hexdigest() + md5_calculator = hashlib.md5() + with open(filename, 'rb') as f: + while True: + chunk = f.read(1024 * 1024) + if not chunk: + break + md5_calculator.update(chunk) + return md5_calculator.hexdigest() def get_md5_cached(filename): - """Don't calculate the MD5 if we can find a .md5 file.""" - # See if we can find an existing MD5 sum stored in a file. - if os.path.exists('%s.md5' % filename): - with open('%s.md5' % filename, 'rb') as f: - md5_match = re.search('([a-z0-9]{32})', f.read().decode()) - if md5_match: - return md5_match.group(1) - else: - md5_hash = get_md5(filename) - with open('%s.md5' % filename, 'wb') as f: - f.write(md5_hash.encode()) - return md5_hash + """Don't calculate the MD5 if we can find a .md5 file.""" + # See if we can find an existing MD5 sum stored in a file. + if os.path.exists('%s.md5' % filename): + with open('%s.md5' % filename, 'rb') as f: + md5_match = re.search('([a-z0-9]{32})', f.read().decode()) + if md5_match: + return md5_match.group(1) + else: + md5_hash = get_md5(filename) + with open('%s.md5' % filename, 'wb') as f: + f.write(md5_hash.encode()) + return md5_hash -def _upload_worker( - thread_num, upload_queue, base_url, gsutil, md5_lock, force, - use_md5, stdout_queue, ret_codes, gzip): - while True: - filename, sha1_sum = upload_queue.get() - if not filename: - break - file_url = '%s/%s' % (base_url, sha1_sum) - if gsutil.check_call('ls', file_url)[0] == 0 and not force: - # File exists, check MD5 hash. - _, out, _ = gsutil.check_call_with_retries('ls', '-L', file_url) - etag_match = re.search(r'ETag:\s+\S+', out) - if etag_match: - stdout_queue.put( - '%d> File with url %s already exists' % (thread_num, file_url)) - remote_md5 = etag_match.group(0).split()[1] - # Calculate the MD5 checksum to match it to Google Storage's ETag. - with md5_lock: - if use_md5: - local_md5 = get_md5_cached(filename) - else: - local_md5 = get_md5(filename) - if local_md5 == remote_md5: - stdout_queue.put( - '%d> File %s already exists and MD5 matches, upload skipped' % - (thread_num, filename)) - continue - stdout_queue.put('%d> Uploading %s...' % ( - thread_num, filename)) - gsutil_args = ['-h', 'Cache-Control:public, max-age=31536000', 'cp'] - if gzip: - gsutil_args.extend(['-z', gzip]) - gsutil_args.extend([filename, file_url]) - code, _, err = gsutil.check_call_with_retries(*gsutil_args) - if code != 0: - ret_codes.put( - (code, - 'Encountered error on uploading %s to %s\n%s' % - (filename, file_url, err))) - continue - - # Mark executable files with the header "x-goog-meta-executable: 1" which - # the download script will check for to preserve the executable bit. - if not sys.platform.startswith('win'): - if os.stat(filename).st_mode & stat.S_IEXEC: - code, _, err = gsutil.check_call_with_retries( - 'setmeta', '-h', 'x-goog-meta-executable:1', file_url) +def _upload_worker(thread_num, upload_queue, base_url, gsutil, md5_lock, force, + use_md5, stdout_queue, ret_codes, gzip): + while True: + filename, sha1_sum = upload_queue.get() + if not filename: + break + file_url = '%s/%s' % (base_url, sha1_sum) + if gsutil.check_call('ls', file_url)[0] == 0 and not force: + # File exists, check MD5 hash. + _, out, _ = gsutil.check_call_with_retries('ls', '-L', file_url) + etag_match = re.search(r'ETag:\s+\S+', out) + if etag_match: + stdout_queue.put('%d> File with url %s already exists' % + (thread_num, file_url)) + remote_md5 = etag_match.group(0).split()[1] + # Calculate the MD5 checksum to match it to Google Storage's + # ETag. + with md5_lock: + if use_md5: + local_md5 = get_md5_cached(filename) + else: + local_md5 = get_md5(filename) + if local_md5 == remote_md5: + stdout_queue.put( + '%d> File %s already exists and MD5 matches, upload ' + 'skipped' % (thread_num, filename)) + continue + stdout_queue.put('%d> Uploading %s...' % (thread_num, filename)) + gsutil_args = ['-h', 'Cache-Control:public, max-age=31536000', 'cp'] + if gzip: + gsutil_args.extend(['-z', gzip]) + gsutil_args.extend([filename, file_url]) + code, _, err = gsutil.check_call_with_retries(*gsutil_args) if code != 0: - ret_codes.put( - (code, - 'Encountered error on setting metadata on %s\n%s' % - (file_url, err))) + ret_codes.put((code, 'Encountered error on uploading %s to %s\n%s' % + (filename, file_url, err))) + continue + + # Mark executable files with the header "x-goog-meta-executable: 1" + # which the download script will check for to preserve the executable + # bit. + if not sys.platform.startswith('win'): + if os.stat(filename).st_mode & stat.S_IEXEC: + code, _, err = gsutil.check_call_with_retries( + 'setmeta', '-h', 'x-goog-meta-executable:1', file_url) + if code != 0: + ret_codes.put( + (code, + 'Encountered error on setting metadata on %s\n%s' % + (file_url, err))) def get_targets(args, parser, use_null_terminator): - if not args: - parser.error('Missing target.') + if not args: + parser.error('Missing target.') - if len(args) == 1 and args[0] == '-': - # Take stdin as a newline or null separated list of files. - if use_null_terminator: - return sys.stdin.read().split('\0') + if len(args) == 1 and args[0] == '-': + # Take stdin as a newline or null separated list of files. + if use_null_terminator: + return sys.stdin.read().split('\0') - return sys.stdin.read().splitlines() + return sys.stdin.read().splitlines() - return args + return args -def upload_to_google_storage( - input_filenames, base_url, gsutil, force, - use_md5, num_threads, skip_hashing, gzip): - # We only want one MD5 calculation happening at a time to avoid HD thrashing. - md5_lock = threading.Lock() +def upload_to_google_storage(input_filenames, base_url, gsutil, force, use_md5, + num_threads, skip_hashing, gzip): + # We only want one MD5 calculation happening at a time to avoid HD + # thrashing. + md5_lock = threading.Lock() - # Start up all the worker threads plus the printer thread. - all_threads = [] - ret_codes = queue.Queue() - ret_codes.put((0, None)) - upload_queue = queue.Queue() - upload_timer = time.time() - stdout_queue = queue.Queue() - printer_thread = PrinterThread(stdout_queue) - printer_thread.daemon = True - printer_thread.start() - for thread_num in range(num_threads): - t = threading.Thread( - target=_upload_worker, - args=[thread_num, upload_queue, base_url, gsutil, md5_lock, - force, use_md5, stdout_queue, ret_codes, gzip]) - t.daemon = True - t.start() - all_threads.append(t) + # Start up all the worker threads plus the printer thread. + all_threads = [] + ret_codes = queue.Queue() + ret_codes.put((0, None)) + upload_queue = queue.Queue() + upload_timer = time.time() + stdout_queue = queue.Queue() + printer_thread = PrinterThread(stdout_queue) + printer_thread.daemon = True + printer_thread.start() + for thread_num in range(num_threads): + t = threading.Thread(target=_upload_worker, + args=[ + thread_num, upload_queue, base_url, gsutil, + md5_lock, force, use_md5, stdout_queue, + ret_codes, gzip + ]) + t.daemon = True + t.start() + all_threads.append(t) - # We want to hash everything in a single thread since its faster. - # The bottleneck is in disk IO, not CPU. - hashing_start = time.time() - has_missing_files = False - for filename in input_filenames: - if not os.path.exists(filename): - stdout_queue.put('Main> Error: %s not found, skipping.' % filename) - has_missing_files = True - continue - if os.path.exists('%s.sha1' % filename) and skip_hashing: - stdout_queue.put( - 'Main> Found hash for %s, sha1 calculation skipped.' % filename) - with open(filename + '.sha1', 'rb') as f: - sha1_file = f.read(1024) - if not re.match('^([a-z0-9]{40})$', sha1_file.decode()): - print('Invalid sha1 hash file %s.sha1' % filename, file=sys.stderr) - return 1 - upload_queue.put((filename, sha1_file.decode())) - continue - stdout_queue.put('Main> Calculating hash for %s...' % filename) - sha1_sum = get_sha1(filename) - with open(filename + '.sha1', 'wb') as f: - f.write(sha1_sum.encode()) - stdout_queue.put('Main> Done calculating hash for %s.' % filename) - upload_queue.put((filename, sha1_sum)) - hashing_duration = time.time() - hashing_start + # We want to hash everything in a single thread since its faster. + # The bottleneck is in disk IO, not CPU. + hashing_start = time.time() + has_missing_files = False + for filename in input_filenames: + if not os.path.exists(filename): + stdout_queue.put('Main> Error: %s not found, skipping.' % filename) + has_missing_files = True + continue + if os.path.exists('%s.sha1' % filename) and skip_hashing: + stdout_queue.put( + 'Main> Found hash for %s, sha1 calculation skipped.' % filename) + with open(filename + '.sha1', 'rb') as f: + sha1_file = f.read(1024) + if not re.match('^([a-z0-9]{40})$', sha1_file.decode()): + print('Invalid sha1 hash file %s.sha1' % filename, + file=sys.stderr) + return 1 + upload_queue.put((filename, sha1_file.decode())) + continue + stdout_queue.put('Main> Calculating hash for %s...' % filename) + sha1_sum = get_sha1(filename) + with open(filename + '.sha1', 'wb') as f: + f.write(sha1_sum.encode()) + stdout_queue.put('Main> Done calculating hash for %s.' % filename) + upload_queue.put((filename, sha1_sum)) + hashing_duration = time.time() - hashing_start - # Wait for everything to finish. - for _ in all_threads: - upload_queue.put((None, None)) # To mark the end of the work queue. - for t in all_threads: - t.join() - stdout_queue.put(None) - printer_thread.join() + # Wait for everything to finish. + for _ in all_threads: + upload_queue.put((None, None)) # To mark the end of the work queue. + for t in all_threads: + t.join() + stdout_queue.put(None) + printer_thread.join() - # Print timing information. - print('Hashing %s files took %1f seconds' % ( - len(input_filenames), hashing_duration)) - print('Uploading took %1f seconds' % (time.time() - upload_timer)) + # Print timing information. + print('Hashing %s files took %1f seconds' % + (len(input_filenames), hashing_duration)) + print('Uploading took %1f seconds' % (time.time() - upload_timer)) - # See if we ran into any errors. - max_ret_code = 0 - for ret_code, message in ret_codes.queue: - max_ret_code = max(ret_code, max_ret_code) - if message: - print(message, file=sys.stderr) - if has_missing_files: - print('One or more input files missing', file=sys.stderr) - max_ret_code = max(1, max_ret_code) + # See if we ran into any errors. + max_ret_code = 0 + for ret_code, message in ret_codes.queue: + max_ret_code = max(ret_code, max_ret_code) + if message: + print(message, file=sys.stderr) + if has_missing_files: + print('One or more input files missing', file=sys.stderr) + max_ret_code = max(1, max_ret_code) - if not max_ret_code: - print('Success!') + if not max_ret_code: + print('Success!') - return max_ret_code + return max_ret_code def create_archives(dirs): - archive_names = [] - for name in dirs: - tarname = '%s.tar.gz' % name - with tarfile.open(tarname, 'w:gz') as tar: - tar.add(name) - archive_names.append(tarname) - return archive_names + archive_names = [] + for name in dirs: + tarname = '%s.tar.gz' % name + with tarfile.open(tarname, 'w:gz') as tar: + tar.add(name) + archive_names.append(tarname) + return archive_names def validate_archive_dirs(dirs): - for d in dirs: - # We don't allow .. in paths in our archives. - if d == '..': - return False - # We only allow dirs. - if not os.path.isdir(d): - return False - # We don't allow sym links in our archives. - if os.path.islink(d): - return False - # We required that the subdirectories we are archiving are all just below - # cwd. - if d not in next(os.walk('.'))[1]: - return False + for d in dirs: + # We don't allow .. in paths in our archives. + if d == '..': + return False + # We only allow dirs. + if not os.path.isdir(d): + return False + # We don't allow sym links in our archives. + if os.path.islink(d): + return False + # We required that the subdirectories we are archiving are all just + # below cwd. + if d not in next(os.walk('.'))[1]: + return False - return True + return True def main(): - parser = optparse.OptionParser(USAGE_STRING) - parser.add_option('-b', '--bucket', - help='Google Storage bucket to upload to.') - parser.add_option('-e', '--boto', help='Specify a custom boto file.') - parser.add_option('-a', '--archive', action='store_true', - help='Archive directory as a tar.gz file') - parser.add_option('-f', '--force', action='store_true', - help='Force upload even if remote file exists.') - parser.add_option('-g', '--gsutil_path', default=GSUTIL_DEFAULT_PATH, - help='Path to the gsutil script.') - parser.add_option('-m', '--use_md5', action='store_true', - help='Generate MD5 files when scanning, and don\'t check ' - 'the MD5 checksum if a .md5 file is found.') - parser.add_option('-t', '--num_threads', default=1, type='int', - help='Number of uploader threads to run.') - parser.add_option('-s', '--skip_hashing', action='store_true', - help='Skip hashing if .sha1 file exists.') - parser.add_option('-0', '--use_null_terminator', action='store_true', - help='Use \\0 instead of \\n when parsing ' - 'the file list from stdin. This is useful if the input ' - 'is coming from "find ... -print0".') - parser.add_option('-z', '--gzip', metavar='ext', - help='Gzip files which end in ext. ' - 'ext is a comma-separated list') - (options, args) = parser.parse_args() + parser = optparse.OptionParser(USAGE_STRING) + parser.add_option('-b', + '--bucket', + help='Google Storage bucket to upload to.') + parser.add_option('-e', '--boto', help='Specify a custom boto file.') + parser.add_option('-a', + '--archive', + action='store_true', + help='Archive directory as a tar.gz file') + parser.add_option('-f', + '--force', + action='store_true', + help='Force upload even if remote file exists.') + parser.add_option('-g', + '--gsutil_path', + default=GSUTIL_DEFAULT_PATH, + help='Path to the gsutil script.') + parser.add_option('-m', + '--use_md5', + action='store_true', + help='Generate MD5 files when scanning, and don\'t check ' + 'the MD5 checksum if a .md5 file is found.') + parser.add_option('-t', + '--num_threads', + default=1, + type='int', + help='Number of uploader threads to run.') + parser.add_option('-s', + '--skip_hashing', + action='store_true', + help='Skip hashing if .sha1 file exists.') + parser.add_option('-0', + '--use_null_terminator', + action='store_true', + help='Use \\0 instead of \\n when parsing ' + 'the file list from stdin. This is useful if the input ' + 'is coming from "find ... -print0".') + parser.add_option('-z', + '--gzip', + metavar='ext', + help='Gzip files which end in ext. ' + 'ext is a comma-separated list') + (options, args) = parser.parse_args() - # Enumerate our inputs. - input_filenames = get_targets(args, parser, options.use_null_terminator) + # Enumerate our inputs. + input_filenames = get_targets(args, parser, options.use_null_terminator) - if options.archive: - if not validate_archive_dirs(input_filenames): - parser.error('Only directories just below cwd are valid entries when ' - 'using the --archive argument. Entries can not contain .. ' - ' and entries can not be symlinks. Entries was %s' % - input_filenames) - return 1 - input_filenames = create_archives(input_filenames) + if options.archive: + if not validate_archive_dirs(input_filenames): + parser.error( + 'Only directories just below cwd are valid entries when ' + 'using the --archive argument. Entries can not contain .. ' + ' and entries can not be symlinks. Entries was %s' % + input_filenames) + return 1 + input_filenames = create_archives(input_filenames) - # Make sure we can find a working instance of gsutil. - if os.path.exists(GSUTIL_DEFAULT_PATH): - gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto) - else: - gsutil = None - for path in os.environ["PATH"].split(os.pathsep): - if os.path.exists(path) and 'gsutil' in os.listdir(path): - gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto) - if not gsutil: - parser.error('gsutil not found in %s, bad depot_tools checkout?' % - GSUTIL_DEFAULT_PATH) + # Make sure we can find a working instance of gsutil. + if os.path.exists(GSUTIL_DEFAULT_PATH): + gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto) + else: + gsutil = None + for path in os.environ["PATH"].split(os.pathsep): + if os.path.exists(path) and 'gsutil' in os.listdir(path): + gsutil = Gsutil(os.path.join(path, 'gsutil'), + boto_path=options.boto) + if not gsutil: + parser.error('gsutil not found in %s, bad depot_tools checkout?' % + GSUTIL_DEFAULT_PATH) - base_url = 'gs://%s' % options.bucket + base_url = 'gs://%s' % options.bucket - return upload_to_google_storage( - input_filenames, base_url, gsutil, options.force, options.use_md5, - options.num_threads, options.skip_hashing, options.gzip) + return upload_to_google_storage(input_filenames, base_url, gsutil, + options.force, options.use_md5, + options.num_threads, options.skip_hashing, + options.gzip) if __name__ == '__main__': - try: - sys.exit(main()) - except KeyboardInterrupt: - sys.stderr.write('interrupted\n') - sys.exit(1) + try: + sys.exit(main()) + except KeyboardInterrupt: + sys.stderr.write('interrupted\n') + sys.exit(1) diff --git a/utils.py b/utils.py index 3f16ce98ba..a85d2f722e 100644 --- a/utils.py +++ b/utils.py @@ -7,19 +7,19 @@ import subprocess def depot_tools_version(): - depot_tools_root = os.path.dirname(os.path.abspath(__file__)) - try: - commit_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], - cwd=depot_tools_root).decode( - 'utf-8', 'ignore') - return 'git-%s' % commit_hash - except Exception: - pass + depot_tools_root = os.path.dirname(os.path.abspath(__file__)) + try: + commit_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], + cwd=depot_tools_root).decode( + 'utf-8', 'ignore') + return 'git-%s' % commit_hash + except Exception: + pass - # git check failed, let's check last modification of frequently checked file - try: - mtime = os.path.getmtime( - os.path.join(depot_tools_root, 'infra', 'config', 'recipes.cfg')) - return 'recipes.cfg-%d' % (mtime) - except Exception: - return 'unknown' + # git check failed, let's check last modification of frequently checked file + try: + mtime = os.path.getmtime( + os.path.join(depot_tools_root, 'infra', 'config', 'recipes.cfg')) + return 'recipes.cfg-%d' % (mtime) + except Exception: + return 'unknown' diff --git a/watchlists.py b/watchlists.py index fca78a9238..7c6d776d69 100755 --- a/watchlists.py +++ b/watchlists.py @@ -2,7 +2,6 @@ # Copyright (c) 2011 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Watchlists Watchlists is a mechanism that allow a developer (a "watcher") to watch over @@ -27,7 +26,7 @@ import sys class Watchlists(object): - """Manage Watchlists. + """Manage Watchlists. This class provides mechanism to load watchlists for a repo and identify watchers. @@ -37,77 +36,79 @@ class Watchlists(object): "/path/to/file2",]) """ - _RULES = "WATCHLISTS" - _RULES_FILENAME = _RULES - _repo_root = None - _defns = {} # Definitions - _path_regexps = {} # Name -> Regular expression mapping - _watchlists = {} # name to email mapping + _RULES = "WATCHLISTS" + _RULES_FILENAME = _RULES + _repo_root = None + _defns = {} # Definitions + _path_regexps = {} # Name -> Regular expression mapping + _watchlists = {} # name to email mapping - def __init__(self, repo_root): - self._repo_root = repo_root - self._LoadWatchlistRules() + def __init__(self, repo_root): + self._repo_root = repo_root + self._LoadWatchlistRules() - def _GetRulesFilePath(self): - """Returns path to WATCHLISTS file.""" - return os.path.join(self._repo_root, self._RULES_FILENAME) + def _GetRulesFilePath(self): + """Returns path to WATCHLISTS file.""" + return os.path.join(self._repo_root, self._RULES_FILENAME) - def _HasWatchlistsFile(self): - """Determine if watchlists are available for this repo.""" - return os.path.exists(self._GetRulesFilePath()) + def _HasWatchlistsFile(self): + """Determine if watchlists are available for this repo.""" + return os.path.exists(self._GetRulesFilePath()) - def _ContentsOfWatchlistsFile(self): - """Read the WATCHLISTS file and return its contents.""" - try: - watchlists_file = open(self._GetRulesFilePath()) - contents = watchlists_file.read() - watchlists_file.close() - return contents - except IOError as e: - logging.error("Cannot read %s: %s" % (self._GetRulesFilePath(), e)) - return '' + def _ContentsOfWatchlistsFile(self): + """Read the WATCHLISTS file and return its contents.""" + try: + watchlists_file = open(self._GetRulesFilePath()) + contents = watchlists_file.read() + watchlists_file.close() + return contents + except IOError as e: + logging.error("Cannot read %s: %s" % (self._GetRulesFilePath(), e)) + return '' - def _LoadWatchlistRules(self): - """Load watchlists from WATCHLISTS file. Does nothing if not present.""" - if not self._HasWatchlistsFile(): - return + def _LoadWatchlistRules(self): + """Load watchlists from WATCHLISTS file. Does nothing if not present.""" + if not self._HasWatchlistsFile(): + return - contents = self._ContentsOfWatchlistsFile() - watchlists_data = None - try: - watchlists_data = eval(contents, {'__builtins__': None}, None) - except SyntaxError as e: - logging.error("Cannot parse %s. %s" % (self._GetRulesFilePath(), e)) - return + contents = self._ContentsOfWatchlistsFile() + watchlists_data = None + try: + watchlists_data = eval(contents, {'__builtins__': None}, None) + except SyntaxError as e: + logging.error("Cannot parse %s. %s" % (self._GetRulesFilePath(), e)) + return - defns = watchlists_data.get("WATCHLIST_DEFINITIONS") - if not defns: - logging.error("WATCHLIST_DEFINITIONS not defined in %s" % - self._GetRulesFilePath()) - return - watchlists = watchlists_data.get("WATCHLISTS") - if not watchlists: - logging.error("WATCHLISTS not defined in %s" % self._GetRulesFilePath()) - return - self._defns = defns - self._watchlists = watchlists + defns = watchlists_data.get("WATCHLIST_DEFINITIONS") + if not defns: + logging.error("WATCHLIST_DEFINITIONS not defined in %s" % + self._GetRulesFilePath()) + return + watchlists = watchlists_data.get("WATCHLISTS") + if not watchlists: + logging.error("WATCHLISTS not defined in %s" % + self._GetRulesFilePath()) + return + self._defns = defns + self._watchlists = watchlists - # Compile the regular expressions ahead of time to avoid creating them - # on-the-fly multiple times per file. - self._path_regexps = {} - for name, rule in defns.items(): - filepath = rule.get('filepath') - if not filepath: - continue - self._path_regexps[name] = re.compile(filepath) + # Compile the regular expressions ahead of time to avoid creating them + # on-the-fly multiple times per file. + self._path_regexps = {} + for name, rule in defns.items(): + filepath = rule.get('filepath') + if not filepath: + continue + self._path_regexps[name] = re.compile(filepath) - # Verify that all watchlist names are defined - for name in watchlists: - if name not in defns: - logging.error("%s not defined in %s" % (name, self._GetRulesFilePath())) + # Verify that all watchlist names are defined + for name in watchlists: + if name not in defns: + logging.error("%s not defined in %s" % + (name, self._GetRulesFilePath())) - def GetWatchersForPaths(self, paths): - """Fetch the list of watchers for |paths| + def GetWatchersForPaths(self, paths): + """Fetch the list of watchers for |paths| Args: paths: [path1, path2, ...] @@ -115,28 +116,28 @@ class Watchlists(object): Returns: [u1@chromium.org, u2@gmail.com, ...] """ - watchers = set() # A set, to avoid duplicates - for path in paths: - path = path.replace(os.sep, '/') - for name, rule in self._path_regexps.items(): - if name not in self._watchlists: - continue - if rule.search(path): - for watchlist in self._watchlists[name]: - watchers.add(watchlist) - return sorted(watchers) + watchers = set() # A set, to avoid duplicates + for path in paths: + path = path.replace(os.sep, '/') + for name, rule in self._path_regexps.items(): + if name not in self._watchlists: + continue + if rule.search(path): + for watchlist in self._watchlists[name]: + watchers.add(watchlist) + return sorted(watchers) def main(argv): - # Confirm that watchlists can be parsed and spew out the watchers - if len(argv) < 2: - print("Usage (from the base of repo):") - print(" %s [file-1] [file-2] ...." % argv[0]) - return 1 - wl = Watchlists(os.getcwd()) - watchers = wl.GetWatchersForPaths(argv[1:]) - print(watchers) + # Confirm that watchlists can be parsed and spew out the watchers + if len(argv) < 2: + print("Usage (from the base of repo):") + print(" %s [file-1] [file-2] ...." % argv[0]) + return 1 + wl = Watchlists(os.getcwd()) + watchers = wl.GetWatchersForPaths(argv[1:]) + print(watchers) if __name__ == '__main__': - main(sys.argv) + main(sys.argv) diff --git a/weekly b/weekly index 853522597d..431440205a 100755 --- a/weekly +++ b/weekly @@ -2,7 +2,6 @@ # Copyright (c) 2010 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Display log of checkins of one particular developer since a particular date. Only works on git dependencies at the moment.""" @@ -17,38 +16,40 @@ import sys def show_log(path, authors, since='1 week ago'): - """Display log in a single git repo.""" + """Display log in a single git repo.""" - author_option = ' '.join(['--author=' + author for author in authors]) - command = ' '.join(['git log', author_option, '--since="%s"' % since, - 'origin/HEAD', '| git shortlog']) - status = subprocess.Popen(['sh', '-c', command], - cwd=path, - stdout=subprocess.PIPE).communicate()[0].rstrip() + author_option = ' '.join(['--author=' + author for author in authors]) + command = ' '.join([ + 'git log', author_option, + '--since="%s"' % since, 'origin/HEAD', '| git shortlog' + ]) + status = subprocess.Popen(['sh', '-c', command], + cwd=path, + stdout=subprocess.PIPE).communicate()[0].rstrip() - if len(status.splitlines()) > 0: - print('---------- %s ----------' % path) - print(status) + if len(status.splitlines()) > 0: + print('---------- %s ----------' % path) + print(status) def main(): - """Take no arguments.""" + """Take no arguments.""" - option_parser = optparse.OptionParser() - option_parser.add_option("-a", "--author", action="append", default=[]) - option_parser.add_option("-s", "--since", default="1 week ago") - options, args = option_parser.parse_args() + option_parser = optparse.OptionParser() + option_parser.add_option("-a", "--author", action="append", default=[]) + option_parser.add_option("-s", "--since", default="1 week ago") + options, args = option_parser.parse_args() - root, entries = gclient_utils.GetGClientRootAndEntries() + root, entries = gclient_utils.GetGClientRootAndEntries() - # which entries map to a git repos? - paths = [k for k, v in entries.items() if not re.search('svn', v)] - paths.sort() + # which entries map to a git repos? + paths = [k for k, v in entries.items() if not re.search('svn', v)] + paths.sort() - for path in paths: - dir = os.path.normpath(os.path.join(root, path)) - show_log(dir, options.author, options.since) + for path in paths: + dir = os.path.normpath(os.path.join(root, path)) + show_log(dir, options.author, options.since) if __name__ == '__main__': - main() + main() diff --git a/win32imports.py b/win32imports.py index 6de5471126..d5c2867fcf 100644 --- a/win32imports.py +++ b/win32imports.py @@ -14,13 +14,13 @@ LOCKFILE_FAIL_IMMEDIATELY = 0x00000001 class Overlapped(ctypes.Structure): - """Overlapped is required and used in LockFileEx and UnlockFileEx.""" - _fields_ = [('Internal', ctypes.wintypes.LPVOID), - ('InternalHigh', ctypes.wintypes.LPVOID), - ('Offset', ctypes.wintypes.DWORD), - ('OffsetHigh', ctypes.wintypes.DWORD), - ('Pointer', ctypes.wintypes.LPVOID), - ('hEvent', ctypes.wintypes.HANDLE)] + """Overlapped is required and used in LockFileEx and UnlockFileEx.""" + _fields_ = [('Internal', ctypes.wintypes.LPVOID), + ('InternalHigh', ctypes.wintypes.LPVOID), + ('Offset', ctypes.wintypes.DWORD), + ('OffsetHigh', ctypes.wintypes.DWORD), + ('Pointer', ctypes.wintypes.LPVOID), + ('hEvent', ctypes.wintypes.HANDLE)] # https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew diff --git a/win_toolchain/get_toolchain_if_necessary.py b/win_toolchain/get_toolchain_if_necessary.py index e4d606f2f0..7942f9a11c 100755 --- a/win_toolchain/get_toolchain_if_necessary.py +++ b/win_toolchain/get_toolchain_if_necessary.py @@ -2,7 +2,6 @@ # Copyright 2013 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Downloads and unpacks a toolchain for building on Windows. The contents are matched by sha1 which will be updated when the toolchain is updated. @@ -42,568 +41,597 @@ ENV_TOOLCHAIN_ROOT = 'DEPOT_TOOLS_WIN_TOOLCHAIN_ROOT' # winreg isn't natively available under CygWin if sys.platform == "win32": - try: - import winreg - except ImportError: - import _winreg as winreg + try: + import winreg + except ImportError: + import _winreg as winreg elif sys.platform == "cygwin": - try: - import cygwinreg as winreg - except ImportError: - print('') - print('CygWin does not natively support winreg but a replacement exists.') - print('https://pypi.python.org/pypi/cygwinreg/') - print('') - print('Try: easy_install cygwinreg') - print('') - raise + try: + import cygwinreg as winreg + except ImportError: + print('') + print( + 'CygWin does not natively support winreg but a replacement exists.') + print('https://pypi.python.org/pypi/cygwinreg/') + print('') + print('Try: easy_install cygwinreg') + print('') + raise BASEDIR = os.path.dirname(os.path.abspath(__file__)) DEPOT_TOOLS_PATH = os.path.join(BASEDIR, '..') sys.path.append(DEPOT_TOOLS_PATH) try: - import download_from_google_storage + import download_from_google_storage except ImportError: - # Allow use of utility functions in this script from package_from_installed - # on bare VM that doesn't have a full depot_tools. - pass + # Allow use of utility functions in this script from package_from_installed + # on bare VM that doesn't have a full depot_tools. + pass def GetFileList(root): - """Gets a normalized list of files under |root|.""" - assert not os.path.isabs(root) - assert os.path.normpath(root) == root - file_list = [] - # Ignore WER ReportQueue entries that vctip/cl leave in the bin dir if/when - # they crash. Also ignores the content of the - # Windows Kits/10/debuggers/x(86|64)/(sym|src)/ directories as this is just - # the temporarily location that Windbg might use to store the symbol files - # and downloaded sources. - # - # Note: These files are only created on a Windows host, so the - # ignored_directories list isn't relevant on non-Windows hosts. + """Gets a normalized list of files under |root|.""" + assert not os.path.isabs(root) + assert os.path.normpath(root) == root + file_list = [] + # Ignore WER ReportQueue entries that vctip/cl leave in the bin dir if/when + # they crash. Also ignores the content of the + # Windows Kits/10/debuggers/x(86|64)/(sym|src)/ directories as this is just + # the temporarily location that Windbg might use to store the symbol files + # and downloaded sources. + # + # Note: These files are only created on a Windows host, so the + # ignored_directories list isn't relevant on non-Windows hosts. - # The Windows SDK is either in `win_sdk` or in `Windows Kits\10`. This - # script must work with both layouts, so check which one it is. - # This can be different in each |root|. - if os.path.isdir(os.path.join(root, 'Windows Kits', '10')): - win_sdk = 'Windows Kits\\10' - else: - win_sdk = 'win_sdk' + # The Windows SDK is either in `win_sdk` or in `Windows Kits\10`. This + # script must work with both layouts, so check which one it is. + # This can be different in each |root|. + if os.path.isdir(os.path.join(root, 'Windows Kits', '10')): + win_sdk = 'Windows Kits\\10' + else: + win_sdk = 'win_sdk' - ignored_directories = ['wer\\reportqueue', - win_sdk + '\\debuggers\\x86\\sym\\', - win_sdk + '\\debuggers\\x64\\sym\\', - win_sdk + '\\debuggers\\x86\\src\\', - win_sdk + '\\debuggers\\x64\\src\\'] - ignored_directories = [d.lower() for d in ignored_directories] + ignored_directories = [ + 'wer\\reportqueue', win_sdk + '\\debuggers\\x86\\sym\\', + win_sdk + '\\debuggers\\x64\\sym\\', + win_sdk + '\\debuggers\\x86\\src\\', win_sdk + '\\debuggers\\x64\\src\\' + ] + ignored_directories = [d.lower() for d in ignored_directories] - for base, _, files in os.walk(root): - paths = [os.path.join(base, f) for f in files] - for p in paths: - if any(ignored_dir in p.lower() for ignored_dir in ignored_directories): - continue - file_list.append(p) - return sorted(file_list, key=lambda s: s.replace('/', '\\').lower()) + for base, _, files in os.walk(root): + paths = [os.path.join(base, f) for f in files] + for p in paths: + if any(ignored_dir in p.lower() + for ignored_dir in ignored_directories): + continue + file_list.append(p) + return sorted(file_list, key=lambda s: s.replace('/', '\\').lower()) def MakeTimestampsFileName(root, sha1): - return os.path.join(root, os.pardir, '%s.timestamps' % sha1) + return os.path.join(root, os.pardir, '%s.timestamps' % sha1) def CalculateHash(root, expected_hash): - """Calculates the sha1 of the paths to all files in the given |root| and the + """Calculates the sha1 of the paths to all files in the given |root| and the contents of those files, and returns as a hex string. |expected_hash| is the expected hash value for this toolchain if it has already been installed. """ - if expected_hash: - full_root_path = os.path.join(root, expected_hash) - else: - full_root_path = root - file_list = GetFileList(full_root_path) - # Check whether we previously saved timestamps in $root/../{sha1}.timestamps. - # If we didn't, or they don't match, then do the full calculation, otherwise - # return the saved value. - timestamps_file = MakeTimestampsFileName(root, expected_hash) - timestamps_data = {'files': [], 'sha1': ''} - if os.path.exists(timestamps_file): - with open(timestamps_file, 'rb') as f: - try: - timestamps_data = json.load(f) - except ValueError: - # json couldn't be loaded, empty data will force a re-hash. - pass - - matches = len(file_list) == len(timestamps_data['files']) - # Don't check the timestamp of the version file as we touch this file to - # indicates which versions of the toolchain are still being used. - vc_dir = os.path.join(full_root_path, 'VC').lower() - if matches: - for disk, cached in zip(file_list, timestamps_data['files']): - if disk != cached[0] or ( - disk != vc_dir and os.path.getmtime(disk) != cached[1]): - matches = False - break - elif os.path.exists(timestamps_file): - # Print some information about the extra/missing files. Don't do this if we - # don't have a timestamp file, as all the files will be considered as - # missing. - timestamps_data_files = [] - for f in timestamps_data['files']: - timestamps_data_files.append(f[0]) - missing_files = [f for f in timestamps_data_files if f not in file_list] - if len(missing_files): - print('%d files missing from the %s version of the toolchain:' % - (len(missing_files), expected_hash)) - for f in missing_files[:10]: - print('\t%s' % f) - if len(missing_files) > 10: - print('\t...') - extra_files = [f for f in file_list if f not in timestamps_data_files] - if len(extra_files): - print('%d extra files in the %s version of the toolchain:' % - (len(extra_files), expected_hash)) - for f in extra_files[:10]: - print('\t%s' % f) - if len(extra_files) > 10: - print('\t...') - if matches: - return timestamps_data['sha1'] - - # Make long hangs when updating the toolchain less mysterious. - print('Calculating hash of toolchain in %s. Please wait...' % full_root_path) - sys.stdout.flush() - digest = hashlib.sha1() - for path in file_list: - path_without_hash = str(path).replace('/', '\\') if expected_hash: - path_without_hash = path_without_hash.replace( - os.path.join(root, expected_hash).replace('/', '\\'), root) - digest.update(bytes(path_without_hash.lower(), 'utf-8')) - with open(path, 'rb') as f: - digest.update(f.read()) + full_root_path = os.path.join(root, expected_hash) + else: + full_root_path = root + file_list = GetFileList(full_root_path) + # Check whether we previously saved timestamps in + # $root/../{sha1}.timestamps. If we didn't, or they don't match, then do the + # full calculation, otherwise return the saved value. + timestamps_file = MakeTimestampsFileName(root, expected_hash) + timestamps_data = {'files': [], 'sha1': ''} + if os.path.exists(timestamps_file): + with open(timestamps_file, 'rb') as f: + try: + timestamps_data = json.load(f) + except ValueError: + # json couldn't be loaded, empty data will force a re-hash. + pass - # Save the timestamp file if the calculated hash is the expected one. - # The expected hash may be shorter, to reduce path lengths, in which case just - # compare that many characters. - if expected_hash and digest.hexdigest().startswith(expected_hash): - SaveTimestampsAndHash(root, digest.hexdigest()) - # Return the (potentially truncated) expected_hash. - return expected_hash - return digest.hexdigest() + matches = len(file_list) == len(timestamps_data['files']) + # Don't check the timestamp of the version file as we touch this file to + # indicates which versions of the toolchain are still being used. + vc_dir = os.path.join(full_root_path, 'VC').lower() + if matches: + for disk, cached in zip(file_list, timestamps_data['files']): + if disk != cached[0] or (disk != vc_dir + and os.path.getmtime(disk) != cached[1]): + matches = False + break + elif os.path.exists(timestamps_file): + # Print some information about the extra/missing files. Don't do this if + # we don't have a timestamp file, as all the files will be considered as + # missing. + timestamps_data_files = [] + for f in timestamps_data['files']: + timestamps_data_files.append(f[0]) + missing_files = [f for f in timestamps_data_files if f not in file_list] + if len(missing_files): + print('%d files missing from the %s version of the toolchain:' % + (len(missing_files), expected_hash)) + for f in missing_files[:10]: + print('\t%s' % f) + if len(missing_files) > 10: + print('\t...') + extra_files = [f for f in file_list if f not in timestamps_data_files] + if len(extra_files): + print('%d extra files in the %s version of the toolchain:' % + (len(extra_files), expected_hash)) + for f in extra_files[:10]: + print('\t%s' % f) + if len(extra_files) > 10: + print('\t...') + if matches: + return timestamps_data['sha1'] + + # Make long hangs when updating the toolchain less mysterious. + print('Calculating hash of toolchain in %s. Please wait...' % + full_root_path) + sys.stdout.flush() + digest = hashlib.sha1() + for path in file_list: + path_without_hash = str(path).replace('/', '\\') + if expected_hash: + path_without_hash = path_without_hash.replace( + os.path.join(root, expected_hash).replace('/', '\\'), root) + digest.update(bytes(path_without_hash.lower(), 'utf-8')) + with open(path, 'rb') as f: + digest.update(f.read()) + + # Save the timestamp file if the calculated hash is the expected one. + # The expected hash may be shorter, to reduce path lengths, in which case + # just compare that many characters. + if expected_hash and digest.hexdigest().startswith(expected_hash): + SaveTimestampsAndHash(root, digest.hexdigest()) + # Return the (potentially truncated) expected_hash. + return expected_hash + return digest.hexdigest() def CalculateToolchainHashes(root, remove_corrupt_toolchains): - """Calculate the hash of the different toolchains installed in the |root| + """Calculate the hash of the different toolchains installed in the |root| directory.""" - hashes = [] - dir_list = [ - d for d in os.listdir(root) if os.path.isdir(os.path.join(root, d))] - for d in dir_list: - toolchain_hash = CalculateHash(root, d) - if toolchain_hash != d: - print('The hash of a version of the toolchain has an unexpected value (' - '%s instead of %s)%s.' % (toolchain_hash, d, - ', removing it' if remove_corrupt_toolchains else '')) - if remove_corrupt_toolchains: - RemoveToolchain(root, d, True) - else: - hashes.append(toolchain_hash) - return hashes + hashes = [] + dir_list = [ + d for d in os.listdir(root) if os.path.isdir(os.path.join(root, d)) + ] + for d in dir_list: + toolchain_hash = CalculateHash(root, d) + if toolchain_hash != d: + print( + 'The hash of a version of the toolchain has an unexpected value (' + '%s instead of %s)%s.' % + (toolchain_hash, d, + ', removing it' if remove_corrupt_toolchains else '')) + if remove_corrupt_toolchains: + RemoveToolchain(root, d, True) + else: + hashes.append(toolchain_hash) + return hashes def SaveTimestampsAndHash(root, sha1): - """Saves timestamps and the final hash to be able to early-out more quickly + """Saves timestamps and the final hash to be able to early-out more quickly next time.""" - file_list = GetFileList(os.path.join(root, sha1)) - timestamps_data = { - 'files': [[f, os.path.getmtime(f)] for f in file_list], - 'sha1': sha1, - } - with open(MakeTimestampsFileName(root, sha1), 'wb') as f: - f.write(json.dumps(timestamps_data).encode('utf-8')) + file_list = GetFileList(os.path.join(root, sha1)) + timestamps_data = { + 'files': [[f, os.path.getmtime(f)] for f in file_list], + 'sha1': sha1, + } + with open(MakeTimestampsFileName(root, sha1), 'wb') as f: + f.write(json.dumps(timestamps_data).encode('utf-8')) def HaveSrcInternalAccess(): - """Checks whether access to src-internal is available.""" - with open(os.devnull, 'w') as nul: - # This is required to avoid modal dialog boxes after Git 2.14.1 and Git - # Credential Manager for Windows 1.12. See https://crbug.com/755694 and - # https://github.com/Microsoft/Git-Credential-Manager-for-Windows/issues/482. - child_env = dict(os.environ, GCM_INTERACTIVE='NEVER') - return subprocess.call( - ['git', '-c', 'core.askpass=true', 'remote', 'show', - 'https://chrome-internal.googlesource.com/chrome/src-internal/'], - shell=True, stdin=nul, stdout=nul, stderr=nul, env=child_env) == 0 + """Checks whether access to src-internal is available.""" + with open(os.devnull, 'w') as nul: + # This is required to avoid modal dialog boxes after Git 2.14.1 and Git + # Credential Manager for Windows 1.12. See https://crbug.com/755694 and + # https://github.com/Microsoft/Git-Credential-Manager-for-Windows/issues/482. + child_env = dict(os.environ, GCM_INTERACTIVE='NEVER') + return subprocess.call([ + 'git', '-c', 'core.askpass=true', 'remote', 'show', + 'https://chrome-internal.googlesource.com/chrome/src-internal/' + ], + shell=True, + stdin=nul, + stdout=nul, + stderr=nul, + env=child_env) == 0 def LooksLikeGoogler(): - """Checks for a USERDOMAIN environment variable of 'GOOGLE', which + """Checks for a USERDOMAIN environment variable of 'GOOGLE', which probably implies the current user is a Googler.""" - return os.environ.get('USERDOMAIN', '').upper() == 'GOOGLE' + return os.environ.get('USERDOMAIN', '').upper() == 'GOOGLE' def CanAccessToolchainBucket(): - """Checks whether the user has access to gs://chrome-wintoolchain/.""" - gsutil = download_from_google_storage.Gsutil( - download_from_google_storage.GSUTIL_DEFAULT_PATH, boto_path=None) - code, stdout, stderr = gsutil.check_call('ls', 'gs://chrome-wintoolchain/') - if code != 0: - # Make sure any error messages are made visible to the user. - print(stderr, file=sys.stderr, end='') - print(stdout, end='') - return code == 0 + """Checks whether the user has access to gs://chrome-wintoolchain/.""" + gsutil = download_from_google_storage.Gsutil( + download_from_google_storage.GSUTIL_DEFAULT_PATH, boto_path=None) + code, stdout, stderr = gsutil.check_call('ls', 'gs://chrome-wintoolchain/') + if code != 0: + # Make sure any error messages are made visible to the user. + print(stderr, file=sys.stderr, end='') + print(stdout, end='') + return code == 0 def ToolchainBaseURL(): - base_url = os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN_BASE_URL', '') - if base_url.startswith('file://'): - base_url = base_url[len('file://'):] - return base_url + base_url = os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN_BASE_URL', '') + if base_url.startswith('file://'): + base_url = base_url[len('file://'):] + return base_url def UsesToolchainFromFile(): - return os.path.isdir(ToolchainBaseURL()) + return os.path.isdir(ToolchainBaseURL()) def UsesToolchainFromHttp(): - url = ToolchainBaseURL() - return url.startswith('http://') or url.startswith('https://') + url = ToolchainBaseURL() + return url.startswith('http://') or url.startswith('https://') def RequestGsAuthentication(): - """Requests that the user authenticate to be able to access gs:// as a + """Requests that the user authenticate to be able to access gs:// as a Googler. This allows much faster downloads, and pulling (old) toolchains that match src/ revisions. """ - print('Access to gs://chrome-wintoolchain/ not configured.') - print('-----------------------------------------------------------------') - print() - print('You appear to be a Googler.') - print() - print('I\'m sorry for the hassle, but you need to do a one-time manual') - print('authentication. Please run:') - print() - print(' download_from_google_storage --config') - print() - print('and follow the instructions.') - print() - print('NOTE 1: Use your google.com credentials, not chromium.org.') - print('NOTE 2: Enter 0 when asked for a "project-id".') - print() - print('-----------------------------------------------------------------') - print() - sys.stdout.flush() - sys.exit(1) + print('Access to gs://chrome-wintoolchain/ not configured.') + print('-----------------------------------------------------------------') + print() + print('You appear to be a Googler.') + print() + print('I\'m sorry for the hassle, but you need to do a one-time manual') + print('authentication. Please run:') + print() + print(' download_from_google_storage --config') + print() + print('and follow the instructions.') + print() + print('NOTE 1: Use your google.com credentials, not chromium.org.') + print('NOTE 2: Enter 0 when asked for a "project-id".') + print() + print('-----------------------------------------------------------------') + print() + sys.stdout.flush() + sys.exit(1) def DelayBeforeRemoving(target_dir): - """A grace period before deleting the out of date toolchain directory.""" - if (os.path.isdir(target_dir) and - not bool(int(os.environ.get('CHROME_HEADLESS', '0')))): - for i in range(9, 0, -1): - sys.stdout.write( - '\rRemoving old toolchain in %ds... (Ctrl-C to cancel)' % i) - sys.stdout.flush() - time.sleep(1) - print() + """A grace period before deleting the out of date toolchain directory.""" + if (os.path.isdir(target_dir) + and not bool(int(os.environ.get('CHROME_HEADLESS', '0')))): + for i in range(9, 0, -1): + sys.stdout.write( + '\rRemoving old toolchain in %ds... (Ctrl-C to cancel)' % i) + sys.stdout.flush() + time.sleep(1) + print() def DownloadUsingHttp(filename): - """Downloads the given file from a url defined in + """Downloads the given file from a url defined in DEPOT_TOOLS_WIN_TOOLCHAIN_BASE_URL environment variable.""" - temp_dir = tempfile.mkdtemp() - assert os.path.basename(filename) == filename - target_path = os.path.join(temp_dir, filename) - base_url = ToolchainBaseURL() - src_url = urljoin(base_url, filename) - try: - with closing(urlopen(src_url)) as fsrc, \ - open(target_path, 'wb') as fdst: - shutil.copyfileobj(fsrc, fdst) - except URLError as e: - RmDir(temp_dir) - sys.exit('Failed to retrieve file: %s' % e) - return temp_dir, target_path + temp_dir = tempfile.mkdtemp() + assert os.path.basename(filename) == filename + target_path = os.path.join(temp_dir, filename) + base_url = ToolchainBaseURL() + src_url = urljoin(base_url, filename) + try: + with closing(urlopen(src_url)) as fsrc, \ + open(target_path, 'wb') as fdst: + shutil.copyfileobj(fsrc, fdst) + except URLError as e: + RmDir(temp_dir) + sys.exit('Failed to retrieve file: %s' % e) + return temp_dir, target_path def DownloadUsingGsutil(filename): - """Downloads the given file from Google Storage chrome-wintoolchain bucket.""" - temp_dir = tempfile.mkdtemp() - assert os.path.basename(filename) == filename - target_path = os.path.join(temp_dir, filename) - gsutil = download_from_google_storage.Gsutil( - download_from_google_storage.GSUTIL_DEFAULT_PATH, boto_path=None) - code = gsutil.call('cp', 'gs://chrome-wintoolchain/' + filename, target_path) - if code != 0: - sys.exit('gsutil failed') - return temp_dir, target_path + """Downloads the given file from Google Storage chrome-wintoolchain bucket.""" + temp_dir = tempfile.mkdtemp() + assert os.path.basename(filename) == filename + target_path = os.path.join(temp_dir, filename) + gsutil = download_from_google_storage.Gsutil( + download_from_google_storage.GSUTIL_DEFAULT_PATH, boto_path=None) + code = gsutil.call('cp', 'gs://chrome-wintoolchain/' + filename, + target_path) + if code != 0: + sys.exit('gsutil failed') + return temp_dir, target_path def RmDir(path): - """Deletes path and all the files it contains.""" - if sys.platform != 'win32': - shutil.rmtree(path, ignore_errors=True) - else: - # shutil.rmtree() doesn't delete read-only files on Windows. - subprocess.check_call('rmdir /s/q "%s"' % path, shell=True) + """Deletes path and all the files it contains.""" + if sys.platform != 'win32': + shutil.rmtree(path, ignore_errors=True) + else: + # shutil.rmtree() doesn't delete read-only files on Windows. + subprocess.check_call('rmdir /s/q "%s"' % path, shell=True) def DoTreeMirror(target_dir, tree_sha1): - """In order to save temporary space on bots that do not have enough space to + """In order to save temporary space on bots that do not have enough space to download ISOs, unpack them, and copy to the target location, the whole tree is uploaded as a zip to internal storage, and then mirrored here.""" - if UsesToolchainFromFile(): - temp_dir = None - local_zip = os.path.join(ToolchainBaseURL(), tree_sha1 + '.zip') - if not os.path.isfile(local_zip): - sys.exit('%s is not a valid file.' % local_zip) - elif UsesToolchainFromHttp(): - temp_dir, local_zip = DownloadUsingHttp(tree_sha1 + '.zip') - else: - temp_dir, local_zip = DownloadUsingGsutil(tree_sha1 + '.zip') - sys.stdout.write('Extracting %s...\n' % local_zip) - sys.stdout.flush() - with zipfile.ZipFile(local_zip, 'r', zipfile.ZIP_DEFLATED, True) as zf: - zf.extractall(target_dir) - if temp_dir: - RmDir(temp_dir) + if UsesToolchainFromFile(): + temp_dir = None + local_zip = os.path.join(ToolchainBaseURL(), tree_sha1 + '.zip') + if not os.path.isfile(local_zip): + sys.exit('%s is not a valid file.' % local_zip) + elif UsesToolchainFromHttp(): + temp_dir, local_zip = DownloadUsingHttp(tree_sha1 + '.zip') + else: + temp_dir, local_zip = DownloadUsingGsutil(tree_sha1 + '.zip') + sys.stdout.write('Extracting %s...\n' % local_zip) + sys.stdout.flush() + with zipfile.ZipFile(local_zip, 'r', zipfile.ZIP_DEFLATED, True) as zf: + zf.extractall(target_dir) + if temp_dir: + RmDir(temp_dir) def RemoveToolchain(root, sha1, delay_before_removing): - """Remove the |sha1| version of the toolchain from |root|.""" - toolchain_target_dir = os.path.join(root, sha1) - if delay_before_removing: - DelayBeforeRemoving(toolchain_target_dir) - if sys.platform == 'win32': - # These stay resident and will make the rmdir below fail. - kill_list = [ - 'mspdbsrv.exe', - 'vctip.exe', # Compiler and tools experience improvement data uploader. - ] - for process_name in kill_list: - with open(os.devnull, 'wb') as nul: - subprocess.call(['taskkill', '/f', '/im', process_name], - stdin=nul, stdout=nul, stderr=nul) - if os.path.isdir(toolchain_target_dir): - RmDir(toolchain_target_dir) + """Remove the |sha1| version of the toolchain from |root|.""" + toolchain_target_dir = os.path.join(root, sha1) + if delay_before_removing: + DelayBeforeRemoving(toolchain_target_dir) + if sys.platform == 'win32': + # These stay resident and will make the rmdir below fail. + kill_list = [ + 'mspdbsrv.exe', + 'vctip.exe', # Compiler and tools experience improvement data uploader. + ] + for process_name in kill_list: + with open(os.devnull, 'wb') as nul: + subprocess.call(['taskkill', '/f', '/im', process_name], + stdin=nul, + stdout=nul, + stderr=nul) + if os.path.isdir(toolchain_target_dir): + RmDir(toolchain_target_dir) - timestamp_file = MakeTimestampsFileName(root, sha1) - if os.path.exists(timestamp_file): - os.remove(timestamp_file) + timestamp_file = MakeTimestampsFileName(root, sha1) + if os.path.exists(timestamp_file): + os.remove(timestamp_file) def RemoveUnusedToolchains(root): - """Remove the versions of the toolchain that haven't been used recently.""" - valid_toolchains = [] - dirs_to_remove = [] + """Remove the versions of the toolchain that haven't been used recently.""" + valid_toolchains = [] + dirs_to_remove = [] - for d in os.listdir(root): - full_path = os.path.join(root, d) - if os.path.isdir(full_path): - if not os.path.exists(MakeTimestampsFileName(root, d)): - dirs_to_remove.append(d) - else: - vc_dir = os.path.join(full_path, 'VC') - valid_toolchains.append((os.path.getmtime(vc_dir), d)) - elif os.path.isfile(full_path): - os.remove(full_path) + for d in os.listdir(root): + full_path = os.path.join(root, d) + if os.path.isdir(full_path): + if not os.path.exists(MakeTimestampsFileName(root, d)): + dirs_to_remove.append(d) + else: + vc_dir = os.path.join(full_path, 'VC') + valid_toolchains.append((os.path.getmtime(vc_dir), d)) + elif os.path.isfile(full_path): + os.remove(full_path) - for d in dirs_to_remove: - print('Removing %s as it doesn\'t correspond to any known toolchain.' % - os.path.join(root, d)) - # Use the RemoveToolchain function to remove these directories as they might - # contain an older version of the toolchain. - RemoveToolchain(root, d, False) + for d in dirs_to_remove: + print('Removing %s as it doesn\'t correspond to any known toolchain.' % + os.path.join(root, d)) + # Use the RemoveToolchain function to remove these directories as they + # might contain an older version of the toolchain. + RemoveToolchain(root, d, False) - # Remove the versions of the toolchains that haven't been used in the past 30 - # days. - toolchain_expiration_time = 60 * 60 * 24 * 30 - for toolchain in valid_toolchains: - toolchain_age_in_sec = time.time() - toolchain[0] - if toolchain_age_in_sec > toolchain_expiration_time: - print('Removing version %s of the Win toolchain as it hasn\'t been used' - ' in the past %d days.' % (toolchain[1], - toolchain_age_in_sec / 60 / 60 / 24)) - RemoveToolchain(root, toolchain[1], True) + # Remove the versions of the toolchains that haven't been used in the past + # 30 days. + toolchain_expiration_time = 60 * 60 * 24 * 30 + for toolchain in valid_toolchains: + toolchain_age_in_sec = time.time() - toolchain[0] + if toolchain_age_in_sec > toolchain_expiration_time: + print( + 'Removing version %s of the Win toolchain as it hasn\'t been used' + ' in the past %d days.' % + (toolchain[1], toolchain_age_in_sec / 60 / 60 / 24)) + RemoveToolchain(root, toolchain[1], True) def EnableCrashDumpCollection(): - """Tell Windows Error Reporting to record crash dumps so that we can diagnose + """Tell Windows Error Reporting to record crash dumps so that we can diagnose linker crashes and other toolchain failures. Documented at: https://msdn.microsoft.com/en-us/library/windows/desktop/bb787181.aspx """ - if sys.platform == 'win32' and os.environ.get('CHROME_HEADLESS') == '1': - key_name = r'SOFTWARE\Microsoft\Windows\Windows Error Reporting' - try: - key = winreg.CreateKeyEx(winreg.HKEY_LOCAL_MACHINE, key_name, 0, - winreg.KEY_WOW64_64KEY | winreg.KEY_ALL_ACCESS) - # Merely creating LocalDumps is sufficient to enable the defaults. - winreg.CreateKey(key, "LocalDumps") - # Disable the WER UI, as documented here: - # https://msdn.microsoft.com/en-us/library/windows/desktop/bb513638.aspx - winreg.SetValueEx(key, "DontShowUI", 0, winreg.REG_DWORD, 1) - # Trap OSError instead of WindowsError so pylint will succeed on Linux. - # Catching errors is important because some build machines are not elevated - # and writing to HKLM requires elevation. - except OSError: - pass + if sys.platform == 'win32' and os.environ.get('CHROME_HEADLESS') == '1': + key_name = r'SOFTWARE\Microsoft\Windows\Windows Error Reporting' + try: + key = winreg.CreateKeyEx( + winreg.HKEY_LOCAL_MACHINE, key_name, 0, + winreg.KEY_WOW64_64KEY | winreg.KEY_ALL_ACCESS) + # Merely creating LocalDumps is sufficient to enable the defaults. + winreg.CreateKey(key, "LocalDumps") + # Disable the WER UI, as documented here: + # https://msdn.microsoft.com/en-us/library/windows/desktop/bb513638.aspx + winreg.SetValueEx(key, "DontShowUI", 0, winreg.REG_DWORD, 1) + # Trap OSError instead of WindowsError so pylint will succeed on Linux. + # Catching errors is important because some build machines are not + # elevated and writing to HKLM requires elevation. + except OSError: + pass def main(): - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument('--output-json', metavar='FILE', - help='write information about toolchain to FILE') - parser.add_argument('--force', action='store_true', - help='force script to run on non-Windows hosts') - parser.add_argument('--no-download', action='store_true', - help='configure if present but don\'t download') - parser.add_argument('--toolchain-dir', - default=os.getenv(ENV_TOOLCHAIN_ROOT, BASEDIR), - help='directory to install toolchain into') - parser.add_argument('desired_hash', metavar='desired-hash', - help='toolchain hash to download') - args = parser.parse_args() + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument('--output-json', + metavar='FILE', + help='write information about toolchain to FILE') + parser.add_argument('--force', + action='store_true', + help='force script to run on non-Windows hosts') + parser.add_argument('--no-download', + action='store_true', + help='configure if present but don\'t download') + parser.add_argument('--toolchain-dir', + default=os.getenv(ENV_TOOLCHAIN_ROOT, BASEDIR), + help='directory to install toolchain into') + parser.add_argument('desired_hash', + metavar='desired-hash', + help='toolchain hash to download') + args = parser.parse_args() - if not (sys.platform.startswith(('cygwin', 'win32')) or args.force): - return 0 + if not (sys.platform.startswith(('cygwin', 'win32')) or args.force): + return 0 - if sys.platform == 'cygwin': - # This script requires Windows Python, so invoke with depot_tools' Python. - def winpath(path): - return subprocess.check_output(['cygpath', '-w', path]).strip() - python = os.path.join(DEPOT_TOOLS_PATH, 'python3.bat') - cmd = [python, winpath(__file__)] - if args.output_json: - cmd.extend(['--output-json', winpath(args.output_json)]) - cmd.append(args.desired_hash) - sys.exit(subprocess.call(cmd)) - assert sys.platform != 'cygwin' + if sys.platform == 'cygwin': + # This script requires Windows Python, so invoke with depot_tools' + # Python. + def winpath(path): + return subprocess.check_output(['cygpath', '-w', path]).strip() - # Create our toolchain destination and "chdir" to it. - toolchain_dir = os.path.abspath(args.toolchain_dir) - if not os.path.isdir(toolchain_dir): - os.makedirs(toolchain_dir) - os.chdir(toolchain_dir) + python = os.path.join(DEPOT_TOOLS_PATH, 'python3.bat') + cmd = [python, winpath(__file__)] + if args.output_json: + cmd.extend(['--output-json', winpath(args.output_json)]) + cmd.append(args.desired_hash) + sys.exit(subprocess.call(cmd)) + assert sys.platform != 'cygwin' - # Move to depot_tools\win_toolchain where we'll store our files, and where - # the downloader script is. - target_dir = 'vs_files' - if not os.path.isdir(target_dir): - os.mkdir(target_dir) - toolchain_target_dir = os.path.join(target_dir, args.desired_hash) + # Create our toolchain destination and "chdir" to it. + toolchain_dir = os.path.abspath(args.toolchain_dir) + if not os.path.isdir(toolchain_dir): + os.makedirs(toolchain_dir) + os.chdir(toolchain_dir) - abs_toolchain_target_dir = os.path.abspath(toolchain_target_dir) + # Move to depot_tools\win_toolchain where we'll store our files, and where + # the downloader script is. + target_dir = 'vs_files' + if not os.path.isdir(target_dir): + os.mkdir(target_dir) + toolchain_target_dir = os.path.join(target_dir, args.desired_hash) - got_new_toolchain = False + abs_toolchain_target_dir = os.path.abspath(toolchain_target_dir) - # If the current hash doesn't match what we want in the file, nuke and pave. - # Typically this script is only run when the .sha1 one file is updated, but - # directly calling "gclient runhooks" will also run it, so we cache - # based on timestamps to make that case fast. - current_hashes = CalculateToolchainHashes(target_dir, True) - if args.desired_hash not in current_hashes: - if args.no_download: - raise SystemExit('Toolchain is out of date. Run "gclient runhooks" to ' - 'update the toolchain, or set ' - 'DEPOT_TOOLS_WIN_TOOLCHAIN=0 to use the locally ' - 'installed toolchain.') - should_use_file = False - should_use_http = False - should_use_gs = False - if UsesToolchainFromFile(): - should_use_file = True - elif UsesToolchainFromHttp(): - should_use_http = True - elif (HaveSrcInternalAccess() or - LooksLikeGoogler() or - CanAccessToolchainBucket()): - should_use_gs = True - if not CanAccessToolchainBucket(): - RequestGsAuthentication() - if not should_use_file and not should_use_gs and not should_use_http: - if sys.platform not in ('win32', 'cygwin'): - doc = 'https://chromium.googlesource.com/chromium/src/+/HEAD/docs/' \ - 'win_cross.md' - print('\n\n\nPlease follow the instructions at %s\n\n' % doc) - else: - doc = 'https://chromium.googlesource.com/chromium/src/+/HEAD/docs/' \ - 'windows_build_instructions.md' - print('\n\n\nNo downloadable toolchain found. In order to use your ' - 'locally installed version of Visual Studio to build Chrome ' - 'please set DEPOT_TOOLS_WIN_TOOLCHAIN=0.\n' - 'For details search for DEPOT_TOOLS_WIN_TOOLCHAIN in the ' - 'instructions at %s\n\n' % doc) - return 1 - print('Windows toolchain out of date or doesn\'t exist, updating (Pro)...') - print(' current_hashes: %s' % ', '.join(current_hashes)) - print(' desired_hash: %s' % args.desired_hash) - sys.stdout.flush() + got_new_toolchain = False - DoTreeMirror(toolchain_target_dir, args.desired_hash) - - got_new_toolchain = True - - # The Windows SDK is either in `win_sdk` or in `Windows Kits\10`. This - # script must work with both layouts, so check which one it is. - win_sdk_in_windows_kits = os.path.isdir( - os.path.join(abs_toolchain_target_dir, 'Windows Kits', '10')) - if win_sdk_in_windows_kits: - win_sdk = os.path.join(abs_toolchain_target_dir, 'Windows Kits', '10') - else: - win_sdk = os.path.join(abs_toolchain_target_dir, 'win_sdk') - - version_file = os.path.join(toolchain_target_dir, 'VS_VERSION') - vc_dir = os.path.join(toolchain_target_dir, 'VC') - with open(version_file, 'rb') as f: - vs_version = f.read().decode('utf-8').strip() - # Touch the VC directory so we can use its timestamp to know when this - # version of the toolchain has been used for the last time. - os.utime(vc_dir, None) - - data = { - 'path': abs_toolchain_target_dir, - 'version': vs_version, - 'win_sdk': win_sdk, - 'wdk': os.path.join(abs_toolchain_target_dir, 'wdk'), - 'runtime_dirs': [ - os.path.join(abs_toolchain_target_dir, 'sys64'), - os.path.join(abs_toolchain_target_dir, 'sys32'), - os.path.join(abs_toolchain_target_dir, 'sysarm64'), - ], - } - data_json = json.dumps(data, indent=2) - data_path = os.path.join(target_dir, '..', 'data.json') - if not os.path.exists(data_path) or open(data_path).read() != data_json: - with open(data_path, 'w') as f: - f.write(data_json) - - if got_new_toolchain: - current_hashes = CalculateToolchainHashes(target_dir, False) + # If the current hash doesn't match what we want in the file, nuke and pave. + # Typically this script is only run when the .sha1 one file is updated, but + # directly calling "gclient runhooks" will also run it, so we cache + # based on timestamps to make that case fast. + current_hashes = CalculateToolchainHashes(target_dir, True) if args.desired_hash not in current_hashes: - print( - 'Got wrong hash after pulling a new toolchain. ' - 'Wanted \'%s\', got one of \'%s\'.' % ( - args.desired_hash, ', '.join(current_hashes)), file=sys.stderr) - return 1 - SaveTimestampsAndHash(target_dir, args.desired_hash) + if args.no_download: + raise SystemExit( + 'Toolchain is out of date. Run "gclient runhooks" to ' + 'update the toolchain, or set ' + 'DEPOT_TOOLS_WIN_TOOLCHAIN=0 to use the locally ' + 'installed toolchain.') + should_use_file = False + should_use_http = False + should_use_gs = False + if UsesToolchainFromFile(): + should_use_file = True + elif UsesToolchainFromHttp(): + should_use_http = True + elif (HaveSrcInternalAccess() or LooksLikeGoogler() + or CanAccessToolchainBucket()): + should_use_gs = True + if not CanAccessToolchainBucket(): + RequestGsAuthentication() + if not should_use_file and not should_use_gs and not should_use_http: + if sys.platform not in ('win32', 'cygwin'): + doc = 'https://chromium.googlesource.com/chromium/src/+/HEAD/docs/' \ + 'win_cross.md' + print('\n\n\nPlease follow the instructions at %s\n\n' % doc) + else: + doc = 'https://chromium.googlesource.com/chromium/src/+/HEAD/docs/' \ + 'windows_build_instructions.md' + print( + '\n\n\nNo downloadable toolchain found. In order to use your ' + 'locally installed version of Visual Studio to build Chrome ' + 'please set DEPOT_TOOLS_WIN_TOOLCHAIN=0.\n' + 'For details search for DEPOT_TOOLS_WIN_TOOLCHAIN in the ' + 'instructions at %s\n\n' % doc) + return 1 + print( + 'Windows toolchain out of date or doesn\'t exist, updating (Pro)...' + ) + print(' current_hashes: %s' % ', '.join(current_hashes)) + print(' desired_hash: %s' % args.desired_hash) + sys.stdout.flush() - if args.output_json: - if (not os.path.exists(args.output_json) or - not filecmp.cmp(data_path, args.output_json)): - shutil.copyfile(data_path, args.output_json) + DoTreeMirror(toolchain_target_dir, args.desired_hash) - EnableCrashDumpCollection() + got_new_toolchain = True - RemoveUnusedToolchains(target_dir) + # The Windows SDK is either in `win_sdk` or in `Windows Kits\10`. This + # script must work with both layouts, so check which one it is. + win_sdk_in_windows_kits = os.path.isdir( + os.path.join(abs_toolchain_target_dir, 'Windows Kits', '10')) + if win_sdk_in_windows_kits: + win_sdk = os.path.join(abs_toolchain_target_dir, 'Windows Kits', '10') + else: + win_sdk = os.path.join(abs_toolchain_target_dir, 'win_sdk') - return 0 + version_file = os.path.join(toolchain_target_dir, 'VS_VERSION') + vc_dir = os.path.join(toolchain_target_dir, 'VC') + with open(version_file, 'rb') as f: + vs_version = f.read().decode('utf-8').strip() + # Touch the VC directory so we can use its timestamp to know when this + # version of the toolchain has been used for the last time. + os.utime(vc_dir, None) + + data = { + 'path': + abs_toolchain_target_dir, + 'version': + vs_version, + 'win_sdk': + win_sdk, + 'wdk': + os.path.join(abs_toolchain_target_dir, 'wdk'), + 'runtime_dirs': [ + os.path.join(abs_toolchain_target_dir, 'sys64'), + os.path.join(abs_toolchain_target_dir, 'sys32'), + os.path.join(abs_toolchain_target_dir, 'sysarm64'), + ], + } + data_json = json.dumps(data, indent=2) + data_path = os.path.join(target_dir, '..', 'data.json') + if not os.path.exists(data_path) or open(data_path).read() != data_json: + with open(data_path, 'w') as f: + f.write(data_json) + + if got_new_toolchain: + current_hashes = CalculateToolchainHashes(target_dir, False) + if args.desired_hash not in current_hashes: + print('Got wrong hash after pulling a new toolchain. ' + 'Wanted \'%s\', got one of \'%s\'.' % + (args.desired_hash, ', '.join(current_hashes)), + file=sys.stderr) + return 1 + SaveTimestampsAndHash(target_dir, args.desired_hash) + + if args.output_json: + if (not os.path.exists(args.output_json) + or not filecmp.cmp(data_path, args.output_json)): + shutil.copyfile(data_path, args.output_json) + + EnableCrashDumpCollection() + + RemoveUnusedToolchains(target_dir) + + return 0 if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/win_toolchain/package_from_installed.py b/win_toolchain/package_from_installed.py index 38430d448d..2567b926fa 100644 --- a/win_toolchain/package_from_installed.py +++ b/win_toolchain/package_from_installed.py @@ -2,7 +2,6 @@ # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """ From a system-installed copy of the toolchain, packages all the required bits into a .zip file. @@ -47,7 +46,6 @@ import zipfile import get_toolchain_if_necessary - _vs_version = None _win_version = None _vc_tools = None @@ -56,527 +54,579 @@ _allow_multiple_vs_installs = False def GetVSPath(): - # Use vswhere to find the VS installation. This will find prerelease - # versions because -prerelease is specified. This assumes that only one - # version is installed. - command = (r'C:\Program Files (x86)\Microsoft Visual Studio\Installer' - r'\vswhere.exe -prerelease') - vs_version_marker = 'catalog_productLineVersion: ' - vs_path_marker = 'installationPath: ' - output = subprocess.check_output(command, universal_newlines=True) - vs_path = None - vs_installs_count = 0 - matching_vs_path = "" - for line in output.splitlines(): - if line.startswith(vs_path_marker): - # The path information comes first - vs_path = line[len(vs_path_marker):] - vs_installs_count += 1 - if line.startswith(vs_version_marker): - # The version for that path comes later - if line[len(vs_version_marker):] == _vs_version: - matching_vs_path = vs_path + # Use vswhere to find the VS installation. This will find prerelease + # versions because -prerelease is specified. This assumes that only one + # version is installed. + command = (r'C:\Program Files (x86)\Microsoft Visual Studio\Installer' + r'\vswhere.exe -prerelease') + vs_version_marker = 'catalog_productLineVersion: ' + vs_path_marker = 'installationPath: ' + output = subprocess.check_output(command, universal_newlines=True) + vs_path = None + vs_installs_count = 0 + matching_vs_path = "" + for line in output.splitlines(): + if line.startswith(vs_path_marker): + # The path information comes first + vs_path = line[len(vs_path_marker):] + vs_installs_count += 1 + if line.startswith(vs_version_marker): + # The version for that path comes later + if line[len(vs_version_marker):] == _vs_version: + matching_vs_path = vs_path - if vs_installs_count == 0: - raise Exception('VS %s path not found in vswhere output' % (_vs_version)) - if vs_installs_count > 1: - if not _allow_multiple_vs_installs: - raise Exception('Multiple VS installs detected. This is unsupported. ' - 'It is recommended that packaging be done on a clean VM ' - 'with just one version installed. To proceed anyway add ' - 'the --allow_multiple_vs_installs flag to this script') - else: - print('Multiple VS installs were detected. This is unsupported. ' - 'Proceeding anyway') - return matching_vs_path + if vs_installs_count == 0: + raise Exception('VS %s path not found in vswhere output' % + (_vs_version)) + if vs_installs_count > 1: + if not _allow_multiple_vs_installs: + raise Exception( + 'Multiple VS installs detected. This is unsupported. ' + 'It is recommended that packaging be done on a clean VM ' + 'with just one version installed. To proceed anyway add ' + 'the --allow_multiple_vs_installs flag to this script') + else: + print('Multiple VS installs were detected. This is unsupported. ' + 'Proceeding anyway') + return matching_vs_path def ExpandWildcards(root, sub_dir): - # normpath is needed to change '/' to '\\' characters. - path = os.path.normpath(os.path.join(root, sub_dir)) - matches = glob.glob(path) - if len(matches) != 1: - raise Exception('%s had %d matches - should be one' % (path, len(matches))) - return matches[0] + # normpath is needed to change '/' to '\\' characters. + path = os.path.normpath(os.path.join(root, sub_dir)) + matches = glob.glob(path) + if len(matches) != 1: + raise Exception('%s had %d matches - should be one' % + (path, len(matches))) + return matches[0] def BuildRepackageFileList(src_dir): - # Strip off a trailing separator if present - if src_dir.endswith(os.path.sep): - src_dir = src_dir[:-len(os.path.sep)] + # Strip off a trailing separator if present + if src_dir.endswith(os.path.sep): + src_dir = src_dir[:-len(os.path.sep)] - # Ensure .\Windows Kits\10\Debuggers exists and fail to repackage if it - # doesn't. - debuggers_path = os.path.join(src_dir, 'Windows Kits', '10', 'Debuggers') - if not os.path.exists(debuggers_path): - raise Exception('Repacking failed. Missing %s.' % (debuggers_path)) + # Ensure .\Windows Kits\10\Debuggers exists and fail to repackage if it + # doesn't. + debuggers_path = os.path.join(src_dir, 'Windows Kits', '10', 'Debuggers') + if not os.path.exists(debuggers_path): + raise Exception('Repacking failed. Missing %s.' % (debuggers_path)) - result = [] - for root, _, files in os.walk(src_dir): - for f in files: - final_from = os.path.normpath(os.path.join(root, f)) - dest = final_from[len(src_dir) + 1:] - result.append((final_from, dest)) - return result + result = [] + for root, _, files in os.walk(src_dir): + for f in files: + final_from = os.path.normpath(os.path.join(root, f)) + dest = final_from[len(src_dir) + 1:] + result.append((final_from, dest)) + return result def BuildFileList(override_dir, include_arm, vs_path): - result = [] + result = [] - # Subset of VS corresponding roughly to VC. - paths = [ - 'DIA SDK/bin', - 'DIA SDK/idl', - 'DIA SDK/include', - 'DIA SDK/lib', - _vc_tools + '/atlmfc', - _vc_tools + '/crt', - 'VC/redist', - ] + # Subset of VS corresponding roughly to VC. + paths = [ + 'DIA SDK/bin', + 'DIA SDK/idl', + 'DIA SDK/include', + 'DIA SDK/lib', + _vc_tools + '/atlmfc', + _vc_tools + '/crt', + 'VC/redist', + ] + + if override_dir: + paths += [ + (os.path.join(override_dir, 'bin'), _vc_tools + '/bin'), + (os.path.join(override_dir, 'include'), _vc_tools + '/include'), + (os.path.join(override_dir, 'lib'), _vc_tools + '/lib'), + ] + else: + paths += [ + _vc_tools + '/bin', + _vc_tools + '/include', + _vc_tools + '/lib', + ] - if override_dir: paths += [ - (os.path.join(override_dir, 'bin'), _vc_tools + '/bin'), - (os.path.join(override_dir, 'include'), _vc_tools + '/include'), - (os.path.join(override_dir, 'lib'), _vc_tools + '/lib'), + ('VC/redist/MSVC/14.*.*/x86/Microsoft.VC*.CRT', 'sys32'), + ('VC/redist/MSVC/14.*.*/x86/Microsoft.VC*.CRT', + 'Windows Kits/10//bin/x86'), + ('VC/redist/MSVC/14.*.*/debug_nonredist/x86/Microsoft.VC*.DebugCRT', + 'sys32'), + ('VC/redist/MSVC/14.*.*/x64/Microsoft.VC*.CRT', 'sys64'), + ('VC/redist/MSVC/14.*.*/x64/Microsoft.VC*.CRT', 'VC/bin/amd64_x86'), + ('VC/redist/MSVC/14.*.*/x64/Microsoft.VC*.CRT', 'VC/bin/amd64'), + ('VC/redist/MSVC/14.*.*/x64/Microsoft.VC*.CRT', + 'Windows Kits/10/bin/x64'), + ('VC/redist/MSVC/14.*.*/debug_nonredist/x64/Microsoft.VC*.DebugCRT', + 'sys64'), ] - else: - paths += [ - _vc_tools + '/bin', - _vc_tools + '/include', - _vc_tools + '/lib', + if include_arm: + paths += [ + ('VC/redist/MSVC/14.*.*/arm64/Microsoft.VC*.CRT', 'sysarm64'), + ('VC/redist/MSVC/14.*.*/arm64/Microsoft.VC*.CRT', + 'VC/bin/amd64_arm64'), + ('VC/redist/MSVC/14.*.*/arm64/Microsoft.VC*.CRT', 'VC/bin/arm64'), + ('VC/redist/MSVC/14.*.*/arm64/Microsoft.VC*.CRT', + 'Windows Kits/10/bin/arm64'), + ('VC/redist/MSVC/14.*.*/debug_nonredist/arm64/Microsoft.VC*.DebugCRT', + 'sysarm64'), + ] + + for path in paths: + src = path[0] if isinstance(path, tuple) else path + # Note that vs_path is ignored if src is an absolute path. + combined = ExpandWildcards(vs_path, src) + if not os.path.exists(combined): + raise Exception('%s missing.' % combined) + if not os.path.isdir(combined): + raise Exception('%s not a directory.' % combined) + for root, _, files in os.walk(combined): + for f in files: + # vctip.exe doesn't shutdown, leaving locks on directories. It's + # optional so let's avoid this problem by not packaging it. + # https://crbug.com/735226 + if f.lower() == 'vctip.exe': + continue + final_from = os.path.normpath(os.path.join(root, f)) + if isinstance(path, tuple): + assert final_from.startswith(combined) + dest = final_from[len(combined) + 1:] + result.append((final_from, + os.path.normpath(os.path.join(path[1], + dest)))) + else: + assert final_from.startswith(vs_path) + dest = final_from[len(vs_path) + 1:] + result.append((final_from, dest)) + + command = ( + r'reg query "HKLM\SOFTWARE\Microsoft\Windows Kits\Installed Roots"' + r' /v KitsRoot10') + marker = " KitsRoot10 REG_SZ " + sdk_path = None + output = subprocess.check_output(command, universal_newlines=True) + for line in output.splitlines(): + if line.startswith(marker): + sdk_path = line[len(marker):] + + # Strip off a trailing slash if present + if sdk_path.endswith(os.path.sep): + sdk_path = sdk_path[:-len(os.path.sep)] + + debuggers_path = os.path.join(sdk_path, 'Debuggers') + if not os.path.exists(debuggers_path): + raise Exception('Packaging failed. Missing %s.' % (debuggers_path)) + + for root, _, files in os.walk(sdk_path): + for f in files: + combined = os.path.normpath(os.path.join(root, f)) + # Some of the files in this directory are exceedingly long (and + # exceed _MAX_PATH for any moderately long root), so exclude them. + # We don't need them anyway. Exclude/filter/skip others just to save + # space. + tail = combined[len(sdk_path) + 1:] + skip_dir = False + for dir in [ + 'References\\', 'Windows Performance Toolkit\\', + 'Testing\\', 'App Certification Kit\\', 'Extension SDKs\\', + 'Assessment and Deployment Kit\\' + ]: + if tail.startswith(dir): + skip_dir = True + if skip_dir: + continue + # There may be many Include\Lib\Source\bin directories for many + # different versions of Windows and packaging them all wastes ~450 + # MB (uncompressed) per version and wastes time. Only copy the + # specified version. Note that the SDK version number started being + # part of the bin path with 10.0.15063.0. + if (tail.startswith('Include\\') or tail.startswith('Lib\\') + or tail.startswith('Source\\') or tail.startswith('bin\\')): + if tail.count(_win_version) == 0: + continue + to = os.path.join('Windows Kits', '10', tail) + result.append((combined, to)) + + # Copy the x86 ucrt DLLs to all directories with x86 binaries that are + # added to the path by SetEnv.cmd, and to sys32. Starting with the 17763 + # SDK the ucrt files are in _win_version\ucrt instead of just ucrt. + ucrt_dir = os.path.join(sdk_path, 'redist', _win_version, r'ucrt\dlls\x86') + if not os.path.exists(ucrt_dir): + ucrt_dir = os.path.join(sdk_path, r'redist\ucrt\dlls\x86') + ucrt_paths = glob.glob(ucrt_dir + r'\*') + assert (len(ucrt_paths) > 0) + for ucrt_path in ucrt_paths: + ucrt_file = os.path.split(ucrt_path)[1] + for dest_dir in [r'Windows Kits\10\bin\x86', 'sys32']: + result.append((ucrt_path, os.path.join(dest_dir, ucrt_file))) + + # Copy the x64 ucrt DLLs to all directories with x64 binaries that are + # added to the path by SetEnv.cmd, and to sys64. + ucrt_dir = os.path.join(sdk_path, 'redist', _win_version, r'ucrt\dlls\x64') + if not os.path.exists(ucrt_dir): + ucrt_dir = os.path.join(sdk_path, r'redist\ucrt\dlls\x64') + ucrt_paths = glob.glob(ucrt_dir + r'\*') + assert (len(ucrt_paths) > 0) + for ucrt_path in ucrt_paths: + ucrt_file = os.path.split(ucrt_path)[1] + for dest_dir in [ + r'VC\bin\amd64_x86', r'VC\bin\amd64', + r'Windows Kits\10\bin\x64', 'sys64' + ]: + result.append((ucrt_path, os.path.join(dest_dir, ucrt_file))) + + system_crt_files = [ + # Needed to let debug binaries run. + 'ucrtbased.dll', ] - - paths += [ - ('VC/redist/MSVC/14.*.*/x86/Microsoft.VC*.CRT', 'sys32'), - ('VC/redist/MSVC/14.*.*/x86/Microsoft.VC*.CRT', - 'Windows Kits/10//bin/x86'), - ('VC/redist/MSVC/14.*.*/debug_nonredist/x86/Microsoft.VC*.DebugCRT', - 'sys32'), - ('VC/redist/MSVC/14.*.*/x64/Microsoft.VC*.CRT', 'sys64'), - ('VC/redist/MSVC/14.*.*/x64/Microsoft.VC*.CRT', 'VC/bin/amd64_x86'), - ('VC/redist/MSVC/14.*.*/x64/Microsoft.VC*.CRT', 'VC/bin/amd64'), - ('VC/redist/MSVC/14.*.*/x64/Microsoft.VC*.CRT', - 'Windows Kits/10/bin/x64'), - ('VC/redist/MSVC/14.*.*/debug_nonredist/x64/Microsoft.VC*.DebugCRT', - 'sys64'), - ] - if include_arm: - paths += [ - ('VC/redist/MSVC/14.*.*/arm64/Microsoft.VC*.CRT', 'sysarm64'), - ('VC/redist/MSVC/14.*.*/arm64/Microsoft.VC*.CRT', 'VC/bin/amd64_arm64'), - ('VC/redist/MSVC/14.*.*/arm64/Microsoft.VC*.CRT', 'VC/bin/arm64'), - ('VC/redist/MSVC/14.*.*/arm64/Microsoft.VC*.CRT', - 'Windows Kits/10/bin/arm64'), - ('VC/redist/MSVC/14.*.*/debug_nonredist/arm64/Microsoft.VC*.DebugCRT', - 'sysarm64'), + cpu_pairs = [ + ('x86', 'sys32'), + ('x64', 'sys64'), ] + if include_arm: + cpu_pairs += [ + ('arm64', 'sysarm64'), + ] + for system_crt_file in system_crt_files: + for cpu_pair in cpu_pairs: + target_cpu, dest_dir = cpu_pair + src_path = os.path.join(sdk_path, 'bin', _win_version, target_cpu, + 'ucrt') + result.append((os.path.join(src_path, system_crt_file), + os.path.join(dest_dir, system_crt_file))) - for path in paths: - src = path[0] if isinstance(path, tuple) else path - # Note that vs_path is ignored if src is an absolute path. - combined = ExpandWildcards(vs_path, src) - if not os.path.exists(combined): - raise Exception('%s missing.' % combined) - if not os.path.isdir(combined): - raise Exception('%s not a directory.' % combined) - for root, _, files in os.walk(combined): - for f in files: - # vctip.exe doesn't shutdown, leaving locks on directories. It's - # optional so let's avoid this problem by not packaging it. - # https://crbug.com/735226 - if f.lower() =='vctip.exe': - continue - final_from = os.path.normpath(os.path.join(root, f)) - if isinstance(path, tuple): - assert final_from.startswith(combined) - dest = final_from[len(combined) + 1:] - result.append( - (final_from, os.path.normpath(os.path.join(path[1], dest)))) - else: - assert final_from.startswith(vs_path) - dest = final_from[len(vs_path) + 1:] - result.append((final_from, dest)) + # Generically drop all arm stuff that we don't need, and + # drop .msi files because we don't need installers and drop + # samples since those are not used by any tools. + def is_skippable(f): + return ('arm\\' in f.lower() + or (not include_arm and 'arm64\\' in f.lower()) + or 'samples\\' in f.lower() or f.lower().endswith( + ('.msi', '.msm'))) - command = (r'reg query "HKLM\SOFTWARE\Microsoft\Windows Kits\Installed Roots"' - r' /v KitsRoot10') - marker = " KitsRoot10 REG_SZ " - sdk_path = None - output = subprocess.check_output(command, universal_newlines=True) - for line in output.splitlines(): - if line.startswith(marker): - sdk_path = line[len(marker):] + return [(f, t) for f, t in result if not is_skippable(f)] - # Strip off a trailing slash if present - if sdk_path.endswith(os.path.sep): - sdk_path = sdk_path[:-len(os.path.sep)] - - debuggers_path = os.path.join(sdk_path, 'Debuggers') - if not os.path.exists(debuggers_path): - raise Exception('Packaging failed. Missing %s.' % (debuggers_path)) - - for root, _, files in os.walk(sdk_path): - for f in files: - combined = os.path.normpath(os.path.join(root, f)) - # Some of the files in this directory are exceedingly long (and exceed - # _MAX_PATH for any moderately long root), so exclude them. We don't need - # them anyway. Exclude/filter/skip others just to save space. - tail = combined[len(sdk_path) + 1:] - skip_dir = False - for dir in ['References\\', 'Windows Performance Toolkit\\', 'Testing\\', - 'App Certification Kit\\', 'Extension SDKs\\', - 'Assessment and Deployment Kit\\']: - if tail.startswith(dir): - skip_dir = True - if skip_dir: - continue - # There may be many Include\Lib\Source\bin directories for many different - # versions of Windows and packaging them all wastes ~450 MB - # (uncompressed) per version and wastes time. Only copy the specified - # version. Note that the SDK version number started being part of the bin - # path with 10.0.15063.0. - if (tail.startswith('Include\\') or tail.startswith('Lib\\') or - tail.startswith('Source\\') or tail.startswith('bin\\')): - if tail.count(_win_version) == 0: - continue - to = os.path.join('Windows Kits', '10', tail) - result.append((combined, to)) - - # Copy the x86 ucrt DLLs to all directories with x86 binaries that are - # added to the path by SetEnv.cmd, and to sys32. Starting with the 17763 - # SDK the ucrt files are in _win_version\ucrt instead of just ucrt. - ucrt_dir = os.path.join(sdk_path, 'redist', _win_version, r'ucrt\dlls\x86') - if not os.path.exists(ucrt_dir): - ucrt_dir = os.path.join(sdk_path, r'redist\ucrt\dlls\x86') - ucrt_paths = glob.glob(ucrt_dir + r'\*') - assert(len(ucrt_paths) > 0) - for ucrt_path in ucrt_paths: - ucrt_file = os.path.split(ucrt_path)[1] - for dest_dir in [ r'Windows Kits\10\bin\x86', 'sys32' ]: - result.append((ucrt_path, os.path.join(dest_dir, ucrt_file))) - - # Copy the x64 ucrt DLLs to all directories with x64 binaries that are - # added to the path by SetEnv.cmd, and to sys64. - ucrt_dir = os.path.join(sdk_path, 'redist', _win_version, r'ucrt\dlls\x64') - if not os.path.exists(ucrt_dir): - ucrt_dir = os.path.join(sdk_path, r'redist\ucrt\dlls\x64') - ucrt_paths = glob.glob(ucrt_dir + r'\*') - assert(len(ucrt_paths) > 0) - for ucrt_path in ucrt_paths: - ucrt_file = os.path.split(ucrt_path)[1] - for dest_dir in [ r'VC\bin\amd64_x86', r'VC\bin\amd64', - r'Windows Kits\10\bin\x64', 'sys64']: - result.append((ucrt_path, os.path.join(dest_dir, ucrt_file))) - - system_crt_files = [ - # Needed to let debug binaries run. - 'ucrtbased.dll', - ] - cpu_pairs = [ - ('x86', 'sys32'), - ('x64', 'sys64'), - ] - if include_arm: - cpu_pairs += [ - ('arm64', 'sysarm64'), - ] - for system_crt_file in system_crt_files: - for cpu_pair in cpu_pairs: - target_cpu, dest_dir = cpu_pair - src_path = os.path.join(sdk_path, 'bin', _win_version, target_cpu, 'ucrt') - result.append((os.path.join(src_path, system_crt_file), - os.path.join(dest_dir, system_crt_file))) - - # Generically drop all arm stuff that we don't need, and - # drop .msi files because we don't need installers and drop - # samples since those are not used by any tools. - def is_skippable(f): - return ('arm\\' in f.lower() or - (not include_arm and 'arm64\\' in f.lower()) or - 'samples\\' in f.lower() or - f.lower().endswith(('.msi', - '.msm'))) - return [(f, t) for f, t in result if not is_skippable(f)] def GenerateSetEnvCmd(target_dir): - """Generate a batch file that gyp expects to exist to set up the compiler + """Generate a batch file that gyp expects to exist to set up the compiler environment. This is normally generated by a full install of the SDK, but we do it here manually since we do not do a full install.""" - vc_tools_parts = _vc_tools.split('/') + vc_tools_parts = _vc_tools.split('/') - # All these paths are relative to the root of the toolchain package. - include_dirs = [ - ['Windows Kits', '10', 'Include', _win_version, 'um'], - ['Windows Kits', '10', 'Include', _win_version, 'shared'], - ['Windows Kits', '10', 'Include', _win_version, 'winrt'], - ] - include_dirs.append(['Windows Kits', '10', 'Include', _win_version, 'ucrt']) - include_dirs.extend([ - vc_tools_parts + ['include'], - vc_tools_parts + ['atlmfc', 'include'], - ]) - libpath_dirs = [ - vc_tools_parts + ['lib', 'x86', 'store', 'references'], - ['Windows Kits', '10', 'UnionMetadata', _win_version], - ] - # Common to x86, x64, and arm64 - env = collections.OrderedDict([ - # Yuck: These have a trailing \ character. No good way to represent this in - # an OS-independent way. - ('VSINSTALLDIR', [['.\\']]), - ('VCINSTALLDIR', [['VC\\']]), - ('INCLUDE', include_dirs), - ('LIBPATH', libpath_dirs), - ]) - # x86. Always use amd64_x86 cross, not x86 on x86. - env['VCToolsInstallDir'] = [vc_tools_parts[:]] - # Yuck: This one ends in a path separator as well. - env['VCToolsInstallDir'][0][-1] += os.path.sep - env_x86 = collections.OrderedDict([ - ( - 'PATH', - [ - ['Windows Kits', '10', 'bin', _win_version, 'x64'], - vc_tools_parts + ['bin', 'HostX64', 'x86'], - vc_tools_parts + ['bin', 'HostX64', 'x64' - ], # Needed for mspdb1x0.dll. - ]), - ('LIB', [ - vc_tools_parts + ['lib', 'x86'], - ['Windows Kits', '10', 'Lib', _win_version, 'um', 'x86'], - ['Windows Kits', '10', 'Lib', _win_version, 'ucrt', 'x86'], - vc_tools_parts + ['atlmfc', 'lib', 'x86'], - ]), - ]) + # All these paths are relative to the root of the toolchain package. + include_dirs = [ + ['Windows Kits', '10', 'Include', _win_version, 'um'], + ['Windows Kits', '10', 'Include', _win_version, 'shared'], + ['Windows Kits', '10', 'Include', _win_version, 'winrt'], + ] + include_dirs.append(['Windows Kits', '10', 'Include', _win_version, 'ucrt']) + include_dirs.extend([ + vc_tools_parts + ['include'], + vc_tools_parts + ['atlmfc', 'include'], + ]) + libpath_dirs = [ + vc_tools_parts + ['lib', 'x86', 'store', 'references'], + ['Windows Kits', '10', 'UnionMetadata', _win_version], + ] + # Common to x86, x64, and arm64 + env = collections.OrderedDict([ + # Yuck: These have a trailing \ character. No good way to represent this + # in an OS-independent way. + ('VSINSTALLDIR', [['.\\']]), + ('VCINSTALLDIR', [['VC\\']]), + ('INCLUDE', include_dirs), + ('LIBPATH', libpath_dirs), + ]) + # x86. Always use amd64_x86 cross, not x86 on x86. + env['VCToolsInstallDir'] = [vc_tools_parts[:]] + # Yuck: This one ends in a path separator as well. + env['VCToolsInstallDir'][0][-1] += os.path.sep + env_x86 = collections.OrderedDict([ + ( + 'PATH', + [ + ['Windows Kits', '10', 'bin', _win_version, 'x64'], + vc_tools_parts + ['bin', 'HostX64', 'x86'], + vc_tools_parts + + ['bin', 'HostX64', 'x64'], # Needed for mspdb1x0.dll. + ]), + ('LIB', [ + vc_tools_parts + ['lib', 'x86'], + ['Windows Kits', '10', 'Lib', _win_version, 'um', 'x86'], + ['Windows Kits', '10', 'Lib', _win_version, 'ucrt', 'x86'], + vc_tools_parts + ['atlmfc', 'lib', 'x86'], + ]), + ]) - # x64. - env_x64 = collections.OrderedDict([ - ('PATH', [ - ['Windows Kits', '10', 'bin', _win_version, 'x64'], - vc_tools_parts + ['bin', 'HostX64', 'x64'], - ]), - ('LIB', [ - vc_tools_parts + ['lib', 'x64'], - ['Windows Kits', '10', 'Lib', _win_version, 'um', 'x64'], - ['Windows Kits', '10', 'Lib', _win_version, 'ucrt', 'x64'], - vc_tools_parts + ['atlmfc', 'lib', 'x64'], - ]), - ]) + # x64. + env_x64 = collections.OrderedDict([ + ('PATH', [ + ['Windows Kits', '10', 'bin', _win_version, 'x64'], + vc_tools_parts + ['bin', 'HostX64', 'x64'], + ]), + ('LIB', [ + vc_tools_parts + ['lib', 'x64'], + ['Windows Kits', '10', 'Lib', _win_version, 'um', 'x64'], + ['Windows Kits', '10', 'Lib', _win_version, 'ucrt', 'x64'], + vc_tools_parts + ['atlmfc', 'lib', 'x64'], + ]), + ]) - # arm64. - env_arm64 = collections.OrderedDict([ - ('PATH', [ - ['Windows Kits', '10', 'bin', _win_version, 'x64'], - vc_tools_parts + ['bin', 'HostX64', 'arm64'], - vc_tools_parts + ['bin', 'HostX64', 'x64'], - ]), - ('LIB', [ - vc_tools_parts + ['lib', 'arm64'], - ['Windows Kits', '10', 'Lib', _win_version, 'um', 'arm64'], - ['Windows Kits', '10', 'Lib', _win_version, 'ucrt', 'arm64'], - vc_tools_parts + ['atlmfc', 'lib', 'arm64'], - ]), - ]) + # arm64. + env_arm64 = collections.OrderedDict([ + ('PATH', [ + ['Windows Kits', '10', 'bin', _win_version, 'x64'], + vc_tools_parts + ['bin', 'HostX64', 'arm64'], + vc_tools_parts + ['bin', 'HostX64', 'x64'], + ]), + ('LIB', [ + vc_tools_parts + ['lib', 'arm64'], + ['Windows Kits', '10', 'Lib', _win_version, 'um', 'arm64'], + ['Windows Kits', '10', 'Lib', _win_version, 'ucrt', 'arm64'], + vc_tools_parts + ['atlmfc', 'lib', 'arm64'], + ]), + ]) - def BatDirs(dirs): - return ';'.join(['%cd%\\' + os.path.join(*d) for d in dirs]) - set_env_prefix = os.path.join(target_dir, r'Windows Kits\10\bin\SetEnv') - with open(set_env_prefix + '.cmd', 'w') as f: - # The prologue changes the current directory to the root of the toolchain - # package, so that path entries can be set up without needing ..\..\..\ - # components. - f.write('@echo off\n' - ':: Generated by win_toolchain\\package_from_installed.py.\n' - 'pushd %~dp0..\..\..\n') - for var, dirs in env.items(): - f.write('set %s=%s\n' % (var, BatDirs(dirs))) - f.write('if "%1"=="/x64" goto x64\n') - f.write('if "%1"=="/arm64" goto arm64\n') + def BatDirs(dirs): + return ';'.join(['%cd%\\' + os.path.join(*d) for d in dirs]) - for var, dirs in env_x86.items(): - f.write('set %s=%s%s\n' % ( - var, BatDirs(dirs), ';%PATH%' if var == 'PATH' else '')) - f.write('goto :END\n') + set_env_prefix = os.path.join(target_dir, r'Windows Kits\10\bin\SetEnv') + with open(set_env_prefix + '.cmd', 'w') as f: + # The prologue changes the current directory to the root of the + # toolchain package, so that path entries can be set up without needing + # ..\..\..\ components. + f.write('@echo off\n' + ':: Generated by win_toolchain\\package_from_installed.py.\n' + 'pushd %~dp0..\..\..\n') + for var, dirs in env.items(): + f.write('set %s=%s\n' % (var, BatDirs(dirs))) + f.write('if "%1"=="/x64" goto x64\n') + f.write('if "%1"=="/arm64" goto arm64\n') - f.write(':x64\n') - for var, dirs in env_x64.items(): - f.write('set %s=%s%s\n' % ( - var, BatDirs(dirs), ';%PATH%' if var == 'PATH' else '')) - f.write('goto :END\n') + for var, dirs in env_x86.items(): + f.write('set %s=%s%s\n' % + (var, BatDirs(dirs), ';%PATH%' if var == 'PATH' else '')) + f.write('goto :END\n') - f.write(':arm64\n') - for var, dirs in env_arm64.items(): - f.write('set %s=%s%s\n' % ( - var, BatDirs(dirs), ';%PATH%' if var == 'PATH' else '')) - f.write('goto :END\n') - f.write(':END\n') - # Restore the original directory. - f.write('popd\n') - with open(set_env_prefix + '.x86.json', 'wt', newline='') as f: - assert not set(env.keys()) & set(env_x86.keys()), 'dupe keys' - json.dump({'env': collections.OrderedDict(list(env.items()) + list(env_x86.items()))}, - f) - with open(set_env_prefix + '.x64.json', 'wt', newline='') as f: - assert not set(env.keys()) & set(env_x64.keys()), 'dupe keys' - json.dump({'env': collections.OrderedDict(list(env.items()) + list(env_x64.items()))}, - f) - with open(set_env_prefix + '.arm64.json', 'wt', newline='') as f: - assert not set(env.keys()) & set(env_arm64.keys()), 'dupe keys' - json.dump({'env': collections.OrderedDict(list(env.items()) + list(env_arm64.items()))}, - f) + f.write(':x64\n') + for var, dirs in env_x64.items(): + f.write('set %s=%s%s\n' % + (var, BatDirs(dirs), ';%PATH%' if var == 'PATH' else '')) + f.write('goto :END\n') + + f.write(':arm64\n') + for var, dirs in env_arm64.items(): + f.write('set %s=%s%s\n' % + (var, BatDirs(dirs), ';%PATH%' if var == 'PATH' else '')) + f.write('goto :END\n') + f.write(':END\n') + # Restore the original directory. + f.write('popd\n') + with open(set_env_prefix + '.x86.json', 'wt', newline='') as f: + assert not set(env.keys()) & set(env_x86.keys()), 'dupe keys' + json.dump( + { + 'env': + collections.OrderedDict( + list(env.items()) + list(env_x86.items())) + }, f) + with open(set_env_prefix + '.x64.json', 'wt', newline='') as f: + assert not set(env.keys()) & set(env_x64.keys()), 'dupe keys' + json.dump( + { + 'env': + collections.OrderedDict( + list(env.items()) + list(env_x64.items())) + }, f) + with open(set_env_prefix + '.arm64.json', 'wt', newline='') as f: + assert not set(env.keys()) & set(env_arm64.keys()), 'dupe keys' + json.dump( + { + 'env': + collections.OrderedDict( + list(env.items()) + list(env_arm64.items())) + }, f) def AddEnvSetup(files, include_arm): - """We need to generate this file in the same way that the "from pieces" + """We need to generate this file in the same way that the "from pieces" script does, so pull that in here.""" - tempdir = tempfile.mkdtemp() - os.makedirs(os.path.join(tempdir, 'Windows Kits', '10', 'bin')) - GenerateSetEnvCmd(tempdir) - files.append(( - os.path.join(tempdir, 'Windows Kits', '10', 'bin', 'SetEnv.cmd'), - 'Windows Kits\\10\\bin\\SetEnv.cmd')) - files.append(( - os.path.join(tempdir, 'Windows Kits', '10', 'bin', 'SetEnv.x86.json'), - 'Windows Kits\\10\\bin\\SetEnv.x86.json')) - files.append(( - os.path.join(tempdir, 'Windows Kits', '10', 'bin', 'SetEnv.x64.json'), - 'Windows Kits\\10\\bin\\SetEnv.x64.json')) - if include_arm: - files.append(( - os.path.join(tempdir, 'Windows Kits', '10', 'bin', 'SetEnv.arm64.json'), - 'Windows Kits\\10\\bin\\SetEnv.arm64.json')) - vs_version_file = os.path.join(tempdir, 'VS_VERSION') - with open(vs_version_file, 'wt', newline='') as version: - print(_vs_version, file=version) - files.append((vs_version_file, 'VS_VERSION')) + tempdir = tempfile.mkdtemp() + os.makedirs(os.path.join(tempdir, 'Windows Kits', '10', 'bin')) + GenerateSetEnvCmd(tempdir) + files.append( + (os.path.join(tempdir, 'Windows Kits', '10', 'bin', + 'SetEnv.cmd'), 'Windows Kits\\10\\bin\\SetEnv.cmd')) + files.append((os.path.join(tempdir, 'Windows Kits', '10', 'bin', + 'SetEnv.x86.json'), + 'Windows Kits\\10\\bin\\SetEnv.x86.json')) + files.append((os.path.join(tempdir, 'Windows Kits', '10', 'bin', + 'SetEnv.x64.json'), + 'Windows Kits\\10\\bin\\SetEnv.x64.json')) + if include_arm: + files.append((os.path.join(tempdir, 'Windows Kits', '10', 'bin', + 'SetEnv.arm64.json'), + 'Windows Kits\\10\\bin\\SetEnv.arm64.json')) + vs_version_file = os.path.join(tempdir, 'VS_VERSION') + with open(vs_version_file, 'wt', newline='') as version: + print(_vs_version, file=version) + files.append((vs_version_file, 'VS_VERSION')) def RenameToSha1(output): - """Determine the hash in the same way that the unzipper does to rename the + """Determine the hash in the same way that the unzipper does to rename the # .zip file.""" - print('Extracting to determine hash...') - tempdir = tempfile.mkdtemp() - old_dir = os.getcwd() - os.chdir(tempdir) - rel_dir = 'vs_files' - with zipfile.ZipFile( - os.path.join(old_dir, output), 'r', zipfile.ZIP_DEFLATED, True) as zf: - zf.extractall(rel_dir) - print('Hashing...') - sha1 = get_toolchain_if_necessary.CalculateHash(rel_dir, None) - # Shorten from forty characters to ten. This is still enough to avoid - # collisions, while being less unwieldy and reducing the risk of MAX_PATH - # failures. - sha1 = sha1[:10] - os.chdir(old_dir) - shutil.rmtree(tempdir) - final_name = sha1 + '.zip' - os.rename(output, final_name) - print('Renamed %s to %s.' % (output, final_name)) + print('Extracting to determine hash...') + tempdir = tempfile.mkdtemp() + old_dir = os.getcwd() + os.chdir(tempdir) + rel_dir = 'vs_files' + with zipfile.ZipFile(os.path.join(old_dir, output), 'r', + zipfile.ZIP_DEFLATED, True) as zf: + zf.extractall(rel_dir) + print('Hashing...') + sha1 = get_toolchain_if_necessary.CalculateHash(rel_dir, None) + # Shorten from forty characters to ten. This is still enough to avoid + # collisions, while being less unwieldy and reducing the risk of MAX_PATH + # failures. + sha1 = sha1[:10] + os.chdir(old_dir) + shutil.rmtree(tempdir) + final_name = sha1 + '.zip' + os.rename(output, final_name) + print('Renamed %s to %s.' % (output, final_name)) def main(): - if sys.version_info[0] < 3: - print('This script requires Python 3') - sys.exit(10) - usage = 'usage: %prog [options] 2022' - parser = optparse.OptionParser(usage) - parser.add_option('-w', '--winver', action='store', type='string', - dest='winver', default='10.0.22621.0', - help='Windows SDK version, such as 10.0.22621.0') - parser.add_option('-d', '--dryrun', action='store_true', dest='dryrun', - default=False, - help='scan for file existence and prints statistics') - parser.add_option('--noarm', action='store_false', dest='arm', - default=True, - help='Avoids arm parts of the SDK') - parser.add_option('--override', action='store', type='string', - dest='override_dir', default=None, - help='Specify alternate bin/include/lib directory') - parser.add_option('--repackage', action='store', type='string', - dest='repackage_dir', default=None, - help='Specify raw directory to be packaged, for hot fixes.') - parser.add_option('--allow_multiple_vs_installs', action='store_true', - default=False, dest='allow_multiple_vs_installs', - help='Specify if multiple VS installs are allowed.') - (options, args) = parser.parse_args() + if sys.version_info[0] < 3: + print('This script requires Python 3') + sys.exit(10) + usage = 'usage: %prog [options] 2022' + parser = optparse.OptionParser(usage) + parser.add_option('-w', + '--winver', + action='store', + type='string', + dest='winver', + default='10.0.22621.0', + help='Windows SDK version, such as 10.0.22621.0') + parser.add_option('-d', + '--dryrun', + action='store_true', + dest='dryrun', + default=False, + help='scan for file existence and prints statistics') + parser.add_option('--noarm', + action='store_false', + dest='arm', + default=True, + help='Avoids arm parts of the SDK') + parser.add_option('--override', + action='store', + type='string', + dest='override_dir', + default=None, + help='Specify alternate bin/include/lib directory') + parser.add_option( + '--repackage', + action='store', + type='string', + dest='repackage_dir', + default=None, + help='Specify raw directory to be packaged, for hot fixes.') + parser.add_option('--allow_multiple_vs_installs', + action='store_true', + default=False, + dest='allow_multiple_vs_installs', + help='Specify if multiple VS installs are allowed.') + (options, args) = parser.parse_args() - if options.repackage_dir: - files = BuildRepackageFileList(options.repackage_dir) - else: - if len(args) != 1 or args[0] not in SUPPORTED_VS_VERSIONS: - print('Must specify 2022') - parser.print_help(); - return 1 + if options.repackage_dir: + files = BuildRepackageFileList(options.repackage_dir) + else: + if len(args) != 1 or args[0] not in SUPPORTED_VS_VERSIONS: + print('Must specify 2022') + parser.print_help() + return 1 - if options.override_dir: - if (not os.path.exists(os.path.join(options.override_dir, 'bin')) or - not os.path.exists(os.path.join(options.override_dir, 'include')) or - not os.path.exists(os.path.join(options.override_dir, 'lib'))): - print('Invalid override directory - must contain bin/include/lib dirs') - return 1 + if options.override_dir: + if (not os.path.exists(os.path.join(options.override_dir, 'bin')) + or not os.path.exists( + os.path.join(options.override_dir, 'include')) + or not os.path.exists( + os.path.join(options.override_dir, 'lib'))): + print( + 'Invalid override directory - must contain bin/include/lib dirs' + ) + return 1 - global _vs_version - _vs_version = args[0] - global _win_version - _win_version = options.winver - global _vc_tools - global _allow_multiple_vs_installs - _allow_multiple_vs_installs = options.allow_multiple_vs_installs - vs_path = GetVSPath() - temp_tools_path = ExpandWildcards(vs_path, 'VC/Tools/MSVC/14.*.*') - # Strip off the leading vs_path characters and switch back to / separators. - _vc_tools = temp_tools_path[len(vs_path) + 1:].replace('\\', '/') + global _vs_version + _vs_version = args[0] + global _win_version + _win_version = options.winver + global _vc_tools + global _allow_multiple_vs_installs + _allow_multiple_vs_installs = options.allow_multiple_vs_installs + vs_path = GetVSPath() + temp_tools_path = ExpandWildcards(vs_path, 'VC/Tools/MSVC/14.*.*') + # Strip off the leading vs_path characters and switch back to / + # separators. + _vc_tools = temp_tools_path[len(vs_path) + 1:].replace('\\', '/') - print('Building file list for VS %s Windows %s...' % (_vs_version, _win_version)) - files = BuildFileList(options.override_dir, options.arm, vs_path) + print('Building file list for VS %s Windows %s...' % + (_vs_version, _win_version)) + files = BuildFileList(options.override_dir, options.arm, vs_path) - AddEnvSetup(files, options.arm) + AddEnvSetup(files, options.arm) + + if False: + for f in files: + print(f[0], '->', f[1]) + return 0 + + output = 'out.zip' + if os.path.exists(output): + os.unlink(output) + count = 0 + version_match_count = 0 + total_size = 0 + missing_files = False + with zipfile.ZipFile(output, 'w', zipfile.ZIP_DEFLATED, True) as zf: + for disk_name, archive_name in files: + sys.stdout.write('\r%d/%d ...%s' % + (count, len(files), disk_name[-40:])) + sys.stdout.flush() + count += 1 + if not options.repackage_dir and disk_name.count(_win_version) > 0: + version_match_count += 1 + if os.path.exists(disk_name): + total_size += os.path.getsize(disk_name) + if not options.dryrun: + zf.write(disk_name, archive_name) + else: + missing_files = True + sys.stdout.write('\r%s does not exist.\n\n' % disk_name) + sys.stdout.flush() + sys.stdout.write( + '\r%1.3f GB of data in %d files, %d files for %s.%s\n' % + (total_size / 1e9, count, version_match_count, _win_version, ' ' * 50)) + if options.dryrun: + return 0 + if missing_files: + raise Exception('One or more files were missing - aborting') + if not options.repackage_dir and version_match_count == 0: + raise Exception('No files found that match the specified winversion') + sys.stdout.write('\rWrote to %s.%s\n' % (output, ' ' * 50)) + sys.stdout.flush() + + RenameToSha1(output) - if False: - for f in files: - print(f[0], '->', f[1]) return 0 - output = 'out.zip' - if os.path.exists(output): - os.unlink(output) - count = 0 - version_match_count = 0 - total_size = 0 - missing_files = False - with zipfile.ZipFile(output, 'w', zipfile.ZIP_DEFLATED, True) as zf: - for disk_name, archive_name in files: - sys.stdout.write('\r%d/%d ...%s' % (count, len(files), disk_name[-40:])) - sys.stdout.flush() - count += 1 - if not options.repackage_dir and disk_name.count(_win_version) > 0: - version_match_count += 1 - if os.path.exists(disk_name): - total_size += os.path.getsize(disk_name) - if not options.dryrun: - zf.write(disk_name, archive_name) - else: - missing_files = True - sys.stdout.write('\r%s does not exist.\n\n' % disk_name) - sys.stdout.flush() - sys.stdout.write('\r%1.3f GB of data in %d files, %d files for %s.%s\n' % - (total_size / 1e9, count, version_match_count, _win_version, ' '*50)) - if options.dryrun: - return 0 - if missing_files: - raise Exception('One or more files were missing - aborting') - if not options.repackage_dir and version_match_count == 0: - raise Exception('No files found that match the specified winversion') - sys.stdout.write('\rWrote to %s.%s\n' % (output, ' '*50)) - sys.stdout.flush() - - RenameToSha1(output) - - return 0 - if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/wtf b/wtf index da4095d9cd..3ac4c57928 100755 --- a/wtf +++ b/wtf @@ -2,7 +2,6 @@ # Copyright (c) 2010 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Display active git branches and code changes in a chromiumos workspace.""" from __future__ import print_function @@ -15,67 +14,66 @@ import sys def show_dir(full_name, relative_name, color): - """Display active work in a single git repo.""" + """Display active work in a single git repo.""" + def show_name(): + """Display the directory name.""" - def show_name(): - """Display the directory name.""" + if color: + sys.stdout.write('========= %s[44m%s[37m%s%s[0m ========\n' % + (chr(27), chr(27), relative_name, chr(27))) + else: + sys.stdout.write('========= %s ========\n' % relative_name) + lines_printed = 0 + + cmd = ['git', 'branch', '-v'] if color: - sys.stdout.write('========= %s[44m%s[37m%s%s[0m ========\n' % - (chr(27), chr(27), relative_name, chr(27))) - else: - sys.stdout.write('========= %s ========\n' % relative_name) + cmd.append('--color') - lines_printed = 0 + branch = subprocess.Popen(cmd, cwd=full_name, + stdout=subprocess.PIPE).communicate()[0].rstrip() - cmd = ['git', 'branch', '-v'] - if color: - cmd.append('--color') + if len(branch.splitlines()) > 1: + if lines_printed == 0: + show_name() + lines_printed += 1 + print(branch) - branch = subprocess.Popen(cmd, - cwd=full_name, - stdout=subprocess.PIPE).communicate()[0].rstrip() + status = subprocess.Popen(['git', 'status'], + cwd=full_name, + stdout=subprocess.PIPE).communicate()[0].rstrip() - if len(branch.splitlines()) > 1: - if lines_printed == 0: - show_name() - lines_printed += 1 - print(branch) - - status = subprocess.Popen(['git', 'status'], - cwd=full_name, - stdout=subprocess.PIPE).communicate()[0].rstrip() - - if len(status.splitlines()) > 2: - if lines_printed == 0: - show_name() - if lines_printed == 1: - print('---------------') - print(status) + if len(status.splitlines()) > 2: + if lines_printed == 0: + show_name() + if lines_printed == 1: + print('---------------') + print(status) def main(): - """Take no arguments.""" + """Take no arguments.""" - color = False + color = False - if os.isatty(1): - color = True + if os.isatty(1): + color = True - base = os.path.basename(os.getcwd()) - root, entries = gclient_utils.GetGClientRootAndEntries() + base = os.path.basename(os.getcwd()) + root, entries = gclient_utils.GetGClientRootAndEntries() - # which entries map to a git repos? - raw = [k for k, v in entries.items() if v and not re.search('svn', v)] - raw.sort() + # which entries map to a git repos? + raw = [k for k, v in entries.items() if v and not re.search('svn', v)] + raw.sort() - # We want to use the full path for testing, but we want to use the relative - # path for display. - fulldirs = [os.path.normpath(os.path.join(root, p)) for p in raw] - reldirs = [re.sub('^' + base, '.', p) for p in raw] + # We want to use the full path for testing, but we want to use the relative + # path for display. + fulldirs = [os.path.normpath(os.path.join(root, p)) for p in raw] + reldirs = [re.sub('^' + base, '.', p) for p in raw] + + for full_path, relative_path in zip(fulldirs, reldirs): + show_dir(full_path, relative_path, color) - for full_path, relative_path in zip(fulldirs, reldirs): - show_dir(full_path, relative_path, color) if __name__ == '__main__': - main() + main()