diff --git a/git-retry b/git-retry new file mode 100755 index 0000000000..2c039426d6 --- /dev/null +++ b/git-retry @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# git_freezer.py freeze -- a git-command to suspend all existing working +# directory modifications. This can be reversed with the 'git thaw' command. + +SCRIPT=git_retry.py +set -- retry "$@" +. $(type -P python_git_runner.sh) diff --git a/git_common.py b/git_common.py index 298ecc4ff3..0c537134ff 100644 --- a/git_common.py +++ b/git_common.py @@ -42,6 +42,55 @@ FREEZE_SECTIONS = { FREEZE_MATCHER = re.compile(r'%s.(%s)' % (FREEZE, '|'.join(FREEZE_SECTIONS))) +# Retry a git operation if git returns a error response with any of these +# messages. It's all observed 'bad' GoB responses so far. +# +# This list is inspired/derived from the one in ChromiumOS's Chromite: +# /lib/git.py::GIT_TRANSIENT_ERRORS +# +# It was last imported from '7add3ac29564d98ac35ce426bc295e743e7c0c02'. +GIT_TRANSIENT_ERRORS = ( + # crbug.com/285832 + r'! \[remote rejected\].*\(error in hook\)', + + # crbug.com/289932 + r'! \[remote rejected\].*\(failed to lock\)', + + # crbug.com/307156 + r'! \[remote rejected\].*\(error in Gerrit backend\)', + + # crbug.com/285832 + r'remote error: Internal Server Error', + + # crbug.com/294449 + r'fatal: Couldn\'t find remote ref ', + + # crbug.com/220543 + r'git fetch_pack: expected ACK/NAK, got', + + # crbug.com/189455 + r'protocol error: bad pack header', + + # crbug.com/202807 + r'The remote end hung up unexpectedly', + + # crbug.com/298189 + r'TLS packet with unexpected length was received', + + # crbug.com/187444 + r'RPC failed; result=\d+, HTTP code = \d+', + + # crbug.com/315421 + r'The requested URL returned error: 500 while accessing', + + # crbug.com/388876 + r'Connection timed out', +) + +GIT_TRANSIENT_ERRORS_RE = re.compile('|'.join(GIT_TRANSIENT_ERRORS), + re.IGNORECASE) + + class BadCommitRefException(Exception): def __init__(self, refs): msg = ('one of %s does not seem to be a valid commitref.' % diff --git a/git_retry.py b/git_retry.py new file mode 100755 index 0000000000..b40e6d2ef8 --- /dev/null +++ b/git_retry.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import logging +import optparse +import subprocess +import sys +import threading +import time + +from git_common import GIT_EXE, GIT_TRANSIENT_ERRORS_RE + + +class TeeThread(threading.Thread): + + def __init__(self, fd, out_fd, name): + super(TeeThread, self).__init__(name='git-retry.tee.%s' % (name,)) + self.data = None + self.fd = fd + self.out_fd = out_fd + + def run(self): + chunks = [] + for line in self.fd: + chunks.append(line) + self.out_fd.write(line) + self.data = ''.join(chunks) + + +class GitRetry(object): + + logger = logging.getLogger('git-retry') + DEFAULT_DELAY_SECS = 3.0 + DEFAULT_RETRY_COUNT = 5 + + def __init__(self, retry_count=None, delay=None, delay_factor=None): + self.retry_count = retry_count or self.DEFAULT_RETRY_COUNT + self.delay = max(delay, 0) if delay else 0 + self.delay_factor = max(delay_factor, 0) if delay_factor else 0 + + def shouldRetry(self, stderr): + m = GIT_TRANSIENT_ERRORS_RE.search(stderr) + if not m: + return False + self.logger.info("Encountered known transient error: [%s]", + stderr[m.start(): m.end()]) + return True + + @staticmethod + def execute(*args): + args = (GIT_EXE,) + args + proc = subprocess.Popen( + args, + stderr=subprocess.PIPE, + ) + stderr_tee = TeeThread(proc.stderr, sys.stderr, 'stderr') + + # Start our process. Collect/tee 'stdout' and 'stderr'. + stderr_tee.start() + try: + proc.wait() + except KeyboardInterrupt: + proc.kill() + raise + finally: + stderr_tee.join() + return proc.returncode, None, stderr_tee.data + + def computeDelay(self, iteration): + """Returns: the delay (in seconds) for a given iteration + + The first iteration has a delay of '0'. + + Args: + iteration: (int) The iteration index (starting with zero as the first + iteration) + """ + if (not self.delay) or (iteration == 0): + return 0 + if self.delay_factor == 0: + # Linear delay + return iteration * self.delay + # Exponential delay + return (self.delay_factor ** (iteration - 1)) * self.delay + + def __call__(self, *args): + returncode = 0 + for i in xrange(self.retry_count): + # If the previous run failed and a delay is configured, delay before the + # next run. + delay = self.computeDelay(i) + if delay > 0: + self.logger.info("Delaying for [%s second(s)] until next retry", delay) + time.sleep(delay) + + self.logger.debug("Executing subprocess (%d/%d) with arguments: %s", + (i+1), self.retry_count, args) + returncode, _, stderr = self.execute(*args) + + self.logger.debug("Process terminated with return code: %d", returncode) + if returncode == 0: + break + + if not self.shouldRetry(stderr): + self.logger.error("Process failure was not known to be transient; " + "terminating with return code %d", returncode) + break + return returncode + + +def main(args): + parser = optparse.OptionParser() + parser.disable_interspersed_args() + parser.add_option('-v', '--verbose', + action='count', default=0, + help="Increase verbosity; can be specified multiple times") + parser.add_option('-c', '--retry-count', metavar='COUNT', + type=int, default=GitRetry.DEFAULT_RETRY_COUNT, + help="Number of times to retry (default=%default)") + parser.add_option('-d', '--delay', metavar='SECONDS', + type=float, default=GitRetry.DEFAULT_DELAY_SECS, + help="Specifies the amount of time (in seconds) to wait " + "between successive retries (default=%default). This " + "can be zero.") + parser.add_option('-D', '--delay-factor', metavar='FACTOR', + type=int, default=2, + help="The exponential factor to apply to delays in between " + "successive failures (default=%default). If this is " + "zero, delays will increase linearly. Set this to " + "one to have a constant (non-increasing) delay.") + + opts, args = parser.parse_args(args) + + # Configure logging verbosity + if opts.verbose == 0: + logging.getLogger().setLevel(logging.WARNING) + elif opts.verbose == 1: + logging.getLogger().setLevel(logging.INFO) + else: + logging.getLogger().setLevel(logging.DEBUG) + + # Execute retries + retry = GitRetry( + retry_count=opts.retry_count, + delay=opts.delay, + delay_factor=opts.delay_factor, + ) + return retry(*args) + + +if __name__ == '__main__': + logging.basicConfig() + logging.getLogger().setLevel(logging.WARNING) + sys.exit(main(sys.argv[2:])) diff --git a/man/html/git-retry.html b/man/html/git-retry.html new file mode 100644 index 0000000000..1feaaae2ed --- /dev/null +++ b/man/html/git-retry.html @@ -0,0 +1,858 @@ + + + + + +git-retry(1) + + + + + +
+
+

SYNOPSIS

+
+
+
git retry [-v] [-c COUNT] [-d DELAY] [-e] — <git_subcommand>
+
+
+
+
+
+

DESCRIPTION

+
+

git retry is a bootstrap that wraps a standard git command execution in +a fault-tolerant retry wrapper.

+

If a retry succeeds, the return code of the successful attempt is returned. +Otherwise, the return code of the last failed attempt is returned.

+

The wrapper is aware of git-specific failure conditions and will only consider +retrying if a given failure can be linked to such a condition.

+
+
+
+

OPTIONS

+
+
+
+<git_subcommand> +
+
+

+ The git command to retry. This should omit the actual git command (e.g., + to retry git clone, use git retry clone). +

+
+
+-v, --verbose +
+
+

+ Increases logging verbosity. By default, no additional logging is generated + by the git retry command. This can be specified multiple times. +

+
+
+-c, --retry-count count +
+
+

+ Specify the number of retries that should be performed before giving up. The + default retry count is 5. +

+
+
+-d, --delay seconds +
+
+

+ Floating-point value that specifies the amount of time (in seconds) to wait + after a failure. This can be zero to specify no delay. The default delay is + 3 seconds. +

+
+
+-D, --delay-factor +
+
+

+ The exponential factor to apply to the delay. By default this is 2. + For a given retry round n, the delay for that round will be + (<delay-factor>^(n-1) * delay). If no delay is specified, this will have + no effect. + If the delay factor is 0, the delay will increase linearly (for a given + retry round n, the delay will be (n * delay)). +

+
+
+
(Note that a delay factor of *1* will result in a constant delay.)
+
+
+
+
+
+
+

EXIT STATUS

+
+

Upon success, git retry will exit with the successful exit code of 0. On +failure, it will exit with the exit code of the last failed attempt.

+
+
+
+

CHROMIUM DEPOT_TOOLS

+
+

Part of the chromium depot_tools(7) suite. These tools are meant to +assist with the development of chromium and related projects. Download the tools +from here.

+
+
+
+
+ + + diff --git a/man/man1/git-retry.1 b/man/man1/git-retry.1 new file mode 100644 index 0000000000..18538b1d6b --- /dev/null +++ b/man/man1/git-retry.1 @@ -0,0 +1,108 @@ +'\" t +.\" Title: git-retry +.\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] +.\" Generator: DocBook XSL Stylesheets v1.78.1 +.\" Date: 07/22/2014 +.\" Manual: Chromium depot_tools Manual +.\" Source: depot_tools 7242c60 +.\" Language: English +.\" +.TH "GIT\-RETRY" "1" "07/22/2014" "depot_tools 7242c60" "Chromium depot_tools Manual" +.\" ----------------------------------------------------------------- +.\" * Define some portability stuff +.\" ----------------------------------------------------------------- +.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.\" http://bugs.debian.org/507673 +.\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html +.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.ie \n(.g .ds Aq \(aq +.el .ds Aq ' +.\" ----------------------------------------------------------------- +.\" * set default formatting +.\" ----------------------------------------------------------------- +.\" disable hyphenation +.nh +.\" disable justification (adjust text to left margin only) +.ad l +.\" ----------------------------------------------------------------- +.\" * MAIN CONTENT STARTS HERE * +.\" ----------------------------------------------------------------- +.SH "NAME" +git-retry \- Bootstrap function to retry a git command\&. +.SH "SYNOPSIS" +.sp +.nf +\fIgit retry\fR [\-v] [\-c COUNT] [\-d DELAY] [\-e] \(em \fI\fR +.fi +.sp +.SH "DESCRIPTION" +.sp +git retry is a bootstrap that wraps a standard git command execution in a fault\-tolerant retry wrapper\&. +.sp +If a retry succeeds, the return code of the successful attempt is returned\&. Otherwise, the return code of the last failed attempt is returned\&. +.sp +The wrapper is aware of git\-specific failure conditions and will only consider retrying if a given failure can be linked to such a condition\&. +.SH "OPTIONS" +.PP + +.RS 4 +The +git +command to retry\&. This should omit the actual +git +command (e\&.g\&., to retry +git clone, use +git retry clone)\&. +.RE +.PP +\-v, \-\-verbose +.RS 4 +Increases logging verbosity\&. By default, no additional logging is generated by the +git retry +command\&. This can be specified multiple times\&. +.RE +.PP +\-c, \-\-retry\-count \fIcount\fR +.RS 4 +Specify the number of retries that should be performed before giving up\&. The default retry count is +\fB5\fR\&. +.RE +.PP +\-d, \-\-delay \fIseconds\fR +.RS 4 +Floating\-point value that specifies the amount of time (in seconds) to wait after a failure\&. This can be zero to specify no delay\&. The default delay is +\fB3 seconds\fR\&. +.RE +.PP +\-D, \-\-delay\-factor +.RS 4 +The exponential factor to apply to the delay\&. By default this is +\fB2\fR\&. For a given retry round +\fBn\fR, the delay for that round will be +\fB(^(n\-1) * delay)\fR\&. If no delay is specified, this will have no effect\&. If the delay factor is +\fB0\fR, the delay will increase linearly (for a given retry round +\fBn\fR, the delay will be +\fB(n * delay)\fR)\&. +.sp +.if n \{\ +.RS 4 +.\} +.nf +(Note that a delay factor of *1* will result in a constant delay\&.) +.fi +.if n \{\ +.RE +.\} +.RE +.SH "EXIT STATUS" +.sp +Upon success, git retry will exit with the successful exit code of \fB0\fR\&. On failure, it will exit with the exit code of the last failed attempt\&. +.SH "CHROMIUM DEPOT_TOOLS" +.sp +Part of the chromium \fBdepot_tools\fR(7) suite\&. These tools are meant to assist with the development of chromium and related projects\&. Download the tools from \m[blue]\fBhere\fR\m[]\&\s-2\u[1]\d\s+2\&. +.SH "NOTES" +.IP " 1." 4 +here +.RS 4 +\%https://chromium.googlesource.com/chromium/tools/depot_tools.git +.RE diff --git a/man/src/_git-retry_desc.helper.txt b/man/src/_git-retry_desc.helper.txt new file mode 100644 index 0000000000..e2bc2d8a05 --- /dev/null +++ b/man/src/_git-retry_desc.helper.txt @@ -0,0 +1,2 @@ +Bootstrap function to retry a git command. + diff --git a/man/src/git-retry.txt b/man/src/git-retry.txt new file mode 100644 index 0000000000..055691a251 --- /dev/null +++ b/man/src/git-retry.txt @@ -0,0 +1,67 @@ +git-retry(1) +============= + +NAME +---- +git-retry - +include::_git-retry_desc.helper.txt[] + +SYNOPSIS +-------- +[verse] +'git retry' [-v] [-c COUNT] [-d DELAY] [-e] -- __ + +DESCRIPTION +----------- + +`git retry` is a bootstrap that wraps a standard `git` command execution in +a fault-tolerant retry wrapper. + +If a retry succeeds, the return code of the successful attempt is returned. +Otherwise, the return code of the last failed attempt is returned. + +The wrapper is aware of `git`-specific failure conditions and will only consider +retrying if a given failure can be linked to such a condition. + + +OPTIONS +------- + +:: + The `git` command to retry. This should omit the actual `git` command (e.g., + to retry `git clone`, use `git retry clone`). + +-v, --verbose:: + Increases logging verbosity. By default, no additional logging is generated + by the `git retry` command. This can be specified multiple times. + +-c, --retry-count _count_:: + Specify the number of retries that should be performed before giving up. The + default retry count is *5*. + +-d, --delay _seconds_:: + Floating-point value that specifies the amount of time (in seconds) to wait + after a failure. This can be zero to specify no delay. The default delay is + *3 seconds*. + +-D, --delay-factor:: + The exponential factor to apply to the delay. By default this is *2*. + For a given retry round *n*, the delay for that round will be + *(^(n-1) * delay)*. If no delay is specified, this will have + no effect. + If the delay factor is *0*, the delay will increase linearly (for a given + retry round *n*, the delay will be *(n * delay)*). + + (Note that a delay factor of *1* will result in a constant delay.) + + +EXIT STATUS +----------- + +Upon success, `git retry` will exit with the successful exit code of *0*. On +failure, it will exit with the exit code of the last failed attempt. + + +include::_footer.txt[] + +// vim: ft=asciidoc: