InTheForest/chromium_depot_tools
1
0
Fork 0
mirror of https://chromium.googlesource.com/chromium/tools/depot_tools.git synced 2026-01-11 10:41:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

[git_cache] Track if git cache is fully initialized

Browse Source 
git_cache populate can get interrupted midway (e.g. by LUCI CV sending a
signal that build is no longer needed). When that happens, a git mirror
may be in a state where some commits are available, but cloning such
repositry results in an empty repsitory.

This change ensures that the initial fetch operation finished
successfully.

R=ddoman@google.com

Bug: 1517944
Change-Id: I1ee860860877dbfff7a444b45fe4515fe26b248c
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5219032
Commit-Queue: Josip Sokcevic <sokcevic@chromium.org>
Reviewed-by: Scott Lee <ddoman@chromium.org>
This commit is contained in:
Josip Sokcevic
2024-01-22 21:15:40 +00:00
committed by LUCI CQ
parent dbbe224787
commit 6fc0c97ab2
2 changed files with 56 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
git_cache.py
View File

@@ -26,6 +26,7 @@ import subcommand
GC_AUTOPACKLIMIT = 50
GIT_CACHE_CORRUPT_MESSAGE = 'WARNING: The Git cache is corrupt.'
INIT_SENTIENT_FILE = ".mirror_init"
# gsutil creates many processes and threads. Creating too many gsutil cp
# processes may result in running out of resources, and may perform worse due to
@@ -136,6 +137,10 @@ class Mirror(object):
self.print('%s took %.1f minutes' % (what,
(time.time() - start) / 60.0))
@property
def _init_sentient_file(self):
return os.path.join(self.mirror_path, INIT_SENTIENT_FILE)
@property
def bootstrap_bucket(self):
b = os.getenv('OVERRIDE_BOOTSTRAP_BUCKET')
@@ -459,6 +464,9 @@ class Mirror(object):
# 2. Project doesn't have a bootstrap folder.
# Start with a bare git dir.
self.RunGit(['init', '--bare'])
with open(self._init_sentient_file, 'w'):
# Create sentient file
pass
# Set appropriate symbolic-ref
remote_info = exponential_backoff_retry(
lambda: subprocess.check_output(
@@ -523,6 +531,8 @@ class Mirror(object):
self.RunGit(['fetch', 'origin', commit], retry=True)
except subprocess.CalledProcessError:
logging.warning('Fetch of %s failed' % commit)
if os.path.isfile(self._init_sentient_file):
os.remove(self._init_sentient_file)
def populate(self,
depth=None,
@@ -537,20 +547,29 @@ class Mirror(object):
depth = 10000
gclient_utils.safe_makedirs(self.GetCachePath())
def bootstrap(force=False):
self._ensure_bootstrapped(depth,
bootstrap,
reset_fetch_config,
force=force)
self._fetch(verbose, depth, no_fetch_tags, reset_fetch_config)
def wipe_cache():
self.print(GIT_CACHE_CORRUPT_MESSAGE)
gclient_utils.rmtree(self.mirror_path)
with lockfile.lock(self.mirror_path, lock_timeout):
if os.path.isfile(self._init_sentient_file):
# Previous bootstrap didn't finish
wipe_cache()
try:
self._ensure_bootstrapped(depth, bootstrap, reset_fetch_config)
self._fetch(verbose, depth, no_fetch_tags, reset_fetch_config)
bootstrap()
except ClobberNeeded:
# This is a major failure, we need to clean and force a
# bootstrap.
gclient_utils.rmtree(self.mirror_path)
self.print(GIT_CACHE_CORRUPT_MESSAGE)
self._ensure_bootstrapped(depth,
bootstrap,
reset_fetch_config,
force=True)
self._fetch(verbose, depth, no_fetch_tags, reset_fetch_config)
wipe_cache()
bootstrap(force=True)
def update_bootstrap(self, prune=False, gc_aggressive=False):
# NOTE: There have been cases where repos were being recursively