From 38d1673525c0259abaf8d991e472040404dde70c Mon Sep 17 00:00:00 2001 From: Joanna Wang Date: Mon, 10 Oct 2022 17:12:47 +0000 Subject: [PATCH] Prevent unexpected directories from being uploaded to google storage. Bug: 1372658 Change-Id: Ifc19420ee703fcdd0ff8c3d53ddcfeabc499d605 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/3938927 Reviewed-by: Josip Sokcevic Commit-Queue: Joanna Wang --- git_cache.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/git_cache.py b/git_cache.py index 43f4391359..6d827a5318 100755 --- a/git_cache.py +++ b/git_cache.py @@ -510,6 +510,19 @@ class Mirror(object): reset_fetch_config) def update_bootstrap(self, prune=False, gc_aggressive=False, branch='main'): + # NOTE: There have been cases where repos were being recursively uploaded + # to google storage. + # E.g. `-//-/` in GS and + # -/-/ on the bot. + # Check for recursed files on the bot here and remove them if found + # before we upload to GS. + # See crbug.com/1370443; keep this check until root cause is found. + recursed_dir = os.path.join(self.mirror_path, + self.mirror_path.split(os.path)[-1]) + if os.path.exists(recursed_dir): + self.print('Deleting unexpected directory: %s' % recursed_dir) + os.remove(recursed_dir) + # The folder is gen_number = subprocess.check_output( [self.git_exe, 'number', branch], @@ -519,8 +532,13 @@ class Mirror(object): dest_prefix = '%s/%s' % (self._gs_path, gen_number) # ls_out lists contents in the format: gs://blah/blah/123... - _, ls_out, _ = gsutil.check_call('ls', self._gs_path) - self.print('ran "gsutil ls %s": %s' % (self._gs_path, ls_out)) + self.print('running "gsutil ls %s":' % self.gs_path) + ls_code, ls_out, ls_error = gsutil.check_call_with_retries( + 'ls', self._gs_path) + if ls_code != 0: + self.print(ls_error) + else: + self.print(ls_out) # Check to see if folder already exists in gs ls_out_set = set(ls_out.strip().splitlines()) @@ -560,9 +578,9 @@ class Mirror(object): # getting compressed enough. self.RunGit(gc_args) - self.print('running "gsutil -m cp -r %s %s"' % + self.print('running "gsutil -m rsync -r -d %s %s"' % (self.mirror_path, dest_prefix)) - gsutil.call('-m', 'cp', '-r', self.mirror_path, dest_prefix) + gsutil.call('-m', 'rsync', '-r', '-d', self.mirror_path, dest_prefix) # Create .ready file and upload _, ready_file_name = tempfile.mkstemp(suffix='.ready')