InTheForest/chromium_depot_tools
1
0
Fork 0
mirror of https://chromium.googlesource.com/chromium/tools/depot_tools.git synced 2026-01-11 18:51:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reland "Stop using compression for git cache update bootstrap."

Browse Source 
This is a reland of 749139969f

Original change's description:
> Stop using compression for git cache update bootstrap.
>
> Change git cache to stop zipping the git directory and instead upload directly to GS.
>
> Bug: 943696
> Change-Id: I310fb48ff3d7fd6781914e14e0e17530d5d9b328
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1585029
> Commit-Queue: Karen Qian <karenqian@google.com>
> Reviewed-by: Andrii Shyshkalov <tandrii@chromium.org>

Bug: 943696
Change-Id: Ifd3a3a07445345ab647572682ebc6423fb7135b6
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1585272
Reviewed-by: Andrii Shyshkalov <tandrii@chromium.org>
Commit-Queue: Karen Qian <karenqian@google.com>
This commit is contained in:
Karen Qian
2019-04-26 03:11:16 +00:00
committed by Commit Bot
parent 9b51f3e394
commit dcad7498c3
1 changed files with 36 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
git_cache.py
View File

@@ -258,6 +258,10 @@ class Mirror(object):
# Not recognized.
return None
@property
def _gs_path(self):
return 'gs://%s/v2/%s' % (self.bootstrap_bucket, self.basedir)
@classmethod
def FromPath(cls, path):
return cls(cls.CacheDirToUrl(path))
@@ -600,28 +604,43 @@ class Mirror(object):
lockfile.unlock()
def update_bootstrap(self, prune=False):
# The files are named <git number>.zip
# The folder is <git number>
gen_number = subprocess.check_output(
[self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip()
# Run Garbage Collect to compress packfile.
self.RunGit(['gc', '--prune=all'])
# Creating a temp file and then deleting it ensures we can use this name.
_, tmp_zipfile = tempfile.mkstemp(suffix='.zip')
os.remove(tmp_zipfile)
subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=self.mirror_path)
gsutil = Gsutil(path=self.gsutil_exe, boto_path=None)
gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir)
dest_name = '%s/%s.zip' % (gs_folder, gen_number)
gsutil.call('cp', tmp_zipfile, dest_name)
os.remove(tmp_zipfile)
# Remove all other files in the same directory.
if prune:
_, ls_out, _ = gsutil.check_call('ls', gs_folder)
for filename in ls_out.splitlines():
if filename == dest_name:
continue
gsutil.call('rm', filename)
gsutil = Gsutil(path=self.gsutil_exe, boto_path=None)
src_name = self.mirror_path
dest_name = '%s/%s' % (self._gs_path, gen_number)
# check to see if folder already exists in gs
_, ls_out, ls_err = gsutil.check_call('ls', dest_name)
_, ls_out_ready, ls_err_ready = (
gsutil.check_call('ls', dest_name + '.ready'))
# only printing out errors because the folder/ready file
# might not exist yet, so it will error no matter what
if ls_err:
print('Failed to check GS:\n%s' % (ls_err))
if ls_err_ready:
print('Failed to check GS:\n%s' % (ls_err_ready))
if not (ls_out == '' and ls_out_ready == ''):
return
gsutil.call('-m', 'cp', '-r', src_name, dest_name)
#TODO(karenqian): prune old caches
# create .ready file and upload
_, ready_file_name = tempfile.mkstemp(suffix='.ready')
try:
gsutil.call('cp', ready_file_name, '%s.ready' % (dest_name))
finally:
os.remove(ready_file_name)
@staticmethod
def DeleteTmpPackFiles(path):