InTheForest/chromium_depot_tools
1
0
Fork 0
mirror of https://chromium.googlesource.com/chromium/tools/depot_tools.git synced 2026-01-11 10:41:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reland "Add support for GCS deps"

Browse Source 
This is a reland of commit 3eedee7b55

Original change's description:
> Add support for GCS deps
>
> Also take out GCS calling logic from download_google_storage and
> into call_google_storage.
>
> GCS deps look like:
>    'src/third_party/node/linux': {
>        'dep_type': 'gcs',
>        'condition': 'checkout_linux',
>        'bucket': 'chromium-nodejs/20.11.0',
>        'object_name': '46795170ff5df9831955f163f6966abde581c8af',
>        'sha256sum': '887504c37404898ca41b896f448ee6d7fc24179d8fb6a4b79d028ab7e1b7153d',
>    },
>
>    'src/third_party/llvm-build/Release+Asserts': {
>        'dep_type': 'gcs',
>        'condition': 'checkout_linux',
>        'bucket': 'chromium-browser-clang',
>        'object_name': 'Linux_x64/clang-llvmorg-18-init-17730-gf670112a-2.tar.xz',
>        'sha256sum': '1e46df9b4e63c074064d75646310cb76be2f19815997a8486987189d80f991e8',
>    },
>
> Example directory for src/third_party/node/linux after gclient sync:
> - tar_file.gz is the downloaded file from GCS.
> - node_linux_x64/ is extracted in its path.
> - `hash` contains the sha of GCS filename.
> ```
> chromium/src/ ->
>    third_party/node/linux/ ->
>        hash, tar_file.gz, node_linux_x64/
> ```
>
> Bug: b/324418194
> Change-Id: Ibcbbff27e211f194ddb8a08494af56570a84a12b
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5299722
> Commit-Queue: Stephanie Kim <kimstephanie@google.com>
> Reviewed-by: Joanna Wang <jojwang@chromium.org>

Bug: b/324418194
Change-Id: I8b960f6dd3534658a9ee0b7688e68eaa15df1444
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5334791
Commit-Queue: Stephanie Kim <kimstephanie@google.com>
Reviewed-by: Josip Sokcevic <sokcevic@chromium.org>
Reviewed-by: Joanna Wang <jojwang@chromium.org>
This commit is contained in:
Stephanie Kim
2024-03-05 17:35:23 +00:00
committed by LUCI CQ
parent 771b8d7983
commit d94a03f21e
12 changed files with 619 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
download_from_google_storage.py
View File

@@ -18,25 +18,7 @@ import threading
import time
import subprocess2
# Env vars that tempdir can be gotten from; minimally, this
# needs to match python's tempfile module and match normal
# unix standards.
_TEMPDIR_ENV_VARS = ('TMPDIR', 'TEMP', 'TMP')
GSUTIL_DEFAULT_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'gsutil.py')
# Maps sys.platform to what we actually want to call them.
PLATFORM_MAPPING = {
'cygwin': 'win',
'darwin': 'mac',
'linux': 'linux', # Python 3.3+.
'linux2': 'linux', # Python < 3.3 uses "linux2" / "linux3".
'win32': 'win',
'aix6': 'aix',
'aix7': 'aix',
'zos': 'zos',
}
import call_google_storage
class InvalidFileError(IOError):
@@ -47,91 +29,6 @@ class InvalidPlatformError(Exception):
pass
def GetNormalizedPlatform():
"""Returns the result of sys.platform accounting for cygwin.
Under cygwin, this will always return "win32" like the native Python."""
if sys.platform == 'cygwin':
return 'win32'
return sys.platform
# Common utilities
class Gsutil(object):
"""Call gsutil with some predefined settings. This is a convenience object,
and is also immutable.
HACK: This object is used directly by the external script
`<depot_tools>/win_toolchain/get_toolchain_if_necessary.py`
"""
MAX_TRIES = 5
RETRY_BASE_DELAY = 5.0
RETRY_DELAY_MULTIPLE = 1.3
VPYTHON3 = ('vpython3.bat'
if GetNormalizedPlatform() == 'win32' else 'vpython3')
def __init__(self, path, boto_path=None):
if not os.path.exists(path):
raise FileNotFoundError('GSUtil not found in %s' % path)
self.path = path
self.boto_path = boto_path
def get_sub_env(self):
env = os.environ.copy()
if self.boto_path == os.devnull:
env['AWS_CREDENTIAL_FILE'] = ''
env['BOTO_CONFIG'] = ''
elif self.boto_path:
env['AWS_CREDENTIAL_FILE'] = self.boto_path
env['BOTO_CONFIG'] = self.boto_path
if PLATFORM_MAPPING[sys.platform] != 'win':
env.update((x, "/tmp") for x in _TEMPDIR_ENV_VARS)
return env
def call(self, *args):
cmd = [self.VPYTHON3, self.path]
cmd.extend(args)
return subprocess2.call(cmd, env=self.get_sub_env())
def check_call(self, *args):
cmd = [self.VPYTHON3, self.path]
cmd.extend(args)
((out, err), code) = subprocess2.communicate(cmd,
stdout=subprocess2.PIPE,
stderr=subprocess2.PIPE,
env=self.get_sub_env())
out = out.decode('utf-8', 'replace')
err = err.decode('utf-8', 'replace')
# Parse output.
status_code_match = re.search('status=([0-9]+)', err)
if status_code_match:
return (int(status_code_match.group(1)), out, err)
if ('ServiceException: 401 Anonymous' in err):
return (401, out, err)
if ('You are attempting to access protected data with '
'no configured credentials.' in err):
return (403, out, err)
if 'matched no objects' in err or 'No URLs matched' in err:
return (404, out, err)
return (code, out, err)
def check_call_with_retries(self, *args):
delay = self.RETRY_BASE_DELAY
for i in range(self.MAX_TRIES):
code, out, err = self.check_call(*args)
if not code or i == self.MAX_TRIES - 1:
break
time.sleep(delay)
delay *= self.RETRY_DELAY_MULTIPLE
return code, out, err
def check_platform(target):
"""Checks if any parent directory of target matches (win|mac|linux)."""
assert os.path.isabs(target)
@@ -629,11 +526,12 @@ def main(args):
options.boto = os.environ.get('NO_AUTH_BOTO_CONFIG', os.devnull)
# Make sure gsutil exists where we expect it to.
if os.path.exists(GSUTIL_DEFAULT_PATH):
gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto)
if os.path.exists(call_google_storage.GSUTIL_DEFAULT_PATH):
gsutil = call_google_storage.Gsutil(
call_google_storage.GSUTIL_DEFAULT_PATH, boto_path=options.boto)
else:
parser.error('gsutil not found in %s, bad depot_tools checkout?' %
GSUTIL_DEFAULT_PATH)
call_google_storage.GSUTIL_DEFAULT_PATH)
# Passing in -g/--config will run our copy of GSUtil, then quit.
if options.config: