InTheForest/chromium_depot_tools
1
0
Fork 0
mirror of https://chromium.googlesource.com/chromium/tools/depot_tools.git synced 2026-01-11 10:41:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert "Add support for GCS deps"

Browse Source 
This reverts commit 3eedee7b55.

Reason for revert: https://buganizer.corp.google.com/issues/324418194#comment7

Original change's description:
> Add support for GCS deps
>
> Also take out GCS calling logic from download_google_storage and
> into call_google_storage.
>
> GCS deps look like:
>    'src/third_party/node/linux': {
>        'dep_type': 'gcs',
>        'condition': 'checkout_linux',
>        'bucket': 'chromium-nodejs/20.11.0',
>        'object_name': '46795170ff5df9831955f163f6966abde581c8af',
>        'sha256sum': '887504c37404898ca41b896f448ee6d7fc24179d8fb6a4b79d028ab7e1b7153d',
>    },
>
>    'src/third_party/llvm-build/Release+Asserts': {
>        'dep_type': 'gcs',
>        'condition': 'checkout_linux',
>        'bucket': 'chromium-browser-clang',
>        'object_name': 'Linux_x64/clang-llvmorg-18-init-17730-gf670112a-2.tar.xz',
>        'sha256sum': '1e46df9b4e63c074064d75646310cb76be2f19815997a8486987189d80f991e8',
>    },
>
> Example directory for src/third_party/node/linux after gclient sync:
> - tar_file.gz is the downloaded file from GCS.
> - node_linux_x64/ is extracted in its path.
> - `hash` contains the sha of GCS filename.
> ```
> chromium/src/ ->
>    third_party/node/linux/ ->
>        hash, tar_file.gz, node_linux_x64/
> ```
>
> Bug: b/324418194
> Change-Id: Ibcbbff27e211f194ddb8a08494af56570a84a12b
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5299722
> Commit-Queue: Stephanie Kim <kimstephanie@google.com>
> Reviewed-by: Joanna Wang <jojwang@chromium.org>

Bug: b/324418194
Change-Id: Ie74df90065c1fea087b240d98005b9d9b4f44411
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5336079
Commit-Queue: Stephanie Kim <kimstephanie@google.com>
Bot-Commit: Rubber Stamper <rubber-stamper@appspot.gserviceaccount.com>
Reviewed-by: Josip Sokcevic <sokcevic@chromium.org>
This commit is contained in:
Stephanie Kim
2024-02-29 19:59:10 +00:00
committed by LUCI CQ
parent 3eedee7b55
commit 064e03aa1c
12 changed files with 118 additions and 619 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
download_from_google_storage.py
View File

@@ -18,7 +18,25 @@ import threading
import time
import subprocess2
import call_google_storage
# Env vars that tempdir can be gotten from; minimally, this
# needs to match python's tempfile module and match normal
# unix standards.
_TEMPDIR_ENV_VARS = ('TMPDIR', 'TEMP', 'TMP')
GSUTIL_DEFAULT_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'gsutil.py')
# Maps sys.platform to what we actually want to call them.
PLATFORM_MAPPING = {
'cygwin': 'win',
'darwin': 'mac',
'linux': 'linux', # Python 3.3+.
'linux2': 'linux', # Python < 3.3 uses "linux2" / "linux3".
'win32': 'win',
'aix6': 'aix',
'aix7': 'aix',
'zos': 'zos',
}
class InvalidFileError(IOError):
@@ -29,6 +47,91 @@ class InvalidPlatformError(Exception):
pass
def GetNormalizedPlatform():
"""Returns the result of sys.platform accounting for cygwin.
Under cygwin, this will always return "win32" like the native Python."""
if sys.platform == 'cygwin':
return 'win32'
return sys.platform
# Common utilities
class Gsutil(object):
"""Call gsutil with some predefined settings. This is a convenience object,
and is also immutable.
HACK: This object is used directly by the external script
`<depot_tools>/win_toolchain/get_toolchain_if_necessary.py`
"""
MAX_TRIES = 5
RETRY_BASE_DELAY = 5.0
RETRY_DELAY_MULTIPLE = 1.3
VPYTHON3 = ('vpython3.bat'
if GetNormalizedPlatform() == 'win32' else 'vpython3')
def __init__(self, path, boto_path=None):
if not os.path.exists(path):
raise FileNotFoundError('GSUtil not found in %s' % path)
self.path = path
self.boto_path = boto_path
def get_sub_env(self):
env = os.environ.copy()
if self.boto_path == os.devnull:
env['AWS_CREDENTIAL_FILE'] = ''
env['BOTO_CONFIG'] = ''
elif self.boto_path:
env['AWS_CREDENTIAL_FILE'] = self.boto_path
env['BOTO_CONFIG'] = self.boto_path
if PLATFORM_MAPPING[sys.platform] != 'win':
env.update((x, "/tmp") for x in _TEMPDIR_ENV_VARS)
return env
def call(self, *args):
cmd = [self.VPYTHON3, self.path]
cmd.extend(args)
return subprocess2.call(cmd, env=self.get_sub_env())
def check_call(self, *args):
cmd = [self.VPYTHON3, self.path]
cmd.extend(args)
((out, err), code) = subprocess2.communicate(cmd,
stdout=subprocess2.PIPE,
stderr=subprocess2.PIPE,
env=self.get_sub_env())
out = out.decode('utf-8', 'replace')
err = err.decode('utf-8', 'replace')
# Parse output.
status_code_match = re.search('status=([0-9]+)', err)
if status_code_match:
return (int(status_code_match.group(1)), out, err)
if ('ServiceException: 401 Anonymous' in err):
return (401, out, err)
if ('You are attempting to access protected data with '
'no configured credentials.' in err):
return (403, out, err)
if 'matched no objects' in err or 'No URLs matched' in err:
return (404, out, err)
return (code, out, err)
def check_call_with_retries(self, *args):
delay = self.RETRY_BASE_DELAY
for i in range(self.MAX_TRIES):
code, out, err = self.check_call(*args)
if not code or i == self.MAX_TRIES - 1:
break
time.sleep(delay)
delay *= self.RETRY_DELAY_MULTIPLE
return code, out, err
def check_platform(target):
"""Checks if any parent directory of target matches (win|mac|linux)."""
assert os.path.isabs(target)
@@ -526,12 +629,11 @@ def main(args):
options.boto = os.environ.get('NO_AUTH_BOTO_CONFIG', os.devnull)
# Make sure gsutil exists where we expect it to.
if os.path.exists(call_google_storage.GSUTIL_DEFAULT_PATH):
gsutil = Gsutil(call_google_storage.GSUTIL_DEFAULT_PATH,
boto_path=options.boto)
if os.path.exists(GSUTIL_DEFAULT_PATH):
gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto)
else:
parser.error('gsutil not found in %s, bad depot_tools checkout?' %
call_google_storage.GSUTIL_DEFAULT_PATH)
GSUTIL_DEFAULT_PATH)
# Passing in -g/--config will run our copy of GSUtil, then quit.
if options.config: