Add support for GCS deps

Also take out GCS calling logic from download_google_storage and
into call_google_storage.

GCS deps look like:
   'src/third_party/node/linux': {
       'dep_type': 'gcs',
       'condition': 'checkout_linux',
       'bucket': 'chromium-nodejs/20.11.0',
       'object_name': '46795170ff5df9831955f163f6966abde581c8af',
       'sha256sum': '887504c37404898ca41b896f448ee6d7fc24179d8fb6a4b79d028ab7e1b7153d',
   },

   'src/third_party/llvm-build/Release+Asserts': {
       'dep_type': 'gcs',
       'condition': 'checkout_linux',
       'bucket': 'chromium-browser-clang',
       'object_name': 'Linux_x64/clang-llvmorg-18-init-17730-gf670112a-2.tar.xz',
       'sha256sum': '1e46df9b4e63c074064d75646310cb76be2f19815997a8486987189d80f991e8',
   },

Example directory for src/third_party/node/linux after gclient sync:
- tar_file.gz is the downloaded file from GCS.
- node_linux_x64/ is extracted in its path.
- `hash` contains the sha of GCS filename.
```
chromium/src/ ->
   third_party/node/linux/ ->
       hash, tar_file.gz, node_linux_x64/
```

Bug: b/324418194
Change-Id: Ibcbbff27e211f194ddb8a08494af56570a84a12b
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5299722
Commit-Queue: Stephanie Kim <kimstephanie@google.com>
Reviewed-by: Joanna Wang <jojwang@chromium.org>
This commit is contained in:
Stephanie Kim
2024-02-29 19:18:32 +00:00
committed by LUCI CQ
parent 1ac3eb7b98
commit 3eedee7b55
12 changed files with 619 additions and 118 deletions

View File

@@ -16,6 +16,7 @@ import random
import re
import socket
import sys
import tarfile
import tempfile
import textwrap
import time
@@ -47,6 +48,9 @@ def read_tree(tree_root):
for f in [join(root, f) for f in files if not f.startswith('.')]:
filepath = f[len(tree_root) + 1:].replace(os.sep, '/')
assert len(filepath) > 0, f
if tarfile.is_tarfile(join(root, f)):
tree[filepath] = 'tarfile'
continue
with io.open(join(root, f), encoding='utf-8') as f:
tree[filepath] = f.read()
return tree
@@ -210,7 +214,7 @@ class FakeReposBase(object):
class FakeRepos(FakeReposBase):
"""Implements populateGit()."""
NB_GIT_REPOS = 21
NB_GIT_REPOS = 23
def populateGit(self):
# Testing:
@@ -881,6 +885,63 @@ deps = {
},
)
self._commit_git(
'repo_22', {
'DEPS':
textwrap.dedent("""\
vars = {}
deps = {
'src/gcs_dep': {
'bucket': '123bucket',
'object_name': 'deadbeef',
'dep_type': 'gcs',
'sha256sum': 'abcd123',
},
'src/another_gcs_dep': {
'bucket': '456bucket',
'object_name': 'Linux/llvmfile.tar.gz',
'dep_type': 'gcs',
'sha256sum': 'abcd123',
},
}"""),
'origin':
'git/repo_22@1\n'
})
self._commit_git(
'repo_23', {
'DEPS': """
deps = {
'src/repo12': '/repo_12',
}""",
'origin': 'git/repo_23@1\n',
})
self._commit_git(
'repo_23', {
'DEPS': """
deps = {
'src/repo12': '/repo_12@refs/changes/1212',
}""",
'origin': 'git/repo_23@2\n',
})
# src/repo12 is now a GCS dependency.
self._commit_git(
'repo_23', {
'DEPS': """
deps = {
'src/repo12': {
'bucket': 'bucket123',
'object_name': 'path_to_file.tar.gz',
'dep_type': 'gcs',
'sha256sum': 'abcd123',
},
}
""",
'origin': 'git/repo_23@3\n'
})
class FakeRepoSkiaDEPS(FakeReposBase):
"""Simulates the Skia DEPS transition in Chrome."""