Revert "Add support for GCS deps"

This reverts commit 3eedee7b55.

Reason for revert: https://buganizer.corp.google.com/issues/324418194#comment7

Original change's description:
> Add support for GCS deps
>
> Also take out GCS calling logic from download_google_storage and
> into call_google_storage.
>
> GCS deps look like:
>    'src/third_party/node/linux': {
>        'dep_type': 'gcs',
>        'condition': 'checkout_linux',
>        'bucket': 'chromium-nodejs/20.11.0',
>        'object_name': '46795170ff5df9831955f163f6966abde581c8af',
>        'sha256sum': '887504c37404898ca41b896f448ee6d7fc24179d8fb6a4b79d028ab7e1b7153d',
>    },
>
>    'src/third_party/llvm-build/Release+Asserts': {
>        'dep_type': 'gcs',
>        'condition': 'checkout_linux',
>        'bucket': 'chromium-browser-clang',
>        'object_name': 'Linux_x64/clang-llvmorg-18-init-17730-gf670112a-2.tar.xz',
>        'sha256sum': '1e46df9b4e63c074064d75646310cb76be2f19815997a8486987189d80f991e8',
>    },
>
> Example directory for src/third_party/node/linux after gclient sync:
> - tar_file.gz is the downloaded file from GCS.
> - node_linux_x64/ is extracted in its path.
> - `hash` contains the sha of GCS filename.
> ```
> chromium/src/ ->
>    third_party/node/linux/ ->
>        hash, tar_file.gz, node_linux_x64/
> ```
>
> Bug: b/324418194
> Change-Id: Ibcbbff27e211f194ddb8a08494af56570a84a12b
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5299722
> Commit-Queue: Stephanie Kim <kimstephanie@google.com>
> Reviewed-by: Joanna Wang <jojwang@chromium.org>

Bug: b/324418194
Change-Id: Ie74df90065c1fea087b240d98005b9d9b4f44411
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5336079
Commit-Queue: Stephanie Kim <kimstephanie@google.com>
Bot-Commit: Rubber Stamper <rubber-stamper@appspot.gserviceaccount.com>
Reviewed-by: Josip Sokcevic <sokcevic@chromium.org>

call_google_storage.py

@@ -1,144 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) 2024 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-"""Download files from Google Storage, given the bucket and file."""
-
-import optparse
-import os
-import re
-import sys
-
-import subprocess2
-
-# Env vars that tempdir can be gotten from; minimally, this
-# needs to match python's tempfile module and match normal
-# unix standards.
-_TEMPDIR_ENV_VARS = ('TMPDIR', 'TEMP', 'TMP')
-
-GSUTIL_DEFAULT_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
-                                   'gsutil.py')
-
-# Maps sys.platform to what we actually want to call them.
-PLATFORM_MAPPING = {
-    'cygwin': 'win',
-    'darwin': 'mac',
-    'linux': 'linux',  # Python 3.3+.
-    'win32': 'win',
-    'aix6': 'aix',
-    'aix7': 'aix',
-    'zos': 'zos',
-}
-
-
-def GetNormalizedPlatform():
-    """Returns the result of sys.platform accounting for cygwin.
-    Under cygwin, this will always return "win32" like the native Python."""
-    if sys.platform == 'cygwin':
-        return 'win32'
-    return sys.platform
-
-
-# Common utilities
-class Gsutil(object):
-    """Call gsutil with some predefined settings.  This is a convenience object,
-    and is also immutable.
-
-    HACK: This object is used directly by the external script
-        `<depot_tools>/win_toolchain/get_toolchain_if_necessary.py`
-    """
-
-    MAX_TRIES = 5
-    RETRY_BASE_DELAY = 5.0
-    RETRY_DELAY_MULTIPLE = 1.3
-    VPYTHON3 = ('vpython3.bat'
-                if GetNormalizedPlatform() == 'win32' else 'vpython3')
-
-    def __init__(self, path, boto_path=None):
-        if not os.path.exists(path):
-            raise FileNotFoundError('GSUtil not found in %s' % path)
-        self.path = path
-        self.boto_path = boto_path
-
-    def get_sub_env(self):
-        env = os.environ.copy()
-        if self.boto_path == os.devnull:
-            env['AWS_CREDENTIAL_FILE'] = ''
-            env['BOTO_CONFIG'] = ''
-        elif self.boto_path:
-            env['AWS_CREDENTIAL_FILE'] = self.boto_path
-            env['BOTO_CONFIG'] = self.boto_path
-
-        if PLATFORM_MAPPING[sys.platform] != 'win':
-            env.update((x, "/tmp") for x in _TEMPDIR_ENV_VARS)
-
-        return env
-
-    def call(self, *args):
-        cmd = [self.VPYTHON3, self.path]
-        cmd.extend(args)
-        return subprocess2.call(cmd, env=self.get_sub_env())
-
-    def check_call(self, *args):
-        cmd = [self.VPYTHON3, self.path]
-        cmd.extend(args)
-        ((out, err), code) = subprocess2.communicate(cmd,
-                                                     stdout=subprocess2.PIPE,
-                                                     stderr=subprocess2.PIPE,
-                                                     env=self.get_sub_env())
-
-        out = out.decode('utf-8', 'replace')
-        err = err.decode('utf-8', 'replace')
-
-        # Parse output.
-        status_code_match = re.search('status=([0-9]+)', err)
-        if status_code_match:
-            return (int(status_code_match.group(1)), out, err)
-        if ('ServiceException: 401 Anonymous' in err):
-            return (401, out, err)
-        if ('You are attempting to access protected data with '
-                'no configured credentials.' in err):
-            return (403, out, err)
-        if 'matched no objects' in err or 'No URLs matched' in err:
-            return (404, out, err)
-        return (code, out, err)
-
-    def check_call_with_retries(self, *args):
-        delay = self.RETRY_BASE_DELAY
-        for i in range(self.MAX_TRIES):
-            code, out, err = self.check_call(*args)
-            if not code or i == self.MAX_TRIES - 1:
-                break
-
-            time.sleep(delay)
-            delay *= self.RETRY_DELAY_MULTIPLE
-
-        return code, out, err
-
-
-def main(args):
-    parser = optparse.OptionParser()
-    parser.add_option('-b',
-                      '--bucket',
-                      help='Google Storage bucket to fetch from.')
-    parser.add_option('-p', '--file', help='Path of file to fetch.')
-    parser.add_option('-o',
-                      '--output',
-                      help='Path where GCS contents should be downloaded.')
-    parser.add_option('-e', '--boto', help='Specify a custom boto file.')
-    (options, args) = parser.parse_args()
-
-    file_url = 'gs://%s/%s' % (options.bucket, options.file)
-
-    # Make sure gsutil exists where we expect it to.
-    if os.path.exists(GSUTIL_DEFAULT_PATH):
-        gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto)
-    else:
-        parser.error('gsutil not found in %s, bad depot_tools checkout?' %
-                     GSUTIL_DEFAULT_PATH)
-
-    gsutil.check_call('cp', file_url, options.output)
-
-
-if __name__ == '__main__':
-    sys.exit(main(sys.argv))

download_from_google_storage.py

@@ -18,7 +18,25 @@ import threading
 import time
 
 import subprocess2
-import call_google_storage
+
+# Env vars that tempdir can be gotten from; minimally, this
+# needs to match python's tempfile module and match normal
+# unix standards.
+_TEMPDIR_ENV_VARS = ('TMPDIR', 'TEMP', 'TMP')
+
+GSUTIL_DEFAULT_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                                   'gsutil.py')
+# Maps sys.platform to what we actually want to call them.
+PLATFORM_MAPPING = {
+    'cygwin': 'win',
+    'darwin': 'mac',
+    'linux': 'linux',  # Python 3.3+.
+    'linux2': 'linux',  # Python < 3.3 uses "linux2" / "linux3".
+    'win32': 'win',
+    'aix6': 'aix',
+    'aix7': 'aix',
+    'zos': 'zos',
+}
 
 
 class InvalidFileError(IOError):
@@ -29,6 +47,91 @@ class InvalidPlatformError(Exception):
     pass
 
 
+def GetNormalizedPlatform():
+    """Returns the result of sys.platform accounting for cygwin.
+    Under cygwin, this will always return "win32" like the native Python."""
+    if sys.platform == 'cygwin':
+        return 'win32'
+    return sys.platform
+
+
+# Common utilities
+class Gsutil(object):
+    """Call gsutil with some predefined settings.  This is a convenience object,
+    and is also immutable.
+
+    HACK: This object is used directly by the external script
+        `<depot_tools>/win_toolchain/get_toolchain_if_necessary.py`
+    """
+
+    MAX_TRIES = 5
+    RETRY_BASE_DELAY = 5.0
+    RETRY_DELAY_MULTIPLE = 1.3
+    VPYTHON3 = ('vpython3.bat'
+                if GetNormalizedPlatform() == 'win32' else 'vpython3')
+
+    def __init__(self, path, boto_path=None):
+        if not os.path.exists(path):
+            raise FileNotFoundError('GSUtil not found in %s' % path)
+        self.path = path
+        self.boto_path = boto_path
+
+    def get_sub_env(self):
+        env = os.environ.copy()
+        if self.boto_path == os.devnull:
+            env['AWS_CREDENTIAL_FILE'] = ''
+            env['BOTO_CONFIG'] = ''
+        elif self.boto_path:
+            env['AWS_CREDENTIAL_FILE'] = self.boto_path
+            env['BOTO_CONFIG'] = self.boto_path
+
+        if PLATFORM_MAPPING[sys.platform] != 'win':
+            env.update((x, "/tmp") for x in _TEMPDIR_ENV_VARS)
+
+        return env
+
+    def call(self, *args):
+        cmd = [self.VPYTHON3, self.path]
+        cmd.extend(args)
+        return subprocess2.call(cmd, env=self.get_sub_env())
+
+    def check_call(self, *args):
+        cmd = [self.VPYTHON3, self.path]
+        cmd.extend(args)
+        ((out, err), code) = subprocess2.communicate(cmd,
+                                                     stdout=subprocess2.PIPE,
+                                                     stderr=subprocess2.PIPE,
+                                                     env=self.get_sub_env())
+
+        out = out.decode('utf-8', 'replace')
+        err = err.decode('utf-8', 'replace')
+
+        # Parse output.
+        status_code_match = re.search('status=([0-9]+)', err)
+        if status_code_match:
+            return (int(status_code_match.group(1)), out, err)
+        if ('ServiceException: 401 Anonymous' in err):
+            return (401, out, err)
+        if ('You are attempting to access protected data with '
+                'no configured credentials.' in err):
+            return (403, out, err)
+        if 'matched no objects' in err or 'No URLs matched' in err:
+            return (404, out, err)
+        return (code, out, err)
+
+    def check_call_with_retries(self, *args):
+        delay = self.RETRY_BASE_DELAY
+        for i in range(self.MAX_TRIES):
+            code, out, err = self.check_call(*args)
+            if not code or i == self.MAX_TRIES - 1:
+                break
+
+            time.sleep(delay)
+            delay *= self.RETRY_DELAY_MULTIPLE
+
+        return code, out, err
+
+
 def check_platform(target):
     """Checks if any parent directory of target matches (win|mac|linux)."""
     assert os.path.isabs(target)
@@ -526,12 +629,11 @@ def main(args):
         options.boto = os.environ.get('NO_AUTH_BOTO_CONFIG', os.devnull)
 
     # Make sure gsutil exists where we expect it to.
-    if os.path.exists(call_google_storage.GSUTIL_DEFAULT_PATH):
+    if os.path.exists(GSUTIL_DEFAULT_PATH):
-        gsutil = Gsutil(call_google_storage.GSUTIL_DEFAULT_PATH,
+        gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto)
-                        boto_path=options.boto)
     else:
         parser.error('gsutil not found in %s, bad depot_tools checkout?' %
-                     call_google_storage.GSUTIL_DEFAULT_PATH)
+                     GSUTIL_DEFAULT_PATH)
 
     # Passing in -g/--config will run our copy of GSUtil, then quit.
     if options.config:

gclient.py

@@ -82,7 +82,6 @@
 __version__ = '0.7'
 
 import copy
-import hashlib
 import json
 import logging
 import optparse
@@ -92,9 +91,6 @@ import posixpath
 import pprint
 import re
 import sys
-import shutil
-import tarfile
-import tempfile
 import time
 import urllib.parse
 
@@ -752,17 +748,6 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
                                        should_process=should_process,
                                        relative=use_relative_paths,
                                        condition=condition))
-            elif dep_type == 'gcs':
-                deps_to_add.append(
-                    GcsDependency(parent=self,
-                                  name=name,
-                                  bucket=dep_value['bucket'],
-                                  object_name=dep_value['object_name'],
-                                  sha256sum=dep_value['sha256sum'],
-                                  custom_vars=self.custom_vars,
-                                  should_process=should_process,
-                                  relative=use_relative_paths,
-                                  condition=condition))
             else:
                 url = dep_value.get('url')
                 deps_to_add.append(
@@ -2498,162 +2483,6 @@ it or fix the checkout.
         return self._enforced_cpu
 
 
-class GcsDependency(Dependency):
-    """A Dependency object that represents a single GCS bucket and object"""
-
-    def __init__(self, parent, name, bucket, object_name, sha256sum,
-                 custom_vars, should_process, relative, condition):
-        self.bucket = bucket
-        self.object_name = object_name
-        self.sha256sum = sha256sum
-        url = 'gs://{bucket}/{object_name}'.format(
-            bucket=self.bucket,
-            object_name=self.object_name,
-        )
-        super(GcsDependency, self).__init__(parent=parent,
-                                            name=name,
-                                            url=url,
-                                            managed=None,
-                                            custom_deps=None,
-                                            custom_vars=custom_vars,
-                                            custom_hooks=None,
-                                            deps_file=None,
-                                            should_process=should_process,
-                                            should_recurse=False,
-                                            relative=relative,
-                                            condition=condition)
-
-    #override
-    def run(self, revision_overrides, command, args, work_queue, options,
-            patch_refs, target_branches, skip_sync_revisions):
-        """Downloads GCS package."""
-        logging.info('GcsDependency(%s).run()' % self.name)
-        if not self.should_process:
-            return
-        self.DownloadGoogleStorage()
-        super(GcsDependency,
-              self).run(revision_overrides, command, args, work_queue, options,
-                        patch_refs, target_branches, skip_sync_revisions)
-
-    def WriteFilenameHash(self, sha1, hash_file):
-        with open(hash_file, 'w') as f:
-            f.write(sha1)
-            f.write('\n')
-
-    def IsDownloadNeeded(self, output_dir, output_file):
-        """Check if download and extract is needed."""
-        download_needed = False
-        if not os.path.exists(output_file):
-            download_needed = True
-
-        hash_file = os.path.join(output_dir, 'hash')
-        existing_hash = None
-        if os.path.exists(hash_file):
-            try:
-                with open(hash_file, 'r') as f:
-                    existing_hash = f.read().rstrip()
-            except IOError:
-                download_needed = True
-        else:
-            download_needed = True
-
-        if existing_hash != self.sha256sum:
-            download_needed = True
-        return download_needed
-
-    def GetSha256Sum(self, filename):
-        sha = hashlib.sha256()
-        with open(filename, 'rb') as f:
-            while True:
-                # Read in 1mb chunks, so it doesn't all have to be loaded into
-                # memory.
-                chunk = f.read(1024 * 1024)
-                if not chunk:
-                    break
-                sha.update(chunk)
-        return sha.hexdigest()
-
-    def DownloadGoogleStorage(self):
-        """Calls call_google_storage.py script."""
-        gcs_file_name = self.object_name.split('/')[-1]
-        root_dir = self.root.root_dir
-
-        # Directory of the extracted tarfile contents
-        output_dir = os.path.join(root_dir, self.name)
-        output_file = os.path.join(output_dir, gcs_file_name)
-
-        if not self.IsDownloadNeeded(output_dir, output_file):
-            return
-
-        # Remove hashfile
-        hash_file = os.path.join(output_dir, 'hash')
-        if os.path.exists(hash_file):
-            os.remove(hash_file)
-
-        # Remove tarfile
-        if os.path.exists(output_file):
-            os.remove(output_file)
-
-        # Remove extracted contents
-        if os.path.exists(output_dir):
-            shutil.rmtree(output_dir)
-        os.makedirs(output_dir)
-
-        call_google_storage_path = os.path.join(DEPOT_TOOLS_DIR,
-                                                'call_google_storage.py')
-        cmd = [
-            'vpython3',
-            call_google_storage_path,
-            '--bucket',
-            self.bucket,
-            '--file',
-            self.object_name,
-            '--output',
-            output_file,
-        ]
-        if os.getenv('GCLIENT_TEST') == '1':
-            # Create fake tar file and extracted tar contents
-            tmpdir = tempfile.mkdtemp()
-            copy_dir = os.path.join(tmpdir, self.name, 'extracted_dir')
-            if os.path.exists(copy_dir):
-                shutil.rmtree(copy_dir)
-            os.makedirs(copy_dir)
-            with open(os.path.join(copy_dir, 'extracted_file'), 'w+') as f:
-                f.write('extracted text')
-            with tarfile.open(output_file, "w:gz") as tar:
-                tar.add(copy_dir, arcname=os.path.basename(copy_dir))
-        else:
-            subprocess2.call(cmd)
-
-        calculated_sha256sum = ''
-        if os.getenv('GCLIENT_TEST') == '1':
-            calculated_sha256sum = 'abcd123'
-        else:
-            calculated_sha256sum = self.GetSha256Sum(output_file)
-
-        if calculated_sha256sum != self.sha256sum:
-            raise Exception('sha256sum does not match calculated hash. '
-                            '{original} vs {calculated}'.format(
-                                original=self.sha256sum,
-                                calculated=calculated_sha256sum,
-                            ))
-
-        with tarfile.open(output_file, 'r:*') as tar:
-            tar.extractall(path=output_dir)
-        self.WriteFilenameHash(calculated_sha256sum, hash_file)
-
-    #override
-    def GetScmName(self):
-        """Always 'gcs'."""
-        return 'gcs'
-
-    #override
-    def CreateSCM(self, out_cb=None):
-        """Create a Wrapper instance suitable for handling this GCS dependency."""
-        return gclient_scm.GcsWrapper(self.url, self.root.root_dir, self.name,
-                                      self.outbuf, out_cb)
-
-
 class CipdDependency(Dependency):
     """A Dependency object that represents a single CIPD package."""
     def __init__(self, parent, name, dep_value, cipd_root, custom_vars,

gclient_eval.py

@@ -131,14 +131,6 @@ _GCLIENT_DEPS_SCHEMA = _NodeDictSchema({
             schema.Optional('dep_type', default='cipd'):
             str,
         }),
-        # GCS content.
-        _NodeDictSchema({
-            'bucket': str,
-            'object_name': str,
-            'sha256sum': str,
-            schema.Optional('condition'): str,
-            schema.Optional('dep_type', default='gcs'): str,
-        }),
     ),
 })
 

gclient_scm.py

@@ -1913,57 +1913,6 @@ class CipdWrapper(SCMWrapper):
     """
 
 
-class GcsWrapper(SCMWrapper):
-    """Wrapper for GCS.
-
-  Currently only supports content from Google Cloud Storage.
-  """
-    name = 'gcs'
-
-    def __init__(self,
-                 url=None,
-                 root_dir=None,
-                 relpath=None,
-                 out_fh=None,
-                 out_cb=None):
-        super(GcsWrapper, self).__init__(url=url,
-                                         root_dir=root_dir,
-                                         relpath=relpath,
-                                         out_fh=out_fh,
-                                         out_cb=out_cb)
-
-    #override
-    def GetCacheMirror(self):
-        return None
-
-    #override
-    def GetActualRemoteURL(self, options):
-        return None
-
-    #override
-    def DoesRemoteURLMatch(self, options):
-        del options
-        return True
-
-    def revert(self, options, args, file_list):
-        """Does nothing."""
-
-    def diff(self, options, args, file_list):
-        """GCS has no notion of diffing."""
-
-    def pack(self, options, args, file_list):
-        """GCS has no notion of diffing."""
-
-    def revinfo(self, options, args, file_list):
-        """Does nothing"""
-
-    def status(self, options, args, file_list):
-        pass
-
-    def update(self, options, args, file_list):
-        """Does nothing."""
-
-
 class CogWrapper(SCMWrapper):
     """Wrapper for Cog, all no-op."""
     name = 'cog'

git_cache.py

@@ -16,7 +16,7 @@ import threading
 import time
 import urllib.parse
 
-from call_google_storage import Gsutil
+from download_from_google_storage import Gsutil
 import gclient_utils
 import lockfile
 import metrics

testing_support/fake_repos.py

@@ -16,7 +16,6 @@ import random
 import re
 import socket
 import sys
-import tarfile
 import tempfile
 import textwrap
 import time
@@ -48,9 +47,6 @@ def read_tree(tree_root):
         for f in [join(root, f) for f in files if not f.startswith('.')]:
             filepath = f[len(tree_root) + 1:].replace(os.sep, '/')
             assert len(filepath) > 0, f
-            if tarfile.is_tarfile(join(root, f)):
-                tree[filepath] = 'tarfile'
-                continue
             with io.open(join(root, f), encoding='utf-8') as f:
                 tree[filepath] = f.read()
     return tree
@@ -214,7 +210,7 @@ class FakeReposBase(object):
 
 class FakeRepos(FakeReposBase):
     """Implements populateGit()."""
-    NB_GIT_REPOS = 23
+    NB_GIT_REPOS = 21
 
     def populateGit(self):
         # Testing:
@@ -885,63 +881,6 @@ deps = {
             },
         )
 
-        self._commit_git(
-            'repo_22', {
-                'DEPS':
-                textwrap.dedent("""\
-        vars = {}
-        deps = {
-          'src/gcs_dep': {
-            'bucket': '123bucket',
-            'object_name': 'deadbeef',
-            'dep_type': 'gcs',
-            'sha256sum': 'abcd123',
-          },
-          'src/another_gcs_dep': {
-            'bucket': '456bucket',
-            'object_name': 'Linux/llvmfile.tar.gz',
-            'dep_type': 'gcs',
-            'sha256sum': 'abcd123',
-          },
-        }"""),
-                'origin':
-                'git/repo_22@1\n'
-            })
-
-        self._commit_git(
-            'repo_23', {
-                'DEPS': """
-deps = {
-  'src/repo12': '/repo_12',
-}""",
-                'origin': 'git/repo_23@1\n',
-            })
-
-        self._commit_git(
-            'repo_23', {
-                'DEPS': """
-deps = {
-  'src/repo12': '/repo_12@refs/changes/1212',
-}""",
-                'origin': 'git/repo_23@2\n',
-            })
-
-        # src/repo12 is now a GCS dependency.
-        self._commit_git(
-            'repo_23', {
-                'DEPS': """
-deps = {
-  'src/repo12': {
-    'bucket': 'bucket123',
-    'object_name': 'path_to_file.tar.gz',
-    'dep_type': 'gcs',
-    'sha256sum': 'abcd123',
-  },
-}
-""",
-                'origin': 'git/repo_23@3\n'
-            })
-
 
 class FakeRepoSkiaDEPS(FakeReposBase):
     """Simulates the Skia DEPS transition in Chrome."""

tests/download_from_google_storage_unittest.py

@@ -18,7 +18,6 @@ import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-import call_google_storage
 import upload_to_google_storage
 import download_from_google_storage
 
@@ -152,7 +151,7 @@ class GstoolsUnitTests(unittest.TestCase):
 
     def test_gsutil(self):
         # This will download a real gsutil package from Google Storage.
-        gsutil = call_google_storage.Gsutil(GSUTIL_DEFAULT_PATH, None)
+        gsutil = download_from_google_storage.Gsutil(GSUTIL_DEFAULT_PATH, None)
         self.assertEqual(gsutil.path, GSUTIL_DEFAULT_PATH)
         code, _, err = gsutil.check_call()
         self.assertEqual(code, 0, err)

tests/gclient_gcs_smoketest.py

@@ -1,140 +0,0 @@
-#!/usr/bin/env vpython3
-# Copyright (c) 2024 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-"""Smoke tests for gclient.py.
-
-Shell out 'gclient' and run gcs tests.
-"""
-
-import logging
-import os
-import sys
-import unittest
-
-from unittest import mock
-import gclient_smoketest_base
-import subprocess2
-
-ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-
-
-class GClientSmokeGcs(gclient_smoketest_base.GClientSmokeBase):
-
-    def setUp(self):
-        super(GClientSmokeGcs, self).setUp()
-        self.enabled = self.FAKE_REPOS.set_up_git()
-        if not self.enabled:
-            self.skipTest('git fake repos not available')
-        self.env['PATH'] = (os.path.join(ROOT_DIR, 'testing_support') +
-                            os.pathsep + self.env['PATH'])
-
-    def testSyncGcs(self):
-        self.gclient(['config', self.git_base + 'repo_22', '--name', 'src'])
-        self.gclient(['sync'])
-
-        tree = self.mangle_git_tree(('repo_22@1', 'src'))
-        tree.update({
-            'src/another_gcs_dep/hash':
-            'abcd123\n',
-            'src/another_gcs_dep/llvmfile.tar.gz':
-            'tarfile',
-            'src/another_gcs_dep/extracted_dir/extracted_file':
-            'extracted text',
-            'src/gcs_dep/deadbeef':
-            'tarfile',
-            'src/gcs_dep/hash':
-            'abcd123\n',
-            'src/gcs_dep/extracted_dir/extracted_file':
-            'extracted text',
-        })
-        self.assertTree(tree)
-
-    def testConvertGitToGcs(self):
-        self.gclient(['config', self.git_base + 'repo_23', '--name', 'src'])
-
-        # repo_13@1 has src/repo12 as a git dependency.
-        self.gclient([
-            'sync', '-v', '-v', '-v', '--revision',
-            self.githash('repo_23', 1)
-        ])
-
-        tree = self.mangle_git_tree(('repo_23@1', 'src'),
-                                    ('repo_12@1', 'src/repo12'))
-        self.assertTree(tree)
-
-        # repo_23@3 has src/repo12 as a gcs dependency.
-        self.gclient([
-            'sync', '-v', '-v', '-v', '--revision',
-            self.githash('repo_23', 3), '--delete_unversioned_trees'
-        ])
-
-        tree = self.mangle_git_tree(('repo_23@3', 'src'))
-        tree.update({
-            'src/repo12/extracted_dir/extracted_file': 'extracted text',
-            'src/repo12/hash': 'abcd123\n',
-            'src/repo12/path_to_file.tar.gz': 'tarfile',
-        })
-        self.assertTree(tree)
-
-    def testConvertGcsToGit(self):
-        self.gclient(['config', self.git_base + 'repo_23', '--name', 'src'])
-
-        # repo_13@3 has src/repo12 as a cipd dependency.
-        self.gclient([
-            'sync', '-v', '-v', '-v', '--revision',
-            self.githash('repo_23', 3), '--delete_unversioned_trees'
-        ])
-
-        tree = self.mangle_git_tree(('repo_23@3', 'src'))
-        tree.update({
-            'src/repo12/extracted_dir/extracted_file': 'extracted text',
-            'src/repo12/hash': 'abcd123\n',
-            'src/repo12/path_to_file.tar.gz': 'tarfile',
-        })
-        self.assertTree(tree)
-
-        # repo_23@1 has src/repo12 as a git dependency.
-        self.gclient([
-            'sync', '-v', '-v', '-v', '--revision',
-            self.githash('repo_23', 1)
-        ])
-
-        tree = self.mangle_git_tree(('repo_23@1', 'src'),
-                                    ('repo_12@1', 'src/repo12'))
-        tree.update({
-            'src/repo12/extracted_dir/extracted_file': 'extracted text',
-            'src/repo12/hash': 'abcd123\n',
-            'src/repo12/path_to_file.tar.gz': 'tarfile',
-        })
-        self.assertTree(tree)
-
-    def testRevInfo(self):
-        self.gclient(['config', self.git_base + 'repo_22', '--name', 'src'])
-        self.gclient(['sync'])
-        results = self.gclient(['revinfo'])
-        out = ('src: %(base)srepo_22\n'
-               'src/another_gcs_dep: gs://456bucket/Linux/llvmfile.tar.gz\n'
-               'src/gcs_dep: gs://123bucket/deadbeef\n' % {
-                   'base': self.git_base,
-               })
-        self.check((out, '', 0), results)
-
-    def testRevInfoActual(self):
-        self.gclient(['config', self.git_base + 'repo_22', '--name', 'src'])
-        self.gclient(['sync'])
-        results = self.gclient(['revinfo', '--actual'])
-        out = (
-            'src: %(base)srepo_22@%(hash1)s\n'
-            'src/another_gcs_dep: gs://456bucket/Linux/llvmfile.tar.gz@None\n'
-            'src/gcs_dep: gs://123bucket/deadbeef@None\n' % {
-                'base': self.git_base,
-                'hash1': self.githash('repo_22', 1),
-            })
-        self.check((out, '', 0), results)
-
-
-if __name__ == '__main__':
-    if '-v' in sys.argv:
-        logging.basicConfig(level=logging.DEBUG)
-    unittest.main()

tests/gclient_scm_test.py

@@ -1022,33 +1022,6 @@ class CipdWrapperTestCase(unittest.TestCase):
         scm.update(None, (), [])
 
 
-class GcsWrapperTestCase(unittest.TestCase):
-
-    def setUp(self):
-        self.workdir = tempfile.mkdtemp()
-        self.url = 'gs://123bucket/path_to_tar.gz'
-
-    def createScm(self):
-        return gclient_scm.GcsWrapper(url=self.url,
-                                      root_dir=self.workdir,
-                                      relpath='fake_relpath')
-
-    def testRevert(self):
-        """Checks that revert does nothing."""
-        scm = self.createScm()
-        scm.revert(None, (), [])
-
-    def testRevinfo(self):
-        """Checks that revinfo does nothing."""
-        scm = self.createScm()
-        scm.revinfo(None, (), [])
-
-    def testUpdate(self):
-        """Checks that update does nothing."""
-        scm = self.createScm()
-        scm.update(None, (), [])
-
-
 class BranchHeadsFakeRepo(fake_repos.FakeReposBase):
     def populateGit(self):
         # Creates a tree that looks like this:

upload_to_google_storage.py

@@ -16,10 +16,10 @@ import tarfile
 import threading
 import time
 
-from call_google_storage import Gsutil
-from call_google_storage import GSUTIL_DEFAULT_PATH
 from download_from_google_storage import get_sha1
+from download_from_google_storage import Gsutil
 from download_from_google_storage import PrinterThread
+from download_from_google_storage import GSUTIL_DEFAULT_PATH
 
 USAGE_STRING = """%prog [options] target [target2 ...].
 Target is the file intended to be uploaded to Google Storage.

win_toolchain/get_toolchain_if_necessary.py

@@ -60,7 +60,7 @@ BASEDIR = os.path.dirname(os.path.abspath(__file__))
 DEPOT_TOOLS_PATH = os.path.join(BASEDIR, '..')
 sys.path.append(DEPOT_TOOLS_PATH)
 try:
-    import call_google_storage
+    import download_from_google_storage
 except ImportError:
     # Allow use of utility functions in this script from package_from_installed
     # on bare VM that doesn't have a full depot_tools.
@@ -255,8 +255,8 @@ def LooksLikeGoogler():
 
 def CanAccessToolchainBucket():
     """Checks whether the user has access to gs://chrome-wintoolchain/."""
-    gsutil = call_google_storage.Gsutil(call_google_storage.GSUTIL_DEFAULT_PATH,
+    gsutil = download_from_google_storage.Gsutil(
-                                        boto_path=None)
+        download_from_google_storage.GSUTIL_DEFAULT_PATH, boto_path=None)
     code, stdout, stderr = gsutil.check_call('ls', 'gs://chrome-wintoolchain/')
     if code != 0:
         # Make sure any error messages are made visible to the user.
@@ -294,7 +294,7 @@ def RequestGsAuthentication():
     print('I\'m sorry for the hassle, but you need to do a one-time manual')
     print('authentication. Please run:')
     print()
-    print('    call_google_storage --config')
+    print('    download_from_google_storage --config')
     print()
     print('and follow the instructions.')
     print()
@@ -342,8 +342,8 @@ def DownloadUsingGsutil(filename):
     temp_dir = tempfile.mkdtemp()
     assert os.path.basename(filename) == filename
     target_path = os.path.join(temp_dir, filename)
-    gsutil = call_google_storage.Gsutil(call_google_storage.GSUTIL_DEFAULT_PATH,
+    gsutil = download_from_google_storage.Gsutil(
-                                        boto_path=None)
+        download_from_google_storage.GSUTIL_DEFAULT_PATH, boto_path=None)
     code = gsutil.call('cp', 'gs://chrome-wintoolchain/' + filename,
                        target_path)
     if code != 0: