Add is_first_class_gcs migration file

If is_first_class_gcs file exists with contents '1'
- GCS hooks will download content and then overwrite the file with '0'
- First class GCS deps will download as necessary by checking the hash
file.

If there is no is_first_class_gcs file or its content is '0' for a GCS hook:
- Hook will run normally and, if content is downloaded, an is_first_class_gcs
file will be written with '0'

If there is no is_first_class_gcs file or its content is '0' for a first
class gcs dep:
- Content will be downloaded and is_first_class_gcs will be set to '1'

Verified locally with migrating and unmigrating a GCS hook.
TODO: update other GCS hook scripts in chromium/src to check migration
file.

Bug: b/324418194
Change-Id: Ida8541cb70839b86e99115bcabfc38bc87e40136
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5368250
Reviewed-by: Joanna Wang <jojwang@chromium.org>
Commit-Queue: Stephanie Kim <kimstephanie@google.com>

download_from_google_storage.py

@@ -38,6 +38,9 @@ PLATFORM_MAPPING = {
     'zos': 'zos',
 }
 
+# (b/328065301): Remove when all GCS hooks are migrated to first class deps
+MIGRATION_TOGGLE_FILE_NAME = 'is_first_class_gcs'
+
 
 class InvalidFileError(IOError):
     pass
@@ -255,6 +258,12 @@ def _downloader_worker_thread(thread_num,
         input_sha1_sum, output_filename = q.get()
         if input_sha1_sum is None:
             return
+        working_dir = os.path.dirname(output_filename)
+        if not working_dir:
+            raise Exception(
+                'Unable to construct a working_dir from the output_filename.')
+        migration_file_name = os.path.join(working_dir,
+                                           MIGRATION_TOGGLE_FILE_NAME)
         extract_dir = None
         if extract:
             if not output_filename.endswith('.tar.gz'):
@@ -284,6 +293,14 @@ def _downloader_worker_thread(thread_num,
                               're-downloading...' %
                               (thread_num, output_filename))
                     skip = False
+            # (b/328065301): Remove when all GCS hooks are migrated to first
+            # class deps
+            # If the directory was created by a first class GCS
+            # dep, remove the migration file and re-download using the
+            # latest hook.
+            is_first_class_gcs = os.path.exists(migration_file_name)
+            if is_first_class_gcs:
+                skip = False
             if skip:
                 continue
 
@@ -371,6 +388,8 @@ def _downloader_worker_thread(thread_num,
                 with open(extract_dir + '.tmp', 'a'):
                     tar.extractall(path=dirname)
                 os.remove(extract_dir + '.tmp')
+        if os.path.exists(migration_file_name):
+            os.remove(migration_file_name)
         # Set executable bit.
         if sys.platform == 'cygwin':
             # Under cygwin, mark all files as executable. The executable flag in
@@ -448,7 +467,23 @@ def download_from_google_storage(input_filename, base_url, gsutil, num_threads,
 
     # Sequentially check for the most common case and see if we can bail out
     # early before making any slow calls to gsutil.
-    if not force and all(
+    if directory:
+        working_dir = input_filename
+    elif os.path.dirname(output):
+        working_dir = os.path.dirname(output)
+
+    if not working_dir:
+        raise Exception(
+            'Unable to construct a working_dir from the inputted directory'
+            ' or sha1 file name.')
+
+    # (b/328065301): Remove when all GCS hooks are migrated to first class deps
+    # If the directory was created by a first class GCS
+    # dep, remove the migration file and re-download using the
+    # latest hook.
+    migration_file = os.path.join(working_dir, MIGRATION_TOGGLE_FILE_NAME)
+    is_first_class_gcs = os.path.exists(migration_file)
+    if not force and not is_first_class_gcs and all(
             _data_exists(sha1, path, extract) for sha1, path in input_data):
         return 0