Update roll_downstream_gcs_deps to take a list of source and destination packages.

This is needed to support rolling multiple packages at once when package
names don't match.

Change-Id: I20bb5edfee87bb3e089a2c25f597e4778e0fe60f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5867086
Reviewed-by: Stephanie Kim <kimstephanie@google.com>
Auto-Submit: Josip Sokcevic <sokcevic@chromium.org>
Commit-Queue: Stephanie Kim <kimstephanie@google.com>

roll_downstream_gcs_deps.py

@@ -138,7 +138,9 @@ def _replace_ast(destination: str, dest_ast: ast.Module, source: str,
     return result
 
 
-def copy_packages(source: str, destination: str, packages: List[str]) -> str:
+def copy_packages(source_content: str, destination_content: str,
+                  source_packages: List[str],
+                  destination_packages: List[str]) -> str:
     """Copies GCS packages from source to destination.
 
     Args:
@@ -149,49 +151,65 @@ def copy_packages(source: str, destination: str, packages: List[str]) -> str:
     Returns:
       Destination DEPS file content with packages copied.
     """
-    source_deps = _get_deps(ast.parse(source, mode='exec'))
+    source_deps = _get_deps(ast.parse(source_content, mode='exec'))
-    for package in packages:
+    for i in range(len(source_packages)):
-        if package not in source_deps:
+        source_package = source_packages[i]
-            raise Exception('Package %s not found in source' % package)
+        destination_package = destination_packages[i]
-        dest_deps = _get_deps(ast.parse(destination, mode='exec'))
+        if source_package not in source_deps:
-        if package not in dest_deps:
+            raise Exception('Package %s not found in source' % source_package)
-            raise Exception('Package %s not found in destination' % package)
+        dest_deps = _get_deps(ast.parse(destination_content, mode='exec'))
-        destination = _replace_ast(
+        if destination_package not in dest_deps:
-            destination, _get_gcs_object_list_ast(dest_deps[package]), source,
+            raise Exception('Package %s not found in destination' %
-            _get_gcs_object_list_ast(source_deps[package]))
+                            destination_package)
+        destination_content = _replace_ast(
+            destination_content,
+            _get_gcs_object_list_ast(dest_deps[destination_package]),
+            source_content,
+            _get_gcs_object_list_ast(source_deps[source_package]))
 
-    return destination
+    return destination_content
 
 
 def main():
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument('--source',
+    parser.add_argument('--source-deps',
                         required=True,
                         help='Source DEPS file where content will be copied '
                         'from')
-    parser.add_argument('--package',
+    parser.add_argument('--source-package',
                         action='append',
                         required=True,
                         help='List of DEPS packages to update')
-    parser.add_argument('--destination',
+    parser.add_argument('--destination-deps',
                         required=True,
                         help='Destination DEPS file, where content will be '
                         'saved')
+    parser.add_argument('--destination-package',
+                        action='append',
+                        required=True,
+                        help='List of DEPS packages to update')
     args = parser.parse_args()
 
-    if not args.package:
+    if not args.source_package:
-        parser.error('No packages specified to roll, aborting...')
+        parser.error('No source packages specified to roll, aborting...')
 
-    with open(args.source) as f:
+    if not args.destination_package:
+        parser.error('No destination packages specified to roll, aborting...')
+
+    if len(args.destination_package) != len(args.source_package):
+        parser.error('Source and destination packages must be of the same '
+                     'length, aborting...')
+
+    with open(args.source_deps) as f:
         source_content = f.read()
 
-    with open(args.destination) as f:
+    with open(args.destination_deps) as f:
         destination_content = f.read()
 
     new_content = copy_packages(source_content, destination_content,
-                                args.package)
+                                args.source_package, args.destination_package)
 
-    with open(args.destination, 'w') as f:
+    with open(args.destination_deps, 'w') as f:
         f.write(new_content)
 
     print('Run:')

tests/roll_downstream_gcs_deps_test.py

@@ -18,16 +18,17 @@ class CopyPackageTest(unittest.TestCase):
 
     def testNoDepsToRoll(self):
         with self.assertRaises(Exception):
-            roll_downstream_gcs_deps.copy_packages('', '', ['foo'])
+            roll_downstream_gcs_deps.copy_packages('', '', ['foo'], ['foo'])
 
         with self.assertRaises(Exception):
             roll_downstream_gcs_deps.copy_packages('deps = {"foo": ""}',
-                                                   'deps = {}', ['foo'])
+                                                   'deps = {}', ['foo'],
+                                                   ['foo'])
 
         with self.assertRaises(Exception):
             roll_downstream_gcs_deps.copy_packages('deps = {"foo": ""}',
                                                    'deps = {"bar": ""}',
-                                                   ['foo'])
+                                                   ['foo'], ['baz'])
 
     def testNoGCSDeps(self):
         source_deps = '''
@@ -62,7 +63,8 @@ deps = {
 '''
         with self.assertRaises(Exception):
             roll_downstream_gcs_deps.copy_packages(source_deps,
-                                                   destination_deps, ['foo'])
+                                                   destination_deps, ['foo'],
+                                                   ['foo'])
 
     def testObjectInLineUpdate(self):
         source_deps = '''
@@ -125,7 +127,7 @@ deps = {
 '''
         result = roll_downstream_gcs_deps.copy_packages(source_deps,
                                                         destination_deps,
-                                                        ['foo'])
+                                                        ['foo'], ['foo'])
         self.assertEqual(result, expected_deps)
 
     def testGCSRustPackageNewPlatform(self):
@@ -169,7 +171,7 @@ deps = {
 '''
         destination_deps = '''
 deps = {
-  'src/third_party/rust-toolchain': {
+  'third_party/rust-toolchain': {
     'dep_type': 'gcs',
     'bucket': 'chromium-browser-clang',
     'objects': [
@@ -183,10 +185,56 @@ deps = {
     ],
   },
 }
+'''
+        expected_deps = '''
+deps = {
+  'third_party/rust-toolchain': {
+    'dep_type': 'gcs',
+    'bucket': 'chromium-browser-clang',
+    'objects': [
+      {
+        'object_name': 'Linux_x64/rust-toolchain-595316b4006932405a63862d8fe65f71a6356293-3-llvmorg-20-init-1009-g7088a5ed.tar.xz',
+        'sha256sum': '560c02da5300f40441992ef639d83cee96cae3584c3d398704fdb2f02e475bbf',
+        'size_bytes': 152024840,
+        'generation': 1722663990116408,
+        'condition': 'host_os == "linux" and non_git_source',
+      },
+      {
+        'object_name': 'Mac/rust-toolchain-595316b4006932405a63862d8fe65f71a6356293-3-llvmorg-20-init-1009-g7088a5ed.tar.xz',
+        'sha256sum': '9f39154b4337438fd170e729ed2ae4c978b22f11708d683c28265bd096df17a5',
+        'size_bytes': 144459260,
+        'generation': 1722663991651609,
+        'condition': 'host_os == "mac" and host_cpu == "x64"',
+      },
+      {
+        'object_name': 'Mac_arm64/rust-toolchain-595316b4006932405a63862d8fe65f71a6356293-3-llvmorg-20-init-1009-g7088a5ed.tar.xz',
+        'sha256sum': '4b89cf125ffa39e8fc74f01ec3beeb632fd3069478d8c6cc4fcae506b4917151',
+        'size_bytes': 135571272,
+        'generation': 1722663993205996,
+        'condition': 'host_os == "mac" and host_cpu == "arm64"',
+      },
+      {
+        'object_name': 'Win/rust-toolchain-595316b4006932405a63862d8fe65f71a6356293-3-llvmorg-20-init-1009-g7088a5ed.tar.xz',
+        'sha256sum': '3f6a1a87695902062a6575632552b9f2cbbbcda1907fe3232f49b8ea29baecf5',
+        'size_bytes': 208844028,
+        'generation': 1722663994756449,
+        'condition': 'host_os == "win"',
+      },
+    ],
+  },
+}
 '''
         result = roll_downstream_gcs_deps.copy_packages(
-            source_deps, destination_deps, ['src/third_party/rust-toolchain'])
+            source_deps, destination_deps, ['src/third_party/rust-toolchain'],
-        self.assertEqual(result, source_deps)
+            ['third_party/rust-toolchain'])
+        self.assertEqual(result, expected_deps)
+
+        with self.assertRaises(Exception):
+            # no destination_package match, so expect a failure.
+            roll_downstream_gcs_deps.copy_packages(
+                source_deps, destination_deps,
+                ['src/third_party/rust-toolchain'],
+                ['src/third_party/rust-toolchain'])
 
 
 if __name__ == '__main__':