Add --use-submodules to update_checkout

Not all repositories in `update-checkout-config.json` are tagged. For contributors that would like to achieve reproducible builds, it's useful to be able to track exact state of checkouts with submodules. New `--use-submodules` flag is added to `update-checkout`, which uses `git submodule add` instead of `git clone` when checking out new repositories.

Author: MaxDesiatov

utils/update_checkout/update_checkout/update_checkout.py

@@ -368,7 +368,7 @@ def update_all_repositories(args, config, scheme_name, scheme_map, cross_repos_p
 
 def obtain_additional_swift_sources(pool_args):
     (args, repo_name, repo_info, repo_branch, remote, with_ssh, scheme_name,
-     skip_history, skip_tags, skip_repository_list) = pool_args
+     skip_history, skip_tags, skip_repository_list, use_submodules) = pool_args
 
     env = dict(os.environ)
     env.update({'GIT_TERMINAL_PROMPT': '0'})
@@ -383,6 +383,11 @@ def obtain_additional_swift_sources(pool_args):
                       (['--no-tags'] if skip_tags else []),
                       env=env,
                       echo=True)
+        elif use_submodules:
+            shell.run(['git', 'submodule', 'add', remote, repo_name] +
+                      (['--no-tags'] if skip_tags else []),
+                      env=env,
+                      echo=True)
         else:
             shell.run(['git', 'clone', '--recursive', remote, repo_name] +
                       (['--no-tags'] if skip_tags else []),
@@ -406,7 +411,7 @@ def obtain_additional_swift_sources(pool_args):
 
 def obtain_all_additional_swift_sources(args, config, with_ssh, scheme_name,
                                         skip_history, skip_tags,
-                                        skip_repository_list):
+                                        skip_repository_list, use_submodules):
 
     pool_args = []
     with shell.pushd(args.source_root, dry_run=False, echo=False):
@@ -416,7 +421,20 @@ def obtain_all_additional_swift_sources(args, config, with_ssh, scheme_name,
                       "user")
                 continue
 
-            if os.path.isdir(os.path.join(repo_name, ".git")):
+            if use_submodules:
+              repo_exists = False
+              submodules_status = shell.capture(['git', 'submodule', 'status'],
+                                                echo=False)
+              if submodules_status:
+                for line in submodules_status.splitlines():
+                  if line[0].endswith(repo_name):
+                    repo_exists = True
+                    break
+
+            else:
+              repo_exists = os.path.isdir(os.path.join(repo_name, ".git"))
+
+            if repo_exists:
                 print("Skipping clone of '" + repo_name + "', directory "
                       "already exists")
                 continue
@@ -450,16 +468,23 @@ def obtain_all_additional_swift_sources(args, config, with_ssh, scheme_name,
             if repo_not_in_scheme:
                 continue
 
-            pool_args.append([args, repo_name, repo_info, repo_branch, remote,
+
+            new_args = [args, repo_name, repo_info, repo_branch, remote,
                               with_ssh, scheme_name, skip_history, skip_tags,
-                              skip_repository_list])
+                              skip_repository_list, use_submodules]
 
-    if not pool_args:
-        print("Not cloning any repositories.")
-        return
+            if use_submodules:
+              obtain_additional_swift_sources(new_args)
+            else:
+              pool_args.append(new_args)
 
-    return run_parallel(
-        obtain_additional_swift_sources, pool_args, args.n_processes)
+    if not use_submodules:
+      if not pool_args:
+          print("Not cloning any repositories.")
+          return
+
+      return run_parallel(
+          obtain_additional_swift_sources, pool_args, args.n_processes)
 
 
 def dump_repo_hashes(args, config, branch_scheme_name='repro'):
@@ -672,6 +697,10 @@ def main():
         help="The root directory to checkout repositories",
         default=SWIFT_SOURCE_ROOT,
         dest='source_root')
+    parser.add_argument(
+        "--use-submodules",
+        help="Checkout repositories as git submodules.",
+        action='store_true')
     args = parser.parse_args()
 
     if not args.scheme:
@@ -693,6 +722,7 @@ def main():
     scheme_name = args.scheme
     github_comment = args.github_comment
     all_repos = args.all_repositories
+    use_submodules = args.use_submodules
 
     with open(args.config) as f:
         config = json.load(f)
@@ -726,7 +756,8 @@ def main():
                                                             scheme_name,
                                                             skip_history,
                                                             skip_tags,
-                                                            skip_repo_list)
+                                                            skip_repo_list,
+                                                            use_submodules)
 
     swift_repo_path = os.path.join(args.source_root, 'swift')
     if 'swift' not in skip_repo_list and os.path.exists(swift_repo_path):