Ensure duplicate arguments are only checked within their respective argument groups

Differential Revision: D57459718

Pull Request resolved: #911

Author: hstonec

torchx/cli/cmd_run.py

@@ -13,6 +13,7 @@
 import threading
 from collections import Counter
 from dataclasses import asdict
+from itertools import groupby
 from pathlib import Path
 from pprint import pformat
 from typing import Dict, List, Optional, Tuple
@@ -85,16 +86,19 @@ def _parse_component_name_and_args(
             component = args[0]
             component_args = args[1:]
 
-    # Error if there are repeated command line arguments
-    all_options = [
-        x
-        for x in component_args
-        if x.startswith("-") and x.strip() != "-" and x.strip() != "--"
-    ]
-    arg_count = Counter(all_options)
-    duplicates = [arg for arg, count in arg_count.items() if count > 1]
-    if len(duplicates) > 0:
-        subparser.error(f"Repeated Command Line Arguments: {duplicates}")
+    # Error if there are repeated command line arguments each group of arguments,
+    # where the groups are separated by "--"
+    arg_groups = [list(g) for _, g in groupby(component_args, key=lambda x: x == "--")]
+    for arg_group in arg_groups:
+        all_options = [
+            x
+            for x in arg_group
+            if x.startswith("-") and x.strip() != "-" and x.strip() != "--"
+        ]
+        arg_count = Counter(all_options)
+        duplicates = [arg for arg, count in arg_count.items() if count > 1]
+        if len(duplicates) > 0:
+            subparser.error(f"Repeated Command Line Arguments: {duplicates}")
 
     if not component:
         subparser.error(MISSING_COMPONENT_ERROR_MSG)

torchx/cli/test/cmd_run_test.py

@@ -251,6 +251,32 @@ def test_parse_component_name_and_args_no_default(self) -> None:
             ),
         )
 
+        self.assertEqual(
+            (
+                "fb.python.binary",
+                [
+                    "--img",
+                    "lex_ig_o3_package",
+                    "-m",
+                    "dper_lib.instagram.pyper_v2.teams.stories.train",
+                    "--",
+                    "-m",
+                ],
+            ),
+            _parse_component_name_and_args(
+                [
+                    "fb.python.binary",
+                    "--img",
+                    "lex_ig_o3_package",
+                    "-m",
+                    "dper_lib.instagram.pyper_v2.teams.stories.train",
+                    "--",
+                    "-m",
+                ],
+                sp,
+            ),
+        )
+
         with self.assertRaises(SystemExit):
             _parse_component_name_and_args(["--"], sp)
 
@@ -271,6 +297,11 @@ def test_parse_component_name_and_args_no_default(self) -> None:
                 ["--msg  ", "hello", "--msg     ", "repeate"], sp
             )
 
+        with self.assertRaises(SystemExit):
+            _parse_component_name_and_args(
+                ["--m", "hello", "--", "--msg", "msg", "--msg", "repeate"], sp
+            )
+
     def test_parse_component_name_and_args_with_default(self) -> None:
         sp = argparse.ArgumentParser(prog="test")
         dirs = [str(self.tmpdir)]