fix overrides dropped when --multirun flag appears between them

hydra/_internal/utils.py

@@ -619,7 +619,29 @@ def __repr__(self) -> str:
 
 
 def get_args(args: Optional[Sequence[str]] = None) -> Any:
-    return get_args_parser().parse_args(args=args)
+    return _parse_args(get_args_parser(), args)
+
+
+def _parse_args(
+    parser: argparse.ArgumentParser, args: Optional[Sequence[str]] = None
+) -> argparse.Namespace:
+    """Parse arguments, handling overrides that appear after optional flags.
+
+    Python's argparse does not correctly collect positional arguments
+    (nargs="*") when an optional flag like ``--multirun`` appears between
+    them.  For example::
+
+        python app.py task=1 --multirun db=mysql
+
+    would only see ``task=1`` as an override, leaving ``db=mysql`` as
+    unrecognised.  This helper uses ``parse_known_args`` and appends the
+    remaining tokens to the ``overrides`` list so that flag ordering no
+    longer matters.
+    """
+    parsed, remaining = parser.parse_known_args(args=args)
+    if remaining:
+        parsed.overrides.extend(remaining)
+    return parsed
 
 
 def get_column_widths(matrix: List[List[str]]) -> List[int]:

hydra/main.py

@@ -11,7 +11,7 @@
 
 from . import version
 from ._internal.deprecation_warning import deprecation_warning
-from ._internal.utils import _run_hydra, get_args_parser
+from ._internal.utils import _parse_args, _run_hydra, get_args_parser
 from .core.hydra_config import HydraConfig
 from .core.utils import _flush_loggers, configure_log
 from .types import TaskFunction
@@ -83,7 +83,7 @@ def decorated_main(cfg_passthrough: Optional[DictConfig] = None) -> Any:
                 return task_function(cfg_passthrough)
             else:
                 args_parser = get_args_parser()
-                args = args_parser.parse_args()
+                args = _parse_args(args_parser)
                 if args.experimental_rerun is not None:
                     cfg = _get_rerun_conf(args.experimental_rerun, args.overrides)
                     task_function(cfg)

tests/test_internal_utils.py

@@ -5,6 +5,7 @@
 from pytest import mark, param
 
 from hydra._internal import utils
+from hydra._internal.utils import get_args
 from tests import data
 
 
@@ -53,3 +54,46 @@ def test_detect_calling_file_or_module_from_task_function(
     file, module = utils.detect_calling_file_or_module_from_task_function(task_function)
     assert file == expected_file
     assert module == expected_module
+
+
+@mark.parametrize(
+    "args, expected_overrides, expected_multirun",
+    [
+        param(
+            ["--multirun", "task=1", "db=mysql"],
+            ["task=1", "db=mysql"],
+            True,
+            id="multirun-first",
+        ),
+        param(
+            ["task=1", "db=mysql", "--multirun"],
+            ["task=1", "db=mysql"],
+            True,
+            id="multirun-last",
+        ),
+        param(
+            ["task=1", "--multirun", "db=mysql"],
+            ["task=1", "db=mysql"],
+            True,
+            id="multirun-between-overrides",
+        ),
+        param(
+            ["task=1", "-m", "db=mysql"],
+            ["task=1", "db=mysql"],
+            True,
+            id="short-flag-between-overrides",
+        ),
+        param(
+            ["task=1", "db=mysql"],
+            ["task=1", "db=mysql"],
+            False,
+            id="no-multirun",
+        ),
+    ],
+)
+def test_get_args_override_ordering(
+    args: Any, expected_overrides: Any, expected_multirun: bool
+) -> None:
+    parsed = get_args(args)
+    assert parsed.overrides == expected_overrides
+    assert parsed.multirun == expected_multirun