diff --git a/bin/common.ml b/bin/common.ml index 2536c178507..17642a4cabf 100644 --- a/bin/common.ml +++ b/bin/common.ml @@ -408,6 +408,10 @@ let shared_with_config_file = ( (fun s -> Result.map_error (Concurrency.of_string s) ~f:(fun s -> `Msg s)) , fun pp x -> Format.pp_print_string pp (Concurrency.to_string x) ) in + let doc = + "Run no more than $(i,JOBS) commands simultaneously. $(i,JOBS) must be a positive \ + integer or $(b,auto) to auto-detect the number of cores (the default)." + in Arg.( value & opt (some arg) None @@ -415,7 +419,8 @@ let shared_with_config_file = [ "j" ] ~docs ~docv:"JOBS" - ~doc:(Some "Run no more than $(i,JOBS) commands simultaneously.")) + ~env:(Cmd.Env.info ~doc "DUNE_JOBS") + ~doc:(Some doc)) and+ sandboxing_preference = let all = List.map Dune_engine.Sandbox_mode.all_except_patch_back_source_tree ~f:(fun s -> diff --git a/bin/rpc/rpc_common.ml b/bin/rpc/rpc_common.ml index 1bbbba44569..0139e5d5e54 100644 --- a/bin/rpc/rpc_common.ml +++ b/bin/rpc/rpc_common.ml @@ -110,7 +110,9 @@ let warn_ignore_arguments lock_held_by = ;; let should_warn ~warn_forwarding builder = - warn_forwarding && not (Common.Builder.equal builder Common.Builder.default) + (not Execution_env.inside_dune) + && warn_forwarding + && not (Common.Builder.equal builder Common.Builder.default) ;; let send_request ~f connection name = diff --git a/doc/changes/added/12800.md b/doc/changes/added/12800.md new file mode 100644 index 00000000000..2e14d63aff0 --- /dev/null +++ b/doc/changes/added/12800.md @@ -0,0 +1,3 @@ +- Add `DUNE_JOBS` environment variable for controlling concurrency of Dune from + environment. The `INSIDE_DUNE` variable also now no longer controls + concurrency (#12800, @Alizter) diff --git a/doc/reference/config/jobs.rst b/doc/reference/config/jobs.rst index 7db0821b58d..22bec1c39e1 100644 --- a/doc/reference/config/jobs.rst +++ b/doc/reference/config/jobs.rst @@ -13,3 +13,8 @@ where ```` is one of: - ````, a positive integer specifying the maximum number of jobs Dune may use simultaneously. + +This setting can also be controlled via the ``-j`` command-line option or the +``DUNE_JOBS`` environment variable. The command-line option takes precedence +over the environment variable, which takes precedence over the configuration +file. diff --git a/doc/usage.rst b/doc/usage.rst index e0d21d151d9..ea1b4add8c5 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -418,6 +418,25 @@ to the user's workspace. However, one can customize this directory by using the # Absolute paths are also allowed $ dune build --build-dir /tmp/build foo.exe +Controlling Concurrency +======================= + +By default Dune automatically detects the number of CPU cores and runs that +many jobs in parallel. You can override this using the ``-j`` flag or the +``DUNE_JOBS`` environment variable with either a positive integer or ``auto`` +to use the default auto-detection. + +.. code:: console + + $ dune build -j 4 + + # this is equivalent to: + $ DUNE_JOBS=4 dune build + +The command-line option takes precedence over the environment variable, which +takes precedence over the :doc:`jobs ` setting in the +configuration file. + Installing a Package ==================== diff --git a/src/dune_config_file/dune_config_file.ml b/src/dune_config_file/dune_config_file.ml index 1d54477a0ac..7e585918f06 100644 --- a/src/dune_config_file/dune_config_file.ml +++ b/src/dune_config_file/dune_config_file.ml @@ -484,7 +484,7 @@ module Dune_config = struct let default = { display = Simple { verbosity = Quiet; status_line = not Execution_env.inside_dune } - ; concurrency = (if Execution_env.inside_dune then Fixed 1 else Auto) + ; concurrency = Auto ; terminal_persistence = Clear_on_rebuild ; sandboxing_preference = [] ; cache_enabled = Enabled_except_user_rules diff --git a/test/blackbox-tests/test-cases/dune b/test/blackbox-tests/test-cases/dune index 522adfbe694..ccf1c33b58b 100644 --- a/test/blackbox-tests/test-cases/dune +++ b/test/blackbox-tests/test-cases/dune @@ -3,6 +3,7 @@ (env-vars (DUNE_CONFIG__BACKGROUND_SANDBOXES disabled) (DUNE_CONFIG__BACKGROUND_DIGESTS disabled) + (DUNE_JOBS 1) ;; We set ocaml to always be colored since it changes the output of ;; ocamlc error messages. See https://github.com/ocaml/ocaml/issues/14144 (OCAML_COLOR always)) @@ -23,6 +24,7 @@ (applies_to :whole_subtree) (deps (env_var OCAML_COLOR) + (env_var DUNE_JOBS) %{bin:dune_cmd} (package dune)) ;; We don't allow conflict markers in tests diff --git a/test/blackbox-tests/test-cases/watching/watching-eager-concurrent-build-command.t b/test/blackbox-tests/test-cases/watching/watching-eager-concurrent-build-command.t index be7aac33141..f737158706d 100644 --- a/test/blackbox-tests/test-cases/watching/watching-eager-concurrent-build-command.t +++ b/test/blackbox-tests/test-cases/watching/watching-eager-concurrent-build-command.t @@ -7,7 +7,6 @@ Demonstrate running "dune build" concurrently with an eager rpc server. $ dune build --watch & Success, waiting for filesystem changes... Success, waiting for filesystem changes... - Success, waiting for filesystem changes... File "foo.ml", line 1, characters 9-21: 1 | let () = print_endlin "Hello, World!" ^^^^^^^^^^^^ @@ -29,11 +28,6 @@ Demonstrate that we can run "dune build" while the watch server is running. $ dune build Success -Demonstrate that a warning is displayed when extra arguments are passed to -"dune build", since those arguments will be ignored. - $ dune build --auto-promote 2>&1 | tr '\n' ' ' | sed 's/(pid: [0-9]*)/(pid: PID)/' - Warning: Your build request is being forwarded to a running Dune instance (pid: PID). Note that certain command line arguments may be ignored. Success - Demonstrate that error messages are still printed by "dune build" when it's acting as an RPC client while running concurrently with an RPC server. $ echo 'let () = print_endlin "Hello, World!"' > foo.ml diff --git a/test/blackbox-tests/test-cases/watching/watching-eager-concurrent-exec-command.t b/test/blackbox-tests/test-cases/watching/watching-eager-concurrent-exec-command.t index 43a2af88f5f..a8100bb0909 100644 --- a/test/blackbox-tests/test-cases/watching/watching-eager-concurrent-exec-command.t +++ b/test/blackbox-tests/test-cases/watching/watching-eager-concurrent-exec-command.t @@ -10,7 +10,6 @@ testing the --no-build option: $ dune build README.md --watch & Success, waiting for filesystem changes... Success, waiting for filesystem changes... - Success, waiting for filesystem changes... Make sure the RPC server is properly started: $ dune rpc ping --wait @@ -33,11 +32,6 @@ Demonstrate running an executable from PATH: executable's name within your PATH only. bar -Demonstrate printing a warning if arguments are passed that would be ignored -due to how Dune builds via RPC: - $ dune exec --force ./foo.exe 2>&1 | tr '\n' ' ' | sed 's/(pid: [0-9]*)/(pid: PID)/' - Warning: Your build request is being forwarded to a running Dune instance (pid: PID). Note that certain command line arguments may be ignored. foo - Demonstrate trying to run exec in watch mode while another watch server is running: $ dune exec ./foo.exe --watch 2>&1 | tr '\n' ' ' | sed 's/(pid: [0-9]*)/(pid: PID)/' Error: Another instance of dune (pid: PID) has locked the _build directory. Refusing to start a new watch server until no other instances of dune are running. diff --git a/test/blackbox-tests/test-cases/watching/watching-eager-concurrent-runtest-command.t b/test/blackbox-tests/test-cases/watching/watching-eager-concurrent-runtest-command.t index 637cd5adfbf..4e2504c0ece 100644 --- a/test/blackbox-tests/test-cases/watching/watching-eager-concurrent-runtest-command.t +++ b/test/blackbox-tests/test-cases/watching/watching-eager-concurrent-runtest-command.t @@ -12,7 +12,6 @@ Define a test that just prints "Hello, World!" $ dune build --watch & Success, waiting for filesystem changes... Success, waiting for filesystem changes... - Success, waiting for filesystem changes... Hello, World! Make sure the RPC server is properly started: @@ -24,10 +23,5 @@ mode: $ dune runtest 2>&1 Success -Test that passing extra arguments to `dune runtest` prints a warning when -running concurrently with another instance of dune in watch mode: - $ dune runtest --auto-promote 2>&1 | tr '\n' ' ' | sed 's/(pid: [0-9]*)/(pid: PID)/' - Warning: Your build request is being forwarded to a running Dune instance (pid: PID). Note that certain command line arguments may be ignored. Success - $ dune shutdown $ wait diff --git a/test/expect-tests/dune_config_file/dune_config_test.ml b/test/expect-tests/dune_config_file/dune_config_test.ml index 85a5441fc5a..3130c2c20e0 100644 --- a/test/expect-tests/dune_config_file/dune_config_test.ml +++ b/test/expect-tests/dune_config_file/dune_config_test.ml @@ -19,7 +19,7 @@ let%expect_test "cache-check-probability 0.1" = [%expect {| { display = Simple { verbosity = Quiet; status_line = false } - ; concurrency = Fixed 1 + ; concurrency = Auto ; terminal_persistence = Clear_on_rebuild ; sandboxing_preference = [] ; cache_enabled = Enabled_except_user_rules @@ -44,7 +44,7 @@ let%expect_test "cache-storage-mode copy" = [%expect {| { display = Simple { verbosity = Quiet; status_line = false } - ; concurrency = Fixed 1 + ; concurrency = Auto ; terminal_persistence = Clear_on_rebuild ; sandboxing_preference = [] ; cache_enabled = Enabled_except_user_rules @@ -69,7 +69,7 @@ let%expect_test "cache-storage-mode hardlink" = [%expect {| { display = Simple { verbosity = Quiet; status_line = false } - ; concurrency = Fixed 1 + ; concurrency = Auto ; terminal_persistence = Clear_on_rebuild ; sandboxing_preference = [] ; cache_enabled = Enabled_except_user_rules