Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion bin/common.ml
Original file line number Diff line number Diff line change
Expand Up @@ -408,14 +408,19 @@ let shared_with_config_file =
( (fun s -> Result.map_error (Concurrency.of_string s) ~f:(fun s -> `Msg s))
, fun pp x -> Format.pp_print_string pp (Concurrency.to_string x) )
in
let doc =
"Run no more than $(i,JOBS) commands simultaneously. $(i,JOBS) must be a positive \
integer or $(b,auto) to auto-detect the number of cores (the default)."
in
Arg.(
value
& opt (some arg) None
& info
[ "j" ]
~docs
~docv:"JOBS"
~doc:(Some "Run no more than $(i,JOBS) commands simultaneously."))
~env:(Cmd.Env.info ~doc "DUNE_JOBS")
~doc:(Some doc))
and+ sandboxing_preference =
let all =
List.map Dune_engine.Sandbox_mode.all_except_patch_back_source_tree ~f:(fun s ->
Expand Down
4 changes: 3 additions & 1 deletion bin/rpc/rpc_common.ml
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,9 @@ let warn_ignore_arguments lock_held_by =
;;

let should_warn ~warn_forwarding builder =
warn_forwarding && not (Common.Builder.equal builder Common.Builder.default)
(not Execution_env.inside_dune)
&& warn_forwarding
&& not (Common.Builder.equal builder Common.Builder.default)
;;

let send_request ~f connection name =
Expand Down
3 changes: 3 additions & 0 deletions doc/changes/added/12800.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
- Add `DUNE_JOBS` environment variable for controlling concurrency of Dune from
environment. The `INSIDE_DUNE` variable also now no longer controls
concurrency (#12800, @Alizter)
5 changes: 5 additions & 0 deletions doc/reference/config/jobs.rst
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,8 @@ where ``<setting>`` is one of:

- ``<number>``, a positive integer specifying the maximum number of jobs Dune
may use simultaneously.

This setting can also be controlled via the ``-j`` command-line option or the
``DUNE_JOBS`` environment variable. The command-line option takes precedence
over the environment variable, which takes precedence over the configuration
file.
19 changes: 19 additions & 0 deletions doc/usage.rst
Original file line number Diff line number Diff line change
Expand Up @@ -418,6 +418,25 @@ to the user's workspace. However, one can customize this directory by using the
# Absolute paths are also allowed
$ dune build --build-dir /tmp/build foo.exe

Controlling Concurrency
=======================

By default Dune automatically detects the number of CPU cores and runs that
many jobs in parallel. You can override this using the ``-j`` flag or the
``DUNE_JOBS`` environment variable with either a positive integer or ``auto``
to use the default auto-detection.

.. code:: console

$ dune build -j 4

# this is equivalent to:
$ DUNE_JOBS=4 dune build

The command-line option takes precedence over the environment variable, which
takes precedence over the :doc:`jobs </reference/config/jobs>` setting in the
configuration file.

Installing a Package
====================

Expand Down
2 changes: 1 addition & 1 deletion src/dune_config_file/dune_config_file.ml
Original file line number Diff line number Diff line change
Expand Up @@ -484,7 +484,7 @@ module Dune_config = struct

let default =
{ display = Simple { verbosity = Quiet; status_line = not Execution_env.inside_dune }
; concurrency = (if Execution_env.inside_dune then Fixed 1 else Auto)
; concurrency = Auto
; terminal_persistence = Clear_on_rebuild
; sandboxing_preference = []
; cache_enabled = Enabled_except_user_rules
Expand Down
2 changes: 2 additions & 0 deletions test/blackbox-tests/test-cases/dune
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
(env-vars
(DUNE_CONFIG__BACKGROUND_SANDBOXES disabled)
(DUNE_CONFIG__BACKGROUND_DIGESTS disabled)
(DUNE_JOBS 1)
;; We set ocaml to always be colored since it changes the output of
;; ocamlc error messages. See https://github.com/ocaml/ocaml/issues/14144
(OCAML_COLOR always))
Expand All @@ -23,6 +24,7 @@
(applies_to :whole_subtree)
(deps
(env_var OCAML_COLOR)
(env_var DUNE_JOBS)
%{bin:dune_cmd}
(package dune))
;; We don't allow conflict markers in tests
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ Demonstrate running "dune build" concurrently with an eager rpc server.
$ dune build --watch &
Success, waiting for filesystem changes...
Success, waiting for filesystem changes...
Success, waiting for filesystem changes...
File "foo.ml", line 1, characters 9-21:
1 | let () = print_endlin "Hello, World!"
^^^^^^^^^^^^
Expand All @@ -29,11 +28,6 @@ Demonstrate that we can run "dune build" while the watch server is running.
$ dune build
Success

Demonstrate that a warning is displayed when extra arguments are passed to
"dune build", since those arguments will be ignored.
$ dune build --auto-promote 2>&1 | tr '\n' ' ' | sed 's/(pid: [0-9]*)/(pid: PID)/'
Warning: Your build request is being forwarded to a running Dune instance (pid: PID). Note that certain command line arguments may be ignored. Success

Demonstrate that error messages are still printed by "dune build" when it's
acting as an RPC client while running concurrently with an RPC server.
$ echo 'let () = print_endlin "Hello, World!"' > foo.ml
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ testing the --no-build option:
$ dune build README.md --watch &
Success, waiting for filesystem changes...
Success, waiting for filesystem changes...
Success, waiting for filesystem changes...

Make sure the RPC server is properly started:
$ dune rpc ping --wait
Expand All @@ -33,11 +32,6 @@ Demonstrate running an executable from PATH:
executable's name within your PATH only.
bar

Demonstrate printing a warning if arguments are passed that would be ignored
due to how Dune builds via RPC:
$ dune exec --force ./foo.exe 2>&1 | tr '\n' ' ' | sed 's/(pid: [0-9]*)/(pid: PID)/'
Warning: Your build request is being forwarded to a running Dune instance (pid: PID). Note that certain command line arguments may be ignored. foo

Demonstrate trying to run exec in watch mode while another watch server is running:
$ dune exec ./foo.exe --watch 2>&1 | tr '\n' ' ' | sed 's/(pid: [0-9]*)/(pid: PID)/'
Error: Another instance of dune (pid: PID) has locked the _build directory. Refusing to start a new watch server until no other instances of dune are running.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ Define a test that just prints "Hello, World!"
$ dune build --watch &
Success, waiting for filesystem changes...
Success, waiting for filesystem changes...
Success, waiting for filesystem changes...
Hello, World!

Make sure the RPC server is properly started:
Expand All @@ -24,10 +23,5 @@ mode:
$ dune runtest 2>&1
Success

Test that passing extra arguments to `dune runtest` prints a warning when
running concurrently with another instance of dune in watch mode:
$ dune runtest --auto-promote 2>&1 | tr '\n' ' ' | sed 's/(pid: [0-9]*)/(pid: PID)/'
Warning: Your build request is being forwarded to a running Dune instance (pid: PID). Note that certain command line arguments may be ignored. Success

$ dune shutdown
$ wait
6 changes: 3 additions & 3 deletions test/expect-tests/dune_config_file/dune_config_test.ml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ let%expect_test "cache-check-probability 0.1" =
[%expect
{|
{ display = Simple { verbosity = Quiet; status_line = false }
; concurrency = Fixed 1
; concurrency = Auto
; terminal_persistence = Clear_on_rebuild
; sandboxing_preference = []
; cache_enabled = Enabled_except_user_rules
Expand All @@ -44,7 +44,7 @@ let%expect_test "cache-storage-mode copy" =
[%expect
{|
{ display = Simple { verbosity = Quiet; status_line = false }
; concurrency = Fixed 1
; concurrency = Auto
; terminal_persistence = Clear_on_rebuild
; sandboxing_preference = []
; cache_enabled = Enabled_except_user_rules
Expand All @@ -69,7 +69,7 @@ let%expect_test "cache-storage-mode hardlink" =
[%expect
{|
{ display = Simple { verbosity = Quiet; status_line = false }
; concurrency = Fixed 1
; concurrency = Auto
; terminal_persistence = Clear_on_rebuild
; sandboxing_preference = []
; cache_enabled = Enabled_except_user_rules
Expand Down
Loading