Skip to content

o2-sim: Better signal propagation; small fix in jobutils #5156

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sawenzel merged 1 commit into from
Jan 12, 2021
Merged

o2-sim: Better signal propagation; small fix in jobutils #5156

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions Utilities/Tools/jobutils.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,12 @@ o2_cleanup_shm_files() {
}

# Function to find out all the (recursive) child processes starting from a parent PID.
# The output includes includes the parent
# output is saved in child_pid_list
# The output includes the parent
childprocs() {
local parent=$1
if [ ! "$2" ]; then
child_pid_list=""
fi
if [ "$parent" ] ; then
child_pid_list="$child_pid_list $parent"
for childpid in $(pgrep -P ${parent}); do
Expand Down
22 changes: 14 additions & 8 deletions run/o2sim_parallel.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,12 +118,18 @@ int checkresult()
return errors;
}

std::vector<int> gChildProcesses; // global vector of child pids

// signal handler for graceful exit
void sighandler(int signal)
{
if (signal == SIGINT || signal == SIGTERM) {
LOG(INFO) << "signal caught ... clean up and exit";
LOG(INFO) << "o2-sim driver: Signal caught ... clean up and exit";
cleanup();
// forward signal to all children
for (auto& pid : gChildProcesses) {
kill(pid, signal);
}
exit(0);
}
}
Expand Down Expand Up @@ -346,7 +352,7 @@ int main(int argc, char* argv[])
}
return r;
} else {
childpids.push_back(pid);
gChildProcesses.push_back(pid);
close(pipe_serverdriver_fd[1]);
std::cout << "Spawning particle server on PID " << pid << "; Redirect output to " << getServerLogName() << "\n";
launchThreadMonitoringEvents(pipe_serverdriver_fd[0], "DISTRIBUTING EVENT : ");
Expand Down Expand Up @@ -380,7 +386,7 @@ int main(int argc, char* argv[])
"worker", "--mq-config", localconfig.c_str(), "--severity", "info", (char*)nullptr);
return 0;
} else {
childpids.push_back(pid);
gChildProcesses.push_back(pid);
std::cout << "Spawning sim worker " << id << " on PID " << pid
<< "; Redirect output to " << workerlogss.str() << "\n";
}
Expand Down Expand Up @@ -409,13 +415,13 @@ int main(int argc, char* argv[])
return 0;
} else {
std::cout << "Spawning hit merger on PID " << pid << "; Redirect output to " << getMergerLogName() << "\n";
childpids.push_back(pid);
gChildProcesses.push_back(pid);
close(pipe_mergerdriver_fd[1]);
launchThreadMonitoringEvents(pipe_mergerdriver_fd[0], "EVENT FINISHED : ");
}

// wait on merger (which when exiting completes the workflow)
auto mergerpid = childpids.back();
auto mergerpid = gChildProcesses.back();

int status, cpid;
// wait just blocks and waits until any child returns; but we make sure to wait until merger is here
Expand All @@ -426,8 +432,8 @@ int main(int argc, char* argv[])
}
// we bring down all processes if one of them aborts
if (WTERMSIG(status) == SIGABRT) {
for (auto p : childpids) {
kill(p, SIGABRT);
for (auto p : gChildProcesses) {
kill(p, SIGTERM);
}
cleanup();
LOG(FATAL) << "ABORTING DUE TO ABORT IN COMPONENT";
Expand All @@ -438,7 +444,7 @@ int main(int argc, char* argv[])
LOG(INFO) << "Simulation process took " << timer.RealTime() << " s";

// make sure the rest shuts down
for (auto p : childpids) {
for (auto p : gChildProcesses) {
if (p != mergerpid) {
LOG(DEBUG) << "SHUTTING DOWN CHILD PROCESS " << p;
kill(p, SIGTERM);
Expand Down