From 44e075bb376a536ee94ca3a3b2d78fd798d799bf Mon Sep 17 00:00:00 2001 From: Michael Hines Date: Tue, 8 Sep 2020 17:48:32 -0400 Subject: [PATCH 1/5] If a netcon_srcgid is negative, need to determine the thread. Only for direct transfer mode is it allowed that a negative srcgid is not in the same thread as the NetCon target. To enable thread determination an std::vector involving the name netcon_negsrcgid_tid is associated with netcon_srcgid in that when a negative gid appears in netcon_srcgid, the tid is the value of the element in netcon_negsrcgid_tid. --- coreneuron/io/nrn2core_direct.h | 3 ++- coreneuron/io/nrn_setup.cpp | 34 ++++++++++++++++++++++++++++++--- coreneuron/io/phase1.cpp | 23 ++++++++++++++++++++-- coreneuron/io/phase1.hpp | 1 + coreneuron/network/netpar.cpp | 7 +++++++ coreneuron/nrniv/nrniv_decl.h | 5 ++++- 6 files changed, 66 insertions(+), 7 deletions(-) diff --git a/coreneuron/io/nrn2core_direct.h b/coreneuron/io/nrn2core_direct.h index 3c4990a05..f7f190c47 100644 --- a/coreneuron/io/nrn2core_direct.h +++ b/coreneuron/io/nrn2core_direct.h @@ -30,7 +30,8 @@ extern int (*nrn2core_get_dat1_)(int tid, int& n_presyn, int& n_netcon, int*& output_gid, - int*& netcon_srcgid); + int*& netcon_srcgid, + std::vector& netcon_negsrcgid_tid); extern int (*nrn2core_get_dat2_1_)(int tid, int& ngid, diff --git a/coreneuron/io/nrn_setup.cpp b/coreneuron/io/nrn_setup.cpp index 28fa1de07..e927c79c1 100644 --- a/coreneuron/io/nrn_setup.cpp +++ b/coreneuron/io/nrn_setup.cpp @@ -113,6 +113,14 @@ void (*nrn2core_all_weights_return_)(std::vector& weights); // encode the thread number into the negative gid // (i.e -ith - nth*(type +1000*index)) failed due to not large enough // integer domain size. +// Note that for file transfer it is an error if a negative srcgid is +// not in the same thread as the target. This is because there it may +// not be the case that threads in a NEURON process end up on same process +// in CoreNEURON. NEURON will raise an error if this +// is the case. However, for direct memory transfer, it is allowed that +// a negative srcgid may be in a different thread than the target. So +// nrn2core_get_dat1 has a last arg netcon_negsrcgid_tid that specifies +// for the negative gids in netcon_srcgid (in that order) the source thread. // // _2.dat // n_output n_real_output, nnode @@ -180,6 +188,10 @@ std::vector netcon_in_presyn_order_; /// Only for setup vector of netcon source gids std::vector netcon_srcgid; +/// If a netcon_srcgid is negative, need to determine the thread when +/// in order to use the correct neg_gid2out[tid] map +std::vector > netcon_negsrcgid_tid; + /* read files.dat file and distribute cellgroups to all mpi ranks */ void nrn_read_filesdat(int& ngrp, int*& grp, const char* filesdat) { patstimtype = nrn_get_mechtype("PatternStim"); @@ -275,6 +287,9 @@ void determine_inputpresyn() { NrnThread& nt = nrn_threads[ith]; // associate gid with InputPreSyn and increase PreSyn and InputPreSyn count nt.n_input_presyn = 0; + // if empty then not to be used. + std::vector& negsrcgid_tid = netcon_negsrcgid_tid[ith]; + size_t i_tid = 0; for (int i = 0; i < nt.n_netcon; ++i) { int gid = netcon_srcgid[ith][i]; if (gid >= 0) { @@ -299,8 +314,12 @@ void determine_inputpresyn() { inputpresyn_.push_back(psi); ++nt.n_input_presyn; } else { - auto gid2out_it = neg_gid2out[nt.id].find(gid); - if (gid2out_it != neg_gid2out[nt.id].end()) { + int tid = nt.id; + if (!negsrcgid_tid.empty()) { + tid = negsrcgid_tid[i_tid++]; + } + auto gid2out_it = neg_gid2out[tid].find(gid); + if (gid2out_it != neg_gid2out[tid].end()) { /// Increase negative PreSyn count ++gid2out_it->second->nc_cnt_; } @@ -365,12 +384,19 @@ void determine_inputpresyn() { // only used via ps.nc_index_ and ps.nc_cnt_; for (int ith = 0; ith < nrn_nthread; ++ith) { NrnThread& nt = nrn_threads[ith]; + // if empty then not to be used. + std::vector& negsrcgid_tid = netcon_negsrcgid_tid[ith]; + size_t i_tid = 0; for (int i = 0; i < nt.n_netcon; ++i) { NetCon* nc = nt.netcons + i; int gid = netcon_srcgid[ith][i]; + int tid = ith; + if (!negsrcgid_tid.empty() && gid < -1) { + tid = negsrcgid_tid[i_tid++]; + } PreSyn* ps; InputPreSyn* psi; - netpar_tid_gid2ps(ith, gid, &ps, &psi); + netpar_tid_gid2ps(tid, gid, &ps, &psi); if (ps) { netcon_in_presyn_order_[ps->nc_index_ + ps->nc_cnt_] = nc; ++ps->nc_cnt_; @@ -394,6 +420,7 @@ void nrn_setup_cleanup() { delete[] netcon_srcgid[ith]; } netcon_srcgid.clear(); + netcon_negsrcgid_tid.clear(); neg_gid2out.clear(); } @@ -473,6 +500,7 @@ void nrn_setup(const char* filesdat, if (!corenrn_embedded) { coreneuron::phase_wrapper(userParams); } else { + netcon_negsrcgid_tid.resize(nrn_nthread); nrn_multithread_job([](NrnThread* n) { Phase1 p1; p1.read_direct(n->id); diff --git a/coreneuron/io/phase1.cpp b/coreneuron/io/phase1.cpp index dd96bc4e1..fd5d3287b 100644 --- a/coreneuron/io/phase1.cpp +++ b/coreneuron/io/phase1.cpp @@ -10,7 +10,8 @@ int (*nrn2core_get_dat1_)(int tid, int& n_presyn, int& n_netcon, int*& output_gid, - int*& netcon_srcgid); + int*& netcon_srcgid, + std::vector& netcon_negsrcgid_tid); namespace coreneuron { void Phase1::read_file(FileHandler& F) { @@ -20,10 +21,24 @@ void Phase1::read_file(FileHandler& F) { this->output_gids = F.read_vector(n_presyn); this->netcon_srcgids = F.read_vector(n_netcon); + // For file mode transfer, it is not allowed that negative gids exist + // in different threads. So this->netcon_tids remains clear. F.close(); } +static void prsrctid(int thread_id, int n, int* srcgid, std::vector& srctid) { + if (!srctid.empty()) { + int i = 0; + for (int i_nc=0; i_nc < n; ++i_nc) { + if (srcgid[i_nc] < -1) { + printf("srcgid=%d tid=%d srctid=%d\n", srcgid[i_nc], thread_id, srctid[i]); + ++i; + } + } + } +} + void Phase1::read_direct(int thread_id) { int* output_gids; int* netcon_srcgid; @@ -32,7 +47,7 @@ void Phase1::read_direct(int thread_id) { // TODO : check error codes for NEURON - CoreNEURON communication int valid = - (*nrn2core_get_dat1_)(thread_id, n_presyn, n_netcon, output_gids, netcon_srcgid); + (*nrn2core_get_dat1_)(thread_id, n_presyn, n_netcon, output_gids, netcon_srcgid, this->netcon_negsrcgid_tid); if (!valid) { return; } @@ -51,6 +66,10 @@ void Phase1::populate(NrnThread& nt, OMP_Mutex& mut) { std::copy(this->netcon_srcgids.begin(), this->netcon_srcgids.end(), netcon_srcgid[nt.id]); + if (!coreneuron::netcon_negsrcgid_tid.empty()) { // multiple threads and direct mode. + coreneuron::netcon_negsrcgid_tid[nt.id] = this->netcon_negsrcgid_tid; + } + nt.netcons = new NetCon[nt.n_netcon]; nt.presyns_helper = (PreSynHelper*)ecalloc_align(nt.n_presyn, sizeof(PreSynHelper)); diff --git a/coreneuron/io/phase1.hpp b/coreneuron/io/phase1.hpp index 7d5ec5a05..8ab8d999b 100644 --- a/coreneuron/io/phase1.hpp +++ b/coreneuron/io/phase1.hpp @@ -18,6 +18,7 @@ class Phase1 { private: std::vector output_gids; std::vector netcon_srcgids; + std::vector netcon_negsrcgid_tid; // entries only for negative srcgids }; } // namespace coreneuron diff --git a/coreneuron/network/netpar.cpp b/coreneuron/network/netpar.cpp index 7c815690e..7a7204d68 100644 --- a/coreneuron/network/netpar.cpp +++ b/coreneuron/network/netpar.cpp @@ -641,10 +641,17 @@ double set_mindelay(double maxdelay) { for (int ith = 0; ith < nrn_nthread; ++ith) { NrnThread& nt = nrn_threads[ith]; + // if empty then not to be used. + std::vector& negsrcgid_tid = netcon_negsrcgid_tid[ith]; + size_t i_tid = 0; for (int i = 0; i < nt.n_netcon; ++i) { NetCon* nc = nt.netcons + i; bool chk = false; // ignore nc.delay_ int gid = netcon_srcgid[ith][i]; + int tid = ith; + if (!negsrcgid_tid.empty() && gid < -1) { + tid = negsrcgid_tid[i_tid++]; + } PreSyn* ps; InputPreSyn* psi; netpar_tid_gid2ps(ith, gid, &ps, &psi); diff --git a/coreneuron/nrniv/nrniv_decl.h b/coreneuron/nrniv/nrniv_decl.h index 2d053962f..b460975bf 100644 --- a/coreneuron/nrniv/nrniv_decl.h +++ b/coreneuron/nrniv/nrniv_decl.h @@ -48,8 +48,11 @@ extern std::map gid2in; /// InputPreSyn.nc_index_ to + InputPreSyn.nc_cnt_ give the NetCon* extern std::vector netcon_in_presyn_order_; -/// Only for setup vector of netcon source gids +/// Only for setup vector of netcon source gids and mindelay determination extern std::vector netcon_srcgid; +/// Companion to netcon_srcgid when src gid is negative to allow +/// determination of the NrnThread of the source PreSyn. +extern std::vector > netcon_negsrcgid_tid; extern void mk_mech(const char* path); extern void set_globals(const char* path, bool cli_global_seed, int cli_global_seed_value); From 7af3e9369ae6b69a4db17604cf6f1f90ced84855 Mon Sep 17 00:00:00 2001 From: Alexandru Savulescu Date: Sat, 12 Sep 2020 14:55:11 +0200 Subject: [PATCH 2/5] always resize netcon_negsrcgid_tid --- coreneuron/io/nrn_setup.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coreneuron/io/nrn_setup.cpp b/coreneuron/io/nrn_setup.cpp index e927c79c1..92049b3f3 100644 --- a/coreneuron/io/nrn_setup.cpp +++ b/coreneuron/io/nrn_setup.cpp @@ -497,10 +497,10 @@ void nrn_setup(const char* filesdat, nrn_partrans::gap_mpi_setup(userParams.ngroup); } + netcon_negsrcgid_tid.resize(nrn_nthread); if (!corenrn_embedded) { coreneuron::phase_wrapper(userParams); } else { - netcon_negsrcgid_tid.resize(nrn_nthread); nrn_multithread_job([](NrnThread* n) { Phase1 p1; p1.read_direct(n->id); From c870bf32b03e936654394789a48e9a0bdb749a13 Mon Sep 17 00:00:00 2001 From: Alexandru Savulescu Date: Sat, 12 Sep 2020 15:43:50 +0200 Subject: [PATCH 3/5] link netcon_negsrcgid_tid usage to corenrn_embedded --- coreneuron/io/nrn_setup.cpp | 10 +++++----- coreneuron/io/phase1.cpp | 6 +++++- coreneuron/network/netpar.cpp | 8 ++++++-- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/coreneuron/io/nrn_setup.cpp b/coreneuron/io/nrn_setup.cpp index 92049b3f3..713527435 100644 --- a/coreneuron/io/nrn_setup.cpp +++ b/coreneuron/io/nrn_setup.cpp @@ -287,8 +287,8 @@ void determine_inputpresyn() { NrnThread& nt = nrn_threads[ith]; // associate gid with InputPreSyn and increase PreSyn and InputPreSyn count nt.n_input_presyn = 0; - // if empty then not to be used. - std::vector& negsrcgid_tid = netcon_negsrcgid_tid[ith]; + std::vector dummy; + std::vector& negsrcgid_tid = corenrn_embedded ? netcon_negsrcgid_tid[ith] : dummy; size_t i_tid = 0; for (int i = 0; i < nt.n_netcon; ++i) { int gid = netcon_srcgid[ith][i]; @@ -384,8 +384,8 @@ void determine_inputpresyn() { // only used via ps.nc_index_ and ps.nc_cnt_; for (int ith = 0; ith < nrn_nthread; ++ith) { NrnThread& nt = nrn_threads[ith]; - // if empty then not to be used. - std::vector& negsrcgid_tid = netcon_negsrcgid_tid[ith]; + std::vector dummy; + std::vector& negsrcgid_tid = corenrn_embedded ? netcon_negsrcgid_tid[ith] : dummy; size_t i_tid = 0; for (int i = 0; i < nt.n_netcon; ++i) { NetCon* nc = nt.netcons + i; @@ -497,10 +497,10 @@ void nrn_setup(const char* filesdat, nrn_partrans::gap_mpi_setup(userParams.ngroup); } - netcon_negsrcgid_tid.resize(nrn_nthread); if (!corenrn_embedded) { coreneuron::phase_wrapper(userParams); } else { + netcon_negsrcgid_tid.resize(nrn_nthread); nrn_multithread_job([](NrnThread* n) { Phase1 p1; p1.read_direct(n->id); diff --git a/coreneuron/io/phase1.cpp b/coreneuron/io/phase1.cpp index fd5d3287b..eff0dc4c7 100644 --- a/coreneuron/io/phase1.cpp +++ b/coreneuron/io/phase1.cpp @@ -58,6 +58,10 @@ void Phase1::read_direct(int thread_id) { delete[] netcon_srcgid; } +extern "C" { +extern bool corenrn_embedded; +} + void Phase1::populate(NrnThread& nt, OMP_Mutex& mut) { nt.n_presyn = this->output_gids.size(); nt.n_netcon = this->netcon_srcgids.size(); @@ -66,7 +70,7 @@ void Phase1::populate(NrnThread& nt, OMP_Mutex& mut) { std::copy(this->netcon_srcgids.begin(), this->netcon_srcgids.end(), netcon_srcgid[nt.id]); - if (!coreneuron::netcon_negsrcgid_tid.empty()) { // multiple threads and direct mode. + if (corenrn_embedded) { // multiple threads and direct mode. coreneuron::netcon_negsrcgid_tid[nt.id] = this->netcon_negsrcgid_tid; } diff --git a/coreneuron/network/netpar.cpp b/coreneuron/network/netpar.cpp index 7a7204d68..46f8a0ab2 100644 --- a/coreneuron/network/netpar.cpp +++ b/coreneuron/network/netpar.cpp @@ -619,6 +619,10 @@ void BBS_netpar_solve(double tstop) { } } +extern "C" { +extern bool corenrn_embedded; +} + double set_mindelay(double maxdelay) { double mindelay = maxdelay; last_maxstep_arg_ = maxdelay; @@ -641,8 +645,8 @@ double set_mindelay(double maxdelay) { for (int ith = 0; ith < nrn_nthread; ++ith) { NrnThread& nt = nrn_threads[ith]; - // if empty then not to be used. - std::vector& negsrcgid_tid = netcon_negsrcgid_tid[ith]; + std::vector dummy; + std::vector& negsrcgid_tid = corenrn_embedded ? netcon_negsrcgid_tid[ith] : dummy; size_t i_tid = 0; for (int i = 0; i < nt.n_netcon; ++i) { NetCon* nc = nt.netcons + i; From 6958a33dcc94e80c7d4161bbb122fb86e8efe3e3 Mon Sep 17 00:00:00 2001 From: Michael Hines Date: Sun, 13 Sep 2020 08:03:50 -0400 Subject: [PATCH 4/5] nrnthreads_netcon_srcgid is more meaninfgul name. --- coreneuron/io/nrn_setup.cpp | 32 ++++++++++++++++---------------- coreneuron/io/phase1.cpp | 18 +++--------------- coreneuron/network/netpar.cpp | 4 ++-- coreneuron/nrniv/nrniv_decl.h | 6 +++--- 4 files changed, 24 insertions(+), 36 deletions(-) diff --git a/coreneuron/io/nrn_setup.cpp b/coreneuron/io/nrn_setup.cpp index 713527435..21e2b9b73 100644 --- a/coreneuron/io/nrn_setup.cpp +++ b/coreneuron/io/nrn_setup.cpp @@ -186,11 +186,11 @@ std::map gid2in; std::vector netcon_in_presyn_order_; /// Only for setup vector of netcon source gids -std::vector netcon_srcgid; +std::vector nrnthreads_netcon_srcgid; -/// If a netcon_srcgid is negative, need to determine the thread when +/// If a nrnthreads_netcon_srcgid is negative, need to determine the thread when /// in order to use the correct neg_gid2out[tid] map -std::vector > netcon_negsrcgid_tid; +std::vector > nrnthreads_netcon_negsrcgid_tid; /* read files.dat file and distribute cellgroups to all mpi ranks */ void nrn_read_filesdat(int& ngrp, int*& grp, const char* filesdat) { @@ -288,10 +288,10 @@ void determine_inputpresyn() { // associate gid with InputPreSyn and increase PreSyn and InputPreSyn count nt.n_input_presyn = 0; std::vector dummy; - std::vector& negsrcgid_tid = corenrn_embedded ? netcon_negsrcgid_tid[ith] : dummy; + std::vector& negsrcgid_tid = corenrn_embedded ? nrnthreads_netcon_negsrcgid_tid[ith] : dummy; size_t i_tid = 0; for (int i = 0; i < nt.n_netcon; ++i) { - int gid = netcon_srcgid[ith][i]; + int gid = nrnthreads_netcon_srcgid[ith][i]; if (gid >= 0) { /// If PreSyn or InputPreSyn is already in the map auto gid2out_it = gid2out.find(gid); @@ -380,16 +380,16 @@ void determine_inputpresyn() { // fill the netcon_in_presyn_order and recompute nc_cnt_ // note that not all netcon_in_presyn will be filled if there are netcon - // with no presyn (ie. netcon_srcgid[nt.id][i] = -1) but that is ok since they are + // with no presyn (ie. nrnthreads_netcon_srcgid[nt.id][i] = -1) but that is ok since they are // only used via ps.nc_index_ and ps.nc_cnt_; for (int ith = 0; ith < nrn_nthread; ++ith) { NrnThread& nt = nrn_threads[ith]; std::vector dummy; - std::vector& negsrcgid_tid = corenrn_embedded ? netcon_negsrcgid_tid[ith] : dummy; + std::vector& negsrcgid_tid = corenrn_embedded ? nrnthreads_netcon_negsrcgid_tid[ith] : dummy; size_t i_tid = 0; for (int i = 0; i < nt.n_netcon; ++i) { NetCon* nc = nt.netcons + i; - int gid = netcon_srcgid[ith][i]; + int gid = nrnthreads_netcon_srcgid[ith][i]; int tid = ith; if (!negsrcgid_tid.empty() && gid < -1) { tid = negsrcgid_tid[i_tid++]; @@ -416,11 +416,11 @@ void determine_inputpresyn() { /// Clean up void nrn_setup_cleanup() { for (int ith = 0; ith < nrn_nthread; ++ith) { - if (netcon_srcgid[ith]) - delete[] netcon_srcgid[ith]; + if (nrnthreads_netcon_srcgid[ith]) + delete[] nrnthreads_netcon_srcgid[ith]; } - netcon_srcgid.clear(); - netcon_negsrcgid_tid.clear(); + nrnthreads_netcon_srcgid.clear(); + nrnthreads_netcon_negsrcgid_tid.clear(); neg_gid2out.clear(); } @@ -476,9 +476,9 @@ void nrn_setup(const char* filesdat, /// std::map gid2out; gid2out.clear(); - netcon_srcgid.resize(nrn_nthread); + nrnthreads_netcon_srcgid.resize(nrn_nthread); for (int i = 0; i < nrn_nthread; ++i) - netcon_srcgid[i] = nullptr; + nrnthreads_netcon_srcgid[i] = nullptr; // gap junctions if (nrn_have_gaps) { @@ -500,7 +500,7 @@ void nrn_setup(const char* filesdat, if (!corenrn_embedded) { coreneuron::phase_wrapper(userParams); } else { - netcon_negsrcgid_tid.resize(nrn_nthread); + nrnthreads_netcon_negsrcgid_tid.resize(nrn_nthread); nrn_multithread_job([](NrnThread* n) { Phase1 p1; p1.read_direct(n->id); @@ -509,7 +509,7 @@ void nrn_setup(const char* filesdat, }); } - // from the gid2out map and the netcon_srcgid array, + // from the gid2out map and the nrnthreads_netcon_srcgid array, // fill the gid2in, and from the number of entries, // allocate the process wide InputPreSyn array determine_inputpresyn(); diff --git a/coreneuron/io/phase1.cpp b/coreneuron/io/phase1.cpp index eff0dc4c7..d1cd1f134 100644 --- a/coreneuron/io/phase1.cpp +++ b/coreneuron/io/phase1.cpp @@ -27,18 +27,6 @@ void Phase1::read_file(FileHandler& F) { F.close(); } -static void prsrctid(int thread_id, int n, int* srcgid, std::vector& srctid) { - if (!srctid.empty()) { - int i = 0; - for (int i_nc=0; i_nc < n; ++i_nc) { - if (srcgid[i_nc] < -1) { - printf("srcgid=%d tid=%d srctid=%d\n", srcgid[i_nc], thread_id, srctid[i]); - ++i; - } - } - } -} - void Phase1::read_direct(int thread_id) { int* output_gids; int* netcon_srcgid; @@ -66,12 +54,12 @@ void Phase1::populate(NrnThread& nt, OMP_Mutex& mut) { nt.n_presyn = this->output_gids.size(); nt.n_netcon = this->netcon_srcgids.size(); - netcon_srcgid[nt.id] = new int[nt.n_netcon]; + nrnthreads_netcon_srcgid[nt.id] = new int[nt.n_netcon]; std::copy(this->netcon_srcgids.begin(), this->netcon_srcgids.end(), - netcon_srcgid[nt.id]); + nrnthreads_netcon_srcgid[nt.id]); if (corenrn_embedded) { // multiple threads and direct mode. - coreneuron::netcon_negsrcgid_tid[nt.id] = this->netcon_negsrcgid_tid; + coreneuron::nrnthreads_netcon_negsrcgid_tid[nt.id] = this->netcon_negsrcgid_tid; } nt.netcons = new NetCon[nt.n_netcon]; diff --git a/coreneuron/network/netpar.cpp b/coreneuron/network/netpar.cpp index 46f8a0ab2..e860ea6f1 100644 --- a/coreneuron/network/netpar.cpp +++ b/coreneuron/network/netpar.cpp @@ -646,12 +646,12 @@ double set_mindelay(double maxdelay) { for (int ith = 0; ith < nrn_nthread; ++ith) { NrnThread& nt = nrn_threads[ith]; std::vector dummy; - std::vector& negsrcgid_tid = corenrn_embedded ? netcon_negsrcgid_tid[ith] : dummy; + std::vector& negsrcgid_tid = corenrn_embedded ? nrnthreads_netcon_negsrcgid_tid[ith] : dummy; size_t i_tid = 0; for (int i = 0; i < nt.n_netcon; ++i) { NetCon* nc = nt.netcons + i; bool chk = false; // ignore nc.delay_ - int gid = netcon_srcgid[ith][i]; + int gid = nrnthreads_netcon_srcgid[ith][i]; int tid = ith; if (!negsrcgid_tid.empty() && gid < -1) { tid = negsrcgid_tid[i_tid++]; diff --git a/coreneuron/nrniv/nrniv_decl.h b/coreneuron/nrniv/nrniv_decl.h index b460975bf..6f92c2970 100644 --- a/coreneuron/nrniv/nrniv_decl.h +++ b/coreneuron/nrniv/nrniv_decl.h @@ -49,10 +49,10 @@ extern std::map gid2in; /// InputPreSyn.nc_index_ to + InputPreSyn.nc_cnt_ give the NetCon* extern std::vector netcon_in_presyn_order_; /// Only for setup vector of netcon source gids and mindelay determination -extern std::vector netcon_srcgid; -/// Companion to netcon_srcgid when src gid is negative to allow +extern std::vector nrnthreads_netcon_srcgid; +/// Companion to nrnthreads_netcon_srcgid when src gid is negative to allow /// determination of the NrnThread of the source PreSyn. -extern std::vector > netcon_negsrcgid_tid; +extern std::vector > nrnthreads_netcon_negsrcgid_tid; extern void mk_mech(const char* path); extern void set_globals(const char* path, bool cli_global_seed, int cli_global_seed_value); From 9dc707924c00e9ebfad3d75ccafbc3b02e71ae22 Mon Sep 17 00:00:00 2001 From: Michael Hines Date: Mon, 14 Sep 2020 14:48:55 -0400 Subject: [PATCH 5/5] Revert "link netcon_negsrcgid_tid usage to corenrn_embedded" Comments indicate that nrnthreads_netcon_negsrc_gid_tid subvectors are definitely empty when single thread or file transfer. (will be empty anyway if there are no negative gids) This reverts commit c870bf32b03e936654394789a48e9a0bdb749a13. --- coreneuron/io/nrn_setup.cpp | 10 +++++----- coreneuron/io/phase1.cpp | 9 ++------- coreneuron/network/netpar.cpp | 8 ++------ 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/coreneuron/io/nrn_setup.cpp b/coreneuron/io/nrn_setup.cpp index 21e2b9b73..c3d2e9df5 100644 --- a/coreneuron/io/nrn_setup.cpp +++ b/coreneuron/io/nrn_setup.cpp @@ -287,8 +287,8 @@ void determine_inputpresyn() { NrnThread& nt = nrn_threads[ith]; // associate gid with InputPreSyn and increase PreSyn and InputPreSyn count nt.n_input_presyn = 0; - std::vector dummy; - std::vector& negsrcgid_tid = corenrn_embedded ? nrnthreads_netcon_negsrcgid_tid[ith] : dummy; + // if single thread or file transfer then definitely empty. + std::vector& negsrcgid_tid = nrnthreads_netcon_negsrcgid_tid[ith]; size_t i_tid = 0; for (int i = 0; i < nt.n_netcon; ++i) { int gid = nrnthreads_netcon_srcgid[ith][i]; @@ -384,8 +384,8 @@ void determine_inputpresyn() { // only used via ps.nc_index_ and ps.nc_cnt_; for (int ith = 0; ith < nrn_nthread; ++ith) { NrnThread& nt = nrn_threads[ith]; - std::vector dummy; - std::vector& negsrcgid_tid = corenrn_embedded ? nrnthreads_netcon_negsrcgid_tid[ith] : dummy; + // if single thread or file transfer then definitely empty. + std::vector& negsrcgid_tid = nrnthreads_netcon_negsrcgid_tid[ith]; size_t i_tid = 0; for (int i = 0; i < nt.n_netcon; ++i) { NetCon* nc = nt.netcons + i; @@ -497,10 +497,10 @@ void nrn_setup(const char* filesdat, nrn_partrans::gap_mpi_setup(userParams.ngroup); } + nrnthreads_netcon_negsrcgid_tid.resize(nrn_nthread); if (!corenrn_embedded) { coreneuron::phase_wrapper(userParams); } else { - nrnthreads_netcon_negsrcgid_tid.resize(nrn_nthread); nrn_multithread_job([](NrnThread* n) { Phase1 p1; p1.read_direct(n->id); diff --git a/coreneuron/io/phase1.cpp b/coreneuron/io/phase1.cpp index d1cd1f134..5055ee98e 100644 --- a/coreneuron/io/phase1.cpp +++ b/coreneuron/io/phase1.cpp @@ -46,10 +46,6 @@ void Phase1::read_direct(int thread_id) { delete[] netcon_srcgid; } -extern "C" { -extern bool corenrn_embedded; -} - void Phase1::populate(NrnThread& nt, OMP_Mutex& mut) { nt.n_presyn = this->output_gids.size(); nt.n_netcon = this->netcon_srcgids.size(); @@ -58,9 +54,8 @@ void Phase1::populate(NrnThread& nt, OMP_Mutex& mut) { std::copy(this->netcon_srcgids.begin(), this->netcon_srcgids.end(), nrnthreads_netcon_srcgid[nt.id]); - if (corenrn_embedded) { // multiple threads and direct mode. - coreneuron::nrnthreads_netcon_negsrcgid_tid[nt.id] = this->netcon_negsrcgid_tid; - } + // netcon_negsrcgid_tid is empty if file transfer or single thread + coreneuron::nrnthreads_netcon_negsrcgid_tid[nt.id] = this->netcon_negsrcgid_tid; nt.netcons = new NetCon[nt.n_netcon]; nt.presyns_helper = (PreSynHelper*)ecalloc_align(nt.n_presyn, sizeof(PreSynHelper)); diff --git a/coreneuron/network/netpar.cpp b/coreneuron/network/netpar.cpp index e860ea6f1..13fdf9e6d 100644 --- a/coreneuron/network/netpar.cpp +++ b/coreneuron/network/netpar.cpp @@ -619,10 +619,6 @@ void BBS_netpar_solve(double tstop) { } } -extern "C" { -extern bool corenrn_embedded; -} - double set_mindelay(double maxdelay) { double mindelay = maxdelay; last_maxstep_arg_ = maxdelay; @@ -645,8 +641,8 @@ double set_mindelay(double maxdelay) { for (int ith = 0; ith < nrn_nthread; ++ith) { NrnThread& nt = nrn_threads[ith]; - std::vector dummy; - std::vector& negsrcgid_tid = corenrn_embedded ? nrnthreads_netcon_negsrcgid_tid[ith] : dummy; + // if single thread or file transfer then definitely empty. + std::vector& negsrcgid_tid = nrnthreads_netcon_negsrcgid_tid[ith]; size_t i_tid = 0; for (int i = 0; i < nt.n_netcon; ++i) { NetCon* nc = nt.netcons + i;