diff --git a/FDTD/engine_mpi.cpp b/FDTD/engine_mpi.cpp index bf5139f..0db978b 100644 --- a/FDTD/engine_mpi.cpp +++ b/FDTD/engine_mpi.cpp @@ -78,6 +78,9 @@ void Engine_MPI::Reset() void Engine_MPI::SendReceiveVoltages() { + if (!m_Op_MPI->GetMPIEnabled()) + return; + unsigned int pos[3]; //non-blocking prepare for receive... @@ -95,9 +98,9 @@ void Engine_MPI::SendReceiveVoltages() pos[n]=numLines[n]-1; if (m_Op_MPI->m_NeighborUp[n]>=0) { - for (pos[nP]=0; pos[nP]GetMPIEnabled()) + return; + unsigned int pos[3]; //non-blocking prepare for receive... @@ -145,9 +151,9 @@ void Engine_MPI::SendReceiveCurrents() pos[n]=0; if (m_Op_MPI->m_NeighborDown[n]>=0) { - for (pos[nP]=0; pos[nP]1) + m_MPI_Enabled=true; + else + m_MPI_Enabled=false; + if (m_MyID==0) { m_Gather_Buffer = new int[m_NumProc]; @@ -82,14 +87,7 @@ bool openEMS_FDTD_MPI::SetupOperator(TiXmlElement* FDTD_Opts) { if (m_engine == EngineType_MPI) { - if (m_MyID>0) - { - //higher ranks never abort the simulation - endCrit = 0; - } - FDTD_Op = Operator_MPI::New(); - return true; } else @@ -200,11 +198,12 @@ bool openEMS_FDTD_MPI::SetupProcessing() } } } + return ret; } void openEMS_FDTD_MPI::RunFDTD() { - if (m_engine != EngineType_MPI) + if (!m_MPI_Enabled) return openEMS::RunFDTD(); cout << "Running MPI-FDTD engine... this may take a while... grab a cup of coffee?!?" << endl; @@ -256,6 +255,9 @@ void openEMS_FDTD_MPI::RunFDTD() if (CheckEnergyCalc()) currE = CalcEnergy(); + //make sure all processes are at the same simulation time + MPI_Bcast(&t_diff, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD); + if (t_diff>4) { if (currE==0) diff --git a/FDTD/openems_fdtd_mpi.h b/FDTD/openems_fdtd_mpi.h index f59d365..2ce7028 100644 --- a/FDTD/openems_fdtd_mpi.h +++ b/FDTD/openems_fdtd_mpi.h @@ -35,6 +35,7 @@ public: protected: int m_MyID; int m_NumProc; + bool m_MPI_Enabled; unsigned int m_NumberCells; virtual bool SetupOperator(TiXmlElement* FDTD_Opts); diff --git a/main.cpp b/main.cpp index 3835683..bc64168 100644 --- a/main.cpp +++ b/main.cpp @@ -100,6 +100,7 @@ int main(int argc, char *argv[]) FDTD.RunFDTD(); #ifdef MPI_SUPPORT + FDTD.Reset(); //make sure everything is cleaned-up before calling MPI::Finalize() MPI::Finalize(); #endif