diff --git a/FDTD/engine_cylindermultigrid.cpp b/FDTD/engine_cylindermultigrid.cpp index 480f23a..39b6bad 100644 --- a/FDTD/engine_cylindermultigrid.cpp +++ b/FDTD/engine_cylindermultigrid.cpp @@ -62,6 +62,11 @@ Engine_CylinderMultiGrid::Engine_CylinderMultiGrid(const Operator_CylinderMultiG Engine_CylinderMultiGrid::~Engine_CylinderMultiGrid() { +#ifdef MPI_SUPPORT + delete m_InnerEngine->m_MPI_Barrier; + m_InnerEngine->m_MPI_Barrier = NULL; +#endif + m_Thread_NumTS = 0; m_startBarrier->wait(); @@ -81,7 +86,6 @@ Engine_CylinderMultiGrid::~Engine_CylinderMultiGrid() m_startBarrier = NULL; delete m_stopBarrier; m_stopBarrier = NULL; - } void Engine_CylinderMultiGrid::Init() @@ -103,6 +107,11 @@ void Engine_CylinderMultiGrid::Init() m_InnerEngine->SortExtensionByPriority(); SortExtensionByPriority(); + +#ifdef MPI_SUPPORT + //assign an MPI barrier to inner Engine + m_InnerEngine->m_MPI_Barrier = new boost::barrier(2); +#endif } bool Engine_CylinderMultiGrid::IterateTS(unsigned int iterTS) @@ -246,9 +255,29 @@ void Engine_CylinderMultiGrid::InterpolCurrChild2Base(unsigned int rzPlane) f4_curr[2][pos[0]][pos[1]][pos[2]].v = m_InnerEngine->f4_curr[2][pos[0]][pos[1]/2][pos[2]].v + one_fourth.v * m_InnerEngine->f4_curr[2][pos[0]][pos[1]/2][pos[2]].v - one_fourth.v * m_InnerEngine->f4_curr[2][pos[0]][pos[1]/2-1][pos[2]].v; } } - } +#ifdef MPI_SUPPORT + void Engine_CylinderMultiGrid::SendReceiveVoltages() + { + //do the local voltage sync, child is waiting... + Engine_Multithread::SendReceiveVoltages(); + + //run inner voltage sync + m_InnerEngine->m_MPI_Barrier->wait(); + } + + void Engine_CylinderMultiGrid::SendReceiveCurrents() + { + //do the local current sync, child is waiting... + Engine_Multithread::SendReceiveCurrents(); + + //run inner voltage sync + m_InnerEngine->m_MPI_Barrier->wait(); + } + +#endif + /****************************************************************************************/ Engine_CylinderMultiGrid_Thread::Engine_CylinderMultiGrid_Thread( Engine_Multithread* engine, boost::barrier *start, boost::barrier *stop, volatile unsigned int* numTS, bool isBase) { diff --git a/FDTD/engine_cylindermultigrid.h b/FDTD/engine_cylindermultigrid.h index 6adaa55..9776ba9 100644 --- a/FDTD/engine_cylindermultigrid.h +++ b/FDTD/engine_cylindermultigrid.h @@ -60,6 +60,11 @@ protected: boost::barrier *m_WaitOnSync; Engine_Ext_CylinderMultiGrid* m_Eng_Ext_MG; + +#ifdef MPI_SUPPORT + virtual void SendReceiveVoltages(); + virtual void SendReceiveCurrents(); +#endif }; diff --git a/FDTD/engine_mpi.h b/FDTD/engine_mpi.h index d4ceb81..5aa1226 100644 --- a/FDTD/engine_mpi.h +++ b/FDTD/engine_mpi.h @@ -49,9 +49,9 @@ protected: float* m_BufferDown[3]; //! Transfer all tangential voltages at the upper bounds to the lower bounds of the neighbouring MPI-processes - void SendReceiveVoltages(); + virtual void SendReceiveVoltages(); //! Transfer all tangential currents at the lower bounds to the upper bounds of the neighbouring MPI-processes - void SendReceiveCurrents(); + virtual void SendReceiveCurrents(); }; #endif // ENGINE_MPI_H diff --git a/FDTD/engine_multithread.cpp b/FDTD/engine_multithread.cpp index 1179b38..66e7e21 100644 --- a/FDTD/engine_multithread.cpp +++ b/FDTD/engine_multithread.cpp @@ -51,6 +51,10 @@ Engine_Multithread::Engine_Multithread(const Operator_Multithread* op) : ENGINE_ m_IterateBarrier = 0; m_startBarrier = 0; m_stopBarrier = 0; + +#ifdef ENABLE_DEBUG_TIME + m_MPI_Barrier = 0; +#endif } Engine_Multithread::~Engine_Multithread() @@ -100,6 +104,9 @@ void Engine_Multithread::Init() m_startBarrier = new boost::barrier(m_numThreads+1); // numThread workers + 1 controller m_stopBarrier = new boost::barrier(m_numThreads+1); // numThread workers + 1 controller +#ifdef MPI_SUPPORT + m_MPI_Barrier = 0; +#endif for (unsigned int n=0; nm_MPI_Barrier) + m_enginePtr->m_MPI_Barrier->wait(); m_enginePtr->SendReceiveVoltages(); + } m_enginePtr->m_IterateBarrier->wait(); #endif @@ -312,7 +323,11 @@ void thread::operator()() #ifdef MPI_SUPPORT if (m_threadID==0) + { + if (m_enginePtr->m_MPI_Barrier) + m_enginePtr->m_MPI_Barrier->wait(); m_enginePtr->SendReceiveCurrents(); + } m_enginePtr->m_IterateBarrier->wait(); #endif diff --git a/FDTD/engine_multithread.h b/FDTD/engine_multithread.h index 536c0f7..f0c8a5a 100644 --- a/FDTD/engine_multithread.h +++ b/FDTD/engine_multithread.h @@ -110,6 +110,15 @@ protected: unsigned int m_numThreads; //!< number of worker threads volatile bool m_stopThreads; +#ifdef MPI_SUPPORT + /*! Workaround needed for subgridding scheme... (see Engine_CylinderMultiGrid) + Some engines may need an additional barrier for synchronizing MPI communication. + This engine will not initialize or cleanup this barrier, but check for it and wait before executing any MPI sync. + Make sure to cleanup (delete) this barriere before Engine_Multithread::Reset() is called. + */ + boost::barrier *m_MPI_Barrier; +#endif + #ifdef ENABLE_DEBUG_TIME std::map > m_timer_list; #endif diff --git a/FDTD/operator_cylindermultigrid.cpp b/FDTD/operator_cylindermultigrid.cpp index 3b1a49a..e78e4d7 100644 --- a/FDTD/operator_cylindermultigrid.cpp +++ b/FDTD/operator_cylindermultigrid.cpp @@ -127,6 +127,26 @@ void Operator_CylinderMultiGrid::Init() m_InnerOp = Operator_CylinderMultiGrid::New(m_Split_Radii,m_numThreads); } +#ifdef MPI_SUPPORT +void Operator_CylinderMultiGrid::SetTag(int tag) +{ + m_MyTag = tag; + m_InnerOp->SetTag(tag+1); +} + +void Operator_CylinderMultiGrid::SetNeighborUp(int ny, int id) +{ + Operator_Cylinder::SetNeighborUp(ny,id); + m_InnerOp->SetNeighborUp(ny,id); +} + +void Operator_CylinderMultiGrid::SetNeighborDown(int ny, int id) +{ + Operator_Cylinder::SetNeighborDown(ny,id); + m_InnerOp->SetNeighborDown(ny,id); +} +#endif + void Operator_CylinderMultiGrid::CalcStartStopLines(unsigned int &numThreads, vector &start, vector &stop) const { vector jpt = AssignJobs2Threads(numLines[0]- m_Split_Pos + 1, numThreads, true); diff --git a/FDTD/operator_cylindermultigrid.h b/FDTD/operator_cylindermultigrid.h index dce639d..6e7a147 100644 --- a/FDTD/operator_cylindermultigrid.h +++ b/FDTD/operator_cylindermultigrid.h @@ -53,6 +53,12 @@ public: virtual void ShowStat() const; +#ifdef MPI_SUPPORT + virtual void SetTag(int tag); + virtual void SetNeighborUp(int ny, int id); + virtual void SetNeighborDown(int ny, int id); +#endif + protected: Operator_CylinderMultiGrid(vector Split_Radii); virtual void Init();