diff --git a/FDTD/engine_mpi.cpp b/FDTD/engine_mpi.cpp index de1a19c..28bfe76 100644 --- a/FDTD/engine_mpi.cpp +++ b/FDTD/engine_mpi.cpp @@ -39,9 +39,165 @@ void Engine_MPI::Init() { Engine_SSE_Compressed::Init(); + for (int i=0;i<3;++i) + { + m_BufferUp[i]=NULL; + m_BufferDown[i]=NULL; + m_BufferSize[i]=0; + } + + if (m_Op_MPI->m_NumProc>1) + { + // init buffers, nx*ny*2 for the tangential electric or magnetic fields at the interface + m_BufferSize[2] = m_Op_MPI->numLines[0]*m_Op_MPI->numLines[1]*2; + if (m_Op_MPI->m_NeighborDown[2]>=0) + { + m_BufferDown[2] = new float[m_BufferSize[2]]; + } + if (m_Op_MPI->m_NeighborUp[2]>=0) + { + m_BufferUp[2] = new float[m_BufferSize[2]]; + } + } + } void Engine_MPI::Reset() { + for (int i=0;i<3;++i) + { + delete[] m_BufferUp[i]; + delete[] m_BufferDown[i]; + m_BufferUp[i]=NULL; + m_BufferDown[i]=NULL; + m_BufferSize[i]=0; + } + Engine_SSE_Compressed::Reset(); } + +void Engine_MPI::SendReceiveVoltages() +{ + unsigned int pos[3]; + + //non-blocking prepare for receive... + for (int n=0;n<3;++n) + if (m_Op_MPI->m_NeighborDown[n]>=0) + MPI_Irecv( m_BufferDown[n] , m_BufferSize[n], MPI_FLOAT, m_Op_MPI->m_NeighborDown[n], m_Op_MPI->m_MyTag, MPI_COMM_WORLD, &Recv_Request[n]); + + for (int n=0;n<3;++n) + { + int nP = (n+1)%3; + int nPP = (n+2)%3; + + //send voltages + unsigned int iPos=0; + pos[n]=numLines[n]-1; + if (m_Op_MPI->m_NeighborUp[n]>=0) + { + for (pos[nP]=0; pos[nP]m_NeighborUp[n], m_Op_MPI->m_MyTag, MPI_COMM_WORLD, &Send_Request[n]); + } + + //receive voltages + pos[n]=0; + iPos=0; + if (m_Op_MPI->m_NeighborDown[n]>=0) + { + //wait for receive to finish... + MPI_Wait(&Recv_Request[n],&stat); + for (pos[nP]=0; pos[nP]m_NeighborUp[n]>=0) + MPI_Irecv( m_BufferUp[n] , m_BufferSize[n], MPI_FLOAT, m_Op_MPI->m_NeighborUp[n], m_Op_MPI->m_MyTag, MPI_COMM_WORLD, &Recv_Request[n]); + + for (int n=0;n<3;++n) + { + int nP = (n+1)%3; + int nPP = (n+2)%3; + + //send currents + unsigned int iPos=0; + pos[n]=0; + if (m_Op_MPI->m_NeighborDown[n]>=0) + { + for (pos[nP]=0; pos[nP]m_NeighborDown[n], m_Op_MPI->m_MyTag, MPI_COMM_WORLD, &Send_Request[n]); + } + + //receive currents + pos[n]=numLines[n]-1; + iPos=0; + if (m_Op_MPI->m_NeighborUp[n]>=0) + { + //wait for receive to finish... + MPI_Wait(&Recv_Request[n],&stat); + for (pos[nP]=0; pos[nP]1) + { + CSRectGrid* grid = geo->GetGrid(); + int nz = grid->GetQtyLines(2); + std::vector jobs = AssignJobs2Threads(nz, m_NumProc); + double z_lines[jobs.at(m_MyID)]; + unsigned int z_start=0; + for (int n=0;nGetLine(2,z_start+n); + grid->ClearLines(2); + grid->AddDiscLines(2,jobs.at(m_MyID),z_lines); + + //lower neighbor is ID-1 + if (m_MyID>0) + m_NeighborDown[2]=m_MyID-1; + //upper neighbor is ID+1 + if (m_MyID=0) + BCs[2*n+1] = 0; + if (m_NeighborDown[n]>=0) + BCs[2*n] = 0; + } + Operator_SSE_Compressed::SetBoundaryCondition(BCs); +} + +void Operator_MPI::ApplyElectricBC(bool* dirs) +{ + for (int n=0;n<3;++n) + { + //do not delete operator at upper inteface + if (m_NeighborUp[n]>=0) + dirs[2*n+1] = false; + } + Operator_SSE_Compressed::ApplyElectricBC(dirs); +} + Engine* Operator_MPI::CreateEngine() const { - Engine_MPI* eng = Engine_MPI::New(this); - return eng; + if (m_NumProc>1) + return Engine_MPI::New(this); + else + return Engine_SSE_Compressed::New(this); } void Operator_MPI::Init() { - Operator_sse::Init(); + Operator_SSE_Compressed::Init(); + + m_MyTag = 0; + + for (int i=0;i<3;++i) + { + m_NeighborUp[i]=-1; + m_NeighborDown[i]=-1; + } + + int namelen; + m_NumProc = MPI::COMM_WORLD.Get_size(); + m_MyID = MPI::COMM_WORLD.Get_rank(); + + m_Processor_Name = new char[MPI_MAX_PROCESSOR_NAME]; + MPI::Get_processor_name(m_Processor_Name,namelen); + + cerr << "Operator_MPI::Init(): Running on " << m_Processor_Name << endl; } void Operator_MPI::Delete() { + delete[] m_Processor_Name; } void Operator_MPI::Reset() { Delete(); - Operator_sse::Reset(); + Operator_SSE_Compressed::Reset(); } diff --git a/FDTD/operator_mpi.h b/FDTD/operator_mpi.h index c00f29b..4cae2fb 100644 --- a/FDTD/operator_mpi.h +++ b/FDTD/operator_mpi.h @@ -22,11 +22,17 @@ class Operator_MPI : public Operator_SSE_Compressed { + friend class Engine_MPI; public: //! Create a new operator static Operator_MPI* New(); virtual ~Operator_MPI(); + virtual bool SetGeometryCSX(ContinuousStructure* geo); + + virtual void SetBoundaryCondition(int* BCs); + virtual void ApplyElectricBC(bool* dirs); + virtual Engine* CreateEngine() const; protected: @@ -34,6 +40,15 @@ protected: virtual void Init(); void Delete(); virtual void Reset(); + + int m_MyID; + int m_NumProc; + int m_MyTag; + char* m_Processor_Name; + + //the up and down neighbors, -1 if non for the given direction + int m_NeighborUp[3]; + int m_NeighborDown[3]; }; #endif // OPERATOR_MPI_H