From eb7524b6f600d08a15203c381df9a16d9248b136 Mon Sep 17 00:00:00 2001 From: Thorsten Liebig Date: Fri, 25 Feb 2011 11:58:23 +0100 Subject: [PATCH] MPI: Setup MPI now allows split in all directions with defitions read from xml-file Warning: depends on API changes in CSXCAD! --- FDTD/openems_fdtd_mpi.cpp | 141 ++++++++++++++++++++++++++++---------- FDTD/openems_fdtd_mpi.h | 2 + matlab/SetupMPI.m | 13 ++++ 3 files changed, 121 insertions(+), 35 deletions(-) create mode 100644 matlab/SetupMPI.m diff --git a/FDTD/openems_fdtd_mpi.cpp b/FDTD/openems_fdtd_mpi.cpp index 27b5b82..5c30024 100644 --- a/FDTD/openems_fdtd_mpi.cpp +++ b/FDTD/openems_fdtd_mpi.cpp @@ -32,6 +32,7 @@ #include #include "mpi.h" #include "tools/useful.h" +#include "tinyxml.h" openEMS_FDTD_MPI::openEMS_FDTD_MPI() : openEMS() { @@ -57,12 +58,18 @@ openEMS_FDTD_MPI::openEMS_FDTD_MPI() : openEMS() m_Gather_Buffer = NULL; m_Energy_Buffer = NULL; } + + m_Original_Grid = NULL; } openEMS_FDTD_MPI::~openEMS_FDTD_MPI() { delete[] m_Gather_Buffer; + m_Gather_Buffer = NULL; delete[] m_Energy_Buffer; + m_Energy_Buffer = NULL; + delete m_Original_Grid; + m_Original_Grid = NULL; } bool openEMS_FDTD_MPI::parseCommandLineArgument( const char *argv ) @@ -90,43 +97,107 @@ bool openEMS_FDTD_MPI::SetupMPI(TiXmlElement* FDTD_Opts) //manipulate geometry for this part... UNUSED(FDTD_Opts); - if (m_MPI_Enabled) + if (!m_MPI_Enabled) { - CSRectGrid* grid = m_CSX->GetGrid(); - int nz = grid->GetQtyLines(2); - std::vector jobs = AssignJobs2Threads(nz, m_NumProc); - double z_lines[jobs.at(m_MyID)+1]; - - if (m_MyID==0) - { - for (unsigned int n=0;nGetLine(2,n); - grid->ClearLines(2); - grid->AddDiscLines(2,jobs.at(0),z_lines); - - } - else - { - unsigned int z_start=0; - for (int n=0;nGetLine(2,z_start+n-1); - grid->ClearLines(2); - grid->AddDiscLines(2,jobs.at(m_MyID)+1,z_lines); - } - - //lower neighbor is ID-1 - if (m_MyID>0) - m_MPI_Op->SetNeighborDown(2,m_MyID-1); - //upper neighbor is ID+1 - if (m_MyIDSetNeighborUp(2,m_MyID+1); - - m_MPI_Op->SetTag(0); + if (g_settings.GetVerboseLevel()>0) + cerr << "openEMS_FDTD_MPI::SetupMPI: Warning: Number of MPI processes is 1, skipping MPI engine... " << endl; + return true; } - else - cerr << "openEMS_FDTD_MPI::SetupMPI: Warning: Number of MPI processes is 1, skipping MPI engine... " << endl; + + TiXmlElement* MPI_Elem = FDTD_Opts->FirstChildElement("MPI"); + + CSRectGrid* grid = m_CSX->GetGrid(); + delete m_Original_Grid; + m_Original_Grid = CSRectGrid::Clone(grid); + + vector SplitNumber[3]; + + string argNames[] = {"SplitPos_X", "SplitPos_Y", "SplitPos_Z"}; + const char* tmp = NULL; + for (int n=0;n<3;++n) + { + SplitNumber[n].push_back(0); + tmp = MPI_Elem->Attribute(argNames[n].c_str()); + if (tmp) + { + vector SplitLines = SplitString2Double(tmp, ','); + bool inside; + unsigned int line; + for (size_t lineN = 0; lineNSnap2LineNumber(n, SplitLines.at(lineN), inside); + if (inside) + SplitNumber[n].push_back(line); + } + } + + SplitNumber[n].push_back(m_Original_Grid->GetQtyLines(n)-1); + unique(SplitNumber[n].begin(), SplitNumber[n].end()); + } + + MPI_Barrier(MPI_COMM_WORLD); + + //validate number of processes + int numProcs = (SplitNumber[0].size()-1)*(SplitNumber[1].size()-1)*(SplitNumber[2].size()-1); + if (numProcs!=m_NumProc) + { + if (m_MyID==0) + cerr << "openEMS_FDTD_MPI::SetupMPI: Error: Requested splits require " << numProcs << " processes, but only " << m_NumProc << " were found! Exit! " << endl; + exit(10); + } + + //create process table + unsigned int procN = 0; + int procTable[SplitNumber[0].size()-1][SplitNumber[1].size()-1][SplitNumber[2].size()-1]; + for (size_t i=0;iClearLines(0); + grid->ClearLines(1); + grid->ClearLines(2); + + for (unsigned int n=SplitNumber[0].at(i);n<=SplitNumber[0].at(i+1);++n) + grid->AddDiscLine(0, m_Original_Grid->GetLine(0,n) ); + for (unsigned int n=SplitNumber[1].at(j);n<=SplitNumber[1].at(j+1);++n) + grid->AddDiscLine(1, m_Original_Grid->GetLine(1,n) ); + for (unsigned int n=SplitNumber[2].at(k);n<=SplitNumber[2].at(k+1);++n) + grid->AddDiscLine(2, m_Original_Grid->GetLine(2,n) ); + + if (i>0) + m_MPI_Op->SetNeighborDown(0,procTable[i-1][j][k]); + if (iSetNeighborUp(0,procTable[i+1][j][k]); + + if (j>0) + m_MPI_Op->SetNeighborDown(1,procTable[i][j-1][k]); + if (jSetNeighborUp(1,procTable[i][j+1][k]); + + if (k>0) + m_MPI_Op->SetNeighborDown(2,procTable[i][j][k-1]); + if (kSetNeighborUp(2,procTable[i][j][k+1]); + + } + } + } + } + + m_MPI_Op->SetTag(0); return true; } diff --git a/FDTD/openems_fdtd_mpi.h b/FDTD/openems_fdtd_mpi.h index 46e0085..dd38bfd 100644 --- a/FDTD/openems_fdtd_mpi.h +++ b/FDTD/openems_fdtd_mpi.h @@ -22,6 +22,7 @@ class ProcessFields; class Operator_MPI; +class CSRectGrid; class openEMS_FDTD_MPI : public openEMS { @@ -35,6 +36,7 @@ public: protected: Operator_MPI* m_MPI_Op; + CSRectGrid* m_Original_Grid; int m_MyID; int m_NumProc; bool m_MPI_Enabled; diff --git a/matlab/SetupMPI.m b/matlab/SetupMPI.m new file mode 100644 index 0000000..f941609 --- /dev/null +++ b/matlab/SetupMPI.m @@ -0,0 +1,13 @@ +function FDTD = SetupMPI(FDTD, varargin) +% function FDTD = SetupMPI(FDTD, varargin); +% +% % example +% FDTD = SetupMPI(FDTD,'SplitPos_X', '-500,500','SplitPos_Z', '-500,500'); +% +% openEMS matlab interface +% ----------------------- +% author: Thorsten Liebig + +for n=1:(nargin-1)/2 + FDTD.MPI.ATTRIBUTE.(varargin{2*n-1})=varargin{2*n}; +end