zcy
/
openEMS
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MPI: update for MPI usage 

Signed-off-by: Thorsten Liebig <Thorsten.Liebig@gmx.de>
pull/14/head
Thorsten Liebig 2016-01-27 18:37:07 +01:00
parent 7c0d75c930
commit d75069cfef
7 changed files with 108 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
CMakeLists.txt
View File

@ -59,6 +59,16 @@ if(UNIX AND ENABLE_RPATH)
set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
endif()
if (WITH_MPI)
ADD_DEFINITIONS(-DMPI_SUPPORT)
# Require MPI for this project:
find_package(MPI REQUIRED)
INCLUDE_DIRECTORIES( ${MPI_INCLUDE_PATH} )
message(STATUS "Found MPI. INCLUDE " ${MPI_INCLUDE_PATH})
endif()
# Set locations of extra CMake modules
SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${openEMS_SOURCE_DIR}/cmake/Modules/")
@ -158,6 +168,13 @@ INCLUDE_DIRECTORIES( ${openEMS_SOURCE_DIR} ) #find tools
message(STATUS "Sources:" ${SOURCES})
if (WITH_MPI)
set(CMAKE_CXX_COMPILE_FLAGS ${CMAKE_CXX_COMPILE_FLAGS} ${MPI_COMPILE_FLAGS})
set(CMAKE_CXX_LINK_FLAGS ${CMAKE_CXX_LINK_FLAGS} ${MPI_LINK_FLAGS})
SET(CMAKE_C_COMPILER /usr/bin/mpicc)
SET(CMAKE_CXX_COMPILER /usr/bin/mpicxx)
endif()
add_library( openEMS SHARED ${SOURCES})
set_target_properties(openEMS PROPERTIES VERSION ${LIB_VERSION_STRING} SOVERSION ${LIB_VERSION_MAJOR})
TARGET_LINK_LIBRARIES( openEMS
@ -167,6 +184,7 @@ TARGET_LINK_LIBRARIES( openEMS
${HDF5_LIBRARIES}
${Boost_LIBRARIES}
${vtk_LIBS}
${MPI_LIBRARIES}
)
# main program
@ -176,10 +194,16 @@ TARGET_LINK_LIBRARIES(openEMS_bin openEMS)
INSTALL(TARGETS openEMS DESTINATION lib${LIB_SUFFIX})
INSTALL(TARGETS openEMS_bin DESTINATION bin )
if (UNIX)
INSTALL( FILES openEMS.sh
DESTINATION bin
PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ OWNER_EXECUTE GROUP_EXECUTE WORLD_EXECUTE)
if (WITH_MPI)
INSTALL( FILES openEMS_MPI.sh
DESTINATION bin
PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ OWNER_EXECUTE GROUP_EXECUTE WORLD_EXECUTE)
endif()
endif ()
INSTALL( DIRECTORY matlab DESTINATION share/openEMS )
# TODO mpi, tarball, debug, release

13
FDTD/CMakeLists.txt
View File

@ -1,8 +1,15 @@
#INCLUDE_DIRECTORIES( ${openEMS_SOURCE_DIR} )
if (WITH_MPI)
set(MPI_SOURCES
${CMAKE_CURRENT_SOURCE_DIR}/openems_fdtd_mpi.cpp
${CMAKE_CURRENT_SOURCE_DIR}/operator_mpi.cpp
${CMAKE_CURRENT_SOURCE_DIR}/engine_mpi.cpp
)
endif()
set(SOURCES
${SOURCES}
${MPI_SOURCES}
${CMAKE_CURRENT_SOURCE_DIR}/engine.cpp
${CMAKE_CURRENT_SOURCE_DIR}/operator.cpp
${CMAKE_CURRENT_SOURCE_DIR}/engine_multithread.cpp
@ -21,7 +28,3 @@ set(SOURCES
${CMAKE_CURRENT_SOURCE_DIR}/engine_interface_cylindrical_fdtd.cpp
PARENT_SCOPE
)
# FDTD lib
#add_library( FDTD STATIC ${SOURCES} )

6
FDTD/engine_mpi.cpp
View File

@ -83,9 +83,6 @@ void Engine_MPI::Reset()
void Engine_MPI::SendReceiveVoltages()
{
if (!m_Op_MPI->GetMPIEnabled())
return;
unsigned int pos[3];
//non-blocking prepare for receive...
@ -136,9 +133,6 @@ void Engine_MPI::SendReceiveVoltages()
void Engine_MPI::SendReceiveCurrents()
{
if (!m_Op_MPI->GetMPIEnabled())
return;
unsigned int pos[3];
//non-blocking prepare for receive...

87
FDTD/openems_fdtd_mpi.cpp
View File

@ -60,6 +60,7 @@ openEMS_FDTD_MPI::openEMS_FDTD_MPI(bool m_MPI_Debug) : openEMS()
m_Energy_Buffer = NULL;
}
m_MPI_Elem = NULL;
m_Original_Grid = NULL;
//redirect output to file for all ranks > 0
@ -129,18 +130,17 @@ bool openEMS_FDTD_MPI::parseCommandLineArgument( const char *argv )
return false;
}
bool openEMS_FDTD_MPI::SetupMPI(TiXmlElement* FDTD_Opts)
bool openEMS_FDTD_MPI::Parse_XML_FDTDSetup(TiXmlElement* FDTD_Opts)
{
TiXmlElement* MPI_Elem = FDTD_Opts->FirstChildElement("MPI");
m_MPI_Elem = FDTD_Opts->FirstChildElement("MPI");
if (!m_MPI_Enabled)
{
if ((MPI_Elem!=NULL))
if ((m_MPI_Elem!=NULL))
cerr << "openEMS_FDTD_MPI::SetupMPI: Warning: Number of MPI processes is 1, skipping MPI engine... " << endl;
return true;
}
if (MPI_Elem==NULL)
if (m_MPI_Elem==NULL)
{
MPI_Barrier(MPI_COMM_WORLD);
if (m_MyID==0)
@ -152,15 +152,14 @@ bool openEMS_FDTD_MPI::SetupMPI(TiXmlElement* FDTD_Opts)
delete m_Original_Grid;
m_Original_Grid = CSRectGrid::Clone(grid);
vector<unsigned int> SplitNumber[3];
string arg_Pos_Names[] = {"SplitPos_X", "SplitPos_Y", "SplitPos_Z"};
string arg_N_Names[] = {"SplitN_X", "SplitN_Y", "SplitN_Z"};
const char* tmp = NULL;
for (int n=0;n<3;++n)
{
SplitNumber[n].push_back(0);
tmp = MPI_Elem->Attribute(arg_Pos_Names[n].c_str());
m_SplitNumber[n].clear();
m_SplitNumber[n].push_back(0);
tmp = m_MPI_Elem->Attribute(arg_Pos_Names[n].c_str());
if (tmp) //check if a split position is requested
{
vector<double> SplitLines = SplitString2Double(tmp, ',');
@ -170,13 +169,13 @@ bool openEMS_FDTD_MPI::SetupMPI(TiXmlElement* FDTD_Opts)
{
line = m_Original_Grid->Snap2LineNumber(n, SplitLines.at(lineN), inside);
if (inside)
SplitNumber[n].push_back(line);
m_SplitNumber[n].push_back(line);
}
}
else //check if a number of splits is requested
{
int SplitN=0;
if (MPI_Elem->QueryIntAttribute( arg_N_Names[n].c_str(), &SplitN) == TIXML_SUCCESS)
if (m_MPI_Elem->QueryIntAttribute( arg_N_Names[n].c_str(), &SplitN) == TIXML_SUCCESS)
{
if (SplitN>1)
{
@ -186,20 +185,28 @@ bool openEMS_FDTD_MPI::SetupMPI(TiXmlElement* FDTD_Opts)
for (size_t i = 0; i<jobs.size()-1;++i)
{
line += jobs.at(i);
SplitNumber[n].push_back(line);
m_SplitNumber[n].push_back(line);
}
}
}
}
SplitNumber[n].push_back(m_Original_Grid->GetQtyLines(n)-1);
unique(SplitNumber[n].begin(), SplitNumber[n].end());
m_SplitNumber[n].push_back(m_Original_Grid->GetQtyLines(n)-1);
unique(m_SplitNumber[n].begin(), m_SplitNumber[n].end());
}
return openEMS::Parse_XML_FDTDSetup(FDTD_Opts);
}
bool openEMS_FDTD_MPI::SetupMPI()
{
if (!m_MPI_Enabled)
return true;
MPI_Barrier(MPI_COMM_WORLD);
//validate number of processes
unsigned int numProcs = (SplitNumber[0].size()-1)*(SplitNumber[1].size()-1)*(SplitNumber[2].size()-1);
unsigned int numProcs = (m_SplitNumber[0].size()-1)*(m_SplitNumber[1].size()-1)*(m_SplitNumber[2].size()-1);
if (numProcs!=m_NumProc)
{
if (m_MyID==0)
@ -209,26 +216,27 @@ bool openEMS_FDTD_MPI::SetupMPI(TiXmlElement* FDTD_Opts)
//create process table
unsigned int procN = 0;
unsigned int splits[] = {(unsigned int)SplitNumber[0].size()-1, (unsigned int)SplitNumber[1].size()-1, (unsigned int)SplitNumber[2].size()-1};
unsigned int splits[] = {(unsigned int)m_SplitNumber[0].size()-1, (unsigned int)m_SplitNumber[1].size()-1, (unsigned int)m_SplitNumber[2].size()-1};
m_MPI_Op->SetSplitNumbers(0,splits[0]);
m_MPI_Op->SetSplitNumbers(1,splits[1]);
m_MPI_Op->SetSplitNumbers(2,splits[2]);
unsigned int*** procTable=Create3DArray<unsigned int>(splits);
for (size_t i=0;i<SplitNumber[0].size()-1;++i)
for (size_t j=0;j<SplitNumber[1].size()-1;++j)
for (size_t k=0;k<SplitNumber[2].size()-1;++k)
for (size_t i=0;i<m_SplitNumber[0].size()-1;++i)
for (size_t j=0;j<m_SplitNumber[1].size()-1;++j)
for (size_t k=0;k<m_SplitNumber[2].size()-1;++k)
{
procTable[i][j][k] = procN;
++procN;
}
m_MPI_Op->SetProcessTable(procTable);
CSRectGrid* grid = m_CSX->GetGrid();
//assign mesh and neighbors to this process
for (size_t i=0;i<SplitNumber[0].size()-1;++i)
for (size_t i=0;i<m_SplitNumber[0].size()-1;++i)
{
for (size_t j=0;j<SplitNumber[1].size()-1;++j)
for (size_t j=0;j<m_SplitNumber[1].size()-1;++j)
{
for (size_t k=0;k<SplitNumber[2].size()-1;++k)
for (size_t k=0;k<m_SplitNumber[2].size()-1;++k)
{
if (procTable[i][j][k] == m_MyID)
{
@ -240,41 +248,41 @@ bool openEMS_FDTD_MPI::SetupMPI(TiXmlElement* FDTD_Opts)
grid->ClearLines(1);
grid->ClearLines(2);
for (unsigned int n=SplitNumber[0].at(i);n<=SplitNumber[0].at(i+1);++n)
for (unsigned int n=m_SplitNumber[0].at(i);n<=m_SplitNumber[0].at(i+1);++n)
grid->AddDiscLine(0, m_Original_Grid->GetLine(0,n) );
for (unsigned int n=SplitNumber[1].at(j);n<=SplitNumber[1].at(j+1);++n)
for (unsigned int n=m_SplitNumber[1].at(j);n<=m_SplitNumber[1].at(j+1);++n)
grid->AddDiscLine(1, m_Original_Grid->GetLine(1,n) );
for (unsigned int n=SplitNumber[2].at(k);n<=SplitNumber[2].at(k+1);++n)
for (unsigned int n=m_SplitNumber[2].at(k);n<=m_SplitNumber[2].at(k+1);++n)
grid->AddDiscLine(2, m_Original_Grid->GetLine(2,n) );
m_MPI_Op->SetSplitPos(0,SplitNumber[0].at(i));
m_MPI_Op->SetSplitPos(1,SplitNumber[1].at(i));
m_MPI_Op->SetSplitPos(2,SplitNumber[2].at(i));
m_MPI_Op->SetSplitPos(0,m_SplitNumber[0].at(i));
m_MPI_Op->SetSplitPos(1,m_SplitNumber[1].at(i));
m_MPI_Op->SetSplitPos(2,m_SplitNumber[2].at(i));
if (i>0)
m_MPI_Op->SetNeighborDown(0,procTable[i-1][j][k]);
if (i<SplitNumber[0].size()-2)
if (i<m_SplitNumber[0].size()-2)
{
//add one additional line
grid->AddDiscLine(0, m_Original_Grid->GetLine(0,SplitNumber[0].at(i+1)+1 ));
grid->AddDiscLine(0, m_Original_Grid->GetLine(0,m_SplitNumber[0].at(i+1)+1 ));
m_MPI_Op->SetNeighborUp(0,procTable[i+1][j][k]);
}
if (j>0)
m_MPI_Op->SetNeighborDown(1,procTable[i][j-1][k]);
if (j<SplitNumber[1].size()-2)
if (j<m_SplitNumber[1].size()-2)
{
//add one additional line
grid->AddDiscLine(1, m_Original_Grid->GetLine(1,SplitNumber[1].at(j+1)+1 ));
grid->AddDiscLine(1, m_Original_Grid->GetLine(1,m_SplitNumber[1].at(j+1)+1 ));
m_MPI_Op->SetNeighborUp(1,procTable[i][j+1][k]);
}
if (k>0)
m_MPI_Op->SetNeighborDown(2,procTable[i][j][k-1]);
if (k<SplitNumber[2].size()-2)
if (k<m_SplitNumber[2].size()-2)
{
//add one additional line
grid->AddDiscLine(2, m_Original_Grid->GetLine(2,SplitNumber[2].at(k+1)+1 ));
grid->AddDiscLine(2, m_Original_Grid->GetLine(2,m_SplitNumber[2].at(k+1)+1 ));
m_MPI_Op->SetNeighborUp(2,procTable[i][j][k+1]);
}
@ -291,7 +299,7 @@ bool openEMS_FDTD_MPI::SetupMPI(TiXmlElement* FDTD_Opts)
}
bool openEMS_FDTD_MPI::SetupOperator(TiXmlElement* FDTD_Opts)
bool openEMS_FDTD_MPI::SetupOperator()
{
bool ret = true;
if (m_engine == EngineType_MPI)
@ -300,7 +308,7 @@ bool openEMS_FDTD_MPI::SetupOperator(TiXmlElement* FDTD_Opts)
}
else
{
ret = openEMS::SetupOperator(FDTD_Opts);
ret = openEMS::SetupOperator();
}
m_MPI_Op = dynamic_cast<Operator_MPI*>(FDTD_Op);
@ -312,7 +320,7 @@ bool openEMS_FDTD_MPI::SetupOperator(TiXmlElement* FDTD_Opts)
exit(0);
}
ret &=SetupMPI(FDTD_Opts);
ret &= SetupMPI();
return ret;
}
@ -421,6 +429,11 @@ bool openEMS_FDTD_MPI::SetupProcessing()
return ret;
}
int openEMS_FDTD_MPI::SetupFDTD()
{
return openEMS::SetupFDTD();
}
void openEMS_FDTD_MPI::RunFDTD()
{
if (!m_MPI_Enabled)

12
FDTD/openems_fdtd_mpi.h
View File

@ -30,11 +30,13 @@ public:
openEMS_FDTD_MPI(bool m_MPI_Debug=false);
virtual ~openEMS_FDTD_MPI();
virtual bool Parse_XML_FDTDSetup(TiXmlElement* FDTD_Opts);
virtual int SetupFDTD();
virtual void RunFDTD();
virtual bool parseCommandLineArgument( const char *argv );
static string GetExtLibsInfo();
static std::string GetExtLibsInfo();
protected:
Operator_MPI* m_MPI_Op;
@ -44,8 +46,10 @@ protected:
bool m_MPI_Enabled;
unsigned int m_NumberCells;
virtual bool SetupMPI(TiXmlElement* FDTD_Opts);
virtual bool SetupOperator(TiXmlElement* FDTD_Opts);
std::vector<unsigned int> m_SplitNumber[3];
TiXmlElement* m_MPI_Elem;
virtual bool SetupMPI();
virtual bool SetupOperator();
int* m_Gather_Buffer;
unsigned int GetNextStep();
@ -62,7 +66,7 @@ protected:
virtual bool SetupProcessing();
//output redirection to file for ranks > 0
ofstream* m_Output;
std::ofstream* m_Output;
};
#endif // OPENEMS_FDTD_MPI_H

26
openems.cpp
View File

@ -284,7 +284,7 @@ void openEMS::WelcomeScreen()
cout << " ---------------------------------------------------------------------- " << endl;
cout << " | openEMS " << bits << " -- version " GIT_VERSION << endl;
cout << " | (C) 2010-2015 Thorsten Liebig <thorsten.liebig@gmx.de> GPL license" << endl;
cout << " | (C) 2010-2016 Thorsten Liebig <thorsten.liebig@gmx.de> GPL license" << endl;
cout << " ---------------------------------------------------------------------- " << endl;
cout << openEMS::GetExtLibsInfo("\t") << endl;
}
@ -667,7 +667,6 @@ bool openEMS::ParseFDTDSetup(std::string file)
cerr << "Can't read openEMS ... " << endl;
exit(-1);
}
TiXmlElement* FDTD_Opts = openEMSxml->FirstChildElement("FDTD");
if (FDTD_Opts==NULL)
@ -675,6 +674,21 @@ bool openEMS::ParseFDTDSetup(std::string file)
cerr << "Can't read openEMS FDTD Settings... " << endl;
exit(-1);
}
if (g_settings.GetVerboseLevel()>0)
cout << "Read Geometry..." << endl;
ContinuousStructure* csx = new ContinuousStructure();
string EC(csx->ReadFromXML(openEMSxml));
if (EC.empty()==false)
cerr << EC << endl;
this->SetCSX(csx);
return this->Parse_XML_FDTDSetup(FDTD_Opts);
}
bool openEMS::Parse_XML_FDTDSetup(TiXmlElement* FDTD_Opts)
{
double dhelp=0;
FDTD_Opts->QueryDoubleAttribute("NumberOfTimesteps",&dhelp);
if (dhelp<0)
@ -764,14 +778,6 @@ bool openEMS::ParseFDTDSetup(std::string file)
if (BC->QueryDoubleAttribute(mur_v_ph_names[n].c_str(),&dhelp) == TIXML_SUCCESS)
this->Set_Mur_PhaseVel(n, dhelp);
if (g_settings.GetVerboseLevel()>0)
cout << "Read Geometry..." << endl;
ContinuousStructure* csx = new ContinuousStructure();
string EC(csx->ReadFromXML(openEMSxml));
if (EC.empty()==false)
cerr << EC << endl;
this->SetCSX(csx);
TiXmlElement* m_Excite_Elem = FDTD_Opts->FirstChildElement("Excitation");
if (!m_Excite_Elem)
{

3
openems.h
View File

@ -49,7 +49,8 @@ public:
static void showUsage();
bool ParseFDTDSetup(std::string file);
int SetupFDTD();
virtual bool Parse_XML_FDTDSetup(TiXmlElement* openEMSxml);
virtual int SetupFDTD();
virtual void RunFDTD();
void Reset();