From d75069cfef465c052a3871b28c601e03e7ade4a7 Mon Sep 17 00:00:00 2001 From: Thorsten Liebig Date: Wed, 27 Jan 2016 18:37:07 +0100 Subject: [PATCH] MPI: update for MPI usage Signed-off-by: Thorsten Liebig --- CMakeLists.txt | 24 +++++++++++ FDTD/CMakeLists.txt | 13 +++--- FDTD/engine_mpi.cpp | 6 --- FDTD/openems_fdtd_mpi.cpp | 87 ++++++++++++++++++++++----------------- FDTD/openems_fdtd_mpi.h | 12 ++++-- openems.cpp | 26 +++++++----- openems.h | 3 +- 7 files changed, 108 insertions(+), 63 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 90427e5..5ed737d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,6 +59,16 @@ if(UNIX AND ENABLE_RPATH) set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) endif() + +if (WITH_MPI) + ADD_DEFINITIONS(-DMPI_SUPPORT) + # Require MPI for this project: + find_package(MPI REQUIRED) + INCLUDE_DIRECTORIES( ${MPI_INCLUDE_PATH} ) + + message(STATUS "Found MPI. INCLUDE " ${MPI_INCLUDE_PATH}) +endif() + # Set locations of extra CMake modules SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${openEMS_SOURCE_DIR}/cmake/Modules/") @@ -158,6 +168,13 @@ INCLUDE_DIRECTORIES( ${openEMS_SOURCE_DIR} ) #find tools message(STATUS "Sources:" ${SOURCES}) +if (WITH_MPI) + set(CMAKE_CXX_COMPILE_FLAGS ${CMAKE_CXX_COMPILE_FLAGS} ${MPI_COMPILE_FLAGS}) + set(CMAKE_CXX_LINK_FLAGS ${CMAKE_CXX_LINK_FLAGS} ${MPI_LINK_FLAGS}) + SET(CMAKE_C_COMPILER /usr/bin/mpicc) + SET(CMAKE_CXX_COMPILER /usr/bin/mpicxx) +endif() + add_library( openEMS SHARED ${SOURCES}) set_target_properties(openEMS PROPERTIES VERSION ${LIB_VERSION_STRING} SOVERSION ${LIB_VERSION_MAJOR}) TARGET_LINK_LIBRARIES( openEMS @@ -167,6 +184,7 @@ TARGET_LINK_LIBRARIES( openEMS ${HDF5_LIBRARIES} ${Boost_LIBRARIES} ${vtk_LIBS} + ${MPI_LIBRARIES} ) # main program @@ -176,10 +194,16 @@ TARGET_LINK_LIBRARIES(openEMS_bin openEMS) INSTALL(TARGETS openEMS DESTINATION lib${LIB_SUFFIX}) INSTALL(TARGETS openEMS_bin DESTINATION bin ) + if (UNIX) INSTALL( FILES openEMS.sh DESTINATION bin PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ OWNER_EXECUTE GROUP_EXECUTE WORLD_EXECUTE) + if (WITH_MPI) + INSTALL( FILES openEMS_MPI.sh + DESTINATION bin + PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ OWNER_EXECUTE GROUP_EXECUTE WORLD_EXECUTE) + endif() endif () INSTALL( DIRECTORY matlab DESTINATION share/openEMS ) # TODO mpi, tarball, debug, release diff --git a/FDTD/CMakeLists.txt b/FDTD/CMakeLists.txt index 630f611..b0358d3 100644 --- a/FDTD/CMakeLists.txt +++ b/FDTD/CMakeLists.txt @@ -1,8 +1,15 @@ -#INCLUDE_DIRECTORIES( ${openEMS_SOURCE_DIR} ) +if (WITH_MPI) + set(MPI_SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/openems_fdtd_mpi.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/operator_mpi.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/engine_mpi.cpp + ) +endif() set(SOURCES ${SOURCES} + ${MPI_SOURCES} ${CMAKE_CURRENT_SOURCE_DIR}/engine.cpp ${CMAKE_CURRENT_SOURCE_DIR}/operator.cpp ${CMAKE_CURRENT_SOURCE_DIR}/engine_multithread.cpp @@ -21,7 +28,3 @@ set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/engine_interface_cylindrical_fdtd.cpp PARENT_SCOPE ) - -# FDTD lib -#add_library( FDTD STATIC ${SOURCES} ) - diff --git a/FDTD/engine_mpi.cpp b/FDTD/engine_mpi.cpp index 8334e0d..c6a5b2c 100644 --- a/FDTD/engine_mpi.cpp +++ b/FDTD/engine_mpi.cpp @@ -83,9 +83,6 @@ void Engine_MPI::Reset() void Engine_MPI::SendReceiveVoltages() { - if (!m_Op_MPI->GetMPIEnabled()) - return; - unsigned int pos[3]; //non-blocking prepare for receive... @@ -136,9 +133,6 @@ void Engine_MPI::SendReceiveVoltages() void Engine_MPI::SendReceiveCurrents() { - if (!m_Op_MPI->GetMPIEnabled()) - return; - unsigned int pos[3]; //non-blocking prepare for receive... diff --git a/FDTD/openems_fdtd_mpi.cpp b/FDTD/openems_fdtd_mpi.cpp index 0e94885..746d0c5 100644 --- a/FDTD/openems_fdtd_mpi.cpp +++ b/FDTD/openems_fdtd_mpi.cpp @@ -60,6 +60,7 @@ openEMS_FDTD_MPI::openEMS_FDTD_MPI(bool m_MPI_Debug) : openEMS() m_Energy_Buffer = NULL; } + m_MPI_Elem = NULL; m_Original_Grid = NULL; //redirect output to file for all ranks > 0 @@ -129,18 +130,17 @@ bool openEMS_FDTD_MPI::parseCommandLineArgument( const char *argv ) return false; } -bool openEMS_FDTD_MPI::SetupMPI(TiXmlElement* FDTD_Opts) +bool openEMS_FDTD_MPI::Parse_XML_FDTDSetup(TiXmlElement* FDTD_Opts) { - TiXmlElement* MPI_Elem = FDTD_Opts->FirstChildElement("MPI"); - + m_MPI_Elem = FDTD_Opts->FirstChildElement("MPI"); if (!m_MPI_Enabled) { - if ((MPI_Elem!=NULL)) + if ((m_MPI_Elem!=NULL)) cerr << "openEMS_FDTD_MPI::SetupMPI: Warning: Number of MPI processes is 1, skipping MPI engine... " << endl; return true; } - if (MPI_Elem==NULL) + if (m_MPI_Elem==NULL) { MPI_Barrier(MPI_COMM_WORLD); if (m_MyID==0) @@ -152,15 +152,14 @@ bool openEMS_FDTD_MPI::SetupMPI(TiXmlElement* FDTD_Opts) delete m_Original_Grid; m_Original_Grid = CSRectGrid::Clone(grid); - vector SplitNumber[3]; - string arg_Pos_Names[] = {"SplitPos_X", "SplitPos_Y", "SplitPos_Z"}; string arg_N_Names[] = {"SplitN_X", "SplitN_Y", "SplitN_Z"}; const char* tmp = NULL; for (int n=0;n<3;++n) { - SplitNumber[n].push_back(0); - tmp = MPI_Elem->Attribute(arg_Pos_Names[n].c_str()); + m_SplitNumber[n].clear(); + m_SplitNumber[n].push_back(0); + tmp = m_MPI_Elem->Attribute(arg_Pos_Names[n].c_str()); if (tmp) //check if a split position is requested { vector SplitLines = SplitString2Double(tmp, ','); @@ -170,13 +169,13 @@ bool openEMS_FDTD_MPI::SetupMPI(TiXmlElement* FDTD_Opts) { line = m_Original_Grid->Snap2LineNumber(n, SplitLines.at(lineN), inside); if (inside) - SplitNumber[n].push_back(line); + m_SplitNumber[n].push_back(line); } } else //check if a number of splits is requested { int SplitN=0; - if (MPI_Elem->QueryIntAttribute( arg_N_Names[n].c_str(), &SplitN) == TIXML_SUCCESS) + if (m_MPI_Elem->QueryIntAttribute( arg_N_Names[n].c_str(), &SplitN) == TIXML_SUCCESS) { if (SplitN>1) { @@ -186,20 +185,28 @@ bool openEMS_FDTD_MPI::SetupMPI(TiXmlElement* FDTD_Opts) for (size_t i = 0; iGetQtyLines(n)-1); - unique(SplitNumber[n].begin(), SplitNumber[n].end()); + m_SplitNumber[n].push_back(m_Original_Grid->GetQtyLines(n)-1); + unique(m_SplitNumber[n].begin(), m_SplitNumber[n].end()); } + return openEMS::Parse_XML_FDTDSetup(FDTD_Opts); +} + +bool openEMS_FDTD_MPI::SetupMPI() +{ + if (!m_MPI_Enabled) + return true; + MPI_Barrier(MPI_COMM_WORLD); //validate number of processes - unsigned int numProcs = (SplitNumber[0].size()-1)*(SplitNumber[1].size()-1)*(SplitNumber[2].size()-1); + unsigned int numProcs = (m_SplitNumber[0].size()-1)*(m_SplitNumber[1].size()-1)*(m_SplitNumber[2].size()-1); if (numProcs!=m_NumProc) { if (m_MyID==0) @@ -209,26 +216,27 @@ bool openEMS_FDTD_MPI::SetupMPI(TiXmlElement* FDTD_Opts) //create process table unsigned int procN = 0; - unsigned int splits[] = {(unsigned int)SplitNumber[0].size()-1, (unsigned int)SplitNumber[1].size()-1, (unsigned int)SplitNumber[2].size()-1}; + unsigned int splits[] = {(unsigned int)m_SplitNumber[0].size()-1, (unsigned int)m_SplitNumber[1].size()-1, (unsigned int)m_SplitNumber[2].size()-1}; m_MPI_Op->SetSplitNumbers(0,splits[0]); m_MPI_Op->SetSplitNumbers(1,splits[1]); m_MPI_Op->SetSplitNumbers(2,splits[2]); unsigned int*** procTable=Create3DArray(splits); - for (size_t i=0;iSetProcessTable(procTable); + CSRectGrid* grid = m_CSX->GetGrid(); //assign mesh and neighbors to this process - for (size_t i=0;iClearLines(1); grid->ClearLines(2); - for (unsigned int n=SplitNumber[0].at(i);n<=SplitNumber[0].at(i+1);++n) + for (unsigned int n=m_SplitNumber[0].at(i);n<=m_SplitNumber[0].at(i+1);++n) grid->AddDiscLine(0, m_Original_Grid->GetLine(0,n) ); - for (unsigned int n=SplitNumber[1].at(j);n<=SplitNumber[1].at(j+1);++n) + for (unsigned int n=m_SplitNumber[1].at(j);n<=m_SplitNumber[1].at(j+1);++n) grid->AddDiscLine(1, m_Original_Grid->GetLine(1,n) ); - for (unsigned int n=SplitNumber[2].at(k);n<=SplitNumber[2].at(k+1);++n) + for (unsigned int n=m_SplitNumber[2].at(k);n<=m_SplitNumber[2].at(k+1);++n) grid->AddDiscLine(2, m_Original_Grid->GetLine(2,n) ); - m_MPI_Op->SetSplitPos(0,SplitNumber[0].at(i)); - m_MPI_Op->SetSplitPos(1,SplitNumber[1].at(i)); - m_MPI_Op->SetSplitPos(2,SplitNumber[2].at(i)); + m_MPI_Op->SetSplitPos(0,m_SplitNumber[0].at(i)); + m_MPI_Op->SetSplitPos(1,m_SplitNumber[1].at(i)); + m_MPI_Op->SetSplitPos(2,m_SplitNumber[2].at(i)); if (i>0) m_MPI_Op->SetNeighborDown(0,procTable[i-1][j][k]); - if (iAddDiscLine(0, m_Original_Grid->GetLine(0,SplitNumber[0].at(i+1)+1 )); + grid->AddDiscLine(0, m_Original_Grid->GetLine(0,m_SplitNumber[0].at(i+1)+1 )); m_MPI_Op->SetNeighborUp(0,procTable[i+1][j][k]); } if (j>0) m_MPI_Op->SetNeighborDown(1,procTable[i][j-1][k]); - if (jAddDiscLine(1, m_Original_Grid->GetLine(1,SplitNumber[1].at(j+1)+1 )); + grid->AddDiscLine(1, m_Original_Grid->GetLine(1,m_SplitNumber[1].at(j+1)+1 )); m_MPI_Op->SetNeighborUp(1,procTable[i][j+1][k]); } if (k>0) m_MPI_Op->SetNeighborDown(2,procTable[i][j][k-1]); - if (kAddDiscLine(2, m_Original_Grid->GetLine(2,SplitNumber[2].at(k+1)+1 )); + grid->AddDiscLine(2, m_Original_Grid->GetLine(2,m_SplitNumber[2].at(k+1)+1 )); m_MPI_Op->SetNeighborUp(2,procTable[i][j][k+1]); } @@ -291,7 +299,7 @@ bool openEMS_FDTD_MPI::SetupMPI(TiXmlElement* FDTD_Opts) } -bool openEMS_FDTD_MPI::SetupOperator(TiXmlElement* FDTD_Opts) +bool openEMS_FDTD_MPI::SetupOperator() { bool ret = true; if (m_engine == EngineType_MPI) @@ -300,7 +308,7 @@ bool openEMS_FDTD_MPI::SetupOperator(TiXmlElement* FDTD_Opts) } else { - ret = openEMS::SetupOperator(FDTD_Opts); + ret = openEMS::SetupOperator(); } m_MPI_Op = dynamic_cast(FDTD_Op); @@ -312,7 +320,7 @@ bool openEMS_FDTD_MPI::SetupOperator(TiXmlElement* FDTD_Opts) exit(0); } - ret &=SetupMPI(FDTD_Opts); + ret &= SetupMPI(); return ret; } @@ -421,6 +429,11 @@ bool openEMS_FDTD_MPI::SetupProcessing() return ret; } +int openEMS_FDTD_MPI::SetupFDTD() +{ + return openEMS::SetupFDTD(); +} + void openEMS_FDTD_MPI::RunFDTD() { if (!m_MPI_Enabled) diff --git a/FDTD/openems_fdtd_mpi.h b/FDTD/openems_fdtd_mpi.h index 4894968..efb1f00 100644 --- a/FDTD/openems_fdtd_mpi.h +++ b/FDTD/openems_fdtd_mpi.h @@ -30,11 +30,13 @@ public: openEMS_FDTD_MPI(bool m_MPI_Debug=false); virtual ~openEMS_FDTD_MPI(); + virtual bool Parse_XML_FDTDSetup(TiXmlElement* FDTD_Opts); + virtual int SetupFDTD(); virtual void RunFDTD(); virtual bool parseCommandLineArgument( const char *argv ); - static string GetExtLibsInfo(); + static std::string GetExtLibsInfo(); protected: Operator_MPI* m_MPI_Op; @@ -44,8 +46,10 @@ protected: bool m_MPI_Enabled; unsigned int m_NumberCells; - virtual bool SetupMPI(TiXmlElement* FDTD_Opts); - virtual bool SetupOperator(TiXmlElement* FDTD_Opts); + std::vector m_SplitNumber[3]; + TiXmlElement* m_MPI_Elem; + virtual bool SetupMPI(); + virtual bool SetupOperator(); int* m_Gather_Buffer; unsigned int GetNextStep(); @@ -62,7 +66,7 @@ protected: virtual bool SetupProcessing(); //output redirection to file for ranks > 0 - ofstream* m_Output; + std::ofstream* m_Output; }; #endif // OPENEMS_FDTD_MPI_H diff --git a/openems.cpp b/openems.cpp index e5bd6ce..72476d3 100644 --- a/openems.cpp +++ b/openems.cpp @@ -284,7 +284,7 @@ void openEMS::WelcomeScreen() cout << " ---------------------------------------------------------------------- " << endl; cout << " | openEMS " << bits << " -- version " GIT_VERSION << endl; - cout << " | (C) 2010-2015 Thorsten Liebig GPL license" << endl; + cout << " | (C) 2010-2016 Thorsten Liebig GPL license" << endl; cout << " ---------------------------------------------------------------------- " << endl; cout << openEMS::GetExtLibsInfo("\t") << endl; } @@ -667,7 +667,6 @@ bool openEMS::ParseFDTDSetup(std::string file) cerr << "Can't read openEMS ... " << endl; exit(-1); } - TiXmlElement* FDTD_Opts = openEMSxml->FirstChildElement("FDTD"); if (FDTD_Opts==NULL) @@ -675,6 +674,21 @@ bool openEMS::ParseFDTDSetup(std::string file) cerr << "Can't read openEMS FDTD Settings... " << endl; exit(-1); } + + if (g_settings.GetVerboseLevel()>0) + cout << "Read Geometry..." << endl; + ContinuousStructure* csx = new ContinuousStructure(); + string EC(csx->ReadFromXML(openEMSxml)); + if (EC.empty()==false) + cerr << EC << endl; + this->SetCSX(csx); + + + return this->Parse_XML_FDTDSetup(FDTD_Opts); +} + +bool openEMS::Parse_XML_FDTDSetup(TiXmlElement* FDTD_Opts) +{ double dhelp=0; FDTD_Opts->QueryDoubleAttribute("NumberOfTimesteps",&dhelp); if (dhelp<0) @@ -764,14 +778,6 @@ bool openEMS::ParseFDTDSetup(std::string file) if (BC->QueryDoubleAttribute(mur_v_ph_names[n].c_str(),&dhelp) == TIXML_SUCCESS) this->Set_Mur_PhaseVel(n, dhelp); - if (g_settings.GetVerboseLevel()>0) - cout << "Read Geometry..." << endl; - ContinuousStructure* csx = new ContinuousStructure(); - string EC(csx->ReadFromXML(openEMSxml)); - if (EC.empty()==false) - cerr << EC << endl; - this->SetCSX(csx); - TiXmlElement* m_Excite_Elem = FDTD_Opts->FirstChildElement("Excitation"); if (!m_Excite_Elem) { diff --git a/openems.h b/openems.h index 1221a96..d194050 100644 --- a/openems.h +++ b/openems.h @@ -49,7 +49,8 @@ public: static void showUsage(); bool ParseFDTDSetup(std::string file); - int SetupFDTD(); + virtual bool Parse_XML_FDTDSetup(TiXmlElement* openEMSxml); + virtual int SetupFDTD(); virtual void RunFDTD(); void Reset();