From e2628740fcbd475fb484495ec1820a5792af8cfa Mon Sep 17 00:00:00 2001 From: Thorsten Liebig Date: Fri, 11 Feb 2011 13:52:48 +0100 Subject: [PATCH] new openEMS_FDTD_MPI for better MPI handling --- FDTD/openems_fdtd_mpi.cpp | 236 ++++++++++++++++++++++++++++++++++++++ FDTD/openems_fdtd_mpi.h | 54 +++++++++ main.cpp | 9 +- openEMS.pro | 6 +- openems.cpp | 80 ++++++------- openems.h | 14 ++- 6 files changed, 344 insertions(+), 55 deletions(-) create mode 100644 FDTD/openems_fdtd_mpi.cpp create mode 100644 FDTD/openems_fdtd_mpi.h diff --git a/FDTD/openems_fdtd_mpi.cpp b/FDTD/openems_fdtd_mpi.cpp new file mode 100644 index 0000000..92b0b1a --- /dev/null +++ b/FDTD/openems_fdtd_mpi.cpp @@ -0,0 +1,236 @@ +/* +* Copyright (C) 2011 Thorsten Liebig (Thorsten.Liebig@gmx.de) +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY{} without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +*/ + +#include "openems_fdtd_mpi.h" +#include "FDTD/engine_interface_fdtd.h" +#include "FDTD/operator_mpi.h" +#include "FDTD/engine_mpi.h" +#include "Common/processfields.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "mpi.h" + +openEMS_FDTD_MPI::openEMS_FDTD_MPI() : openEMS() +{ + m_MyID = MPI::COMM_WORLD.Get_rank(); + m_NumProc = MPI::COMM_WORLD.Get_size(); + + m_MaxEnergy = 0; + m_EnergyDecrement = 1; + + if (m_MyID==0) + { + m_Gather_Buffer = new int[m_NumProc]; + m_Energy_Buffer = new double[m_NumProc]; + } + else + { + m_Gather_Buffer = NULL; + m_Energy_Buffer = NULL; + } +} + +openEMS_FDTD_MPI::~openEMS_FDTD_MPI() +{ + delete[] m_Gather_Buffer; + delete[] m_Energy_Buffer; +} + +bool openEMS_FDTD_MPI::parseCommandLineArgument( const char *argv ) +{ + if (!argv) + return false; + + bool ret = openEMS::parseCommandLineArgument( argv ); + + if (ret) + return ret; + + if (strcmp(argv,"--engine=MPI")==0) + { + cout << "openEMS_FDTD_MPI - enabled MPI parallel processing" << endl; + m_engine = EngineType_MPI; + return true; + } + + return false; +} + +bool openEMS_FDTD_MPI::SetupOperator(TiXmlElement* FDTD_Opts) +{ + if (m_engine == EngineType_MPI) + { + if (m_MyID>0) + { + //higher ranks never abort the simulation + endCrit = 0; + } + + FDTD_Op = Operator_MPI::New(); + + return true; + } + else + { + return openEMS::SetupOperator(FDTD_Opts); + } +} + +unsigned int openEMS_FDTD_MPI::GetNextStep() +{ + //start processing and get local next step + int step=PA->Process(); + double currTS = FDTD_Eng->GetNumberOfTimesteps(); + if ((step<0) || (step>(int)(NrTS - currTS))) step=NrTS - currTS; + + int local_step=step; + + //find the smallest next step requestes by all processings + MPI_Reduce(&local_step, &step, 1, MPI_INT, MPI_MIN, 0, MPI_COMM_WORLD); + //send the smallest next step to all + MPI_Bcast(&step, 1, MPI_INT, 0, MPI_COMM_WORLD); + + return step; +} + +bool openEMS_FDTD_MPI::CheckEnergyCalc() +{ + int local_Check = (int)m_ProcField->CheckTimestep(); + int result; + + //check if some process request an energy calculation --> the sum is larger than 0 + MPI_Reduce(&local_Check, &result, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); + //send result to all + MPI_Bcast(&result, 1, MPI_INT, 0, MPI_COMM_WORLD); + + //calc energy if result is non-zero + return result>0; +} + +double openEMS_FDTD_MPI::CalcEnergy() +{ + double energy = 0; + double loc_energy= m_ProcField->CalcTotalEnergy(); + + //calc the sum of all local energies + MPI_Reduce(&loc_energy, &energy, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); + //send sum-energy to all processes + MPI_Bcast(&energy, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD); + + if (energy>m_MaxEnergy) + m_MaxEnergy = energy; + if (m_MaxEnergy) + m_EnergyDecrement = energy/m_MaxEnergy; + + return energy; +} + +void openEMS_FDTD_MPI::RunFDTD() +{ + if (m_engine != EngineType_MPI) + return openEMS::RunFDTD(); + + cout << "Running MPI-FDTD engine... this may take a while... grab a cup of coffee?!?" << endl; + + //get the sum of all cells + unsigned int local_NrCells=FDTD_Op->GetNumberCells(); + MPI_Reduce(&local_NrCells, &m_NumberCells, 1, MPI_UNSIGNED, MPI_SUM, 0, MPI_COMM_WORLD); + MPI_Bcast(&m_NumberCells, 1, MPI_UNSIGNED, 0, MPI_COMM_WORLD); + + //special handling of a field processing, needed to realize the end criteria... + m_ProcField = new ProcessFields(new Engine_Interface_FDTD(FDTD_Op,FDTD_Eng)); + PA->AddProcessing(m_ProcField); + + double currE=0; + + //add all timesteps to end-crit field processing with max excite amplitude + unsigned int maxExcite = FDTD_Op->Exc->GetMaxExcitationTimestep(); + for (unsigned int n=0; nExc->Volt_Count; ++n) + m_ProcField->AddStep(FDTD_Op->Exc->Volt_delay[n]+maxExcite); + + int prevTS=0,currTS=0; + double speed = m_NumberCells/1e6; + double t_diff; + + timeval currTime; + gettimeofday(&currTime,NULL); + timeval startTime = currTime; + timeval prevTime= currTime; + + //*************** simulate ************// + + PA->PreProcess(); + int step = GetNextStep(); + + while ((step>0) && !CheckAbortCond()) + { + FDTD_Eng->IterateTS(step); + step = GetNextStep(); + + currTS = FDTD_Eng->GetNumberOfTimesteps(); + + currE = 0; + gettimeofday(&currTime,NULL); + t_diff = CalcDiffTime(currTime,prevTime); + + if (CheckEnergyCalc()) + currE = CalcEnergy(); + + if (t_diff>4) + { + if (currE==0) + currE = CalcEnergy(); + if (m_MyID==0) + { + cout << "[@" << FormatTime(CalcDiffTime(currTime,startTime)) << "] Timestep: " << setw(12) << currTS << " (" << setw(6) << setprecision(2) << std::fixed << (double)currTS/(double)NrTS*100.0 << "%)" ; + cout << " || Speed: " << setw(6) << setprecision(1) << std::fixed << speed*(currTS-prevTS)/t_diff << " MC/s (" << setw(4) << setprecision(3) << std::scientific << t_diff/(currTS-prevTS) << " s/TS)" ; + cout << " || Energy: ~" << setw(6) << setprecision(2) << std::scientific << currE << " (-" << setw(5) << setprecision(2) << std::fixed << fabs(10.0*log10(m_EnergyDecrement)) << "dB)" << endl; + + //set step to zero to abort simulation and send to all + if (m_EnergyDecrementFlushNext(); + } + } + PA->PostProcess(); + + //*************** postproc ************// + prevTime = currTime; + gettimeofday(&currTime,NULL); + + t_diff = CalcDiffTime(currTime,startTime); + + if (m_MyID==0) + { + cout << "Time for " << FDTD_Eng->GetNumberOfTimesteps() << " iterations with " << FDTD_Op->GetNumberCells() << " cells : " << t_diff << " sec" << endl; + cout << "Speed: " << speed*(double)FDTD_Eng->GetNumberOfTimesteps()/t_diff << " MCells/s " << endl; + } +} diff --git a/FDTD/openems_fdtd_mpi.h b/FDTD/openems_fdtd_mpi.h new file mode 100644 index 0000000..5cfac8d --- /dev/null +++ b/FDTD/openems_fdtd_mpi.h @@ -0,0 +1,54 @@ +/* +* Copyright (C) 2011 Thorsten Liebig (Thorsten.Liebig@gmx.de) +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY{} without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +*/ + +#ifndef OPENEMS_FDTD_MPI_H +#define OPENEMS_FDTD_MPI_H + +#include "openems.h" + +class ProcessFields; + +class openEMS_FDTD_MPI : public openEMS +{ +public: + openEMS_FDTD_MPI(); + virtual ~openEMS_FDTD_MPI(); + + virtual void RunFDTD(); + + virtual bool parseCommandLineArgument( const char *argv ); + +protected: + int m_MyID; + int m_NumProc; + unsigned int m_NumberCells; + virtual bool SetupOperator(TiXmlElement* FDTD_Opts); + + int* m_Gather_Buffer; + unsigned int GetNextStep(); + + ProcessFields* m_ProcField; + double m_MaxEnergy; + double m_EnergyDecrement; + double* m_Energy_Buffer; + //! Check if energy calc is requested... + bool CheckEnergyCalc(); + //! Calc energy in all processes and add up + double CalcEnergy(); +}; + +#endif // OPENEMS_FDTD_MPI_H diff --git a/main.cpp b/main.cpp index 349bdd7..3835683 100644 --- a/main.cpp +++ b/main.cpp @@ -23,9 +23,11 @@ #ifdef MPI_SUPPORT #include "mpi.h" +#include "FDTD/openems_fdtd_mpi.h" +#else +#include "openems.h" #endif -#include "openems.h" #include "tools/global.h" #ifndef GIT_VERSION @@ -39,9 +41,10 @@ int main(int argc, char *argv[]) #ifdef MPI_SUPPORT //init MPI MPI::Init(argc,argv); -#endif - + openEMS_FDTD_MPI FDTD; +#else openEMS FDTD; +#endif #ifdef _LP64 string bits = "64bit"; diff --git a/openEMS.pro b/openEMS.pro index ac2eb0f..87cbbac 100644 --- a/openEMS.pro +++ b/openEMS.pro @@ -158,9 +158,11 @@ MPI_SUPPORT { INCLUDEPATH += /usr/include/mpich2 LIBS += -lmpich -lmpichcxx HEADERS += FDTD/operator_mpi.h \ - FDTD/engine_mpi.h + FDTD/engine_mpi.h \ + FDTD/openems_fdtd_mpi.h SOURCES += FDTD/operator_mpi.cpp \ - FDTD/engine_mpi.cpp + FDTD/engine_mpi.cpp \ + FDTD/openems_fdtd_mpi.cpp } QMAKE_CXXFLAGS_RELEASE = -O3 \ diff --git a/openems.cpp b/openems.cpp index 23e7e48..14849aa 100644 --- a/openems.cpp +++ b/openems.cpp @@ -34,15 +34,9 @@ #include "Common/processfields_td.h" #include "Common/processfields_fd.h" #include "Common/processfields_sar.h" -#include -#include #include // only for H5get_libversion() #include // only for BOOST_LIB_VERSION -#ifdef MPI_SUPPORT -#include "FDTD/operator_mpi.h" -#endif - //external libs #include "tinyxml.h" #include "ContinuousStructure.h" @@ -161,14 +155,6 @@ bool openEMS::parseCommandLineArgument( const char *argv ) m_engine = EngineType_Multithreaded; return true; } -#ifdef MPI_SUPPORT - else if (strcmp(argv,"--engine=MPI")==0) - { - cout << "openEMS - enabled MPI parallel processing" << endl; - m_engine = EngineType_MPI; - return true; - } -#endif else if (strncmp(argv,"--numThreads=",13)==0) { m_engine_numThreads = atoi(argv+13); @@ -462,6 +448,38 @@ bool openEMS::SetupMaterialStorages() return true; } +bool openEMS::SetupOperator(TiXmlElement* FDTD_Opts) +{ + if (CylinderCoords) + { + const char* radii = FDTD_Opts->Attribute("MultiGrid"); + if (radii) + { + string rad(radii); + FDTD_Op = Operator_CylinderMultiGrid::New(SplitString2Double(rad,','),m_engine_numThreads); + } + else + FDTD_Op = Operator_Cylinder::New(m_engine_numThreads); + } + else if (m_engine == EngineType_SSE) + { + FDTD_Op = Operator_sse::New(); + } + else if (m_engine == EngineType_SSE_Compressed) + { + FDTD_Op = Operator_SSE_Compressed::New(); + } + else if (m_engine == EngineType_Multithreaded) + { + FDTD_Op = Operator_Multithread::New(m_engine_numThreads); + } + else + { + FDTD_Op = Operator::New(); + } + return true; +} + int openEMS::SetupFDTD(const char* file) { @@ -549,39 +567,7 @@ int openEMS::SetupFDTD(const char* file) m_CSX->Write2XML("debugm_CSX->xml"); //*************** setup operator ************// - if (CylinderCoords) - { - const char* radii = FDTD_Opts->Attribute("MultiGrid"); - if (radii) - { - string rad(radii); - FDTD_Op = Operator_CylinderMultiGrid::New(SplitString2Double(rad,','),m_engine_numThreads); - } - else - FDTD_Op = Operator_Cylinder::New(m_engine_numThreads); - } - else if (m_engine == EngineType_SSE) - { - FDTD_Op = Operator_sse::New(); - } - else if (m_engine == EngineType_SSE_Compressed) - { - FDTD_Op = Operator_SSE_Compressed::New(); - } - else if (m_engine == EngineType_Multithreaded) - { - FDTD_Op = Operator_Multithread::New(m_engine_numThreads); - } -#ifdef MPI_SUPPORT - else if (m_engine == EngineType_MPI) - { - FDTD_Op = Operator_MPI::New(); - } -#endif - else - { - FDTD_Op = Operator::New(); - } + SetupOperator(FDTD_Opts); if (FDTD_Op->SetGeometryCSX(m_CSX)==false) return(2); diff --git a/openems.h b/openems.h index 1cd2a6e..dddfe8b 100644 --- a/openems.h +++ b/openems.h @@ -19,6 +19,8 @@ #define OPENEMS_H #include +#include +#include using namespace std; @@ -29,17 +31,20 @@ class ProcessingArray; class TiXmlElement; class ContinuousStructure; +double CalcDiffTime(timeval t1, timeval t2); +string FormatTime(int sec); + class openEMS { public: openEMS(); - ~openEMS(); + virtual ~openEMS(); - bool parseCommandLineArgument( const char *argv ); + virtual bool parseCommandLineArgument( const char *argv ); int SetupFDTD(const char* file); - void RunFDTD(); + virtual void RunFDTD(); void Reset(); @@ -86,6 +91,9 @@ protected: EngineType m_engine; unsigned int m_engine_numThreads; + //! Setup an operator matching the requested engine + virtual bool SetupOperator(TiXmlElement* FDTD_Opts); + //! Read boundary conditions from xml element and apply to FDTD operator bool SetupBoundaryConditions(TiXmlElement* BC);