zcy
/
openEMS
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

new openEMS_FDTD_MPI for better MPI handling

pull/1/head
Thorsten Liebig 2011-02-11 13:52:48 +01:00
parent 9364ecd292
commit e2628740fc
6 changed files with 344 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

236
FDTD/openems_fdtd_mpi.cpp Normal file
View File

@ -0,0 +1,236 @@
/*
* Copyright (C) 2011 Thorsten Liebig (Thorsten.Liebig@gmx.de)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY{} without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "openems_fdtd_mpi.h"
#include "FDTD/engine_interface_fdtd.h"
#include "FDTD/operator_mpi.h"
#include "FDTD/engine_mpi.h"
#include "Common/processfields.h"
#include <stdio.h>
#include <stdlib.h>
#include <iostream>
#include <sstream>
#include <vector>
#include <iomanip>
#include <string.h>
#include <sys/time.h>
#include <time.h>
#include "mpi.h"
openEMS_FDTD_MPI::openEMS_FDTD_MPI() : openEMS()
{
m_MyID = MPI::COMM_WORLD.Get_rank();
m_NumProc = MPI::COMM_WORLD.Get_size();
m_MaxEnergy = 0;
m_EnergyDecrement = 1;
if (m_MyID==0)
{
m_Gather_Buffer = new int[m_NumProc];
m_Energy_Buffer = new double[m_NumProc];
}
else
{
m_Gather_Buffer = NULL;
m_Energy_Buffer = NULL;
}
}
openEMS_FDTD_MPI::~openEMS_FDTD_MPI()
{
delete[] m_Gather_Buffer;
delete[] m_Energy_Buffer;
}
bool openEMS_FDTD_MPI::parseCommandLineArgument( const char *argv )
{
if (!argv)
return false;
bool ret = openEMS::parseCommandLineArgument( argv );
if (ret)
return ret;
if (strcmp(argv,"--engine=MPI")==0)
{
cout << "openEMS_FDTD_MPI - enabled MPI parallel processing" << endl;
m_engine = EngineType_MPI;
return true;
}
return false;
}
bool openEMS_FDTD_MPI::SetupOperator(TiXmlElement* FDTD_Opts)
{
if (m_engine == EngineType_MPI)
{
if (m_MyID>0)
{
//higher ranks never abort the simulation
endCrit = 0;
}
FDTD_Op = Operator_MPI::New();
return true;
}
else
{
return openEMS::SetupOperator(FDTD_Opts);
}
}
unsigned int openEMS_FDTD_MPI::GetNextStep()
{
//start processing and get local next step
int step=PA->Process();
double currTS = FDTD_Eng->GetNumberOfTimesteps();
if ((step<0) || (step>(int)(NrTS - currTS))) step=NrTS - currTS;
int local_step=step;
//find the smallest next step requestes by all processings
MPI_Reduce(&local_step, &step, 1, MPI_INT, MPI_MIN, 0, MPI_COMM_WORLD);
//send the smallest next step to all
MPI_Bcast(&step, 1, MPI_INT, 0, MPI_COMM_WORLD);
return step;
}
bool openEMS_FDTD_MPI::CheckEnergyCalc()
{
int local_Check = (int)m_ProcField->CheckTimestep();
int result;
//check if some process request an energy calculation --> the sum is larger than 0
MPI_Reduce(&local_Check, &result, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
//send result to all
MPI_Bcast(&result, 1, MPI_INT, 0, MPI_COMM_WORLD);
//calc energy if result is non-zero
return result>0;
}
double openEMS_FDTD_MPI::CalcEnergy()
{
double energy = 0;
double loc_energy= m_ProcField->CalcTotalEnergy();
//calc the sum of all local energies
MPI_Reduce(&loc_energy, &energy, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
//send sum-energy to all processes
MPI_Bcast(&energy, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
if (energy>m_MaxEnergy)
m_MaxEnergy = energy;
if (m_MaxEnergy)
m_EnergyDecrement = energy/m_MaxEnergy;
return energy;
}
void openEMS_FDTD_MPI::RunFDTD()
{
if (m_engine != EngineType_MPI)
return openEMS::RunFDTD();
cout << "Running MPI-FDTD engine... this may take a while... grab a cup of coffee?!?" << endl;
//get the sum of all cells
unsigned int local_NrCells=FDTD_Op->GetNumberCells();
MPI_Reduce(&local_NrCells, &m_NumberCells, 1, MPI_UNSIGNED, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Bcast(&m_NumberCells, 1, MPI_UNSIGNED, 0, MPI_COMM_WORLD);
//special handling of a field processing, needed to realize the end criteria...
m_ProcField = new ProcessFields(new Engine_Interface_FDTD(FDTD_Op,FDTD_Eng));
PA->AddProcessing(m_ProcField);
double currE=0;
//add all timesteps to end-crit field processing with max excite amplitude
unsigned int maxExcite = FDTD_Op->Exc->GetMaxExcitationTimestep();
for (unsigned int n=0; n<FDTD_Op->Exc->Volt_Count; ++n)
m_ProcField->AddStep(FDTD_Op->Exc->Volt_delay[n]+maxExcite);
int prevTS=0,currTS=0;
double speed = m_NumberCells/1e6;
double t_diff;
timeval currTime;
gettimeofday(&currTime,NULL);
timeval startTime = currTime;
timeval prevTime= currTime;
//*************** simulate ************//
PA->PreProcess();
int step = GetNextStep();
while ((step>0) && !CheckAbortCond())
{
FDTD_Eng->IterateTS(step);
step = GetNextStep();
currTS = FDTD_Eng->GetNumberOfTimesteps();
currE = 0;
gettimeofday(&currTime,NULL);
t_diff = CalcDiffTime(currTime,prevTime);
if (CheckEnergyCalc())
currE = CalcEnergy();
if (t_diff>4)
{
if (currE==0)
currE = CalcEnergy();
if (m_MyID==0)
{
cout << "[@" << FormatTime(CalcDiffTime(currTime,startTime)) << "] Timestep: " << setw(12) << currTS << " (" << setw(6) << setprecision(2) << std::fixed << (double)currTS/(double)NrTS*100.0 << "%)" ;
cout << " || Speed: " << setw(6) << setprecision(1) << std::fixed << speed*(currTS-prevTS)/t_diff << " MC/s (" << setw(4) << setprecision(3) << std::scientific << t_diff/(currTS-prevTS) << " s/TS)" ;
cout << " || Energy: ~" << setw(6) << setprecision(2) << std::scientific << currE << " (-" << setw(5) << setprecision(2) << std::fixed << fabs(10.0*log10(m_EnergyDecrement)) << "dB)" << endl;
//set step to zero to abort simulation and send to all
if (m_EnergyDecrement<endCrit)
step=0;
}
MPI_Bcast(&step, 1, MPI_INT, 0, MPI_COMM_WORLD);
prevTime=currTime;
prevTS=currTS;
PA->FlushNext();
}
}
PA->PostProcess();
//*************** postproc ************//
prevTime = currTime;
gettimeofday(&currTime,NULL);
t_diff = CalcDiffTime(currTime,startTime);
if (m_MyID==0)
{
cout << "Time for " << FDTD_Eng->GetNumberOfTimesteps() << " iterations with " << FDTD_Op->GetNumberCells() << " cells : " << t_diff << " sec" << endl;
cout << "Speed: " << speed*(double)FDTD_Eng->GetNumberOfTimesteps()/t_diff << " MCells/s " << endl;
}
}

54
FDTD/openems_fdtd_mpi.h Normal file
View File

@ -0,0 +1,54 @@
/*
* Copyright (C) 2011 Thorsten Liebig (Thorsten.Liebig@gmx.de)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY{} without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef OPENEMS_FDTD_MPI_H
#define OPENEMS_FDTD_MPI_H
#include "openems.h"
class ProcessFields;
class openEMS_FDTD_MPI : public openEMS
{
public:
openEMS_FDTD_MPI();
virtual ~openEMS_FDTD_MPI();
virtual void RunFDTD();
virtual bool parseCommandLineArgument( const char *argv );
protected:
int m_MyID;
int m_NumProc;
unsigned int m_NumberCells;
virtual bool SetupOperator(TiXmlElement* FDTD_Opts);
int* m_Gather_Buffer;
unsigned int GetNextStep();
ProcessFields* m_ProcField;
double m_MaxEnergy;
double m_EnergyDecrement;
double* m_Energy_Buffer;
//! Check if energy calc is requested...
bool CheckEnergyCalc();
//! Calc energy in all processes and add up
double CalcEnergy();
};
#endif // OPENEMS_FDTD_MPI_H

9
main.cpp
View File

@ -23,9 +23,11 @@
#ifdef MPI_SUPPORT
#include "mpi.h"
#include "FDTD/openems_fdtd_mpi.h"
#else
#include "openems.h"
#endif
#include "openems.h"
#include "tools/global.h"
#ifndef GIT_VERSION
@ -39,9 +41,10 @@ int main(int argc, char *argv[])
#ifdef MPI_SUPPORT
//init MPI
MPI::Init(argc,argv);
#endif
openEMS_FDTD_MPI FDTD;
#else
openEMS FDTD;
#endif
#ifdef _LP64
string bits = "64bit";

6
openEMS.pro
View File

@ -158,9 +158,11 @@ MPI_SUPPORT {
INCLUDEPATH += /usr/include/mpich2
LIBS += -lmpich -lmpichcxx
HEADERS += FDTD/operator_mpi.h \
FDTD/engine_mpi.h
FDTD/engine_mpi.h \
FDTD/openems_fdtd_mpi.h
SOURCES += FDTD/operator_mpi.cpp \
FDTD/engine_mpi.cpp
FDTD/engine_mpi.cpp \
FDTD/openems_fdtd_mpi.cpp
}
QMAKE_CXXFLAGS_RELEASE = -O3 \

80
openems.cpp
View File

@ -34,15 +34,9 @@
#include "Common/processfields_td.h"
#include "Common/processfields_fd.h"
#include "Common/processfields_sar.h"
#include <sys/time.h>
#include <time.h>
#include <H5Cpp.h> // only for H5get_libversion()
#include <boost/version.hpp> // only for BOOST_LIB_VERSION
#ifdef MPI_SUPPORT
#include "FDTD/operator_mpi.h"
#endif
//external libs
#include "tinyxml.h"
#include "ContinuousStructure.h"
@ -161,14 +155,6 @@ bool openEMS::parseCommandLineArgument( const char *argv )
m_engine = EngineType_Multithreaded;
return true;
}
#ifdef MPI_SUPPORT
else if (strcmp(argv,"--engine=MPI")==0)
{
cout << "openEMS - enabled MPI parallel processing" << endl;
m_engine = EngineType_MPI;
return true;
}
#endif
else if (strncmp(argv,"--numThreads=",13)==0)
{
m_engine_numThreads = atoi(argv+13);
@ -462,6 +448,38 @@ bool openEMS::SetupMaterialStorages()
return true;
}
bool openEMS::SetupOperator(TiXmlElement* FDTD_Opts)
{
if (CylinderCoords)
{
const char* radii = FDTD_Opts->Attribute("MultiGrid");
if (radii)
{
string rad(radii);
FDTD_Op = Operator_CylinderMultiGrid::New(SplitString2Double(rad,','),m_engine_numThreads);
}
else
FDTD_Op = Operator_Cylinder::New(m_engine_numThreads);
}
else if (m_engine == EngineType_SSE)
{
FDTD_Op = Operator_sse::New();
}
else if (m_engine == EngineType_SSE_Compressed)
{
FDTD_Op = Operator_SSE_Compressed::New();
}
else if (m_engine == EngineType_Multithreaded)
{
FDTD_Op = Operator_Multithread::New(m_engine_numThreads);
}
else
{
FDTD_Op = Operator::New();
}
return true;
}
int openEMS::SetupFDTD(const char* file)
{
@ -549,39 +567,7 @@ int openEMS::SetupFDTD(const char* file)
m_CSX->Write2XML("debugm_CSX->xml");
//*************** setup operator ************//
if (CylinderCoords)
{
const char* radii = FDTD_Opts->Attribute("MultiGrid");
if (radii)
{
string rad(radii);
FDTD_Op = Operator_CylinderMultiGrid::New(SplitString2Double(rad,','),m_engine_numThreads);
}
else
FDTD_Op = Operator_Cylinder::New(m_engine_numThreads);
}
else if (m_engine == EngineType_SSE)
{
FDTD_Op = Operator_sse::New();
}
else if (m_engine == EngineType_SSE_Compressed)
{
FDTD_Op = Operator_SSE_Compressed::New();
}
else if (m_engine == EngineType_Multithreaded)
{
FDTD_Op = Operator_Multithread::New(m_engine_numThreads);
}
#ifdef MPI_SUPPORT
else if (m_engine == EngineType_MPI)
{
FDTD_Op = Operator_MPI::New();
}
#endif
else
{
FDTD_Op = Operator::New();
}
SetupOperator(FDTD_Opts);
if (FDTD_Op->SetGeometryCSX(m_CSX)==false) return(2);

14
openems.h
View File

@ -19,6 +19,8 @@
#define OPENEMS_H
#include <sstream>
#include <sys/time.h>
#include <time.h>
using namespace std;
@ -29,17 +31,20 @@ class ProcessingArray;
class TiXmlElement;
class ContinuousStructure;
double CalcDiffTime(timeval t1, timeval t2);
string FormatTime(int sec);
class openEMS
{
public:
openEMS();
~openEMS();
virtual ~openEMS();
bool parseCommandLineArgument( const char *argv );
virtual bool parseCommandLineArgument( const char *argv );
int SetupFDTD(const char* file);
void RunFDTD();
virtual void RunFDTD();
void Reset();
@ -86,6 +91,9 @@ protected:
EngineType m_engine;
unsigned int m_engine_numThreads;
//! Setup an operator matching the requested engine
virtual bool SetupOperator(TiXmlElement* FDTD_Opts);
//! Read boundary conditions from xml element and apply to FDTD operator
bool SetupBoundaryConditions(TiXmlElement* BC);