diff --git a/FDTD/openems_fdtd_mpi.cpp b/FDTD/openems_fdtd_mpi.cpp
new file mode 100644
index 0000000..92b0b1a
--- /dev/null
+++ b/FDTD/openems_fdtd_mpi.cpp
@@ -0,0 +1,236 @@
+/*
+* Copyright (C) 2011 Thorsten Liebig (Thorsten.Liebig@gmx.de)
+*
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY{} without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU General Public License for more details.
+*
+* You should have received a copy of the GNU General Public License
+* along with this program. If not, see .
+*/
+
+#include "openems_fdtd_mpi.h"
+#include "FDTD/engine_interface_fdtd.h"
+#include "FDTD/operator_mpi.h"
+#include "FDTD/engine_mpi.h"
+#include "Common/processfields.h"
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include "mpi.h"
+
+openEMS_FDTD_MPI::openEMS_FDTD_MPI() : openEMS()
+{
+ m_MyID = MPI::COMM_WORLD.Get_rank();
+ m_NumProc = MPI::COMM_WORLD.Get_size();
+
+ m_MaxEnergy = 0;
+ m_EnergyDecrement = 1;
+
+ if (m_MyID==0)
+ {
+ m_Gather_Buffer = new int[m_NumProc];
+ m_Energy_Buffer = new double[m_NumProc];
+ }
+ else
+ {
+ m_Gather_Buffer = NULL;
+ m_Energy_Buffer = NULL;
+ }
+}
+
+openEMS_FDTD_MPI::~openEMS_FDTD_MPI()
+{
+ delete[] m_Gather_Buffer;
+ delete[] m_Energy_Buffer;
+}
+
+bool openEMS_FDTD_MPI::parseCommandLineArgument( const char *argv )
+{
+ if (!argv)
+ return false;
+
+ bool ret = openEMS::parseCommandLineArgument( argv );
+
+ if (ret)
+ return ret;
+
+ if (strcmp(argv,"--engine=MPI")==0)
+ {
+ cout << "openEMS_FDTD_MPI - enabled MPI parallel processing" << endl;
+ m_engine = EngineType_MPI;
+ return true;
+ }
+
+ return false;
+}
+
+bool openEMS_FDTD_MPI::SetupOperator(TiXmlElement* FDTD_Opts)
+{
+ if (m_engine == EngineType_MPI)
+ {
+ if (m_MyID>0)
+ {
+ //higher ranks never abort the simulation
+ endCrit = 0;
+ }
+
+ FDTD_Op = Operator_MPI::New();
+
+ return true;
+ }
+ else
+ {
+ return openEMS::SetupOperator(FDTD_Opts);
+ }
+}
+
+unsigned int openEMS_FDTD_MPI::GetNextStep()
+{
+ //start processing and get local next step
+ int step=PA->Process();
+ double currTS = FDTD_Eng->GetNumberOfTimesteps();
+ if ((step<0) || (step>(int)(NrTS - currTS))) step=NrTS - currTS;
+
+ int local_step=step;
+
+ //find the smallest next step requestes by all processings
+ MPI_Reduce(&local_step, &step, 1, MPI_INT, MPI_MIN, 0, MPI_COMM_WORLD);
+ //send the smallest next step to all
+ MPI_Bcast(&step, 1, MPI_INT, 0, MPI_COMM_WORLD);
+
+ return step;
+}
+
+bool openEMS_FDTD_MPI::CheckEnergyCalc()
+{
+ int local_Check = (int)m_ProcField->CheckTimestep();
+ int result;
+
+ //check if some process request an energy calculation --> the sum is larger than 0
+ MPI_Reduce(&local_Check, &result, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
+ //send result to all
+ MPI_Bcast(&result, 1, MPI_INT, 0, MPI_COMM_WORLD);
+
+ //calc energy if result is non-zero
+ return result>0;
+}
+
+double openEMS_FDTD_MPI::CalcEnergy()
+{
+ double energy = 0;
+ double loc_energy= m_ProcField->CalcTotalEnergy();
+
+ //calc the sum of all local energies
+ MPI_Reduce(&loc_energy, &energy, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ //send sum-energy to all processes
+ MPI_Bcast(&energy, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
+
+ if (energy>m_MaxEnergy)
+ m_MaxEnergy = energy;
+ if (m_MaxEnergy)
+ m_EnergyDecrement = energy/m_MaxEnergy;
+
+ return energy;
+}
+
+void openEMS_FDTD_MPI::RunFDTD()
+{
+ if (m_engine != EngineType_MPI)
+ return openEMS::RunFDTD();
+
+ cout << "Running MPI-FDTD engine... this may take a while... grab a cup of coffee?!?" << endl;
+
+ //get the sum of all cells
+ unsigned int local_NrCells=FDTD_Op->GetNumberCells();
+ MPI_Reduce(&local_NrCells, &m_NumberCells, 1, MPI_UNSIGNED, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Bcast(&m_NumberCells, 1, MPI_UNSIGNED, 0, MPI_COMM_WORLD);
+
+ //special handling of a field processing, needed to realize the end criteria...
+ m_ProcField = new ProcessFields(new Engine_Interface_FDTD(FDTD_Op,FDTD_Eng));
+ PA->AddProcessing(m_ProcField);
+
+ double currE=0;
+
+ //add all timesteps to end-crit field processing with max excite amplitude
+ unsigned int maxExcite = FDTD_Op->Exc->GetMaxExcitationTimestep();
+ for (unsigned int n=0; nExc->Volt_Count; ++n)
+ m_ProcField->AddStep(FDTD_Op->Exc->Volt_delay[n]+maxExcite);
+
+ int prevTS=0,currTS=0;
+ double speed = m_NumberCells/1e6;
+ double t_diff;
+
+ timeval currTime;
+ gettimeofday(&currTime,NULL);
+ timeval startTime = currTime;
+ timeval prevTime= currTime;
+
+ //*************** simulate ************//
+
+ PA->PreProcess();
+ int step = GetNextStep();
+
+ while ((step>0) && !CheckAbortCond())
+ {
+ FDTD_Eng->IterateTS(step);
+ step = GetNextStep();
+
+ currTS = FDTD_Eng->GetNumberOfTimesteps();
+
+ currE = 0;
+ gettimeofday(&currTime,NULL);
+ t_diff = CalcDiffTime(currTime,prevTime);
+
+ if (CheckEnergyCalc())
+ currE = CalcEnergy();
+
+ if (t_diff>4)
+ {
+ if (currE==0)
+ currE = CalcEnergy();
+ if (m_MyID==0)
+ {
+ cout << "[@" << FormatTime(CalcDiffTime(currTime,startTime)) << "] Timestep: " << setw(12) << currTS << " (" << setw(6) << setprecision(2) << std::fixed << (double)currTS/(double)NrTS*100.0 << "%)" ;
+ cout << " || Speed: " << setw(6) << setprecision(1) << std::fixed << speed*(currTS-prevTS)/t_diff << " MC/s (" << setw(4) << setprecision(3) << std::scientific << t_diff/(currTS-prevTS) << " s/TS)" ;
+ cout << " || Energy: ~" << setw(6) << setprecision(2) << std::scientific << currE << " (-" << setw(5) << setprecision(2) << std::fixed << fabs(10.0*log10(m_EnergyDecrement)) << "dB)" << endl;
+
+ //set step to zero to abort simulation and send to all
+ if (m_EnergyDecrementFlushNext();
+ }
+ }
+ PA->PostProcess();
+
+ //*************** postproc ************//
+ prevTime = currTime;
+ gettimeofday(&currTime,NULL);
+
+ t_diff = CalcDiffTime(currTime,startTime);
+
+ if (m_MyID==0)
+ {
+ cout << "Time for " << FDTD_Eng->GetNumberOfTimesteps() << " iterations with " << FDTD_Op->GetNumberCells() << " cells : " << t_diff << " sec" << endl;
+ cout << "Speed: " << speed*(double)FDTD_Eng->GetNumberOfTimesteps()/t_diff << " MCells/s " << endl;
+ }
+}
diff --git a/FDTD/openems_fdtd_mpi.h b/FDTD/openems_fdtd_mpi.h
new file mode 100644
index 0000000..5cfac8d
--- /dev/null
+++ b/FDTD/openems_fdtd_mpi.h
@@ -0,0 +1,54 @@
+/*
+* Copyright (C) 2011 Thorsten Liebig (Thorsten.Liebig@gmx.de)
+*
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY{} without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU General Public License for more details.
+*
+* You should have received a copy of the GNU General Public License
+* along with this program. If not, see .
+*/
+
+#ifndef OPENEMS_FDTD_MPI_H
+#define OPENEMS_FDTD_MPI_H
+
+#include "openems.h"
+
+class ProcessFields;
+
+class openEMS_FDTD_MPI : public openEMS
+{
+public:
+ openEMS_FDTD_MPI();
+ virtual ~openEMS_FDTD_MPI();
+
+ virtual void RunFDTD();
+
+ virtual bool parseCommandLineArgument( const char *argv );
+
+protected:
+ int m_MyID;
+ int m_NumProc;
+ unsigned int m_NumberCells;
+ virtual bool SetupOperator(TiXmlElement* FDTD_Opts);
+
+ int* m_Gather_Buffer;
+ unsigned int GetNextStep();
+
+ ProcessFields* m_ProcField;
+ double m_MaxEnergy;
+ double m_EnergyDecrement;
+ double* m_Energy_Buffer;
+ //! Check if energy calc is requested...
+ bool CheckEnergyCalc();
+ //! Calc energy in all processes and add up
+ double CalcEnergy();
+};
+
+#endif // OPENEMS_FDTD_MPI_H
diff --git a/main.cpp b/main.cpp
index 349bdd7..3835683 100644
--- a/main.cpp
+++ b/main.cpp
@@ -23,9 +23,11 @@
#ifdef MPI_SUPPORT
#include "mpi.h"
+#include "FDTD/openems_fdtd_mpi.h"
+#else
+#include "openems.h"
#endif
-#include "openems.h"
#include "tools/global.h"
#ifndef GIT_VERSION
@@ -39,9 +41,10 @@ int main(int argc, char *argv[])
#ifdef MPI_SUPPORT
//init MPI
MPI::Init(argc,argv);
-#endif
-
+ openEMS_FDTD_MPI FDTD;
+#else
openEMS FDTD;
+#endif
#ifdef _LP64
string bits = "64bit";
diff --git a/openEMS.pro b/openEMS.pro
index ac2eb0f..87cbbac 100644
--- a/openEMS.pro
+++ b/openEMS.pro
@@ -158,9 +158,11 @@ MPI_SUPPORT {
INCLUDEPATH += /usr/include/mpich2
LIBS += -lmpich -lmpichcxx
HEADERS += FDTD/operator_mpi.h \
- FDTD/engine_mpi.h
+ FDTD/engine_mpi.h \
+ FDTD/openems_fdtd_mpi.h
SOURCES += FDTD/operator_mpi.cpp \
- FDTD/engine_mpi.cpp
+ FDTD/engine_mpi.cpp \
+ FDTD/openems_fdtd_mpi.cpp
}
QMAKE_CXXFLAGS_RELEASE = -O3 \
diff --git a/openems.cpp b/openems.cpp
index 23e7e48..14849aa 100644
--- a/openems.cpp
+++ b/openems.cpp
@@ -34,15 +34,9 @@
#include "Common/processfields_td.h"
#include "Common/processfields_fd.h"
#include "Common/processfields_sar.h"
-#include
-#include
#include // only for H5get_libversion()
#include // only for BOOST_LIB_VERSION
-#ifdef MPI_SUPPORT
-#include "FDTD/operator_mpi.h"
-#endif
-
//external libs
#include "tinyxml.h"
#include "ContinuousStructure.h"
@@ -161,14 +155,6 @@ bool openEMS::parseCommandLineArgument( const char *argv )
m_engine = EngineType_Multithreaded;
return true;
}
-#ifdef MPI_SUPPORT
- else if (strcmp(argv,"--engine=MPI")==0)
- {
- cout << "openEMS - enabled MPI parallel processing" << endl;
- m_engine = EngineType_MPI;
- return true;
- }
-#endif
else if (strncmp(argv,"--numThreads=",13)==0)
{
m_engine_numThreads = atoi(argv+13);
@@ -462,6 +448,38 @@ bool openEMS::SetupMaterialStorages()
return true;
}
+bool openEMS::SetupOperator(TiXmlElement* FDTD_Opts)
+{
+ if (CylinderCoords)
+ {
+ const char* radii = FDTD_Opts->Attribute("MultiGrid");
+ if (radii)
+ {
+ string rad(radii);
+ FDTD_Op = Operator_CylinderMultiGrid::New(SplitString2Double(rad,','),m_engine_numThreads);
+ }
+ else
+ FDTD_Op = Operator_Cylinder::New(m_engine_numThreads);
+ }
+ else if (m_engine == EngineType_SSE)
+ {
+ FDTD_Op = Operator_sse::New();
+ }
+ else if (m_engine == EngineType_SSE_Compressed)
+ {
+ FDTD_Op = Operator_SSE_Compressed::New();
+ }
+ else if (m_engine == EngineType_Multithreaded)
+ {
+ FDTD_Op = Operator_Multithread::New(m_engine_numThreads);
+ }
+ else
+ {
+ FDTD_Op = Operator::New();
+ }
+ return true;
+}
+
int openEMS::SetupFDTD(const char* file)
{
@@ -549,39 +567,7 @@ int openEMS::SetupFDTD(const char* file)
m_CSX->Write2XML("debugm_CSX->xml");
//*************** setup operator ************//
- if (CylinderCoords)
- {
- const char* radii = FDTD_Opts->Attribute("MultiGrid");
- if (radii)
- {
- string rad(radii);
- FDTD_Op = Operator_CylinderMultiGrid::New(SplitString2Double(rad,','),m_engine_numThreads);
- }
- else
- FDTD_Op = Operator_Cylinder::New(m_engine_numThreads);
- }
- else if (m_engine == EngineType_SSE)
- {
- FDTD_Op = Operator_sse::New();
- }
- else if (m_engine == EngineType_SSE_Compressed)
- {
- FDTD_Op = Operator_SSE_Compressed::New();
- }
- else if (m_engine == EngineType_Multithreaded)
- {
- FDTD_Op = Operator_Multithread::New(m_engine_numThreads);
- }
-#ifdef MPI_SUPPORT
- else if (m_engine == EngineType_MPI)
- {
- FDTD_Op = Operator_MPI::New();
- }
-#endif
- else
- {
- FDTD_Op = Operator::New();
- }
+ SetupOperator(FDTD_Opts);
if (FDTD_Op->SetGeometryCSX(m_CSX)==false) return(2);
diff --git a/openems.h b/openems.h
index 1cd2a6e..dddfe8b 100644
--- a/openems.h
+++ b/openems.h
@@ -19,6 +19,8 @@
#define OPENEMS_H
#include
+#include
+#include
using namespace std;
@@ -29,17 +31,20 @@ class ProcessingArray;
class TiXmlElement;
class ContinuousStructure;
+double CalcDiffTime(timeval t1, timeval t2);
+string FormatTime(int sec);
+
class openEMS
{
public:
openEMS();
- ~openEMS();
+ virtual ~openEMS();
- bool parseCommandLineArgument( const char *argv );
+ virtual bool parseCommandLineArgument( const char *argv );
int SetupFDTD(const char* file);
- void RunFDTD();
+ virtual void RunFDTD();
void Reset();
@@ -86,6 +91,9 @@ protected:
EngineType m_engine;
unsigned int m_engine_numThreads;
+ //! Setup an operator matching the requested engine
+ virtual bool SetupOperator(TiXmlElement* FDTD_Opts);
+
//! Read boundary conditions from xml element and apply to FDTD operator
bool SetupBoundaryConditions(TiXmlElement* BC);