From b395546483dbcc781cd95fab421d925ffb5cffc3 Mon Sep 17 00:00:00 2001
From: Thorsten Liebig <Thorsten.Liebig@gmx.de>
Date: Wed, 16 Feb 2011 15:17:51 +0100
Subject: [PATCH] MPI: some fixes in handling the MPI engine

---
 FDTD/engine_mpi.cpp       | 22 ++++++++++++++--------
 FDTD/openems_fdtd_mpi.cpp | 18 ++++++++++--------
 FDTD/openems_fdtd_mpi.h   |  1 +
 main.cpp                  |  1 +
 4 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/FDTD/engine_mpi.cpp b/FDTD/engine_mpi.cpp
index bf5139f..0db978b 100644
--- a/FDTD/engine_mpi.cpp
+++ b/FDTD/engine_mpi.cpp
@@ -78,6 +78,9 @@ void Engine_MPI::Reset()
 
 void Engine_MPI::SendReceiveVoltages()
 {
+	if (!m_Op_MPI->GetMPIEnabled())
+		return;
+
 	unsigned int pos[3];
 
 	//non-blocking prepare for receive...
@@ -95,9 +98,9 @@ void Engine_MPI::SendReceiveVoltages()
 		pos[n]=numLines[n]-1;
 		if (m_Op_MPI->m_NeighborUp[n]>=0)
 		{
-			for (pos[nP]=0; pos[nP]<numLines[nP]-1; ++pos[nP])
+			for (pos[nP]=0; pos[nP]<numLines[nP]; ++pos[nP])
 			{
-				for (pos[nPP]=0; pos[nPP]<numLines[nPP]-1; ++pos[nPP])
+				for (pos[nPP]=0; pos[nPP]<numLines[nPP]; ++pos[nPP])
 				{
 					m_BufferUp[n][iPos++] = Engine_SSE_Compressed::GetVolt(nP ,pos);
 					m_BufferUp[n][iPos++] = Engine_SSE_Compressed::GetVolt(nPP,pos);
@@ -113,9 +116,9 @@ void Engine_MPI::SendReceiveVoltages()
 		{
 			//wait for receive to finish...
 			MPI_Wait(&Recv_Request[n],&stat);
-			for (pos[nP]=0; pos[nP]<numLines[nP]-1; ++pos[nP])
+			for (pos[nP]=0; pos[nP]<numLines[nP]; ++pos[nP])
 			{
-				for (pos[nPP]=0; pos[nPP]<numLines[nPP]-1; ++pos[nPP])
+				for (pos[nPP]=0; pos[nPP]<numLines[nPP]; ++pos[nPP])
 				{
 					Engine_SSE_Compressed::SetVolt(nP ,pos,m_BufferDown[n][iPos++]);
 					Engine_SSE_Compressed::SetVolt(nPP,pos,m_BufferDown[n][iPos++]);
@@ -128,6 +131,9 @@ void Engine_MPI::SendReceiveVoltages()
 
 void Engine_MPI::SendReceiveCurrents()
 {
+	if (!m_Op_MPI->GetMPIEnabled())
+		return;
+
 	unsigned int pos[3];
 
 	//non-blocking prepare for receive...
@@ -145,9 +151,9 @@ void Engine_MPI::SendReceiveCurrents()
 		pos[n]=0;
 		if (m_Op_MPI->m_NeighborDown[n]>=0)
 		{
-			for (pos[nP]=0; pos[nP]<numLines[nP]-1; ++pos[nP])
+			for (pos[nP]=0; pos[nP]<numLines[nP]; ++pos[nP])
 			{
-				for (pos[nPP]=0; pos[nPP]<numLines[nPP]-1; ++pos[nPP])
+				for (pos[nPP]=0; pos[nPP]<numLines[nPP]; ++pos[nPP])
 				{
 					m_BufferDown[n][iPos++] = Engine_SSE_Compressed::GetCurr(nP ,pos);
 					m_BufferDown[n][iPos++] = Engine_SSE_Compressed::GetCurr(nPP,pos);
@@ -163,9 +169,9 @@ void Engine_MPI::SendReceiveCurrents()
 		{
 			//wait for receive to finish...
 			MPI_Wait(&Recv_Request[n],&stat);
-			for (pos[nP]=0; pos[nP]<numLines[nP]-1; ++pos[nP])
+			for (pos[nP]=0; pos[nP]<numLines[nP]; ++pos[nP])
 			{
-				for (pos[nPP]=0; pos[nPP]<numLines[nPP]-1; ++pos[nPP])
+				for (pos[nPP]=0; pos[nPP]<numLines[nPP]; ++pos[nPP])
 				{
 					Engine_SSE_Compressed::SetCurr(nP ,pos,m_BufferUp[n][iPos++]);
 					Engine_SSE_Compressed::SetCurr(nPP,pos,m_BufferUp[n][iPos++]);
diff --git a/FDTD/openems_fdtd_mpi.cpp b/FDTD/openems_fdtd_mpi.cpp
index 362f6f0..3e46ad2 100644
--- a/FDTD/openems_fdtd_mpi.cpp
+++ b/FDTD/openems_fdtd_mpi.cpp
@@ -40,6 +40,11 @@ openEMS_FDTD_MPI::openEMS_FDTD_MPI() : openEMS()
 	m_MaxEnergy = 0;
 	m_EnergyDecrement = 1;
 
+	if (m_NumProc>1)
+		m_MPI_Enabled=true;
+	else
+		m_MPI_Enabled=false;
+
 	if (m_MyID==0)
 	{
 		m_Gather_Buffer = new int[m_NumProc];
@@ -82,14 +87,7 @@ bool openEMS_FDTD_MPI::SetupOperator(TiXmlElement* FDTD_Opts)
 {
 	if (m_engine == EngineType_MPI)
 	{
-		if (m_MyID>0)
-		{
-			//higher ranks never abort the simulation
-			endCrit = 0;
-		}
-
 		FDTD_Op = Operator_MPI::New();
-
 		return true;
 	}
 	else
@@ -200,11 +198,12 @@ bool openEMS_FDTD_MPI::SetupProcessing()
 			}
 		}
 	}
+	return ret;
 }
 
 void openEMS_FDTD_MPI::RunFDTD()
 {
-	if (m_engine != EngineType_MPI)
+	if (!m_MPI_Enabled)
 		return openEMS::RunFDTD();
 
 	cout << "Running MPI-FDTD engine... this may take a while... grab a cup of coffee?!?" << endl;
@@ -256,6 +255,9 @@ void openEMS_FDTD_MPI::RunFDTD()
 		if (CheckEnergyCalc())
 			currE = CalcEnergy();
 
+		//make sure all processes are at the same simulation time
+		MPI_Bcast(&t_diff, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
+
 		if (t_diff>4)
 		{
 			if (currE==0)
diff --git a/FDTD/openems_fdtd_mpi.h b/FDTD/openems_fdtd_mpi.h
index f59d365..2ce7028 100644
--- a/FDTD/openems_fdtd_mpi.h
+++ b/FDTD/openems_fdtd_mpi.h
@@ -35,6 +35,7 @@ public:
 protected:
 	int m_MyID;
 	int m_NumProc;
+	bool m_MPI_Enabled;
 	unsigned int m_NumberCells;
 	virtual bool SetupOperator(TiXmlElement* FDTD_Opts);
 
diff --git a/main.cpp b/main.cpp
index 3835683..bc64168 100644
--- a/main.cpp
+++ b/main.cpp
@@ -100,6 +100,7 @@ int main(int argc, char *argv[])
 	FDTD.RunFDTD();
 
 #ifdef MPI_SUPPORT
+	FDTD.Reset(); //make sure everything is cleaned-up before calling MPI::Finalize()
 	MPI::Finalize();
 #endif