From 06eb82832e54203743314ebdf304f43992e766dc Mon Sep 17 00:00:00 2001
From: Thorsten Liebig <Thorsten.Liebig@gmx.de>
Date: Mon, 28 Mar 2011 10:26:08 +0200
Subject: [PATCH] MPI: operator saves process table

---
 FDTD/openems_fdtd_mpi.cpp | 13 +++++++++++--
 FDTD/openems_fdtd_mpi.h   |  4 ++--
 FDTD/operator_mpi.cpp     | 11 ++++++++---
 FDTD/operator_mpi.h       | 16 ++++++++++++++--
 4 files changed, 35 insertions(+), 9 deletions(-)

diff --git a/FDTD/openems_fdtd_mpi.cpp b/FDTD/openems_fdtd_mpi.cpp
index 86c729f..9cda209 100644
--- a/FDTD/openems_fdtd_mpi.cpp
+++ b/FDTD/openems_fdtd_mpi.cpp
@@ -193,7 +193,7 @@ bool openEMS_FDTD_MPI::SetupMPI(TiXmlElement* FDTD_Opts)
 	MPI_Barrier(MPI_COMM_WORLD);
 
 	//validate number of processes
-	int numProcs = (SplitNumber[0].size()-1)*(SplitNumber[1].size()-1)*(SplitNumber[2].size()-1);
+	unsigned int numProcs = (SplitNumber[0].size()-1)*(SplitNumber[1].size()-1)*(SplitNumber[2].size()-1);
 	if (numProcs!=m_NumProc)
 	{
 		if (m_MyID==0)
@@ -203,7 +203,11 @@ bool openEMS_FDTD_MPI::SetupMPI(TiXmlElement* FDTD_Opts)
 
 	//create process table
 	unsigned int procN = 0;
-	int procTable[SplitNumber[0].size()-1][SplitNumber[1].size()-1][SplitNumber[2].size()-1];
+	unsigned int splits[] = {SplitNumber[0].size()-1, SplitNumber[1].size()-1, SplitNumber[2].size()-1};
+	m_MPI_Op->SetSplitNumbers(0,splits[0]);
+	m_MPI_Op->SetSplitNumbers(1,splits[1]);
+	m_MPI_Op->SetSplitNumbers(2,splits[2]);
+	unsigned int*** procTable=Create3DArray<unsigned int>(splits);
 	for (size_t i=0;i<SplitNumber[0].size()-1;++i)
 		for (size_t j=0;j<SplitNumber[1].size()-1;++j)
 			for (size_t k=0;k<SplitNumber[2].size()-1;++k)
@@ -211,6 +215,7 @@ bool openEMS_FDTD_MPI::SetupMPI(TiXmlElement* FDTD_Opts)
 				procTable[i][j][k] = procN;
 				++procN;
 			}
+	m_MPI_Op->SetProcessTable(procTable);
 
 	//assign mesh and neighbors to this process
 	for (size_t i=0;i<SplitNumber[0].size()-1;++i)
@@ -221,6 +226,10 @@ bool openEMS_FDTD_MPI::SetupMPI(TiXmlElement* FDTD_Opts)
 			{
 				if (procTable[i][j][k] == m_MyID)
 				{
+					m_MPI_Op->SetProcessTablePosition(0,i);
+					m_MPI_Op->SetProcessTablePosition(1,j);
+					m_MPI_Op->SetProcessTablePosition(2,k);
+
 					grid->ClearLines(0);
 					grid->ClearLines(1);
 					grid->ClearLines(2);
diff --git a/FDTD/openems_fdtd_mpi.h b/FDTD/openems_fdtd_mpi.h
index a281914..236a345 100644
--- a/FDTD/openems_fdtd_mpi.h
+++ b/FDTD/openems_fdtd_mpi.h
@@ -39,8 +39,8 @@ public:
 protected:
 	Operator_MPI* m_MPI_Op;
 	CSRectGrid* m_Original_Grid;
-	int m_MyID;
-	int m_NumProc;
+	unsigned int m_MyID;
+	unsigned int m_NumProc;
 	bool m_MPI_Enabled;
 	unsigned int m_NumberCells;
 
diff --git a/FDTD/operator_mpi.cpp b/FDTD/operator_mpi.cpp
index 0d6635c..bdb4f4a 100644
--- a/FDTD/operator_mpi.cpp
+++ b/FDTD/operator_mpi.cpp
@@ -117,6 +117,11 @@ void Operator_MPI::Init()
 		m_OrigDiscLines[i]=NULL;
 	}
 
+	m_ProcTable = NULL;
+	m_SplitNumber[0]=0;
+	m_SplitNumber[1]=0;
+	m_SplitNumber[2]=0;
+
 	int  namelen;
 	m_NumProc = MPI::COMM_WORLD.Get_size();
 	m_MyID = MPI::COMM_WORLD.Get_rank();
@@ -137,7 +142,8 @@ void Operator_MPI::Delete()
 		delete[] m_OrigDiscLines[i];
 		m_OrigDiscLines[i] = NULL;
 	}
-
+	Delete3DArray(m_ProcTable,m_SplitNumber);
+	m_ProcTable=NULL;
 }
 
 void Operator_MPI::Reset()
@@ -151,8 +157,7 @@ void Operator_MPI::SetOriginalMesh(CSRectGrid* orig_Mesh)
 	for (int n=0;n<3;++n)
 	{
 		delete[] m_OrigDiscLines[n];
-		m_OrigDiscLines[n] = NULL;
-		orig_Mesh->GetLines(n,m_OrigDiscLines[n],m_OrigNumLines[n]);
+		m_OrigDiscLines[n] = orig_Mesh->GetLines(n,NULL,m_OrigNumLines[n]);
 	}
 }
 
diff --git a/FDTD/operator_mpi.h b/FDTD/operator_mpi.h
index b2b7bad..fbb2413 100644
--- a/FDTD/operator_mpi.h
+++ b/FDTD/operator_mpi.h
@@ -36,11 +36,19 @@ public:
 
 	virtual void SetTag(int tag) {m_MyTag=tag;}
 
+	//! Set the number of splits for a given direction. This also defines the size of the process table. \sa SetProcessTable
+	virtual void SetSplitNumbers(int ny, unsigned int splits) {m_SplitNumber[ny]=splits;}
+	//! Set the table containing a list of all MPI rank ID's and there mesh affiliation. \sa SetProcessTablePosition
+	virtual void SetProcessTable(unsigned int*** procTable) {m_ProcTable=procTable;}
+	//! Save the position for this rank in the process table. \sa SetProcessTable
+	virtual void SetProcessTablePosition(int ny, unsigned int pos) {m_ProcTablePos[ny]=pos;}
+
 	virtual void SetNeighborUp(int ny, int id);
 	virtual void SetNeighborDown(int ny, int id);
 
 	virtual void DumpExciationSignals();
 
+	//! Set the lower original mesh position
 	virtual void SetSplitPos(int ny, unsigned int pos) {m_SplitPos[ny]=pos;}
 	virtual void SetOriginalMesh(CSRectGrid* orig_Mesh);
 
@@ -55,8 +63,8 @@ protected:
 
 	virtual double CalcTimestep();
 
-	int m_MyID;
-	int m_NumProc;
+	unsigned int m_MyID;
+	unsigned int m_NumProc;
 	int m_MyTag;
 	char* m_Processor_Name;
 
@@ -64,6 +72,10 @@ protected:
 	int m_NeighborUp[3];
 	int m_NeighborDown[3];
 
+	unsigned int m_SplitNumber[3];
+	unsigned int m_ProcTablePos[3];
+	unsigned int*** m_ProcTable;
+
 	double* m_OrigDiscLines[3];
 	unsigned int m_OrigNumLines[3];
 	unsigned int m_SplitPos[3];