package org.openpixi.pixi.distributed;
import org.openpixi.pixi.distributed.ibis.IbisRegistry;
import org.openpixi.pixi.distributed.ibis.MasterToWorkers;
import org.openpixi.pixi.distributed.partitioning.Partitioner;
import org.openpixi.pixi.distributed.partitioning.SimplePartitioner;
import org.openpixi.pixi.distributed.util.CountLock;
import org.openpixi.pixi.distributed.util.IncomingResultHandler;
import org.openpixi.pixi.physics.particles.Particle;
import org.openpixi.pixi.physics.ParticleGridInitializer;
import org.openpixi.pixi.physics.Settings;
import org.openpixi.pixi.physics.grid.Cell;
import org.openpixi.pixi.physics.grid.Grid;
import org.openpixi.pixi.physics.grid.LocalInterpolation;
import org.openpixi.pixi.physics.util.ClassCopier;
import org.openpixi.pixi.physics.util.IntBox;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* Distributes the problem and collects the results.
* To each worker we assign exactly one problem.
*/
public class Master {
private MasterToWorkers communicator;
private Settings settings;
private Grid initialGrid;
private List<Particle> initialParticles;
private Grid finalGrid;
private List<Particle> finalParticles;
/**
* Problem decomposition table.
* N-th element in the table belongs to the n-th worker.
*/
private IntBox[] partitions;
/* Results received from workers */
private Cell[][][] gridPartitions;
private List<List<Particle>> particlePartitions = new ArrayList<List<Particle>>();
private CountLock resultsLock;
public Grid getFinalGrid() {
return finalGrid;
}
public List<Particle> getFinalParticles() {
return finalParticles;
}
public Master(IbisRegistry registry, Settings settings) {
this.settings = settings;
try {
communicator = new MasterToWorkers(registry, new ResultHandler());
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
// Initialize the grid
initialGrid = new Grid(settings);
initialParticles = settings.getParticles();
ParticleGridInitializer pgi = new ParticleGridInitializer();
pgi.initialize(
new LocalInterpolation(
settings.getInterpolator(), settings.getParticleIterator()),
settings.getPoissonSolver(),
initialParticles,
initialGrid);
// Initialize the classes holding the result
gridPartitions = new Cell[settings.getNumOfNodes()][][];
for (int i = 0; i < settings.getNumOfNodes(); i++) {
particlePartitions.add(new ArrayList<Particle>());
}
resultsLock = new CountLock(settings.getNumOfNodes());
}
public void distributeProblem() {
// Partition the problem
Partitioner partitioner = new SimplePartitioner();
partitions = partitioner.partition(
settings.getGridCellsX(), settings.getGridCellsY(), settings.getNumOfNodes());
// Log the partitioning scheme
Logger logger = LoggerFactory.getLogger(this.getClass());
logger.debug("Problem partitioning:\n{}", partitioner);
// Partition the data
List<Particle> initialParticlesCopy = copyInitialParticles();
List<List<Particle>> particlePartitions = partitionParticles(
partitions, initialParticlesCopy);
Cell[][][] gridPartitions = partitionGrid(partitions, initialGrid);
// Send to each worker
for (int workerID = 0; workerID < partitions.length; workerID++) {
try {
communicator.sendProblem(
workerID, partitions,
particlePartitions.get(workerID), gridPartitions[workerID]);
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
}
private List<Particle> copyInitialParticles() {
List<Particle> copy = new ArrayList<Particle>();
for (Particle p: initialParticles) {
copy.add(ClassCopier.copy(p));
}
return copy;
}
/**
* Divides cells according to partitions.
*/
private Cell[][][] partitionGrid(IntBox[] partitions, Grid grid) {
Cell[][][] gridPartitions = new Cell[partitions.length][][];
for (int i = 0; i < partitions.length; ++i) {
gridPartitions[i] = getSubgrid(partitions[i], grid);
}
return gridPartitions;
}
/**
* At the global simulation edges we also the extra cells!
* Since there is some initialization done on the grid
* (density charge interpolation + poisson solver)
* we need to distribute also the extra cells.
*
* At the inner partitions of global simulation we still need the ghost cells
* for correct field solving.
*/
private Cell[][] getSubgrid(IntBox partition, Grid grid) {
int startX = partition.xmin() - Grid.EXTRA_CELLS_BEFORE_GRID;
int endX = partition.xmax() + Grid.EXTRA_CELLS_AFTER_GRID;
int startY = partition.ymin() - Grid.EXTRA_CELLS_BEFORE_GRID;
int endY = partition.ymax() + Grid.EXTRA_CELLS_AFTER_GRID;
Cell[][] subGrid = new Cell[endX - startX + 1][endY - startY + 1];
for (int x = startX; x <= endX ; x++) {
for (int y = startY; y <= endY; y++) {
// We create a new copy of the cell so that each cell is unique.
// In another words we want to avoid shared references
// present in periodic boundaries.
subGrid[x - startX][y - startY] = ClassCopier.copy(grid.getCell(x,y));
}
}
return subGrid;
}
/**
* Divides particles according to partitions they belong to.
*/
private List<List<Particle>> partitionParticles(IntBox[] partitions, List<Particle> particles) {
List<List<Particle>> particlePartitions = new ArrayList<List<Particle>>();
for (int i = 0; i < partitions.length; ++i) {
particlePartitions.add(new ArrayList<Particle>());
}
for (Particle p: particles) {
int partitionIndex = -1;
int cellX = (int)Math.floor(p.getX() / initialGrid.getCellWidth());
int cellY = (int)Math.floor(p.getY() / initialGrid.getCellHeight());
for (int i = 0; i < partitions.length; ++i) {
if (partitions[i].contains(cellX, cellY)) {
partitionIndex = i;
}
}
assert partitionIndex != -1;
// Translate particle's position
double xmin = partitions[partitionIndex].xmin() * settings.getCellWidth();
double ymin = partitions[partitionIndex].ymin() * settings.getCellHeight();
p.setX(p.getX() - xmin);
p.setY(p.getY() - ymin);
particlePartitions.get(partitionIndex).add(p);
}
return particlePartitions;
}
public void collectResults() {
resultsLock.waitForCount();
resultsLock.reset();
finalParticles = assembleParticles(particlePartitions);
finalGrid = assembleGrid(gridPartitions);
}
/**
* Puts together the particle lists coming from workers.
*/
private List<Particle> assembleParticles(List<List<Particle>> particlePartitions) {
List<Particle> assembledParticles = new ArrayList<Particle>();
for (int i = 0; i < particlePartitions.size(); ++i) {
double xmin = partitions[i].xmin() * settings.getCellWidth();
double ymin = partitions[i].ymin() * settings.getCellHeight();
for (Particle p: particlePartitions.get(i)) {
// Translate particles position
p.setX(p.getX() + xmin);
p.setY(p.getY() + ymin);
assembledParticles.add(p);
}
}
return assembledParticles;
}
/**
* Puts together the subgrids coming from workers.
*/
private Grid assembleGrid(Cell[][][] gridPartitions) {
int totalXCells =
Grid.EXTRA_CELLS_BEFORE_GRID +
settings.getGridCellsX() +
Grid.EXTRA_CELLS_AFTER_GRID;
int totalYCells =
Grid.EXTRA_CELLS_BEFORE_GRID +
settings.getGridCellsY() +
Grid.EXTRA_CELLS_AFTER_GRID;
Cell[][] cells = new Cell[totalXCells][totalYCells];
for (int workerID = 0; workerID < partitions.length; ++workerID) {
fillSubgrid(partitions[workerID], gridPartitions[workerID], cells);
}
return new Grid(settings, cells);
}
/**
* We fill also the extra cells.
* In case of hardwall boundary we have additional information in them.
*/
private void fillSubgrid(IntBox partition, Cell[][] subgrid, Cell[][] cells) {
int startX = partition.xmin();
int endX = partition.xmax();
int startY = partition.ymin();
int endY = partition.ymax();
if (startX == 0) {
startX -= Grid.EXTRA_CELLS_BEFORE_GRID;
}
if (endX == settings.getGridCellsX() - 1) {
endX += Grid.EXTRA_CELLS_AFTER_GRID;
}
if (startY == 0) {
startY -= Grid.EXTRA_CELLS_BEFORE_GRID;
}
if (endY == settings.getGridCellsY() - 1) {
endY += Grid.EXTRA_CELLS_AFTER_GRID;
}
for (int x = startX; x <= endX ; x++) {
for (int y = startY; y <= endY; y++) {
cells[x + Grid.EXTRA_CELLS_BEFORE_GRID][y + Grid.EXTRA_CELLS_BEFORE_GRID] =
subgrid[x - startX][y - startY];
}
}
}
public void close() {
communicator.close();
}
private class ResultHandler implements IncomingResultHandler {
public void handle(int workerID, List<Particle> particles, Cell[][] cells) {
gridPartitions[workerID] = cells;
particlePartitions.set(workerID, particles);
resultsLock.increase();
}
}
}