voloctree_parallel.cpp

/*---------------------------------------------------------------------------*\
 *
 *  bitpit
 *
 *  Copyright (C) 2015-2021 OPTIMAD engineering Srl
 *
 *  -------------------------------------------------------------------------
 *  License
 *  This file is part of bitpit.
 *
 *  bitpit is free software: you can redistribute it and/or modify it
 *  under the terms of the GNU Lesser General Public License v3 (LGPL)
 *  as published by the Free Software Foundation.
 *
 *  bitpit is distributed in the hope that it will be useful, but WITHOUT
 *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
 *  License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public License
 *  along with bitpit. If not, see <http://www.gnu.org/licenses/>.
 *
\*---------------------------------------------------------------------------*/
#if BITPIT_ENABLE_MPI==1
 
#include <mpi.h>
#include <unordered_set>
 
#include "voloctree.hpp"
 
using namespace std;
 
namespace bitpit {
 
void VolOctree::initializeTreePartitioning()
{
    // Move all the octants to the first processor
    //
    // Assigning a null weight to every octant but the last octant and then
    // doing a load balance will have the effect of moving all the octants
    // to the first process.
    std::vector<double> octantWeightStorage;
    const std::vector<double> *octantWeights;
 
    std::size_t nOctants = m_tree->getNumOctants();
    if (nOctants > 1) {
        octantWeightStorage.assign(nOctants, 0.);
        octantWeightStorage[nOctants - 1] = 1.;
        octantWeights = &octantWeightStorage;
    } else {
        octantWeights = nullptr;
    }
 
    m_tree->loadBalance(octantWeights);
}
 
void VolOctree::initializeTreeHaloSize(std::size_t haloSize)
{
    m_tree->setNofGhostLayers(haloSize);
}
 
void VolOctree::setCommunicator(MPI_Comm communicator)
{
    PatchKernel::setCommunicator(communicator);
 
    m_tree->setComm(communicator);
}
 
std::size_t VolOctree::_getMaxHaloSize()
{
    return std::numeric_limits<uint32_t>::max();
}
 
void VolOctree::_setHaloSize(std::size_t haloSize)
{
    initializeTreeHaloSize(haloSize);
}
 
std::vector<adaption::Info> VolOctree::_partitioningPrepare(const std::unordered_map<long, double> &cellWeights, double defaultWeight, bool trackPartitioning)
{
    // Eval partitioning weights
    computePartitioningOctantWeights(cellWeights, defaultWeight);
 
    // Generate partitioning information
    std::vector<adaption::Info> partitioningData;
    if (trackPartitioning) {
        int currentRank = getRank();
        PabloUniform::LoadBalanceRanges loadBalanceRanges = m_tree->evalLoadBalanceRanges(m_partitioningOctantWeights.get());
        for (const auto &entry : loadBalanceRanges.sendRanges) {
            int receiver = entry.first;
            if (receiver == currentRank) {
                continue;
            }
 
            adaption::Type adaptionType;
            if (loadBalanceRanges.sendAction == PabloUniform::LoadBalanceRanges::ACTION_DELETE) {
                adaptionType = adaption::TYPE_DELETION;
            } else {
                adaptionType = adaption::TYPE_PARTITION_SEND;
            }
 
            partitioningData.emplace_back();
            adaption::Info &partitioningInfo = partitioningData.back();
            partitioningInfo.entity = adaption::ENTITY_CELL;
            partitioningInfo.type   = adaptionType;
            partitioningInfo.rank   = receiver;
 
            const std::array<uint32_t, 2> &sendRange = entry.second;
            uint32_t beginTreeId = sendRange[0];
            uint32_t endTreeId   = sendRange[1];
            partitioningInfo.previous.reserve(endTreeId - beginTreeId);
            for (uint32_t treeId = beginTreeId; treeId < endTreeId; ++treeId) {
                OctantInfo octantInfo(treeId, true);
                partitioningInfo.previous.emplace_back();
                long &cellId = partitioningInfo.previous.back();
                cellId = getOctantId(octantInfo);
            }
        }
    }
 
    return partitioningData;
}
 
std::vector<adaption::Info> VolOctree::_partitioningAlter(bool trackPartitioning)
{
    std::vector<adaption::Info> partitioningData;
 
    // Early return if the dimension of the tree is null
    if (m_tree->getDim() == 0) {
        return partitioningData;
    }
 
    // Updating the tree
    m_tree->loadBalance(m_partitioningOctantWeights.get());
 
    // Sync the patch
    partitioningData = sync(trackPartitioning);
 
    // The bounding box is frozen, it is not updated automatically
    setBoundingBox();
 
    return partitioningData;
}
 
void VolOctree::_partitioningCleanup()
{
    // Clear partitioning weights
    clearPartitioningOctantWeights();
}
 
std::vector<long> VolOctree::_findGhostCellExchangeSources(int rank)
{
    const std::vector<uint32_t> &ghostOctantSources = m_tree->getBordersPerProc().at(rank);
    std::size_t nGhostSources = ghostOctantSources.size();
 
    std::vector<long> ghostCellSources(nGhostSources);
    for (std::size_t k = 0; k < nGhostSources; ++k) {
        OctantInfo octantInfo(ghostOctantSources[k], true);
        ghostCellSources[k] = getOctantId(octantInfo);
    }
 
    return ghostCellSources;
}
 
void VolOctree::computePartitioningOctantWeights(const std::unordered_map<long, double> &cellWeights, double defaultWeight)
{
    bool useWeights = !cellWeights.empty();
    MPI_Allreduce(MPI_IN_PLACE, &useWeights, 1, MPI_C_BOOL, MPI_LOR, getCommunicator());
    if (!useWeights) {
        clearPartitioningOctantWeights();
        return;
    }
 
    CellConstIterator beginItr = internalCellConstBegin();
    CellConstIterator endItr   = internalCellConstEnd();
 
    std::size_t nOctants = m_tree->getNumOctants();
    m_partitioningOctantWeights = std::unique_ptr<std::vector<double>>(new std::vector<double>(nOctants, defaultWeight));
    for (CellConstIterator cellItr = beginItr; cellItr != endItr; ++cellItr) {
        long cellId = cellItr.getId();
        auto weightItr = cellWeights.find(cellId);
        if (weightItr != cellWeights.end()) {
            OctantInfo octantInfo = getCellOctant(cellId);
            uint32_t treeId = octantInfo.id;
            (*m_partitioningOctantWeights)[treeId] = weightItr->second;
        }
    }
}
 
void VolOctree::clearPartitioningOctantWeights()
{
    m_partitioningOctantWeights.reset();
}
 
}
 
#endif