patch_info.cpp

/*---------------------------------------------------------------------------*\
 *
 *  bitpit
 *
 *  Copyright (C) 2015-2021 OPTIMAD engineering Srl
 *
 *  -------------------------------------------------------------------------
 *  License
 *  This file is part of bitpit.
 *
 *  bitpit is free software: you can redistribute it and/or modify it
 *  under the terms of the GNU Lesser General Public License v3 (LGPL)
 *  as published by the Free Software Foundation.
 *
 *  bitpit is distributed in the hope that it will be useful, but WITHOUT
 *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
 *  License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public License
 *  along with bitpit. If not, see <http://www.gnu.org/licenses/>.
 *
\*---------------------------------------------------------------------------*/
 
#if BITPIT_ENABLE_MPI==1
#   include <mpi.h>
#endif
 
#include "patch_info.hpp"
#include "patch_kernel.hpp"
 
namespace bitpit {


PatchInfo::PatchInfo(PatchKernel const *patch)
    : m_patch(nullptr)
{
    setPatch(patch, false);
}

PatchKernel const & PatchInfo::getPatch() const
{
    return *m_patch;
}

void PatchInfo::setPatch(PatchKernel const *patch)
{
    setPatch(patch, true);
}

void PatchInfo::setPatch(PatchKernel const *patch, bool initialize)
{
    // Reset information
    if (m_patch != nullptr) {
        reset();
    }
 
    // Set the patch
    m_patch = patch;
 
    // Call initialization
    if (initialize) {
        _init();
    }
}

void PatchInfo::reset()
{
    _reset();
}

void PatchInfo::extract()
{
    if (m_patch == nullptr) {
        return;
    }
 
    // Reset information
    reset();
 
    // Extract new information
    _extract();
}

void PatchInfo::update()
{
    extract();
}


PatchNumberingInfo::PatchNumberingInfo(PatchKernel const *patch)
    : PatchInfo(patch)
{
    PatchNumberingInfo::_init();
 
    extract();
}

void PatchNumberingInfo::_init()
{
    m_cellConsecutiveOffset = -1;
}

void PatchNumberingInfo::_reset()
{
    // Clear data
    m_cellLocalToConsecutiveMap.clear();
#if BITPIT_ENABLE_MPI==1
    m_nGlobalInternalCells.clear();
#endif
 
    // Initialize data
    _init();
}

void PatchNumberingInfo::_extract()
{
    long consecutiveId;
 
#if BITPIT_ENABLE_MPI==1
    // Initialize communications
    size_t exchangeDataSize = sizeof(consecutiveId);
    std::unique_ptr<DataCommunicator> dataCommunicator;
 
    if (m_patch->isPartitioned()) {
        // Create the data communicator
        dataCommunicator = std::unique_ptr<DataCommunicator>(new DataCommunicator(m_patch->getCommunicator()));
 
        // Set and start the receives
        for (const auto &entry : m_patch->getGhostCellExchangeTargets()) {
            const int rank = entry.first;
            const auto &list = entry.second;
 
            dataCommunicator->setRecv(rank, list.size() * exchangeDataSize);
            dataCommunicator->startRecv(rank);
        }
    }
#endif
 
#if BITPIT_ENABLE_MPI==1
    // Get the internal cell count of all the partitions
    if (m_patch->isPartitioned()) {
        m_nGlobalInternalCells.resize(m_patch->getProcessorCount());
        long nLocalInternalCells = m_patch->getInternalCellCount();
        MPI_Allgather(&nLocalInternalCells, 1, MPI_LONG, m_nGlobalInternalCells.data(), 1, MPI_LONG, m_patch->getCommunicator());
    } else {
        m_nGlobalInternalCells.resize(1);
        m_nGlobalInternalCells[0] = m_patch->getInternalCellCount();
    }
#endif
 
    // Initialize data structure for consecutive ids
    m_cellLocalToConsecutiveMap.reserve(m_patch->getCellCount());
 
    // Evalaute the consecutive id of the internal cells
    if (m_patch->getInternalCellCount() > 0) {
        PatchKernel::CellConstIterator beginItr = m_patch->internalCellConstBegin();
        PatchKernel::CellConstIterator endItr   = m_patch->internalCellConstEnd();
 
        std::size_t index = 0;
        std::vector<std::pair<long, long>> nativeIds(m_patch->getInternalCellCount());
        for (PatchKernel::CellConstIterator itr = beginItr; itr != endItr; ++itr) {
            long id = itr.getId();
            long nativeId = m_patch->_getCellNativeIndex(id);
 
            nativeIds[index++] = std::make_pair(nativeId, id);
        }
 
        auto nativeIdsBegin = nativeIds.begin();
        auto nativeIdsEnd   = nativeIds.end();
 
        std::sort(
            nativeIdsBegin,
            nativeIdsEnd,
            [](const std::pair<long, long> &x, const std::pair<long, long> &y) -> bool
            {
                return x.first < y.first;
            }
        );
 
#if BITPIT_ENABLE_MPI==1
        m_cellConsecutiveOffset = getCellGlobalCountOffset();
#else
        m_cellConsecutiveOffset = 0;
#endif
 
        consecutiveId = m_cellConsecutiveOffset;
        for (auto itr = nativeIdsBegin; itr != nativeIdsEnd; ++itr) {
            m_cellLocalToConsecutiveMap.insert({itr->second, consecutiveId++});
        }
    }
 
#if BITPIT_ENABLE_MPI==1
    // Communicate the consecutive id of the ghost cells
    if (m_patch->isPartitioned()) {
        // Set and start the sends
        for (const auto &entry : m_patch->getGhostCellExchangeSources()) {
            const int rank = entry.first;
            auto &list = entry.second;
 
            dataCommunicator->setSend(rank, list.size() * exchangeDataSize);
            SendBuffer &buffer = dataCommunicator->getSendBuffer(rank);
            for (long id : list) {
                buffer << m_cellLocalToConsecutiveMap.at(id);
            }
            dataCommunicator->startSend(rank);
        }
 
        // Receive the consecutive ids of the ghosts
        int nCompletedRecvs = 0;
        while (nCompletedRecvs < dataCommunicator->getRecvCount()) {
            int rank = dataCommunicator->waitAnyRecv();
            const auto &list = m_patch->getGhostCellExchangeTargets(rank);
 
            RecvBuffer &buffer = dataCommunicator->getRecvBuffer(rank);
            for (long id : list) {
                buffer >> consecutiveId;
                m_cellLocalToConsecutiveMap.insert({id, consecutiveId});
            }
 
            ++nCompletedRecvs;
        }
 
        // Wait for the sends to finish
        dataCommunicator->waitAllSends();
    }
#endif
}

long PatchNumberingInfo::getCellConsecutiveOffset() const
{
    return m_cellConsecutiveOffset;
}

long PatchNumberingInfo::getCellConsecutiveId(long id) const
{
    return m_cellLocalToConsecutiveMap.at(id);
}

const std::unordered_map<long, long> & PatchNumberingInfo::getCellConsecutiveMap() const
{
    return m_cellLocalToConsecutiveMap;
}
 
 
#if BITPIT_ENABLE_MPI==1
long PatchNumberingInfo::getCellGlobalCount() const
{
    long nGlobalCells = 0;
    for (long count : m_nGlobalInternalCells) {
        nGlobalCells += count;
    }
 
    return nGlobalCells;
}

long PatchNumberingInfo::getCellGlobalCountOffset() const
{
    return getCellGlobalCountOffset(m_patch->getRank());
}

long PatchNumberingInfo::getCellGlobalCountOffset(int rank) const
{
    long offset = 0;
    for (int i = 0; i < rank; ++i) {
        offset += m_nGlobalInternalCells[i];
    }
 
    return offset;
}

long PatchNumberingInfo::getCellGlobalId(long id) const
{
    // Global ids and consecutive ids are the same
    return m_cellLocalToConsecutiveMap.at(id);
}

const std::unordered_map<long, long> & PatchNumberingInfo::getCellGlobalMap() const
{
    // Global ids and consecutive ids are the same
    return m_cellLocalToConsecutiveMap;
}
 

int PatchNumberingInfo::getCellOwnerFromLocal(long id) const
{
    auto ghostCellOwnerItr = m_patch->m_ghostCellInfo.find(id);
    if (ghostCellOwnerItr != m_patch->m_ghostCellInfo.end()) {
        return ghostCellOwnerItr->second.owner;
    } else {
        return m_patch->getRank();
    }
}

int PatchNumberingInfo::getCellOwnerFromConsecutive(long id) const
{
    if (!m_patch->isPartitioned()) {
        return m_patch->getRank();
    }
 
    long offset = 0;
    for (int k = 0; k < m_patch->getProcessorCount(); ++k) {
        offset += m_nGlobalInternalCells[k];
        if (id < offset) {
            return k;
        }
    }
 
    return -1;
}

int PatchNumberingInfo::getCellOwnerFromGlobal(long id) const
{
    // Global ids and consecutive ids are the same
    return getCellOwnerFromConsecutive(id);
}
#endif
 
}