My Project
PropsCentroidsDataHandle.hpp
1 /*
2  Copyright 2020 Equinor AS.
3 
4  This file is part of the Open Porous Media project (OPM).
5 
6  OPM is free software: you can redistribute it and/or modify
7  it under the terms of the GNU General Public License as published by
8  the Free Software Foundation, either version 3 of the License, or
9  (at your option) any later version.
10 
11  OPM is distributed in the hope that it will be useful,
12  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  GNU General Public License for more details.
15 
16  You should have received a copy of the GNU General Public License
17  along with OPM. If not, see <http://www.gnu.org/licenses/>.
18 */
26 #ifndef PROPS_CENTROIDS_DATAHANDLE_HPP
27 #define PROPS_CENTROIDS_DATAHANDLE_HPP
28 
29 #if HAVE_MPI
30 #include <opm/simulators/utils/ParallelEclipseState.hpp>
31 #include <opm/simulators/utils/ParallelRestart.hpp>
32 #include <dune/grid/common/datahandleif.hh>
33 #include <dune/grid/common/mcmgmapper.hh>
34 #include <dune/grid/common/partitionset.hh>
35 #include <dune/common/parallel/mpihelper.hh>
36 #include <unordered_map>
37 #include <iostream>
38 
39 namespace Opm
40 {
41 
47 template<class Grid>
48 class PropsCentroidsDataHandle
49  : public Dune::CommDataHandleIF< PropsCentroidsDataHandle<Grid>, double>
50 {
51 public:
53  using DataType = std::pair<double, unsigned char>;
54 
64  PropsCentroidsDataHandle(const Grid& grid, ParallelEclipseState& eclState,
65  const EclipseGrid* eclGridOnRoot,
66  std::vector<double>& centroids,
67  const typename Dune::CartesianIndexMapper<Grid>& cartMapper)
68  : m_grid(grid), m_distributed_fieldProps(eclState.m_fieldProps),
69  m_centroids(centroids)
70  {
71  // Scatter the keys
72  const Parallel::Communication comm = m_grid.comm();
73  if (comm.rank() == 0)
74  {
75  const auto& globalProps = eclState.globalFieldProps();
76  m_intKeys = globalProps.keys<int>();
77  m_doubleKeys = globalProps.keys<double>();
78  std::size_t packSize = Mpi::packSize(m_intKeys, comm) +
79  Mpi::packSize(m_doubleKeys,comm);
80  std::vector<char> buffer(packSize);
81  int position = 0;
82  Mpi::pack(m_intKeys, buffer, position, comm);
83  Mpi::pack(m_doubleKeys, buffer, position, comm);
84  int calcStart = position;
85  {
86  std::vector<char> tran_buffer = globalProps.serialize_tran();
87  position += tran_buffer.size();
88  buffer.insert(buffer.end(), std::make_move_iterator(tran_buffer.begin()), std::make_move_iterator(tran_buffer.end()));
89  }
90  comm.broadcast(&position, 1, 0);
91  comm.broadcast(buffer.data(), position, 0);
92 
93  // Unpack Calculator as we need it here, too.
94  m_distributed_fieldProps.deserialize_tran( std::vector<char>(buffer.begin() + calcStart, buffer.end()) );
95 
96  // copy data to persistent map based on local id
97  m_no_data = m_intKeys.size() + m_doubleKeys.size() +
98  Grid::dimensionworld;
99  const auto& idSet = m_grid.localIdSet();
100  const auto& gridView = m_grid.levelGridView(0);
101  using ElementMapper =
102  Dune::MultipleCodimMultipleGeomTypeMapper<typename Grid::LevelGridView>;
103  ElementMapper elemMapper(gridView, Dune::mcmgElementLayout());
104 
105  for( const auto &element : elements( gridView, Dune::Partitions::interiorBorder ) )
106  {
107  const auto& id = idSet.id(element);
108  auto index = elemMapper.index(element);
109  auto& data = elementData_[id];
110  data.reserve(m_no_data);
111 
112  for (const auto& intKey : m_intKeys)
113  {
114  const auto& fieldData = globalProps.get_int_field_data(intKey);
115  data.emplace_back(fieldData.data[index],
116  static_cast<unsigned char>(fieldData.value_status[index]));
117  }
118 
119  for (const auto& doubleKey : m_doubleKeys)
120  {
121  // We need to allow unsupported keywords to get the data
122  // for TranCalculator, too.
123  const auto& fieldData = globalProps.get_double_field_data(doubleKey,
124  /* allow_unsupported = */ true);
125  data.emplace_back(fieldData.data[index],
126  static_cast<unsigned char>(fieldData.value_status[index]));
127  }
128 
129  auto cartIndex = cartMapper.cartesianIndex(index);
130  const auto& center = eclGridOnRoot->getCellCenter(cartIndex);
131  for (int dim = 0; dim < Grid::dimensionworld; ++dim)
132  data.emplace_back(center[dim], '1'); // write garbage for value_status
133  }
134  }
135  else
136  {
137  int bufferSize;
138  comm.broadcast(&bufferSize, 1, 0);
139  std::vector<char> buffer(bufferSize);
140  comm.broadcast(buffer.data(), bufferSize, 0);
141  int position{};
142  Mpi::unpack(m_intKeys, buffer, position, comm);
143  Mpi::unpack(m_doubleKeys, buffer, position, comm);
144  m_distributed_fieldProps.deserialize_tran( std::vector<char>(buffer.begin() + position, buffer.end()) );
145  m_no_data = m_intKeys.size() + m_doubleKeys.size() +
146  Grid::dimensionworld;
147  }
148  }
149 
150  ~PropsCentroidsDataHandle()
151  {
152  // distributed grid is now correctly set up.
153  for(const auto& intKey : m_intKeys)
154  {
155  m_distributed_fieldProps.m_intProps[intKey].data.resize(m_grid.size(0));
156  m_distributed_fieldProps.m_intProps[intKey].value_status.resize(m_grid.size(0));
157  }
158 
159  for(const auto& doubleKey : m_doubleKeys)
160  {
161  m_distributed_fieldProps.m_doubleProps[doubleKey].data.resize(m_grid.size(0));
162  m_distributed_fieldProps.m_doubleProps[doubleKey].value_status.resize(m_grid.size(0));
163  }
164 
165  m_centroids.resize(m_grid.size(0) * Grid::dimensionworld);
166 
167  // copy data for the persistent mao to the field properties
168  const auto& idSet = m_grid.localIdSet();
169  const auto& gridView = m_grid.levelGridView(0);
170  using ElementMapper =
171  Dune::MultipleCodimMultipleGeomTypeMapper<typename Grid::LevelGridView>;
172  ElementMapper elemMapper(gridView, Dune::mcmgElementLayout());
173 
174  for( const auto &element : elements( gridView, Dune::Partitions::all ) )
175  {
176  std::size_t counter{};
177  const auto& id = idSet.id(element);
178  auto index = elemMapper.index(element);
179  auto data = elementData_.find(id);
180  assert(data != elementData_.end());
181 
182  for(const auto& intKey : m_intKeys)
183  {
184  const auto& pair = data->second[counter++];
185  m_distributed_fieldProps.m_intProps[intKey].data[index] = static_cast<int>(pair.first);
186  m_distributed_fieldProps.m_intProps[intKey].value_status[index] = static_cast<value::status>(pair.second);
187  }
188 
189  for(const auto& doubleKey : m_doubleKeys)
190  {
191  const auto& pair = data->second[counter++];
192  m_distributed_fieldProps.m_doubleProps[doubleKey].data[index] = pair.first;
193  m_distributed_fieldProps.m_doubleProps[doubleKey].value_status[index] = static_cast<value::status>(pair.second);
194  }
195 
196  auto centroidIter = m_centroids.begin() + Grid::dimensionworld * index;
197  auto centroidIterEnd = centroidIter + Grid::dimensionworld;
198  for ( ; centroidIter != centroidIterEnd; ++centroidIter )
199  *centroidIter = data->second[counter++].first; // value_status discarded
200  }
201  }
202 
203  bool contains(int /* dim */, int codim)
204  {
205  return codim == 0;
206  }
207 
208  bool fixedsize(int /* dim */, int /* codim */)
209  {
210  return true;
211  }
212  bool fixedSize(int /* dim */, int /* codim */)
213  {
214  return true;
215  }
216 
217  template<class EntityType>
218  std::size_t size(const EntityType /* entity */)
219  {
220  return m_no_data;
221  }
222 
223  template<class BufferType, class EntityType>
224  void gather(BufferType& buffer, const EntityType& e) const
225  {
226  auto iter = elementData_.find(m_grid.localIdSet().id(e));
227  assert(iter != elementData_.end());
228  for(const auto& data : iter->second)
229  {
230  buffer.write(data);
231  }
232  }
233 
234  template<class BufferType, class EntityType>
235  void scatter(BufferType& buffer, const EntityType& e, std::size_t n)
236  {
237  assert(n == m_no_data);
238  auto& array = elementData_[m_grid.localIdSet().id(e)];
239  array.resize(n);
240  for(auto& data: array)
241  {
242  buffer.read(data);
243  }
244  }
245 
246 private:
247  using LocalIdSet = typename Grid::LocalIdSet;
248  const Grid& m_grid;
250  ParallelFieldPropsManager& m_distributed_fieldProps;
252  std::vector<std::string> m_intKeys;
254  std::vector<std::string> m_doubleKeys;
258  std::unordered_map<typename LocalIdSet::IdType, std::vector<std::pair<double,unsigned char> > > elementData_;
260  std::vector<double>& m_centroids;
262  std::size_t m_no_data;
263 };
264 
265 } // end namespace Opm
266 #endif // HAVE_MPI
267 #endif // PROPS_CENTROIDS_DATAHANDLE_HPP
268 
This file contains a set of helper functions used by VFPProd / VFPInj.
Definition: BlackoilPhases.hpp:26