My Project
Loading...
Searching...
No Matches
ISTLSolverEbos.hpp
1/*
2 Copyright 2016 IRIS AS
3 Copyright 2019, 2020 Equinor ASA
4 Copyright 2020 SINTEF Digital, Mathematics and Cybernetics
5
6 This file is part of the Open Porous Media project (OPM).
7
8 OPM is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 3 of the License, or
11 (at your option) any later version.
12
13 OPM is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with OPM. If not, see <http://www.gnu.org/licenses/>.
20*/
21
22#ifndef OPM_ISTLSOLVER_EBOS_HEADER_INCLUDED
23#define OPM_ISTLSOLVER_EBOS_HEADER_INCLUDED
24
25#include <dune/istl/owneroverlapcopy.hh>
26#include <dune/istl/solver.hh>
27
28#include <ebos/eclbasevanguard.hh>
29
30#include <opm/common/ErrorMacros.hpp>
31#include <opm/common/Exceptions.hpp>
32#include <opm/common/TimingMacros.hpp>
33
34#include <opm/models/discretization/common/fvbaseproperties.hh>
35#include <opm/models/common/multiphasebaseproperties.hh>
36#include <opm/models/utils/parametersystem.hh>
37#include <opm/models/utils/propertysystem.hh>
38#include <opm/simulators/flow/BlackoilModelParametersEbos.hpp>
39#include <opm/simulators/linalg/ExtractParallelGridInformationToISTL.hpp>
40#include <opm/simulators/linalg/FlowLinearSolverParameters.hpp>
41#include <opm/simulators/linalg/matrixblock.hh>
42#include <opm/simulators/linalg/istlsparsematrixadapter.hh>
43#include <opm/simulators/linalg/PreconditionerWithUpdate.hpp>
44#include <opm/simulators/linalg/WellOperators.hpp>
45#include <opm/simulators/linalg/WriteSystemMatrixHelper.hpp>
46#include <opm/simulators/linalg/findOverlapRowsAndColumns.hpp>
47#include <opm/simulators/linalg/getQuasiImpesWeights.hpp>
48#include <opm/simulators/linalg/setupPropertyTree.hpp>
49
50#include <any>
51#include <cstddef>
52#include <functional>
53#include <memory>
54#include <set>
55#include <sstream>
56#include <string>
57#include <tuple>
58#include <vector>
59
60namespace Opm::Properties {
61
62namespace TTag {
64 using InheritsFrom = std::tuple<FlowIstlSolverParams>;
65};
66}
67
68template <class TypeTag, class MyTypeTag>
70
73template<class TypeTag>
74struct SparseMatrixAdapter<TypeTag, TTag::FlowIstlSolver>
75{
76private:
80
81public:
82 using type = typename Linear::IstlSparseMatrixAdapter<Block>;
83};
84
85} // namespace Opm::Properties
86
87namespace Opm
88{
89
90
91namespace detail
92{
93
94template<class Matrix, class Vector, class Comm>
95struct FlexibleSolverInfo
96{
97 using AbstractSolverType = Dune::InverseOperator<Vector, Vector>;
98 using AbstractOperatorType = Dune::AssembledLinearOperator<Matrix, Vector, Vector>;
99 using AbstractPreconditionerType = Dune::PreconditionerWithUpdate<Vector, Vector>;
100
101 void create(const Matrix& matrix,
102 bool parallel,
103 const PropertyTree& prm,
104 std::size_t pressureIndex,
105 std::function<Vector()> trueFunc,
106 const bool forceSerial,
107 Comm& comm);
108
109 std::unique_ptr<AbstractSolverType> solver_;
110 std::unique_ptr<AbstractOperatorType> op_;
111 std::unique_ptr<LinearOperatorExtra<Vector,Vector>> wellOperator_;
112 AbstractPreconditionerType* pre_ = nullptr;
113 std::size_t interiorCellNum_ = 0;
114};
115
116
117#ifdef HAVE_MPI
119void copyParValues(std::any& parallelInformation, std::size_t size,
120 Dune::OwnerOverlapCopyCommunication<int,int>& comm);
121#endif
122
125template<class Matrix>
126void makeOverlapRowsInvalid(Matrix& matrix,
127 const std::vector<int>& overlapRows);
128
131template<class Matrix, class Grid>
132std::unique_ptr<Matrix> blockJacobiAdjacency(const Grid& grid,
133 const std::vector<int>& cell_part,
134 std::size_t nonzeroes,
135 const std::vector<std::set<int>>& wellConnectionsGraph);
136}
137
142 template <class TypeTag>
144 {
145 protected:
153 using Matrix = typename SparseMatrixAdapter::IstlMatrix;
156 using AbstractSolverType = Dune::InverseOperator<Vector, Vector>;
157 using AbstractOperatorType = Dune::AssembledLinearOperator<Matrix, Vector, Vector>;
161 constexpr static std::size_t pressureIndex = GetPropType<TypeTag, Properties::Indices>::pressureSwitchIdx;
162
163#if HAVE_MPI
164 using CommunicationType = Dune::OwnerOverlapCopyCommunication<int,int>;
165#else
166 using CommunicationType = Dune::CollectiveCommunication<int>;
167#endif
168
169 public:
170 using AssembledLinearOperatorType = Dune::AssembledLinearOperator< Matrix, Vector, Vector >;
171
172 static void registerParameters()
173 {
174 FlowLinearSolverParameters::registerParameters<TypeTag>();
175 }
176
185 : simulator_(simulator),
186 iterations_( 0 ),
187 converged_(false),
188 matrix_(nullptr),
189 parameters_{parameters},
190 forceSerial_(forceSerial)
191 {
192 initialize();
193 }
194
197 explicit ISTLSolverEbos(const Simulator& simulator)
198 : simulator_(simulator),
199 iterations_( 0 ),
200 solveCount_(0),
201 converged_(false),
202 matrix_(nullptr)
203 {
204 parameters_.resize(1);
205 parameters_[0].template init<TypeTag>(simulator_.vanguard().eclState().getSimulationConfig().useCPR());
206 initialize();
207 }
208
209 void initialize()
210 {
212
213 if (parameters_[0].linsolver_ == "hybrid") {
214 // Experimental hybrid configuration.
215 // When chosen, will set up two solvers, one with CPRW
216 // and the other with ILU0 preconditioner. More general
217 // options may be added later.
218 prm_.clear();
219 parameters_.clear();
220 {
222 para.init<TypeTag>(false);
223 para.linsolver_ = "cprw";
224 parameters_.push_back(para);
225 prm_.push_back(setupPropertyTree(parameters_[0],
226 EWOMS_PARAM_IS_SET(TypeTag, int, LinearSolverMaxIter),
227 EWOMS_PARAM_IS_SET(TypeTag, double, LinearSolverReduction)));
228 }
229 {
230 FlowLinearSolverParameters para;
231 para.init<TypeTag>(false);
232 para.linsolver_ = "ilu0";
233 parameters_.push_back(para);
234 prm_.push_back(setupPropertyTree(parameters_[1],
235 EWOMS_PARAM_IS_SET(TypeTag, int, LinearSolverMaxIter),
236 EWOMS_PARAM_IS_SET(TypeTag, double, LinearSolverReduction)));
237 }
238 // ------------
239 } else {
240 // Do a normal linear solver setup.
241 assert(parameters_.size() == 1);
242 assert(prm_.empty());
243 prm_.push_back(setupPropertyTree(parameters_[0],
244 EWOMS_PARAM_IS_SET(TypeTag, int, LinearSolverMaxIter),
245 EWOMS_PARAM_IS_SET(TypeTag, double, LinearSolverReduction)));
246 }
247 flexibleSolver_.resize(prm_.size());
248
249 const bool on_io_rank = (simulator_.gridView().comm().rank() == 0);
250#if HAVE_MPI
251 comm_.reset( new CommunicationType( simulator_.vanguard().grid().comm() ) );
252#endif
253 extractParallelGridInformationToISTL(simulator_.vanguard().grid(), parallelInformation_);
254
255 // For some reason simulator_.model().elementMapper() is not initialized at this stage
256 //const auto& elemMapper = simulator_.model().elementMapper(); //does not work.
257 // Set it up manually
258 ElementMapper elemMapper(simulator_.vanguard().gridView(), Dune::mcmgElementLayout());
259 detail::findOverlapAndInterior(simulator_.vanguard().grid(), elemMapper, overlapRows_, interiorRows_);
260 useWellConn_ = EWOMS_GET_PARAM(TypeTag, bool, MatrixAddWellContributions);
261 const bool ownersFirst = EWOMS_GET_PARAM(TypeTag, bool, OwnerCellsFirst);
262 if (!ownersFirst) {
263 const std::string msg = "The linear solver no longer supports --owner-cells-first=false.";
264 if (on_io_rank) {
265 OpmLog::error(msg);
266 }
267 OPM_THROW_NOLOG(std::runtime_error, msg);
268 }
269
270 const int interiorCellNum_ = detail::numMatrixRowsToUseInSolver(simulator_.vanguard().grid(), true);
271 for (auto& f : flexibleSolver_) {
272 f.interiorCellNum_ = interiorCellNum_;
273 }
274
275 // Print parameters to PRT/DBG logs.
276 if (on_io_rank && parameters_[activeSolverNum_].linear_solver_print_json_definition_) {
277 std::ostringstream os;
278 os << "Property tree for linear solvers:\n";
279 for (std::size_t i = 0; i<prm_.size(); i++) {
280 prm_[i].write_json(os, true);
281 }
282 OpmLog::note(os.str());
283 }
284 }
285
286 // nothing to clean here
287 void eraseMatrix()
288 {
289 }
290
291 void setActiveSolver(const int num)
292 {
293 if (num > static_cast<int>(prm_.size()) - 1) {
294 OPM_THROW(std::logic_error, "Solver number " + std::to_string(num) + " not available.");
295 }
296 activeSolverNum_ = num;
297 auto cc = Dune::MPIHelper::getCollectiveCommunication();
298 if (cc.rank() == 0) {
299 OpmLog::debug("Active solver = " + std::to_string(activeSolverNum_)
300 + " (" + parameters_[activeSolverNum_].linsolver_ + ")");
301 }
302 }
303
304 int numAvailableSolvers()
305 {
306 return flexibleSolver_.size();
307 }
308
309 void prepare(const SparseMatrixAdapter& M, Vector& b)
310 {
311 prepare(M.istlMatrix(), b);
312 }
313
314 void prepare(const Matrix& M, Vector& b)
315 {
316 OPM_TIMEBLOCK(istlSolverEbosPrepare);
317 const bool firstcall = (matrix_ == nullptr);
318#if HAVE_MPI
319 if (firstcall && isParallel()) {
320 const std::size_t size = M.N();
321 detail::copyParValues(parallelInformation_, size, *comm_);
322 }
323#endif
324
325 // update matrix entries for solvers.
326 if (firstcall) {
327 // ebos will not change the matrix object. Hence simply store a pointer
328 // to the original one with a deleter that does nothing.
329 // Outch! We need to be able to scale the linear system! Hence const_cast
330 matrix_ = const_cast<Matrix*>(&M);
331
332 useWellConn_ = EWOMS_GET_PARAM(TypeTag, bool, MatrixAddWellContributions);
333 // setup sparsity pattern for jacobi matrix for preconditioner (only used for openclSolver)
334 } else {
335 // Pointers should not change
336 if ( &M != matrix_ ) {
337 OPM_THROW(std::logic_error,
338 "Matrix objects are expected to be reused when reassembling!");
339 }
340 }
341 rhs_ = &b;
342
343 // TODO: check all solvers, not just one.
344 if (isParallel() && prm_[activeSolverNum_].template get<std::string>("preconditioner.type") != "ParOverILU0") {
345 detail::makeOverlapRowsInvalid(getMatrix(), overlapRows_);
346 }
347 prepareFlexibleSolver();
348 }
349
350
351 void setResidual(Vector& /* b */)
352 {
353 // rhs_ = &b; // Must be handled in prepare() instead.
354 }
355
356 void getResidual(Vector& b) const
357 {
358 b = *rhs_;
359 }
360
361 void setMatrix(const SparseMatrixAdapter& /* M */)
362 {
363 // matrix_ = &M.istlMatrix(); // Must be handled in prepare() instead.
364 }
365
366 int getSolveCount() const {
367 return solveCount_;
368 }
369
370 void resetSolveCount() {
371 solveCount_ = 0;
372 }
373
374 bool solve(Vector& x)
375 {
376 OPM_TIMEBLOCK(istlSolverEbosSolve);
377 ++solveCount_;
378 // Write linear system if asked for.
379 const int verbosity = prm_[activeSolverNum_].get("verbosity", 0);
380 const bool write_matrix = verbosity > 10;
381 if (write_matrix) {
382 Helper::writeSystem(simulator_, //simulator is only used to get names
383 getMatrix(),
384 *rhs_,
385 comm_.get());
386 }
387
388 // Solve system.
389 Dune::InverseOperatorResult result;
390 {
391 OPM_TIMEBLOCK(flexibleSolverApply);
392 assert(flexibleSolver_[activeSolverNum_].solver_);
393 flexibleSolver_[activeSolverNum_].solver_->apply(x, *rhs_, result);
394 }
395
396 // Check convergence, iterations etc.
397 checkConvergence(result);
398
399 return converged_;
400 }
401
402
408
410 int iterations () const { return iterations_; }
411
413 const std::any& parallelInformation() const { return parallelInformation_; }
414
415 const CommunicationType* comm() const { return comm_.get(); }
416
417 protected:
418#if HAVE_MPI
419 using Comm = Dune::OwnerOverlapCopyCommunication<int, int>;
420#endif
421
422 void checkConvergence( const Dune::InverseOperatorResult& result ) const
423 {
424 // store number of iterations
425 iterations_ = result.iterations;
426 converged_ = result.converged;
427 if(!converged_){
428 if(result.reduction < parameters_[activeSolverNum_].relaxed_linear_solver_reduction_){
429 std::stringstream ss;
430 ss<< "Full linear solver tolerance not achieved. The reduction is:" << result.reduction
431 << " after " << result.iterations << " iterations ";
432 OpmLog::warning(ss.str());
433 converged_ = true;
434 }
435 }
436 // Check for failure of linear solver.
437 if (!parameters_[activeSolverNum_].ignoreConvergenceFailure_ && !converged_) {
438 const std::string msg("Convergence failure for linear solver.");
439 OPM_THROW_NOLOG(NumericalProblem, msg);
440 }
441 }
442 protected:
443
444 bool isParallel() const {
445#if HAVE_MPI
446 return !forceSerial_ && comm_->communicator().size() > 1;
447#else
448 return false;
449#endif
450 }
451
452 void prepareFlexibleSolver()
453 {
454 OPM_TIMEBLOCK(flexibleSolverPrepare);
455 if (shouldCreateSolver()) {
456 std::function<Vector()> trueFunc =
457 [this]
458 {
459 return this->getTrueImpesWeights(pressureIndex);
460 };
461
462 if (!useWellConn_) {
463 auto wellOp = std::make_unique<WellModelOperator>(simulator_.problem().wellModel());
464 flexibleSolver_[activeSolverNum_].wellOperator_ = std::move(wellOp);
465 }
466 OPM_TIMEBLOCK(flexibleSolverCreate);
467 flexibleSolver_[activeSolverNum_].create(getMatrix(),
468 isParallel(),
469 prm_[activeSolverNum_],
470 pressureIndex,
471 trueFunc,
472 forceSerial_,
473 *comm_);
474 }
475 else
476 {
477 OPM_TIMEBLOCK(flexibleSolverUpdate);
478 flexibleSolver_[activeSolverNum_].pre_->update();
479 }
480 }
481
482
486 {
487 // Decide if we should recreate the solver or just do
488 // a minimal preconditioner update.
489 if (flexibleSolver_.empty()) {
490 return true;
491 }
492 if (!flexibleSolver_[activeSolverNum_].solver_) {
493 return true;
494 }
495 if (this->parameters_[activeSolverNum_].cpr_reuse_setup_ == 0) {
496 // Always recreate solver.
497 return true;
498 }
499 if (this->parameters_[activeSolverNum_].cpr_reuse_setup_ == 1) {
500 // Recreate solver on the first iteration of every timestep.
501 const int newton_iteration = this->simulator_.model().newtonMethod().numIterations();
502 return newton_iteration == 0;
503 }
504 if (this->parameters_[activeSolverNum_].cpr_reuse_setup_ == 2) {
505 // Recreate solver if the last solve used more than 10 iterations.
506 return this->iterations() > 10;
507 }
508 if (this->parameters_[activeSolverNum_].cpr_reuse_setup_ == 3) {
509 // Recreate solver if the last solve used more than 10 iterations.
510 return false;
511 }
512 if (this->parameters_[activeSolverNum_].cpr_reuse_setup_ == 4) {
513 // Recreate solver every 'step' solve calls.
514 const int step = this->parameters_[activeSolverNum_].cpr_reuse_interval_;
515 const bool create = ((solveCount_ % step) == 0);
516 return create;
517 }
518
519 // If here, we have an invalid parameter.
520 const bool on_io_rank = (simulator_.gridView().comm().rank() == 0);
521 std::string msg = "Invalid value: " + std::to_string(this->parameters_[activeSolverNum_].cpr_reuse_setup_)
522 + " for --cpr-reuse-setup parameter, run with --help to see allowed values.";
523 if (on_io_rank) {
524 OpmLog::error(msg);
525 }
526 throw std::runtime_error(msg);
527
528 // Never reached.
529 return false;
530 }
531
532
533 // Weights to make approximate pressure equations.
534 // Calculated from the storage terms (only) of the
535 // conservation equations, ignoring all other terms.
536 Vector getTrueImpesWeights(int pressureVarIndex) const
537 {
538 OPM_TIMEBLOCK(getTrueImpesWeights);
539 Vector weights(rhs_->size());
540 ElementContext elemCtx(simulator_);
541 Amg::getTrueImpesWeights(pressureVarIndex, weights,
542 simulator_.vanguard().gridView(),
543 elemCtx, simulator_.model(),
544 ThreadManager::threadId());
545 return weights;
546 }
547
548 Matrix& getMatrix()
549 {
550 return *matrix_;
551 }
552
553 const Matrix& getMatrix() const
554 {
555 return *matrix_;
556 }
557
558 const Simulator& simulator_;
559 mutable int iterations_;
560 mutable int solveCount_;
561 mutable bool converged_;
562 std::any parallelInformation_;
563
564 // non-const to be able to scale the linear system
565 Matrix* matrix_;
566 Vector *rhs_;
567
568 int activeSolverNum_ = 0;
569 std::vector<detail::FlexibleSolverInfo<Matrix,Vector,CommunicationType>> flexibleSolver_;
570 std::vector<int> overlapRows_;
571 std::vector<int> interiorRows_;
572
573 bool useWellConn_;
574
575 std::vector<FlowLinearSolverParameters> parameters_;
576 bool forceSerial_ = false;
577 std::vector<PropertyTree> prm_;
578
579 std::shared_ptr< CommunicationType > comm_;
580 }; // end ISTLSolver
581
582} // namespace Opm
583#endif
Interface class adding the update() method to the preconditioner interface.
Definition PreconditionerWithUpdate.hpp:32
Definition AquiferInterface.hpp:35
This class solves the fully implicit black-oil system by solving the reduced system (after eliminatin...
Definition ISTLSolverEbos.hpp:144
ISTLSolverEbos(const Simulator &simulator, const FlowLinearSolverParameters &parameters, bool forceSerial=false)
Construct a system solver.
Definition ISTLSolverEbos.hpp:184
int iterations() const
Solve the system of linear equations Ax = b, with A being the combined derivative matrix of the resid...
Definition ISTLSolverEbos.hpp:410
const std::any & parallelInformation() const
Definition ISTLSolverEbos.hpp:413
ISTLSolverEbos(const Simulator &simulator)
Construct a system solver.
Definition ISTLSolverEbos.hpp:197
bool shouldCreateSolver() const
Return true if we should (re)create the whole solver, instead of just calling update() on the precond...
Definition ISTLSolverEbos.hpp:485
Definition PropertyTree.hpp:37
This file contains a set of helper functions used by VFPProd / VFPInj.
Definition BlackoilPhases.hpp:27
PropertyTree setupPropertyTree(FlowLinearSolverParameters p, bool linearSolverMaxIterSet, bool linearSolverReductionSet)
Set up a property tree intended for FlexibleSolver by either reading the tree from a JSON file or cre...
Definition setupPropertyTree.cpp:40
This class carries all parameters for the NewtonIterationBlackoilInterleaved class.
Definition FlowLinearSolverParameters.hpp:237
Definition ISTLSolverEbos.hpp:69
Definition ISTLSolverEbos.hpp:63