Add volume data output to Schwarz solver

src/ParallelAlgorithms/LinearSolver/Multigrid/ElementActions.hpp

@@ -68,12 +68,12 @@ struct InitializeElement : tt::ConformsTo<amr::protocols::Projector> {
                  Tags::ParentMesh<Dim>,
                  observers::Tags::ObservationKey<Tags::MultigridLevel>,
                  observers::Tags::ObservationKey<Tags::IsFinestGrid>,
-                 Tags::ObservationId<OptionsGroup>,
+                 LinearSolver::Tags::ObservationId<OptionsGroup>,
                  Tags::VolumeDataForOutput<OptionsGroup, FieldsTag>>;
   using compute_tags = tmpl::list<>;
   using const_global_cache_tags =
       tmpl::list<Tags::MaxLevels<OptionsGroup>,
-                 Tags::OutputVolumeData<OptionsGroup>>;
+                 LinearSolver::Tags::OutputVolumeData<OptionsGroup>>;
 
   template <typename DbTagsList, typename... InboxTags, typename Metavariables,
             typename ActionList, typename ParallelComponent>
@@ -91,7 +91,7 @@ struct InitializeElement : tt::ConformsTo<amr::protocols::Projector> {
   using argument_tags =
       tmpl::list<domain::Tags::Mesh<Dim>, domain::Tags::Element<Dim>,
                  domain::Tags::InitialRefinementLevels<Dim>,
-                 Tags::OutputVolumeData<OptionsGroup>>;
+                 LinearSolver::Tags::OutputVolumeData<OptionsGroup>>;
   using return_tags = tmpl::append<simple_tags, simple_tags_from_options>;
 
   template <typename... AmrData>
@@ -147,11 +147,12 @@ struct InitializeElement : tt::ConformsTo<amr::protocols::Projector> {
       // Preserve state of observation ID
       if constexpr (tt::is_a_v<tuples::TaggedTuple, AmrData...>) {
         // h-refinement: copy from the parent
-        *observation_id = get<Tags::ObservationId<OptionsGroup>>(amr_data...);
+        *observation_id =
+            get<LinearSolver::Tags::ObservationId<OptionsGroup>>(amr_data...);
       } else if constexpr (tt::is_a_v<std::unordered_map, AmrData...>) {
         // h-coarsening: copy from one of the children (doesn't matter which)
-        *observation_id =
-            get<Tags::ObservationId<OptionsGroup>>(amr_data.begin()->second...);
+        *observation_id = get<LinearSolver::Tags::ObservationId<OptionsGroup>>(
+            amr_data.begin()->second...);
       } else {
         (void)observation_id;
       }
@@ -235,7 +236,7 @@ struct PreparePreSmoothing {
     }
 
     // Record pre-smoothing initial fields and source
-    if (db::get<Tags::OutputVolumeData<OptionsGroup>>(box)) {
+    if (db::get<LinearSolver::Tags::OutputVolumeData<OptionsGroup>>(box)) {
       db::mutate<Tags::VolumeDataForOutput<OptionsGroup, FieldsTag>>(
           [](const auto volume_data, const auto& initial_fields,
              const auto& source) {
@@ -293,7 +294,7 @@ struct SkipPostSmoothingAtBottom {
         not db::get<Tags::ParentId<Dim>>(box).has_value();
 
     // Record pre-smoothing result fields and residual
-    if (db::get<Tags::OutputVolumeData<OptionsGroup>>(box)) {
+    if (db::get<LinearSolver::Tags::OutputVolumeData<OptionsGroup>>(box)) {
       db::mutate<Tags::VolumeDataForOutput<OptionsGroup, FieldsTag>>(
           [](const auto volume_data, const auto& result_fields,
              const auto& residuals) {
@@ -362,7 +363,7 @@ struct SendCorrectionToFinerGrid {
     const auto& child_ids = db::get<Tags::ChildIds<Dim>>(box);
 
     // Record post-smoothing result fields and residual
-    if (db::get<Tags::OutputVolumeData<OptionsGroup>>(box)) {
+    if (db::get<LinearSolver::Tags::OutputVolumeData<OptionsGroup>>(box)) {
       db::mutate<Tags::VolumeDataForOutput<OptionsGroup, FieldsTag>>(
           [](const auto volume_data, const auto& result_fields,
              const auto& residuals) {
@@ -475,7 +476,7 @@ struct ReceiveCorrectionFromCoarserGrid {
         make_not_null(&box));
 
     // Record post-smoothing initial fields and source
-    if (db::get<Tags::OutputVolumeData<OptionsGroup>>(box)) {
+    if (db::get<LinearSolver::Tags::OutputVolumeData<OptionsGroup>>(box)) {
       db::mutate<Tags::VolumeDataForOutput<OptionsGroup, FieldsTag>>(
           [](const auto volume_data, const auto& initial_fields,
              const auto& source) {

src/ParallelAlgorithms/LinearSolver/Multigrid/ObserveVolumeData.hpp

@@ -69,12 +69,12 @@
       Parallel::GlobalCache<Metavariables>& cache,
       const ElementId<Dim>& element_id, const ActionList /*meta*/,
       const ParallelComponent* const /*meta*/) {
-    if (not db::get<Tags::OutputVolumeData<OptionsGroup>>(box)) {
+    if (not db::get<LinearSolver::Tags::OutputVolumeData<OptionsGroup>>(box)) {
       return {Parallel::AlgorithmExecution::Continue, std::nullopt};
     }
     const auto& volume_data = db::get<volume_data_tag>(box);
     const auto& observation_id =
-        db::get<Tags::ObservationId<OptionsGroup>>(box);
+        db::get<LinearSolver::Tags::ObservationId<OptionsGroup>>(box);
     const auto& mesh = db::get<domain::Tags::Mesh<Dim>>(box);
     const auto& inertial_coords =
         db::get<domain::Tags::Coordinates<Dim, Frame::Inertial>>(box);
@@ -124,7 +124,7 @@
         ElementVolumeData{element_id, std::move(components), mesh});
 
     // Increment observation ID
-    db::mutate<Tags::ObservationId<OptionsGroup>>(
+    db::mutate<LinearSolver::Tags::ObservationId<OptionsGroup>>(
         [](const auto local_observation_id) { ++(*local_observation_id); },
         make_not_null(&box));
     return {Parallel::AlgorithmExecution::Continue, std::nullopt};

src/ParallelAlgorithms/LinearSolver/Multigrid/Tags.hpp

@@ -122,7 +122,8 @@
 struct OutputVolumeData : db::SimpleTag {
   using type = bool;
   static constexpr bool pass_metavariables = false;
-  using option_tags = tmpl::list<OptionTags::OutputVolumeData<OptionsGroup>>;
+  using option_tags =
+      tmpl::list<LinearSolver::OptionTags::OutputVolumeData<OptionsGroup>>;
   static type create_from_options(const type value) { return value; };
   static std::string name() {
     return "OutputVolumeData(" + pretty_type::name<OptionsGroup>() + ")";
@@ -189,16 +190,6 @@
   using type = std::optional<Mesh<Dim>>;
 };
 
-// The following tags are related to volume data output
-
-/// Continuously incrementing ID for volume observations
-template <typename OptionsGroup>
-struct ObservationId : db::SimpleTag {
-  using type = size_t;
-  static std::string name() {
-    return "ObservationId(" + pretty_type::name<OptionsGroup>() + ")";
-  }
-};
 /// @{
 /// Prefix tag for recording volume data in
 /// `LinearSolver::multigrid::Tags::VolumeDataForOutput`

src/ParallelAlgorithms/LinearSolver/Schwarz/CMakeLists.txt

@@ -19,6 +19,7 @@ spectre_target_headers(
   ComputeTags.hpp
   ElementActions.hpp
   ElementCenteredSubdomainData.hpp
+  ObserveVolumeData.hpp
   OverlapHelpers.hpp
   Schwarz.hpp
   SubdomainOperator.hpp

src/ParallelAlgorithms/LinearSolver/Schwarz/ElementActions.hpp

@@ -206,17 +206,21 @@ struct InitializeElement : tt::ConformsTo<amr::protocols::Projector> {
       subdomain_solver<FieldsTag, SubdomainOperator, SubdomainPreconditioners>;
   using subdomain_solver_tag =
       Tags::SubdomainSolver<std::unique_ptr<SubdomainSolver>, OptionsGroup>;
+  using volume_data_tag =
+      Tags::VolumeDataForOutput<SubdomainData, OptionsGroup>;
 
  public:  // Iterable action
   using simple_tags_from_options = tmpl::list<subdomain_solver_tag>;
 
-  using const_global_cache_tags = tmpl::list<Tags::MaxOverlap<OptionsGroup>>;
+  using const_global_cache_tags =
+      tmpl::list<Tags::MaxOverlap<OptionsGroup>,
+                 LinearSolver::Tags::OutputVolumeData<OptionsGroup>>;
 
-  using simple_tags =
-      tmpl::list<Tags::IntrudingExtents<Dim, OptionsGroup>,
-                 Tags::Weight<OptionsGroup>,
-                 domain::Tags::Faces<Dim, Tags::Weight<OptionsGroup>>,
-                 SubdomainDataBufferTag<SubdomainData, OptionsGroup>>;
+  using simple_tags = tmpl::list<
+      Tags::IntrudingExtents<Dim, OptionsGroup>, Tags::Weight<OptionsGroup>,
+      domain::Tags::Faces<Dim, Tags::Weight<OptionsGroup>>,
+      SubdomainDataBufferTag<SubdomainData, OptionsGroup>,
+      LinearSolver::Tags::ObservationId<OptionsGroup>, volume_data_tag>;
   using compute_tags = tmpl::list<>;
   template <typename DbTagsList, typename... InboxTags, typename Metavariables,
             typename ActionList, typename ParallelComponent>
@@ -244,7 +248,9 @@ struct InitializeElement : tt::ConformsTo<amr::protocols::Projector> {
       const gsl::not_null<DirectionMap<Dim, Scalar<DataVector>>*>
           intruding_overlap_weights,
       const gsl::not_null<SubdomainData*> subdomain_data,
-      [[maybe_unused]] const gsl::not_null<std::unique_ptr<SubdomainSolver>*>
+      const gsl::not_null<size_t*> observation_id,
+      const gsl::not_null<SubdomainData*> /*volume_data_for_output*/,
+      const gsl::not_null<std::unique_ptr<SubdomainSolver>*>
           subdomain_solver,
       const Element<Dim>& element, const Mesh<Dim>& mesh,
       const tnsr::I<DataVector, Dim, Frame::ElementLogical>& logical_coords,
@@ -299,14 +305,23 @@ struct InitializeElement : tt::ConformsTo<amr::protocols::Projector> {
     // Subdomain solver
     // The subdomain solver initially gets created from options on each element.
     // Then we have to copy it around during AMR.
-    if constexpr (sizeof...(AmrData) == 1) {
+    if constexpr (sizeof...(AmrData) == 0) {
+      *observation_id = 0;
+      (void)subdomain_solver;
+    } else {
       if constexpr (tt::is_a_v<tuples::TaggedTuple, AmrData...>) {
         // h-refinement: copy from the parent
+        *observation_id =
+            get<LinearSolver::Tags::ObservationId<OptionsGroup>>(amr_data...);
         *subdomain_solver = get<subdomain_solver_tag>(amr_data...)->get_clone();
       } else if constexpr (tt::is_a_v<std::unordered_map, AmrData...>) {
         // h-coarsening, copy from one of the children (doesn't matter which)
+        *observation_id = get<LinearSolver::Tags::ObservationId<OptionsGroup>>(
+            amr_data.begin()->second...);
         *subdomain_solver =
             get<subdomain_solver_tag>(amr_data.begin()->second...)->get_clone();
+      } else {
+        (void)observation_id;
       }
       (*subdomain_solver)->reset();
     }
@@ -453,6 +468,15 @@ struct SolveSubdomain {
           db::get<Tags::ObservePerCoreReductions<OptionsGroup>>(box));
     }
 
+    // Record subdomain solution for volume data output
+    if (db::get<LinearSolver::Tags::OutputVolumeData<OptionsGroup>>(box)) {
+      db::mutate<Tags::VolumeDataForOutput<SubdomainData, OptionsGroup>>(
+          [&subdomain_solution](const auto volume_data) {
+            volume_data->element_data = subdomain_solution.element_data;
+          },
+          make_not_null(&box));
+    }
+
     // Apply weighting
     if (LIKELY(max_overlap > 0)) {
       subdomain_solution.element_data *=
@@ -537,11 +561,22 @@ struct ReceiveOverlapSolution {
                        pretty_type::name<OptionsGroup>(), iteration_id);
     }
 
-    // Add solutions on overlaps to this element's solution in a weighted sum
+    // Extract overlap solutions from inbox
     const auto received_overlap_solutions =
         std::move(tuples::get<overlap_solution_inbox_tag>(inboxes)
                       .extract(iteration_id)
                       .mapped());
+
+    // Record overlap solutions for volume data output
+    if (db::get<LinearSolver::Tags::OutputVolumeData<OptionsGroup>>(box)) {
+      db::mutate<Tags::VolumeDataForOutput<SubdomainData, OptionsGroup>>(
+          [&received_overlap_solutions](const auto volume_data) {
+            volume_data->overlap_data = received_overlap_solutions;
+          },
+          make_not_null(&box));
+    }
+
+    // Add overlap solutions to this element's solution in a weighted sum
     db::mutate<fields_tag>(
         [&received_overlap_solutions](
             const auto fields, const Index<Dim>& full_extents,