eschnett/cactusamrex - Change 5XGIB7XMEZNBLA5ZLQTXRTC3AZ5CTRGMXWBPVMWXG4DPHKWDF4ZAC

More work on I/O

Created by Erik Schnetter on August 19, 2019

5XGIB7XMEZNBLA5ZLQTXRTC3AZ5CTRGMXWBPVMWXG4DPHKWDF4ZAC

Dependencies

In channels

master

main

Change contents

Replacement in AMReX/src/driver.cxx at line 91 [5.15972]

B:BD[4.1568] → [4.1568:1615]

    const Array4<char> &tag = tags.array(mfi);

[4.1568]

[6.1697]

    const Array4<char> &tags_array4 = tags.array(mfi);

Replacement in AMReX/src/driver.cxx at line 94 [5.15972]

B:BD[7.193] → [7.193:299]

      tag(grid.cactus_offset.x + p.i, grid.cactus_offset.y + p.j,
          grid.cactus_offset.z + p.k) =

[7.193]

[7.299]

      tags_array4(grid.cactus_offset.x + p.i, grid.cactus_offset.y + p.j,
                  grid.cactus_offset.z + p.k) =

Replacement in AMReX/src/driver.hxx at line 77 [5.15275]
B:BD[8.199] → [7.4275:4307]
```
      array<int,dim> indextype;
```
[8.199]
[9.1601]
```
      array<int, dim> indextype;
```
Insertion in AMReX/src/io.cxx at line 5 [10.107]
[4.4552]
[10.147]
```
#include "timer.hxx"
```

Insertion in AMReX/src/io.cxx at line 32 [10.107]

[7.4412]


  static Timer timer("OutputPlotfile");
  Interval interval(timer);

Replacement in AMReX/src/io.cxx at line 46 [10.107]

B:BD[7.4821] → [7.4821:4988]

      ostringstream buf;
      buf << out_dir << "/" << groupname;
      buf << ".it" << setw(6) << setfill('0') << cctk_iteration;
      string filename = buf.str();

[7.4821]

[7.4988]

      string filename = [&]() {
        ostringstream buf;
        buf << out_dir << "/" << groupname << ".it" << setw(6) << setfill('0')
            << cctk_iteration;
        return buf.str();
      }();

Insertion in AMReX/src/io.cxx at line 76 [10.107]

[7.6038]


      const bool is_root = CCTK_MyProc(nullptr) == 0;
      if (is_root) {
        string visitname = [&]() {
          ostringstream buf;
          buf << out_dir << "/" << groupname << ".visit";
          return buf.str();
        }();
        ofstream visit(visitname, ios::app);
        assert(visit.good());
        // visit << filename << "/Header\n";
        visit << groupname << ".it" << setw(6) << setfill('0') << cctk_iteration
              << "/Header\n";
        visit.close();
      }

Insertion in AMReX/src/io.cxx at line 228 [10.107]

[11.197]

[10.610]

  static Timer timer("OutputASCII");
  Interval interval(timer);

Insertion in AMReX/src/io.cxx at line 265 [10.107]

[12.1091]

[7.8248]

  static Timer timer("OutputNorms");
  Interval interval(timer);

Replacement in AMReX/src/io.cxx at line 276 [10.107]

B:BD[13.529] → [13.529:639]

    const int numvars = groupdata.numvars;
    for (int vi = 0; vi < numvars; ++vi) {
      const int tl = 0;

[13.529]

[13.639]


    const int tl = 0;
    for (int vi = 0; vi < groupdata.numvars; ++vi) {
#if 1

Insertion in AMReX/src/io.cxx at line 282 [10.107]

[13.692]

[6.8197]

#else
      reduction<CCTK_REAL> red;
      for (auto &restrict leveldata : ghext->leveldata) {
        auto &restrict groupdata = leveldata.groupdata.at(gi);
        MultiFab &mfab = *groupdata.mfab.at(tl);
        reduction<CCTK_REAL> red1;
        red1.min = mfab.min(vi);
        red1.max = mfab.max(vi);
        red1.sum = mfab.sum(vi);
        // red1.sum2 = mfab.sum2(vi);
        // red1.vol = mfab.vol(vi);
        red1.maxabs = mfab.norminf(vi);
        red1.sumabs = mfab.norm1(vi, mfab.fb_period);
        red1.sum2abs = pow(mfab.norm2(vi), 2);
        red += red1;
      }
#endif

Replacement in AMReX/src/io.cxx at line 304 [10.107]

B:BD[6.8342] → [6.8342:8393]

∅:D[6.8393] → [14.0:322]

B:BD[13.800] → [14.0:322]

             << ": maxabs=" << red.maxabs << "\n";
      // CCTK_REAL maxabs = 0.0;
      // for (auto &restrict leveldata : ghext->leveldata) {
      //   auto &restrict groupdata = leveldata.groupdata.at(gi);
      //   MultiFab &mfab = *groupdata.mfab.at(tl);
      //   maxabs = fmax(maxabs, mfab.norminf(vi));
      // }
      // CCTK_VINFO("            %g", maxabs);

[6.8342]

[8.2278]

             << ": maxabs=" << red.maxabs << " vol=" << red.vol << "\n";

Insertion in AMReX/src/io.cxx at line 313 [10.107]

[7.8442]

  static Timer timer("OutputGH");
  Interval interval(timer);

Replacement in AMReX/src/io.cxx at line 321 [10.107]

B:BD[7.8642] → [7.8642:8786]

  if (out_every <= 0 || cctk_iteration % out_every != 0)
    return 0;
  OutputPlotfile(cctkGH);
  OutputASCII(cctkGH);
  OutputNorms(cctkGH);

[7.8642]

[7.8786]

  if (out_every > 0 && cctk_iteration % out_every == 0) {
    OutputPlotfile(cctkGH);
    OutputASCII(cctkGH);
    OutputNorms(cctkGH);
  }

Replacement in AMReX/src/make.code.defn at line 4 [5.15114]

B:BD[5.15192] → [15.4735:4797]

SRCS = driver.cxx io.cxx level.cxx reduction.cxx schedule.cxx

[5.15192]

[5.15209]

SRCS = driver.cxx io.cxx level.cxx reduction.cxx schedule.cxx timer.cxx

Insertion in AMReX/src/reduction.cxx at line 7 [13.870]
[13.919]
[13.919]
```
#include <AMReX_MultiFabUtil.H>
```
Insertion in AMReX/src/reduction.cxx at line 9 [13.870]
[13.950]
[13.950]
```
#include <cstdlib>
```

Insertion in AMReX/src/reduction.cxx at line 15 [13.870]

[13.1013]

[13.1185]


namespace {
template <typename T>
void mpi_reduce_typed(const void *restrict x0, void *restrict y0,
                      const int *restrict length) {
  const T *restrict x = static_cast<const T *>(x0);
  T *restrict y = static_cast<T *>(y0);
#pragma omp simd
  for (int i = 0; i < *length; ++i)
    y[i] += x[i];
}
void mpi_reduce(void *restrict x, void *restrict y, int *restrict length,
                MPI_Datatype *restrict datatype) {
  if (*datatype == MPI_FLOAT)
    return mpi_reduce_typed<float>(x, y, length);
  if (*datatype == MPI_DOUBLE)
    return mpi_reduce_typed<double>(x, y, length);
  if (*datatype == MPI_LONG_DOUBLE)
    return mpi_reduce_typed<long double>(x, y, length);
  abort();
}
} // namespace
MPI_Op reduction_mpi_op() {
  static MPI_Op op = MPI_OP_NULL;
  if (op == MPI_OP_NULL)
    MPI_Op_create(mpi_reduce, 1, &op);
  return op;
}
////////////////////////////////////////////////////////////////////////////////

Replacement in AMReX/src/reduction.cxx at line 51 [13.870]

B:BD[7.15266] → [7.15266:15339]

                          const T *restrict ptr, const GridDesc &grid) {

[7.15266]

[16.4011]

                          const GridPtrDesc &grid, const T *restrict ptr,
                          const Array4<const int> *restrict finemask_array4) {

Replacement in AMReX/src/reduction.cxx at line 60 [13.870]

B:BD[7.15409] → [7.15409:15468]

    red = reduction<T>(red, reduction<T>(dV, ptr[p.idx]));

[7.15409]

[4.9750]

    if (!finemask_array4 || !(*finemask_array4)(grid.cactus_offset.x + p.i,
                                                grid.cactus_offset.y + p.j,
                                                grid.cactus_offset.z + p.k))
      red = reduction<T>(red, reduction<T>(dV, ptr[p.idx]));

Replacement in AMReX/src/reduction.cxx at line 73 [13.870]

B:BD[13.1791] → [13.1791:1899]

B:BD[13.1899] → [15.4798:4871]

∅:D[16.4268] → [13.1899:1960]

∅:D[15.4871] → [13.1899:1960]

B:BD[13.1899] → [13.1899:1960]

B:BD[13.2007] → [6.8588:8716]

B:BD[6.8716] → [7.15469:15544]

∅:D[7.15544] → [4.9758:9800]

B:BD[13.2163] → [4.9758:9800]

#pragma omp parallel reduction(reduction : red)
  {
    for (auto &restrict leveldata : ghext->leveldata) {
      const auto &restrict geom = ghext->amrcore->Geom(leveldata.level);
      auto &restrict groupdata = leveldata.groupdata.at(gi);
      auto mfitinfo = MFItInfo().SetDynamic(true).EnableTiling(
          {max_tile_size_x, max_tile_size_y, max_tile_size_z});
      for (MFIter mfi(*leveldata.mfab0, mfitinfo); mfi.isValid(); ++mfi) {
        GridPtrDesc grid(leveldata, mfi);

[13.1791]

[13.2241]

  for (auto &restrict leveldata : ghext->leveldata) {
    const auto &restrict geom = ghext->amrcore->Geom(leveldata.level);
    const auto &restrict groupdata = leveldata.groupdata.at(gi);
    const MultiFab &mfab = *groupdata.mfab.at(tl);
    unique_ptr<iMultiFab> finemask;

Replacement in AMReX/src/reduction.cxx at line 79 [13.870]

B:BD[13.2507] → [13.2507:2582]

B:BD[13.2582] → [4.9801:9861]

        const Array4<CCTK_REAL> &vars = groupdata.mfab.at(tl)->array(mfi);
        const CCTK_REAL *restrict ptr = grid.ptr(vars, vi);

[13.2242]

[13.2660]

    const int fine_level = leveldata.level + 1;
    if (fine_level < int(ghext->leveldata.size())) {
      const auto &restrict fine_leveldata = ghext->leveldata.at(fine_level);
      const auto &restrict fine_groupdata = fine_leveldata.groupdata.at(gi);
      const MultiFab &fine_mfab = *fine_groupdata.mfab.at(tl);
      const IntVect reffact{2, 2, 2};
      finemask = make_unique<iMultiFab>(
          makeFineMask(mfab, fine_mfab.boxArray(), reffact));
    }

Replacement in AMReX/src/reduction.cxx at line 90 [13.870]

B:BD[13.2661] → [6.8867:8905]

B:BD[6.8905] → [7.15545:15602]

∅:D[16.4319] → [13.2705:2713]

∅:D[6.8955] → [13.2705:2713]

∅:D[7.15602] → [13.2705:2713]

∅:D[4.9908] → [13.2705:2713]

B:BD[13.2705] → [13.2705:2713]

#warning "TODO: Skip refined regions"
        red += reduce_array(geom, groupdata, ptr, grid);
      }

[13.2661]

[13.2713]

    auto mfitinfo = MFItInfo().SetDynamic(true).EnableTiling(
        {max_tile_size_x, max_tile_size_y, max_tile_size_z});
#pragma omp parallel reduction(reduction : red)
    for (MFIter mfi(*leveldata.mfab0, mfitinfo); mfi.isValid(); ++mfi) {
      GridPtrDesc grid(leveldata, mfi);
      unique_ptr<Array4<const int> > finemask_array4;
      if (finemask)
        finemask_array4 = make_unique<Array4<const int> >(finemask->array(mfi));
      const Array4<const CCTK_REAL> &vars = mfab.array(mfi);
      const CCTK_REAL *restrict ptr = grid.ptr(vars, vi);
      red += reduce_array(geom, groupdata, grid, ptr, finemask_array4.get());

Insertion in AMReX/src/reduction.cxx at line 103 [13.870]

[13.2723]


  MPI_Op op = reduction_mpi_op();
  MPI_Allreduce(MPI_IN_PLACE, &red, 1, MPI_DOUBLE, op, MPI_COMM_WORLD);

Insertion in AMReX/src/reduction.hxx at line 12 [13.2777]
[2.431]
[13.2841]
```
#include <mpi.h>
```
Insertion in AMReX/src/reduction.hxx at line 71 [13.2777]
[13.4064]
[13.4064]
```
MPI_Op reduction_mpi_op();
```
Insertion in AMReX/src/schedule.cxx at line 3 [17.531]
[4.10576]
[17.577]
```
#include "timer.hxx"
```
Insertion in AMReX/src/schedule.cxx at line 453 [17.531]
[18.1983]
[18.1983]
```
  static Timer timer("Initialise");
  Interval interval(timer);
```
Insertion in AMReX/src/schedule.cxx at line 487 [17.531]
[16.6056]
[19.1106]
```
    const int *restrict nghostzones = cctkGH->cctk_nghostzones;
```
Replacement in AMReX/src/schedule.cxx at line 492 [17.531]
∅:D[6.9961] → [16.6116:6154]
B:BD[19.1350] → [16.6116:6154]
```
      x1[d] = x0[d] + gsh[d] * dx[d];
```
[6.9961]
[6.9962]
```
      x1[d] = x0[d] + (gsh[d] - 2 * nghostzones[d]) * dx[d];
```

Replacement in AMReX/src/schedule.cxx at line 544 [17.531]

B:BD[15.5232] → [20.7635:7671]

      CreateRefinedGrid(level + 1);

[15.5232]

[15.5233]

      {
        static Timer timer("InitialiseRegrid");
        Interval interval(timer);
        CreateRefinedGrid(level + 1);
      }

Insertion in AMReX/src/schedule.cxx at line 644 [17.531]
[20.7802]
[20.7802]
```
  static Timer timer("Evolve");
  Interval interval(timer);
```

Replacement in AMReX/src/schedule.cxx at line 661 [17.531]

B:BD[20.8023] → [20.8023:8062]

      ghext->amrcore->regrid(0, time);

[20.8023]

[20.8062]

      {
        static Timer timer("EvolveRegrid");
        Interval interval(timer);
        ghext->amrcore->regrid(0, time);
      }

Insertion in AMReX/src/schedule.cxx at line 698 [17.531]
[6.10413]
[6.10413]
```
  static Timer timer("Shutdown");
  Interval interval(timer);
```
Insertion in AMReX/src/schedule.cxx at line 772 [17.531]
[21.2422]
[21.2422]
```
  static Timer timer("Sync");
  Interval interval(timer);
```
Insertion in AMReX/src/schedule.cxx at line 841 [17.531]
[22.456]
[20.8407]
```
  static Timer timer("Restrict");
  Interval interval(timer);
```

File addition: timer.cxx (-rw-r--r--)

[5.15096]

#include "timer.hxx"
namespace AMReX {
Timer::Timer(const string &name)
    : name(name), handle(CCTK_TimerCreate(name.c_str())) {}
Interval::Interval(const Timer &timer) : timer(timer) {
  CCTK_TimerStartI(timer.handle);
}
Interval::~Interval() { CCTK_TimerStopI(timer.handle); }
} // namespace AMReX

File addition: timer.hxx (-rw-r--r--)

[5.15096]

#ifndef TIMER_HXX
#define TIMER_HXX
#include <cctk.h>
#undef copysign
#undef fpclassify
#undef isfinite
#undef isinf
#undef isnan
#undef isnormal
#undef signbit
#include <string>
namespace AMReX {
using namespace std;
class Interval;
class Timer {
  friend class Interval;
  string name;
  int handle;
public:
  Timer() = delete;
  Timer(const string &name);
};
class Interval {
  const Timer &timer;
public:
  Interval() = delete;
  Interval(const Timer &timer);
  ~Interval();
};
} // namespace AMReX
#endif // #ifndef TIMER_HXX

Replacement in MaxwellToyAMReX/interface.ccl at line 15 [23.13068]

B:BD[23.13268] → [7.17788:17861]

# d*dA = mu J   A^a,^b_b - A^b,^a_b = mu J^a
# d*A  = 0      A^a,a   = 0

[23.13268]

[7.17861]

# 4d:
#
# d*dA = mu J      A^a,^b_b - A^b,^a_b = mu J^a
# d*A  = 0         A^a,a               = 0
#
# F   = dA        F_ab                     = A_a,b - A_b,a
# dF  = 0         F_ab,c + F_bc,a + F_ca,b = 0
# d*F = mu J      F^ab,a                   = mu J^b

Insertion in MaxwellToyAMReX/interface.ccl at line 24 [23.13068]
[7.17862]
[7.17862]
```
# 3d:
#
```

Replacement in MaxwellToyAMReX/interface.ccl at line 28 [23.13068]

B:BD[7.17929] → [7.17929:17986]

# *phi,t + d*A         = 0             phi,t + A^i,i = 0

[7.17929]

[7.17986]

# *phi,t + d*A         = 0                phi,t + A^i,i = 0
#
# E          = -dphi - A,t      E_i                      = - phi,i - A_i,t
# B          = dA               B_ij                     = A_j,i - A_i,j
# dE + B,t   = 0                E_j,i - E_i,j + B_ij,t   = 0
# dB         = 0                B_jk,i + B_ki,j + B_ij,k = 0
# d*E        = rho/epsilon      E^i,i                    = rho/epsilon
# d*B - dt*E = mu J             - B^ij,i - E^j,t         = mu J^j

Deletion in MaxwellToyAMReX/interface.ccl at line 37 [23.13068]

B:BD[7.17987] → [7.17987:18136]

# F   = dA     F_ab                     = A_a,b - A_b,a
# dF  = 0      F_ab,c + F_bc,a + F_ca,b = 0
# d*F = mu J   F^ab,a                   = mu J^b

Deletion in MaxwellToyAMReX/interface.ccl at line 38 [23.13068]

B:BD[23.13389] → [7.18137:18522]

# E          = -dphi - A,t   E_i                     = - phi,i - A_i,t
# B          = dA            B_ij                    = A_j,i - A_i,j
# dE + B,t   = 0             E_j,i - E_i,j + B_ij,t  = 0
# dB         = 0             B_jk,i + B_ki,j + B_ijk = 0
# d*E        = rho/epsilon   E^i,i                   = rho/epsilon
# d*B - dt*E = mu J          - B^ij,i - E^j,t        = mu J^j

Insertion in MaxwellToyAMReX/interface.ccl at line 66 [23.13068]
[7.18999]
[23.13519]
```
# 
# CCTK_REAL constraints TYPE=gf
# {
#   dive
#   divb
# } "Constraints"
```

Replacement in MaxwellToyAMReX/interface.ccl at line 199 [23.13068]

B:BD[23.14451] → [23.14451:14526]

# CCTK_REAL constraints TYPE=gf
# {
#   dive
#   divb
# } "Constraints"
#

[23.14451]

[23.14526]


CCTK_REAL constraints_dive TYPE=gf TAGS='index={0 0 0}'
{
  dive
} "Constraints"
CCTK_REAL constraints_divb TYPE=gf TAGS='index={1 1 1}'
{
  divb
} "Constraints"

File addition: maxwell-gaussian-nob.par (-rw-r--r--)

[24.6]

ActiveThorns = "
    AMReX
    Formaline
    IOUtil
    MaxwellToyAMReX
    SystemTopology
    # TerminationTrigger
    TimerReport
"
 
$nlevels = 5
$ncells = 32
Cactus::cctk_show_schedule = no
# Cactus::terminate = "iteration"
# Cactus::cctk_itlast = 0
Cactus::terminate = "time"
Cactus::cctk_final_time = 2.0
# AMReX::verbose = yes
AMReX::xmin = -1.0
AMReX::ymin = -1.0
AMReX::zmin = -1.0
AMReX::xmax = +1.0
AMReX::ymax = +1.0
AMReX::zmax = +1.0
AMReX::ncells_x = $ncells
AMReX::ncells_y = $ncells
AMReX::ncells_z = $ncells
AMReX::max_num_levels = $nlevels
AMReX::regrid_every = 16
AMReX::regrid_error_threshold = 0.01
AMReX::dtfac = 0.5
MaxwellToyAMReX::evolve_b = no
MaxwellToyAMReX::initial_condition = "Gaussian wave"
MaxwellToyAMReX::spatial_frequency_x = 0.5
MaxwellToyAMReX::spatial_frequency_y = 0.0
MaxwellToyAMReX::spatial_frequency_z = 0.0
MaxwellToyAMReX::analyse_every = $ncells * 2 ** ($nlevels - 1) / 32
IO::out_dir = $parfile
IO::out_every = $ncells * 2 ** ($nlevels - 1) / 32
# AMReX::out_tsv = yes
TimerReport::out_every = $ncells * 2 ** ($nlevels - 1) / 32
TimerReport::out_filename = "TimerReport"
TimerReport::output_all_timers_together = yes
TimerReport::output_all_timers_readable = yes
TimerReport::n_top_timers = 50
# TerminationTrigger::termination_from_file = yes
# TerminationTrigger::create_termination_file = yes
# TerminationTrigger::termination_file = "../TERMINATE"

Insertion in MaxwellToyAMReX/par/maxwell-gaussian.par at line 6 [24.30]
[24.103]
[24.103]
```
    SystemTopology
    # TerminationTrigger
    TimerReport
```
Insertion in MaxwellToyAMReX/par/maxwell-gaussian.par at line 41 [24.30]
[24.619]
[24.619]
```
MaxwellToyAMReX::evolve_b = yes
```
Insertion in MaxwellToyAMReX/par/maxwell-gaussian.par at line 49 [24.30]
[24.803]
[24.803]
```
MaxwellToyAMReX::analyse_every = $ncells * 2 ** ($nlevels - 1) / 32
```

Insertion in MaxwellToyAMReX/par/maxwell-gaussian.par at line 55 [24.30]

[24.901]


TimerReport::out_every = $ncells * 2 ** ($nlevels - 1) / 32
TimerReport::out_filename = "TimerReport"
TimerReport::output_all_timers_together = yes
TimerReport::output_all_timers_readable = yes
TimerReport::n_top_timers = 50
# TerminationTrigger::termination_from_file = yes
# TerminationTrigger::create_termination_file = yes
# TerminationTrigger::termination_file = "../TERMINATE"

Insertion in MaxwellToyAMReX/param.ccl at line 2 [23.14686]
[23.14737]
[23.14737]
```
BOOLEAN evolve_b "Evolve magnetic field" STEERABLE=never
{
} yes
```
Insertion in MaxwellToyAMReX/param.ccl at line 13 [23.14686]
[23.14788]
[23.14788]
```
  "linear" :: "for testing"
```

Insertion in MaxwellToyAMReX/param.ccl at line 37 [23.14686]

[3.2412]


CCTK_INT analyse_every "How often to calculate analysis quantities" STEERABLE=always
{
  0 :: "never"
  1:* :: "every that many iterations"
} 1

Replacement in MaxwellToyAMReX/schedule.ccl at line 3 [23.15053]

B:BD[23.15104] → [23.15104:15219]

SCHEDULE MaxwellToyAMReX_Initialize AT initial
{
  LANG: C
} "Set up initial conditions for the Maxwell equations"

[23.15104]

[23.15219]

if (evolve_b) {
  SCHEDULE MaxwellToyAMReX_Initialize AT initial
  {
    LANG: C
    SYNC: potential_phi
    SYNC: potential_ax potential_ay potential_az
    SYNC: field_ex field_ey field_ez
    SYNC: field_bx field_by field_bz
    SYNC: current_rho
    SYNC: current_jx current_jy current_jz
  } "Set up initial conditions for the Maxwell equations"
} else {
  SCHEDULE MaxwellToyAMReX_Initialize AT initial
  {
    LANG: C
    SYNC: potential_phi
    SYNC: potential_ax potential_ay potential_az
    SYNC: field_ex field_ey field_ez
    SYNC: current_rho
    SYNC: current_jx current_jy current_jz
  } "Set up initial conditions for the Maxwell equations"
  SCHEDULE MaxwellToyAMReX_Dependents1 AT initial AFTER MaxwellToyAMReX_Initialize
  {
    LANG: C
    SYNC: field_bx field_by field_bz
  } "Calculate the magnetic field"
}

Insertion in MaxwellToyAMReX/schedule.ccl at line 39 [23.15053]

[23.15348]

if (evolve_b) {
  SCHEDULE MaxwellToyAMReX_Evolve1 AT evol
  {
    LANG: C
    SYNC: potential_ax potential_ay potential_az
    SYNC: field_bx field_by field_bz
    SYNC: current_jx current_jy current_jz
  } "Evolve the Maxwell equations, step 1"
} else {
  SCHEDULE MaxwellToyAMReX_Evolve1 AT evol
  {
    LANG: C
    SYNC: potential_ax potential_ay potential_az
    SYNC: current_jx current_jy current_jz
  } "Evolve the Maxwell equations, step 1"

Replacement in MaxwellToyAMReX/schedule.ccl at line 55 [23.15053]

B:BD[23.15349] → [23.15349:15402]

B:BD[23.15402] → [7.21591:21714]

∅:D[7.21714] → [23.15434:15475]

B:BD[23.15434] → [23.15434:15475]

SCHEDULE MaxwellToyAMReX_Evolve1 AT evol
{
  LANG: C
  SYNC: potential_ax potential_ay potential_az
  SYNC: field_bx field_by field_bz
  SYNC: current_jx current_jy current_jz
} "Evolve the Maxwell equations, step 1"

[23.15349]

[23.15475]

  SCHEDULE MaxwellToyAMReX_Dependents1 AT evol AFTER MaxwellToyAMReX_Evolve1
  {
    LANG: C
    SYNC: field_bx field_by field_bz
  } "Calculate the magnetic field"
}

Replacement in MaxwellToyAMReX/schedule.ccl at line 62 [23.15053]

B:BD[23.15476] → [23.15476:15547]

SCHEDULE MaxwellToyAMReX_Evolve2 AT evol AFTER MaxwellToyAMReX_Evolve1

[23.15476]

[23.15547]

SCHEDULE MaxwellToyAMReX_Evolve2 AT evol AFTER (MaxwellToyAMReX_Evolve1, MaxwellToyAMReX_Dependents1)

Replacement in MaxwellToyAMReX/schedule.ccl at line 70 [23.15053]

B:BD[23.15632] → [23.15632:15752]

# SCHEDULE MaxwellToyAMReX_Constraints AT analysis
# {
#   LANG: C
# } "Calculate constraints of the Maxwell equations"

[23.15632]

[23.15752]

SCHEDULE MaxwellToyAMReX_Constraints AT analysis
{
  LANG: C
  SYNC: constraints_dive
  SYNC: constraints_divb
  # TRIGGERS: constraints_dive
  # TRIGGERS: constraints_divb
} "Calculate constraints of the Maxwell equations"

Insertion in MaxwellToyAMReX/schedule.ccl at line 92 [23.15053]

[23.16049]

  # TRIGGERS: errors_potential_phi
  # TRIGGERS: errors_potential_ax errors_potential_ay errors_potential_az
  # TRIGGERS: errors_field_ex errors_field_ey errors_field_ez
  # TRIGGERS: errors_field_bx errors_field_by errors_field_bz
  # TRIGGERS: errors_current_rho
  # TRIGGERS: errors_current_jx errors_current_jy errors_current_jz

Insertion in MaxwellToyAMReX/src/maxwelltoy.cxx at line 57 [23.16301]

[3.2495]

[23.17031]


// Linear
template <typename T> potential<T> linear(T t, T x, T y, T z) {
  DECLARE_CCTK_PARAMETERS;
  T phi = t - x;
  // T phi = x;
  // T phi = pow(x, 2);
  // T phi = y;
  // T phi = pow(y, 2);
  return {.phi = phi, .ax = -phi, .ay = 0, .az = 0};
}

Insertion in MaxwellToyAMReX/src/maxwelltoy.cxx at line 146 [23.16301]
[7.23800]
[23.18867]
```
  if (CCTK_EQUALS(initial_condition, "linear")) {
```

Replacement in MaxwellToyAMReX/src/maxwelltoy.cxx at line 149 [23.16301]

B:BD[23.18868] → [23.18868:18922]

  if (CCTK_EQUALS(initial_condition, "plane wave")) {

[23.18868]

[23.18922]

    Loop::loop_all<0, 0, 0>(cctkGH, [&](const Loop::PointDesc &p) {
      phi_(p.i, p.j, p.k) = linear(t + dt / 2, p.x, p.y, p.z).phi;
    });
    Loop::loop_all<1, 0, 0>(cctkGH, [&](const Loop::PointDesc &p) {
      ax_(p.i, p.j, p.k) = linear(t, p.x, p.y, p.z).ax;
    });
    Loop::loop_all<0, 1, 0>(cctkGH, [&](const Loop::PointDesc &p) {
      ay_(p.i, p.j, p.k) = linear(t, p.x, p.y, p.z).ay;
    });
    Loop::loop_all<0, 0, 1>(cctkGH, [&](const Loop::PointDesc &p) {
      az_(p.i, p.j, p.k) = linear(t, p.x, p.y, p.z).az;
    });
    Loop::loop_all<1, 0, 0>(cctkGH, [&](const Loop::PointDesc &p) {
      ex_(p.i, p.j, p.k) =
          -xderiv(linear<CCTK_REAL>, dx)(t + dt / 2, p.x, p.y, p.z).phi -
          tderiv(linear<CCTK_REAL>, dt)(t + dt / 2, p.x, p.y, p.z).ax;
    });
    Loop::loop_all<0, 1, 0>(cctkGH, [&](const Loop::PointDesc &p) {
      ey_(p.i, p.j, p.k) =
          -yderiv(linear<CCTK_REAL>, dy)(t + dt / 2, p.x, p.y, p.z).phi -
          tderiv(linear<CCTK_REAL>, dt)(t + dt / 2, p.x, p.y, p.z).ay;
    });
    Loop::loop_all<0, 0, 1>(cctkGH, [&](const Loop::PointDesc &p) {
      ez_(p.i, p.j, p.k) =
          -zderiv(linear<CCTK_REAL>, dz)(t + dt / 2, p.x, p.y, p.z).phi -
          tderiv(linear<CCTK_REAL>, dt)(t + dt / 2, p.x, p.y, p.z).az;
    });
    if (evolve_b) {
      Loop::loop_all<0, 1, 1>(cctkGH, [&](const Loop::PointDesc &p) {
        bx_(p.i, p.j, p.k) =
            yderiv(linear<CCTK_REAL>, dy)(t, p.x, p.y, p.z).az -
            zderiv(linear<CCTK_REAL>, dz)(t, p.x, p.y, p.z).ay;
      });
      Loop::loop_all<1, 0, 1>(cctkGH, [&](const Loop::PointDesc &p) {
        by_(p.i, p.j, p.k) =
            zderiv(linear<CCTK_REAL>, dz)(t, p.x, p.y, p.z).ax -
            xderiv(linear<CCTK_REAL>, dx)(t, p.x, p.y, p.z).az;
      });
      Loop::loop_all<1, 1, 0>(cctkGH, [&](const Loop::PointDesc &p) {
        bz_(p.i, p.j, p.k) =
            xderiv(linear<CCTK_REAL>, dx)(t, p.x, p.y, p.z).ay -
            yderiv(linear<CCTK_REAL>, dy)(t, p.x, p.y, p.z).ax;
      });
    }
    Loop::loop_all<0, 0, 0>(
        cctkGH, [&](const Loop::PointDesc &p) { rho_(p.i, p.j, p.k) = 0.0; });
    Loop::loop_all<1, 0, 0>(
        cctkGH, [&](const Loop::PointDesc &p) { jx_(p.i, p.j, p.k) = 0.0; });
    Loop::loop_all<0, 1, 0>(
        cctkGH, [&](const Loop::PointDesc &p) { jy_(p.i, p.j, p.k) = 0.0; });
    Loop::loop_all<0, 0, 1>(
        cctkGH, [&](const Loop::PointDesc &p) { jz_(p.i, p.j, p.k) = 0.0; });
  } else if (CCTK_EQUALS(initial_condition, "plane wave")) {

Replacement in MaxwellToyAMReX/src/maxwelltoy.cxx at line 239 [23.16301]

B:BD[23.20393] → [7.25133:25228]

B:BD[7.25228] → [3.3448:3581]

∅:D[3.3581] → [7.25365:25468]

B:BD[7.25365] → [7.25365:25468]

B:BD[7.25468] → [3.3582:3715]

∅:D[3.3715] → [7.25605:25708]

B:BD[7.25605] → [7.25605:25708]

B:BD[7.25708] → [3.3716:3857]

    Loop::loop_all<0, 1, 1>(cctkGH, [&](const Loop::PointDesc &p) {
      bx_(p.i, p.j, p.k) =
          yderiv(plane_wave<CCTK_REAL>, dy)(t, p.x, p.y, p.z).az -
          zderiv(plane_wave<CCTK_REAL>, dz)(t, p.x, p.y, p.z).ay;
    });
    Loop::loop_all<1, 0, 1>(cctkGH, [&](const Loop::PointDesc &p) {
      by_(p.i, p.j, p.k) =
          zderiv(plane_wave<CCTK_REAL>, dz)(t, p.x, p.y, p.z).ax -
          xderiv(plane_wave<CCTK_REAL>, dx)(t, p.x, p.y, p.z).az;
    });
    Loop::loop_all<1, 1, 0>(cctkGH, [&](const Loop::PointDesc &p) {
      bz_(p.i, p.j, p.k) =
          xderiv(plane_wave<CCTK_REAL>, dx)(t, p.x, p.y, p.z).ay -
          yderiv(plane_wave<CCTK_REAL>, dy)(t, p.x, p.y, p.z).ax;
    });

[23.20393]

[3.3857]

    if (evolve_b) {
      Loop::loop_all<0, 1, 1>(cctkGH, [&](const Loop::PointDesc &p) {
        bx_(p.i, p.j, p.k) =
            yderiv(plane_wave<CCTK_REAL>, dy)(t, p.x, p.y, p.z).az -
            zderiv(plane_wave<CCTK_REAL>, dz)(t, p.x, p.y, p.z).ay;
      });
      Loop::loop_all<1, 0, 1>(cctkGH, [&](const Loop::PointDesc &p) {
        by_(p.i, p.j, p.k) =
            zderiv(plane_wave<CCTK_REAL>, dz)(t, p.x, p.y, p.z).ax -
            xderiv(plane_wave<CCTK_REAL>, dx)(t, p.x, p.y, p.z).az;
      });
      Loop::loop_all<1, 1, 0>(cctkGH, [&](const Loop::PointDesc &p) {
        bz_(p.i, p.j, p.k) =
            xderiv(plane_wave<CCTK_REAL>, dx)(t, p.x, p.y, p.z).ay -
            yderiv(plane_wave<CCTK_REAL>, dy)(t, p.x, p.y, p.z).ax;
      });
    }

Replacement in MaxwellToyAMReX/src/maxwelltoy.cxx at line 299 [23.16301]

B:BD[3.5710] → [3.5710:6428]

∅:D[3.6428] → [23.20478:20486]

∅:D[7.25845] → [23.20478:20486]

B:BD[23.20478] → [23.20478:20486]

    Loop::loop_all<0, 1, 1>(cctkGH, [&](const Loop::PointDesc &p) {
      bx_(p.i, p.j, p.k) =
          yderiv(gaussian_wave<CCTK_REAL>, dy)(t, p.x, p.y, p.z).az -
          zderiv(gaussian_wave<CCTK_REAL>, dz)(t, p.x, p.y, p.z).ay;
    });
    Loop::loop_all<1, 0, 1>(cctkGH, [&](const Loop::PointDesc &p) {
      by_(p.i, p.j, p.k) =
          zderiv(gaussian_wave<CCTK_REAL>, dz)(t, p.x, p.y, p.z).ax -
          xderiv(gaussian_wave<CCTK_REAL>, dx)(t, p.x, p.y, p.z).az;
    });
    Loop::loop_all<1, 1, 0>(cctkGH, [&](const Loop::PointDesc &p) {
      bz_(p.i, p.j, p.k) =
          xderiv(gaussian_wave<CCTK_REAL>, dx)(t, p.x, p.y, p.z).ay -
          yderiv(gaussian_wave<CCTK_REAL>, dy)(t, p.x, p.y, p.z).ax;
    });

[3.5710]

[7.25846]

    if (evolve_b) {
      Loop::loop_all<0, 1, 1>(cctkGH, [&](const Loop::PointDesc &p) {
        bx_(p.i, p.j, p.k) =
            yderiv(gaussian_wave<CCTK_REAL>, dy)(t, p.x, p.y, p.z).az -
            zderiv(gaussian_wave<CCTK_REAL>, dz)(t, p.x, p.y, p.z).ay;
      });
      Loop::loop_all<1, 0, 1>(cctkGH, [&](const Loop::PointDesc &p) {
        by_(p.i, p.j, p.k) =
            zderiv(gaussian_wave<CCTK_REAL>, dz)(t, p.x, p.y, p.z).ax -
            xderiv(gaussian_wave<CCTK_REAL>, dx)(t, p.x, p.y, p.z).az;
      });
      Loop::loop_all<1, 1, 0>(cctkGH, [&](const Loop::PointDesc &p) {
        bz_(p.i, p.j, p.k) =
            xderiv(gaussian_wave<CCTK_REAL>, dx)(t, p.x, p.y, p.z).ay -
            yderiv(gaussian_wave<CCTK_REAL>, dy)(t, p.x, p.y, p.z).ax;
      });
    }

Insertion in MaxwellToyAMReX/src/maxwelltoy.cxx at line 330 [23.16301]

[23.20517]

}
extern "C" void MaxwellToyAMReX_Dependents1(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;
  const CCTK_REAL dx = CCTK_DELTA_SPACE(0);
  const CCTK_REAL dy = CCTK_DELTA_SPACE(1);
  const CCTK_REAL dz = CCTK_DELTA_SPACE(2);
  const Loop::GF3D<const CCTK_REAL, 1, 0, 0> ax_(cctkGH, ax);
  const Loop::GF3D<const CCTK_REAL, 0, 1, 0> ay_(cctkGH, ay);
  const Loop::GF3D<const CCTK_REAL, 0, 0, 1> az_(cctkGH, az);
  const Loop::GF3D<CCTK_REAL, 0, 1, 1> bx_(cctkGH, bx);
  const Loop::GF3D<CCTK_REAL, 1, 0, 1> by_(cctkGH, by);
  const Loop::GF3D<CCTK_REAL, 1, 1, 0> bz_(cctkGH, bz);
  Loop::loop_int<0, 1, 1>(cctkGH, [&](const Loop::PointDesc &p) {
    bx_(p.i, p.j, p.k) = (az_(p.i, p.j + 1, p.k) - az_(p.i, p.j, p.k)) / dy -
                         (ay_(p.i, p.j, p.k + 1) - ay_(p.i, p.j, p.k)) / dz;
  });
  Loop::loop_int<1, 0, 1>(cctkGH, [&](const Loop::PointDesc &p) {
    by_(p.i, p.j, p.k) = (ax_(p.i, p.j, p.k + 1) - ax_(p.i, p.j, p.k)) / dz -
                         (az_(p.i + 1, p.j, p.k) - az_(p.i, p.j, p.k)) / dx;
  });
  Loop::loop_int<1, 1, 0>(cctkGH, [&](const Loop::PointDesc &p) {
    bz_(p.i, p.j, p.k) = (ay_(p.i + 1, p.j, p.k) - ay_(p.i, p.j, p.k)) / dx -
                         (ax_(p.i, p.j + 1, p.k) - ax_(p.i, p.j, p.k)) / dy;
  });

Insertion in MaxwellToyAMReX/src/maxwelltoy.cxx at line 402 [23.16301]
[7.27720]
[7.27720]
```
#if 1
```

Replacement in MaxwellToyAMReX/src/maxwelltoy.cxx at line 415 [23.16301]

B:BD[7.28079] → [7.28079:28201]

B:BD[7.28201] → [3.6562:6704]

∅:D[3.6704] → [7.28347:28475]

B:BD[7.28347] → [7.28347:28475]

B:BD[7.28475] → [3.6705:6847]

∅:D[3.6847] → [7.28621:28749]

B:BD[7.28621] → [7.28621:28749]

B:BD[7.28749] → [3.6848:6990]

∅:D[3.6990] → [7.28895:28901]

B:BD[7.28895] → [7.28895:28901]

  Loop::loop_int<0, 1, 1>(cctkGH, [&](const Loop::PointDesc &p) {
    bx_(p.i, p.j, p.k) =
        bx_p_(p.i, p.j, p.k) +
        dt * ((ey_p_(p.i, p.j, p.k + 1) - ey_p_(p.i, p.j, p.k)) / dz -
              (ez_p_(p.i, p.j + 1, p.k) - ez_p_(p.i, p.j, p.k)) / dy);
  });
  Loop::loop_int<1, 0, 1>(cctkGH, [&](const Loop::PointDesc &p) {
    by_(p.i, p.j, p.k) =
        by_p_(p.i, p.j, p.k) +
        dt * ((ez_p_(p.i + 1, p.j, p.k) - ez_p_(p.i, p.j, p.k)) / dx -
              (ex_p_(p.i, p.j, p.k + 1) - ex_p_(p.i, p.j, p.k)) / dz);
  });
  Loop::loop_int<1, 1, 0>(cctkGH, [&](const Loop::PointDesc &p) {
    bz_(p.i, p.j, p.k) =
        bz_p_(p.i, p.j, p.k) +
        dt * ((ex_p_(p.i, p.j + 1, p.k) - ex_p_(p.i, p.j, p.k)) / dy -
              (ey_p_(p.i + 1, p.j, p.k) - ey_p_(p.i, p.j, p.k)) / dx);
  });

[7.28079]

[7.28901]

  if (evolve_b) {
    Loop::loop_int<0, 1, 1>(cctkGH, [&](const Loop::PointDesc &p) {
      bx_(p.i, p.j, p.k) =
          bx_p_(p.i, p.j, p.k) +
          dt * ((ey_p_(p.i, p.j, p.k + 1) - ey_p_(p.i, p.j, p.k)) / dz -
                (ez_p_(p.i, p.j + 1, p.k) - ez_p_(p.i, p.j, p.k)) / dy);
    });
    Loop::loop_int<1, 0, 1>(cctkGH, [&](const Loop::PointDesc &p) {
      by_(p.i, p.j, p.k) =
          by_p_(p.i, p.j, p.k) +
          dt * ((ez_p_(p.i + 1, p.j, p.k) - ez_p_(p.i, p.j, p.k)) / dx -
                (ex_p_(p.i, p.j, p.k + 1) - ex_p_(p.i, p.j, p.k)) / dz);
    });
    Loop::loop_int<1, 1, 0>(cctkGH, [&](const Loop::PointDesc &p) {
      bz_(p.i, p.j, p.k) =
          bz_p_(p.i, p.j, p.k) +
          dt * ((ex_p_(p.i, p.j + 1, p.k) - ex_p_(p.i, p.j, p.k)) / dy -
                (ey_p_(p.i + 1, p.j, p.k) - ey_p_(p.i, p.j, p.k)) / dx);
    });
  }

Insertion in MaxwellToyAMReX/src/maxwelltoy.cxx at line 453 [23.16301]

[7.29616]

[23.21716]

  });
#else
  Loop::loop_int<1, 0, 0>(cctkGH, [&](const Loop::PointDesc &p) {
    jx_(p.i, p.j, p.k) = jx_p_(p.i, p.j, p.k);
  });
  Loop::loop_int<0, 1, 0>(cctkGH, [&](const Loop::PointDesc &p) {
    jy_(p.i, p.j, p.k) = jy_p_(p.i, p.j, p.k);
  });
  Loop::loop_int<0, 0, 1>(cctkGH, [&](const Loop::PointDesc &p) {
    jz_(p.i, p.j, p.k) = jz_p_(p.i, p.j, p.k);

Insertion in MaxwellToyAMReX/src/maxwelltoy.cxx at line 466 [23.16301]

[23.21722]


  if (evolve_b) {
    Loop::loop_int<0, 1, 1>(cctkGH, [&](const Loop::PointDesc &p) {
      bx_(p.i, p.j, p.k) = bx_p_(p.i, p.j, p.k);
    });
    Loop::loop_int<1, 0, 1>(cctkGH, [&](const Loop::PointDesc &p) {
      by_(p.i, p.j, p.k) = by_p_(p.i, p.j, p.k);
    });
    Loop::loop_int<1, 1, 0>(cctkGH, [&](const Loop::PointDesc &p) {
      bz_(p.i, p.j, p.k) = bz_p_(p.i, p.j, p.k);
    });
  }
  Loop::loop_int<1, 0, 0>(cctkGH, [&](const Loop::PointDesc &p) {
    ax_(p.i, p.j, p.k) = ax_p_(p.i, p.j, p.k);
  });
  Loop::loop_int<0, 1, 0>(cctkGH, [&](const Loop::PointDesc &p) {
    ay_(p.i, p.j, p.k) = ay_p_(p.i, p.j, p.k);
  });
  Loop::loop_int<0, 0, 1>(cctkGH, [&](const Loop::PointDesc &p) {
    az_(p.i, p.j, p.k) = az_p_(p.i, p.j, p.k);
  });
#endif

Insertion in MaxwellToyAMReX/src/maxwelltoy.cxx at line 528 [23.16301]
[7.30802]
[23.22364]
```
#if 1
```

Replacement in MaxwellToyAMReX/src/maxwelltoy.cxx at line 566 [23.16301]

B:BD[3.8087] → [3.8087:8154]

              (ay_(p.i, p.j, p.k) - az_(p.i, p.j, p.k - 1)) / dz);

[3.8087]

[7.32371]

              (az_(p.i, p.j, p.k) - az_(p.i, p.j, p.k - 1)) / dz);
  });
#else
  Loop::loop_int<0, 0, 0>(cctkGH, [&](const Loop::PointDesc &p) {
    rho_(p.i, p.j, p.k) = rho_p_(p.i, p.j, p.k);
  });
  Loop::loop_int<1, 0, 0>(cctkGH, [&](const Loop::PointDesc &p) {
    ex_(p.i, p.j, p.k) = ex_p_(p.i, p.j, p.k);
  });
  Loop::loop_int<0, 1, 0>(cctkGH, [&](const Loop::PointDesc &p) {
    ey_(p.i, p.j, p.k) = ey_p_(p.i, p.j, p.k);
  });
  Loop::loop_int<0, 0, 1>(cctkGH, [&](const Loop::PointDesc &p) {
    ez_(p.i, p.j, p.k) = ez_p_(p.i, p.j, p.k);
  });
  Loop::loop_int<0, 0, 0>(cctkGH, [&](const Loop::PointDesc &p) {
    phi_(p.i, p.j, p.k) = phi_p_(p.i, p.j, p.k);

Insertion in MaxwellToyAMReX/src/maxwelltoy.cxx at line 588 [23.16301]
[7.32377]
[23.22984]
```
#endif
```
Insertion in MaxwellToyAMReX/src/maxwelltoy.cxx at line 595 [23.16301]
[23.23104]
[23.23104]
```
  const Loop::GF3D<const CCTK_REAL, 0, 0, 0> phi_(cctkGH, phi);
```

Insertion in MaxwellToyAMReX/src/maxwelltoy.cxx at line 601 [23.16301]

[3.8341]


  const Loop::GF3D<const CCTK_REAL, 1, 0, 0> ex_(cctkGH, ex);
  const Loop::GF3D<const CCTK_REAL, 0, 1, 0> ey_(cctkGH, ey);
  const Loop::GF3D<const CCTK_REAL, 0, 0, 1> ez_(cctkGH, ez);
  const Loop::GF3D<const CCTK_REAL, 0, 1, 1> bx_(cctkGH, bx);
  const Loop::GF3D<const CCTK_REAL, 1, 0, 1> by_(cctkGH, by);
  const Loop::GF3D<const CCTK_REAL, 1, 1, 0> bz_(cctkGH, bz);
  // Loop::loop_int<1, 1, 1>(cctkGH, [&](const Loop::PointDesc &p) {
  //   regrid_error[p.idx] = fabs(p.x) < 0.5 && fabs(p.y) < 0.5 && fabs(p.z) <
  //   0.5;
  // });

Insertion in MaxwellToyAMReX/src/maxwelltoy.cxx at line 615 [23.16301]

[3.8342]

  // Loop::loop_int<1, 1, 1>(cctkGH, [&](const Loop::PointDesc &p) {
  //   CCTK_REAL err = 0;
  //   for (int b = 0; b < 2; ++b)
  //     for (int a = 0; a < 2; ++a)
  //       err += fabs(ax_(p.i - 1, p.j + a, p.k + b) -
  //                   2 * ax_(p.i, p.j + a, p.k + b) +
  //                   ax_(p.i + 1, p.j + a, p.k + b)) +
  //              fabs(ay_(p.i + a, p.j - 1, p.k + b) -
  //                   2 * ay_(p.i + a, p.j, p.k + b) +
  //                   ay_(p.i + a, p.j + 1, p.k + b)) +
  //              fabs(az_(p.i + a, p.j + b, p.k - 1) -
  //                   2 * az_(p.i + a, p.j + b, p.k) +
  //                   az_(p.i + a, p.j + b, p.k + 1));
  //   regrid_error[p.idx] = err;
  // });

Replacement in MaxwellToyAMReX/src/maxwelltoy.cxx at line 633 [23.16301]

B:BD[3.8431] → [3.8431:8976]

    for (int b = 0; b < 2; ++b)
      for (int a = 0; a < 2; ++a)
        err += fabs(ax_(p.i - 1, p.j + a, p.k + b) -
                    2 * ax_(p.i, p.j + a, p.k + b) +
                    ax_(p.i + 1, p.j + a, p.k + b)) +
               fabs(ay_(p.i + a, p.j - 1, p.k + b) -
                    2 * ay_(p.i + a, p.j, p.k + b) +
                    ay_(p.i + a, p.j + 1, p.k + b)) +
               fabs(az_(p.i + a, p.j + b, p.k - 1) -
                    2 * az_(p.i + a, p.j + b, p.k) +
                    az_(p.i + a, p.j + b, p.k + 1));

[3.8431]

[3.8976]

    const auto accum = [&](CCTK_REAL x) { err += fabs(x); };
    const auto diffx = [&](const auto &var_, int i, int j, int k) {
      for (int b = 0; b < 2; ++b)
        for (int a = 0; a < 2; ++a)
          accum(var_(i + 1, j + a, k + b) - var_(i, j + a, k + b));
    };
    const auto diffy = [&](const auto &var_, int i, int j, int k) {
      for (int b = 0; b < 2; ++b)
        for (int a = 0; a < 2; ++a)
          accum(var_(i + a, j + 1, k + b) - var_(i + a, j, k + b));
    };
    const auto diffz = [&](const auto &var_, int i, int j, int k) {
      for (int b = 0; b < 2; ++b)
        for (int a = 0; a < 2; ++a)
          accum(var_(i + a, j + b, k + 1) - var_(i + a, j + b, k));
    };
    const auto diff000 = [&](const auto &var_) {
      diffx(var_, p.i, p.j, p.k);
      diffy(var_, p.i, p.j, p.k);
      diffz(var_, p.i, p.j, p.k);
    };
    const auto diff100 = [&](const auto &var_) {
      diffx(var_, p.i - 1, p.j, p.k);
      diffx(var_, p.i, p.j, p.k);
      diffy(var_, p.i, p.j, p.k);
      diffz(var_, p.i, p.j, p.k);
    };
    const auto diff010 = [&](const auto &var_) {
      diffx(var_, p.i, p.j, p.k);
      diffy(var_, p.i, p.j - 1, p.k);
      diffy(var_, p.i, p.j, p.k);
      diffz(var_, p.i, p.j, p.k);
    };
    const auto diff001 = [&](const auto &var_) {
      diffx(var_, p.i, p.j, p.k);
      diffy(var_, p.i, p.j, p.k);
      diffz(var_, p.i, p.j, p.k - 1);
      diffz(var_, p.i, p.j, p.k);
    };
    const auto diff011 = [&](const auto &var_) {
      diffx(var_, p.i, p.j, p.k);
      diffy(var_, p.i, p.j - 1, p.k);
      diffy(var_, p.i, p.j, p.k);
      diffz(var_, p.i, p.j, p.k - 1);
      diffz(var_, p.i, p.j, p.k);
    };
    const auto diff101 = [&](const auto &var_) {
      diffx(var_, p.i - 1, p.j, p.k);
      diffx(var_, p.i, p.j, p.k);
      diffy(var_, p.i, p.j, p.k);
      diffz(var_, p.i, p.j, p.k - 1);
      diffz(var_, p.i, p.j, p.k);
    };
    const auto diff110 = [&](const auto &var_) {
      diffx(var_, p.i - 1, p.j, p.k);
      diffx(var_, p.i, p.j, p.k);
      diffy(var_, p.i, p.j - 1, p.k);
      diffy(var_, p.i, p.j, p.k);
      diffz(var_, p.i, p.j, p.k);
    };
    diff000(phi_);
    diff100(ax_);
    diff010(ay_);
    diff001(az_);
    diff100(ex_);
    diff010(ey_);
    diff001(ez_);
    diff011(bx_);
    diff101(by_);
    diff110(bz_);

Insertion in MaxwellToyAMReX/src/maxwelltoy.cxx at line 710 [23.16301]
[23.23322]
[23.23322]
```
  if (analyse_every <= 0 || cctk_iteration % analyse_every != 0)
    return;
```

Insertion in MaxwellToyAMReX/src/maxwelltoy.cxx at line 759 [23.16301]

[7.34231]

[23.23663]


  if (CCTK_EQUALS(initial_condition, "linear")) {
    Loop::loop_all<0, 0, 0>(cctkGH, [&](const Loop::PointDesc &p) {
      phierr_(p.i, p.j, p.k) =
          phi_(p.i, p.j, p.k) - linear(t + dt / 2, p.x, p.y, p.z).phi;
    });
    Loop::loop_all<1, 0, 0>(cctkGH, [&](const Loop::PointDesc &p) {
      axerr_(p.i, p.j, p.k) = ax_(p.i, p.j, p.k) - linear(t, p.x, p.y, p.z).ax;
    });
    Loop::loop_all<0, 1, 0>(cctkGH, [&](const Loop::PointDesc &p) {
      ayerr_(p.i, p.j, p.k) = ay_(p.i, p.j, p.k) - linear(t, p.x, p.y, p.z).ay;
    });
    Loop::loop_all<0, 0, 1>(cctkGH, [&](const Loop::PointDesc &p) {
      azerr_(p.i, p.j, p.k) = az_(p.i, p.j, p.k) - linear(t, p.x, p.y, p.z).az;
    });
    Loop::loop_all<1, 0, 0>(cctkGH, [&](const Loop::PointDesc &p) {
      exerr_(p.i, p.j, p.k) =
          ex_(p.i, p.j, p.k) -
          (-xderiv(linear<CCTK_REAL>, dx)(t + dt / 2, p.x, p.y, p.z).phi -
           tderiv(linear<CCTK_REAL>, dt)(t + dt / 2, p.x, p.y, p.z).ax);
    });
    Loop::loop_all<0, 1, 0>(cctkGH, [&](const Loop::PointDesc &p) {
      eyerr_(p.i, p.j, p.k) =
          ey_(p.i, p.j, p.k) -
          (-yderiv(linear<CCTK_REAL>, dy)(t + dt / 2, p.x, p.y, p.z).phi -
           tderiv(linear<CCTK_REAL>, dt)(t + dt / 2, p.x, p.y, p.z).ay);
    });
    Loop::loop_all<0, 0, 1>(cctkGH, [&](const Loop::PointDesc &p) {
      ezerr_(p.i, p.j, p.k) =
          ez_(p.i, p.j, p.k) -
          (-zderiv(linear<CCTK_REAL>, dz)(t + dt / 2, p.x, p.y, p.z).phi -
           tderiv(linear<CCTK_REAL>, dt)(t + dt / 2, p.x, p.y, p.z).az);
    });

Replacement in MaxwellToyAMReX/src/maxwelltoy.cxx at line 796 [23.16301]

B:BD[23.23664] → [23.23664:23718]

  if (CCTK_EQUALS(initial_condition, "plane wave")) {

[23.23664]

[23.23718]

    Loop::loop_all<0, 1, 1>(cctkGH, [&](const Loop::PointDesc &p) {
      bxerr_(p.i, p.j, p.k) =
          bx_(p.i, p.j, p.k) -
          (yderiv(linear<CCTK_REAL>, dy)(t, p.x, p.y, p.z).az -
           zderiv(linear<CCTK_REAL>, dz)(t, p.x, p.y, p.z).ay);
    });
    Loop::loop_all<1, 0, 1>(cctkGH, [&](const Loop::PointDesc &p) {
      byerr_(p.i, p.j, p.k) =
          by_(p.i, p.j, p.k) -
          (zderiv(linear<CCTK_REAL>, dz)(t, p.x, p.y, p.z).ax -
           xderiv(linear<CCTK_REAL>, dx)(t, p.x, p.y, p.z).az);
    });
    Loop::loop_all<1, 1, 0>(cctkGH, [&](const Loop::PointDesc &p) {
      bzerr_(p.i, p.j, p.k) =
          bz_(p.i, p.j, p.k) -
          (xderiv(linear<CCTK_REAL>, dx)(t, p.x, p.y, p.z).ay -
           yderiv(linear<CCTK_REAL>, dy)(t, p.x, p.y, p.z).ax);
    });
    Loop::loop_all<0, 0, 0>(cctkGH, [&](const Loop::PointDesc &p) {
      rhoerr_(p.i, p.j, p.k) = 0.0;
    });
    Loop::loop_all<1, 0, 0>(
        cctkGH, [&](const Loop::PointDesc &p) { jxerr_(p.i, p.j, p.k) = 0.0; });
    Loop::loop_all<0, 1, 0>(
        cctkGH, [&](const Loop::PointDesc &p) { jyerr_(p.i, p.j, p.k) = 0.0; });
    Loop::loop_all<0, 0, 1>(
        cctkGH, [&](const Loop::PointDesc &p) { jzerr_(p.i, p.j, p.k) = 0.0; });
  } else if (CCTK_EQUALS(initial_condition, "plane wave")) {

Insertion in MaxwellToyAMReX/src/maxwelltoy.cxx at line 971 [23.16301]

[23.25587]


extern "C" void MaxwellToyAMReX_Constraints(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

Insertion in MaxwellToyAMReX/src/maxwelltoy.cxx at line 976 [23.16301]

[23.25588]

  if (analyse_every <= 0 || cctk_iteration % analyse_every != 0)
    return;
  const CCTK_REAL dx = CCTK_DELTA_SPACE(0);
  const CCTK_REAL dy = CCTK_DELTA_SPACE(1);
  const CCTK_REAL dz = CCTK_DELTA_SPACE(2);
  const Loop::GF3D<const CCTK_REAL, 1, 0, 0> ex_(cctkGH, ex);
  const Loop::GF3D<const CCTK_REAL, 0, 1, 0> ey_(cctkGH, ey);
  const Loop::GF3D<const CCTK_REAL, 0, 0, 1> ez_(cctkGH, ez);
  const Loop::GF3D<const CCTK_REAL, 0, 1, 1> bx_(cctkGH, bx);
  const Loop::GF3D<const CCTK_REAL, 1, 0, 1> by_(cctkGH, by);
  const Loop::GF3D<const CCTK_REAL, 1, 1, 0> bz_(cctkGH, bz);
  const Loop::GF3D<const CCTK_REAL, 0, 0, 0> rho_(cctkGH, rho);
  const Loop::GF3D<const CCTK_REAL, 1, 0, 0> jx_(cctkGH, jx);
  const Loop::GF3D<const CCTK_REAL, 0, 1, 0> jy_(cctkGH, jy);
  const Loop::GF3D<const CCTK_REAL, 0, 0, 1> jz_(cctkGH, jz);
  const Loop::GF3D<CCTK_REAL, 0, 0, 0> dive_(cctkGH, dive);
  const Loop::GF3D<CCTK_REAL, 1, 1, 1> divb_(cctkGH, divb);
  Loop::loop_int<0, 0, 0>(cctkGH, [&](const Loop::PointDesc &p) {
    dive_(p.i, p.j, p.k) = (ex_(p.i, p.j, p.k) - ex_(p.i - 1, p.j, p.k)) / dx +
                           (ey_(p.i, p.j, p.k) - ey_(p.i, p.j - 1, p.k)) / dy +
                           (ez_(p.i, p.j, p.k) - ez_(p.i, p.j, p.k - 1)) / dz -
                           rho_(p.i, p.j, p.k);
  });
  Loop::loop_int<1, 1, 1>(cctkGH, [&](const Loop::PointDesc &p) {
    divb_(p.i, p.j, p.k) = (bx_(p.i + 1, p.j, p.k) - bx_(p.i, p.j, p.k)) / dx +
                           (by_(p.i, p.j + 1, p.k) - by_(p.i, p.j, p.k)) / dy +
                           (bz_(p.i, p.j, p.k + 1) - bz_(p.i, p.j, p.k)) / dz;
  });
}

Replacement in azure-pipelines/amrex.th at line 100 [25.11]

B:BD[3.14151] → [3.14151:14239]

# CactusUtils/SystemTopology
# CactusUtils/TerminationTrigger
# CactusUtils/TimerReport

[3.14151]

[3.14239]

CactusUtils/SystemTopology
CactusUtils/TerminationTrigger
CactusUtils/TimerReport

Insertion in azure-pipelines/amrex.th at line 115 [25.11]
[25.2025]
[25.2025]
```
ExternalLibraries/hwloc
ExternalLibraries/zlib
```

More work on I/O

Dependencies

In channels

Change contents

Replacement in AMReX/src/driver.cxx at line 91 [5.15972]

Replacement in AMReX/src/driver.cxx at line 94 [5.15972]

Replacement in AMReX/src/driver.hxx at line 77 [5.15275]

Insertion in AMReX/src/io.cxx at line 5 [10.107]

Insertion in AMReX/src/io.cxx at line 32 [10.107]

Replacement in AMReX/src/io.cxx at line 46 [10.107]

Insertion in AMReX/src/io.cxx at line 76 [10.107]

Insertion in AMReX/src/io.cxx at line 228 [10.107]

Insertion in AMReX/src/io.cxx at line 265 [10.107]

Replacement in AMReX/src/io.cxx at line 276 [10.107]

Insertion in AMReX/src/io.cxx at line 282 [10.107]

Replacement in AMReX/src/io.cxx at line 304 [10.107]

Insertion in AMReX/src/io.cxx at line 313 [10.107]

Replacement in AMReX/src/io.cxx at line 321 [10.107]

Replacement in AMReX/src/make.code.defn at line 4 [5.15114]

Insertion in AMReX/src/reduction.cxx at line 7 [13.870]

Insertion in AMReX/src/reduction.cxx at line 9 [13.870]

Insertion in AMReX/src/reduction.cxx at line 15 [13.870]

Replacement in AMReX/src/reduction.cxx at line 51 [13.870]

Replacement in AMReX/src/reduction.cxx at line 60 [13.870]

Replacement in AMReX/src/reduction.cxx at line 73 [13.870]

Replacement in AMReX/src/reduction.cxx at line 79 [13.870]

Replacement in AMReX/src/reduction.cxx at line 90 [13.870]

Insertion in AMReX/src/reduction.cxx at line 103 [13.870]

Insertion in AMReX/src/reduction.hxx at line 12 [13.2777]

Insertion in AMReX/src/reduction.hxx at line 71 [13.2777]

Insertion in AMReX/src/schedule.cxx at line 3 [17.531]

Insertion in AMReX/src/schedule.cxx at line 453 [17.531]

Insertion in AMReX/src/schedule.cxx at line 487 [17.531]

Replacement in AMReX/src/schedule.cxx at line 492 [17.531]

Replacement in AMReX/src/schedule.cxx at line 544 [17.531]

Insertion in AMReX/src/schedule.cxx at line 644 [17.531]

Replacement in AMReX/src/schedule.cxx at line 661 [17.531]

Insertion in AMReX/src/schedule.cxx at line 698 [17.531]

Insertion in AMReX/src/schedule.cxx at line 772 [17.531]

Insertion in AMReX/src/schedule.cxx at line 841 [17.531]