eschnett/cactusamrex - Change WASO7G5FJXRXWNH2U2FLUNEKU6VE63OI3HUYP64BVD4LMD6KE7OQC

Begin to support multiple refinement levels

Created by Erik Schnetter on July 15, 2019

WASO7G5FJXRXWNH2U2FLUNEKU6VE63OI3HUYP64BVD4LMD6KE7OQC

Dependencies

In channels

master

main

Change contents

Replacement in AMReX/src/AMReX.hxx at line 30 [4.15275]

B:BD[4.15823] → [5.1334:1360]

∅:D[5.1360] → [4.15823:15855]

B:BD[4.15823] → [4.15823:15855]

  // AMReX grid structure
  BoxArray ba;
  Geometry geom;

[4.15695]

[6.0]


  struct Level {
    int level;
    // AMReX grid structure
    BoxArray grids;
    Geometry geom;
    DistributionMapping dmap;

Replacement in AMReX/src/AMReX.hxx at line 39 [4.15275]

B:BD[6.1] → [7.206:309]

  struct GroupData {
    int firstvarindex;
    int numvars;
    int numtimelevels;
    MultiFab mfab;

[6.1]

[7.309]

    struct GroupData {
      int firstvarindex;
      int numvars;
      int numtimelevels;
      MultiFab mfab;
    };
    vector<GroupData> groupdata;

Replacement in AMReX/src/AMReX.hxx at line 47 [4.15275]
B:BD[7.314] → [7.314:345]
```
  vector<GroupData> groupdata;
```
[7.314]
[4.15872]
```
  vector<Level> levels;
```

Insertion in AMReX/src/driver.cxx at line 101 [4.15972]

[8.1195]


  const int nlevels = 1;
  ghext->levels.resize(nlevels);
  const int lev = 0;
  GHExt::Level &level = ghext->levels.at(lev);
  level.level = lev;

Replacement in AMReX/src/driver.cxx at line 112 [4.15972]
B:BD[4.16729] → [4.16729:16757]
```
  ghext->ba.define(domain);
```
[4.16729]
[4.16757]
```
  level.grids.define(domain);
```
Replacement in AMReX/src/driver.cxx at line 117 [4.15972]
B:BD[4.16881] → [4.16881:16917]
```
  ghext->ba.maxSize(max_grid_size);
```
[4.16881]
[4.16917]
```
  level.grids.maxSize(max_grid_size);
```

Replacement in AMReX/src/driver.cxx at line 125 [4.15972]

B:BD[4.17145] → [4.17145:17248]

  ghext->geom.define(domain, &real_box, CoordSys::cartesian,
                     is_periodic.data());

[4.17145]

[4.17300]

  level.geom.define(domain, &real_box, CoordSys::cartesian, is_periodic.data());

Replacement in AMReX/src/driver.cxx at line 128 [4.15972]
B:BD[4.17324] → [4.17324:17361]
```
  DistributionMapping dm(ghext->ba);
```
[4.17324]
[4.17361]
```
  level.dmap = DistributionMapping(level.grids);
```
Replacement in AMReX/src/driver.cxx at line 131 [4.15972]
B:BD[7.495] → [7.495:533]
```
  ghext->groupdata.resize(numgroups);
```
[7.495]
[7.533]
```
  level.groupdata.resize(numgroups);
```

Replacement in AMReX/src/driver.cxx at line 141 [4.15972]

B:BD[7.828] → [7.828:887]

    GHExt::GroupData &groupdata = ghext->groupdata.at(gi);

[7.828]

[7.887]

    GHExt::Level::GroupData &groupdata = level.groupdata.at(gi);

Replacement in AMReX/src/driver.cxx at line 147 [4.15972]

B:BD[7.1064] → [9.865:977]

    groupdata.mfab = MultiFab(
        ghext->ba, dm, groupdata.numvars * groupdata.numtimelevels, ghost_size);

[7.1064]

[7.1200]

    groupdata.mfab =
        MultiFab(level.grids, level.dmap,
                 groupdata.numvars * groupdata.numtimelevels, ghost_size);

Replacement in AMReX/src/io.cxx at line 34 [2.107]

B:BD[2.652] → [2.652:751]

  for (int gi = 0; gi < numgroups; ++gi) {
    auto &restrict groupdata = ghext->groupdata.at(gi);

[2.652]

[2.751]

  for (int gi = 0; gi < numgroups; ++gi)
    for (const auto &restrict level : ghext->levels) {
      auto &restrict groupdata = level.groupdata.at(gi);

Replacement in AMReX/src/io.cxx at line 38 [2.107]

B:BD[2.752] → [2.752:1083]

    string groupname = unique_ptr<char>(CCTK_GroupName(gi)).get();
    groupname = regex_replace(groupname, regex("::"), "-");
    for (auto &c : groupname)
      c = tolower(c);
    ostringstream buf;
    buf << "wavetoy/" << groupname << "." << setw(6) << setfill('0')
        << cctk_iteration;
    string filename = buf.str();

[2.752]

[2.1083]

      string groupname = unique_ptr<char>(CCTK_GroupName(gi)).get();
      groupname = regex_replace(groupname, regex("::"), "-");
      for (auto &c : groupname)
        c = tolower(c);
      ostringstream buf;
      buf << "wavetoy/" << groupname;
      buf << ".rl" << setw(2) << setfill('0') << level.level;
      buf << ".it" << setw(6) << setfill('0') << cctk_iteration;
      string filename = buf.str();

Replacement in AMReX/src/io.cxx at line 48 [2.107]

B:BD[2.1084] → [2.1084:1475]

    Vector<string> varnames(groupdata.numvars * groupdata.numtimelevels);
    for (int tl = 0; tl < groupdata.numtimelevels; ++tl) {
      for (int n = 0; n < groupdata.numvars; ++n) {
        ostringstream buf;
        buf << CCTK_VarName(groupdata.firstvarindex + n);
        for (int i = 0; i < tl; ++i)
          buf << "_p";
        varnames.at(tl * groupdata.numvars + n) = buf.str();

[2.1084]

[2.1475]

      Vector<string> varnames(groupdata.numvars * groupdata.numtimelevels);
      for (int tl = 0; tl < groupdata.numtimelevels; ++tl) {
        for (int n = 0; n < groupdata.numvars; ++n) {
          ostringstream buf;
          buf << CCTK_VarName(groupdata.firstvarindex + n);
          for (int i = 0; i < tl; ++i)
            buf << "_p";
          varnames.at(tl * groupdata.numvars + n) = buf.str();
        }

Deletion in AMReX/src/io.cxx at line 58 [2.107]
B:BD[2.1483] → [2.1483:1489]
```
    }
```

Replacement in AMReX/src/io.cxx at line 59 [2.107]

B:BD[2.1490] → [2.1490:1777]

    // TODO: Output only current time level. This might require having one
    // mfab per time level
    // TODO: Output everything into a single file
    WriteSingleLevelPlotfile(filename, groupdata.mfab, varnames, ghext->geom,
                             cctk_time, cctk_iteration);

[2.1490]

[2.1777]

      // TODO: Output only current time level. This might require having one
      // mfab per time level
      // TODO: Output all levels into a single file
      // TODO: Output everything into a single file
      WriteSingleLevelPlotfile(filename, groupdata.mfab, varnames, level.geom,
                               cctk_time, cctk_iteration);

Replacement in AMReX/src/io.cxx at line 66 [2.107]

B:BD[2.1778] → [2.1778:1819]

    count_vars += groupdata.numvars;
  }

[2.1778]

[2.1819]

      count_vars += groupdata.numvars;
    }

Replacement in AMReX/src/schedule.cxx at line 131 [10.531]

B:BD[10.2407] → [10.2407:2453]

void enter_level_mode(cGH *restrict cctkGH) {

[10.2407]

[9.1955]

void enter_level_mode(cGH *restrict cctkGH,
                      const GHExt::Level &restrict level) {

Insertion in AMReX/src/schedule.cxx at line 137 [10.531]
[9.2014]
[9.2014]
```
#warning "TODO"
```

Replacement in AMReX/src/schedule.cxx at line 142 [10.531]

B:BD[10.2545] → [10.2545:2607]

  // The refinement factor over the top level (coarsest) grid

[10.2545]

[10.2607]

    // The refinement factor over the top level (coarsest) grid
#warning "TODO"

Replacement in AMReX/src/schedule.cxx at line 147 [10.531]

B:BD[10.2680] → [10.2680:2745]

  // Offset between this level's and the coarsest level's origin

[10.2680]

[10.2745]

    // Offset between this level's and the coarsest level's origin
#warning "TODO"

Insertion in AMReX/src/schedule.cxx at line 151 [10.531]
[10.2817]
[10.2817]
```
#warning "TODO"
```

Replacement in AMReX/src/schedule.cxx at line 155 [10.531]

B:BD[10.2896] → [10.2896:2942]

void leave_level_mode(cGH *restrict cctkGH) {

[10.2896]

[10.2942]

void leave_level_mode(cGH *restrict cctkGH,
                      const GHExt::Level &restrict level) {

Replacement in AMReX/src/schedule.cxx at line 168 [10.531]

B:BD[10.3051] → [10.3051:3116]

void enter_local_mode(cGH *restrict cctkGH, const MFIter &mfi) {

[10.3051]

[6.241]

void enter_local_mode(cGH *restrict cctkGH, GHExt::Level &restrict level,
                      const MFIter &mfi) {

Replacement in AMReX/src/schedule.cxx at line 196 [10.531]

B:BD[6.792] → [7.1327:1381]

  for (auto &restrict groupdata : ghext->groupdata) {

[6.792]

[7.1381]

  for (auto &restrict groupdata : level.groupdata) {

Replacement in AMReX/src/schedule.cxx at line 204 [10.531]

B:BD[10.3875] → [10.3875:3921]

void leave_local_mode(cGH *restrict cctkGH) {

[10.3875]

[10.3921]

void leave_local_mode(cGH *restrict cctkGH, const GHExt::Level &restrict level,
                      const MFIter &mfi) {

Replacement in AMReX/src/schedule.cxx at line 217 [10.531]

B:BD[6.1200] → [7.1686:1740]

  for (auto &restrict groupdata : ghext->groupdata) {

[6.1200]

[7.1740]

  for (auto &restrict groupdata : level.groupdata) {

Replacement in AMReX/src/schedule.cxx at line 264 [10.531]
B:BD[11.1678] → [11.1678:1706]
```
  enter_level_mode(cctkGH);
```
[11.1678]
[11.1706]
```
  // enter_level_mode(cctkGH);
```
Replacement in AMReX/src/schedule.cxx at line 272 [10.531]
B:BD[11.1988] → [11.1988:2020]
```
    enter_level_mode(threadGH);
```
[11.1988]
[3.941]
```
    // enter_level_mode(threadGH);
```

Replacement in AMReX/src/schedule.cxx at line 381 [10.531]

∅:D[9.2170] → [7.4994:5237]

∅:D[11.2105] → [7.4994:5237]

B:BD[7.4994] → [7.4994:5237]

B:BD[7.5237] → [9.2171:2224]

∅:D[9.2224] → [7.5298:5302]

B:BD[7.5298] → [7.5298:5302]

  for (auto &restrict groupdata : ghext->groupdata) {
    for (int tl = groupdata.numtimelevels - 1; tl > 0; --tl)
      MultiFab::Copy(groupdata.mfab, groupdata.mfab,
                     (tl - 1) * groupdata.numvars, tl * groupdata.numvars,
                     groupdata.numvars, ghost_size);
  }

[9.2170]

[7.5302]

  for (auto &restrict level : ghext->levels)
    for (auto &restrict groupdata : level.groupdata) {
      for (int tl = groupdata.numtimelevels - 1; tl > 0; --tl)
        MultiFab::Copy(groupdata.mfab, groupdata.mfab,
                       (tl - 1) * groupdata.numvars, tl * groupdata.numvars,
                       groupdata.numvars, ghost_size);
    }

Deletion in AMReX/src/schedule.cxx at line 439 [10.531]

B:BD[10.5189] → [7.6104:6274]

    assert(!ghext->groupdata.empty());
    MultiFab &mfab = ghext->groupdata.at(0).mfab;
    auto mfitinfo = MFItInfo().SetDynamic(true).EnableTiling({1024000, 16, 32});

Replacement in AMReX/src/schedule.cxx at line 443 [10.531]

B:BD[11.2229] → [11.2229:2391]

      for (MFIter mfi(mfab, mfitinfo); mfi.isValid(); ++mfi) {
        enter_local_mode(threadGH, mfi);
        CCTK_CallFunction(function, attribute, threadGH);

[11.2229]

[11.2391]

      for (auto &restrict level : ghext->levels) {
        MultiFab &mfab = level.groupdata.at(0).mfab;
        enter_level_mode(threadGH, level);
        auto mfitinfo =
            MFItInfo().SetDynamic(true).EnableTiling({1024000, 16, 32});
        for (MFIter mfi(mfab, mfitinfo); mfi.isValid(); ++mfi) {
          enter_local_mode(threadGH, level, mfi);
          CCTK_CallFunction(function, attribute, threadGH);
          leave_local_mode(threadGH, level, mfi);
        }
        leave_level_mode(threadGH, level);

Replacement in AMReX/src/schedule.cxx at line 458 [10.531]
B:BD[10.5627] → [10.5627:5649]
```
  case mode_t::level:
```
[10.5627]
[10.5649]
```
  case mode_t::meta:
```
Replacement in AMReX/src/schedule.cxx at line 460 [10.531]
B:BD[10.5672] → [10.5672:5695]
```
  case mode_t::meta: {
```
[10.5672]
[6.1950]
```
  case mode_t::level: {
```
Insertion in AMReX/src/schedule.cxx at line 467 [10.531]
[10.5884]
[10.5884]

Replacement in AMReX/src/schedule.cxx at line 484 [10.531]

B:BD[12.392] → [12.392:613]

  for (int n = 0; n < numgroups; ++n) {
    int gi = groups[n];
    auto &restrict groupdata = ghext->groupdata.at(gi);
    // we always sync all directions
    groupdata.mfab.FillBoundary(ghext->geom.periodicity());
  }

[12.392]

[12.613]

  for (auto &restrict level : ghext->levels)
    for (int n = 0; n < numgroups; ++n) {
      int gi = groups[n];
      auto &restrict groupdata = level.groupdata.at(gi);
      // we always sync all directions
      groupdata.mfab.FillBoundary(level.geom.periodicity());
    }

Begin to support multiple refinement levels

Dependencies

In channels

Change contents

Replacement in AMReX/src/AMReX.hxx at line 30 [4.15275]

Replacement in AMReX/src/AMReX.hxx at line 39 [4.15275]

Replacement in AMReX/src/AMReX.hxx at line 47 [4.15275]

Insertion in AMReX/src/driver.cxx at line 101 [4.15972]

Replacement in AMReX/src/driver.cxx at line 112 [4.15972]

Replacement in AMReX/src/driver.cxx at line 117 [4.15972]

Replacement in AMReX/src/driver.cxx at line 125 [4.15972]

Replacement in AMReX/src/driver.cxx at line 128 [4.15972]

Replacement in AMReX/src/driver.cxx at line 131 [4.15972]