Added NPC directory to plotfiles and checkpoint files for NEUTRINO_DARK_PARTICLES case

Author: jmsexton03

Source/Nyx.H

@@ -656,6 +656,9 @@ protected:
 #ifdef AGN
     std::string retrieveAGN();
 #endif
+#ifdef NEUTRINO_DARK_PARTICLES
+    std::string retrieveNPC();
+#endif
 
 #ifndef NO_HYDRO
     amrex::FluxRegister& get_flux_reg();

Source/Nyx.cpp

@@ -2484,6 +2484,22 @@ Nyx::CreateLevelDirectory (const std::string &dir)
     }
 #endif
 
+#ifdef NEUTRINO_DARK_MATTER
+    std::string npc(dir + "/" + Nyx::retrieveNPC());
+    if(ParallelDescriptor::IOProcessor()) {
+      if( ! amrex::UtilCreateDirectory(npc, 0755)) {
+        amrex::CreateDirectoryFailed(npc);
+      }
+    }
+
+    LevelDirectoryNames(dir, Nyx::retrieveNPC(), LevelDir, FullPath);
+    if(ParallelDescriptor::IOProcessor()) {
+      if( ! amrex::UtilCreateDirectory(FullPath, 0755)) {
+        amrex::CreateDirectoryFailed(FullPath);
+      }
+    }
+#endif
+
     if(parent->UsingPrecreateDirectories()) {
       if(Nyx::theDMPC()) {
         Nyx::theDMPC()->SetLevelDirectoriesCreated(true);

Source/NyxParticles.cpp

@@ -29,6 +29,7 @@ namespace
   // const std::string chk_particle_file("DM");
     const std::string dm_chk_particle_file("DM");
     const std::string agn_chk_particle_file("AGN");
+    const std::string npc_chk_particle_file("NPC");
 
     //
     // We want to call this routine when on exit to clean up particles.
@@ -941,6 +942,45 @@ Nyx::particle_post_restart (const std::string& restart_file, bool is_checkpoint)
         }
     }
 #endif
+
+#ifdef NEUTRINO_DARK_PARTICLES
+    {
+        BL_ASSERT(NPC == 0);
+        NPC = new DarkMatterParticleContainer(parent);
+        ActiveParticles.push_back(NPC);
+ 
+        if (parent->subCycle())
+        {
+          VirtNPC = new DarkMatterParticleContainer(parent);
+          VirtualParticles.push_back(VirtNPC);
+ 
+          GhostNPC = new DarkMatterParticleContainer(parent);
+          GhostParticles.push_back(GhostNPC);
+        }
+
+        //
+        // Make sure to call RemoveParticlesOnExit() on exit.
+        //
+        amrex::ExecOnFinalize(RemoveParticlesOnExit);
+        //
+        // 2 gives more stuff than 1.
+        //
+        NPC->SetVerbose(particle_verbose);
+        NPC->Restart(restart_file, npc_chk_particle_file, is_checkpoint);
+        //
+        // We want the ability to write the particles out to an ascii file.
+        //
+        ParmParse pp("particles");
+
+        std::string npc_particle_output_file;
+        pp.query("npc_particle_output_file", npc_particle_output_file);
+
+        if (!npc_particle_output_file.empty())
+        {
+            NPC->WriteAsciiFile(npc_particle_output_file);
+        }
+    }
+#endif
 }
 
 #ifdef GRAVITY

Source/Nyx_output.cpp

@@ -21,6 +21,10 @@ namespace
 
     const std::string agn_chk_particle_file("AGN");
     const std::string agn_plt_particle_file("AGN");
+
+    const std::string npc_chk_particle_file("NPC");
+    const std::string npc_plt_particle_file("NPC");
+
 }
 
 std::string
@@ -45,6 +49,13 @@ Nyx::retrieveAGN () {
 }
 #endif
 
+#ifdef NEUTRINO_DARK_PARTICLES
+std::string
+Nyx::retrieveNPC () {
+    return npc_chk_particle_file;
+}
+#endif
+
 void
 Nyx::setPlotVariables ()
 {
@@ -358,6 +369,11 @@ Nyx::writePlotFile (const std::string& dir,
       Nyx::theAPC()->SetLevelDirectoriesCreated(false);
     }
 #endif
+#ifdef NEUTRINO_DARK_PARTICLES
+    if(Nyx::theNPC()) {
+      Nyx::theNPC()->SetLevelDirectoriesCreated(false);
+    }
+#endif
 
 }
 
@@ -373,6 +389,12 @@ Nyx::writePlotFilePre (const std::string& dir, ostream& os)
   }
 #endif
 
+#ifdef NEUTRINO_DARK_PARTICLES
+  if(Nyx::theNPC()) {
+    Nyx::theNPC()->WritePlotFilePre();
+  }
+#endif
+
 }
 
 
@@ -387,6 +409,12 @@ Nyx::writePlotFilePost (const std::string& dir, ostream& os)
     Nyx::theAPC()->WritePlotFilePost();
   }
 #endif
+#ifdef NEUTRINO_DARK_PARTICLES
+  if(Nyx::theNPC()) {
+    Nyx::theNPC()->WritePlotFilePost();
+  }
+#endif
+
 }
 
 void
@@ -574,6 +602,13 @@ Nyx::particle_plot_file (const std::string& dir)
           }
 #endif
 
+#ifdef NEUTRINO_DARK_PARTICLES
+        if (Nyx::theNPC())
+          {
+            Nyx::theNPC()->WriteNyxPlotFile(dir, npc_plt_particle_file);
+          }
+#endif
+
 #ifdef NO_HYDRO
         Real cur_time = state[PhiGrav_Type].curTime();
 #else
@@ -628,6 +663,22 @@ Nyx::particle_plot_file (const std::string& dir)
             File.close();
         }
 #endif
+
+#ifdef NEUTRINO_DARK_PARTICLES
+        // Write particle_plotfile_format into its own file in the particle directory
+        if (Nyx::theNPC() && ParallelDescriptor::IOProcessor())
+        {
+            std::string FileName = dir + "/" + npc_plt_particle_file + "/precision";
+            std::ofstream File;
+            File.open(FileName.c_str(), std::ios::out|std::ios::trunc);
+            if ( ! File.good()) {
+                amrex::FileOpenFailed(FileName);
+	    }
+            File.precision(15);
+            File << particle_plotfile_format << '\n';
+            File.close();
+        }
+#endif
     }
 }
 
@@ -647,6 +698,12 @@ Nyx::particle_check_point (const std::string& dir)
           Nyx::theAPC()->NyxCheckpoint(dir, agn_chk_particle_file);
         }
 #endif
+#ifdef NEUTRINO_DARK_PARTICLES
+      if (Nyx::theNPC())
+        {
+          Nyx::theNPC()->NyxCheckpoint(dir, npc_chk_particle_file);
+        }
+#endif
 
 #ifdef NO_HYDRO
         Real cur_time = state[PhiGrav_Type].curTime();
@@ -836,6 +893,11 @@ Nyx::checkPoint (const std::string& dir,
       Nyx::theAPC()->SetLevelDirectoriesCreated(false);
     }
 #endif
+#ifdef NEUTRINO_DARK_PARTICLES
+    if(Nyx::theNPC()) {
+      Nyx::theNPC()->SetLevelDirectoriesCreated(false);
+    }
+#endif
 
 }
 
@@ -851,6 +913,11 @@ Nyx::checkPointPre (const std::string& dir,
     Nyx::theAPC()->CheckpointPre();
   }
 #endif
+#ifdef NEUTRINO_DARK_PARTICLES
+  if(Nyx::theNPC()) {
+    Nyx::theNPC()->CheckpointPre();
+  }
+#endif
 
 }
 
@@ -867,6 +934,11 @@ Nyx::checkPointPost (const std::string& dir,
     Nyx::theAPC()->CheckpointPost();
   }
 #endif
+#ifdef NEUTRINO_DARK_PARTICLES
+  if(Nyx::theNPC()) {
+    Nyx::theNPC()->CheckpointPost();
+  }
+#endif
 }