1147 hi l1c add calibration product coordinate to pset (IMAP-Science-Operations-Center#1172)

* Add calibration product dimension to pset data product

* Add test coverage for coordinate size checks

* fix test broken by addition of calibration_prod to pset

* Fix swapped CATDESC and FIELDNAM

Author: subagonsouth

imap_processing/cdf/config/imap_hi_variable_attrs.yaml

@@ -312,6 +312,20 @@ hi_pset_esa_energy_step:
   VAR_TYPE: support_data
   dtype: uint8
 
+hi_pset_calibration_prod:
+  CATDESC: Coincidence types are combined into various calibration products
+  FIELDNAM: Calibration product number
+  FILLVAL: 65535
+  FORMAT: I1
+  LABLAXIS: Calibration Prod
+  UNITS: " "
+  SCALE_TYP: linear
+  MONOTON: INCREASE
+  VALIDMIN: 0
+  VALIDMAX: 4
+  VAR_TYPE: support_data
+  dtype: uint8
+
 hi_pset_spin_angle_bin:
   CATDESC: 0.1 degree spin angle bins
   FIELDNAM: Spin angle bin index
@@ -377,9 +391,11 @@ hi_pset_counts:
   FIELDNAM: Binned DE counts
   DEPEND_0: epoch
   DEPEND_1: esa_energy_step
-  DEPEND_2: spin_angle_bin
+  DEPEND_2: calibration_prod
+  DEPEND_3: spin_angle_bin
   LABL_PTR_1: esa_energy_step_label
-  LABL_PTR_2: spin_bin_label
+  LABL_PTR_2: calibration_prod_label
+  LABL_PTR_3: spin_bin_label
   DISPLAY_TYPE: stack_plot
   FORMAT: I5
 
@@ -389,9 +405,11 @@ hi_pset_exposure_times:
   FIELDNAM: Bin exposure times
   DEPEND_0: epoch
   DEPEND_1: esa_energy_step
-  DEPEND_2: spin_angle_bin
+  DEPEND_2: calibration_prod
+  DEPEND_3: spin_angle_bin
   LABL_PTR_1: esa_energy_step_label
-  LABL_PTR_2: spin_bin_label
+  LABL_PTR_2: calibration_prod_label
+  LABL_PTR_3: spin_bin_label
   UNITS: sec
   DISPLAY_TYPE: stack_plot
   FORMAT: F6.2
@@ -402,9 +420,11 @@ hi_pset_background_rates:
   FIELDNAM: Background count rates
   DEPEND_0: epoch
   DEPEND_1: esa_energy_step
-  DEPEND_2: spin_angle_bin
+  DEPEND_2: calibration_prod
+  DEPEND_3: spin_angle_bin
   LABL_PTR_1: esa_energy_step_label
-  LABL_PTR_2: spin_bin_label
+  LABL_PTR_2: calibration_prod_label
+  LABL_PTR_3: spin_bin_label
   UNITS: counts / s
   DISPLAY_TYPE: stack_plot
   FORMAT: F4.2
@@ -415,9 +435,11 @@ hi_pset_background_rates_uncertainty:
   FIELDNAM: Background count rate uncertainties
   DEPEND_0: epoch
   DEPEND_1: esa_energy_step
-  DEPEND_2: spin_angle_bin
+  DEPEND_2: calibration_prod
+  DEPEND_3: spin_angle_bin
   LABL_PTR_1: esa_energy_step_label
-  LABL_PTR_2: spin_bin_label
+  LABL_PTR_2: calibration_prod_label
+  LABL_PTR_3: spin_bin_label
   UNITS: counts / s
   DISPLAY_TYPE: stack_plot
   FORMAT: F4.2
@@ -435,6 +457,12 @@ hi_pset_esa_energy_step_label:
   FORMAT: A4
   VAR_TYPE: metadata
 
+hi_pset_calibration_prod_label:
+  CATDESC: Label calibration product
+  FIELDNAM: Label calibration product
+  FORMAT: A4
+  VAR_TYPE: metadata
+
 hi_pset_label_vector_HAE:
   CATDESC: Label cartesian despun_z
   FIELDNAM: Label cartesian despun_z

imap_processing/hi/l1c/hi_l1c.py

@@ -115,6 +115,20 @@ def allocate_pset_dataset(n_esa_steps: int, sensor_str: str) -> xr.Dataset:
         dims=["esa_energy_step"],
         attrs=attrs,
     )
+    # TODO: define calibration product number to coincidence type mapping and
+    #     use the number of calibration products here. I believe it will be 5
+    #     0 for any, 1-4, for the number of detector hits.
+    n_calibration_prod = 5
+    attrs = attr_mgr.get_variable_attributes(
+        "hi_pset_calibration_prod", check_schema=False
+    ).copy()
+    dtype = attrs.pop("dtype")
+    coords["calibration_prod"] = xr.DataArray(
+        np.arange(n_calibration_prod, dtype=dtype),
+        name="calibration_prod",
+        dims=["calibration_prod"],
+        attrs=attrs,
+    )
     # spin angle bins are 0.1 degree bins for full 360 degree spin
     attrs = attr_mgr.get_variable_attributes(
         "hi_pset_spin_angle_bin", check_schema=False
@@ -157,6 +171,14 @@ def allocate_pset_dataset(n_esa_steps: int, sensor_str: str) -> xr.Dataset:
             "hi_pset_esa_energy_step_label", check_schema=False
         ),
     )
+    data_vars["calibration_prod_label"] = xr.DataArray(
+        coords["calibration_prod"].values.astype(str),
+        name="calibration_prod_label",
+        dims=["calibration_prod"],
+        attrs=attr_mgr.get_variable_attributes(
+            "hi_pset_calibration_prod_label", check_schema=False
+        ),
+    )
     data_vars["spin_bin_label"] = xr.DataArray(
         coords["spin_angle_bin"].values.astype(str),
         name="spin_bin_label",

imap_processing/tests/hi/test_hi_l1c.py

@@ -3,6 +3,7 @@
 import numpy as np
 import pytest
 
+from imap_processing.cdf.utils import write_cdf
 from imap_processing.hi.l1a.hi_l1a import hi_l1a
 from imap_processing.hi.l1b.hi_l1b import hi_l1b
 from imap_processing.hi.l1c import hi_l1c
@@ -30,19 +31,26 @@ def test_generate_pset_dataset(create_de_data):
 def test_allocate_pset_dataset():
     """Test coverage for allocate_pset_dataset function"""
     n_esa_steps = 10
+    n_calibration_prods = 5
     sensor_str = HIAPID.H90_SCI_DE.sensor
     dataset = hi_l1c.allocate_pset_dataset(n_esa_steps, sensor_str)
 
     assert dataset.epoch.size == 1
     assert dataset.spin_angle_bin.size == 3600
+    assert dataset.esa_energy_step.size == n_esa_steps
+    assert dataset.calibration_prod.size == n_calibration_prods
     np.testing.assert_array_equal(dataset.despun_z.data.shape, (1, 3))
     np.testing.assert_array_equal(dataset.hae_latitude.data.shape, (1, 3600))
     np.testing.assert_array_equal(dataset.hae_longitude.data.shape, (1, 3600))
-    n_esa_step = dataset.esa_energy_step.data.size
     for var in [
         "counts",
         "exposure_times",
         "background_rates",
         "background_rates_uncertainty",
     ]:
-        np.testing.assert_array_equal(dataset[var].data.shape, (1, n_esa_step, 3600))
+        np.testing.assert_array_equal(
+            dataset[var].data.shape, (1, n_esa_steps, n_calibration_prods, 3600)
+        )
+    # Verify resulting CDF is ISTP compliant by writing to disk
+    dataset.attrs["Data_version"] = 1
+    write_cdf(dataset)

imap_processing/tests/hi/test_utils.py

@@ -48,14 +48,15 @@ def test_parse_sensor_number(test_str, expected):
     [
         ("despun_z", (1, 3), (1, 3)),
         ("hae_latitude", None, (1, 360)),
-        ("counts", None, (1, 10, 360)),
+        ("counts", None, (1, 10, 5, 360)),
     ],
 )
 def test_full_dataarray(name, shape, expected_shape):
     """Test coverage for full_dataarray function"""
     coords = {
         "epoch": xr.DataArray(np.array([0])),
         "esa_energy_step": xr.DataArray(np.arange(10)),
+        "calibration_prod": xr.DataArray(np.arange(5)),
         "spin_angle_bin": xr.DataArray(np.arange(360)),
     }
     cdf_manager = ImapCdfAttributes()