Skip to content

NotImplementedError: Unknown calibration type #3281

Description

@toufikshit

I just wanted to report this issue:

if I load files before 2008-11-30 → works fine.

Files after 2008-11-30 → fail immediately at calibration.

Likely cause: metadata in the file specifies an unsupported value of dataset_id["calibration"].

Could be related to MSG reprocessing phase or calibration table change.


RemoteTraceback                           Traceback (most recent call last)
RemoteTraceback: 
"""
Traceback (most recent call last):
  File "/opt/conda/lib/python3.11/multiprocessing/pool.py", line 125, in worker
    result = (True, func(*args, **kwds))
                    ^^^^^^^^^^^^^^^^^^^
 


 File "/tmp/ipykernel_930349/299977156.py", line 122, in process_product
    scn.load(CHANNELS)
  File "/opt/conda/lib/python3.11/site-packages/satpy/scene.py", line 1474, in load
    self._read_datasets_from_storage(**kwargs)
  File "/opt/conda/lib/python3.11/site-packages/satpy/scene.py", line 1497, in _read_datasets_from_storage
    return self._read_dataset_nodes_from_storage(nodes, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/lib/python3.11/site-packages/satpy/scene.py", line 1503, in _read_dataset_nodes_from_storage
    loaded_datasets = self._load_datasets_by_readers(reader_datasets, **kwargs)
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/lib/python3.11/site-packages/satpy/scene.py", line 1528, in _load_datasets_by_readers
    new_datasets = reader_instance.load(ds_ids, **kwargs)
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py", line 950, in load
    ds = self._load_dataset_with_area(dsid, coords, **kwargs)
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py", line 1147, in _load_dataset_with_area
    ds = super(GEOFlippableFileYAMLReader, self)._load_dataset_with_area(dsid, coords, **kwargs)
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py", line 838, in _load_dataset_with_area
    ds = self._load_dataset_data(file_handlers, dsid, **kwargs)
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py", line 738, in _load_dataset_data
    proj = self._load_dataset(dsid, ds_info, file_handlers, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py", line 714, in _load_dataset
    projectable = fh.get_dataset(dsid, ds_info)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/lib/python3.11/site-packages/satpy/readers/seviri_l1b_native.py", line 585, in get_dataset
    dataset = self.calibrate(xarr, dataset_id)
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/lib/python3.11/site-packages/satpy/readers/seviri_l1b_native.py", line 627, in calibrate
    res = calib.calibrate(data, dataset_id["calibration"])
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/lib/python3.11/site-packages/satpy/readers/core/seviri.py", line 669, in calibrate
    res = self._algo.ir_calibrate(
          ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/lib/python3.11/site-packages/satpy/readers/core/seviri.py", line 592, in ir_calibrate
    raise NotImplementedError("Unknown calibration type")
NotImplementedError: Unknown calibration type
"""

The above exception was the direct cause of the following exception:

NotImplementedError                       Traceback (most recent call last)
Cell In[1], line 175
    173 if products:
    174     with Pool(processes=cpu_count() - 1) as pool:
--> 175         for _ in tqdm(pool.imap_unordered(process_product, products), total=len(products)):
    176             pass
    178 current += datetime.timedelta(days=2)

File /opt/conda/lib/python3.11/site-packages/tqdm/std.py:1182, in tqdm.__iter__(self)
   1179 time = self._time
   1181 try:
-> 1182     for obj in iterable:
   1183         yield obj
   1184         # Update and possibly print the progressbar.
   1185         # Note: does not call self.update(1) for speed optimisation.

File /opt/conda/lib/python3.11/multiprocessing/pool.py:873, in IMapIterator.next(self, timeout)
    871 if success:
    872     return value
--> 873 raise value

File /opt/conda/lib/python3.11/multiprocessing/pool.py:125, in worker()
    123 job, i, func, args, kwds = task
    124 try:
--> 125     result = (True, func(*args, **kwds))
    126 except Exception as e:
    127     if wrap_exception and func is not _helper_reraises_exception:

Cell In[1], line 122, in process_product()
    120 try:
    121     scn = Scene(reader="seviri_l1b_native", filenames=[nat_path])
--> 122     scn.load(CHANNELS)
    123 except ValueError as e:
    124     print(f"❌ Corrupted or partial NAT file: {nat_file} — skipping. ({e})")

File /opt/conda/lib/python3.11/site-packages/satpy/scene.py:1474, in load()
   1470 self._update_dependency_tree(needed_datasets, query)
   1472 self._wishlist |= needed_datasets
-> 1474 self._read_datasets_from_storage(**kwargs)
   1475 if generate:
   1476     self.generate_possible_composites(unload)

File /opt/conda/lib/python3.11/site-packages/satpy/scene.py:1497, in _read_datasets_from_storage()
   1487 """Load datasets from the necessary reader.
   1488 
   1489 Args:
   (...)
   1494 
   1495 """
   1496 nodes = self._dependency_tree.leaves(limit_nodes_to=self.missing_datasets)
-> 1497 return self._read_dataset_nodes_from_storage(nodes, **kwargs)

File /opt/conda/lib/python3.11/site-packages/satpy/scene.py:1503, in _read_dataset_nodes_from_storage()
   1501 # Sort requested datasets by reader
   1502 reader_datasets = self._sort_dataset_nodes_by_reader(reader_nodes)
-> 1503 loaded_datasets = self._load_datasets_by_readers(reader_datasets, **kwargs)
   1504 self._datasets.update(loaded_datasets)
   1505 return loaded_datasets

File /opt/conda/lib/python3.11/site-packages/satpy/scene.py:1528, in _load_datasets_by_readers()
   1526 for reader_name, ds_ids in reader_datasets.items():
   1527     reader_instance = self._readers[reader_name]
-> 1528     new_datasets = reader_instance.load(ds_ids, **kwargs)
   1529     loaded_datasets.update(new_datasets)
   1530 return loaded_datasets

File /opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py:950, in load()
    947     continue
    948 coords = [all_datasets.get(cid, None)
    949           for cid in coordinates.get(dsid, [])]
--> 950 ds = self._load_dataset_with_area(dsid, coords, **kwargs)
    951 if ds is not None:
    952     all_datasets[dsid] = ds

File /opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py:1147, in _load_dataset_with_area()
   1146 def _load_dataset_with_area(self, dsid, coords, upper_right_corner="native", **kwargs):
-> 1147     ds = super(GEOFlippableFileYAMLReader, self)._load_dataset_with_area(dsid, coords, **kwargs)
   1149     if ds is not None:
   1150         ds = _set_orientation(ds, upper_right_corner)

File /opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py:838, in _load_dataset_with_area()
    835     return
    837 try:
--> 838     ds = self._load_dataset_data(file_handlers, dsid, **kwargs)
    839 except (KeyError, ValueError) as err:
    840     logger.exception("Could not load dataset '%s': %s", dsid, str(err))

File /opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py:738, in _load_dataset_data()
    736 def _load_dataset_data(self, file_handlers, dsid, **kwargs):
    737     ds_info = self.all_ids[dsid]
--> 738     proj = self._load_dataset(dsid, ds_info, file_handlers, **kwargs)
    739     # FIXME: areas could be concatenated here
    740     # Update the metadata
    741     proj.attrs["start_time"] = file_handlers[0].start_time

File /opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py:714, in _load_dataset()
    712 for fh in file_handlers:
    713     try:
--> 714         projectable = fh.get_dataset(dsid, ds_info)
    715         if projectable is not None:
    716             slice_list.append(projectable)

File /opt/conda/lib/python3.11/site-packages/satpy/readers/seviri_l1b_native.py:585, in get_dataset()
    582 if xarr is None:
    583     return None
--> 585 dataset = self.calibrate(xarr, dataset_id)
    586 self._add_scanline_acq_time(dataset, dataset_id)
    587 self._update_attrs(dataset, dataset_info)

File /opt/conda/lib/python3.11/site-packages/satpy/readers/seviri_l1b_native.py:627, in calibrate()
    625 tic = dt.datetime.now()
    626 calib = self._get_calibration_handler(dataset_id)
--> 627 res = calib.calibrate(data, dataset_id["calibration"])
    628 logger.debug("Calibration time " + str(dt.datetime.now() - tic))
    629 return res

File /opt/conda/lib/python3.11/site-packages/satpy/readers/core/seviri.py:669, in calibrate()
    667     res = self._algo.vis_calibrate(res, solar_irradiance)
    668 elif calibration == "brightness_temperature":
--> 669     res = self._algo.ir_calibrate(
    670         res, self._scan_params.channel_name, self._calib_params.radiance_type
    671     )
    672 return res

File /opt/conda/lib/python3.11/site-packages/satpy/readers/core/seviri.py:592, in ir_calibrate()
    590     return self._erads2bt(data, channel_name)
    591 else:
--> 592     raise NotImplementedError("Unknown calibration type")

NotImplementedError: Unknown calibration type 


Activity

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Projects

    No projects

      Milestone

      No milestone

      Relationships

      None yet

      Development

      No branches or pull requests

      Issue actions