Merge branch '255_library_molecular_formula' into 'dev'

kheal · kheal · commit f8cd52732fdf · 2026-07-30T09:40:38.000-07:00
Separate MS1 Molecular Formula from library spectral formulas (#255)

See merge request mass-spectrometry/corems!237
diff --git a/corems/mass_spectra/factory/lc_class.py b/corems/mass_spectra/factory/lc_class.py
@@ -2400,8 +2400,11 @@ def feature_annotations_table(
             - sample_id: sample ID
             - Mass Feature ID: mass feature ID within the sample
             - Mass feature attributes (mz, scan_time, intensity, etc.)
-            - MS1 annotations (if molecular_formula_search was run)
-            - MS2 annotations (if ms2_spectral_search was run)
+            - MS1 annotations (if molecular_formula_search was run):
+              ``Molecular Formula``, ``Ion Formula``, ``Calculated m/z``, etc.
+            - MS2 annotations (if ms2_spectral_search was run):
+              ``Library Molecular Formula``, ``Library Ion Formula``,
+              ``Entropy Similarity``, ``name``, etc.
         
         Notes
         -----
@@ -2411,6 +2414,13 @@ def feature_annotations_table(
         Only mass features that are loaded in each sample's mass_features dict
         are included (typically the representative features if load_representatives
         was used in process_consensus_features).
+
+        ``Molecular Formula`` / ``Ion Formula`` are filled only from MS1 molecular
+        formula search. Spectral-library formulas appear under
+        ``Library Molecular Formula`` / ``Library Ion Formula`` so MS2-only hits
+        are not mistaken for MS1 formula assignments. When MS1 and MS2 match
+        (same ion formula), the library formula columns are cleared on that
+        row to avoid repeating the MS1 values.
         
         Raises
         ------
@@ -2504,8 +2514,10 @@ def feature_annotations_table(
             'Isotopologue Similarity',
             'Confidence Score',
             'Ion Formula',
-            'Ion Type',
             'Molecular Formula',
+            'Library Ion Formula',
+            'Library Ion Type',
+            'Library Molecular Formula',
             'inchikey',
             'name',
             'ref_ms_id',
diff --git a/corems/mass_spectra/output/export.py b/corems/mass_spectra/output/export.py
@@ -1489,12 +1489,14 @@ def clean_ms1_report(self, ms1_summary_full):
             self.get_isotope_type(f) for f in ms1_summary["ion_formula"].tolist()
         ]
 
-        # Reorder columns
+        # Keep MS1 neutral Molecular Formula so it is not confused with library formulas
+        # from MS2 spectral metadata.
         ms1_summary = ms1_summary[
             [
                 "mf_id",
                 "ion_formula",
                 "isotopologue_type",
+                "Molecular Formula",
                 "Calculated m/z",
                 "m/z Error (ppm)",
                 "m/z Error Score",
@@ -1580,16 +1582,20 @@ def clean_ms2_report(self, metabolite_summary):
             The cleaned metabolomics summary DataFrame.
         """
         metabolite_summary = metabolite_summary.reset_index()
-        metabolite_summary["ion_formula"] = [
+        # Library (MS2 metadata) formulas — keep separate from MS1 search fields
+        metabolite_summary["library_ion_formula"] = [
             self.get_ion_formula(f, a)
             for f, a in zip(metabolite_summary["formula"], metabolite_summary["ref_ion_type"])
         ]
+        metabolite_summary = metabolite_summary.rename(
+            columns={"formula": "library_formula"}
+        )
 
         col_order = [
             "mf_id",
-            "ion_formula",
+            "library_ion_formula",
             "ref_ion_type",
-            "formula",
+            "library_formula",
             "inchikey",
             "name",
             "inchi",
@@ -1631,6 +1637,14 @@ def combine_reports(self, mf_report, ms1_annot_report, ms2_annot_report):
             The MS1 annotation report DataFrame.
         ms2_annot_report : DataFrame
             The MS2 annotation report DataFrame.
+
+        Notes
+        -----
+        ``Molecular Formula`` / ``Ion Formula`` come only from MS1 molecular
+        formula search. MS2 spectral library formulas are exposed as
+        ``Library Molecular Formula`` / ``Library Ion Formula``.
+        When MS1 and MS2 match (same ion formula), the library formula columns
+        are cleared so they do not repeat the MS1 values on that row.
         """
         # If there is an ms1_annot_report, merge it with the mf_report
         if ms1_annot_report is not None and not ms1_annot_report.empty:
@@ -1642,24 +1656,31 @@ def combine_reports(self, mf_report, ms1_annot_report, ms2_annot_report):
                 on=["mf_id", "isotopologue_type"],
             )
         if ms2_annot_report is not None:
-            # If both reports contain 'ion_formula', prefer a merge that respects it.
-            # Otherwise fall back to merging on 'mf_id' only to remain robust when
-            # MS1 formula assignment wasn't performed or MS2 summary lacks the field.
-            if "ion_formula" in mf_report.columns and "ion_formula" in ms2_annot_report.columns:
+            # Prefer joining MS2 hits to matching MS1 ion formula when both exist.
+            # MS2 uses library_ion_formula so MS1 Ion Formula is not overwritten.
+            if (
+                "ion_formula" in mf_report.columns
+                and "library_ion_formula" in ms2_annot_report.columns
+            ):
                 # pull out the records without ion_formula and merge on mf_id only
                 mf_no_ion_formula = mf_report[mf_report["ion_formula"].isna()]
-                mf_no_ion_formula = mf_no_ion_formula.drop(columns=["ion_formula"]) if "ion_formula" in mf_no_ion_formula.columns else mf_no_ion_formula
+                mf_no_ion_formula = (
+                    mf_no_ion_formula.drop(columns=["ion_formula"])
+                    if "ion_formula" in mf_no_ion_formula.columns
+                    else mf_no_ion_formula
+                )
                 mf_no_ion_formula = pd.merge(
                     mf_no_ion_formula, ms2_annot_report, how="left", on=["mf_id"]
                 )
 
-                # pull out the records with ion_formula and merge on mf_id + ion_formula
+                # pull out the records with ion_formula and merge on mf_id + formula match
                 mf_with_ion_formula = mf_report[~mf_report["ion_formula"].isna()]
                 mf_with_ion_formula = pd.merge(
                     mf_with_ion_formula,
                     ms2_annot_report,
                     how="left",
-                    on=["mf_id", "ion_formula"],
+                    left_on=["mf_id", "ion_formula"],
+                    right_on=["mf_id", "library_ion_formula"],
                 )
 
                 # put back together
@@ -1688,8 +1709,9 @@ def combine_reports(self, mf_report, ms1_annot_report, ms2_annot_report):
             "mass_spectrum_deconvoluted_parent": "Is Largest Ion after Deconvolution",
             "associated_mass_features": "Associated Mass Features after Deconvolution",
             "ion_formula": "Ion Formula",
-            "formula": "Molecular Formula",
-            "ref_ion_type": "Ion Type",
+            "library_ion_formula": "Library Ion Formula",
+            "library_formula": "Library Molecular Formula",
+            "ref_ion_type": "Library Ion Type",
             "annot_level": "Lipid Annotation Level",
             "lipid_molecular_species_id": "Lipid Molecular Species",
             "lipid_summed_name": "Lipid Species",
@@ -1701,6 +1723,35 @@ def combine_reports(self, mf_report, ms1_annot_report, ms2_annot_report):
             "n_spectra_contributing": "Spectra with Annotation (n)",
         }
         mf_report = mf_report.rename(columns=rename_dict)
+
+        # When MS1 and MS2 co-annotate the same ion/molecular formula, library formula
+        # columns only repeat MS1 fields — clear them so the row is not redundant.
+        # Keep Library * columns populated for MS2-only (or mismatched) hits.
+        if (
+            "Ion Formula" in mf_report.columns
+            and "Library Ion Formula" in mf_report.columns
+        ):
+            ion_match = (
+                mf_report["Ion Formula"].notna()
+                & mf_report["Library Ion Formula"].notna()
+                & (mf_report["Ion Formula"] == mf_report["Library Ion Formula"])
+            )
+            mf_report.loc[ion_match, "Library Ion Formula"] = pd.NA
+            if "Library Molecular Formula" in mf_report.columns:
+                mf_report.loc[ion_match, "Library Molecular Formula"] = pd.NA
+        elif (
+            "Molecular Formula" in mf_report.columns
+            and "Library Molecular Formula" in mf_report.columns
+        ):
+            mol_match = (
+                mf_report["Molecular Formula"].notna()
+                & mf_report["Library Molecular Formula"].notna()
+                & (mf_report["Molecular Formula"] == mf_report["Library Molecular Formula"])
+            )
+            mf_report.loc[mol_match, "Library Molecular Formula"] = pd.NA
+            if "Library Ion Formula" in mf_report.columns:
+                mf_report.loc[mol_match, "Library Ion Formula"] = pd.NA
+
         mf_report["Sample Name"] = self.mass_spectra.sample_name
         mf_report["Polarity"] = self.mass_spectra.polarity
         mf_report = mf_report[
@@ -2065,18 +2116,20 @@ def clean_ms2_report(self, lipid_summary):
             The cleaned lipid summary DataFrame.
         """
         lipid_summary = lipid_summary.reset_index()
-        lipid_summary["ion_formula"] = [
+        # Library (MS2 metadata) formulas — keep separate from MS1 search fields
+        lipid_summary["library_ion_formula"] = [
             self.get_ion_formula(f, a)
             for f, a in zip(lipid_summary["formula"], lipid_summary["ref_ion_type"])
         ]
+        lipid_summary = lipid_summary.rename(columns={"formula": "library_formula"})
 
         # Reorder columns
         lipid_summary = lipid_summary[
             [
                 "mf_id",
-                "ion_formula",
+                "library_ion_formula",
                 "ref_ion_type",
-                "formula",
+                "library_formula",
                 "annot_level",
                 "lipid_molecular_species_id",
                 "lipid_summed_name",
diff --git a/tests/test_lcms_collection.py b/tests/test_lcms_collection.py
@@ -530,6 +530,16 @@ def test_lcms_collection_feature_annotations_table(lcms_collection, msp_file_loc
     if 'Entropy Similarity' in annotations_table.columns:
         matched_features = annotations_table[annotations_table['Entropy Similarity'].notna()]
         assert len(matched_features) > 0, "Should have at least some MS2 spectral matches after search"
+        # Library formula must not be written as MS1 Molecular Formula
+        assert 'Library Molecular Formula' in annotations_table.columns
+        lib_hits = matched_features[
+            matched_features['Library Molecular Formula'].notna()
+        ] if 'Library Molecular Formula' in matched_features.columns else matched_features.iloc[0:0]
+        if len(lib_hits) > 0 and 'Molecular Formula' in annotations_table.columns:
+            # Rows with library formula only (no MS1 search here) should leave MS1 Molecular Formula empty
+            assert lib_hits['Molecular Formula'].isna().all() or (
+                lib_hits['Molecular Formula'].fillna('').eq('').all()
+            )
     else:
         # If column doesn't exist, the test should fail
         raise AssertionError("Expected 'Entropy Similarity' column in annotations table after MS2 spectral search")
diff --git a/tests/test_lcms_metabolomics.py b/tests/test_lcms_metabolomics.py
@@ -3,6 +3,7 @@
 import warnings
 
 import numpy as np
+import pandas as pd
 import pytest
 
 from corems.mass_spectra.output.export import LCMSMetabolomicsExport
@@ -131,7 +132,15 @@ def test_lcms_metabolomics(tmp_path, postgres_database, lcms_obj, msp_file_locat
     exporter.to_hdf(overwrite=True)
     exporter.report_to_csv(molecular_metadata=metabolite_metadata_negative)
     report = exporter.to_report(molecular_metadata=metabolite_metadata_negative)
-    assert report['Ion Formula'][1] == 'C24 H47 O2'
+    # MS2 spectral library formula must not fill MS1 "Molecular Formula" / "Ion Formula";
+    # library values live under Library * columns.
+    assert report["Library Ion Formula"][1] == "C24 H47 O2"
+    assert "Library Molecular Formula" in report.columns
+    if "Ion Formula" in report.columns:
+        # No MS1 formula search in this test — MS1 ion formula should be empty
+        assert pd.isna(report["Ion Formula"][1]) or report["Ion Formula"][1] is None
+    if "Molecular Formula" in report.columns:
+        assert pd.isna(report["Molecular Formula"][1]) or report["Molecular Formula"][1] is None
     assert report['chebi'][1] == 28866
 
     # Test plotting mass feature with MS2 mirror plot
diff --git a/tests/test_wf_lipidomics.py b/tests/test_wf_lipidomics.py
@@ -3,6 +3,7 @@
 
 import shutil
 import numpy as np
+import pandas as pd
 import pytest
 
 from corems.encapsulation.constant import Labels
@@ -238,7 +239,10 @@ def test_lipidomics_workflow(tmp_path, postgres_database, lcms_obj, lipidomics_s
     exporter.to_hdf(overwrite=True)
     exporter.report_to_csv(molecular_metadata=lipid_metadata)
     report = exporter.to_report(molecular_metadata=lipid_metadata)
+    # This workflow runs MS1 formula search + MS2 library match. When ion formulas
+    # agree, Library Ion Formula is cleared and the value lives in Ion Formula.
     assert report['Ion Formula'][1] == 'C24 H47 O2'
+    assert pd.isna(report['Library Ion Formula'][1])
     assert report['Lipid Molecular Species'][0] == 'FA 20:5'
 
     # Import the hdf5 file, assert that its df is same as above and that we can plot a mass feature
