Improve unit tests for TIMDEXDatasetMetadata.merge_append_deltas

Author: jonavellecuerdo

tests/conftest.py

@@ -1,5 +1,6 @@
 """tests/conftest.py"""
 
+import shutil
 from collections.abc import Iterator
 
 import boto3
@@ -265,6 +266,26 @@ def timdex_metadata_with_deltas(
     return TIMDEXDatasetMetadata(timdex_dataset_with_runs.location)
 
 
+@pytest.fixture
+def timdex_metadata_merged_deltas(
+    tmp_path, timdex_metadata_with_deltas, timdex_dataset_with_runs
+):
+    """TIMDEXDatasetMetadata after merging append deltas to static database file."""
+    # copy directory of a dataset with runs
+    dataset_location = str(tmp_path / "cloned_dataset_with_runs/")
+    shutil.copytree(timdex_metadata_with_deltas.location, dataset_location)
+
+    # clone dataset with runs using new dataset location
+    td = TIMDEXDataset(dataset_location, config=timdex_dataset_with_runs.config)
+
+    # clone metadata and merge append deltas
+    metadata = TIMDEXDatasetMetadata(td.location)
+    metadata.merge_append_deltas()
+    metadata.refresh()
+
+    return metadata
+
+
 # ================================================================================
 # Utility Fixtures
 # ================================================================================

tests/test_metadata.py

@@ -6,6 +6,18 @@
 
 from timdex_dataset_api import TIMDEXDatasetMetadata
 
+ORDERED_METADATA_COLUMN_NAMES = [
+    "timdex_record_id",
+    "source",
+    "run_date",
+    "run_type",
+    "action",
+    "run_id",
+    "run_record_offset",
+    "run_timestamp",
+    "filename",
+]
+
 
 def test_tdm_init_no_metadata_file_warning_success(caplog, timdex_dataset_with_runs):
     TIMDEXDatasetMetadata(timdex_dataset_with_runs.location)
@@ -264,26 +276,44 @@ def test_tdm_current_records_most_recent_version(timdex_metadata_with_deltas):
         assert current_version.iloc[0]["run_id"] == most_recent.iloc[0]["run_id"]
 
 
-def test_tdm_merge_append_deltas(timdex_metadata_with_deltas):
-    # get record count from static db
-    metadata_db_current_count = timdex_metadata_with_deltas.conn.query(
+def test_tdm_merge_append_deltas_static_counts_match_records_count_before_merge(
+    timdex_metadata_with_deltas, timdex_metadata_merged_deltas
+):
+    static_count_merged_deltas = timdex_metadata_merged_deltas.conn.query(
         """select count(*) as count from static_db.records;"""
     ).fetchone()[0]
-    total_count = timdex_metadata_with_deltas.records_count
+    assert static_count_merged_deltas == timdex_metadata_with_deltas.records_count
 
-    # merge append deltas into static db file
-    timdex_metadata_with_deltas.merge_append_deltas()
-    timdex_metadata_with_deltas.refresh()
 
-    # get updated record count from static db
-    metadata_db_updated_count = timdex_metadata_with_deltas.conn.query(
-        """select count(*) as count from static_db.records;"""
-    ).fetchone()[0]
+def test_tdm_merge_append_deltas_adds_records_to_static_db(
+    timdex_metadata_with_deltas, timdex_metadata_merged_deltas
+):
+    append_deltas = timdex_metadata_with_deltas.conn.query(
+        f"""
+            select
+            {','.join(ORDERED_METADATA_COLUMN_NAMES)}
+            from metadata.append_deltas
+        """
+    ).to_df()
+
+    merged_static_db = timdex_metadata_merged_deltas.conn.query(
+        f"""
+            select
+            {','.join(ORDERED_METADATA_COLUMN_NAMES)}
+            from static_db.records
+        """
+    ).to_df()
+
+    assert set(map(tuple, append_deltas.to_numpy())).issubset(
+        set(map(tuple, merged_static_db.to_numpy()))
+    )
+
 
-    # verify the addition of new records
-    assert metadata_db_current_count < metadata_db_updated_count
-    assert metadata_db_updated_count == total_count
+def test_tdm_merge_append_deltas_deletes_append_deltas(
+    timdex_metadata_with_deltas, timdex_metadata_merged_deltas
+):
+    assert timdex_metadata_with_deltas.append_deltas_count != 0
+    assert os.listdir(timdex_metadata_with_deltas.append_deltas_path)
 
-    # verify append deltas are deleted
-    assert timdex_metadata_with_deltas.append_deltas_count == 0
-    assert os.listdir(timdex_metadata_with_deltas.append_deltas_path) == []
+    assert timdex_metadata_merged_deltas.append_deltas_count == 0
+    assert not os.listdir(timdex_metadata_merged_deltas.append_deltas_path)