Fixed various test issues

Author: Bryan Lawrence

pyproject.toml

@@ -74,7 +74,8 @@ addopts = """
 pythonpath = ["."]
 markers = [
     "slow: marks tests as slow (deselect with '-m \"not slow\"')",
-    "integration: marks tests requiring external services (SSH, remote servers)",
+    "integration: marks tests that exercise multiple internal components together (local/loopback integration)",
+    "external_service: marks tests that depend on externally hosted network services",
 ]
 
 [tool.coverage.run]

tests/integration/test_bigger_data.py

@@ -17,7 +17,7 @@
 @pytest.fixture
 def test_data_path():
     """Path to test data for CMOR fixes."""
-    return Path(__file__).resolve().parent / 'test_data'
+    return Path(__file__).resolve().parents[1] / 'test_data'
 
 
 def create_hyb_pres_file_without_ap(dataset, short_name):

tests/integration/test_cfdm_local.py

@@ -63,7 +63,7 @@ def _stop_loopback_server(
 def test_cfdm_read_local_file_handle_repeatedly() -> None:
     """cfdm.read should work with a local file-handle repeatedly."""
 
-    data_path = Path(__file__).parent / "data" / "test1.nc"
+    data_path = Path(__file__).parents[1] / "test_data" / "test1.nc"
     local_fs = fsspec.filesystem("local")
     with local_fs.open(str(data_path), "rb") as handle:
         handle.seek(0)
@@ -79,7 +79,7 @@ def test_cfdm_read_local_file_handle_repeatedly() -> None:
 def test_cfdm_read_open_pyfive_file() -> None:
     """cfdm.read should work when given an actual open pyfive.File object."""
 
-    data_path = Path(__file__).parent / "data" / "test1.nc"
+    data_path = Path(__file__).parents[1] / "test_data" / "test1.nc"
     with pyfive.File(str(data_path)) as file_obj:
         print(f"\n[DEBUG] pyfive.File test")
         print(f"  type: {type(file_obj)}")
@@ -94,7 +94,7 @@ def test_cfdm_read_remote_rfile_over_loopback() -> None:
 
     connection = _start_loopback_server(ServerStub)
     session = connection[0]
-    data_path = str(Path(__file__).parent / "data" / "test1.nc")
+    data_path = str(Path(__file__).parents[1] / "test_data" / "test1.nc")
     try:
         with session.open(data_path) as remote_file:
             fields = cfdm.read(remote_file)
@@ -108,7 +108,7 @@ def test_remote_dataset_read_after_rfile_close_matches_pyfive() -> None:
 
     connection = _start_loopback_server(ServerStub)
     session = connection[0]
-    data_path = str(Path(__file__).parent / "data" / "test1.nc")
+    data_path = str(Path(__file__).parents[1] / "test_data" / "test1.nc")
     try:
         remote_file = session.open(data_path)
         dataset = remote_file["tas"]

tests/integration/test_compression.py

@@ -101,7 +101,7 @@ def test_compression_and_filters_cmip6_data(storage_options, active_storage_url)
     Test use of datasets with compression and filters applied for a real
     CMIP6 dataset (CMIP6_IPSL-CM6A-LR_tas).
     """
-    test_file = str(Path(__file__).resolve().parent / 'test_data' / 'CMIP6_IPSL-CM6A-LR_tas.nc')
+    test_file = str(Path(__file__).resolve().parents[1] / 'test_data' / 'CMIP6_IPSL-CM6A-LR_tas.nc')
     with Dataset(test_file) as nc_data:
         nc_min = np.min(nc_data["tas"][0:2,4:6,7:9])
     print(f"Numpy min from compressed file {nc_min}")
@@ -131,7 +131,7 @@ def test_compression_and_filters_obs4mips_data(storage_options, active_storage_u
     obs4mips dataset (obs4MIPS_CERES-EBAF_L3B_Ed2-8_rlut.nc) at CMIP5 MIP standard
     but with CMIP6-standard file packaging.
     """
-    test_file = str(Path(__file__).resolve().parent / 'test_data' / 'obs4MIPS_CERES-EBAF_L3B_Ed2-8_rlut.nc')
+    test_file = str(Path(__file__).resolve().parents[1] / 'test_data' / 'obs4MIPS_CERES-EBAF_L3B_Ed2-8_rlut.nc')
     with Dataset(test_file) as nc_data:
         nc_min = np.min(nc_data["rlut"][0:2,4:6,7:9])
     print(f"Numpy min from compressed file {nc_min}")

tests/integration/test_compression_remote_reductionist.py

@@ -47,7 +47,7 @@ def test_compression_and_filters_cmip6_data(storage_options, active_storage_url)
     S3 anon=True Bucket and File: bnl CMIP6-test.nc
     Reductionist request data dictionary: {'source': 'https://uor-aces-o.s3-ext.jc.rl.ac.uk', 'bucket': 'bnl', 'object': 'CMIP6-test.nc', 'dtype': 'float32', 'byte_order': 'little', 'offset': 29385, 'size': 942518, 'order': 'C', 'shape': (15, 143, 144), 'selection': [[0, 2, 1], [4, 6, 1], [7, 9, 1]], 'compression': {'id': 'zlib'}}
     """
-    test_file = str(Path(__file__).resolve().parent / 'test_data' / 'CMIP6-test.nc')
+    test_file = str(Path(__file__).resolve().parents[1] / 'test_data' / 'CMIP6-test.nc')
     with Dataset(test_file) as nc_data:
         nc_min = np.min(nc_data["tas"][0:2,4:6,7:9])
     print(f"Numpy min from compressed file {nc_min}")
@@ -93,7 +93,7 @@ def test_compression_and_filters_cmip6_forced_s3_from_local(storage_options, act
 
     This is for a special anon=True bucket ONLY.
     """
-    test_file = str(Path(__file__).resolve().parent / 'test_data' / 'CMIP6-test.nc')
+    test_file = str(Path(__file__).resolve().parents[1] / 'test_data' / 'CMIP6-test.nc')
     with Dataset(test_file) as nc_data:
         nc_min = np.min(nc_data["tas"][0:2,4:6,7:9])
     print(f"Numpy min from compressed file {nc_min}")
@@ -128,7 +128,7 @@ def test_compression_and_filters_cmip6_forced_s3_from_local_2():
         'client_kwargs': {'endpoint_url': "https://uor-aces-o.s3-ext.jc.rl.ac.uk"}
     }
     active_storage_url = "https://192.171.169.248:8080"
-    test_file = str(Path(__file__).resolve().parent / 'test_data' / 'CMIP6-test.nc')
+    test_file = str(Path(__file__).resolve().parents[1] / 'test_data' / 'CMIP6-test.nc')
     with Dataset(test_file) as nc_data:
         nc_min = np.min(nc_data["tas"][0:2,4:6,7:9])
     print(f"Numpy min from compressed file {nc_min}")
@@ -170,7 +170,7 @@ def test_compression_and_filters_cmip6_forced_s3_using_local_Reductionist():
         'client_kwargs': {'endpoint_url': "https://uor-aces-o.s3-ext.jc.rl.ac.uk"}
     }
 
-    test_file = str(Path(__file__).resolve().parent / 'test_data' / 'CMIP6-test.nc')
+    test_file = str(Path(__file__).resolve().parents[1] / 'test_data' / 'CMIP6-test.nc')
     with Dataset(test_file) as nc_data:
         nc_min = np.min(nc_data["tas"][0:2,4:6,7:9])
     print(f"Numpy min from compressed file {nc_min}")

tests/integration/test_loopback.py

@@ -183,7 +183,7 @@ def test_server_detect_file_format_from_magic(tmp_path: Path) -> None:
 def test_loopback_proxy_round_trip() -> None:
 	connection = _start_loopback_server(ServerStub)
 	session = connection[0]
-	data_path = str(Path(__file__).parent / "data" / "test1.nc")
+	data_path = str(Path(__file__).parents[2] / "p5rem" / "tests" / "data" / "test1.nc")
 	try:
 		with session.open(data_path) as proxy:
 			assert "tas" in proxy.keys()
@@ -228,7 +228,7 @@ def test_loopback_proxy_round_trip() -> None:
 def test_loopback_get_chunk_recovers_when_server_loses_file_state() -> None:
 	connection = _start_loopback_server(DropFileStateBeforeGetChunkServer)
 	session = connection[0]
-	data_path = str(Path(__file__).parent / "data" / "contiguous_eg.nc")
+	data_path = str(Path(__file__).parents[2] / "p5rem" / "tests" / "data" / "contiguous_eg.nc")
 	try:
 		with session.open(data_path) as proxy:
 			q = proxy["q"]
@@ -241,7 +241,7 @@ def test_loopback_get_chunk_recovers_when_server_loses_file_state() -> None:
 def test_loopback_get_chunks_recovers_when_server_loses_file_state() -> None:
 	connection = _start_loopback_server(DropFileStateBeforeGetChunksServer)
 	session = connection[0]
-	data_path = str(Path(__file__).parent / "data" / "test1.nc")
+	data_path = str(Path(__file__).parents[2] / "p5rem" / "tests" / "data" / "test1.nc")
 	try:
 		with session.open(data_path) as proxy:
 			tas = proxy["tas"]
@@ -254,7 +254,7 @@ def test_loopback_get_chunks_recovers_when_server_loses_file_state() -> None:
 def test_loopback_reduce_recovers_when_server_loses_file_state() -> None:
 	connection = _start_loopback_server(DropFileStateBeforeReduceServer)
 	session = connection[0]
-	data_path = str(Path(__file__).parent / "data" / "test1.nc")
+	data_path = str(Path(__file__).parents[2] / "p5rem" / "tests" / "data" / "test1.nc")
 	try:
 		session.file_open(data_path)
 		response = session.reduce_selection(
@@ -272,7 +272,7 @@ def test_loopback_reduce_recovers_when_server_loses_file_state() -> None:
 def test_contiguous_file_data_and_coordinates_round_trip() -> None:
 	connection = _start_loopback_server(ServerStub)
 	session = connection[0]
-	data_path = str(Path(__file__).parent / "data" / "contiguous_eg.nc")
+	data_path = str(Path(__file__).parents[2] / "p5rem" / "tests" / "data" / "contiguous_eg.nc")
 	try:
 		with session.open(data_path) as proxy:
 			ref_file = pyfive.File(data_path)
@@ -310,7 +310,7 @@ def test_contiguous_file_data_and_coordinates_round_trip() -> None:
 def test_enum_file_round_trip() -> None:
 	connection = _start_loopback_server(ServerStub)
 	session = connection[0]
-	data_path = str(Path(__file__).parent / "data" / "enum_variable.nc")
+	data_path = str(Path(__file__).parents[2] / "p5rem" / "tests" / "data" / "enum_variable.nc")
 	try:
 		with session.open(data_path) as proxy:
 			ref_file = pyfive.File(data_path)
@@ -341,7 +341,7 @@ def test_enum_file_round_trip() -> None:
 def test_loopback_shared_roundtrip_assertions(filename: str) -> None:
 	connection = _start_loopback_server(ServerStub)
 	session = connection[0]
-	data_path = Path(__file__).parent / "data" / filename
+	data_path = Path(__file__).parents[2] / "p5rem" / "tests" / "data" / filename
 	try:
 		assert_roundtrip_file_matches(session, data_path, str(data_path))
 	finally:
@@ -371,7 +371,7 @@ def test_unexpected_response_type_raises() -> None:
 def test_real_server_var_open_includes_rich_metadata() -> None:
 	connection = _start_loopback_server(ServerStub)
 	session = connection[0]
-	data_path = Path(__file__).parent / "data" / "test1.nc"
+	data_path = Path(__file__).parents[2] / "p5rem" / "tests" / "data" / "test1.nc"
 	try:
 		file_info = session.file_open(str(data_path))
 		assert "tas" in file_info["keys"]
@@ -421,7 +421,7 @@ def __getitem__(self, key: str) -> FakeDataset:
 		def close(self) -> None:
 			self.closed = True
 
-	data_path = Path(__file__).parent / "data" / "test1.nc"
+	data_path = Path(__file__).parents[2] / "p5rem" / "tests" / "data" / "test1.nc"
 	first_file = FakeFile("old_var", FakeDataset("old_var", 1))
 	second_file = FakeFile("new_var", FakeDataset("new_var", 2))
 	opened_files = iter([first_file, second_file])
@@ -465,7 +465,7 @@ def test_get_dataset_rejects_stale_cache_when_file_not_open() -> None:
 def test_real_server_get_chunk_for_chunked_and_contiguous() -> None:
 	connection = _start_loopback_server(ServerStub)
 	session = connection[0]
-	data_path = Path(__file__).parent / "data" / "test1.nc"
+	data_path = Path(__file__).parents[2] / "p5rem" / "tests" / "data" / "test1.nc"
 	try:
 		session.file_open(str(data_path))
 
@@ -504,7 +504,7 @@ def test_real_server_get_chunk_for_chunked_and_contiguous() -> None:
 def test_real_server_reduce_supports_selection_and_chunk_modes() -> None:
 	connection = _start_loopback_server(ServerStub)
 	session = connection[0]
-	data_path = Path(__file__).parent / "data" / "contiguous_eg.nc"
+	data_path = Path(__file__).parents[2] / "p5rem" / "tests" / "data" / "contiguous_eg.nc"
 	try:
 		session.file_open(str(data_path))
 
@@ -543,7 +543,7 @@ def test_real_server_reduce_supports_selection_and_chunk_modes() -> None:
 def test_real_server_reduce_selection_parallel_full_dataset() -> None:
 	connection = _start_loopback_server(ServerStub)
 	session = connection[0]
-	data_path = Path(__file__).parent / "data" / "test1.nc"
+	data_path = Path(__file__).parents[2] / "p5rem" / "tests" / "data" / "test1.nc"
 	try:
 		session.file_open(str(data_path))
 		response = session.reduce_selection(
@@ -568,7 +568,7 @@ def test_real_server_reduce_selection_parallel_full_dataset() -> None:
 def test_real_server_reduce_selection_parallel_partial_selection() -> None:
 	connection = _start_loopback_server(ServerStub)
 	session = connection[0]
-	data_path = Path(__file__).parent / "data" / "test1.nc"
+	data_path = Path(__file__).parents[2] / "p5rem" / "tests" / "data" / "test1.nc"
 	try:
 		session.file_open(str(data_path))
 		response = session.reduce_selection(
@@ -597,7 +597,7 @@ def test_real_server_reduce_selection_parallel_partial_selection() -> None:
 def test_real_server_reduce_selection_chunk_planned_with_single_thread() -> None:
 	connection = _start_loopback_server(ServerStub)
 	session = connection[0]
-	data_path = Path(__file__).parent / "data" / "test1.nc"
+	data_path = Path(__file__).parents[2] / "p5rem" / "tests" / "data" / "test1.nc"
 	try:
 		session.file_open(str(data_path))
 		response = session.reduce_selection(
@@ -639,7 +639,7 @@ def test_real_server_reduce_selection_chunk_planned_with_single_thread() -> None
 def test_reduce_selection_chunked_all_operations_full_dataset(operation, ref_fn) -> None:
 	connection = _start_loopback_server(ServerStub)
 	session = connection[0]
-	data_path = Path(__file__).parent / "data" / "test1.nc"
+	data_path = Path(__file__).parents[2] / "p5rem" / "tests" / "data" / "test1.nc"
 	try:
 		session.file_open(str(data_path))
 		response = session.reduce_selection(str(data_path), "tas", operation, selection=None, thread_count=4)
@@ -670,7 +670,7 @@ def test_reduce_selection_chunked_all_operations_full_dataset(operation, ref_fn)
 def test_reduce_selection_chunked_all_operations_partial_selection(operation, ref_fn) -> None:
 	connection = _start_loopback_server(ServerStub)
 	session = connection[0]
-	data_path = Path(__file__).parent / "data" / "test1.nc"
+	data_path = Path(__file__).parents[2] / "p5rem" / "tests" / "data" / "test1.nc"
 	sel = [
 		{"type": "slice", "start": 2, "stop": 10, "step": 1},
 		{"type": "slice", "start": 10, "stop": 50, "step": 1},
@@ -707,7 +707,7 @@ def test_reduce_selection_chunked_all_operations_partial_selection(operation, re
 def test_reduce_selection_contiguous_all_operations_full_dataset(operation, ref_fn) -> None:
 	connection = _start_loopback_server(ServerStub)
 	session = connection[0]
-	data_path = Path(__file__).parent / "data" / "contiguous_eg.nc"
+	data_path = Path(__file__).parents[2] / "p5rem" / "tests" / "data" / "contiguous_eg.nc"
 	try:
 		session.file_open(str(data_path))
 		response = session.reduce_selection(str(data_path), "q", operation, selection=None)
@@ -730,7 +730,7 @@ def test_reduce_selection_contiguous_all_operations_full_dataset(operation, ref_
 def test_reduce_chunk_on_chunked_variable() -> None:
 	connection = _start_loopback_server(ServerStub)
 	session = connection[0]
-	data_path = Path(__file__).parent / "data" / "test1.nc"
+	data_path = Path(__file__).parents[2] / "p5rem" / "tests" / "data" / "test1.nc"
 	try:
 		session.file_open(str(data_path))
 		meta = session.var_open(str(data_path), "tas")
@@ -754,7 +754,7 @@ def test_reduce_chunk_on_chunked_variable() -> None:
 def test_reduce_chunk_on_contiguous_variable() -> None:
 	connection = _start_loopback_server(ServerStub)
 	session = connection[0]
-	data_path = Path(__file__).parent / "data" / "contiguous_eg.nc"
+	data_path = Path(__file__).parents[2] / "p5rem" / "tests" / "data" / "contiguous_eg.nc"
 	try:
 		session.file_open(str(data_path))
 		meta = session.var_open(str(data_path), "q")
@@ -791,7 +791,7 @@ def test_reduce_chunk_on_contiguous_variable() -> None:
 def test_reduce_selection_unknown_operation_chunked_raises() -> None:
 	connection = _start_loopback_server(ServerStub)
 	session = connection[0]
-	data_path = Path(__file__).parent / "data" / "test1.nc"
+	data_path = Path(__file__).parents[2] / "p5rem" / "tests" / "data" / "test1.nc"
 	try:
 		session.file_open(str(data_path))
 		with pytest.raises(ResponseError, match="unsupported reduction operation"):
@@ -807,7 +807,7 @@ def test_reduce_selection_unknown_operation_chunked_raises() -> None:
 def test_reduce_selection_thread_count_exceeds_chunk_count() -> None:
 	connection = _start_loopback_server(ServerStub)
 	session = connection[0]
-	data_path = Path(__file__).parent / "data" / "test1.nc"
+	data_path = Path(__file__).parents[2] / "p5rem" / "tests" / "data" / "test1.nc"
 	try:
 		session.file_open(str(data_path))
 		# A 1-timestep selection hits at most a handful of chunks; use 1000 workers.

tests/integration/test_real_https.py

@@ -4,6 +4,8 @@
 from requests.exceptions import MissingSchema
 from activestorage.active import Active, load_from_https
 
+pytestmark = [pytest.mark.external_service]
+
 
 def test_https():
     """

tests/integration/test_real_https_extras.py

@@ -3,6 +3,8 @@
 
 from activestorage.active import Active, load_from_https
 
+pytestmark = [pytest.mark.external_service]
+
 
 @pytest.mark.skip(
     reason="save time: test_https_implicit_storage is more general.")

tests/integration/test_real_s3.py

@@ -6,6 +6,8 @@
 from activestorage.active import Active, load_from_s3
 from activestorage.reductionist import ReductionistError
 
+pytestmark = [pytest.mark.external_service]
+
 
 S3_BUCKET = "bnl"
 

tests/integration/test_real_s3_with_axes.py

@@ -5,6 +5,8 @@
 
 from activestorage.active import Active
 
+pytestmark = [pytest.mark.external_service]
+
 
 S3_BUCKET = "bnl"