Merge pull request #314 from NCAS-CMS/anon_bucket

Anon bucket

Author: valeriupredoi

activestorage/active.py

@@ -519,21 +519,22 @@ def _from_storage(self, ds, indexer, chunks, out_shape, out_dtype,
         # Create a shared session object.
         if self.interface_type == "s3" and self._version == 2:
             if self.storage_options is not None:
-                key, secret = None, None
                 if self.storage_options.get("anon", None) is True:
                     print("Reductionist session for Anon S3 bucket.")
                     session = reductionist.get_session(
                         None, None, S3_ACTIVE_STORAGE_CACERT)
-                if "key" in self.storage_options:
-                    key = self.storage_options["key"]
-                if "secret" in self.storage_options:
-                    secret = self.storage_options["secret"]
-                if key and secret:
-                    session = reductionist.get_session(
-                        key, secret, S3_ACTIVE_STORAGE_CACERT)
                 else:
-                    session = reductionist.get_session(
-                        S3_ACCESS_KEY, S3_SECRET_KEY, S3_ACTIVE_STORAGE_CACERT)
+                    key, secret = None, None
+                    if "key" in self.storage_options:
+                        key = self.storage_options["key"]
+                    if "secret" in self.storage_options:
+                        secret = self.storage_options["secret"]
+                    if key and secret:
+                        session = reductionist.get_session(
+                            key, secret, S3_ACTIVE_STORAGE_CACERT)
+                    else:
+                        session = reductionist.get_session(
+                            S3_ACCESS_KEY, S3_SECRET_KEY, S3_ACTIVE_STORAGE_CACERT)
             else:
                 session = reductionist.get_session(S3_ACCESS_KEY,
                                                    S3_SECRET_KEY,

activestorage/reductionist.py

@@ -24,13 +24,10 @@ def get_session(username: str, password: str,
     :returns: a client session object.
     """
     session = requests.Session()
-    # TODO Stack-HPC
-    # we need to allow Anon buckets. though this
-    # will break connection to data server
-    # if username is None and password is None:
-    #     return session
-    session.auth = (username, password)
     session.verify = cacert or False
+    if username is None and password is None:
+        return session
+    session.auth = (username, password)
     return session
 
 
@@ -261,8 +258,13 @@ def __init__(self, status_code, error):
 
 def decode_and_raise_error(response):
     """Decode an error response and raise ReductionistError."""
-    try:
-        error = json.dumps(response.json())
+    if response.status_code == http.client.INTERNAL_SERVER_ERROR:
+        try:
+            error = json.dumps(response.json())
+        except requests.exceptions.JSONDecodeError as exc:
+            error = http.client.responses.get(response.status_code, "-")
+            raise ReductionistError(response.status_code, error) from exc
         raise ReductionistError(response.status_code, error)
-    except requests.exceptions.JSONDecodeError as exc:
-        raise ReductionistError(response.status_code, "-") from exc
+
+    error = http.client.responses.get(response.status_code, "-")
+    raise ReductionistError(response.status_code, error)

tests/test_real_s3.py

@@ -4,7 +4,6 @@
 import pytest
 
 from activestorage.active import Active, load_from_s3
-from activestorage.reductionist import ReductionistError
 
 
 S3_BUCKET = "bnl"
@@ -32,9 +31,54 @@ def test_anon_s3():
                     active_storage_url=active_storage_url,
                     option_disable_chunk_cache=True)
     active._version = 2
-    with pytest.raises(ReductionistError):
+    result = active.min()[:]
+    assert result[0][0][0] == 197.69595
+
+
+def test_anon_s3_invalid_file():
+    """Test a very basic but real S3 ANON access."""
+    active_storage_url = "https://reductionist.jasmin.ac.uk/"  # Wacasoft
+    bigger_file = "CMIP6-test3.nc"  # not a file
+
+    test_file_uri = os.path.join(
+        "esmvaltool-zarr",
+        bigger_file
+    )
+
+    active = Active(test_file_uri, 'tas',
+                    storage_options={
+                        "anon": True,
+                         'client_kwargs': {
+                             'endpoint_url': "https://uor-aces-o.s3-ext.jc.rl.ac.uk"
+                              }
+                          },
+                    active_storage_url=active_storage_url)
+    active._version = 2
+    with pytest.raises(FileNotFoundError) as red_err:
+        result = active.min()[:]
+
+
+def test_anon_s3_bucket_not_anon():
+    """Test a very basic but real S3 ANON access."""
+    active_storage_url = "https://reductionist.jasmin.ac.uk/"  # Wacasoft
+    bigger_file = "CMIP6-test.nc"  # not a file
+
+    test_file_uri = os.path.join(
+        "repacking",
+        bigger_file
+    )
+
+    active = Active(test_file_uri, 'tas',
+                    storage_options={
+                        "anon": True,
+                         'client_kwargs': {
+                             'endpoint_url': "https://uor-aces-o.s3-ext.jc.rl.ac.uk"
+                              }
+                          },
+                    active_storage_url=active_storage_url)
+    active._version = 2
+    with pytest.raises(PermissionError) as red_err:
         result = active.min()[:]
-        assert result == 197.69595    
 
 
 def test_s3_small_file():

tests/unit/test_reductionist.py

@@ -299,4 +299,42 @@ def test_reduce_chunk_not_found(mock_request):
                                   operation)
 
     print("Not found exc from reductionist", str(exc.value))
-    assert str(exc.value) == 'Reductionist error: HTTP 404: -'
+    assert str(exc.value) == 'Reductionist error: HTTP 404: Not Found'
+
+
+@mock.patch.object(reductionist, 'request')
+def test_reductionist_internal_server_error_json(mock_request):
+    """Unit test for Reductionist 500 JSON response."""
+    response = requests.Response()
+    response.status_code = 500
+    response._content = b'{"detail": "backend exploded"}'
+    response.headers["Content-Type"] = "application/json"
+    mock_request.return_value = response
+
+    active_url = "https://s3.example.com"
+    access_key = "fake-access"
+    secret_key = "fake-secret"
+    cacert = None
+    s3_url = "https://active.example.com"
+    offset = 2
+    size = 128
+    compression = None
+    filters = None
+    missing = []
+    dtype = np.dtype("int32")
+    shape = (32, )
+    axis = (0, )
+    order = "C"
+    chunk_selection = [slice(0, 2, 1)]
+    operation = "min"
+
+    session = reductionist.get_session(access_key, secret_key, cacert)
+    with pytest.raises(reductionist.ReductionistError) as exc:
+        reductionist.reduce_chunk(session, active_url, s3_url,
+                                  offset, size, compression, filters, missing,
+                                  dtype, shape, order, chunk_selection, axis,
+                                  operation)
+
+    assert (
+        str(exc.value) == 'Reductionist error: HTTP 500: {"detail": "backend exploded"}'
+    )