Merge pull request #1230 from clowder-framework/feature/aws_iam

Use AWS IAM when connecting to S3

Author: lmarini

backend/app/config.py

@@ -36,6 +36,9 @@ class Settings(BaseSettings):
     MINIO_EXPIRES: int = 3600  # seconds
     MINIO_SECURE: str = "False"  # http vs https
 
+    # If AWS_IAM is set to True, the MINIO_ACCESS_KEY and MINIO_SECRET_KEY will be ignored and AWS IAM will be used
+    AWS_IAM: bool = False
+
     # Files in the listed directories can be added to Clowder without copying them elsewhere
     LOCAL_WHITELIST: List[str] = []
 

backend/app/dependencies.py

@@ -1,5 +1,7 @@
-from typing import Generator
+import logging
+from typing import AsyncGenerator
 
+import boto3
 import pika
 from app.config import settings
 from app.search.connect import connect_elasticsearch
@@ -8,14 +10,32 @@
 from minio.versioningconfig import VersioningConfig
 from pika.adapters.blocking_connection import BlockingChannel
 
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
 
-async def get_fs() -> Generator:
-    file_system = Minio(
-        settings.MINIO_SERVER_URL,
-        access_key=settings.MINIO_ACCESS_KEY,
-        secret_key=settings.MINIO_SECRET_KEY,
-        secure=False,
-    )
+
+async def get_fs() -> AsyncGenerator[Minio, None]:
+    # Either use AWS Identity and Access Management (IAM) to connect to S3 or connect to Minio server
+    if settings.AWS_IAM:
+        logger.debug("AWS IAM enabled for s3 authentication")
+        session = boto3.Session()
+        credentials = session.get_credentials()
+        credentials = credentials.get_frozen_credentials()
+        file_system = Minio(
+            settings.MINIO_EXTERNAL_SERVER_URL,
+            access_key=credentials.access_key,
+            secret_key=credentials.secret_key,
+            session_token=credentials.token,
+            secure=settings.MINIO_SECURE.lower() == "true",
+        )
+    else:
+        logger.debug("Local MinIO authentication")
+        file_system = Minio(
+            settings.MINIO_EXTERNAL_SERVER_URL,
+            access_key=settings.MINIO_ACCESS_KEY,
+            secret_key=settings.MINIO_SECRET_KEY,
+            secure=settings.MINIO_SECURE.lower() == "true",
+        )
     clowder_bucket = settings.MINIO_BUCKET_NAME
     if not file_system.bucket_exists(clowder_bucket):
         file_system.make_bucket(clowder_bucket)
@@ -24,14 +44,30 @@ async def get_fs() -> Generator:
 
 
 # This will be needed for generating presigned URL for sharing
-async def get_external_fs() -> Generator:
-    file_system = Minio(
-        settings.MINIO_EXTERNAL_SERVER_URL,
-        access_key=settings.MINIO_ACCESS_KEY,
-        secret_key=settings.MINIO_SECRET_KEY,
-        secure=settings.MINIO_SECURE.lower() == "true",
-    )
+async def get_external_fs() -> AsyncGenerator[Minio, None]:
+    # Either use AWS Identity and Access Management (IAM) to connect to S3 or connect to Minio server
+    if settings.AWS_IAM:
+        logger.debug("AWS IAM enabled for s3 authentication")
+        session = boto3.Session()
+        credentials = session.get_credentials()
+        credentials = credentials.get_frozen_credentials()
+        file_system = Minio(
+            settings.MINIO_EXTERNAL_SERVER_URL,
+            access_key=credentials.access_key,
+            secret_key=credentials.secret_key,
+            session_token=credentials.token,
+            secure=settings.MINIO_SECURE.lower() == "true",
+        )
+    else:
+        logger.debug("Local MinIO authentication")
+        file_system = Minio(
+            settings.MINIO_EXTERNAL_SERVER_URL,
+            access_key=settings.MINIO_ACCESS_KEY,
+            secret_key=settings.MINIO_SECRET_KEY,
+            secure=settings.MINIO_SECURE.lower() == "true",
+        )
     clowder_bucket = settings.MINIO_BUCKET_NAME
+    logger.debug("Connecting to bucket %s", clowder_bucket)
     if not file_system.bucket_exists(clowder_bucket):
         file_system.make_bucket(clowder_bucket)
     file_system.set_bucket_versioning(clowder_bucket, VersioningConfig(ENABLED))
@@ -44,6 +80,7 @@ def get_rabbitmq() -> BlockingChannel:
     parameters = pika.ConnectionParameters(
         settings.RABBITMQ_HOST, credentials=credentials
     )
+    logger.debug("Connecting to rabbitmq at %s", settings.RABBITMQ_HOST)
     connection = pika.BlockingConnection(parameters)
     channel = connection.channel()
     return channel

pyproject.toml

@@ -6,10 +6,28 @@ description = """Clowder is a cloud native data management framework to support
                  metadata, and automatic data pipelines."""
 readme = "README.md"
 requires-python = ">=3.10"
-dependencies = ["fastapi==0.95.1", "pydantic==1.10.13", "uvicorn==0.21.1", "motor==3.1.2", "pymongo==4.8.0",
-    "beanie==1.18.0", "passlib==1.7.4", "bcrypt==4.0.1", "pyjwt==2.6.0", "minio==7.1.14", "python-multipart==0.0.6",
-    "email-validator==2.0.0.post2", "python-keycloak==2.15.3", "pika==1.3.1", "aio-pika==9.0.5", "elasticsearch==8.7.0",
-    "rocrate==0.7.0", "itsdangerous==2.1.2", "setuptools"]
+dependencies = [
+    "fastapi==0.95.1",
+    "pydantic==1.10.13",
+    "uvicorn==0.21.1",
+    "motor==3.1.2",
+    "pymongo==4.8.0",
+    "beanie==1.18.0",
+    "passlib==1.7.4",
+    "bcrypt==4.0.1",
+    "pyjwt==2.6.0",
+    "minio==7.1.14",
+    "python-multipart==0.0.6",
+    "email-validator==2.0.0.post2",
+    "python-keycloak==2.15.3",
+    "pika==1.3.1",
+    "aio-pika==9.0.5",
+    "elasticsearch==8.7.0",
+    "rocrate==0.7.0",
+    "itsdangerous==2.1.2",
+    "setuptools",
+    "boto3>=1.37.8",
+]
 
 [dependency-groups]
 dev = [