CUBESTORE_S3_ENDPOINT

Author: jdaripineni

rust/cubestore/cubestore/src/config/mod.rs

@@ -343,6 +343,7 @@ pub enum FileStoreProvider {
         region: String,
         bucket_name: String,
         sub_path: Option<String>,
+        endpoint: Option<String>,
     },
     GCS {
         bucket_name: String,
@@ -1303,6 +1304,7 @@ impl Config {
                                 "CUBESTORE_S3_REGION required when CUBESTORE_S3_BUCKET is set",
                             ),
                             sub_path: env::var("CUBESTORE_S3_SUB_PATH").ok(),
+                            endpoint: env::var("CUBESTORE_S3_ENDPOINT").ok(),
                         }
                     } else if let Ok(bucket_name) = env::var("CUBESTORE_MINIO_BUCKET") {
                         FileStoreProvider::MINIO {
@@ -1946,15 +1948,18 @@ impl Config {
                 region,
                 bucket_name,
                 sub_path,
+                endpoint,
             } => {
                 let data_dir = self.config_obj.data_dir.clone();
                 let region = region.to_string();
                 let bucket_name = bucket_name.to_string();
                 let sub_path = sub_path.clone();
+                let endpoint = endpoint.clone();
                 self.injector
                     .register("original_remote_fs", async move |_| {
                         let arc: Arc<dyn DIService> =
-                            S3RemoteFs::new(data_dir, region, bucket_name, sub_path).unwrap();
+                            S3RemoteFs::new(data_dir, region, bucket_name, sub_path, endpoint)
+                                .unwrap();
                         arc
                     })
                     .await;

rust/cubestore/cubestore/src/remotefs/mod.rs

@@ -687,6 +687,7 @@ mod tests {
             region.clone(),
             bucket_name.clone(),
             None,
+            None,
         )?;
 
         let name_maker = NameMaker::new(Uuid::new_v4().to_string());
@@ -699,6 +700,7 @@ mod tests {
             region.clone(),
             bucket_name.clone(),
             Some("remotefs_test_subpathdir".to_string()),
+            None,
         )
         .unwrap();
 

rust/cubestore/cubestore/src/remotefs/s3.rs

@@ -49,6 +49,7 @@ impl S3RemoteFs {
         region: String,
         bucket_name: String,
         sub_path: Option<String>,
+        endpoint: Option<String>,
     ) -> Result<Arc<Self>, CubeError> {
         // Incorrect naming for ENV variables...
         let access_key = env::var("CUBESTORE_AWS_ACCESS_KEY_ID").ok();
@@ -67,13 +68,21 @@ impl S3RemoteFs {
                 err.to_string()
             ))
         })?;
-        let region = region.parse::<Region>().map_err(|err| {
-            CubeError::internal(format!(
-                "Failed to parse Region '{}': {}",
-                region,
-                err.to_string()
-            ))
-        })?;
+        // If endpoint is provided, create a Custom region to support GovCloud and other non-standard regions
+        let region = if let Some(ep) = &endpoint {
+            Region::Custom {
+                region: region.clone(),
+                endpoint: ep.clone(),
+            }
+        } else {
+            region.parse::<Region>().map_err(|err| {
+                CubeError::internal(format!(
+                    "Failed to parse Region '{}': {}",
+                    region,
+                    err.to_string()
+                ))
+            })?
+        };
         let bucket = Bucket::new(&bucket_name, region.clone(), credentials)?;
         let fs = Arc::new(Self {
             dir,

rust/cubestore/cubestore/src/sql/mod.rs

@@ -3501,6 +3501,7 @@ mod tests {
                     region: "us-west-2".to_string(),
                     bucket_name: "cube-store-ci-test".to_string(),
                     sub_path: Some("high_frequency_inserts_s3".to_string()),
+                    endpoint: None,
                 };
                 c.select_workers = vec!["127.0.0.1:4306".to_string()];
                 c
@@ -3516,6 +3517,7 @@ mod tests {
                             region: "us-west-2".to_string(),
                             bucket_name: "cube-store-ci-test".to_string(),
                             sub_path: Some("high_frequency_inserts_s3".to_string()),
+                            endpoint: None,
                         };
                         c
                     })
@@ -3989,6 +3991,7 @@ mod tests {
                     region: "us-west-2".to_string(),
                     bucket_name: "cube-store-ci-test".to_string(),
                     sub_path: Some("create_table_with_location_cluster".to_string()),
+                    endpoint: None,
                 };
                 c.select_workers = vec!["127.0.0.1:24306".to_string()];
                 c.metastore_bind_address = Some("127.0.0.1:25312".to_string());
@@ -4005,6 +4008,7 @@ mod tests {
                             region: "us-west-2".to_string(),
                             bucket_name: "cube-store-ci-test".to_string(),
                             sub_path: Some("create_table_with_location_cluster".to_string()),
+                            endpoint: None,
                         };
                         c.metastore_remote_address = Some("127.0.0.1:25312".to_string());
                         c