Release 6.3.1 - Update Quickstart Demo to use custom Service Account

Author: TXZebra

deploy.sh

@@ -17,32 +17,61 @@
 # Exit on error.
 set -e
 
-log_bucket=$1
+build_account=""
+log_bucket=""
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+    --build-account)
+        build_account="$2"
+        shift 2
+        ;;
+    --gcs-logs-bucket)
+        log_bucket="$2"
+        shift 2
+        ;;
+    *)
+        # Handle other arguments or flags if needed
+        echo "Unknown option: $1" >&2
+        shift
+        ;;
+    esac
+done
+
 echo "Deploying Cortex Data Foundation."
 
 cloud_build_project=$(cat "config/config.json" | python3 -c "import json,sys; print(str(json.load(sys.stdin)['projectIdSource']))" 2>/dev/null || echo "")
-if [[ "${cloud_build_project}" == "" ]]
-then
-    echo "ERROR: Cortex Data Foundation is not configured."
-    echo "Please read https://github.com/GoogleCloudPlatform/cortex-data-foundation/blob/main/README.md"
+if [[ "${cloud_build_project}" == "" ]]; then
+    echo "ERROR: Cortex Data Foundation source project is not configured."
+    echo "Please read https://cloud.google.com/cortex/docs/deployment-prerequisites"
     exit 1
 fi
+
 echo "Using Cloud Build in project '${cloud_build_project}'"
 
-if [[ "${log_bucket}" == "" ]]
-then
+if [[ "${log_bucket}" == "" ]]; then
     _GCS_BUCKET="${cloud_build_project}_cloudbuild"
 else
     _GCS_BUCKET="${log_bucket}"
 fi
+
 echo "Using logs bucket ${_GCS_BUCKET}"
 
+# Check if bucket already created and exist e.g. if Cloud Build has been used before in the project.
+# If it does not exist Cloud Build might not have been called before and the bucket will be created
+# with correct permissions
+if gcloud storage buckets describe "${_GCS_BUCKET}" --project="$cloud_build_project" >/dev/null 2>&1; then
+    # Bucket exists we can go ahead and set policy binding
+    echo "Ensuring ${build_account} has access to bucket ${_GCS_BUCKET}"
+    gcloud storage buckets add-iam-policy-binding gs://"${_GCS_BUCKET}" --member=serviceAccount:"${build_account}" --role=roles/storage.objectUser
+fi
+
 set +e
 echo -e "\n\033[0;32m\033[1mPlease wait while Data Foundation is being deployed...\033[0m\n"
 gcloud builds submit --config=cloudbuild.yaml --suppress-logs \
     --project "${cloud_build_project}" \
-    --substitutions=_GCS_BUCKET="${_GCS_BUCKET}" . \
-    && _SUCCESS="true"
+    --substitutions=_GCS_BUCKET="${_GCS_BUCKET}",_BUILD_ACCOUNT="projects/${cloud_build_project}/serviceAccounts/${build_account}" . &&
+    _SUCCESS="true"
 if [[ "${_SUCCESS}" != "true" ]]; then
     echo -e "\n🛑 Data Foundation deployment has failed. 🛑"
     exit 1

src/SAP/SAP_CDC/deploy_cdc.sh

@@ -15,6 +15,8 @@
 # limitations under the License.
 
 log_bucket=$1
+build_account="$2"
+
 echo "Deploying CDC and extra data."
 
 if [[ "${log_bucket}" == "" ]]
@@ -27,4 +29,5 @@ else
     _GCS_BUCKET="${log_bucket}"
 fi
 
-gcloud builds submit --config=cloudbuild.cdc.yaml --substitutions=_GCS_BUCKET="${_GCS_BUCKET}" .
+gcloud builds submit --config=cloudbuild.cdc.yaml --substitutions=_GCS_BUCKET="${_GCS_BUCKET}",_BUILD_ACCOUNT="${build_account}" .
+

src/SFDC/deploy.sh

@@ -20,6 +20,7 @@
 # assuming the bucket exists.
 
 GCS_LOGS_BUCKET="$1"
+BUILD_ACCOUNT="$2"
 
 echo -e "🦄🦄🦄 Running Cortex Data Foundation modules for SalesForce.com 🔪🔪🔪\n"
 
@@ -33,4 +34,4 @@ fi
 
 gcloud builds submit . \
         --config=cloudbuild.sfdc.yaml \
-        --substitutions=_GCS_LOGS_BUCKET="${GCS_LOGS_BUCKET}"
+        --substitutions=_GCS_LOGS_BUCKET="${GCS_LOGS_BUCKET}",_BUILD_ACCOUNT="${BUILD_ACCOUNT}"

src/k9/src/meridian/deploy_meridian.sh

@@ -111,55 +111,53 @@ done
 
 validate_args
 
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
 
 export SOURCE_PROJECT=$(cat ${CONFIG_FILE} | python3 -c "import json,sys; print(str(json.load(sys.stdin)['projectIdSource']))" 2>/dev/null || echo "")
 export TARGET_BUCKET=$(cat ${CONFIG_FILE} | python3 -c "import json,sys; print(str(json.load(sys.stdin)['targetBucket']))" 2>/dev/null || echo "")
-if [[ "${SOURCE_PROJECT}" == "" ]]
-then
-    echo "ERROR: projectIdSource value in config.json is empty."
-    exit 1
+if [[ "${SOURCE_PROJECT}" == "" ]]; then
+  echo "ERROR: projectIdSource value in config.json is empty."
+  exit 1
 fi
-if [[ "${TARGET_BUCKET}" == "" ]]
-then
-    echo "ERROR: targetBucket value in config.json is empty."
-    exit 1
+if [[ "${TARGET_BUCKET}" == "" ]]; then
+  echo "ERROR: targetBucket value in config.json is empty."
+  exit 1
 fi
 
-    declare -a _WORKER_POOL_OPTIONS
-    declare -a _REGION_PARAMETER
-
-    if [[ -n "${_WORKER_POOL_NAME}" ]]; then
-      _WORKER_POOL_OPTIONS+=(",_WORKER_POOL_NAME=\"${_WORKER_POOL_NAME}\"")
-    fi
+declare -a _WORKER_POOL_OPTIONS
+declare -a _REGION_PARAMETER
 
-    if [[ -n "${_CLOUD_BUILD_REGION}" ]]; then
-      _WORKER_POOL_OPTIONS+=(",_CLOUD_BUILD_REGION=\"${_CLOUD_BUILD_REGION}\"")
-      _REGION_PARAMETER=(--region "${_CLOUD_BUILD_REGION}")
-    fi
+if [[ -n "${_WORKER_POOL_NAME}" ]]; then
+  _WORKER_POOL_OPTIONS+=(",_WORKER_POOL_NAME=\"${_WORKER_POOL_NAME}\"")
+fi
 
-    if [[ -n "${_BUILD_ACCOUNT}" ]]; then
-      _WORKER_POOL_OPTIONS+=(",_BUILD_ACCOUNT=\"${_BUILD_ACCOUNT}\"")
-    fi
+if [[ -n "${_CLOUD_BUILD_REGION}" ]]; then
+  _WORKER_POOL_OPTIONS+=(",_CLOUD_BUILD_REGION=\"${_CLOUD_BUILD_REGION}\"")
+  _REGION_PARAMETER=(--region "${_CLOUD_BUILD_REGION}")
+fi
 
+if [[ -n "${_BUILD_ACCOUNT}" ]]; then
+  _WORKER_POOL_OPTIONS+=(",_BUILD_ACCOUNT=\"${_BUILD_ACCOUNT}\"")
+fi
 
 echo "Deploying Meridian"
 
 cp -f "${CONFIG_FILE}" "${SCRIPT_DIR}/config/config.json"
 set +e
+
 gcloud builds submit --project="${SOURCE_PROJECT}" \
-    --config="${SCRIPT_DIR}/cloudbuild.meridian.yaml" \
-    --substitutions \
-    _TGT_BUCKET="${TARGET_BUCKET}",_GCS_BUCKET="${GCS_LOGS_BUCKET}" "${_WORKER_POOL_OPTIONS[@]}" \
-    "${_REGION_PARAMETER[@]}" \
-    "${SCRIPT_DIR}"
+  --config="${SCRIPT_DIR}/cloudbuild.meridian.yaml" \
+  --substitutions \
+  _TGT_BUCKET="${TARGET_BUCKET}",_GCS_BUCKET="${GCS_LOGS_BUCKET}",_BUILD_ACCOUNT="${_BUILD_ACCOUNT}" \
+  "${_REGION_PARAMETER[@]}" \
+  "${SCRIPT_DIR}"
+
 _err=$?
 rm -f "${SCRIPT_DIR}/config/config.json"
 
-if [ $_err -ne 0 ]
-then
-    echo "Meridian deployment failed."
-    exit 1
+if [ $_err -ne 0 ]; then
+  echo "Meridian deployment failed."
+  exit 1
 fi
 
-echo "Meridian has been deployed."
+echo "Meridian has been deployed."

src/utils/automatic/deploy_cortex_meridian_1_click.sh

@@ -174,31 +174,61 @@ fi
 
 fancy_echo_done "Done creating Service Account"
 
+# Create Cloud Build SA
+
+fancy_echo_start "Creating Cortex Deployer Service Account for Cloud Build"
+
+# Define the service account ID
+CLOUD_BUILD_SERVICE_ACCOUNT_ID="cortex-deployer"
+
+# Define the full service account email
+CLOUD_BUILD_SERVICE_ACCOUNT_EMAIL="${CLOUD_BUILD_SERVICE_ACCOUNT_ID}@${PROJECT_ID}.iam.gserviceaccount.com"
+
+# Check if the service account exists
+if gcloud iam service-accounts list \
+    --project="$PROJECT_ID" \
+    --filter="email:${CLOUD_BUILD_SERVICE_ACCOUNT_EMAIL}" \
+    --format="value(email)" |
+grep -q "${CLOUD_BUILD_SERVICE_ACCOUNT_EMAIL}"; then
+    fancy_sub_echo "Service account '${CLOUD_BUILD_SERVICE_ACCOUNT_EMAIL}' already exists skipping create 🔐"
+else
+    fancy_sub_echo "Creating service account '${CLOUD_BUILD_SERVICE_ACCOUNT_ID}'..."
+    gcloud iam service-accounts create "${CLOUD_BUILD_SERVICE_ACCOUNT_ID}" --project="$PROJECT_ID" \
+        --description="Cortex Deployer Service Account" \
+        --display-name="Cortex Deployer"
+    if [ $? -eq 0 ]; then
+        fancy_sub_echo "Service account '${CLOUD_BUILD_SERVICE_ACCOUNT_EMAIL}' created successfully 🔐"
+    else
+        fancy_error_echo "Error creating service account '${CLOUD_BUILD_SERVICE_ACCOUNT_ID}' 🔐"
+        exit 1
+    fi
+fi
+
+fancy_echo_done "Done creating Service Account"
+
 # IAM roles assignments
 
-fancy_echo_start "Assigning IAM roles to Cloud Build Default Service Account & Meridian Colab Runner Service Account"
+fancy_echo_start "Assigning IAM roles to Cloud Build Service Account & Meridian Colab Runner Service Account"
 
 CLOUD_BUILD_ROLES=('roles/aiplatform.colabEnterpriseAdmin'
     'roles/storage.objectUser' 'roles/workflows.editor' 'roles/bigquery.jobUser'
-    'roles/bigquery.dataEditor' 'roles/iam.serviceAccountUser')
-
-# Get cloud build service account email
-CLOUD_BUILD_SA=$(gcloud builds get-default-service-account --project="$PROJECT_ID" --format='value(serviceAccountEmail)' | sed 's|.*/||')
-
+    'roles/bigquery.dataEditor' 'roles/iam.serviceAccountUser' 'roles/logging.logWriter' 
+    'roles/cloudbuild.builds.builder')
+	
 for role in "${CLOUD_BUILD_ROLES[@]}"; do
-    fancy_sub_echo "Assigning role: $role to $CLOUD_BUILD_SA 🔑"
+    fancy_sub_echo "Assigning role: $role to $CLOUD_BUILD_SERVICE_ACCOUNT_EMAIL 🔑"
     gcloud projects add-iam-policy-binding "$PROJECT_ID" \
-        --member="serviceAccount:$CLOUD_BUILD_SA" \
+        --member="serviceAccount:$CLOUD_BUILD_SERVICE_ACCOUNT_EMAIL" \
         --role="$role" \
         --condition=None \
         --no-user-output-enabled
     if [ $? -ne 0 ]; then
-        fancy_error_echo "Error assigning role '$role' to service account '$CLOUD_BUILD_SA'"
+        fancy_error_echo "Error assigning role '$role' to service account '$CLOUD_BUILD_SERVICE_ACCOUNT_EMAIL'"
         exit 1
     fi
 done
 
-fancy_sub_echo "Done assigning roles to service account '$CLOUD_BUILD_SA'"
+fancy_sub_echo "Done assigning roles to service account '$CLOUD_BUILD_SERVICE_ACCOUNT_EMAIL'"
 
 MERIDIAN_RUNNER_ROLES=('roles/bigquery.dataViewer' 'roles/bigquery.jobUser'
     'roles/bigquery.readSessionUser' 'roles/cloudbuild.builds.editor'
@@ -317,7 +347,7 @@ fancy_sub_echo "Deploying Cortex Data Foundation via Cloud Build👷"
 
 source_project=$(cat "config/config.json" | python3 -c "import json,sys; print(str(json.load(sys.stdin)['projectIdSource']))" 2>/dev/null || echo "")
 gcloud config set project "${source_project}"
-./deploy.sh
+./deploy.sh --build-account "$CLOUD_BUILD_SERVICE_ACCOUNT_EMAIL"
 popd 1>/dev/null
 
 fancy_echo_done "Done Cortex Data Foundation was deployed via Cloud Build"