Merge pull request #2062 from gooddata/gsan-grif-524

hung-nguyen-hoang · web-flow · commit 96b4b4ff5a95 · 2026-01-28T03:39:42.000Z
GRIF-524 - Refactor of integration-e2e to run on github CI
diff --git a/.gitignore b/.gitignore
@@ -45,3 +45,7 @@ deprecations.txt
 
 # test cache files
 spec/cache/
+
+# local/ci execution
+lib/gooddata/cloud_resources/*/*/*.jar
+ci/*/target/
diff --git a/docker-compose.lcm.yml b/docker-compose.lcm.yml
@@ -1,12 +1,10 @@
-version: '2'
 services:
   appstore:
     image: gooddata/appstore
     build:
       context: .
       dockerfile: Dockerfile.jruby
-    links:
-     - localstack
+    working_dir: /src
     environment:
      - GD_ENV
      - GDC_LOG_LEVEL
@@ -57,16 +55,17 @@ services:
      - GD_STG_DEFAULT_PASSWORD
     volumes:
       - .:/src
-    volumes_from:
-      - bundle
-    mem_limit: 2500m
+      - bundle_cache:/src/bundle
+    deploy:
+      resources:
+        limits:
+          memory: 2500m
   localstack:
     image: hahihula/localstack_dockerfile
     expose:
      - "4572"
     environment:
      - SERVICES=s3:4572
-  bundle:
-    image: busybox
-    volumes:
-      - /bundle
+
+volumes:
+  bundle_cache:
diff --git a/gooddata.gemspec b/gooddata.gemspec
@@ -35,6 +35,7 @@ Gem::Specification.new do |s|
     s.add_development_dependency 'rspec', '~> 3.12.0'
     s.add_development_dependency 'rspec-expectations', '~> 3.12'
     s.add_development_dependency 'rspec_junit_formatter', '~> 0.6.0'
+    s.add_development_dependency 'webmock', '~> 3.26.0'
 
     s.add_dependency 'azure-storage-blob', '~> 2.0'
     s.add_dependency 'nokogiri', '~> 1', '>= 1.10.8'
@@ -47,6 +48,7 @@ Gem::Specification.new do |s|
     s.add_development_dependency 'rspec', '~> 3.5.0'
     s.add_development_dependency 'rspec-expectations', '~> 3.5'
     s.add_development_dependency 'rspec_junit_formatter', '~> 0.3.0'
+    s.add_development_dependency 'webmock', '~> 2.3.1'
 
     s.add_dependency 'azure-storage-blob', '~> 1.1.0'
     s.add_dependency 'nokogiri', '~> 1.10.0'
@@ -56,7 +58,6 @@ Gem::Specification.new do |s|
     s.add_dependency 'unf', '~> 0.1.4'
   end
   s.add_development_dependency 'simplecov', '~> 0.12'
-  s.add_development_dependency 'webmock', '~> 2.3.1'
   s.add_development_dependency 'yard', '~> 0.9.11'
   s.add_development_dependency 'yard-rspec', '~> 0.1'
   s.add_development_dependency 'pry'
@@ -68,7 +69,12 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'vcr', '5.0.0'
   s.add_development_dependency 'hashdiff', '~> 0.4'
 
-  s.add_development_dependency 'sqlite3' if RUBY_PLATFORM != 'java'
+  if RUBY_PLATFORM == 'java'
+    s.add_development_dependency 'activerecord-jdbcsqlite3-adapter'
+    s.add_development_dependency 'activerecord'
+  else
+    s.add_development_dependency 'sqlite3'
+  end
 
   if RUBY_VERSION >= '2.8'
     s.add_dependency 'activesupport', '>= 6.0.3.1'
diff --git a/lib/gooddata/bricks/middleware/aws_middleware.rb b/lib/gooddata/bricks/middleware/aws_middleware.rb
@@ -60,6 +60,7 @@ def rewrite_for_aws_sdk_v2(config)
             'SDK uses SSL everywhere. To disable SSL you must ' \
             'configure an endpoint that uses http://.'
         end
+        config.delete('endpoint') if config['endpoint'].nil? || config['endpoint'].to_s.strip.empty?
         config
       end
     end
diff --git a/lib/gooddata/rest/connection.rb b/lib/gooddata/rest/connection.rb
@@ -177,6 +177,9 @@ def initialize(opts)
 
       # Connect using username and password
       def connect(username, password, options = {})
+        if RUBY_VERSION >= '3.3'
+          Psych::Parser.code_point_limit = 100_000_000
+        end
         server = options[:server] || Helpers::AuthHelper.read_server
         options = DEFAULT_LOGIN_PAYLOAD.merge(options)
         headers = options[:headers] || {}
diff --git a/spec/environment/environment.rb b/spec/environment/environment.rb
@@ -49,6 +49,7 @@ def initial_secrets(env)
           s3_bucket_name: ENV['RT_S3_BUCKET_NAME'],
           s3_access_key_id: ENV['RT_S3_ACCESS_KEY'],
           s3_secret_access_key: ENV['RT_S3_SECRET_KEY'],
+          s3_session_token: ENV['RT_S3_SESSION_TOKEN'],
           redshift_password: ENV['REDSHIFT_PASSWORD'],
           redshift_access_key: ENV['REDSHIFT_ACCESS_KEY'],
           redshift_secret_key: ENV['REDSHIFT_SECRET_KEY'],
diff --git a/spec/lcm/integration/params/provisioning_brick.json.erb b/spec/lcm/integration/params/provisioning_brick.json.erb
@@ -36,7 +36,9 @@
     "aws_client": {
       "access_key_id": "${s3_access_key}",
       "secret_access_key": "${s3_secret_access_key}",
-      "endpoint": "<%= s3_endpoint %>"
+      "session_token": "${s3_session_token}",
+      "endpoint": "<%= s3_endpoint %>",
+      "region": "<%= s3_region %>"
     }
   },
   "gd_encoded_hidden_params": {
@@ -53,5 +55,6 @@
   "GDC_USERNAME": "<%= config[:username] %>",
   "GDC_PASSWORD": "<%= config[:password] %>",
   "s3_secret_access_key": "<%= s3_secret_access_key %>",
-  "s3_access_key": "<%= s3_access_key %>"
+  "s3_access_key": "<%= s3_access_key %>",
+  "s3_session_token": "<%= s3_session_token %>"
 }
diff --git a/spec/lcm/integration/params/release_brick.json.erb b/spec/lcm/integration/params/release_brick.json.erb
@@ -10,8 +10,7 @@
     "SEGMENTS": <%= segments %>,
     "DATA_PRODUCT": "<%= data_product %>",
     "TOKENS": {
-      "pg": "<%= config[:prod_token] %>",
-      "vertica": "<%= config[:vertica_prod_token] %>"
+      "pg": "<%= config[:prod_token] %>"
     },
     <% if defined? jdbc_url %>
     "ads_client": {
diff --git a/spec/lcm/integration/params/release_brick_delete_old_master_project.json.erb b/spec/lcm/integration/params/release_brick_delete_old_master_project.json.erb
@@ -6,8 +6,7 @@
     "SEGMENTS": <%= segments %>,
     "DATA_PRODUCT": "<%= data_product %>",
     "TOKENS": {
-      "pg": "<%= config[:prod_token] %>",
-      "vertica": "<%= config[:vertica_prod_token] %>"
+      "pg": "<%= config[:prod_token] %>"
     },
     <% if defined? jdbc_url %>
     "ads_client": {
diff --git a/spec/lcm/integration/params/release_brick_set_master_project.json.erb b/spec/lcm/integration/params/release_brick_set_master_project.json.erb
@@ -3,20 +3,33 @@
   "CLIENT_GDC_PROTOCOL": "https",
   "CLIENT_GDC_HOSTNAME": "<%= config[:prod_server] %>",
   "gd_encoded_params": {
+    "SEGMENTS": <%= segments %>,
+    "DATA_PRODUCT": "<%= data_product %>",
+    "TOKENS": {
+      "pg": "<%= config[:prod_token] %>"
+    },
     <% if defined? jdbc_url %>
     "ads_client": {
       "username": "${GDC_USERNAME}",
       "password": "${GDC_PASSWORD}",
       "jdbc_url": "<%= jdbc_url %>"
     },
     <% end %>
-    "SEGMENTS": <%= segments %>,
-    "DATA_PRODUCT": "<%= data_product %>"
+    "technical_user": [
+      "${GDC_USERNAME}"
+    ],
+    "development_client": {
+      "protocol": "https",
+      "hostname": "<%= config[:dev_server] %>",
+      "username": "${GDC_USERNAME}",
+      "password": "${GDC_PASSWORD}"
+    }
   },
+  "production_tag": "dashboard, metric",
   "release_table_name": "<%= release_table_name %>",
   "fail_early": "true",
   "strict": "true",
   "GDC_USERNAME": "<%= config[:username] %>",
   "GDC_PASSWORD": "<%= config[:password] %>",
-  "set_master_project": "<%= master_project_id %>"
+  "GDC_LOG_LEVEL": "DEBUG"
 }
diff --git a/spec/lcm/integration/spec/shared_contexts_for_lcm.rb b/spec/lcm/integration/spec/shared_contexts_for_lcm.rb
@@ -123,7 +123,9 @@ def create_workspace_csv(workspaces, client_id_column)
       data = {
         segment_id: "LCM_SPEC_#{segment}_#{i}",
         development_pid: @project.obj_id,
-        driver: segment == 'PREMIUM' ? 'vertica' : 'pg',
+        # TODO: Uncomment below after https://gooddata.atlassian.net/browse/GRIF-655 is fixed
+        # driver: segment == 'PREMIUM' ? 'vertica' : 'pg',
+        driver: 'pg',
         master_name: "LCM spec master project (#{segment} #{i}) " + '##{version}' # rubocop:disable Lint/InterpolationCheck
       }
 
diff --git a/spec/lcm/integration/support/in_memory_ads.rb b/spec/lcm/integration/support/in_memory_ads.rb
@@ -1,11 +1,26 @@
-require 'sqlite3' unless RUBY_PLATFORM == 'java'
+if RUBY_PLATFORM == 'java'
+  require 'active_record'
+  require 'activerecord-jdbcsqlite3-adapter'
+else
+  require 'sqlite3'
+end
 
 module Support
   class InMemoryAds
     def initialize
-      db = SQLite3::Database.new(':memory:')
-      db.results_as_hash = true
-      @db = db
+      if RUBY_PLATFORM == 'java'
+        # Use ActiveRecord with JDBC adapter for JRuby
+        ActiveRecord::Base.establish_connection(
+          adapter: 'sqlite3',
+          database: ':memory:'
+        )
+        @db = ActiveRecord::Base.connection
+      else
+        # Use sqlite3 gem for MRI
+        db = SQLite3::Database.new(':memory:')
+        db.results_as_hash = true
+        @db = db
+      end
     end
 
     def data
@@ -33,12 +48,30 @@ def execute(*args, &block)
     end
 
     def execute_with_headers(*args)
-      res = @db.execute(*args)
-      res.map do |row|
-        # sqlite3 returns hash with both column names and numbers, we want only names
-        res = GoodData::Helpers.symbolize_keys(row.reject { |k, _| k.is_a? Integer })
-        yield res if block_given?
-        res
+      if RUBY_PLATFORM == 'java'
+        # ActiveRecord JDBC adapter
+        result = @db.exec_query(*args)
+        # ActiveRecord returns arrays of arrays, convert to hash format
+        columns = result.columns if result.respond_to?(:columns)
+        result.map do |row|
+          if row.is_a?(Array) && columns
+            # Convert array to hash
+            row_hash = columns.zip(row).to_h
+            res = GoodData::Helpers.symbolize_keys(row_hash)
+          else
+            res = GoodData::Helpers.symbolize_keys(row.is_a?(Hash) ? row : {})
+          end
+          yield res if block_given?
+          res
+        end
+      else
+        res = @db.execute(*args)
+        res.map do |row|
+          # sqlite3 returns hash with both column names and numbers, we want only names
+          res = GoodData::Helpers.symbolize_keys(row.reject { |k, _| k.is_a? Integer })
+          yield res if block_given?
+          res
+        end
       end
     end
 
diff --git a/spec/lcm/integration/support/s3_helper.rb b/spec/lcm/integration/support/s3_helper.rb
@@ -1,10 +1,9 @@
 module Support
   class S3Helper
     LOCALSTACK_ENDPOINT = 'http://localstack:4572'.freeze
-    S3_ENDPOINT = 'https://s3.amazonaws.com'.freeze
     USER_FILTERS_KEY = 'user_filters'
     USERS_KEY = 'users_brick_input'
-    REGION = 'us-east-1'
+    REGION = 'eu-west-1'
 
     class << self
       include GoodData::Environment::ConnectionHelper
@@ -13,16 +12,14 @@ def upload_file(file, object_name)
         bucket_name = SECRETS[:s3_bucket_name]
         s3_endpoint = if GoodData::Environment::LOCALSTACK
                         Support::S3Helper::LOCALSTACK_ENDPOINT
-                      else
-                        Support::S3Helper::S3_ENDPOINT
                       end
 
         s3 = Aws::S3::Resource.new(
           access_key_id: SECRETS[:s3_access_key_id],
           secret_access_key: SECRETS[:s3_secret_access_key],
+          session_token: SECRETS[:s3_session_token],
           endpoint: s3_endpoint,
           region: REGION,
-          force_path_style: true
         )
 
         bucket = s3.bucket(bucket_name)
@@ -32,9 +29,11 @@ def upload_file(file, object_name)
 
         {
           s3_bucket: bucket_name,
-          s3_endpoint: s3_endpoint,
+          s3_endpoint: s3_endpoint ? bucket.url : s3_endpoint,
           s3_access_key: SECRETS[:s3_access_key_id],
-          s3_secret_access_key: SECRETS[:s3_secret_access_key]
+          s3_secret_access_key: SECRETS[:s3_secret_access_key],
+          s3_session_token: SECRETS[:s3_session_token],
+          s3_region: REGION
         }
       end
     end
diff --git a/spec/vcr_configurer.rb b/spec/vcr_configurer.rb
@@ -43,8 +43,12 @@ def self.setup
           vcr_config.allow_http_connections_when_no_cassette = true
           vcr_config.configure_rspec_metadata!
 
-          vcr_config.ignore_request do
-            @ignore_vcr_requests
+          vcr_config.ignore_request do |request|
+            @ignore_vcr_requests ||
+              request.uri.include?('s3.amazonaws.com') ||
+              request.uri.match?(/s3[.-][a-z0-9-]+\.amazonaws\.com/) ||
+              request.uri.include?('staging-lcm-prod.intgdc.com') ||
+              request.uri.include?('staging-lcm-dev.intgdc.com')
           end
 
           vcr_config.default_cassette_options = {
