GRIF-524 - Refactor of integration-e2e to run on github CI

Author: Guilherme Santos

.gitignore

@@ -45,3 +45,7 @@ deprecations.txt
 
 # test cache files
 spec/cache/
+
+# local/ci execution
+lib/gooddata/cloud_resources/*/*/*.jar
+ci/*/target/

docker-compose.lcm.yml

@@ -1,12 +1,10 @@
-version: '2'
 services:
   appstore:
     image: gooddata/appstore
     build:
       context: .
       dockerfile: Dockerfile.jruby
-    links:
-     - localstack
+    working_dir: /src
     environment:
      - GD_ENV
      - GDC_LOG_LEVEL
@@ -57,16 +55,17 @@ services:
      - GD_STG_DEFAULT_PASSWORD
     volumes:
       - .:/src
-    volumes_from:
-      - bundle
-    mem_limit: 2500m
+      - bundle_cache:/src/bundle
+    deploy:
+      resources:
+        limits:
+          memory: 2500m
   localstack:
     image: hahihula/localstack_dockerfile
     expose:
      - "4572"
     environment:
      - SERVICES=s3:4572
-  bundle:
-    image: busybox
-    volumes:
-      - /bundle
+
+volumes:
+  bundle_cache:

gooddata.gemspec

@@ -56,7 +56,7 @@ Gem::Specification.new do |s|
     s.add_dependency 'unf', '~> 0.1.4'
   end
   s.add_development_dependency 'simplecov', '~> 0.12'
-  s.add_development_dependency 'webmock', '~> 2.3.1'
+  s.add_development_dependency 'webmock', '~> 3.26.0'
   s.add_development_dependency 'yard', '~> 0.9.11'
   s.add_development_dependency 'yard-rspec', '~> 0.1'
   s.add_development_dependency 'pry'
@@ -68,7 +68,12 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'vcr', '5.0.0'
   s.add_development_dependency 'hashdiff', '~> 0.4'
 
-  s.add_development_dependency 'sqlite3' if RUBY_PLATFORM != 'java'
+  if RUBY_PLATFORM == 'java'
+    s.add_development_dependency 'activerecord-jdbcsqlite3-adapter'
+    s.add_development_dependency 'activerecord'
+  else
+    s.add_development_dependency 'sqlite3'
+  end
 
   if RUBY_VERSION >= '2.8'
     s.add_dependency 'activesupport', '>= 6.0.3.1'

lib/gooddata/bricks/middleware/aws_middleware.rb

@@ -60,6 +60,7 @@ def rewrite_for_aws_sdk_v2(config)
             'SDK uses SSL everywhere. To disable SSL you must ' \
             'configure an endpoint that uses http://.'
         end
+        config.delete('endpoint') if config['endpoint'].nil? || config['endpoint'].to_s.strip.empty?
         config
       end
     end

lib/gooddata/rest/connection.rb

@@ -177,6 +177,8 @@ def initialize(opts)
 
       # Connect using username and password
       def connect(username, password, options = {})
+        Psych::Parser.code_point_limit = 100_000_000
+
         server = options[:server] || Helpers::AuthHelper.read_server
         options = DEFAULT_LOGIN_PAYLOAD.merge(options)
         headers = options[:headers] || {}

spec/environment/environment.rb

@@ -49,6 +49,7 @@ def initial_secrets(env)
           s3_bucket_name: ENV['RT_S3_BUCKET_NAME'],
           s3_access_key_id: ENV['RT_S3_ACCESS_KEY'],
           s3_secret_access_key: ENV['RT_S3_SECRET_KEY'],
+          s3_session_token: ENV['RT_S3_SESSION_TOKEN'],
           redshift_password: ENV['REDSHIFT_PASSWORD'],
           redshift_access_key: ENV['REDSHIFT_ACCESS_KEY'],
           redshift_secret_key: ENV['REDSHIFT_SECRET_KEY'],

spec/lcm/integration/params/provisioning_brick.json.erb

@@ -36,7 +36,9 @@
     "aws_client": {
       "access_key_id": "${s3_access_key}",
       "secret_access_key": "${s3_secret_access_key}",
-      "endpoint": "<%= s3_endpoint %>"
+      "session_token": "${s3_session_token}",
+      "endpoint": "<%= s3_endpoint %>",
+      "region": "<%= s3_region %>"
     }
   },
   "gd_encoded_hidden_params": {
@@ -53,5 +55,6 @@
   "GDC_USERNAME": "<%= config[:username] %>",
   "GDC_PASSWORD": "<%= config[:password] %>",
   "s3_secret_access_key": "<%= s3_secret_access_key %>",
-  "s3_access_key": "<%= s3_access_key %>"
+  "s3_access_key": "<%= s3_access_key %>",
+  "s3_session_token": "<%= s3_session_token %>"
 }

spec/lcm/integration/params/release_brick.json.erb

@@ -10,8 +10,7 @@
     "SEGMENTS": <%= segments %>,
     "DATA_PRODUCT": "<%= data_product %>",
     "TOKENS": {
-      "pg": "<%= config[:prod_token] %>",
-      "vertica": "<%= config[:vertica_prod_token] %>"
+      "pg": "<%= config[:prod_token] %>"
     },
     <% if defined? jdbc_url %>
     "ads_client": {

spec/lcm/integration/params/release_brick_delete_old_master_project.json.erb

@@ -6,8 +6,7 @@
     "SEGMENTS": <%= segments %>,
     "DATA_PRODUCT": "<%= data_product %>",
     "TOKENS": {
-      "pg": "<%= config[:prod_token] %>",
-      "vertica": "<%= config[:vertica_prod_token] %>"
+      "pg": "<%= config[:prod_token] %>"
     },
     <% if defined? jdbc_url %>
     "ads_client": {

spec/lcm/integration/params/release_brick_set_master_project.json.erb

@@ -3,20 +3,33 @@
   "CLIENT_GDC_PROTOCOL": "https",
   "CLIENT_GDC_HOSTNAME": "<%= config[:prod_server] %>",
   "gd_encoded_params": {
+    "SEGMENTS": <%= segments %>,
+    "DATA_PRODUCT": "<%= data_product %>",
+    "TOKENS": {
+      "pg": "<%= config[:prod_token] %>"
+    },
     <% if defined? jdbc_url %>
     "ads_client": {
       "username": "${GDC_USERNAME}",
       "password": "${GDC_PASSWORD}",
       "jdbc_url": "<%= jdbc_url %>"
     },
     <% end %>
-    "SEGMENTS": <%= segments %>,
-    "DATA_PRODUCT": "<%= data_product %>"
+    "technical_user": [
+      "${GDC_USERNAME}"
+    ],
+    "development_client": {
+      "protocol": "https",
+      "hostname": "<%= config[:dev_server] %>",
+      "username": "${GDC_USERNAME}",
+      "password": "${GDC_PASSWORD}"
+    }
   },
+  "production_tag": "dashboard, metric",
   "release_table_name": "<%= release_table_name %>",
   "fail_early": "true",
   "strict": "true",
   "GDC_USERNAME": "<%= config[:username] %>",
   "GDC_PASSWORD": "<%= config[:password] %>",
-  "set_master_project": "<%= master_project_id %>"
+  "GDC_LOG_LEVEL": "DEBUG"
 }