Merge pull request #1769 from gooddata/develop

Merge develop to master

Reviewed-by: Binh Pham
             https://github.com/helobinvn

Author: gdgate

.travis.yml

@@ -7,6 +7,8 @@ stages:
   if: type IN (api, cron)
 - name: platform-cleanup
   if: type IN (cron)
+- name: license-update
+  if: branch = master AND type = push
 - name: gem-release
   if: branch = master AND type = push
 - name: gem-smoke-test
@@ -73,23 +75,52 @@ jobs:
 
   - name: lcm integration (vcr) tests - e2e
     stage: before-merge
-    script: bundle exec rspec spec/lcm/integration/lcm_end_to_end_spec.rb
+    script: bundle exec rspec spec/lcm/integration/spec/e2e/lcm_end_to_end_spec.rb
     env:
     - VCR_RECORD_MODE=none
     rvm: 2.3
 
   #  AFTER MERGE
   #  staging 1
-  - &lcm-integration-tests
-    name: staging1 - lcm integration tests
+  - &lcm-integration-e2e-tests
+    name: staging1 - lcm integration-e2e tests
     stage: after-merge
     script: |
       bundle exec rake -f lcm.rake docker:build
       bundle exec rake -f lcm.rake docker:bundle
-      bundle exec rake -f lcm.rake test:docker:integration
+      bundle exec rake -f lcm.rake test:docker:integration-e2e
     env:
     - VCR_ON=false
     - GD_ENV=staging
+    - GD_MAX_RETRY=2
+    rvm: jruby-9.1.14
+    jdk: openjdk8
+
+  - &lcm-integration-release-tests
+    name: staging1 - lcm integration-release tests
+    stage: after-merge
+    script: |
+      bundle exec rake -f lcm.rake docker:build
+      bundle exec rake -f lcm.rake docker:bundle
+      bundle exec rake -f lcm.rake test:docker:integration-release
+    env:
+    - VCR_ON=false
+    - GD_ENV=staging
+    - GD_MAX_RETRY=2
+    rvm: jruby-9.1.14
+    jdk: openjdk8
+
+  - &lcm-integration-others-tests
+    name: staging1 - lcm integration-others tests
+    stage: after-merge
+    script: |
+      bundle exec rake -f lcm.rake docker:build
+      bundle exec rake -f lcm.rake docker:bundle
+      bundle exec rake -f lcm.rake test:docker:integration-others
+    env:
+    - VCR_ON=false
+    - GD_ENV=staging
+    - GD_MAX_RETRY=2
     rvm: jruby-9.1.14
     jdk: openjdk8
 
@@ -155,11 +186,26 @@ jobs:
     jdk: openjdk8
 
   # staging 2
-  - <<: *lcm-integration-tests
-    name: staging2 - lcm integration tests
+  - <<: *lcm-integration-e2e-tests
+    name: staging2 - lcm integration-e2e tests
+    env:
+    - GD_ENV=testing
+    - VCR_ON=false
+    - GD_MAX_RETRY=2
+
+  - <<: *lcm-integration-release-tests
+    name: staging2 - lcm integration-release tests
     env:
     - GD_ENV=testing
     - VCR_ON=false
+    - GD_MAX_RETRY=2
+
+  - <<: *lcm-integration-others-tests
+    name: staging2 - lcm integration-others tests
+    env:
+    - GD_ENV=testing
+    - VCR_ON=false
+    - GD_MAX_RETRY=2
 
   - <<: *lcm-slow-tests
     name: staging2 - lcm slow tests
@@ -192,11 +238,26 @@ jobs:
     - VCR_ON=false
 
   # staging 3
-  - <<: *lcm-integration-tests
-    name: staging3 - lcm integration tests
+  - <<: *lcm-integration-e2e-tests
+    name: staging3 - lcm integration-e2e tests
+    env:
+    - GD_ENV=development
+    - VCR_ON=false
+    - GD_MAX_RETRY=2
+
+  - <<: *lcm-integration-release-tests
+    name: staging3 - lcm integration-release tests
+    env:
+    - GD_ENV=development
+    - VCR_ON=false
+    - GD_MAX_RETRY=2
+
+  - <<: *lcm-integration-others-tests
+    name: staging3 - lcm integration-others tests
     env:
     - GD_ENV=development
     - VCR_ON=false
+    - GD_MAX_RETRY=2
 
   - <<: *lcm-slow-tests
     name: staging3 - lcm slow tests
@@ -234,15 +295,21 @@ jobs:
     stage: after-merge
     name: staging1 - test environment clean-up
     script: travis_wait 30 bundle exec ruby bin/test_projects_cleanup.rb -f
-    env: GD_ENV=staging
+    env:
+    - GD_ENV=staging
+    - GD_MAX_RETRY=2
 
   - <<: *cleanup
-    name: travis_wait 30 staging2 - test environment clean-up
-    env: GD_ENV=testing
+    name: staging2 - test environment clean-up
+    env:
+    - GD_ENV=testing
+    - GD_MAX_RETRY=2
 
   - <<: *cleanup
-    name: travis_wait 30 staging3 - test environment clean-up
-    env: GD_ENV=development
+    name: staging3 - test environment clean-up
+    env:
+    - GD_ENV=development
+    - GD_MAX_RETRY=2
 
   # AFTER MERGE UNIT TESTS ON ALTERNATIVE PLATFORM
 
@@ -279,8 +346,18 @@ jobs:
     script: bundle exec rake test:unit
     os: osx
     rvm: jruby-9.1.14
-    osx_image: xcode7.3
+    osx_image: xcode9.4
 
+  - stage: license-update
+    name: Update license file
+    rvm: 2.3
+    script: |
+      bundle exec rake license:update
+  - stage: license-update
+    name: Update license file
+    rvm: jruby-9.1.14
+    script: |
+      bundle exec rake license:update
   - stage: gem-release
     name: deploy MRI gem
     rvm: 2.3

Dockerfile.jruby

@@ -2,7 +2,7 @@ FROM jruby:9.1-alpine
 
 MAINTAINER Tomas Korcak <korczis@gmail.com>
 
-RUN apk add --no-cache curl make gcc git g++ python linux-headers binutils-gold gnupg libstdc++
+RUN apk add --no-cache curl make gcc git g++ python linux-headers binutils-gold gnupg libstdc++ postgresql postgresql-client openssl
 
 # Switch to directory with sources
 WORKDIR /src
@@ -20,3 +20,13 @@ ADD . .
 RUN keytool -importcert -alias gooddata-2008 -file "./data/2008.crt" -keystore $JAVA_HOME/lib/security/cacerts -trustcacerts -storepass 'changeit' -noprompt
 RUN keytool -importcert -alias gooddata-int -file "./data/new_ca.cer" -keystore $JAVA_HOME/lib/security/cacerts -trustcacerts -storepass 'changeit' -noprompt
 RUN keytool -importcert -alias gooddata-prod -file "data/new_prodgdc_ca.crt" -keystore $JAVA_HOME/lib/security/cacerts -trustcacerts -storepass 'changeit' -noprompt
+
+ENV LANG en_US.utf8
+RUN adduser -D -g '' gooddata
+RUN rm -rf $HOME/spec/postgresql
+RUN mkdir $HOME/spec/postgresql
+RUN chmod -R 0750 $HOME/spec/postgresql
+RUN chown gooddata $HOME/spec/postgresql
+RUN mkdir /run/postgresql; chmod 1777 -R /run;
+RUN mkdir $HOME/pre-exec.d; mkdir $HOME/pre-init.d; chmod -R 1755 $HOME/pre-exec.d; chmod -R 1755 $HOME/pre-exec.d
+USER gooddata

Rakefile

@@ -33,14 +33,13 @@ namespace :gem do
   desc "Release gem version #{GoodData::VERSION} to rubygems"
   task :release do
     gem = "gooddata-#{GoodData::VERSION}.gem"
-    origin_license_file = 'LICENSE'
-    new_license_file = 'LICENSE_FOR_RUBY_SDK_COMPONENT.txt'
     notices_file = 'NOTICES.txt'
-    File.delete(origin_license_file) if File.exist?(origin_license_file)
-    File.delete(notices_file) if File.exist?(notices_file)
-    puts "Deleted files: #{origin_license_file} and #{notices_file}"
-    File.rename(new_license_file, origin_license_file) if File.exists?(new_license_file)
-    puts "Renamed file #{new_license_file} to #{origin_license_file}"
+
+    if File.exist?(notices_file)
+      File.delete(notices_file)
+      puts "Deleted files: #{notices_file}"
+    end
+
     puts "Building #{gem} ..."
     res = `gem build ./gooddata.gemspec`
     file = res.match('File: (.*)')[1]
@@ -112,6 +111,24 @@ namespace :license do
     puts 'All licenses seem to be OK'
   end
 
+  desc 'Update LICENSE file'
+  task :update do
+    origin_license_file = 'LICENSE'
+    new_license_file = 'LICENSE_FOR_RUBY_SDK_COMPONENT.txt'
+
+    if File.exist?(origin_license_file)
+      File.delete(origin_license_file)
+      puts "Deleted files: #{origin_license_file}"
+    end
+
+    if File.exists?(new_license_file)
+      File.rename(new_license_file, origin_license_file)
+      puts "Renamed file #{new_license_file} to #{origin_license_file}"
+    end
+
+    puts 'Update LICENSE seem to be OK'
+  end
+
   desc 'Add license header to each file'
   task :add do
     spec = Gem::Specification.load('gooddata.gemspec')

bin/test_projects_cleanup.rb

@@ -22,6 +22,39 @@
 GoodData::Environment.load
 config = GoodData::Environment::ConnectionHelper::LCM_ENVIRONMENT
 secrets = GoodData::Environment::ConnectionHelper::SECRETS
+default_domain = GoodData::Environment::ConnectionHelper::DEFAULT_DOMAIN
+
+def delete_segment_by_title(title, segments, days = 14, force = false)
+  return if segments.empty?
+  dead_line = Time.now - days * 60 * 60 * 24
+
+  filtered_segments = segments.select do |s|
+    if s.id.match(title) && s.id.length > 14 && s.id[s.id.length-14..-1]
+      segment_created = s.id[s.id.length-14..-1]
+      created = Time.parse(segment_created) if segment_created
+
+      s if created.year >= Time.new.year && created < dead_line
+    end
+  end
+
+  filtered_segments.each do |segment|
+    begin
+      if force
+        segment.clients.each do |segment_client|
+          GoodData.logger.info("Deleting segment: #{segment.id} - client: #{segment_client.client_id}")
+          segment_client.dissociate
+        end
+
+        puts "Deleting segment: #{segment.id}"
+        segment && segment.delete(force: true)
+      else
+        puts "Would delete segment: #{segment.id}"
+      end
+    rescue RuntimeError, StandardError => ex
+      puts "Failed to delete segment #{segment.id}, reason: #{ex}"
+    end
+  end
+end
 
 def delete_project_by_title(title, projects, days = 14, force = false)
   dead_line = Time.now - days * 60 * 60 * 24
@@ -73,8 +106,16 @@ def delete_ads_by_title(title, client, days = 14, force = false)
   puts "#{deleted} ADS instances with title \"#{title}\" #{'would be ' unless force}deleted."
 end
 
-def clean_up!(client, force, days)
+def clean_up!(client, force, days, opts = {})
   projects = client.projects
+
+  if opts[:domain_id]
+    domain = client.domain(opts[:domain_id])
+
+    # segments id format: {title}_{hostname}_{datetime}
+    delete_segment_by_title(/CAR_DEMO_PREMIUM/, domain.segments, days, force)
+  end
+
   delete_project_by_title(/Insurance Demo Master/, projects, days, force)
   delete_project_by_title(/Car Demo Master/, projects, days, force)
   delete_project_by_title(/Insurance Demo Workspace/, projects, days, force)
@@ -117,7 +158,8 @@ def init_client(username, password, server)
 
 force = options[:force]
 days = options[:days] || 3
+
 clean_up!(dev_client, force, days)
-clean_up!(prod_client, force, days)
+clean_up!(prod_client, force, days, {domain_id: default_domain})
 
 dev_client.disconnect

docker-compose.lcm.yml

@@ -13,6 +13,9 @@ services:
      - JRUBY_OPTS=-J-Xmx1g
      - GD_SPEC_PASSWORD
      - VCR_ON
+     - POSTGRES_DB=gooddata
+     - POSTGRES_USER=gooddata
+     - POSTGRES_PASSWORD=changeit
     volumes:
       - .:/src
     volumes_from:

lcm.rake

@@ -3,7 +3,7 @@ require 'fileutils'
 require 'pathname'
 require 'rspec/core/rake_task'
 
-test_cases = %i[integration slow userprov load]
+test_cases = %i[integration-e2e integration-release integration-others slow userprov load]
 
 # Schema for new Bricks.
 brick_info_schema = {
@@ -79,7 +79,7 @@ namespace :test do
   test_cases.each do |test_case|
     desc "Run #{test_case} tests"
     RSpec::Core::RakeTask.new(test_case) do |task|
-      task.pattern = "spec/lcm/#{test_case}/**/*_spec.rb"
+      task.pattern = test_case['integration'] ? "spec/lcm/integration/spec/#{test_case.to_s.split('-')[-1]}/*_spec.rb" : "spec/lcm/#{test_case}/**/*_spec.rb"
     end
   end
 
@@ -101,8 +101,14 @@ namespace :test do
     test_cases.each do |t|
       desc "Run #{t} tests in Docker"
       task t do
-        system("docker-compose -f docker-compose.lcm.yml run --rm appstore bundle exec rake -f lcm.rake test:#{t}") ||
-          fail('Test execution failed!')
+        if t.to_s == 'integration'
+          system("docker-compose -f docker-compose.lcm.yml run --rm appstore /bin/bash -c ./spec/integration_with_postgresql.sh") ||
+            fail('Test execution failed!')
+        else
+          system("docker-compose -f docker-compose.lcm.yml run --rm appstore bundle exec rake -f lcm.rake test:#{t}") ||
+            fail('Test execution failed!')
+        end
+
       end
     end
   end