FEATURE: MSF-17743 upgrade custom v2 for rollout brick

Author: Tu Quoc Tran

lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+# (C) 2019-2020 GoodData Corporation
+require_relative 'base_action'
+
+# Migrate date dimension urn:gooddata:date or urn:custom:date to urn:custom_v2:date
+module GoodData
+  module LCM2
+    class MigrateGdcDateDimension < BaseAction
+      DESCRIPTION = 'Migrate Gdc Date Dimension'
+      DATE_DIMENSION_CUSTOM_V2 = 'urn:custom_v2:date'
+      DATE_DIMENSION_OLD = %w[urn:gooddata:date urn:custom:date]
+
+      PARAMS = define_params(self) do
+        description 'Client Used for Connecting to GD'
+        param :gdc_gd_client, instance_of(Type::GdClientType), required: true
+
+        description 'Specifies how to synchronize LDM and resolve possible conflicts'
+        param :synchronize_ldm, instance_of(Type::SynchronizeLDM), required: false, default: 'diff_against_master_with_fallback'
+
+        description 'Synchronization Info'
+        param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
+      end
+
+      RESULT_HEADER = %i[from to status]
+
+      class << self
+        def call(params)
+          results = []
+          params.synchronize.map do |segment_info|
+            result = migrate_date_dimension(params, segment_info)
+            results.concat(result)
+          end
+
+          {
+            results: results
+          }
+        end
+
+        def migrate_date_dimension(params, segment_info)
+          results = []
+          client = params.gdc_gd_client
+          latest_blueprint = segment_info[:from_blueprint]
+          # don't migrate when latest master doesn't contain custom v2 date.
+          return results unless contain_v2?(latest_blueprint)
+
+          previous_blueprint = segment_info[:previous_master]&.blueprint
+          # check latest master and previous master
+          master_upgrade_datasets = get_upgrade_dates(latest_blueprint, previous_blueprint) if params[:synchronize_ldm].downcase == 'diff_against_master' && previous_blueprint
+          unless master_upgrade_datasets&.empty?
+            segment_info[:to].pmap do |entry|
+              pid = entry[:pid]
+              to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
+              to_blueprint = to_project.blueprint
+              upgrade_datasets = get_upgrade_dates(latest_blueprint, to_blueprint)
+              next if upgrade_datasets.empty?
+
+              message = get_upgrade_message(upgrade_datasets)
+
+              results << {
+                from: segment_info[:from],
+                to: pid,
+                status: to_project.upgrade_custom_v2(message)
+              }
+            end
+          end
+
+          results
+        end
+
+        def get_upgrade_dates(src_blueprint, dest_blueprint)
+          dest_dates = get_date_dimensions(dest_blueprint) if dest_blueprint
+          src_dates = get_date_dimensions(src_blueprint) if src_blueprint
+
+          return false if dest_dates.empty? || src_dates.empty?
+
+          upgrade_datasets = []
+          dest_dates.each do |dest|
+            src_dim = get_date_dimension(src_blueprint, dest[:id])
+            next unless src_dim
+
+            upgrade_datasets << src_dim[:identifier] if upgrade?(src_dim, dest) && src_dim[:identifier]
+          end
+
+          upgrade_datasets
+        end
+
+        def get_upgrade_message(upgrade_datasets)
+          {
+            upgrade: {
+              dateDatasets: {
+                upgrade: "exact",
+                datasets: upgrade_datasets
+              }
+            }
+          }
+        end
+
+        def upgrade?(src_dim, dest_dim)
+          src_dim[:urn] == DATE_DIMENSION_CUSTOM_V2 && DATE_DIMENSION_OLD.any? { |e| dest_dim[:urn] == e }
+        end
+
+        def contain_v2?(blueprint)
+          get_date_dimensions(blueprint).any? { |e| e[:urn] == DATE_DIMENSION_CUSTOM_V2 }
+        end
+
+        def get_date_dimension(blueprint, id)
+          GoodData::Model::ProjectBlueprint.find_date_dimension(blueprint, id)
+        end
+
+        def get_date_dimensions(blueprint)
+          GoodData::Model::ProjectBlueprint.date_dimensions(blueprint)
+        end
+      end
+    end
+  end
+end

lib/gooddata/lcm/actions/synchronize_ldm.rb

@@ -49,6 +49,8 @@ class SynchronizeLdm < BaseAction
         param :include_deprecated, instance_of(Type::BooleanType), required: false, default: false
       end
 
+      RESULT_HEADER = %i[from to status]
+
       class << self
         def call(params)
           results = []
@@ -76,9 +78,9 @@ def sync_segment_ldm(params, segment_info)
           include_deprecated = params.include_deprecated.to_b
           from_pid = segment_info[:from]
           from = params.development_client.projects(from_pid) || fail("Invalid 'from' project specified - '#{from_pid}'")
-
           GoodData.logger.info "Creating Blueprint, project: '#{from.title}', PID: #{from_pid}"
           blueprint = from.blueprint(include_ca: params.include_computed_attributes.to_b)
+          segment_info[:from_blueprint] = blueprint
           maql_diff = nil
           previous_master = segment_info[:previous_master]
           diff_against_master = %w(diff_against_master_with_fallback diff_against_master)

lib/gooddata/lcm/lcm2.rb

@@ -138,6 +138,7 @@ def respond_to_missing?(name, *_args)
         EnsureTechnicalUsersDomain,
         EnsureTechnicalUsersProject,
         SynchronizeLdm,
+        MigrateGdcDateDimension,
         SynchronizeClients,
         SynchronizeComputedAttributes,
         CollectDymanicScheduleParams,

lib/gooddata/models/from_wire.rb

@@ -105,6 +105,7 @@ def self.parse_date_dimensions(date_dim)
           d[:title] = date_dim['dateDimension']['title']
           d[:urn] = date_dim['dateDimension']['urn']
           d[:identifier_prefix] = date_dim['dateDimension']['identifierPrefix']
+          d[:identifier] = date_dim['dateDimension']['identifier'] if date_dim['dateDimension']['identifier']
           d[:columns] = parse_bridges(date_dim)
         end
       end

lib/gooddata/models/project.rb

@@ -625,6 +625,7 @@ def attributes_by_title(title)
     def blueprint(options = {})
       options = { include_ca: true }.merge(options)
       result = client.get("/gdc/projects/#{pid}/model/view", params: { includeDeprecated: true, includeGrain: true, includeCA: options[:include_ca] })
+
       polling_url = result['asyncTask']['link']['poll']
       model = client.poll_on_code(polling_url, options)
       bp = GoodData::Model::FromWire.from_wire(model, options)
@@ -1922,6 +1923,20 @@ def resolve_roles(login, desired_roles, options = {})
       [user, roles]
     end
 
+    def upgrade_custom_v2(message, options = {})
+      uri = "/gdc/md/#{pid}/datedimension/upgrade"
+      poll_result = client&.post(uri, message)
+
+      return poll_result['wTaskStatus']['status'] if poll_result['wTaskStatus'] && poll_result['wTaskStatus']['status']
+
+      polling_uri = poll_result['asyncTask']['link']['poll']
+      result = client&.poll_on_response(polling_uri, options) do |body|
+        body && body['wTaskStatus'] && body['wTaskStatus']['status'] == 'RUNNING'
+      end
+
+      result['wTaskStatus']['status'] == 'OK' ? 'OK' : 'FAIL'
+    end
+
     def add
       @add ||= GoodData::AutomatedDataDistribution.new(self)
       @add

spec/data/blueprints/old_date_dimension.json

@@ -0,0 +1,74 @@
+{
+  "title": "blueprint old date dimension",
+  "include_ca": true,
+  "datasets": [
+    {
+      "type": "dataset",
+      "title": "person",
+      "id": "dataset.person",
+      "columns": [
+        {
+          "type": "anchor",
+          "id": "attr.person.factsof",
+          "title": "Records of person",
+          "description": "Records of person",
+          "folder": "person"
+        },
+        {
+          "type": "attribute",
+          "id": "attr.person.name",
+          "title": "name",
+          "description": "name",
+          "folder": "person"
+        },
+        {
+          "type": "label",
+          "id": "label.person.name",
+          "reference": "attr.person.name",
+          "title": "name",
+          "gd_data_type": "VARCHAR(128)",
+          "gd_type": "GDC.text",
+          "default_label": true
+        },
+        {
+          "type": "fact",
+          "id": "fact.person.id",
+          "title": "id",
+          "description": "id",
+          "folder": "person",
+          "gd_data_type": "DECIMAL(12,2)"
+        },
+        {
+          "type": "date",
+          "dataset": "datecustom"
+        },
+        {
+          "type": "date",
+          "dataset": "dategooddata"
+        }
+      ]
+    }
+  ],
+  "date_dimensions": [
+    {
+      "type": "date_dimension",
+      "id": "datecustom",
+      "title": "DateCustom",
+      "urn": "urn:custom:date",
+      "identifier_prefix": "datecustom",
+      "columns": [
+
+      ]
+    },
+    {
+      "type": "date_dimension",
+      "id": "dategooddata",
+      "title": "DateGooddata",
+      "urn": "urn:gooddata:date",
+      "identifier_prefix": "dategooddata",
+      "columns": [
+
+      ]
+    }
+  ]
+}

spec/integration/blueprint_upgrade_spec.rb

@@ -0,0 +1,113 @@
+# (C) 2019-2020 GoodData Corporation
+require 'gooddata'
+
+describe 'Should upgrade custom v2 for project', :vcr, :constraint => 'slow' do
+  before(:all) do
+    @client = ConnectionHelper.create_default_connection
+    @blueprint = GoodData::Model::ProjectBlueprint.from_json('./spec/data/blueprints/old_date_dimension.json')
+  end
+
+  after(:all) do
+    @client&.disconnect
+  end
+
+  it 'upgrade all date dimension to custom v2' do
+    to_project = @client&.create_project_from_blueprint(@blueprint, auth_token: ConnectionHelper::SECRETS[:gd_project_token], environment: ProjectHelper::ENVIRONMENT)
+    blueprint = to_project.blueprint(include_ca: true)
+    date_old = GoodData::Model::ProjectBlueprint.date_dimensions(blueprint)
+
+    expect(date_old).not_to be_nil
+    date_old.each do |date|
+      expect(%w[datecustom dategooddata].any? { |e| date[:id] == e }).to be_truthy
+      expect(GoodData::LCM2::MigrateGdcDateDimension::DATE_DIMENSION_OLD.any? { |e| date[:urn]&.include?(e) }).to be_truthy
+    end
+
+    message = get_upgrade_message(true, nil)
+    to_project&.upgrade_custom_v2(message)
+
+    blueprint = to_project.blueprint(include_ca: true)
+    date_new = GoodData::Model::ProjectBlueprint.date_dimensions(blueprint)
+    expect(date_new).not_to be_nil
+    date_new.each do |date|
+      expect(%w[datecustom dategooddata].any? { |e| date[:id] == e }).to be_truthy
+      expect(date[:urn]&.include?(GoodData::LCM2::MigrateGdcDateDimension::DATE_DIMENSION_CUSTOM_V2)).to be_truthy
+    end
+
+    to_project&.delete
+  end
+
+  it 'upgrade a date dimension to custom v2' do
+    to_project = @client&.create_project_from_blueprint(@blueprint, auth_token: ConnectionHelper::SECRETS[:gd_project_token], environment: ProjectHelper::ENVIRONMENT)
+    blueprint = to_project.blueprint(include_ca: true)
+    date_old = GoodData::Model::ProjectBlueprint.date_dimensions(blueprint)
+
+    expect(date_old).not_to be_nil
+    date_old.each do |date|
+      expect(%w[datecustom dategooddata].any? { |e| date[:id] == e }).to be_truthy
+      expect(GoodData::LCM2::MigrateGdcDateDimension::DATE_DIMENSION_OLD.any? { |e| date[:urn]&.include?(e) }).to be_truthy
+    end
+
+    message = get_upgrade_message(false, ['datecustom.dataset.dt'])
+    to_project&.upgrade_custom_v2(message)
+
+    blueprint = to_project.blueprint(include_ca: true)
+    date_new = GoodData::Model::ProjectBlueprint.date_dimensions(blueprint)
+    expect(date_new).not_to be_nil
+    date_new.each do |date|
+      expect(%w[datecustom dategooddata].any? { |e| date[:id] == e }).to be_truthy
+      if date[:id] == 'datecustom'
+        expect(date[:urn]&.include?(GoodData::LCM2::MigrateGdcDateDimension::DATE_DIMENSION_CUSTOM_V2)).to be_truthy
+      else
+        expect(date[:urn]&.include?('gooddata')).to be_truthy
+      end
+    end
+
+    to_project&.delete
+  end
+
+  it 'upgrade multiple date dimension to custom v2' do
+    to_project = @client&.create_project_from_blueprint(@blueprint, auth_token: ConnectionHelper::SECRETS[:gd_project_token], environment: ProjectHelper::ENVIRONMENT)
+    blueprint = to_project.blueprint(include_ca: true)
+    date_old = GoodData::Model::ProjectBlueprint.date_dimensions(blueprint)
+
+    expect(date_old).not_to be_nil
+    date_old.each do |date|
+      expect(%w[datecustom dategooddata].any? { |e| date[:id] == e }).to be_truthy
+      expect(GoodData::LCM2::MigrateGdcDateDimension::DATE_DIMENSION_OLD.any? { |e| date[:urn]&.include?(e) }).to be_truthy
+    end
+
+    message = get_upgrade_message(false, ['datecustom.dataset.dt','dategooddata.dataset.dt'])
+    to_project&.upgrade_custom_v2(message)
+
+    blueprint = to_project.blueprint(include_ca: true)
+    date_new = GoodData::Model::ProjectBlueprint.date_dimensions(blueprint)
+    expect(date_new).not_to be_nil
+    date_new.each do |date|
+      expect(%w[datecustom dategooddata].any? { |e| date[:id] == e }).to be_truthy
+      expect(date[:urn]&.include?(GoodData::LCM2::MigrateGdcDateDimension::DATE_DIMENSION_CUSTOM_V2)).to be_truthy
+    end
+
+    to_project&.delete
+  end
+
+  def get_upgrade_message(is_all, upgrade_datasets)
+    if is_all
+      {
+          upgrade: {
+              dateDatasets: {
+                  upgrade: "all"
+              }
+          }
+      }
+    else
+      {
+          upgrade: {
+              dateDatasets: {
+                  upgrade: "exact",
+                  datasets: upgrade_datasets
+              }
+          }
+      }
+    end
+  end
+end