Skip to content

Commit 3dd4e65

Browse files
author
gdgate
authored
Merge pull request #1818 from tqtu/MSF-22129
FEATURE: MSF-22129 add support mysql input source Reviewed-by: <phong.nguyen-duy@gooddata.com> https://github.com/phong-nguyen-duy
2 parents 4b63128 + 536ec7c commit 3dd4e65

10 files changed

Lines changed: 205 additions & 3 deletions

File tree

‎Dockerfile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,10 @@ RUN cp -rf ci/postgresql/target/*.jar ./lib/gooddata/cloud_resources/postgresql/
7070
RUN mvn -f ci/mssql/pom.xml clean install -P binary-packaging
7171
RUN cp -rf ci/mssql/target/*.jar ./lib/gooddata/cloud_resources/mssql/drivers/
7272

73+
#build mysql dependencies
74+
RUN mvn -f ci/mysql/pom.xml clean install -P binary-packaging
75+
RUN cp -rf ci/mysql/target/*.jar ./lib/gooddata/cloud_resources/mysql/drivers/
76+
7377
RUN bundle install
7478

7579
ARG GIT_COMMIT=unspecified

‎ci/mysql/pom.xml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<modelVersion>4.0.0</modelVersion>
6+
7+
<groupId>com.gooddata.lcm</groupId>
8+
<artifactId>lcm-mysql-driver</artifactId>
9+
<version>1.0-SNAPSHOT</version>
10+
11+
<dependencies>
12+
<dependency>
13+
<groupId>mysql</groupId>
14+
<artifactId>mysql-connector-java</artifactId>
15+
<version>8.0.25</version>
16+
</dependency>
17+
<dependency>
18+
<groupId>org.slf4j</groupId>
19+
<artifactId>slf4j-api</artifactId>
20+
<version>1.7.2</version>
21+
</dependency>
22+
</dependencies>
23+
24+
<profiles>
25+
<profile>
26+
<id>binary-packaging</id>
27+
<build>
28+
<plugins>
29+
<plugin>
30+
<artifactId>maven-dependency-plugin</artifactId>
31+
<executions>
32+
<execution>
33+
<phase>package</phase>
34+
<goals>
35+
<goal>copy-dependencies</goal>
36+
</goals>
37+
<configuration>
38+
<outputDirectory>${project.build.directory}</outputDirectory>
39+
<!-- compile scope gives runtime and compile dependencies (skips test deps) -->
40+
<includeScope>runtime</includeScope>
41+
</configuration>
42+
</execution>
43+
</executions>
44+
</plugin>
45+
</plugins>
46+
</build>
47+
</profile>
48+
</profiles>
49+
50+
<repositories>
51+
<repository>
52+
<id>my-repo1</id>
53+
<name>my custom repo</name>
54+
<url>https://repository.mulesoft.org/nexus/content/repositories/public/</url>
55+
</repository>
56+
</repositories>
57+
</project>

‎lib/gooddata/cloud_resources/mysql/drivers/.gitkeepme‎

Whitespace-only changes.

‎lib/gooddata/cloud_resources/mysql/mysql_client.rb‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
# encoding: UTF-8
2+
# frozen_string_literal: true
3+
#
4+
# Copyright (c) 2021 GoodData Corporation. All rights reserved.
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
require 'securerandom'
9+
require 'java'
10+
require 'pathname'
11+
require_relative '../cloud_resource_client'
12+
13+
base = Pathname(__FILE__).dirname.expand_path
14+
Dir.glob(base + 'drivers/*.jar').each do |file|
15+
require file unless file.start_with?('lcm-mysql-driver')
16+
end
17+
18+
module GoodData
19+
module CloudResources
20+
class MysqlClient < CloudResourceClient
21+
JDBC_MYSQL_PATTERN = %r{jdbc:mysql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?}
22+
MYSQL_DEFAULT_PORT = 3306
23+
JDBC_MYSQL_PROTOCOL = 'jdbc:mysql://'
24+
VERIFY_FULL = 'VERIFY_IDENTITY'
25+
PREFER = 'PREFERRED'
26+
REQUIRE = 'REQUIRED'
27+
MYSQL_FETCH_SIZE = 1000
28+
29+
class << self
30+
def accept?(type)
31+
type == 'mysql'
32+
end
33+
end
34+
35+
def initialize(options = {})
36+
raise("Data Source needs a client to Mysql to be able to query the storage but 'mysql_client' is empty.") unless options['mysql_client']
37+
38+
if options['mysql_client']['connection'].is_a?(Hash)
39+
@database = options['mysql_client']['connection']['database']
40+
@authentication = options['mysql_client']['connection']['authentication']
41+
@ssl_mode = options['mysql_client']['connection']['sslMode']
42+
raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
43+
44+
@url = build_url(options['mysql_client']['connection']['url'])
45+
else
46+
raise('Missing connection info for Mysql client')
47+
end
48+
49+
Java.com.mysql.cj.jdbc.Driver
50+
end
51+
52+
def realize_query(query, _params)
53+
GoodData.gd_logger.info("Realize SQL query: type=mysql status=started")
54+
55+
connect
56+
filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
57+
measure = Benchmark.measure do
58+
statement = @connection.create_statement
59+
statement.set_fetch_size(MYSQL_FETCH_SIZE)
60+
has_result = statement.execute(query)
61+
if has_result
62+
result = statement.get_result_set
63+
metadata = result.get_meta_data
64+
col_count = metadata.column_count
65+
CSV.open(filename, 'wb') do |csv|
66+
csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
67+
csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
68+
end
69+
end
70+
end
71+
GoodData.gd_logger.info("Realize SQL query: type=mysql status=finished duration=#{measure.real}")
72+
filename
73+
ensure
74+
@connection&.close
75+
@connection = nil
76+
end
77+
78+
def connect
79+
GoodData.logger.info "Setting up connection to Mysql #{@url}"
80+
81+
prop = java.util.Properties.new
82+
prop.setProperty('user', @authentication['basic']['userName'])
83+
prop.setProperty('password', @authentication['basic']['password'])
84+
85+
@connection = java.sql.DriverManager.getConnection(@url, prop)
86+
@connection.set_auto_commit(false)
87+
end
88+
89+
def build_url(url)
90+
matches = url.scan(JDBC_MYSQL_PATTERN)
91+
raise 'Cannot reach the url' unless matches
92+
93+
host = matches[0][0]
94+
port = matches[0][2]&.to_i || MYSQL_DEFAULT_PORT
95+
96+
"#{JDBC_MYSQL_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{get_ssl_mode(@ssl_mode)}&useCursorFetch=true"
97+
end
98+
99+
def get_ssl_mode(ssl_mode)
100+
mode = PREFER
101+
if ssl_mode == 'verify-full'
102+
mode = VERIFY_FULL
103+
elsif ssl_mode == 'require'
104+
mode = REQUIRE
105+
end
106+
107+
mode
108+
end
109+
end
110+
end
111+
end

‎lib/gooddata/cloud_resources/postgresql/postgresql_client.rb‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,6 @@ def build_url(url)
9898

9999
host = matches[0][0]
100100
port = matches[0][2]&.to_i || POSTGRES_DEFAULT_PORT
101-
raise "Custom port #{port} is not supported. Remove it or use the default port '5432'" if POSTGRES_DEFAULT_PORT != port
102101

103102
"#{JDBC_POSTGRES_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{@ssl_mode}#{VERIFY_FULL == @ssl_mode ? SSL_JAVA_FACTORY : ''}"
104103
end

‎lib/gooddata/helpers/data_helper.rb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ def realize(params = {})
4444
realize_link
4545
when 's3'
4646
realize_s3(params)
47-
when 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql'
47+
when 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql', 'mysql'
4848
raise GoodData::InvalidEnvError, "DataSource does not support type \"#{source}\" on the platform #{RUBY_PLATFORM}" unless RUBY_PLATFORM =~ /java/
4949
require_relative '../cloud_resources/cloud_resources'
5050
realize_cloud_resource(source, params)

‎lib/gooddata/lcm/actions/update_metric_formats.rb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ def validate_input_source(input_source)
7575

7676
modified_input_source = input_source
7777
case type
78-
when 'ads', 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql'
78+
when 'ads', 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql', 'mysql'
7979
if metric_format[:query].blank?
8080
GoodData.logger.warn("The metric input_source '#{type}' is missing property 'query'")
8181
return nil

‎spec/data/mysql_data.csv‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
segment_id,client_id,project_title,project_id,project_token
2+
Segment,Client1,Client-1,,token
3+
Segment,Client2,Client-2,,token

‎spec/environment/secrets.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ global:
88
redshift_secret_key: 1JHYwvoIQinjgdKJXQL6RNGFhx4o4M9DiQf9q4jV+dq6xCPLDBCP/8tXBd0H9y8xdOOI78mY/aOQWjiPzgizLA==
99
snowflake_password: 1zg1PDRMQq2DhBG3SwQOA8/POUkeek3gurrmV4MT2Go=
1010
blob_storage_connection: Md/faNEbH3YOsmVCDaUJEH4/eHkABgp2X1V6BIZyMbuMxlAdlCFxY8gLqM1sJUEt2txBp7I6PmDdnG34+wV1nawRO3U9WAwr8wTPI57pkcNj0fpFN9KLycNA8ms6cVklxFlgO1WmCOqBL+wBnIbqRZ8sl9wx2BTFebt8QQSLucGMZtY0oDjy/YeG6SqH+HCzEW70ipU3whVXWJkZStIK8cHy9uxJZF88uqpphJFTQAFMwgCQQ9+vEF+mpt4xaWtF2KnRkif2a2OuYSsEStuA/A==
11+
mysql_connection: PsmFcHvUtff5A2OGYWaI1+KKkQsDrThUf46DAR/m6v69yPBLI65jCZO25XV6R4xU
1112
development:
1213
dev_token: 8qWaLsyWwAUJ7MJJTBdriUvtaWKNidnzmfxVThCrL0c=
1314
prod_token: RitXvhFjpJ8KEpqUqZm57iV3bwVU1zBGDrXNklvwkaE=

‎spec/lcm/integration/spec/others/data_helper_spec.rb‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,26 @@
179179
},
180180
}
181181

182+
mysql_basic_params = {
183+
"input_source"=> {
184+
"type"=> "mysql",
185+
"query"=> "SELECT DISTINCT * FROM clients",
186+
},
187+
"mysql_client" => {
188+
"connection" => {
189+
"url" => "jdbc:mysql://msf-test-database.na.intgdc.com:1435",
190+
"database" => "integration_test",
191+
"authentication" => {
192+
"basic" => {
193+
"userName" => "mysql_integration_test",
194+
"password" => ConnectionHelper::SECRETS[:mysql_connection],
195+
}
196+
},
197+
"sslMode" => "prefer"
198+
},
199+
},
200+
}
201+
182202
describe 'data helper', :vcr do
183203

184204
it 'connect to redshift with IAM authentication' do
@@ -256,4 +276,11 @@
256276
data = File.open('spec/data/mssql_data.csv').read
257277
expect(data).to eq File.open(file_path).read
258278
end
279+
280+
it 'connect to mysql with BASIC authentication' do
281+
data_helper = GoodData::Helpers::DataSource.new(mysql_basic_params['input_source'])
282+
file_path = data_helper.realize(mysql_basic_params)
283+
data = File.open('spec/data/mysql_data.csv').read
284+
expect(data).to eq File.open(file_path).read
285+
end
259286
end

0 commit comments

Comments
 (0)