Merge branch '10io-workhorse-google-native-client' into 'master'

Add support for the workhorse google storage client configuration

See merge request https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/6530



Merged-by: Robert Marshall <rmarshall@gitlab.com>
Approved-by: Jason Young <jyoung@gitlab.com>
Approved-by: Robert Marshall <rmarshall@gitlab.com>
Reviewed-by: David Fernandez <dfernandez@gitlab.com>
Co-authored-by: David Fernandez <dfernandez@gitlab.com>

files/gitlab-cookbooks/gitlab/libraries/helpers/gitlab_workhorse_helper.rb

@@ -29,10 +29,49 @@ class GitlabWorkhorseHelper < BaseHelper
         azure_storage_account_name = #{toml_string(object_store.dig('connection', 'azure_storage_account_name'))}
         azure_storage_access_key = #{toml_string(object_store.dig('connection', 'azure_storage_access_key'))}
       AZURECFG
+    when 'Google'
+      google_config_from(object_store)
     end
   end
+  private
+
   def toml_string(str)
     (str || '').to_json
   end
+
+  def google_config_from(object_store)
+    connection = object_store['connection']
+
+    return unless connection['google_application_default'] ||
+      connection['google_json_key_string'] ||
+      connection['google_json_key_location']
+
+    result = <<~GOOGLECFG
+    [object_storage]
+      provider = "Google"
+    GOOGLECFG
+
+    if connection['google_application_default']
+      value = connection['google_application_default']
+      result << <<~GOOGLECFG
+      [object_storage.google]
+        google_application_default = #{toml_string(value)}
+      GOOGLECFG
+    elsif connection['google_json_key_string']
+      value = connection['google_json_key_string']
+      result << <<~GOOGLECFG
+      [object_storage.google]
+        google_json_key_string = '''#{value}'''
+      GOOGLECFG
+    elsif connection['google_json_key_location']
+      value = connection['google_json_key_location']
+      result << <<~GOOGLECFG
+      [object_storage.google]
+        google_json_key_location = #{toml_string(value)}
+      GOOGLECFG
+    end
+
+    result
+  end
 end

spec/chef/cookbooks/gitlab/libraries/helpers/gitlab_workhorse_helper_spec.rb

@@ -105,10 +105,57 @@ RSpec.describe GitlabWorkhorseHelper do
     end
     context 'with Google' do
-      let(:connection_hash) { { 'provider' => 'Google' } }
-      it 'returns nil' do
-        expect(subject.object_store_toml).to be nil
+      context 'with application default' do
+        let(:connection_hash) { google_connection_hash_with_application_default }
+        it 'returns a valid TOML' do
+          data = Tomlrb.parse(subject.object_store_toml)
+
+          expect(data.dig('object_storage', 'provider')).to eq('Google')
+
+          google_data = data.dig('object_storage', 'google')
+          expect(google_data).to be_a(Hash)
+          expect(google_data.keys.count).to eq(1)
+          expect(google_data['google_application_default']).to eq(true)
+        end
+      end
+
+      context 'with json_key_string' do
+        let(:connection_hash) { google_connection_hash_with_json_key_string }
+
+        it 'returns a valid TOML' do
+          data = Tomlrb.parse(subject.object_store_toml)
+
+          expect(data.dig('object_storage', 'provider')).to eq('Google')
+
+          google_data = data.dig('object_storage', 'google')
+          expect(google_data).to be_a(Hash)
+          expect(google_data.keys.count).to eq(1)
+          expect(google_data['google_json_key_string']).to eq(connection_hash['google_json_key_string'])
+        end
+      end
+
+      context 'with json_key_location' do
+        let(:connection_hash) { google_connection_hash_with_json_key_location }
+
+        it 'returns a valid TOML' do
+          data = Tomlrb.parse(subject.object_store_toml)
+
+          expect(data.dig('object_storage', 'provider')).to eq('Google')
+
+          google_data = data.dig('object_storage', 'google')
+          expect(google_data).to be_a(Hash)
+          expect(google_data.keys.count).to eq(1)
+          expect(google_data['google_connection_hash_with_json_key_location']).to eq(connection_hash['google_connection_hash_with_json_key_location'])
+        end
+      end
+
+      context 'with invalid connection_hash' do
+        let(:connection_hash) { incomplete_google_connection_hash }
+
+        it 'returns a nil TOML' do
+          expect(subject.object_store_toml).to be_nil
+        end
       end
     end
   end

spec/chef/cookbooks/gitlab/recipes/gitlab-workhorse_spec.rb

@@ -276,20 +276,48 @@ RSpec.describe 'gitlab::gitlab-workhorse' do
       end
     end
-    # Workhorse doesn't directly use a Google Cloud client and relies on
-    # pre-signed URLs, but include a test for completeness.
     context 'with Google Cloud config' do
-      let(:connection_hash) do
-        {
-          'provider' => 'Google',
-          'google_application_default' => true
-        }
+      where(:parameter_name, :parameter_value) do
+        'google_application_default' | true
+        'google_application_default' | ''
+        'google_application_default' | nil
+        'google_json_key_string'     | 'test'
+        'google_json_key_string'     | ''
+        'google_json_key_string'     | nil
+        'google_json_key_location'   | 'test'
+        'google_json_key_location'   | ''
+        'google_json_key_location'   | nil
       end
-      it 'does not include object storage config' do
-        expect(chef_run).to render_file(config_file).with_content { |content|
-          expect(content).not_to include(%([object_storage]))
-        }
+      with_them do
+        let(:connection_hash) do
+          {
+            'provider' => 'Google',
+            parameter_name => parameter_value
+          }
+        end
+        let(:expected_parameter_value) do
+          if parameter_name == 'google_json_key_string'
+            "'''#{parameter_value}'''"
+          else
+            parameter_value.to_json
+          end
+        end
+
+        if params[:parameter_value].nil?
+          it 'does not include object storage config' do
+            expect(chef_run).to render_file(config_file).with_content { |content|
+              expect(content).not_to include(%([object_storage]))
+            }
+          end
+        else
+          it 'includes the proper google configuration' do
+            expect(chef_run).to render_file(config_file).with_content { |content|
+              expect(content).to include(%([object_storage]\n  provider = "Google"\n))
+              expect(content).to include(%([object_storage.google]\n  #{parameter_name} = #{expected_parameter_value}))
+            }
+          end
+        end
       end
     end
   end

spec/chef/support/shared_context/object_store_shared_context.rb

@@ -37,6 +37,37 @@ RSpec.shared_context 'object storage config' do
     }
   end
+  let(:google_connection_hash_with_application_default) do
+    {
+      'provider' => 'Google',
+      'google_application_default' => true
+    }
+  end
+  let(:google_connection_hash_with_json_key_string) do
+    {
+      'provider' => 'Google',
+      'google_json_key_string' => '{
+        "type": "service_account",
+        "project_id": "test",
+        "private_key_id": "555555555555555555555",
+        "private_key": "-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----\n",
+        "client_email": "test@test.iam.gserviceaccount.com",
+        "client_id": "555555555555555555555",
+        "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+        "token_uri": "https://oauth2.googleapis.com/token",
+        "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+        "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test%40test.iam.gserviceaccount.com"
+      }'
+    }
+  end
+  let(:google_connection_hash_with_json_key_location) do
+    {
+      'provider' => 'Google',
+      'google_json_key_location' => '/usr/opt/testdata/google_dummy_credentials.json'
+    }
+  end
+  let(:incomplete_google_connection_hash) { { 'provider' => 'Google' } }
+
   let(:aws_connection_data) { JSON.parse(aws_connection_hash.to_json, symbolize_names: true) }
   let(:aws_storage_options) { JSON.parse(aws_storage_options_hash.to_json, symbolize_names: true) }
 end