Merge branch 'sh-add-ldap-tests' into 'master'

Add LDAP integration tests

See merge request gitlab-org/gitlab-qa!124

.gitlab-ci.yml

@@ -138,3 +138,15 @@ ee:geo:
     - bin/qa Test::Integration::Geo ${RELEASE:=EE}
   <<: *test
   <<: *high-capacity
+
+ce:ldap:
+  script:
+    - bin/qa Test::Integration::LDAP ${RELEASE:=CE}
+  <<: *test
+  <<: *high-capacity
+
+ee:ldap:
+  script:
+    - bin/qa Test::Integration::LDAP ${RELEASE:=EE}
+  <<: *test
+  <<: *high-capacity

README.md

@@ -157,6 +157,17 @@ This scenario tests that GitLab instance works as expected when
 enabling the embedded Mattermost server (see `Test::Instance::Image`
 above).
+### `Test::Integration::LDAP CE|EE|<full image address>`
+
+This scenario tests that GitLab instance works as expected with an external
+LDAP server. The scenario spins up an OpenLDAP server, seeds users, and
+attempts to LDAP-related tests against a GitLab instance.
+
+In EE, both the GitLab standard and LDAP credentials are needed:
+
+1. The first is used to login as an Admin to enter in the EE license.
+2. The second is used to conduct LDAP-related tasks
+
 ### `Test::Instance::Any CE|EE|<full image address>`
 This scenario tests that the any GitLab instance works as expected by running
@@ -164,8 +175,11 @@ tests against it (see `Test::Instance::Image` above).
 ## Supported environment variables
-* `GITLAB_USERNAME` - username to use when signing in to GitLab
-* `GITLAB_PASSWORD` - password to use when signing in to GitLab
+* `GITLAB_USERNAME` - username to use when signing into GitLab
+* `GITLAB_PASSWORD` - password to use when signing into GitLab
+* `GITLAB_USER_TYPE` - type of user to use when signing into GitLab: standard (default), ldap
+* `GITLAB_LDAP_USERNAME` - LDAP username to use when signing into GitLab
+* `GITLAB_LDAP_PASSWORD` - LDAP password to use when signing into GitLab
 * `GITLAB_SANDBOX_NAME` - The sandbox group name test suite is going to use (default: `gitlab-qa-sandbox`)
 * `EE_LICENSE` - Enterprise Edition license
 * `QA_SCREENSHOTS_DIR` - Path to a directory where screenshots for failing tests

fixtures/ldap/tanuki.ldif

+dn: uid=tanuki,dc=example,dc=org
+changetype: add
+uid: tanuki
+cn: tanuki
+sn: 3
+objectClass: top
+objectClass: posixAccount
+objectClass: inetOrgPerson
+loginShell: /bin/bash
+homeDirectory: /home/tanuki
+uidNumber: 14583102
+gidNumber: 14564100
+# hashed value for 'password'
+userPassword: {SSHA}ICMhr6Jxt5bk2awD7HL7GxRTM3BZ1pFI
+mail: tanuki@example.org
+gecos: Tanuki User

lib/gitlab/qa.rb

@@ -27,6 +27,7 @@ module Gitlab
         module Integration
           autoload :Geo, 'qa/scenario/test/integration/geo'
+          autoload :LDAP, 'qa/scenario/test/integration/ldap'
           autoload :Mattermost, 'qa/scenario/test/integration/mattermost'
         end
@@ -38,6 +39,7 @@ module Gitlab
     module Component
       autoload :Gitlab, 'qa/component/gitlab'
+      autoload :LDAP, 'qa/component/ldap'
       autoload :Specs, 'qa/component/specs'
       autoload :Staging, 'qa/component/staging'
     end

lib/gitlab/qa/component/ldap.rb

+require 'securerandom'
+
+# This component sets up the docker-openldap (https://github.com/osixia/docker-openldap)
+# image with the proper configuration for GitLab users to login.
+#
+# By default, the docker-openldap image configures the Docker image with a
+# default admin user in the example.org domain. This user does not have a uid
+# attribute that GitLab needs to authenticate, so we seed the LDAP server with
+# a "tanuki" user via a LDIF file in the fixtures/ldap directory.
+#
+# The docker-openldap container has a startup script
+# (https://github.com/osixia/docker-openldap/blob/v1.1.11/image/service/slapd/startup.sh#L74-L78)
+# that looks for custom LDIF files in the BOOTSTRAP_LDIF directory. Note that the LDIF
+# files must have a "changetype" option specified for the script to work.
+module Gitlab
+  module QA
+    module Component
+      class LDAP
+        include Scenario::Actable
+
+        LDAP_IMAGE = 'osixia/openldap'.freeze
+        LDAP_IMAGE_TAG = 'latest'.freeze
+        LDAP_DOMAIN = 'example.org'.freeze
+        ADMIN_USER = 'admin'.freeze
+        ADMIN_PASSWORD = 'admin'.freeze
+        LDAP_USER = 'tanuki'.freeze
+        LDAP_PASSWORD = 'password'.freeze
+        LDAP_PORT = 389
+        BASE_DN = 'dc=example,dc=org'.freeze
+        BIND_DN = 'cn=admin,dc=example,dc=org'.freeze
+        BOOTSTRAP_LDIF = '/container/service/slapd/assets/config/bootstrap/ldif/custom'.freeze
+        GROUP_BASE = 'ou=groups,dc=example,dc=org'.freeze
+        ADMIN_GROUP = 'admin'.freeze
+        FIXTURE_PATH = File.expand_path('../../../../fixtures/ldap'.freeze, __dir__)
+
+        attr_reader :docker
+        attr_accessor :volumes, :network, :environment
+        attr_writer :name
+
+        def initialize
+          @docker = Docker::Engine.new
+          @environment = {}
+          @volumes = {}
+          @network_aliases = []
+
+          @volumes[FIXTURE_PATH] = BOOTSTRAP_LDIF
+        end
+
+        def enable_tls(status)
+          @environment['LDAP_TLS'] = 'false' unless status
+        end
+
+        def username
+          LDAP_USER
+        end
+
+        def password
+          LDAP_PASSWORD
+        end
+
+        def add_network_alias(name)
+          @network_aliases.push(name)
+        end
+
+        def name
+          @name ||= "openldap-#{SecureRandom.hex(4)}"
+        end
+
+        def hostname
+          "#{name}.#{network}"
+        end
+
+        def instance
+          raise 'Please provide a block!' unless block_given?
+
+          prepare
+          start
+
+          yield self
+
+          teardown
+        end
+
+        def prepare
+          @docker.pull(LDAP_IMAGE, LDAP_IMAGE_TAG)
+
+          return if @docker.network_exists?(network)
+
+          @docker.network_create(network)
+        end
+
+        def start
+          # copy-service needed for bootstraping LDAP user:
+          # https://github.com/osixia/docker-openldap#seed-ldap-database-with-ldif
+          docker.run(LDAP_IMAGE, LDAP_IMAGE_TAG, '--copy-service') do |command|
+            command << '-d '
+            command << "--name #{name}"
+            command << "--net #{network}"
+            command << "--hostname #{hostname}"
+
+            @volumes.to_h.each do |to, from|
+              command.volume(to, from, 'Z')
+            end
+
+            @environment.to_h.each do |key, value|
+              command.env(key, value)
+            end
+
+            @network_aliases.to_a.each do |network_alias|
+              command << "--network-alias #{network_alias}"
+            end
+          end
+        end
+
+        def restart
+          @docker.restart(name)
+        end
+
+        def teardown
+          raise 'Invalid instance name!' unless name
+
+          @docker.stop(name)
+          @docker.remove(name)
+        end
+
+        def pull
+          @docker.pull(LDAP_IMAGE, LDAP_IMAGE_TAG)
+        end
+
+        def to_config
+          config = YAML.safe_load <<~CFG
+            main:
+              label: LDAP
+              host: #{hostname}
+              port: #{LDAP_PORT}
+              uid: 'uid'
+              bind_dn: #{BIND_DN}
+              password: #{ADMIN_PASSWORD}
+              method: 'plain'
+              base: #{BASE_DN}
+              user_filter: ''
+              group_base: #{GROUP_BASE}
+              admin_group: #{ADMIN_GROUP}
+              external_groups: ''
+              sync_ssh_keys: false
+          CFG
+
+          # Quotes get eaten up when the string is set in the environment
+          config.to_s.gsub("\"", "\\\"")
+        end
+
+        def set_gitlab_credentials
+          ::Gitlab::QA::Runtime::Env.ldap_username = username
+          ::Gitlab::QA::Runtime::Env.ldap_password = password
+        end
+      end
+    end
+  end
+end

lib/gitlab/qa/component/specs.rb

@@ -20,8 +20,9 @@ module Gitlab
           @docker.run(release.qa_image, release.tag, suite, *args) do |command|
             command << "-t --rm --net=#{network || 'bridge'}"
-            Runtime::Env.delegated.each do |env|
-              command.env(env, "$#{env}")
+            variables = Runtime::Env.variables
+            variables.each do |key, value|
+              command.env(key, value)
             end
             command.volume('/var/run/docker.sock', '/var/run/docker.sock')

lib/gitlab/qa/runtime/env.rb

@@ -4,11 +4,20 @@ module Gitlab
       module Env
         extend self
-        VARIABLES = %w[GITLAB_USERNAME
-                       GITLAB_PASSWORD
-                       GITLAB_SANDBOX_NAME
-                       GITLAB_URL
-                       EE_LICENSE].freeze
+        ENV_VARIABLES = {
+          'GITLAB_USERNAME' => :user_username,
+          'GITLAB_PASSWORD' => :user_password,
+          'GITLAB_LDAP_USERNAME' => :ldap_username,
+          'GITLAB_LDAP_PASSWORD' => :ldap_password,
+          'GITLAB_USER_TYPE' => :user_type,
+          'GITLAB_SANDBOX_NAME' => :gitlab_sandbox_name,
+          'GITLAB_URL' => :gitlab_url,
+          'EE_LICENSE' => :ee_license
+        }.freeze
+
+        ENV_VARIABLES.each_value do |accessor|
+          send(:attr_accessor, accessor) # rubocop:disable GitlabSecurity/PublicSend
+        end
         def screenshots_dir
           ENV['QA_SCREENSHOTS_DIR'] || '/tmp/gitlab-qa/screenshots'
@@ -18,8 +27,17 @@ module Gitlab
           ENV['QA_LOGS_DIR'] || '/tmp/gitlab-qa/logs'
         end
-        def delegated
-          VARIABLES.select { |name| ENV[name] }
+        def variables
+          vars = {}
+
+          ENV_VARIABLES.each do |name, attribute|
+            # Variables that are overriden in the environment take precedence
+            # over the defaults specified by the QA runtime.
+            value = env_value_if_defined(name) || send(attribute) # rubocop:disable GitlabSecurity/PublicSend
+            vars[name] = value if value
+          end
+
+          vars
         end
         def require_license!
@@ -27,6 +45,13 @@ module Gitlab
           raise ArgumentError, 'GitLab License is not available. Please load a license into EE_LICENSE env variable.'
         end
+
+        private
+
+        def env_value_if_defined(variable)
+          # Pass through the variables if they are defined in the environment
+          return "$#{variable}" if ENV[variable]
+        end
       end
     end
   end

lib/gitlab/qa/scenario/test/integration/ldap.rb

+require 'yaml'
+
+module Gitlab
+  module QA
+    module Scenario
+      module Test
+        module Integration
+          class LDAP < Scenario::Template
+            # rubocop:disable Metrics/AbcSize
+            def perform(release)
+              Component::Gitlab.perform do |gitlab|
+                gitlab.release = release
+                gitlab.name = 'gitlab-ldap'
+                gitlab.network = 'test'
+
+                Component::LDAP.perform do |ldap|
+                  ldap.enable_tls(false)
+                  ldap.network = 'test'
+                  ldap.set_gitlab_credentials
+
+                  gitlab.omnibus_config = <<~OMNIBUS
+                    gitlab_rails['ldap_enabled'] = true;
+                    gitlab_rails['ldap_servers'] = #{ldap.to_config};
+                  OMNIBUS
+
+                  ldap.instance do
+                    gitlab.instance do
+                      puts 'Running LDAP specs!'
+
+                      Component::Specs.perform do |specs|
+                        specs.suite = 'Test::Integration::LDAP'
+                        specs.release = gitlab.release
+                        specs.network = gitlab.network
+                        specs.args = [gitlab.address]
+                      end
+                    end
+                  end
+                end
+              end
+            end
+            # rubocop:enable Metrics/AbcSize
+          end
+        end
+      end
+    end
+  end
+end

spec/runtime/env_spec.rb

@@ -45,18 +45,36 @@ describe Gitlab::QA::Runtime::Env do
     end
   end
-  describe '.delegated' do
+  describe '.variables' do
     before do
-      stub_env('GITLAB_USERNAME', 'root')
+      stub_env_values({ 'GITLAB_USERNAME' => 'root',
+                        'EE_LICENSE' => nil })
+      described_class.user_username = nil
+      described_class.user_password = nil
+      described_class.user_type = nil
+      described_class.gitlab_url = nil
+      described_class.ee_license = nil
     end
-    it 'returns a list of envs delegated to tests component' do
-      expect(described_class.delegated).not_to be_empty
+    it 'returns only these delegated variables that are set' do
+      expect(described_class.variables).to eq({ 'GITLAB_USERNAME' => '$GITLAB_USERNAME' })
     end
-    it 'returns only these delegated variables that are set' do
-      expect(described_class.delegated).to include('GITLAB_USERNAME')
-      expect(described_class.delegated).not_to include('GITLAB_PASSWORD')
+    it 'prefers environment variables to defined values' do
+      described_class.user_username = 'tanuki'
+
+      expect(described_class.variables).to eq({ 'GITLAB_USERNAME' => '$GITLAB_USERNAME' })
+    end
+
+    it 'returns values that have been overriden' do
+      described_class.user_password = 'tanuki'
+      described_class.user_type = 'ldap'
+      described_class.gitlab_url = 'http://localhost:9999'
+
+      expect(described_class.variables).to eq({ 'GITLAB_USERNAME' => '$GITLAB_USERNAME',
+                                                'GITLAB_PASSWORD' => 'tanuki',
+                                                'GITLAB_USER_TYPE' => 'ldap',
+                                                'GITLAB_URL' => 'http://localhost:9999' })
     end
   end
@@ -64,4 +82,12 @@ describe Gitlab::QA::Runtime::Env do
     allow(ENV).to receive(:[]).and_call_original
     allow(ENV).to receive(:[]).with(name).and_return(value)
   end
+
+  def stub_env_values(pairs)
+    allow(ENV).to receive(:[]).and_call_original
+
+    pairs.each do |key, value|
+      allow(ENV).to receive(:[]).with(key).and_return(value)
+    end
+  end
 end