Add support for testing the cookbook before building the package.

.gitignore

@@ -15,3 +15,4 @@ Vagrantfile
 .idea
 *.log
 docker/RELEASE
+.kitchen/*

.kitchen.yml

+---
+driver:
+  name: docker
+
+driver_config:
+  privileged: true
+
+provisioner:
+  name: chef_solo
+platforms:
+  - name: ubuntu-14.04
+  - name: centos-7.1
+#  - name: ubuntu-12.04  Docker image for Ubuntu 12.04 fails on provision. Ignore for now.
+  - name: centos-6.4
+  driver_config:
+      provision_command: ln -sf /bin/true /sbin/initctl
+  - name: debian-8
+  - name: debian-7
+
+suites:
+  - name: default
+    run_list:
+      - recipe[kitchen-prepare]
+      - recipe[gitlab]
+    attributes:
+      package:
+        install-dir: '/opt/gitlab'

Berksfile

-site :opscode
-cookbook "omnibus"
+# This Berksfile is used for test-kitchen only
+source 'https://api.berkshelf.com'
+
+cookbook 'gitlab', path: 'files/gitlab-cookbooks/gitlab'
+cookbook 'runit', path: 'files/gitlab-cookbooks/runit'
+cookbook 'kitchen-prepare', path: 'test/cookbooks/kitchen-prepare'
+cookbook 'package', path: 'test/cookbooks/package'

Berksfile.lock

+DEPENDENCIES
+  gitlab
+    path: files/gitlab-cookbooks/gitlab
+  kitchen-prepare
+    path: test/cookbooks/kitchen-prepare
+  package
+    path: test/cookbooks/package
+  runit
+    path: files/gitlab-cookbooks/runit
+
+GRAPH
+  gitlab (0.0.1)
+    package (>= 0.0.0)
+    runit (>= 0.0.0)
+  kitchen-prepare (0.1.0)
+  package (0.1.0)
+  runit (0.14.2)
+    package (>= 0.0.0)

Gemfile

@@ -7,3 +7,10 @@ gem 'package_cloud'
 gem 'thor', '0.18.1' # This specific version is required by package_cloud
 gem 'json'
 gem 'rspec'
+
+# Cookbook testing suite
+gem 'test-kitchen'
+gem 'kitchen-docker'
+# Lock to preven issues starting berkshelf https://github.com/reset/varia_model/issues/9
+gem "hashie", ">= 2.0.2", "< 3.0.0"
+gem "berkshelf"

Gemfile.lock

@@ -23,34 +23,89 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
+    addressable (2.3.8)
     aws-sdk (2.2.3)
       aws-sdk-resources (= 2.2.3)
     aws-sdk-core (2.2.3)
       jmespath (~> 1.0)
     aws-sdk-resources (2.2.3)
       aws-sdk-core (= 2.2.3)
+    berkshelf (3.1.2)
+      addressable (~> 2.3.4)
+      berkshelf-api-client (~> 1.2)
+      buff-config (~> 0.4)
+      buff-extensions (~> 0.4)
+      buff-shell_out (~> 0.1)
+      celluloid (~> 0.16.0.pre)
+      celluloid-io (~> 0.16.0.pre)
+      faraday (~> 0.9.0)
+      minitar (~> 0.5.4)
+      octokit (~> 3.0)
+      retryable (~> 1.3.3)
+      ridley (~> 3.1)
+      solve (~> 1.1)
+      thor (~> 0.18)
+    berkshelf-api-client (1.3.1)
+      faraday (~> 0.9.1)
+      httpclient (~> 2.6.0)
+    buff-config (0.4.0)
+      buff-extensions (~> 0.3)
+      varia_model (~> 0.1)
+    buff-extensions (0.5.0)
+    buff-ignore (1.1.1)
+    buff-ruby_engine (0.1.0)
+    buff-shell_out (0.2.0)
+      buff-ruby_engine (~> 0.1.0)
+    celluloid (0.16.0)
+      timers (~> 4.0.0)
+    celluloid-io (0.16.2)
+      celluloid (>= 0.16.0)
+      nio4r (>= 1.1.0)
     chef-config (12.5.1)
       mixlib-config (~> 2.0)
       mixlib-shellout (~> 2.0)
     chef-sugar (3.1.1)
     cleanroom (1.0.0)
     colorize (0.6.0)
+    dep-selector-libgecode (1.0.2)
+    dep_selector (1.0.3)
+      dep-selector-libgecode (~> 1.0)
+      ffi (~> 1.9)
     diff-lcs (1.2.5)
+    erubis (2.7.0)
+    faraday (0.9.2)
+      multipart-post (>= 1.2, < 3)
     ffi (1.9.10)
     ffi-yajl (2.2.2)
       libyajl2 (~> 1.2)
+    hashie (2.1.2)
     highline (1.6.20)
+    hitimes (1.2.3)
+    httpclient (2.6.0.1)
     ipaddress (0.8.0)
     jmespath (1.1.3)
     json (1.8.2)
     json_pure (1.8.1)
+    kitchen-docker (2.3.0)
+      test-kitchen (>= 1.0.0)
     libyajl2 (1.2.0)
     mime-types (2.99)
+    minitar (0.5.4)
+    mixlib-authentication (1.3.0)
+      mixlib-log
     mixlib-cli (1.5.0)
     mixlib-config (2.2.1)
     mixlib-log (1.6.0)
     mixlib-shellout (2.2.3)
     mixlib-versioning (1.1.0)
+    multipart-post (2.0.0)
+    net-http-persistent (2.9.4)
+    net-scp (1.2.1)
+      net-ssh (>= 2.6.5)
+    net-ssh (2.9.2)
+    nio4r (1.2.0)
+    octokit (3.8.0)
+      sawyer (~> 0.6.0, >= 0.5.3)
     ohai (8.7.0)
       chef-config (>= 12.5.0.alpha.1, < 13)
       ffi (~> 1.9)
@@ -73,6 +128,24 @@ GEM
     rake (10.4.2)
     rest-client (1.6.7)
       mime-types (>= 1.16)
+    retryable (1.3.6)
+    ridley (3.1.0)
+      addressable
+      buff-config (~> 0.2)
+      buff-extensions (~> 0.3)
+      buff-ignore (~> 1.1)
+      buff-shell_out (~> 0.1)
+      celluloid (~> 0.16.0.pre)
+      celluloid-io (~> 0.16.0.pre)
+      erubis
+      faraday (~> 0.9.0)
+      hashie (>= 2.0.2)
+      json (>= 1.7.7)
+      mixlib-authentication (>= 1.3.0)
+      net-http-persistent (>= 2.8)
+      retryable
+      semverse (~> 1.1)
+      varia_model (~> 0.3)
     rspec (3.2.0)
       rspec-core (~> 3.2.0)
       rspec-expectations (~> 3.2.0)
@@ -87,18 +160,41 @@ GEM
       rspec-support (~> 3.2.0)
     rspec-support (3.2.2)
     ruby-progressbar (1.7.5)
+    safe_yaml (1.0.4)
+    sawyer (0.6.0)
+      addressable (~> 2.3.5)
+      faraday (~> 0.8, < 0.10)
+    semverse (1.2.1)
+    solve (1.2.1)
+      dep_selector (~> 1.0)
+      semverse (~> 1.1)
     systemu (2.6.5)
+    test-kitchen (1.4.2)
+      mixlib-shellout (>= 1.2, < 3.0)
+      net-scp (~> 1.1)
+      net-ssh (~> 2.7, < 2.10)
+      safe_yaml (~> 1.0)
+      thor (~> 0.18)
     thor (0.18.1)
+    timers (4.0.4)
+      hitimes
+    varia_model (0.3.2)
+      buff-extensions (~> 0.2)
+      hashie (>= 2.0.2)
     wmi-lite (1.0.0)
 PLATFORMS
   ruby
 DEPENDENCIES
+  berkshelf
+  hashie (>= 2.0.2, < 3.0.0)
   json
+  kitchen-docker
   ohai
   omnibus!
   omnibus-software!
   package_cloud
   rspec
+  test-kitchen
   thor (= 0.18.1)

Makefile

@@ -6,6 +6,7 @@ PLATFORM_DIR:=$(shell bundle exec support/ohai-helper platform-dir)
 PACKAGECLOUD_USER=gitlab
 PACKAGECLOUD_REPO:=$(shell support/repo_name.sh)
 PACKAGECLOUD_OS:=$(shell bundle exec support/ohai-helper repo-string)
+KITCHEN_TEST:=$(shell bundle exec support/ohai-helper kitchen-test)
 ifeq ($(shell support/is_gitlab_ee.sh; echo $$?), 0)
 RELEASE_PACKAGE=gitlab-ee
 else
@@ -33,7 +34,7 @@ do_release: no_changes on_tag purge build move_to_platform_dir sync packagecloud
 # Redefine RELEASE_BUCKET for test builds
 test: RELEASE_BUCKET=omnibus-builds
-test: no_changes purge test_build move_to_platform_dir sync
+test: no_changes purge test_cookbook test_build move_to_platform_dir sync
 ifdef NIGHTLY
 test: PACKAGECLOUD_REPO=nightly-builds
 test: packagecloud
@@ -63,6 +64,9 @@ purge:
 	mkdir -p pkg
 	(cd pkg && find . -delete)
+test_cookbook:
+	kitchen test ${KITCHEN_TEST}
+
 # Instead of pkg/gitlab-xxx.deb, put all files in pkg/ubuntu/gitlab-xxx.deb
 move_to_platform_dir:
 	mv pkg ${PLATFORM_DIR}
@@ -104,7 +108,7 @@ ifeq (,$(findstring rc,$(RELEASE_VERSION)))
 	docker push gitlab/$(RELEASE_PACKAGE):latest
 endif
-do_docker_master: 
+do_docker_master:
 ifdef NIGHTLY
 do_docker_master: docker_build docker_push
 endif

support/ohai-helper

@@ -51,11 +51,21 @@ def ohai
   end.data
 end
+def kitchen_instance
+  instance_name = [ "default",
+                    ohai['platform'],
+                    ohai['platform_version'].gsub(/\..*/, '')
+                  ].join("-")
+  puts instance_name
+end
+
 case ARGV.first
 when 'platform-dir'
   platform_dir
 when 'repo-string'
   repo_string
+when 'kitchen-test'
+  kitchen_instance
 else
   abort "Usage: #{$0} platform-dir|repo-string"
 end

test/cookbooks/kitchen-prepare/CHANGELOG.md

+kitchen-prepare CHANGELOG
+=========================
+
+0.1.0
+-----
+- [Marin Jankovski] - Initial release of kitchen-prepare
+
+- - -
+Check the [Markdown Syntax Guide](http://daringfireball.net/projects/markdown/syntax) for help with Markdown.

test/cookbooks/kitchen-prepare/README.md

+kitchen-prepare Cookbook
+========================
+
+This cookbook is used to prepare the environment so that test-kitchen can converge omnibus-gitlab internal cookbook.
+
+License and Authors
+-------------------
+License: MIT
+Authors: Marin Jankovski

test/cookbooks/kitchen-prepare/files/default/gitlab.rb

+external_url "http://gitlab.example.com"

test/cookbooks/kitchen-prepare/metadata.rb

+name             'kitchen-prepare'
+maintainer       'Marin Jankovski'
+maintainer_email 'marin@gitlab.com'
+license          'All rights reserved'
+description      'Installs/Configures kitchen-prepare'
+long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
+version          '0.1.0'

test/cookbooks/kitchen-prepare/recipes/default.rb

+#
+# Cookbook Name:: kitchen-prepare
+# Recipe:: default
+#
+# Copyright 2016, GitLab B.V.
+#
+# All rights reserved - Do Not Redistribute
+#
+package 'git'
+
+if node['platform'] =~ /centos/
+  [
+    'selinux-policy-devel',
+    'cronie'
+  ].each do |pkg|
+    package pkg
+  end
+else
+  package 'cron'
+end
+
+# Stub directories that will be created by installing the package
+[
+  "/etc/gitlab",
+  "/opt/gitlab/embedded/service/",
+  "/opt/gitlab/embedded/bin/",
+  "/opt/gitlab/sv",
+  "/opt/gitlab/init/",
+  "/opt/gitlab/bin/",
+  "/opt/gitlab/service",
+  "/var/opt/gitlab/postgresql/data"
+].each do |dir|
+  directory dir do
+    recursive true
+  end
+end
+
+# Stub services which will be called during cookbook run
+%w( redis postgresql sidekiq unicorn gitlab-workhorse nginx logrotate svlogd ).each do |service|
+  [
+    "/opt/gitlab/sv/#{service}/supervise",
+    "/opt/gitlab/sv/#{service}/log/supervise/"
+  ].each do |dir|
+    directory dir do
+      recursive true
+    end
+  end
+
+  execute "create a named pipe for #{service}" do
+    command "mkfifo /opt/gitlab/sv/#{service}/supervise/ok"
+    not_if { File.exists?("/opt/gitlab/sv/#{service}/supervise/ok") }
+  end
+end
+
+# Stub files that will be checked/called during the cookbook run
+# and assume they work
+[
+  "/var/opt/gitlab/postgresql/data/PG_VERSION",
+  "/opt/gitlab/bin/gitlab-rake",
+  "/opt/gitlab/bin/gitlab-ctl",
+  "/opt/gitlab/embedded/bin/chpst"
+].each do |file|
+  file file do
+    mode 0755
+    content "exit 0"
+  end
+end
+
+# Supply minimal gitlab.rb file
+cookbook_file "/etc/gitlab/gitlab.rb"
+
+# Clone the required projects
+git "clone gitlab-rails" do
+  destination "/opt/gitlab/embedded/service/gitlab-rails"
+  repository "https://github.com/gitlabhq/gitlabhq.git"
+  enable_checkout true # checkout master
+end
+
+git "clone gitlab-shell" do
+  destination "/opt/gitlab/embedded/service/gitlab-shell"
+  repository "https://github.com/gitlabhq/gitlab-shell.git"
+  enable_checkout true # checkout master
+end
+
+git "clone mattermost" do
+  destination "/opt/gitlab/embedded/service/mattermost"
+  repository "https://github.com/mattermost/platform.git"
+  enable_checkout true # checkout master
+end
+
+# Remove directories which are deleted during build
+[
+  "/opt/gitlab/embedded/service/gitlab-rails/tmp",
+  "/opt/gitlab/embedded/service/gitlab-rails/log"
+].each do |dir|
+  directory dir do
+    recursive true
+    action :delete
+  end
+end
+
+file "/etc/inittab" do
+  content "'CS:123456:respawn:/opt/gitlab/embedded/bin/runsvdir-start'"
+end

test/cookbooks/package/CHANGELOG.md

+package CHANGELOG
+=================
+
+This file is used to list changes made in each version of the package cookbook.
+
+0.1.0
+-----
+- [your_name] - Initial release of package
+
+- - -
+Check the [Markdown Syntax Guide](http://daringfireball.net/projects/markdown/syntax) for help with Markdown.
+
+The [Github Flavored Markdown page](http://github.github.com/github-flavored-markdown/) describes the differences between markdown on github and standard markdown.

test/cookbooks/package/README.md

+package Cookbook
+================
+TODO: Enter the cookbook description here.
+
+e.g.
+This cookbook makes your favorite breakfast sandwich.
+
+Requirements
+------------
+TODO: List your cookbook requirements. Be sure to include any requirements this cookbook has on platforms, libraries, other cookbooks, packages, operating systems, etc.
+
+e.g.
+#### packages
+- `toaster` - package needs toaster to brown your bagel.
+
+Attributes
+----------
+TODO: List your cookbook attributes here.
+
+e.g.
+#### package::default
+<table>
+  <tr>
+    <th>Key</th>
+    <th>Type</th>
+    <th>Description</th>
+    <th>Default</th>
+  </tr>
+  <tr>
+    <td><tt>['package']['bacon']</tt></td>
+    <td>Boolean</td>
+    <td>whether to include bacon</td>
+    <td><tt>true</tt></td>
+  </tr>
+</table>
+
+Usage
+-----
+#### package::default
+TODO: Write usage instructions for each cookbook.
+
+e.g.
+Just include `package` in your node's `run_list`:
+
+```json
+{
+  "name":"my_node",
+  "run_list": [
+    "recipe[package]"
+  ]
+}
+```
+
+Contributing
+------------
+TODO: (optional) If this is a public cookbook, detail the process for contributing. If this is a private cookbook, remove this section.
+
+e.g.
+1. Fork the repository on Github
+2. Create a named feature branch (like `add_component_x`)
+3. Write your change
+4. Write tests for your change (if applicable)
+5. Run the tests, ensuring they all pass
+6. Submit a Pull Request using Github
+
+License and Authors
+-------------------
+Authors: TODO: List authors

test/cookbooks/package/metadata.rb

+name             'package'
+maintainer       'YOUR_COMPANY_NAME'
+maintainer_email 'YOUR_EMAIL'
+license          'All rights reserved'
+description      'Installs/Configures package'
+long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
+version          '0.1.0'

test/cookbooks/package/recipes/default.rb

+#
+# Cookbook Name:: package
+# Recipe:: default
+#
+# Copyright 2016, YOUR_COMPANY_NAME
+#
+# All rights reserved - Do Not Redistribute
+#