Add CI sections to most Rake tasks

lib/gitlab/tasks/build.rake

@@ -11,7 +11,10 @@ require 'json'
 namespace :build do
   desc 'Start project build'
   task project: ["cache:purge", "check:no_changes"] do
+    Gitlab::Util.section_start('build:project')
     Build.exec('gitlab') || raise('Build failed')
+    Gitlab::Util.section_end
+
     Rake::Task["license:check"].invoke
     Rake::Task["build:package:move_to_platform_dir"].invoke
     Rake::Task["build:package:generate_checksums"].invoke
@@ -41,15 +44,20 @@ namespace :build do
     desc "Generate checksums for each file"
     task :generate_checksums do
-      files = Dir.glob('pkg/**/*.{deb,rpm}').select { |f| File.file? f }
+      Gitlab::Util.section_start('build:package:generate_checksums')
+      files = Dir.glob('pkg/**/*.{deb,rpm}').select { |f| File.file? f }
       files.each do |file|
         system('sha256sum', file, out: "#{file}.sha256")
       end
+
+      Gitlab::Util.section_end
     end
     desc "Sync packages to aws"
     task :sync do
+      Gitlab::Util.section_start('build:package:sync')
+
       release_bucket = Build::Info.release_bucket
       release_bucket_region = "eu-west-1"
       system(*%W[aws s3 sync pkg/ s3://#{release_bucket} --acl public-read --region #{release_bucket_region}])
@@ -57,6 +65,8 @@ namespace :build do
       files.each do |file|
         puts file.gsub('pkg', "https://#{release_bucket}.s3.amazonaws.com").gsub('+', '%2B')
       end
+
+      Gitlab::Util.section_end
     end
   end
@@ -69,7 +79,9 @@ namespace :build do
     Gitlab::Util.set_env_if_missing('TOP_UPSTREAM_SOURCE_SHA', Gitlab::Util.get_env('CI_COMMIT_SHA'))
     Gitlab::Util.set_env_if_missing('TOP_UPSTREAM_SOURCE_REF', Gitlab::Util.get_env('CI_COMMIT_REF_NAME'))
+    Gitlab::Util.section_start('build:trigger')
     Build::OmnibusTrigger.invoke!(post_comment: true).wait!
+    Gitlab::Util.section_end
   end
   desc 'Print the current version'
@@ -83,10 +95,14 @@ namespace :build do
     version_manifest_file = Dir.glob('pkg/**/*version-manifest.json').first
     return unless version_manifest_file
+    Gitlab::Util.section_start('build:component_shas')
+
     puts "#### SHAs of GitLab Components"
     json_content = JSON.parse(File.read(version_manifest_file))
     %w[gitlab-rails gitaly gitlab-pages gitlab-shell gitlab-workhorse].each do |component|
       puts "#{component} : #{json_content['software'][component]['locked_version']}"
     end
+
+    Gitlab::Util.section_end
   end
 end

lib/gitlab/tasks/docker_tasks.rake

@@ -10,6 +10,8 @@ namespace :docker do
   namespace :build do
     desc "Build Docker All in one image"
     task :image do
+      Gitlab::Util.section_start('docker:build:image')
+
       Build::GitlabImage.write_release_file
       location = File.absolute_path(File.join(File.dirname(File.expand_path(__FILE__)), "../../../docker"))
       DockerOperations.build(
@@ -17,54 +19,76 @@ namespace :docker do
         Build::GitlabImage.gitlab_registry_image_address,
         'latest'
       )
+
+      Gitlab::Util.section_end
     end
   end
   task :measure_memory do
+    Gitlab::Util.section_start('docker:measure_memory')
+
     image_reference = Gitlab::Util.get_env('IMAGE_REFERENCE') || Build::Info.image_reference
     debug_output_dir = Gitlab::Util.get_env('DEBUG_OUTPUT_DIR')
     docker_image_memory_measurer = Gitlab::DockerImageMemoryMeasurer.new(image_reference, debug_output_dir)
     puts docker_image_memory_measurer.measure
+
+    Gitlab::Util.section_end
   end
   desc "Push Docker Image to Registry"
   namespace :push do
     # Only runs on dev.gitlab.org
     task :staging do
+      Gitlab::Util.section_start('docker:push:staging')
       Build::GitlabImage.tag_and_push_to_gitlab_registry(Build::Info.docker_tag)
+      Gitlab::Util.section_end
     end
     task :stable do
+      Gitlab::Util.section_start('docker:push:stable')
       Build::GitlabImage.tag_and_push_to_dockerhub(Build::Info.docker_tag)
+      Gitlab::Util.section_end
     end
     # Special tags
     task :nightly do
+      Gitlab::Util.section_start('docker:push:nightly')
       Build::GitlabImage.tag_and_push_to_dockerhub('nightly') if Build::Check.is_nightly?
+      Gitlab::Util.section_end
     end
     # push as :rc tag, the :rc is always the latest tagged release
     task :rc do
+      Gitlab::Util.section_start('docker:push:rc')
       Build::GitlabImage.tag_and_push_to_dockerhub('rc') if Build::Check.is_latest_tag?
+      Gitlab::Util.section_end
     end
     # push as :latest tag, the :latest is always the latest stable release
     task :latest do
+      Gitlab::Util.section_start('docker:push:latest')
       Build::GitlabImage.tag_and_push_to_dockerhub('latest') if Build::Check.is_latest_stable_tag?
+      Gitlab::Util.section_end
     end
     desc "Push triggered Docker Image to GitLab Registry"
     task :triggered do
+      Gitlab::Util.section_start('docker:push:triggered')
       Build::GitlabImage.tag_and_push_to_gitlab_registry(Build::Info.docker_tag)
+      Gitlab::Util.section_end
     end
   end
   desc "Pull Docker Image from Registry"
   namespace :pull do
     task :staging do
+      Gitlab::Util.section_start('docker:pull:staging')
+
       DockerOperations.authenticate("gitlab-ci-token", Gitlab::Util.get_env("CI_JOB_TOKEN"), Gitlab::Util.get_env('CI_REGISTRY'))
       Build::GitlabImage.pull
+
+      Gitlab::Util.section_end
     end
   end
 end

lib/gitlab/tasks/license.rake

@@ -5,10 +5,13 @@ require_relative "../build/check.rb"
 require_relative '../license/analyzer.rb'
 require_relative '../license/uploader.rb'
 require_relative '../license/collector.rb'
+require_relative "../util.rb"
 namespace :license do
   desc "Check licenses of bundled softwares"
   task :check do
+    Gitlab::Util.section_start('license:check')
+
     install_dir = File.open('config/projects/gitlab.rb').grep(/install_dir *'/)[0].match(/install_dir[ \t]*'(?<install_dir>.*)'/)['install_dir']
     raise StandardError, "Unable to retrieve install_dir, thus unable to check #{install_dir}/dependency_licenses.json" unless File.exist?(install_dir)
@@ -29,16 +32,24 @@ namespace :license do
       raise "Build Aborted due to license violations"
     end
     puts "###### END LICENSE CHECK ######"
+
+    Gitlab::Util.section_end
   end
   desc "Generate license file of current release and push to AWS bucket"
   task :upload do
+    Gitlab::Util.section_start('license:upload')
+
     # This is done on Ubuntu 18.04 non-rc tag pipeline only
     License::Uploader.new.execute unless Build::Check.is_rc_tag?
+
+    Gitlab::Util.section_end
   end
   desc "Collect all license files and generate index page"
   task :generate_pages do
+    Gitlab::Util.section_start('license:generate_pages')
     License::Collector.new.execute
+    Gitlab::Util.section_end
   end
 end

lib/gitlab/tasks/qa.rake

@@ -12,71 +12,101 @@ require_relative "../util.rb"
 namespace :qa do
   desc "Build QA Docker image"
   task :build do
+    Gitlab::Util.section_start('qa:build')
+
     DockerOperations.build(
       Build::QA.get_gitlab_repo,
       Build::QAImage.gitlab_registry_image_address,
       'latest',
       dockerfile: 'qa/Dockerfile'
     )
+
+    Gitlab::Util.section_end
   end
   namespace :push do
     # Only runs on dev.gitlab.org
     desc "Push unstable or auto-deploy version of gitlab-{ce,ee}-qa to the GitLab registry"
     task :staging do
+      Gitlab::Util.section_start('qa:push:staging')
+
       tag = Build::Check.is_auto_deploy? ? Build::Info.major_minor_version_and_rails_ref : Build::Info.gitlab_version
       Build::QAImage.tag_and_push_to_gitlab_registry(tag)
       Build::QAImage.tag_and_push_to_gitlab_registry(Build::Info.commit_sha)
+
+      Gitlab::Util.section_end
     end
     desc "Push stable version of gitlab-{ce,ee}-qa to the GitLab registry and Docker Hub"
     task :stable do
+      Gitlab::Util.section_start('qa:push:stable')
+
       # Allows to have gitlab/gitlab-{ce,ee}-qa:10.2.0-ee without the build number
       Build::QAImage.tag_and_push_to_gitlab_registry(Build::Info.gitlab_version)
       Build::QAImage.tag_and_push_to_dockerhub(Build::Info.gitlab_version, initial_tag: 'latest')
+
+      Gitlab::Util.section_end
     end
     desc "Push rc version of gitlab-{ce,ee}-qa to Docker Hub"
     task :rc do
+      Gitlab::Util.section_start('qa:push:rc')
       Build::QAImage.tag_and_push_to_dockerhub('rc', initial_tag: 'latest') if Build::Check.is_latest_tag?
+      Gitlab::Util.section_end
     end
     desc "Push nightly version of gitlab-{ce,ee}-qa to Docker Hub"
     task :nightly do
+      Gitlab::Util.section_start('qa:push:nightly')
       Build::QAImage.tag_and_push_to_dockerhub('nightly', initial_tag: 'latest') if Build::Check.is_nightly?
+      Gitlab::Util.section_end
     end
     desc "Push latest version of gitlab-{ce,ee}-qa to Docker Hub"
     task :latest do
+      Gitlab::Util.section_start('qa:push:latest')
       Build::QAImage.tag_and_push_to_dockerhub('latest', initial_tag: 'latest') if Build::Check.is_latest_stable_tag?
+      Gitlab::Util.section_end
     end
     desc "Push triggered version of gitlab-{ce,ee}-qa to the GitLab registry"
     task :triggered do
+      Gitlab::Util.section_start('qa:push:triggered')
       Build::QAImage.tag_and_push_to_gitlab_registry(Build::Info.docker_tag)
+      Gitlab::Util.section_end
     end
   end
   desc "Run QA tests"
   task :test do
+    Gitlab::Util.section_start('qa:test')
+
     image_address = Build::GitlabImage.gitlab_registry_image_address(tag: Build::Info.docker_tag)
     Build::QATrigger.invoke!(image: image_address, post_comment: true).wait!
+
+    Gitlab::Util.section_end
   end
   namespace :ha do
     desc "Validate HA setup"
     task :validate do
+      Gitlab::Util.section_start('qa:ha:validate')
       Build::HA::ValidateTrigger.invoke!.wait!(timeout: 3600 * 4)
+      Gitlab::Util.section_end
     end
     desc 'Validate nightly build'
     task :nightly do
+      Gitlab::Util.section_start('qa:ha:nightly')
       Build::HA::ValidateNightly.invoke!.wait!(timeout: 3600 * 4)
+      Gitlab::Util.section_end
     end
     desc 'Validate tagged build'
     task :tag do
+      Gitlab::Util.section_start('qa:ha:tag')
       Build::HA::ValidateTag.invoke!(timeout: 3600 * 4)
+      Gitlab::Util.section_end
     end
   end
 end

spec/gitlab/tasks/docker_tasks_spec.rb

@@ -39,7 +39,7 @@ RSpec.describe 'docker', type: :rake do
       expect(Gitlab::DockerImageMemoryMeasurer).to receive(:new).with('dev.gitlab.org:5005/gitlab/omnibus-gitlab', 'tmp/debug_folder').and_return(mock_measurer)
       expect(mock_measurer).to receive(:measure).and_return('mock_return')
-      expect { Rake::Task['docker:measure_memory'].invoke }.to output("mock_return\n").to_stdout
+      expect { Rake::Task['docker:measure_memory'].invoke }.to output(/.*mock_return\n.*/).to_stdout
     end
     it 'initialize DockerImageMemoryMeasurer with correct parameters when ENV IMAGE_REFERENCE set' do
@@ -49,7 +49,7 @@ RSpec.describe 'docker', type: :rake do
       expect(Gitlab::DockerImageMemoryMeasurer).to receive(:new).with('env_value_image_reference', 'tmp/debug_folder').and_return(mock_measurer)
       expect(mock_measurer).to receive(:measure).and_return('mock_return')
-      expect { Rake::Task['docker:measure_memory'].invoke }.to output("mock_return\n").to_stdout
+      expect { Rake::Task['docker:measure_memory'].invoke }.to output(/.*mock_return\n.*/).to_stdout
     end
   end

spec/gitlab/tasks/qa_spec.rb

@@ -71,6 +71,7 @@ RSpec.describe 'qa', type: :rake do
     end
     it 'pushes triggered images correctly' do
+      allow(ENV).to receive(:[]).with('CI').and_return('true')
       expect(ENV).to receive(:[]).with('IMAGE_TAG').and_return(image_tag)
       expect(Build::QAImage).to receive(:tag_and_push_to_gitlab_registry).with(image_tag)
@@ -95,6 +96,7 @@ RSpec.describe 'qa', type: :rake do
       end
       it 'pushes staging auto-deploy images correctly' do
+        allow(ENV).to receive(:[]).with('CI').and_return('true')
         allow(ENV).to receive(:[]).with('CI_COMMIT_TAG').and_return('12.0.12345+5159f2949cb.59c9fa631')
         allow(Build::Info).to receive(:current_git_tag).and_return('12.0.12345+5159f2949cb.59c9fa631')