Implement software whitelist/blacklist logic in license checking

lib/gitlab/license_analyzer.rb

+class LicenseAnalyzer
+  @license_acceptable = Regexp.union([/MIT/i, /LGPL/i, /Apache/i, /Ruby/i, /BSD/i,
+                                      /ISO/i, /ISC/i, /Public[- ]Domain/i,
+                                      /Unlicense/i, /Artistic/i, /MPL/i, /AFL/i,
+                                      /CC-BY-[0-9]*/, /^project_license$/, /OpenSSL/i,
+                                      /ZLib/i, /jemalloc/i, /Python/i, /PostgreSQL/i,
+                                      /Info-Zip/i])
+  # TODO: Re-confirm that licenses Python, Info-Zip, OpenSSL and CC-BY are
+  # OK to be shipped. https://gitlab.com/gitlab-org/omnibus-gitlab/issues/2448
+
+  @license_unacceptable = Regexp.union([/GPL/i, /AGPL/i])
+  @software_acceptable = [
+    'git',                # GPL Mere Aggregate Exception - https://www.gnu.org/licenses/gpl-faq.en.html#MereAggregation
+    'config_guess',       # GPL Mere Aggregate Exception - https://www.gnu.org/licenses/gpl-faq.en.html#MereAggregation
+    'pkg-config-lite',    # GPL Mere Aggregate Exception - https://www.gnu.org/licenses/gpl-faq.en.html#MereAggregation
+    'libtool',            # GPL Mere Aggregate Exception - https://www.gnu.org/licenses/gpl-faq.en.html#MereAggregation
+    'logrotate',          # GPL Mere Aggregate Exception - https://www.gnu.org/licenses/gpl-faq.en.html#MereAggregation
+    'rsync',              # GPL Mere Aggregate Exception - https://www.gnu.org/licenses/gpl-faq.en.html#MereAggregation
+    'mysql-client',       # GPL Mere Aggregate Exception - https://www.gnu.org/licenses/gpl-faq.en.html#MereAggregation
+    'blob',               # MIT Licensed - https://github.com/webmodules/blob/blob/master/LICENSE
+    'callsite',           # MIT Licensed - https://github.com/tj/callsite/blob/master/LICENSE
+    'component-bind',     # MIT Licensed - https://github.com/component/bind/blob/master/LICENSE
+    'component-inherit',  # MIT Licensed - https://github.com/component/inherit/blob/master/LICENSE
+    'domelementtype',     # BSD-2-Clause Licensed - https://github.com/fb55/domelementtype/blob/master/LICENSE
+    'domhandler',         # BSD-2-Clause Licensed - https://github.com/fb55/domhandler/blob/master/LICENSE
+    'domutils',           # BSD-2-Clause Licensed - https://github.com/fb55/domutils/blob/master/LICENSE
+    'fsevents',           # MIT Licensed - https://github.com/strongloop/fsevents/blob/master/LICENSE
+    'indexof',            # MIT Licensed - https://github.com/component/indexof/blob/master/LICENSE
+    'map-stream',         # MIT Licensed - https://github.com/dominictarr/map-stream/blob/master/LICENCE
+    'object-component',   # MIT Licensed - https://github.com/component/object/blob/master/LICENSE
+    'select2',            # MIT Licensed - https://github.com/select2/select2/blob/master/LICENSE.md
+  ]
+  # readline is GPL licensed and its use was not mere aggregation. Hence it is
+  # blacklisted.
+  # Details: https://gitlab.com/gitlab-org/omnibus-gitlab/issues/1945#note_29286329
+  @software_unacceptable = ['readline']
+
+  def self.software_check(dependency)
+    if @software_unacceptable.include?(dependency)
+      ['unacceptable', 'Blacklisted software']
+    elsif @software_acceptable.include?(dependency)
+      ['acceptable', 'Whitelisted software']
+    end
+  end
+
+  def self.license_check(license)
+    if license.match(@license_acceptable)
+      ['acceptable', 'Acceptable license']
+    elsif license.match(@license_unacceptable)
+      ['unacceptable', 'Unacceptable license']
+    end
+  end
+
+  def self.acceptable?(dependency, license)
+    # This method returns two values. First one is whether the software is
+    # acceptable or not. Second one is the reason for that decision. This
+    # information is relayed to the user for better transparency.
+
+    software_check_status = software_check(dependency)
+    return software_check_status if software_check_status # status is nil if software is unlisted
+
+    license_check_status = license_check(license)
+    return license_check_status if license_check_status # status is nil if license is unlisted
+
+    ['unacceptable', 'Unknown license']
+  end
+
+  def self.analyze(json_data)
+    violations = []
+    json_data.each do |dependency, attributes|
+      license = attributes['license'].strip.delete('"').delete("'")
+      version = attributes['version']
+      status, reason = acceptable?(dependency, license.strip)
+
+      case status
+      when 'acceptable'
+        puts "Acceptable   : #{dependency} - #{version} uses #{license} - #{reason}"
+      when 'unacceptable'
+        violations << "#{dependency} - #{version} - #{license} - #{reason}"
+        if reason == 'Blacklisted software'
+          puts "Unacceptable ! #{dependency} - #{version} uses #{license} - #{reason}"
+        elsif reason == 'Unknown license'
+          puts "Unknown      ? #{dependency} - #{version} uses #{license} - #{reason}"
+        end
+      end
+    end
+
+    violations
+  end
+end

lib/gitlab/tasks/license_check.rake

 require 'json'
+require_relative '../license_analyzer.rb'
 desc "Check licenses of bundled softwares"
 namespace :license do
   task :check do
-    good = Regexp.union([/^MIT/, /^LGPL/, /^Apache/, /^Ruby/, /^BSD-[23]{1}/, /^ISO/])
-    bad = Regexp.union([/^GPL/, /^AGPL/])
-
-    puts "###### BEGIN LICENSE CHECK ######"
-
     install_dir = File.open('config/projects/gitlab.rb').grep(/install_dir *'/)[0].match(/install_dir[ \t]*'(?<install_dir>.*)'/)['install_dir']
     raise StandardError, "Unable to retrieve install_dir, thus unable to check #{install_dir}/dependency_licenses.json" unless File.exist?(install_dir)
+
     puts "Checking licenses via the contents of '#{install_dir}/dependency_licenses.json'"
-    unless File.exist?("#{install_dir}/dependency_licenses.json")
-      raise StandardError, "Unable to open #{install_dir}/dependency_licenses.json"
-    end
-    content = File.read("#{install_dir}/dependency_licenses.json")
-    JSON.parse(content).each do |dependency, attributes|
-      license = attributes['license']
-      version = attributes['version']
-      if license.match(good)
-        puts "Good   : #{dependency} - #{version} uses #{license}"
-      elsif license.match(bad)
-        puts "Check  ! #{dependency} - #{version} uses #{license}"
-      else
-        puts "Unknown? #{dependency} - #{version} uses #{license}"
+    raise StandardError, "Unable to open #{install_dir}/dependency_licenses.json" unless File.exist?("#{install_dir}/dependency_licenses.json")
+    json_data = JSON.parse(File.read("#{install_dir}/dependency_licenses.json"))
+    puts "###### BEGIN LICENSE CHECK ######"
+    violations = LicenseAnalyzer.analyze(json_data)
+    unless violations.empty?
+      puts "\n\nProblematic softwares: #{violations.count}"
+      violations.each do |violation|
+        puts violation
       end
+      puts "\n\n"
+      raise "Build Aborted due to license violations"
     end
     puts "###### END LICENSE CHECK ######"
   end

spec/gitlab/tasks/license_spec.rb

@@ -7,7 +7,39 @@ describe 'license:check', type: :rake do
   before do
     Rake::Task['license:check'].reenable
-    @license_info = '{
+    allow(File).to receive(:exist?).and_return(true)
+  end
+
+  it 'detects good licenses correctly' do
+    license_info = '{
+      "chef-zero": {
+        "version": "4.8.0",
+        "license": "Apache-2.0"
+      }
+    }'
+    allow(File).to receive(:read).and_return(license_info)
+
+    expect { Rake::Task['license:check'].invoke }.to output(/Acceptable.*chef-zero - 4.8.0.*Apache-2.0/).to_stdout
+  end
+
+  it 'detects blacklisted softwares with good licenses correctly' do
+    license_info = '{
+      "chef-zero": {
+        "version": "4.8.0",
+        "license": "Apache-2.0"
+      },
+      "readline": {
+        "version": "2.3.0",
+        "license": "Apache-2.0"
+      }
+    }'
+    allow(File).to receive(:read).and_return(license_info)
+
+    expect { Rake::Task['license:check'].invoke }.to output(/readline.*Blacklisted software/).to_stdout.and raise_error(RuntimeError, "Build Aborted due to license violations")
+  end
+
+  it 'detects bad licenses correctly' do
+    license_info = '{
       "chef-zero": {
         "version": "4.8.0",
         "license": "Apache-2.0"
@@ -21,22 +53,55 @@ describe 'license:check', type: :rake do
         "license": "GPL-3.0+"
       }
     }'
-    allow(File).to receive(:exist?).and_return(true)
+    allow(File).to receive(:read).and_return(license_info)
+    expect { Rake::Task['license:check'].invoke }.to output(/foo.*Unacceptable license/).to_stdout.and raise_error(RuntimeError, "Build Aborted due to license violations")
   end
-  it 'detects good licenses correctly' do
-    allow(File).to receive(:read).and_return(@license_info)
-    expect { Rake::Task['license:check'].invoke }.to output(/Good.*chef-zero - 4.8.0.*Apache-2.0/).to_stdout
+  it 'detects whitelisted softwares with bad licenses correctly' do
+    license_info = '{
+      "chef-zero": {
+        "version": "4.8.0",
+        "license": "Apache-2.0"
+      },
+      "git": {
+        "version": "1.2.11",
+        "license": "GPL-3.0+"
+      }
+    }'
+    allow(File).to receive(:read).and_return(license_info)
+
+    expect { Rake::Task['license:check'].invoke }.to output(/git.*Whitelisted software/).to_stdout
   end
-  it 'detects bad licenses correctly' do
-    allow(File).to receive(:read).and_return(@license_info)
-    expect { Rake::Task['license:check'].invoke }.to output(/Check.*foo - 1.2.11.*GPL-3.0\+/).to_stdout
+  it 'detects blacklisted softwares with unknown licenses correctly' do
+    license_info = '{
+      "chef-zero": {
+        "version": "4.8.0",
+        "license": "Apache-2.0"
+      },
+      "readline": {
+        "version": "2.3.0",
+        "license": "jargon"
+      }
+    }'
+    allow(File).to receive(:read).and_return(license_info)
+
+    expect { Rake::Task['license:check'].invoke }.to output(/readline.*Blacklisted software/).to_stdout.and raise_error(RuntimeError, "Build Aborted due to license violations")
   end
-  it 'detects unknown licenses correctly' do
-    allow(File).to receive(:read).and_return(@license_info)
-    expect { Rake::Task['license:check'].invoke }.to output(/Unknown.*bar - 2.3.0.*jargon/).to_stdout
+  it 'detects whitelisted software with unknown licenses correctly' do
+    license_info = '{
+      "chef-zero": {
+        "version": "4.8.0",
+        "license": "Apache-2.0"
+      },
+      "git": {
+        "version": "1.2.11",
+        "license": "jargon"
+      }
+    }'
+    allow(File).to receive(:read).and_return(license_info)
+    expect { Rake::Task['license:check'].invoke }.to output(/git.*Whitelisted software/).to_stdout
   end
   it 'should detect if install directory not found' do