Merge pull request #440 from szechyjs/feature/checksums

Allow the use of SHA checksums in source definitions

lib/omnibus/fetchers/net_fetcher.rb

@@ -26,6 +26,9 @@ module Omnibus
     # tar probably has compression scheme linked in, otherwise for tarballs
     TAR_EXTENSIONS = %w(.tar .tar.gz .tgz .bz2 .tar.xz .txz)
+    # Digest types used for verifying file checksums
+    DIGESTS = [:sha512, :sha256, :sha1, :md5]
+
     #
     # A fetch is required if the downloaded_file (such as a tarball) does not
     # exist on disk, or if the checksum of the downloaded file is different
@@ -34,7 +37,7 @@ module Omnibus
     # @return [true, false]
     #
     def fetch_required?
-      !(File.exist?(downloaded_file) && digest(downloaded_file, :md5) == checksum)
+      !(File.exist?(downloaded_file) && digest(downloaded_file, digest_type) == checksum)
     end
     #
@@ -44,7 +47,7 @@ module Omnibus
     # @return [String]
     #
     def version_guid
-      "md5:#{checksum}"
+      "#{digest_type}:#{checksum}"
     end
     #
@@ -82,13 +85,13 @@ module Omnibus
     end
     #
-    # The version for this item in the cache. The is the md5 of downloaded file
-    # and the URL where it was downloaded from.
+    # The version for this item in the cache. This is the digest of downloaded
+    # file and the URL where it was downloaded from.
     #
     # @return [String]
     #
     def version_for_cache
-      "download_url:#{source[:url]}|md5:#{source[:md5]}"
+      "download_url:#{source[:url]}|#{digest_type}:#{checksum}"
     end
     #
@@ -114,12 +117,12 @@ module Omnibus
     end
     #
-    # The checksum (+md5+) as defined by the user in the software definition.
+    # The checksum as defined by the user in the software definition.
     #
     # @return [String]
     #
     def checksum
-      source[:md5]
+      source[digest_type]
     end
     private
@@ -214,6 +217,17 @@ module Omnibus
       end
     end
+    #
+    # The digest type defined in the software definition
+    #
+    # @return [Symbol]
+    #
+    def digest_type
+      DIGESTS.each do |digest|
+        return digest if source.key? digest
+      end
+    end
+
     #
     # Verify the downloaded file has the correct checksum.#
     #
@@ -224,7 +238,7 @@ module Omnibus
       log.info(log_key) { 'Verifying checksum' }
       expected = checksum
-      actual   = digest(downloaded_file, :md5)
+      actual   = digest(downloaded_file, digest_type)
       if expected != actual
         raise ChecksumMismatch.new(self, expected, actual)

spec/functional/fetchers/net_fetcher_spec.rb

@@ -6,6 +6,10 @@ module Omnibus
     let(:source_url) { 'http://downloads.sourceforge.net/project/libpng/zlib/1.2.8/zlib-1.2.8.tar.gz' }
     let(:source_md5) { '44d667c142d7cda120332623eab69f40' }
+    let(:source_sha1) { 'a4d316c404ff54ca545ea71a27af7dbc29817088' }
+    let(:source_sha256) { '36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d' }
+    let(:source_sha512) { 'ece209d4c7ec0cb58ede791444dc754e0d10811cbbdebe3df61c0fd9f9f9867c1c3ccd5f1827f847c005e24eef34fb5bf87b5d3f894d75da04f1797538290e4a' }
+
     let(:source) do
       { url: source_url, md5: source_md5 }
     end
@@ -50,8 +54,40 @@ module Omnibus
     end
     describe '#version_guid' do
-      it 'includes the source md5' do
-        expect(subject.version_guid).to eq("md5:#{source_md5}")
+      context 'source with md5' do
+        it 'includes the md5 digest' do
+          expect(subject.version_guid).to eq("md5:#{source_md5}")
+        end
+      end
+
+      context 'source with sha1' do
+        let(:source) do
+          { url: source_url, sha1: source_sha1 }
+        end
+
+        it 'includes the sha1 digest' do
+          expect(subject.version_guid).to eq("sha1:#{source_sha1}")
+        end
+      end
+
+      context 'source with sha256' do
+        let(:source) do
+          { url: source_url, sha256: source_sha256 }
+        end
+
+        it 'includes the sha256 digest' do
+          expect(subject.version_guid).to eq("sha256:#{source_sha256}")
+        end
+      end
+
+      context 'source with sha512' do
+        let(:source) do
+          { url: source_url, sha512: source_sha512 }
+        end
+
+        it 'includes the sha512 digest' do
+          expect(subject.version_guid).to eq("sha512:#{source_sha512}")
+        end
       end
     end
@@ -85,9 +121,76 @@ module Omnibus
     end
     describe '#fetch' do
-      it 'downloads the file' do
-        fetch!
-        expect(downloaded_file).to be_a_file
+      context 'source with md5' do
+        it 'downloads the file' do
+          fetch!
+          expect(downloaded_file).to be_a_file
+        end
+
+        context 'when the checksum is invalid' do
+          let(:source_md5) { 'bad01234checksum' }
+
+          it 'raises an exception' do
+            expect { fetch! }.to raise_error(ChecksumMismatch)
+          end
+        end
+      end
+
+      context 'source with sha1' do
+        let(:source) do
+          { url: source_url, sha1: source_sha1 }
+        end
+
+        it 'downloads the file' do
+          fetch!
+          expect(downloaded_file).to be_a_file
+        end
+
+        context 'when the checksum is invalid' do
+          let(:source_sha1) { 'bad01234checksum' }
+
+          it 'raises an exception' do
+            expect { fetch! }.to raise_error(ChecksumMismatch)
+          end
+        end
+      end
+
+      context 'source with sha256' do
+        let(:source) do
+          { url: source_url, sha256: source_sha256 }
+        end
+
+        it 'downloads the file' do
+          fetch!
+          expect(downloaded_file).to be_a_file
+        end
+
+        context 'when the checksum is invalid' do
+          let(:source_sha256) { 'bad01234checksum' }
+
+          it 'raises an exception' do
+            expect { fetch! }.to raise_error(ChecksumMismatch)
+          end
+        end
+      end
+
+      context 'source with sha512' do
+        let(:source) do
+          { url: source_url, sha512: source_sha512 }
+        end
+
+        it 'downloads the file' do
+          fetch!
+          expect(downloaded_file).to be_a_file
+        end
+
+        context 'when the checksum is invalid' do
+          let(:source_sha512) { 'bad01234checksum' }
+
+          it 'raises an exception' do
+            expect { fetch! }.to raise_error(ChecksumMismatch)
+          end
+        end
       end
       it 'when the download times out' do
@@ -104,14 +207,6 @@ module Omnibus
         expect(retry_count).to eq(Omnibus::Config.fetcher_retries)
       end
-      context 'when the checksum is invalid' do
-        let(:source_md5) { 'bad01234checksum' }
-
-        it 'raises an exception' do
-          expect { fetch! }.to raise_error(ChecksumMismatch)
-        end
-      end
-
       it 'extracts the file' do
         fetch!
         expect(extracted).to be_a_directory
@@ -135,8 +230,40 @@ module Omnibus
         create_file("#{project_dir}/.file_c")
       end
-      it 'includes the download_url and checksum' do
-        expect(subject.version_for_cache).to eq("download_url:#{source_url}|md5:#{source_md5}")
+      context 'source with md5' do
+        it 'includes the download_url and checksum' do
+          expect(subject.version_for_cache).to eq("download_url:#{source_url}|md5:#{source_md5}")
+        end
+      end
+
+      context 'source with sha1' do
+        let(:source) do
+          { url: source_url, sha1: source_sha1 }
+        end
+
+        it 'includes the download_url and checksum' do
+          expect(subject.version_for_cache).to eq("download_url:#{source_url}|sha1:#{source_sha1}")
+        end
+      end
+
+      context 'source with sha256' do
+        let(:source) do
+          { url: source_url, sha256: source_sha256 }
+        end
+
+        it 'includes the download_url and checksum' do
+          expect(subject.version_for_cache).to eq("download_url:#{source_url}|sha256:#{source_sha256}")
+        end
+      end
+
+      context 'source with sha512' do
+        let(:source) do
+          { url: source_url, sha512: source_sha512 }
+        end
+
+        it 'includes the download_url and checksum' do
+          expect(subject.version_for_cache).to eq("download_url:#{source_url}|sha512:#{source_sha512}")
+        end
       end
     end
   end

spec/unit/fetchers/net_fetcher_spec.rb

@@ -4,12 +4,15 @@ module Omnibus
   describe NetFetcher do
     let(:project_dir) { '/tmp/project' }
     let(:build_dir) { '/tmp/build' }
+    let(:source) do
+      { url: 'https://get.example.com/file.tar.gz', md5: 'abcd1234' }
+    end
     let(:manifest_entry) do
       double(Omnibus::ManifestEntry,
              name: 'file',
              locked_version: "1.2.3",
-             locked_source: { url: 'https://get.example.com/file.tar.gz', md5: 'abcd1234' })
+             locked_source: source)
     end
     let(:cache_dir) { '/cache' }
@@ -57,8 +60,40 @@ module Omnibus
     end
     describe '#version_guid' do
-      it 'returns the shasum' do
-        expect(subject.version_guid).to eq('md5:abcd1234')
+      context 'source with md5' do
+        it 'returns the shasum' do
+          expect(subject.version_guid).to eq('md5:abcd1234')
+        end
+      end
+
+      context 'source with sha1' do
+        let(:source) do
+          { url: 'https://get.example.com/file.tar.gz', sha1: 'abcd1234' }
+        end
+
+        it 'returns the shasum' do
+          expect(subject.version_guid).to eq('sha1:abcd1234')
+        end
+      end
+
+      context 'source with sha256' do
+        let(:source) do
+          { url: 'https://get.example.com/file.tar.gz', sha256: 'abcd1234' }
+        end
+
+        it 'returns the shasum' do
+          expect(subject.version_guid).to eq('sha256:abcd1234')
+        end
+      end
+
+      context 'source with sha512' do
+        let(:source) do
+          { url: 'https://get.example.com/file.tar.gz', sha512: 'abcd1234' }
+        end
+
+        it 'returns the shasum' do
+          expect(subject.version_guid).to eq('sha512:abcd1234')
+        end
       end
     end
@@ -101,8 +136,40 @@ module Omnibus
     end
     describe '#version_for_cache' do
-      it 'returns the download URL and md5' do
-        expect(subject.version_for_cache).to eq('download_url:https://get.example.com/file.tar.gz|md5:abcd1234')
+      context 'source with md5' do
+        it 'returns the download URL and md5' do
+          expect(subject.version_for_cache).to eq('download_url:https://get.example.com/file.tar.gz|md5:abcd1234')
+        end
+      end
+
+      context 'source with sha1' do
+        let(:source) do
+          { url: 'https://get.example.com/file.tar.gz', sha1: 'abcd1234' }
+        end
+
+        it 'returns the download URL and sha1' do
+          expect(subject.version_for_cache).to eq('download_url:https://get.example.com/file.tar.gz|sha1:abcd1234')
+        end
+      end
+
+      context 'source with sha256' do
+        let(:source) do
+          { url: 'https://get.example.com/file.tar.gz', sha256: 'abcd1234' }
+        end
+
+        it 'returns the download URL and sha256' do
+          expect(subject.version_for_cache).to eq('download_url:https://get.example.com/file.tar.gz|sha256:abcd1234')
+        end
+      end
+
+      context 'source with sha512' do
+        let(:source) do
+          { url: 'https://get.example.com/file.tar.gz', sha512: 'abcd1234' }
+        end
+
+        it 'returns the download URL and sha1' do
+          expect(subject.version_for_cache).to eq('download_url:https://get.example.com/file.tar.gz|sha512:abcd1234')
+        end
       end
     end