Merge branch 'master' into docs_refactor

lib/banzai/pipeline/asciidoc_pipeline.rb

-require 'banzai'
-
 module Banzai
   module Pipeline
     class AsciidocPipeline < BasePipeline

lib/banzai/pipeline/atom_pipeline.rb

-require 'banzai'
-
 module Banzai
   module Pipeline
     class AtomPipeline < FullPipeline

lib/banzai/pipeline/base_pipeline.rb

-require 'banzai'
 require 'html/pipeline'
 module Banzai

lib/banzai/pipeline/combined_pipeline.rb

-require 'banzai'
-
 module Banzai
   module Pipeline
     module CombinedPipeline

lib/banzai/pipeline/description_pipeline.rb

-require 'banzai'
-
 module Banzai
   module Pipeline
     class DescriptionPipeline < FullPipeline

lib/banzai/pipeline/email_pipeline.rb

-require 'banzai'
-
 module Banzai
   module Pipeline
     class EmailPipeline < FullPipeline

lib/banzai/pipeline/full_pipeline.rb

-require 'banzai'
-
 module Banzai
   module Pipeline
     class FullPipeline < CombinedPipeline.new(PlainMarkdownPipeline, GfmPipeline)

lib/banzai/pipeline/gfm_pipeline.rb

-require 'banzai'
-
 module Banzai
   module Pipeline
     class GfmPipeline < BasePipeline

lib/banzai/pipeline/note_pipeline.rb

-require 'banzai'
-
 module Banzai
   module Pipeline
     class NotePipeline < FullPipeline

lib/banzai/pipeline/plain_markdown_pipeline.rb

-require 'banzai'
-
 module Banzai
   module Pipeline
     class PlainMarkdownPipeline < BasePipeline

lib/banzai/pipeline/post_process_pipeline.rb

-require 'banzai'
-
 module Banzai
   module Pipeline
     class PostProcessPipeline < BasePipeline

lib/banzai/pipeline/reference_extraction_pipeline.rb

-require 'banzai'
-
 module Banzai
   module Pipeline
     class ReferenceExtractionPipeline < BasePipeline

lib/banzai/pipeline/single_line_pipeline.rb

-require 'banzai'
-
 module Banzai
   module Pipeline
     class SingleLinePipeline < GfmPipeline

lib/banzai/pipeline/wiki_pipeline.rb

+require 'banzai'
+
+module Banzai
+  module Pipeline
+    class WikiPipeline < FullPipeline
+      def self.filters
+        super.insert(1, Filter::GollumTagsFilter)
+      end
+    end
+  end
+end

lib/banzai/reference_extractor.rb

-require 'banzai'
-
 module Banzai
   # Extract possible GFM references from an arbitrary String for further processing.
   class ReferenceExtractor

lib/ci/api/builds.rb

@@ -20,7 +20,7 @@ module Ci
           if build
             update_runner_info
-            present build, with: Entities::Build
+            present build, with: Entities::BuildDetails
           else
             not_found!
           end
@@ -78,11 +78,13 @@ module Ci
         # Parameters:
         #   id (required) - The ID of a build
         #   token (required) - The build authorization token
-        #   file (required) - The uploaded file
+        #   file (required) - Artifacts file
         # Parameters (accelerated by GitLab Workhorse):
         #   file.path - path to locally stored body (generated by Workhorse)
         #   file.name - real filename as send in Content-Disposition
         #   file.type - real content type as send in Content-Type
+        #   metadata.path - path to locally stored body (generated by Workhorse)
+        #   metadata.name - filename (generated by Workhorse)
         # Headers:
         #   BUILD-TOKEN (required) - The build authorization token, the same as token
         # Body:
@@ -96,13 +98,20 @@ module Ci
           build = Ci::Build.find_by_id(params[:id])
           not_found! unless build
           authenticate_build_token!(build)
-          forbidden!('build is not running') unless build.running?
-          file = uploaded_file!(:file, ArtifactUploader.artifacts_upload_path)
-          file_to_large! unless file.size < max_artifacts_size
-          if build.update_attributes(artifacts_file: file)
-            present build, with: Entities::Build
+          forbidden!('Build is not running!') unless build.running?
+
+          artifacts_upload_path = ArtifactUploader.artifacts_upload_path
+          artifacts = uploaded_file(:file, artifacts_upload_path)
+          metadata = uploaded_file(:metadata, artifacts_upload_path)
+
+          bad_request!('Missing artifacts file!') unless artifacts
+          file_to_large! unless artifacts.size < max_artifacts_size
+          build.artifacts_file = artifacts
+          build.artifacts_metadata = metadata
+          if build.save
+            present(build, with: Entities::BuildDetails)
           else
             render_validation_error!(build)
           end
@@ -148,6 +157,7 @@ module Ci
           not_found! unless build
           authenticate_build_token!(build)
           build.remove_artifacts_file!
+          build.remove_artifacts_metadata!
         end
       end
     end

lib/ci/api/entities.rb

@@ -16,10 +16,19 @@ module Ci
       end
       class Build < Grape::Entity
-        expose :id, :commands, :ref, :sha, :status, :project_id, :repo_url,
-          :before_sha, :allow_git_fetch, :project_name
-
+        expose :id, :ref, :tag, :sha, :status
         expose :name, :token, :stage
+        expose :project_id
+        expose :project_name
+        expose :artifacts_file, using: ArtifactFile, if: lambda { |build, opts| build.artifacts? }
+      end
+
+      class BuildDetails < Build
+        expose :commands
+        expose :repo_url
+        expose :before_sha
+        expose :allow_git_fetch
+        expose :token
         expose :options do |model|
           model.options
@@ -30,7 +39,7 @@ module Ci
         end
         expose :variables
-        expose :artifacts_file, using: ArtifactFile
+        expose :depends_on_builds, using: Build
       end
       class Runner < Grape::Entity

lib/gitlab/ci/build/artifacts/metadata.rb

+require 'zlib'
+require 'json'
+
+module Gitlab
+  module Ci
+    module Build
+      module Artifacts
+        class Metadata
+          class ParserError < StandardError; end
+
+          VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/
+          INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)}
+
+          attr_reader :file, :path, :full_version
+
+          def initialize(file, path)
+            @file, @path = file, path
+            @full_version = read_version
+          end
+
+          def version
+            @full_version.match(VERSION_PATTERN)[1]
+          end
+
+          def errors
+            gzip do |gz|
+              read_string(gz) # version
+              errors = read_string(gz)
+              raise ParserError, 'Errors field not found!' unless errors
+
+              begin
+                JSON.parse(errors)
+              rescue JSON::ParserError
+                raise ParserError, 'Invalid errors field!'
+              end
+            end
+          end
+
+          def find_entries!
+            gzip do |gz|
+              2.times { read_string(gz) } # version and errors fields
+              match_entries(gz)
+            end
+          end
+
+          def to_entry
+            entries = find_entries!
+            Entry.new(@path, entries)
+          end
+
+          private
+
+          def match_entries(gz)
+            entries = {}
+            match_pattern = %r{^#{Regexp.escape(@path)}[^/]*/?$}
+
+            until gz.eof? do
+              begin
+                path = read_string(gz).force_encoding('UTF-8')
+                meta = read_string(gz).force_encoding('UTF-8')
+               
+                next unless path.valid_encoding? && meta.valid_encoding?
+                next unless path =~ match_pattern
+                next if path =~ INVALID_PATH_PATTERN
+
+                entries[path] = JSON.parse(meta, symbolize_names: true)
+              rescue JSON::ParserError, Encoding::CompatibilityError
+                next
+              end
+            end
+
+            entries
+          end
+
+          def read_version
+            gzip do |gz|
+              version_string = read_string(gz)
+
+              unless version_string
+                raise ParserError, 'Artifacts metadata file empty!'
+              end
+
+              unless version_string =~ VERSION_PATTERN
+                raise ParserError, 'Invalid version!'
+              end
+
+              version_string.chomp
+            end
+          end
+
+          def read_uint32(gz)
+            binary = gz.read(4)
+            binary.unpack('L>')[0] if binary
+          end
+
+          def read_string(gz)
+            string_size = read_uint32(gz)
+            return nil unless string_size
+            gz.read(string_size)
+          end
+
+          def gzip(&block)
+            Zlib::GzipReader.open(@file, &block)
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/build/artifacts/metadata/entry.rb

+module Gitlab
+  module Ci::Build::Artifacts
+    class Metadata
+      ##
+      # Class that represents an entry (path and metadata) to a file or
+      # directory in GitLab CI Build Artifacts binary file / archive
+      #
+      # This is IO-operations safe class, that does similar job to
+      # Ruby's Pathname but without the risk of accessing filesystem.
+      #
+      # This class is working only with UTF-8 encoded paths.
+      #
+      class Entry
+        attr_reader :path, :entries
+        attr_accessor :name
+
+        def initialize(path, entries)
+          @path = path.dup.force_encoding('UTF-8')
+          @entries = entries
+
+          if path.include?("\0")
+            raise ArgumentError, 'Path contains zero byte character!'
+          end
+
+          unless path.valid_encoding?
+            raise ArgumentError, 'Path contains non-UTF-8 byte sequence!'
+          end
+        end
+
+        def directory?
+          blank_node? || @path.end_with?('/')
+        end
+
+        def file?
+          !directory?
+        end
+
+        def has_parent?
+          nodes > 0
+        end
+
+        def parent
+          return nil unless has_parent?
+          self.class.new(@path.chomp(basename), @entries)
+        end
+
+        def basename
+          (directory? && !blank_node?) ? name + '/' : name
+        end
+
+        def name
+          @name || @path.split('/').last.to_s
+        end
+
+        def children
+          return [] unless directory?
+          return @children if @children
+
+          child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$}
+          @children = select_entries { |path| path =~ child_pattern }
+        end
+
+        def directories(opts = {})
+          return [] unless directory?
+          dirs = children.select(&:directory?)
+          return dirs unless has_parent? && opts[:parent]
+
+          dotted_parent = parent
+          dotted_parent.name = '..'
+          dirs.prepend(dotted_parent)
+        end
+
+        def files
+          return [] unless directory?
+          children.select(&:file?)
+        end
+
+        def metadata
+          @entries[@path] || {}
+        end
+
+        def nodes
+          @path.count('/') + (file? ? 1 : 0)
+        end
+
+        def blank_node?
+          @path.empty? # "" is considered to be './'
+        end
+
+        def exists?
+          blank_node? || @entries.include?(@path)
+        end
+
+        def empty?
+          children.empty?
+        end
+
+        def to_s
+          @path
+        end
+
+        def ==(other)
+          @path == other.path && @entries == other.entries
+        end
+
+        def inspect
+          "#{self.class.name}: #{@path}"
+        end
+
+        private
+
+        def select_entries
+          selected = @entries.select { |path, _metadata| yield path }
+          selected.map { |path, _metadata| self.class.new(path, @entries) }
+        end
+      end
+    end
+  end
+end

lib/gitlab/diff/parser.rb

@@ -56,8 +56,9 @@ module Gitlab
       private
       def filename?(line)
-        line.start_with?('--- /dev/null', '+++ /dev/null', '--- a', '+++ b',
-                         '--- /tmp/diffy', '+++ /tmp/diffy')
+        line.start_with?( '--- /dev/null', '+++ /dev/null', '--- a', '+++ b',
+                          '+++ a', # The line will start with `+++ a` in the reverse diff of an orphan commit
+                          '--- /tmp/diffy', '+++ /tmp/diffy')
       end
       def identification_type(line)