From ff1d6477dade1564d8d8f366375607ce57f3592f Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Tue, 16 Aug 2016 17:32:58 +0200
Subject: [PATCH] Export integration test and attribute change spec - squashed
Export file integration test that checks for sensitive info.
Also added spec to check new added attributes to models that
can be exported.
---
.../import_export/export_file_spec.rb | 162 ++++++++++++++++++
.../attribute_configuration_spec.rb | 83 +++++++++
2 files changed, 245 insertions(+)
create mode 100644 spec/features/projects/import_export/export_file_spec.rb
create mode 100644 spec/lib/gitlab/import_export/attribute_configuration_spec.rb
diff --git a/spec/features/projects/import_export/export_file_spec.rb b/spec/features/projects/import_export/export_file_spec.rb
new file mode 100644
index 00000000000..eea0f595eb0
--- /dev/null
+++ b/spec/features/projects/import_export/export_file_spec.rb
@@ -0,0 +1,162 @@
+require 'spec_helper'
+
+feature 'project export', feature: true, js: true do
+ include Select2Helper
+
+ let(:user) { create(:admin) }
+ let(:export_path) { "#{Dir::tmpdir}/import_file_spec" }
+
+ let(:sensitive_words) { %w[pass secret token key] }
+ let(:safe_hashes) do
+ {
+ token: [
+ { # Triggers
+ "id" => 1,
+ "token" => "token",
+ "project_id" => nil,
+ "deleted_at" => nil,
+ "gl_project_id" => 4
+ },
+ { # Project hooks
+ "id" => 1,
+ "project_id" => 4,
+ "service_id" => nil,
+ "push_events" => true,
+ "issues_events" => false,
+ "merge_requests_events" => false,
+ "tag_push_events" => false,
+ "note_events" => false,
+ "enable_ssl_verification" => true,
+ "build_events" => false,
+ "wiki_page_events" => false,
+ "pipeline_events" => false,
+ "token" => "token"
+ }
+ ]
+ }
+ end
+
+ let(:project) { setup_project }
+
+ background do
+ allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
+ end
+
+ after do
+ FileUtils.rm_rf(export_path, secure: true)
+ end
+
+ context 'admin user' do
+ before do
+ login_as(user)
+ end
+
+ scenario 'user imports an exported project successfully' do
+ visit edit_namespace_project_path(project.namespace, project)
+
+ expect(page).to have_content('Export project')
+
+ click_link 'Export project'
+
+ visit edit_namespace_project_path(project.namespace, project)
+
+ expect(page).to have_content('Download export')
+
+ in_directory_with_expanded_export(project) do |exit_status, tmpdir|
+ expect(exit_status).to eq(0)
+
+ project_json_path = File.join(tmpdir, 'project.json')
+ expect(File).to exist(project_json_path)
+
+ project_hash = JSON.parse(IO.read(project_json_path))
+
+ sensitive_words.each do |sensitive_word|
+ expect(has_sensitive_attributes?(sensitive_word, project_hash)).to be false
+ end
+ end
+ end
+ end
+
+ def setup_project
+ issue = create(:issue, assignee: user)
+ snippet = create(:project_snippet)
+ release = create(:release)
+
+ project = create(:project,
+ :public,
+ issues: [issue],
+ snippets: [snippet],
+ releases: [release]
+ )
+ label = create(:label, project: project)
+ create(:label_link, label: label, target: issue)
+ milestone = create(:milestone, project: project)
+ merge_request = create(:merge_request, source_project: project, milestone: milestone)
+ commit_status = create(:commit_status, project: project)
+
+ ci_pipeline = create(:ci_pipeline,
+ project: project,
+ sha: merge_request.diff_head_sha,
+ ref: merge_request.source_branch,
+ statuses: [commit_status])
+
+ create(:ci_build, pipeline: ci_pipeline, project: project)
+ create(:milestone, project: project)
+ create(:note, noteable: issue, project: project)
+ create(:note, noteable: merge_request, project: project)
+ create(:note, noteable: snippet, project: project)
+ create(:note_on_commit,
+ author: user,
+ project: project,
+ commit_id: ci_pipeline.sha)
+
+ create(:event, target: milestone, project: project, action: Event::CREATED, author: user)
+ create(:project_member, :master, user: user, project: project)
+ create(:ci_variable, project: project)
+ create(:ci_trigger, project: project)
+ key = create(:deploy_key)
+ key.projects << project
+ create(:service, project: project)
+ create(:project_hook, project: project, token: 'token')
+ create(:protected_branch, project: project)
+
+ project
+ end
+
+ # Expands the compressed file for an exported project into +tmpdir+
+ def in_directory_with_expanded_export(project)
+ Dir.mktmpdir do |tmpdir|
+ export_file = project.export_project_path
+ _output, exit_status = Gitlab::Popen.popen(%W{tar -zxf #{export_file} -C #{tmpdir}})
+
+ yield(exit_status, tmpdir)
+ end
+ end
+
+ # Recursively finds key/values including +key+ as part of the key, inside a nested hash
+ def deep_find_with_parent(key, object, found = nil)
+ if object.respond_to?(:key?) && object.keys.any? { |k| k.include?(key) }
+ [object[key], object] if object[key]
+ elsif object.is_a? Enumerable
+ object.find { |*a| found, object = deep_find_with_parent(key, a.last, found) }
+ [found, object] if found
+ end
+ end
+
+ # Returns true if a sensitive word is found inside a hash, excluding safe hashes
+ def has_sensitive_attributes?(sensitive_word, project_hash)
+ loop do
+ object, parent = deep_find_with_parent(sensitive_word, project_hash)
+ parent.except!('created_at', 'updated_at', 'url') if parent
+
+ if object && safe_hashes[sensitive_word.to_sym].include?(parent)
+ # It's in the safe list, remove hash and keep looking
+ parent.delete(object)
+ elsif object
+ return true
+ else
+ return false
+ end
+ end
+ end
+end
diff --git a/spec/lib/gitlab/import_export/attribute_configuration_spec.rb b/spec/lib/gitlab/import_export/attribute_configuration_spec.rb
new file mode 100644
index 00000000000..7872f0c2257
--- /dev/null
+++ b/spec/lib/gitlab/import_export/attribute_configuration_spec.rb
@@ -0,0 +1,83 @@
+require 'spec_helper'
+
+# Checks whether there are new attributes in models that are currently being exported as part of the
+# project Import/Export feature.
+# If there are new attributes, these will have to either be added to this spec in case we want them
+# to be included as part of the export, or blacklist them using the import_export.yml configuration file.
+# Likewise, new models added to import_export.yml, will need to be added with their correspondent attributes
+# to this spec.
+describe 'Attribute configuration', lib: true do
+ let(:config_hash) { YAML.load_file(Gitlab::ImportExport.config_file).deep_stringify_keys }
+ let(:relation_names) do
+ names = config_hash['project_tree'].to_s.gsub(/[\[{}\]=>\"\:]/, ',').split(',').delete_if(&:blank?)
+ names.uniq - ['author', 'milestones', 'labels'] + ['project'] # Remove duplicated or add missing models
+ end
+
+ let(:safe_model_attributes) do
+ {
+ 'Issue' => %w[id title assignee_id author_id project_id created_at updated_at position branch_name description state iid updated_by_id confidential deleted_at due_date moved_to_id lock_version],
+ 'Event' => %w[id target_type target_id title data project_id created_at updated_at action author_id],
+ 'Note' => %w[id note noteable_type author_id created_at updated_at project_id attachment line_code commit_id noteable_id system st_diff updated_by_id type position original_position],
+ 'LabelLink' => %w[id label_id target_id target_type created_at updated_at],
+ 'Label' => %w[id title color project_id created_at updated_at template description priority],
+ 'Milestone' => %w[id title project_id description due_date created_at updated_at state iid],
+ 'ProjectSnippet' => %w[id title content author_id project_id created_at updated_at file_name type visibility_level],
+ 'Release' => %w[id tag description project_id created_at updated_at],
+ 'ProjectMember' => %w[id access_level source_id source_type user_id notification_level type created_at updated_at created_by_id invite_email invite_token invite_accepted_at requested_at],
+ 'User' => %w[id username email],
+ 'MergeRequest' => %w[id target_branch source_branch source_project_id author_id assignee_id title created_at updated_at state merge_status target_project_id iid description position locked_at updated_by_id merge_error merge_params merge_when_build_succeeds merge_user_id merge_commit_sha deleted_at in_progress_merge_commit_sha lock_version],
+ 'MergeRequestDiff' => %w[id state st_commits merge_request_id created_at updated_at base_commit_sha real_size head_commit_sha start_commit_sha],
+ 'Ci::Pipeline' => %w[id project_id ref sha before_sha push_data created_at updated_at tag yaml_errors committed_at gl_project_id status started_at finished_at duration user_id],
+ 'CommitStatus' => %w[id project_id status finished_at trace created_at updated_at started_at runner_id coverage commit_id commands job_id name deploy options allow_failure stage trigger_request_id stage_idx tag ref user_id type target_url description artifacts_file gl_project_id artifacts_metadata erased_by_id erased_at artifacts_expire_at environment artifacts_size when yaml_variables queued_at],
+ 'Ci::Variable' => %w[id project_id key value encrypted_value encrypted_value_salt encrypted_value_iv gl_project_id],
+ 'Ci::Trigger' => %w[id token project_id deleted_at created_at updated_at gl_project_id],
+ 'DeployKey' => %w[id user_id created_at updated_at key title type fingerprint public],
+ 'Service' => %w[id type title project_id created_at updated_at active properties template push_events issues_events merge_requests_events tag_push_events note_events pipeline_events build_events category default wiki_page_events],
+ 'ProjectHook' => %w[id url project_id created_at updated_at type service_id push_events issues_events merge_requests_events tag_push_events note_events pipeline_events enable_ssl_verification build_events wiki_page_events token],
+ 'ProtectedBranch' => %w[id project_id name created_at updated_at],
+ 'Project' => %w[description issues_enabled merge_requests_enabled wiki_enabled snippets_enabled visibility_level archived]
+ }
+ end
+
+ it 'has no new columns' do
+ relation_names.each do |relation_name|
+ relation_class = relation_class_for_name(relation_name)
+
+ expect(safe_model_attributes[relation_class.to_s]).not_to be_nil
+
+ current_attributes = parsed_attributes(relation_name, relation_class.attribute_names)
+ safe_attributes = safe_model_attributes[relation_class.to_s]
+ new_attributes = current_attributes - safe_attributes
+
+ expect(new_attributes).to be_empty, failure_message(relation_class.to_s, new_attributes)
+ end
+ end
+
+ def relation_class_for_name(relation_name)
+ relation_name = Gitlab::ImportExport::RelationFactory::OVERRIDES[relation_name.to_sym] || relation_name
+ relation_name.to_s.classify.constantize
+ end
+
+ def failure_message(relation_class, new_attributes)
+ <<-MSG
+ It looks like #{relation_class}, which is exported using the project Import/Export, has new attributes: #{new_attributes.join(',')}
+
+ Please add the attribute(s) to +safe_model_attributes+ in CURRENT_SPEC if you consider this can be exported.
+ Otherwise, please blacklist the attribute(s) in IMPORT_EXPORT_CONFIG by adding it to its correspondent
+ model in the +excluded_attributes+ section.
+
+ CURRENT_SPEC: #{__FILE__}
+ IMPORT_EXPORT_CONFIG: #{Gitlab::ImportExport.config_file}
+ MSG
+ end
+
+ def parsed_attributes(relation_name, attributes)
+ excluded_attributes = config_hash['excluded_attributes'][relation_name]
+ included_attributes = config_hash['included_attributes'][relation_name]
+
+ attributes = attributes - JSON[excluded_attributes.to_json] if excluded_attributes
+ attributes = attributes & JSON[included_attributes.to_json] if included_attributes
+
+ attributes
+ end
+end
--
GitLab