WIP - building up import tree with sub relations, etc

app/services/projects/import_export.rb

@@ -10,6 +10,10 @@ module Projects
       %i(name path description issues_enabled wall_enabled merge_requests_enabled wiki_enabled snippets_enabled visibility_level archived)
     end
+    def project_tree_list
+      project_tree.map {|r| r.is_a?(Hash) ? r.keys.first : r }
+    end
+
     def project_tree
       %i(issues labels milestones snippets releases events) + [members, merge_requests, commit_statuses]
     end

app/services/projects/import_export/project_tree_restorer.rb

@@ -12,6 +12,7 @@ module Projects
         json = IO.read(@path)
         @tree_hash = ActiveSupport::JSON.decode(json)
         create_relations
+        puts project.inspect
       end
       private
@@ -21,11 +22,33 @@ module Projects
           exported_members: @tree_hash.delete('project_members'), user: @user, project_id: project.id)
       end
-      def create_relations
-        (ImportExport.project_tree - [:project_members]).each do |relation|
-          next if @tree_hash[relation.to_s].empty?
-          relation_hash = create_relation(relation, @tree_hash[relation.to_s])
-          project.update_attribute(relation, relation_hash)
+      #TODO Definitely refactor this method!
+      def create_relations(relation_list = default_relation_list, tree_hash = @tree_hash)
+        relation_list.each do |relation|
+          relation_hash = nil
+          # FIXME
+          # next if tree_hash[relation.to_s].blank?
+          if (relation.is_a?(Hash) && relation.values.first[:include])
+            #TODO name stuff properly
+            relation_sym = relation.keys.first
+            #TODO remove sub-relation hashes from here so we can save the parent relation first
+            relation_hash = create_relation(relation_sym, tree_hash[relation_sym.to_s])
+            sub_relations = []
+            sub_relation = relation.values.first[:include]
+            sub_relation_hash_list = tree_hash[relation.keys.first.to_s]
+            sub_relation_hash_list.each do |sub_relation_hash|
+              sub_relations << create_relation(relation, sub_relation_hash[relation.to_s])
+            end
+            relation_hash.update_attribute(sub_relation, sub_relations)
+          end
+            relation_hash ||= create_relation(relation, tree_hash[relation.to_s])
+            project.update_attribute(relation, relation_hash)
+        end
+      end
+
+      def default_relation_list
+        ImportExport.project_tree.reject do |rel|
+          rel.is_a?(Hash) && !rel[:project_members].blank?
         end
       end
@@ -42,7 +65,7 @@ module Projects
       end
       def create_relation(relation, relation_hash_list)
-        relation_hash_list.map do |relation_hash|
+        [relation_hash_list].flatten.map do |relation_hash|
           Projects::ImportExport::RelationFactory.create(
             relation_sym: relation, relation_hash: relation_hash.merge('project_id' => project.id), members_map: members_map)
         end

fixtures/import_export/project.json

-{"name":"searchable_project","path":"gitlabhq","description":null,"issues_enabled":true,"wall_enabled":false,"merge_requests_enabled":true,"wiki_enabled":true,"snippets_enabled":true,"visibility_level":20,"archived":false,"issues":[{"id":2,"title":"Voluptatem quia repudiandae id rem ducimus.","assignee_id":19,"author_id":20,"project_id":14,"created_at":"2016-03-11T10:21:37.525Z","updated_at":"2016-03-11T10:21:44.516Z","position":0,"branch_name":null,"description":null,"milestone_id":null,"state":"opened","iid":1,"updated_by_id":null}],"merge_requests":[{"id":2,"target_branch":"feature","source_branch":"master","source_project_id":9,"author_id":23,"assignee_id":null,"title":"Enim dolor veritatis suscipit sed laboriosam consequatur doloremque.","created_at":"2016-03-11T10:21:38.628Z","updated_at":"2016-03-11T10:21:38.628Z","milestone_id":null,"state":"opened","merge_status":"can_be_merged","target_project_id":14,"iid":1,"description":null,"position":0,"locked_at":null,"updated_by_id":null,"merge_error":null,"merge_params":{},"merge_when_build_succeeds":false,"merge_user_id":null,"merge_commit_sha":null}],"labels":[{"id":2,"title":"Bug","color":"#990000","project_id":14,"created_at":"2016-03-11T10:21:39.964Z","updated_at":"2016-03-11T10:21:44.539Z","template":false,"description":null}],"milestones":[{"id":2,"title":"Milestone v1.2","project_id":14,"description":null,"due_date":null,"created_at":"2016-03-11T10:21:44.688Z","updated_at":"2016-03-11T10:21:44.688Z","state":"active","iid":1}],"snippets":[{"id":2,"title":"Dolorem perspiciatis unde aut similique aperiam totam qui iste.","content":"Sed similique praesentium tempora et ad dolor quis ut.","author_id":30,"project_id":14,"created_at":"2016-03-11T10:21:41.034Z","updated_at":"2016-03-11T10:21:44.557Z","file_name":"dee","expires_at":null,"visibility_level":0}],"releases":[{"id":2,"tag":"v1.1.0","description":"Awesome release","project_id":14,"created_at":"2016-03-11T10:21:42.531Z","updated_at":"2016-03-11T10:21:44.576Z"}],"events":[{"id":2,"target_type":null,"target_id":null,"title":null,"data":null,"project_id":14,"created_at":"2016-03-11T10:21:44.892Z","updated_at":"2016-03-11T10:21:44.892Z","action":8,"author_id":19}],"commit_statuses":[{"id":2,"project_id":null,"status":"success","finished_at":"2016-01-26T07:23:42.000Z","trace":null,"created_at":"2016-03-11T10:21:43.493Z","updated_at":"2016-03-11T10:21:44.620Z","started_at":"2016-01-26T07:21:42.000Z","runner_id":null,"coverage":null,"commit_id":2,"commands":null,"job_id":null,"name":"default","deploy":false,"options":null,"allow_failure":false,"stage":null,"trigger_request_id":null,"stage_idx":null,"tag":null,"ref":null,"user_id":null,"target_url":null,"description":"commit status","artifacts_file":null,"gl_project_id":14,"artifacts_metadata":null,"erased_by_id":null,"erased_at":null}],"project_members":[{"id":2,"access_level":40,"source_id":14,"source_type":"Project","user_id":19,"notification_level":3,"created_at":"2016-03-11T10:21:44.822Z","updated_at":"2016-03-11T10:21:44.822Z","created_by_id":null,"invite_email":null,"invite_token":null,"invite_accepted_at":null,"user":{"email":"claudie.lemke@prohaskamarquardt.co.uk","username":"hiram_conroy19"}}]}
+{"name":"searchable_project","path":"gitlabhq","description":null,"issues_enabled":true,"wall_enabled":false,"merge_requests_enabled":true,"wiki_enabled":true,"snippets_enabled":true,"visibility_level":20,"archived":false,"issues":[{"id":2,"title":"Libero explicabo dolores atque quae debitis sit ipsam unde.","assignee_id":19,"author_id":20,"project_id":14,"created_at":"2016-03-14T11:56:57.324Z","updated_at":"2016-03-14T11:57:02.118Z","position":0,"branch_name":null,"description":null,"milestone_id":null,"state":"opened","iid":1,"updated_by_id":null}],"labels":[{"id":2,"title":"Bug","color":"#990000","project_id":14,"created_at":"2016-03-14T11:56:59.538Z","updated_at":"2016-03-14T11:57:02.132Z","template":false,"description":null}],"milestones":[{"id":2,"title":"Milestone v1.2","project_id":14,"description":null,"due_date":null,"created_at":"2016-03-14T11:57:02.241Z","updated_at":"2016-03-14T11:57:02.241Z","state":"active","iid":1}],"releases":[{"id":2,"tag":"v1.1.0","description":"Awesome release","project_id":14,"created_at":"2016-03-14T11:57:00.950Z","updated_at":"2016-03-14T11:57:02.159Z"}],"events":[{"id":2,"target_type":null,"target_id":null,"title":null,"data":null,"project_id":14,"created_at":"2016-03-14T11:57:02.392Z","updated_at":"2016-03-14T11:57:02.392Z","action":8,"author_id":19}],"snippets":[{"id":2,"title":"Et qui optio blanditiis non.","content":"Repudiandae hic id vero adipisci.","author_id":28,"project_id":14,"created_at":"2016-03-14T11:57:00.372Z","updated_at":"2016-03-14T11:57:02.144Z","file_name":"krystina","visibility_level":0}],"project_members":[{"id":2,"access_level":40,"source_id":14,"source_type":"Project","user_id":19,"notification_level":3,"created_at":"2016-03-14T11:57:02.335Z","updated_at":"2016-03-14T11:57:02.335Z","created_by_id":null,"invite_email":null,"invite_token":null,"invite_accepted_at":null,"user":{"id":19,"email":"braxton@greenholttromp.ca","username":"elena19"}}],"merge_requests":[{"id":2,"target_branch":"feature","source_branch":"master","source_project_id":9,"author_id":23,"assignee_id":null,"title":"Cum est in est et.","created_at":"2016-03-14T11:56:58.333Z","updated_at":"2016-03-14T11:56:58.333Z","milestone_id":null,"state":"opened","merge_status":"can_be_merged","target_project_id":14,"iid":1,"description":null,"position":0,"locked_at":null,"updated_by_id":null,"merge_error":null,"merge_params":{},"merge_when_build_succeeds":false,"merge_user_id":null,"merge_commit_sha":null,"merge_request_diff":{"id":2,"state":"collected","st_commits":[{"id":"5937ac0a7beb003549fc5fd26fc247adbce4a52e","message":"Add submodule from gitlab.com\n\nSigned-off-by: Dmitriy Zaporozhets \u003cdmitriy.zaporozhets@gmail.com\u003e\n","parent_ids":["570e7b2abdd848b95f2f578043fc23bd6f6fd24d"],"authored_date":"2014-02-27T10:01:38.000+01:00","author_name":"Dmitriy Zaporozhets","author_email":"dmitriy.zaporozhets@gmail.com","committed_date":"2014-02-27T10:01:38.000+01:00","committer_name":"Dmitriy Zaporozhets","committer_email":"dmitriy.zaporozhets@gmail.com"},{"id":"570e7b2abdd848b95f2f578043fc23bd6f6fd24d","message":"Change some files\n\nSigned-off-by: Dmitriy Zaporozhets \u003cdmitriy.zaporozhets@gmail.com\u003e\n","parent_ids":["6f6d7e7ed97bb5f0054f2b1df789b39ca89b6ff9"],"authored_date":"2014-02-27T09:57:31.000+01:00","author_name":"Dmitriy Zaporozhets","author_email":"dmitriy.zaporozhets@gmail.com","committed_date":"2014-02-27T09:57:31.000+01:00","committer_name":"Dmitriy Zaporozhets","committer_email":"dmitriy.zaporozhets@gmail.com"},{"id":"6f6d7e7ed97bb5f0054f2b1df789b39ca89b6ff9","message":"More submodules\n\nSigned-off-by: Dmitriy Zaporozhets \u003cdmitriy.zaporozhets@gmail.com\u003e\n","parent_ids":["d14d6c0abdd253381df51a723d58691b2ee1ab08"],"authored_date":"2014-02-27T09:54:21.000+01:00","author_name":"Dmitriy Zaporozhets","author_email":"dmitriy.zaporozhets@gmail.com","committed_date":"2014-02-27T09:54:21.000+01:00","committer_name":"Dmitriy Zaporozhets","committer_email":"dmitriy.zaporozhets@gmail.com"},{"id":"d14d6c0abdd253381df51a723d58691b2ee1ab08","message":"Remove ds_store files\n\nSigned-off-by: Dmitriy Zaporozhets \u003cdmitriy.zaporozhets@gmail.com\u003e\n","parent_ids":["c1acaa58bbcbc3eafe538cb8274ba387047b69f8"],"authored_date":"2014-02-27T09:49:50.000+01:00","author_name":"Dmitriy Zaporozhets","author_email":"dmitriy.zaporozhets@gmail.com","committed_date":"2014-02-27T09:49:50.000+01:00","committer_name":"Dmitriy Zaporozhets","committer_email":"dmitriy.zaporozhets@gmail.com"},{"id":"c1acaa58bbcbc3eafe538cb8274ba387047b69f8","message":"Ignore DS files\n\nSigned-off-by: Dmitriy Zaporozhets \u003cdmitriy.zaporozhets@gmail.com\u003e\n","parent_ids":["ae73cb07c9eeaf35924a10f713b364d32b2dd34f"],"authored_date":"2014-02-27T09:48:32.000+01:00","author_name":"Dmitriy Zaporozhets","author_email":"dmitriy.zaporozhets@gmail.com","committed_date":"2014-02-27T09:48:32.000+01:00","committer_name":"Dmitriy Zaporozhets","committer_email":"dmitriy.zaporozhets@gmail.com"}],"st_diffs":[{"diff":"Binary files a/.DS_Store and /dev/null differ\n","new_path":".DS_Store","old_path":".DS_Store","a_mode":"100644","b_mode":"0","new_file":false,"renamed_file":false,"deleted_file":true},{"diff":"--- a/.gitignore\n+++ b/.gitignore\n@@ -17,3 +17,4 @@ rerun.txt\n pickle-email-*.html\n .project\n config/initializers/secret_token.rb\n+.DS_Store\n","new_path":".gitignore","old_path":".gitignore","a_mode":"100644","b_mode":"100644","new_file":false,"renamed_file":false,"deleted_file":false},{"diff":"--- a/.gitmodules\n+++ b/.gitmodules\n@@ -1,3 +1,9 @@\n [submodule \"six\"]\n \tpath = six\n \turl = git://github.com/randx/six.git\n+[submodule \"gitlab-shell\"]\n+\tpath = gitlab-shell\n+\turl = https://github.com/gitlabhq/gitlab-shell.git\n+[submodule \"gitlab-grack\"]\n+\tpath = gitlab-grack\n+\turl = https://gitlab.com/gitlab-org/gitlab-grack.git\n","new_path":".gitmodules","old_path":".gitmodules","a_mode":"100644","b_mode":"100644","new_file":false,"renamed_file":false,"deleted_file":false},{"diff":"Binary files a/files/.DS_Store and /dev/null differ\n","new_path":"files/.DS_Store","old_path":"files/.DS_Store","a_mode":"100644","b_mode":"0","new_file":false,"renamed_file":false,"deleted_file":true},{"diff":"--- a/files/ruby/popen.rb\n+++ b/files/ruby/popen.rb\n@@ -6,12 +6,18 @@ module Popen\n \n   def popen(cmd, path=nil)\n     unless cmd.is_a?(Array)\n-      raise \"System commands must be given as an array of strings\"\n+      raise RuntimeError, \"System commands must be given as an array of strings\"\n     end\n \n     path ||= Dir.pwd\n-    vars = { \"PWD\" =\u003e path }\n-    options = { chdir: path }\n+\n+    vars = {\n+      \"PWD\" =\u003e path\n+    }\n+\n+    options = {\n+      chdir: path\n+    }\n \n     unless File.directory?(path)\n       FileUtils.mkdir_p(path)\n@@ -19,6 +25,7 @@ module Popen\n \n     @cmd_output = \"\"\n     @cmd_status = 0\n+\n     Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr|\n       @cmd_output \u003c\u003c stdout.read\n       @cmd_output \u003c\u003c stderr.read\n","new_path":"files/ruby/popen.rb","old_path":"files/ruby/popen.rb","a_mode":"100644","b_mode":"100644","new_file":false,"renamed_file":false,"deleted_file":false},{"diff":"--- a/files/ruby/regex.rb\n+++ b/files/ruby/regex.rb\n@@ -19,14 +19,12 @@ module Gitlab\n     end\n \n     def archive_formats_regex\n-      #|zip|tar|    tar.gz    |         tar.bz2         |\n-      /(zip|tar|tar\\.gz|tgz|gz|tar\\.bz2|tbz|tbz2|tb2|bz2)/\n+      /(zip|tar|7z|tar\\.gz|tgz|gz|tar\\.bz2|tbz|tbz2|tb2|bz2)/\n     end\n \n     def git_reference_regex\n       # Valid git ref regex, see:\n       # https://www.kernel.org/pub/software/scm/git/docs/git-check-ref-format.html\n-\n       %r{\n         (?!\n            (?# doesn't begins with)\n","new_path":"files/ruby/regex.rb","old_path":"files/ruby/regex.rb","a_mode":"100644","b_mode":"100644","new_file":false,"renamed_file":false,"deleted_file":false},{"diff":"--- /dev/null\n+++ b/gitlab-grack\n@@ -0,0 +1 @@\n+Subproject commit 645f6c4c82fd3f5e06f67134450a570b795e55a6\n","new_path":"gitlab-grack","old_path":"gitlab-grack","a_mode":"0","b_mode":"160000","new_file":true,"renamed_file":false,"deleted_file":false},{"diff":"--- /dev/null\n+++ b/gitlab-shell\n@@ -0,0 +1 @@\n+Subproject commit 79bceae69cb5750d6567b223597999bfa91cb3b9\n","new_path":"gitlab-shell","old_path":"gitlab-shell","a_mode":"0","b_mode":"160000","new_file":true,"renamed_file":false,"deleted_file":false}],"merge_request_id":2,"created_at":"2016-03-14T11:56:58.353Z","updated_at":"2016-03-14T11:56:58.635Z","base_commit_sha":"ae73cb07c9eeaf35924a10f713b364d32b2dd34f","real_size":"8"}}],"commit_statuses":[{"id":2,"project_id":null,"status":"success","finished_at":"2016-01-26T07:23:42.000Z","trace":null,"created_at":"2016-03-14T11:57:01.483Z","updated_at":"2016-03-14T11:57:02.190Z","started_at":"2016-01-26T07:21:42.000Z","runner_id":null,"coverage":null,"commit_id":2,"commands":null,"job_id":null,"name":"default","deploy":false,"options":null,"allow_failure":false,"stage":null,"trigger_request_id":null,"stage_idx":null,"tag":null,"ref":null,"user_id":null,"target_url":null,"description":"commit status","artifacts_file":null,"gl_project_id":14,"artifacts_metadata":null,"erased_by_id":null,"erased_at":null,"commit":{"id":2,"project_id":13,"ref":null,"sha":"97de212e80737a608d939f648d959671fb0a0142","before_sha":null,"push_data":null,"created_at":"2016-03-14T11:57:01.468Z","updated_at":"2016-03-14T11:57:01.468Z","tag":false,"yaml_errors":null,"committed_at":null,"gl_project_id":13}}]}
\ No newline at end of file