Merge branch 'jt/partial-clone-proto-v2'

Transfer protocol v2 learned to support the partial clone.

* jt/partial-clone-proto-v2:
  {fetch,upload}-pack: support filter in protocol v2
  upload-pack: read config when serving protocol v2
  upload-pack: fix error message typo

Documentation/technical/protocol-v2.txt

@@ -290,6 +290,15 @@ included in the clients request as well as the potential addition of the
 	Cannot be used with "deepen", but can be used with
 	"deepen-since".
+If the 'filter' feature is advertised, the following argument can be
+included in the client's request:
+
+    filter <filter-spec>
+	Request that various objects from the packfile be omitted
+	using one of several filtering techniques. These are intended
+	for use with partial clone and partial fetch operations. See
+	`rev-list` for possible "filter-spec" values.
+
 The response of `fetch` is broken into a number of sections separated by
 delimiter packets (0001), with each section beginning with its section
 header.

fetch-pack.c

@@ -1198,14 +1198,29 @@ static int send_fetch_request(int fd_out, const struct fetch_pack_args *args,
 	else if (is_repository_shallow() || args->deepen)
 		die(_("Server does not support shallow requests"));
+	/* Add filter */
+	if (server_supports_feature("fetch", "filter", 0) &&
+	    args->filter_options.choice) {
+		print_verbose(args, _("Server supports filter"));
+		packet_buf_write(&req_buf, "filter %s",
+				 args->filter_options.filter_spec);
+	} else if (args->filter_options.choice) {
+		warning("filtering not recognized by server, ignoring");
+	}
+
 	/* add wants */
 	add_wants(wants, &req_buf);
-	/* Add all of the common commits we've found in previous rounds */
-	add_common(&req_buf, common);
-	/* Add initial haves */
-	ret = add_haves(&req_buf, haves_to_send, in_vain);
+	if (args->no_dependents) {
+		packet_buf_write(&req_buf, "done");
+		ret = 1;
+	} else {
+		/* Add all of the common commits we've found in previous rounds */
+		add_common(&req_buf, common);
+		/* Add initial haves */
+		ret = add_haves(&req_buf, haves_to_send, in_vain);
+	}
 	/* Send request */
 	packet_buf_flush(&req_buf);

t/t5701-git-serve.sh

@@ -194,4 +194,18 @@ test_expect_success 'sending server-options' '
 	test_cmp actual expect
 '
+test_expect_success 'unexpected lines are not allowed in fetch request' '
+	git init server &&
+
+	test-pkt-line pack >in <<-EOF &&
+	command=fetch
+	0001
+	this-is-not-a-command
+	0000
+	EOF
+
+	test_must_fail git -C server serve --stateless-rpc <in >/dev/null 2>err &&
+	grep "unexpected line: .this-is-not-a-command." err
+'
+
 test_done

t/t5702-protocol-v2.sh

@@ -233,6 +233,118 @@ test_expect_success 'server-options are sent when fetching' '
 	grep "server-option=world" log
 '
+test_expect_success 'upload-pack respects config using protocol v2' '
+	git init server &&
+	write_script server/.git/hook <<-\EOF &&
+		touch hookout
+		"$@"
+	EOF
+	test_commit -C server one &&
+
+	test_config_global uploadpack.packobjectshook ./hook &&
+	test_path_is_missing server/.git/hookout &&
+	git -c protocol.version=2 clone "file://$(pwd)/server" client &&
+	test_path_is_file server/.git/hookout
+'
+
+test_expect_success 'setup filter tests' '
+	rm -rf server client &&
+	git init server &&
+
+	# 1 commit to create a file, and 1 commit to modify it
+	test_commit -C server message1 a.txt &&
+	test_commit -C server message2 a.txt &&
+	git -C server config protocol.version 2 &&
+	git -C server config uploadpack.allowfilter 1 &&
+	git -C server config uploadpack.allowanysha1inwant 1 &&
+	git -C server config protocol.version 2
+'
+
+test_expect_success 'partial clone' '
+	GIT_TRACE_PACKET="$(pwd)/trace" git -c protocol.version=2 \
+		clone --filter=blob:none "file://$(pwd)/server" client &&
+	grep "version 2" trace &&
+
+	# Ensure that the old version of the file is missing
+	git -C client rev-list master --quiet --objects --missing=print \
+		>observed.oids &&
+	grep "$(git -C server rev-parse message1:a.txt)" observed.oids &&
+
+	# Ensure that client passes fsck
+	git -C client fsck
+'
+
+test_expect_success 'dynamically fetch missing object' '
+	rm "$(pwd)/trace" &&
+	GIT_TRACE_PACKET="$(pwd)/trace" git -C client -c protocol.version=2 \
+		cat-file -p $(git -C server rev-parse message1:a.txt) &&
+	grep "version 2" trace
+'
+
+test_expect_success 'partial fetch' '
+	rm -rf client "$(pwd)/trace" &&
+	git init client &&
+	SERVER="file://$(pwd)/server" &&
+	test_config -C client extensions.partialClone "$SERVER" &&
+
+	GIT_TRACE_PACKET="$(pwd)/trace" git -C client -c protocol.version=2 \
+		fetch --filter=blob:none "$SERVER" master:refs/heads/other &&
+	grep "version 2" trace &&
+
+	# Ensure that the old version of the file is missing
+	git -C client rev-list other --quiet --objects --missing=print \
+		>observed.oids &&
+	grep "$(git -C server rev-parse message1:a.txt)" observed.oids &&
+
+	# Ensure that client passes fsck
+	git -C client fsck
+'
+
+test_expect_success 'do not advertise filter if not configured to do so' '
+	SERVER="file://$(pwd)/server" &&
+
+	rm "$(pwd)/trace" &&
+	git -C server config uploadpack.allowfilter 1 &&
+	GIT_TRACE_PACKET="$(pwd)/trace" git -c protocol.version=2 \
+		ls-remote "$SERVER" &&
+	grep "fetch=.*filter" trace &&
+
+	rm "$(pwd)/trace" &&
+	git -C server config uploadpack.allowfilter 0 &&
+	GIT_TRACE_PACKET="$(pwd)/trace" git -c protocol.version=2 \
+		ls-remote "$SERVER" &&
+	grep "fetch=" trace >fetch_capabilities &&
+	! grep filter fetch_capabilities
+'
+
+test_expect_success 'partial clone warns if filter is not advertised' '
+	rm -rf client &&
+	git -C server config uploadpack.allowfilter 0 &&
+	git -c protocol.version=2 \
+		clone --filter=blob:none "file://$(pwd)/server" client 2>err &&
+	test_i18ngrep "filtering not recognized by server, ignoring" err
+'
+
+test_expect_success 'even with handcrafted request, filter does not work if not advertised' '
+	git -C server config uploadpack.allowfilter 0 &&
+
+	# Custom request that tries to filter even though it is not advertised.
+	test-pkt-line pack >in <<-EOF &&
+	command=fetch
+	0001
+	want $(git -C server rev-parse master)
+	filter blob:none
+	0000
+	EOF
+
+	test_must_fail git -C server serve --stateless-rpc <in >/dev/null 2>err &&
+	grep "unexpected line: .filter blob:none." err &&
+
+	# Exercise to ensure that if advertised, filter works
+	git -C server config uploadpack.allowfilter 1 &&
+	git -C server serve --stateless-rpc <in >/dev/null
+'
+
 # Test protocol v2 with 'http://' transport
 #
 . "$TEST_DIRECTORY"/lib-httpd.sh

upload-pack.c

@@ -1205,6 +1205,7 @@ static void process_args(struct packet_reader *request,
 {
 	while (packet_reader_read(request) != PACKET_READ_FLUSH) {
 		const char *arg = request->line;
+		const char *p;
 		/* process want */
 		if (parse_want(arg))
@@ -1251,8 +1252,13 @@ static void process_args(struct packet_reader *request,
 			continue;
 		}
+		if (allow_filter && skip_prefix(arg, "filter ", &p)) {
+			parse_list_objects_filter(&filter_options, p);
+			continue;
+		}
+
 		/* ignore unknown lines maybe? */
-		die("unexpect line: '%s'", arg);
+		die("unexpected line: '%s'", arg);
 	}
 }
@@ -1376,6 +1382,8 @@ int upload_pack_v2(struct repository *r, struct argv_array *keys,
 	enum fetch_state state = FETCH_PROCESS_ARGS;
 	struct upload_pack_data data;
+	git_config(upload_pack_config, NULL);
+
 	upload_pack_data_init(&data);
 	use_sideband = LARGE_PACKET_MAX;
@@ -1428,7 +1436,14 @@ int upload_pack_v2(struct repository *r, struct argv_array *keys,
 int upload_pack_advertise(struct repository *r,
 			  struct strbuf *value)
 {
-	if (value)
+	if (value) {
+		int allow_filter_value;
 		strbuf_addstr(value, "shallow");
+		if (!repo_config_get_bool(the_repository,
+					 "uploadpack.allowfilter",
+					 &allow_filter_value) &&
+		    allow_filter_value)
+			strbuf_addstr(value, " filter");
+	}
 	return 1;
 }