status: deserialization wait

Teach `git status --deserialize` to either wait indefintely
or immediately fail if the status serialization cache file
is stale.

Signed-off-by: Jeff Hostetler <jeffhost@microsoft.com>

Author: jeffhostetler

Documentation/config/status.txt

@@ -81,3 +81,19 @@ status.deserializePath::
 	generated by `--serialize`.  This will be overridden by
 	`--deserialize=<path>` on the command line.  If the cache file is
 	invalid or stale, git will fall-back and compute status normally.
+
+status.deserializeWait::
+	EXPERIMENTAL, Specifies what `git status --deserialize` should do
+	if the serialization cache file is stale and whether it should
+	fall-back and compute status normally.  This will be overridden by
+	`--deserialize-wait=<value>` on the command line.
++
+--
+* `fail` - cause git to exit with an error when the status cache file
+is stale; this is intended for testing and debugging.
+* `block` - cause git to spin and periodically retry the cache file
+every 100 ms; this is intended to help coordinate with another git
+instance concurrently computing the cache file.
+* `no` - to immediately fall-back if cache file is stale.  This is the default.
+* `<timeout>` - time (in tenths of a second) to spin and retry.
+--

builtin/commit.c

@@ -182,6 +182,9 @@ static int do_implicit_deserialize = 0;
 static int do_explicit_deserialize = 0;
 static char *deserialize_path = NULL;
 
+static enum wt_status_deserialize_wait implicit_deserialize_wait = DESERIALIZE_WAIT__UNSET;
+static enum wt_status_deserialize_wait explicit_deserialize_wait = DESERIALIZE_WAIT__UNSET;
+
 /*
  * --serialize | --serialize=<path>
  *
@@ -247,6 +250,40 @@ static int opt_parse_deserialize(const struct option *opt, const char *arg, int
 	return 0;
 }
 
+static enum wt_status_deserialize_wait parse_dw(const char *arg)
+{
+	int tenths;
+
+	if (!strcmp(arg, "fail"))
+		return DESERIALIZE_WAIT__FAIL;
+	else if (!strcmp(arg, "block"))
+		return DESERIALIZE_WAIT__BLOCK;
+	else if (!strcmp(arg, "no"))
+		return DESERIALIZE_WAIT__NO;
+
+	/*
+	 * Otherwise, assume it is a timeout in tenths of a second.
+	 * If it contains a bogus value, atol() will return zero
+	 * which is OK.
+	 */
+	tenths = atol(arg);
+	if (tenths < 0)
+		tenths = DESERIALIZE_WAIT__NO;
+	return tenths;
+}
+
+static int opt_parse_deserialize_wait(const struct option *opt,
+				      const char *arg,
+				      int unset)
+{
+	if (unset)
+		explicit_deserialize_wait = DESERIALIZE_WAIT__UNSET;
+	else
+		explicit_deserialize_wait = parse_dw(arg);
+
+	return 0;
+}
+
 static int opt_parse_m(const struct option *opt, const char *arg, int unset)
 {
 	struct strbuf *buf = opt->value;
@@ -1550,6 +1587,13 @@ static int git_status_config(const char *k, const char *v,
 		}
 		return 0;
 	}
+	if (!strcmp(k, "status.deserializewait")) {
+		if (!v || !*v)
+			implicit_deserialize_wait = DESERIALIZE_WAIT__UNSET;
+		else
+			implicit_deserialize_wait = parse_dw(v);
+		return 0;
+	}
 	if (!strcmp(k, "status.showuntrackedfiles")) {
 		if (!v)
 			return config_error_nonbool(k);
@@ -1612,6 +1656,9 @@ int cmd_status(int argc, const char **argv, const char *prefix)
 		{ OPTION_CALLBACK, 0, "deserialize", NULL,
 		  N_("path"), N_("deserialize raw status data from file"),
 		  PARSE_OPT_OPTARG, opt_parse_deserialize },
+		{ OPTION_CALLBACK, 0, "deserialize-wait", NULL,
+		  N_("fail|block|no"), N_("how to wait if status cache file is invalid"),
+		  PARSE_OPT_OPTARG, opt_parse_deserialize_wait },
 		OPT_SET_INT(0, "long", &status_format,
 			    N_("show status in long format (default)"),
 			    STATUS_FORMAT_LONG),
@@ -1708,11 +1755,21 @@ int cmd_status(int argc, const char **argv, const char *prefix)
 	}
 
 	if (try_deserialize) {
+		int result;
+		enum wt_status_deserialize_wait dw = implicit_deserialize_wait;
+		if (explicit_deserialize_wait != DESERIALIZE_WAIT__UNSET)
+			dw = explicit_deserialize_wait;
+		if (dw == DESERIALIZE_WAIT__UNSET)
+			dw = DESERIALIZE_WAIT__NO;
+
 		if (s.relative_paths)
 			s.prefix = prefix;
 
-		if (wt_status_deserialize(&s, deserialize_path) == DESERIALIZE_OK)
+		result = wt_status_deserialize(&s, deserialize_path, dw);
+		if (result == DESERIALIZE_OK)
 			return 0;
+		if (dw == DESERIALIZE_WAIT__FAIL)
+			die(_("Rejected status serialization cache"));
 
 		/* deserialize failed, so force the initialization we skipped above. */
 		enable_fscache(1);

t/t7522-serialized-status.sh

@@ -199,6 +199,58 @@ test_expect_success 'verify new --serialize=path mode' '
 	test_cmp expect output.2
 '
 
+test_expect_success 'try deserialize-wait feature' '
+	test_when_finished "rm -f serialized_status.dat dirt expect.* output.* trace.*" &&
+
+	git status --serialize=serialized_status.dat >output.1 &&
+
+	# make status cache stale by updating the mtime on the index.  confirm that
+	# deserialize fails when requested.
+	sleep 1 &&
+	touch .git/index &&
+	test_must_fail git status --deserialize=serialized_status.dat --deserialize-wait=fail &&
+	test_must_fail git -c status.deserializeWait=fail status --deserialize=serialized_status.dat &&
+
+	cat >expect.1 <<-\EOF &&
+	? expect.1
+	? output.1
+	? serialized_status.dat
+	? untracked/
+	? untracked_1.txt
+	EOF
+
+	# refresh the status cache.
+	git status --porcelain=v2 --serialize=serialized_status.dat >output.1 &&
+	test_cmp expect.1 output.1 &&
+
+	# create some dirt. confirm deserialize used the existing status cache.
+	echo x >dirt &&
+	git status --porcelain=v2 --deserialize=serialized_status.dat >output.2 &&
+	test_cmp output.1 output.2 &&
+
+	# make the cache stale and try the timeout feature and wait upto
+	# 2 tenths of a second.  confirm deserialize timed out and rejected
+	# the status cache and did a normal scan.
+
+	cat >expect.2 <<-\EOF &&
+	? dirt
+	? expect.1
+	? expect.2
+	? output.1
+	? output.2
+	? serialized_status.dat
+	? trace.2
+	? untracked/
+	? untracked_1.txt
+	EOF
+
+	sleep 1 &&
+	touch .git/index &&
+	GIT_TRACE_DESERIALIZE=1 git status --porcelain=v2 --deserialize=serialized_status.dat --deserialize-wait=2 >output.2 2>trace.2 &&
+	test_cmp expect.2 output.2 &&
+	grep "wait polled=2 result=1" trace.2 >trace.2g
+'
+
 test_expect_success 'merge conflicts' '
 
 	# create a merge conflict.

wt-status-deserialize.c

@@ -61,7 +61,8 @@ static int my_validate_index(const struct cache_time *mtime_reported)
 	mtime_observed_on_disk.nsec = ST_MTIME_NSEC(st);
 	if ((mtime_observed_on_disk.sec != mtime_reported->sec) ||
 	    (mtime_observed_on_disk.nsec != mtime_reported->nsec)) {
-		trace_printf_key(&trace_deserialize, "index mtime changed [des %d.%d][obs %d.%d]",
+		trace_printf_key(&trace_deserialize,
+				 "index mtime changed [des %d %d][obs %d %d]",
 			     mtime_reported->sec, mtime_reported->nsec,
 			     mtime_observed_on_disk.sec, mtime_observed_on_disk.nsec);
 		return DESERIALIZE_ERR;
@@ -553,6 +554,8 @@ static inline int my_strcmp_null(const char *a, const char *b)
 
 static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *des_s, int fd)
 {
+	memset(des_s, 0, sizeof(*des_s));
+
 	/*
 	 * Check the path spec on the current command
 	 */
@@ -676,33 +679,127 @@ static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *de
 	return DESERIALIZE_OK;
 }
 
+static struct cache_time deserialize_prev_mtime = { 0, 0 };
+
+static int try_deserialize_read_from_file_1(const struct wt_status *cmd_s,
+					    const char *path,
+					    struct wt_status *des_s)
+{
+	struct stat st;
+	int result;
+	int fd;
+
+	/*
+	 * If we are spinning waiting for the status cache to become
+	 * valid, skip re-reading it if the mtime has not changed
+	 * since the last time we read it.
+	 */
+	if (lstat(path, &st)) {
+		trace_printf_key(&trace_deserialize,
+				 "could not lstat '%s'", path);
+		return DESERIALIZE_ERR;
+	}
+	if (st.st_mtime == deserialize_prev_mtime.sec &&
+	    ST_MTIME_NSEC(st) == deserialize_prev_mtime.nsec) {
+		trace_printf_key(&trace_deserialize,
+				 "mtime has not changed '%s'", path);
+		return DESERIALIZE_ERR;
+	}
+
+	fd = xopen(path, O_RDONLY);
+	if (fd == -1) {
+		trace_printf_key(&trace_deserialize,
+				 "could not read '%s'", path);
+		return DESERIALIZE_ERR;
+	}
+
+	deserialize_prev_mtime.sec = st.st_mtime;
+	deserialize_prev_mtime.nsec = ST_MTIME_NSEC(st);
+
+	trace_printf_key(&trace_deserialize,
+			 "reading serialization file (%d %d) '%s'",
+			 deserialize_prev_mtime.sec,
+			 deserialize_prev_mtime.nsec,
+			 path);
+
+	result = wt_deserialize_fd(cmd_s, des_s, fd);
+	close(fd);
+
+	return result;
+}
+
+static int try_deserialize_read_from_file(const struct wt_status *cmd_s,
+					  const char *path,
+					  enum wt_status_deserialize_wait dw,
+					  struct wt_status *des_s)
+{
+	int k, limit;
+	int result = DESERIALIZE_ERR;
+
+	/*
+	 * For "fail" or "no", try exactly once to read the status cache.
+	 * Return an error if the file is stale.
+	 */
+	if (dw == DESERIALIZE_WAIT__FAIL || dw == DESERIALIZE_WAIT__NO)
+		return try_deserialize_read_from_file_1(cmd_s, path, des_s);
+
+	/*
+	 * Wait for the status cache file to refresh.  Wait duration can
+	 * be in tenths of a second or unlimited.  Poll every 100ms.
+	 */
+	if (dw == DESERIALIZE_WAIT__BLOCK) {
+		/*
+		 * Convert "unlimited" to 1 day.
+		 */
+		limit = 10 * 60 * 60 * 24;
+	} else {
+		/* spin for dw tenths of a second */
+		limit = dw;
+	}
+	for (k = 0; k < limit; k++) {
+		result = try_deserialize_read_from_file_1(
+			cmd_s, path, des_s);
+
+		if (result == DESERIALIZE_OK)
+			break;
+
+		sleep_millisec(100);
+	}
+
+	trace_printf_key(&trace_deserialize,
+			 "wait polled=%d result=%d '%s'",
+			 k, result, path);
+	return result;
+}
+
 /*
- * Read raw serialized status data from the given file
+ * Read raw serialized status data from the given file (or STDIN).
  *
  * Verify that the args specified in the current command
  * are compatible with the deserialized data (such as "-uno").
  *
  * Copy display-related fields from the current command
  * into the deserialized data (so that the user can request
  * long or short as they please).
+ *
+ * Print status report using cached data.
  */
 int wt_status_deserialize(const struct wt_status *cmd_s,
-			  const char *path)
+			  const char *path,
+			  enum wt_status_deserialize_wait dw)
 {
 	struct wt_status des_s;
 	int result;
 
 	if (path && *path && strcmp(path, "0")) {
-		int fd = xopen(path, O_RDONLY);
-		if (fd == -1) {
-			trace_printf_key(&trace_deserialize, "could not read '%s'", path);
-			return DESERIALIZE_ERR;
-		}
-		trace_printf_key(&trace_deserialize, "reading serialization file '%s'", path);
-		result = wt_deserialize_fd(cmd_s, &des_s, fd);
-		close(fd);
+		result = try_deserialize_read_from_file(cmd_s, path, dw, &des_s);
 	} else {
 		trace_printf_key(&trace_deserialize, "reading stdin");
+
+		/*
+		 * Read status cache data from stdin.  Ignore the deserialize-wait
+		 * term, since we cannot read stdin multiple times.
+		 */
 		result = wt_deserialize_fd(cmd_s, &des_s, 0);
 	}
 

wt-status.h

@@ -217,6 +217,15 @@ struct wt_status_serialize_data
 		     - sizeof(struct wt_status_serialize_data_fixed)];
 };
 
+enum wt_status_deserialize_wait
+{
+	DESERIALIZE_WAIT__UNSET = -3,
+	DESERIALIZE_WAIT__FAIL = -2, /* return error, do not fallback */
+	DESERIALIZE_WAIT__BLOCK = -1, /* unlimited timeout */
+	DESERIALIZE_WAIT__NO = 0, /* immediately fallback */
+	/* any positive value is a timeout in tenths of a second */
+};
+
 /*
  * Serialize computed status scan results using "version 1" format
  * to the given file.
@@ -231,7 +240,8 @@ void wt_status_serialize_v1(int fd, struct wt_status *s);
  * fields.
  */
 int wt_status_deserialize(const struct wt_status *cmd_s,
-			  const char *path);
+			  const char *path,
+			  enum wt_status_deserialize_wait dw);
 
 /*
  * A helper routine for serialize and deserialize to compute