Sparse Index: log why the index is being expanded (#691)

I will intend to send this upstream after the 2.47.0 release cycle, but
this should get to our microsoft/git users for maximum impact.

Customers have been struggling with explaining why the sparse index
expansion advice message is showing up. The advice to run 'git clean'
has not always helped folks, and sometimes it is very unclear why we are
running into trouble.

These changes introduce a way to log a reason for the expansion into the
trace2 logs so it can be found by requesting that a user enable tracing.

While testing this, I created the most standard case that happens, which
is to have an existing directory match a sparse directory in the index.
In this case, it showed that two log messages were required. See the
last commit for this new log message. Together, these two places show
this kind of message in the `GIT_TRACE2_PERF` output (trimmed for
clarity):

```
region_enter | index        | label:clear_skip_worktree_from_present_files_sparse
data         | sparse-index | ..skip-worktree sparsedir:<my-sparse-path>/
data         | index        | ..sparse_path_count:362
data         | index        | ..sparse_lstat_count:732
region_leave | index        | label:clear_skip_worktree_from_present_files_sparse
data         | sparse-index | expansion-reason:failed to clear skip-worktree while sparse
```

I added some tests to demonstrate that these logs are recorded, but it
also seems difficult to hit some of these cases.

Author: dscho

Documentation/technical/sparse-index.adoc

@@ -206,3 +206,10 @@ Here are some commands that might be useful to update:
 * `git am`
 * `git clean`
 * `git stash`
+
+In order to help identify the cases where remaining index expansion is
+occurring in user machines, calls to `ensure_full_index()` have been
+replaced with `ensure_full_index_with_reason()` or with
+`ensure_full_index_unaudited()`. These versions add tracing that should
+help identify the reason for the index expansion without needing full
+access to someone's repository.

builtin/checkout-index.c

@@ -155,7 +155,7 @@ static int checkout_all(struct index_state *index, const char *prefix, int prefi
 			 * first entry inside the expanded sparse directory).
 			 */
 			if (ignore_skip_worktree) {
-				ensure_full_index(index);
+				ensure_full_index_with_reason(index, "checkout-index");
 				ce = index->cache[i];
 			}
 		}

builtin/commit.c

@@ -385,7 +385,7 @@ static int list_paths(struct string_list *list, const char *with_tree,
 	}
 
 	/* TODO: audit for interaction with sparse-index. */
-	ensure_full_index(the_repository->index);
+	ensure_full_index_unaudited(the_repository->index);
 	for (i = 0; i < the_repository->index->cache_nr; i++) {
 		const struct cache_entry *ce = the_repository->index->cache[i];
 		struct string_list_item *item;
@@ -1135,7 +1135,7 @@ static int prepare_to_commit(const char *index_file, const char *prefix,
 			int i, ita_nr = 0;
 
 			/* TODO: audit for interaction with sparse-index. */
-			ensure_full_index(the_repository->index);
+			ensure_full_index_unaudited(the_repository->index);
 			for (i = 0; i < the_repository->index->cache_nr; i++)
 				if (ce_intent_to_add(the_repository->index->cache[i]))
 					ita_nr++;

builtin/difftool.c

@@ -607,7 +607,7 @@ static int run_dir_diff(struct repository *repo,
 	ret = run_command(&cmd);
 
 	/* TODO: audit for interaction with sparse-index. */
-	ensure_full_index(&wtindex);
+	ensure_full_index_unaudited(&wtindex);
 
 	/*
 	 * If the diff includes working copy files and those

builtin/fsck.c

@@ -815,7 +815,7 @@ static void fsck_index(struct index_state *istate, const char *index_path,
 	unsigned int i;
 
 	/* TODO: audit for interaction with sparse-index. */
-	ensure_full_index(istate);
+	ensure_full_index_unaudited(istate);
 	for (i = 0; i < istate->cache_nr; i++) {
 		unsigned int mode;
 		struct blob *blob;

builtin/ls-files.c

@@ -414,7 +414,7 @@ static void show_files(struct repository *repo, struct dir_struct *dir)
 		return;
 
 	if (!show_sparse_dirs)
-		ensure_full_index(repo->index);
+		ensure_full_index_with_reason(repo->index, "ls-files");
 
 	for (i = 0; i < repo->index->cache_nr; i++) {
 		const struct cache_entry *ce = repo->index->cache[i];

builtin/merge-index.c

@@ -66,7 +66,7 @@ static void merge_all(void)
 {
 	int i;
 	/* TODO: audit for interaction with sparse-index. */
-	ensure_full_index(the_repository->index);
+	ensure_full_index_unaudited(the_repository->index);
 	for (i = 0; i < the_repository->index->cache_nr; i++) {
 		const struct cache_entry *ce = the_repository->index->cache[i];
 		if (!ce_stage(ce))
@@ -98,7 +98,7 @@ int cmd_merge_index(int argc,
 	repo_read_index(the_repository);
 
 	/* TODO: audit for interaction with sparse-index. */
-	ensure_full_index(the_repository->index);
+	ensure_full_index_unaudited(the_repository->index);
 
 	i = 1;
 	if (!strcmp(argv[i], "-o")) {

builtin/read-tree.c

@@ -231,7 +231,8 @@ int cmd_read_tree(int argc,
 		setup_work_tree();
 
 	if (opts.skip_sparse_checkout)
-		ensure_full_index(the_repository->index);
+		ensure_full_index_with_reason(the_repository->index,
+					      "read-tree");
 
 	if (opts.merge) {
 		switch (stage - 1) {

builtin/reset.c

@@ -262,7 +262,8 @@ static int read_from_tree(const struct pathspec *pathspec,
 	opt.add_remove = diff_addremove;
 
 	if (pathspec->nr && pathspec_needs_expanded_index(the_repository->index, pathspec))
-		ensure_full_index(the_repository->index);
+		ensure_full_index_with_reason(the_repository->index,
+					      "reset pathspec");
 
 	if (do_diff_cache(tree_oid, &opt))
 		return 1;

builtin/rm.c

@@ -310,7 +310,8 @@ int cmd_rm(int argc,
 	seen = xcalloc(pathspec.nr, 1);
 
 	if (pathspec_needs_expanded_index(the_repository->index, &pathspec))
-		ensure_full_index(the_repository->index);
+		ensure_full_index_with_reason(the_repository->index,
+					      "rm pathspec");
 
 	for (unsigned int i = 0; i < the_repository->index->cache_nr; i++) {
 		const struct cache_entry *ce = the_repository->index->cache[i];