unpack-trees: retain clean proofs across policy updates

Configured pulls preserve FSMonitor clean proofs when checkout can
authenticate every index change.  Tracked policy files were an
exception: adding or replacing .gitattributes or .gitignore made the
generic semantic transfer reject the whole proof.  Later read-only
status commands then had to rescan the worktree and could not restore
the paired untracked proof.

Let checkout retain history across regular policy-file changes that it
writes itself.  Attribute changes refresh the worktree manifest before
the provider boundary is rebound, and fail closed if that refresh cannot
authenticate the new sources.  Keep the existing untracked-cache
invalidation for ignore changes, and transfer that cache only while the
full tracked proof remains current.

Exercise configured fast-forward pulls in main and linked worktrees.  A
required-filter control also verifies that changed attributes invalidate
the affected tracked entry instead of certifying it.

Author: ttaylorr-oai

clean-status-history.c

@@ -2018,14 +2018,65 @@ int clean_status_transfer_current_proof_if_same_index(
 	return replace_current_fsmonitor_proof(dst, src);
 }
 
-int clean_status_transfer_current_proof_if_semantically_same_index(
-	struct index_state *dst, const struct index_state *src)
+enum checkout_policy_change {
+	CHECKOUT_POLICY_NONE = 0,
+	CHECKOUT_POLICY_IGNORE = 1 << 0,
+	CHECKOUT_POLICY_ATTRIBUTES = 1 << 1,
+};
+
+static enum checkout_policy_change checkout_policy_change_kind(
+	const struct cache_entry *old, const struct cache_entry *new_entry)
+{
+	const struct cache_entry *entry = old ? old : new_entry;
+	const char *base;
+
+	if (!entry ||
+	    (old && (!S_ISREG(old->ce_mode) || ce_stage(old) ||
+		     ce_skip_worktree(old) || ce_intent_to_add(old) ||
+		     (old->ce_flags & CE_VALID))) ||
+	    (new_entry && (!S_ISREG(new_entry->ce_mode) || ce_stage(new_entry) ||
+			    ce_skip_worktree(new_entry) ||
+			    ce_intent_to_add(new_entry) ||
+			    (new_entry->ce_flags & CE_VALID))) ||
+	    (old && new_entry &&
+	     (strcmp(old->name, new_entry->name) || old->ce_mode != new_entry->ce_mode)))
+		return CHECKOUT_POLICY_NONE;
+	base = strrchr(entry->name, '/');
+	base = base ? base + 1 : entry->name;
+	if (!fspathcmp(base, GITATTRIBUTES_FILE))
+		return CHECKOUT_POLICY_ATTRIBUTES;
+	if (!fspathcmp(base, ".gitignore"))
+		return CHECKOUT_POLICY_IGNORE;
+	return CHECKOUT_POLICY_NONE;
+}
+
+static int record_checkout_policy_change(
+	enum checkout_policy_change *policy_changes,
+	const struct cache_entry *old, const struct cache_entry *new_entry)
+{
+	enum checkout_policy_change change =
+		checkout_policy_change_kind(old, new_entry);
+
+	if (!change)
+		return 0;
+	*policy_changes |= change;
+	return 1;
+}
+
+static int transfer_current_proof_if_semantically_same_index(
+	struct index_state *dst, const struct index_state *src,
+	int allow_checkout_policy_changes,
+	int *manifest_refresh_required)
 {
 	const unsigned int semantic_flags =
 		CE_STAGEMASK | CE_VALID | CE_EXTENDED_FLAGS;
 	unsigned int src_pos = 0, dst_pos = 0;
+	enum checkout_policy_change policy_changes = CHECKOUT_POLICY_NONE;
 	int transferred;
 
+	if (manifest_refresh_required)
+		*manifest_refresh_required = 0;
+
 	if (!current_proof_is_writable(src) ||
 	    src->repo != dst->repo || src->split_index || dst->split_index ||
 	    src->sparse_index || dst->sparse_index ||
@@ -2050,25 +2101,37 @@ int clean_status_transfer_current_proof_if_semantically_same_index(
 			cmp = strcmp(old->name, new_entry->name);
 		if (cmp < 0) {
 			if (!clean_status_index_entry_is_semantically_safe(
-				    src, old, NULL))
+				    src, old, NULL) &&
+			    (!allow_checkout_policy_changes ||
+			     !record_checkout_policy_change(&policy_changes,
+							    old, NULL)))
 				return 0;
 			src_pos++;
 		} else if (cmp > 0) {
 			if (!clean_status_index_entry_is_semantically_safe(
-				    src, NULL, new_entry))
+				    src, NULL, new_entry) &&
+			    (!allow_checkout_policy_changes ||
+			     !record_checkout_policy_change(&policy_changes,
+							    NULL, new_entry)))
 				return 0;
 			dst_pos++;
 		} else {
 			if ((old->ce_mode != new_entry->ce_mode ||
 			     !oideq(&old->oid, &new_entry->oid) ||
 			     ((old->ce_flags ^ new_entry->ce_flags) & semantic_flags)) &&
 			    !clean_status_index_entry_is_semantically_safe(
-				    src, old, new_entry))
+				    src, old, new_entry) &&
+			    (!allow_checkout_policy_changes ||
+			     !record_checkout_policy_change(&policy_changes,
+							    old, new_entry)))
 				return 0;
 			src_pos++;
 			dst_pos++;
 		}
 	}
+	if (manifest_refresh_required &&
+	    (policy_changes & CHECKOUT_POLICY_ATTRIBUTES))
+		*manifest_refresh_required = 1;
 
 	if (current_proof_is_writable(dst)) {
 		const struct clean_status_state *src_state = src->clean_status;
@@ -2102,6 +2165,26 @@ int clean_status_transfer_current_proof_if_semantically_same_index(
 	return transferred;
 }
 
+int clean_status_transfer_current_proof_if_semantically_same_index(
+	struct index_state *dst, const struct index_state *src)
+{
+	return transfer_current_proof_if_semantically_same_index(
+		dst, src, 0, NULL);
+}
+
+int clean_status_transfer_current_proof_after_checkout(
+	struct index_state *dst, const struct index_state *src,
+	int *manifest_refresh_required)
+{
+	/*
+	 * A successful checkout owns these worktree writes.  It may therefore
+	 * retain history across policy-file changes, but the caller must refresh
+	 * changed attribute sources before pairing the transferred proof.
+	 */
+	return transfer_current_proof_if_semantically_same_index(
+		dst, src, 1, manifest_refresh_required);
+}
+
 struct clean_status_commit_checkpoint {
 	struct repository *repo;
 	struct lock_file *lock;

clean-status.h

@@ -162,6 +162,9 @@ int clean_status_transfer_current_proof_if_same_index(
 	struct index_state *dst, const struct index_state *src);
 int clean_status_transfer_current_proof_if_semantically_same_index(
 	struct index_state *dst, const struct index_state *src);
+int clean_status_transfer_current_proof_after_checkout(
+	struct index_state *dst, const struct index_state *src,
+	int *manifest_refresh_required);
 
 /*
  * A canonical main-index source may lend suspended historical state to an

t/t7519-status-fsmonitor.sh

@@ -2875,7 +2875,7 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \
 
 test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \
 	'configured pulls preserve authenticated worktree proofs' '
-	test_when_finished "rm -rf pull-proof-origin.git pull-proof-seed pull-proof-ff pull-proof-ff-linked pull-proof-rebase pull-proof-rebase-linked pull-proof-autostash pull-proof-autostash-linked" &&
+	test_when_finished "rm -rf pull-proof-origin.git pull-proof-seed pull-proof-ff pull-proof-ff-linked pull-proof-rebase pull-proof-rebase-linked pull-proof-autostash pull-proof-autostash-linked pull-proof-filter" &&
 	git init --bare pull-proof-origin.git &&
 	test_create_repo pull-proof-seed &&
 	(
@@ -2934,11 +2934,11 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \
 				case "$role" in
 				main)
 					worktree="$repo" &&
-					invalidated=96
+					invalidated=98
 					;;
 				linked)
 					worktree="$linked" &&
-					invalidated=192
+					invalidated=194
 					;;
 				esac &&
 				if test "$mode" != ff
@@ -2969,9 +2969,14 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \
 					do
 						test_write_lines "$mode-$role-$file" \
 							>"upstream-$mode-$role/nested/$file" ||
-							return 1
+								return 1
 					done &&
-					git add "upstream-$mode-$role"
+					test_write_lines "# $mode-$role" \
+						>.gitattributes &&
+					test_write_lines "*.ignored" "# $mode-$role" \
+						>.gitignore &&
+					git add "upstream-$mode-$role" \
+						.gitattributes .gitignore
 				else
 					test_write_lines "$mode-$role" \
 						>"upstream-$mode-$role" &&
@@ -3008,14 +3013,21 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \
 					<"$gitdir/pull.trace" &&
 				! test_trace2_data fsmonitor untracked/proof-missing 1 \
 					<"$gitdir/pull.trace" &&
-				! test_trace2_data fsmonitor \
-					semantic/manifest-scan-count 1 \
-					<"$gitdir/pull.trace" &&
 				if test "$mode" = ff
 				then
+					test_trace2_data fsmonitor \
+						semantic/manifest-scan-count 1 \
+						<"$gitdir/pull.trace" &&
+					test_trace2_data fsmonitor \
+						history/checkout-manifest-refreshed 1 \
+						<"$gitdir/pull.trace" &&
 					test_trace2_data fsmonitor \
 						history/untracked-paired-new-directory-invalidated \
 						"$invalidated" <"$gitdir/pull.trace"
+				else
+					! test_trace2_data fsmonitor \
+						semantic/manifest-scan-count 1 \
+						<"$gitdir/pull.trace"
 				fi &&
 				perl "$PWD/.git/check-pull-proof.pl" \
 					<"$gitdir/index" &&
@@ -3041,7 +3053,51 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \
 					semantic/manifest-scan-count 1 \
 					<"$gitdir/status.trace" || return 1
 			done || return 1
-		done
+		done &&
+		git clone --quiet "$PWD/../pull-proof-origin.git" \
+			"$PWD/../pull-proof-filter" &&
+		filter="$PWD/../pull-proof-filter" &&
+		git -C "$filter" config pull.ff only &&
+		git -C "$filter" config core.untrackedCache true &&
+		git -C "$filter" config core.fsmonitor true &&
+		git -C "$filter" config filter.pullproof.clean false &&
+		git -C "$filter" config filter.pullproof.required true &&
+		filter_gitdir=$(git -C "$filter" rev-parse --absolute-git-dir) &&
+		GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \
+			git -C "$filter" update-index --fsmonitor &&
+		GIT_INDEX_FILE="$filter_gitdir/index" \
+		GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \
+			git -C "$filter" status --porcelain=v2 \
+				>"$filter_gitdir/prime" &&
+		test_must_be_empty "$filter_gitdir/prime" &&
+		test_write_lines "tracked filter=pullproof" >.gitattributes &&
+		git add .gitattributes &&
+		git commit -qm "upstream-filter" &&
+		git push --quiet origin main &&
+		GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCC \
+		GIT_TEST_FSMONITOR_QUERY_PATH=tracked \
+		GIT_TRACE2_EVENT="$filter_gitdir/pull.trace" \
+			git -C "$filter" pull --quiet &&
+		test_trace2_data fsmonitor semantic/manifest-scan-count 1 \
+			<"$filter_gitdir/pull.trace" &&
+		test_trace2_data fsmonitor history/checkout-manifest-refreshed 1 \
+			<"$filter_gitdir/pull.trace" &&
+		test_trace2_data fsmonitor semantic/manifest-invalidated 1 \
+			<"$filter_gitdir/pull.trace" &&
+		test_trace2_data fsmonitor history/untracked-paired-transfer 1 \
+			<"$filter_gitdir/pull.trace" &&
+		cp "$filter_gitdir/index" "$filter_gitdir/status.before" &&
+		test_must_fail env GIT_OPTIONAL_LOCKS=0 \
+			GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \
+			GIT_TRACE2_EVENT="$filter_gitdir/status.trace" \
+			git -C "$filter" status --porcelain=v2 \
+				--untracked-files=no -- tracked \
+				>"$filter_gitdir/status" \
+				2>"$filter_gitdir/status.err" &&
+		test_grep "clean filter .pullproof. failed" \
+			"$filter_gitdir/status.err" &&
+		test_cmp_bin "$filter_gitdir/status.before" \
+			"$filter_gitdir/index"
 	)
 '
 

unpack-trees.c

@@ -2165,6 +2165,7 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options
 	ret = check_updates(o, &o->internal.result) ? (-2) : 0;
 	if (o->dst_index) {
 		int history_transferred = 0;
+		int manifest_refresh_required = 0;
 		int new_indexed_directory = 0;
 
 		if (!ret) {
@@ -2173,14 +2174,44 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options
 					&o->internal.result, o->src_index);
 			if (!history_transferred && o->preserve_semantic_history)
 				history_transferred =
-					clean_status_transfer_current_proof_if_semantically_same_index(
-						&o->internal.result, o->src_index);
+					clean_status_transfer_current_proof_after_checkout(
+						&o->internal.result, o->src_index,
+						&manifest_refresh_required);
 			if (history_transferred && o->preserve_semantic_history)
 				new_indexed_directory =
 					checkout_introduces_new_indexed_directory(
 						o->src_index, &o->internal.result);
 		}
 		move_index_extensions(&o->internal.result, o->src_index);
+		if (!ret && history_transferred && manifest_refresh_required) {
+			int manifest_refreshed = 0;
+
+			/*
+			 * The checkout has installed the new attribute sources.  Refresh
+			 * them before allowing the old provider boundary to authenticate
+			 * the resulting index and paired untracked cache.
+			 */
+			if (clean_status_refresh_worktree_manifest(
+				    &o->internal.result) < 0 ||
+			    clean_status_manifest_global_fallback(
+				    &o->internal.result)) {
+				clean_status_invalidate_current_proof(
+					&o->internal.result);
+				history_transferred = 0;
+			} else {
+				manifest_refreshed = 1;
+				clean_status_mark_fsmonitor_config_valid(
+					&o->internal.result,
+					o->internal.result.fsmonitor_last_update);
+				history_transferred =
+					clean_status_has_current_full_fsmonitor_proof(
+						&o->internal.result);
+			}
+			trace2_data_intmax(
+				"fsmonitor", repo,
+				"history/checkout-manifest-refreshed",
+				manifest_refreshed);
+		}
 		if (!ret && o->internal.backoff_transfer)
 			clean_status_transfer_backoff_history(
 				o->internal.backoff_transfer,