From e8b6f68f33c9b1b540ade46adf60a87d836a7d56 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 16 Jul 2024 09:17:15 -0400 Subject: [PATCH 001/289] t: remove advice from some tests These seem to be custom tests to microsoft/git as they break without these changes, but these changes are not needed upstream. Signed-off-by: Derrick Stolee --- t/t1091-sparse-checkout-builtin.sh | 1 + t/t7002-mv-sparse-checkout.sh | 3 +++ 2 files changed, 4 insertions(+) diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index 74b1761e0c8507..25a5550ee387ca 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -701,6 +701,7 @@ test_expect_success 'pattern-checks: contained glob characters' ' test_expect_success BSLASHPSPEC 'pattern-checks: escaped characters' ' git clone repo escaped && + git -C escaped config advice.sparseIndexExpanded false && TREEOID=$(git -C escaped rev-parse HEAD:folder1) && NEWTREE=$(git -C escaped mktree <<-EOF $(git -C escaped ls-tree HEAD) diff --git a/t/t7002-mv-sparse-checkout.sh b/t/t7002-mv-sparse-checkout.sh index 9c0e82ba3190a7..537948c5850d85 100755 --- a/t/t7002-mv-sparse-checkout.sh +++ b/t/t7002-mv-sparse-checkout.sh @@ -155,6 +155,9 @@ test_expect_success 'mv refuses to move sparse-to-non-sparse' ' test_expect_success 'recursive mv refuses to move (possible) sparse' ' test_when_finished rm -rf b c e sub2 && + + git config advice.sparseIndexExpanded false && + git reset --hard && # Without cone mode, "sub" and "sub2" do not match git sparse-checkout set sub/dir sub2/dir && From a837ddcb1d82213470b27b834791be3feaa24c62 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 29 Apr 2024 12:47:27 -0400 Subject: [PATCH 002/289] survey: calculate more stats on refs Calculate the number of symrefs, loose vs packed, and the maximal/accumulated length of local vs remote branches. Signed-off-by: Jeff Hostetler Signed-off-by: Johannes Schindelin --- Documentation/git-repo.adoc | 9 +++ builtin/repo.c | 65 +++++++++++++++++++- t/t1901-repo-structure.sh | 115 ++++++++++++++++++++++++++++++++++-- 3 files changed, 183 insertions(+), 6 deletions(-) diff --git a/Documentation/git-repo.adoc b/Documentation/git-repo.adoc index fd498abb7fa658..ff64a5f67d614c 100644 --- a/Documentation/git-repo.adoc +++ b/Documentation/git-repo.adoc @@ -61,6 +61,9 @@ supported: following kinds of information are reported: + * Reference counts categorized by type +* Symbolic reference counts +* Loose and packed reference counts (with the `files` reference backend) +* Maximum and total reference-name lengths for local and remote references * Reachable object counts categorized by type * Total inflated size of reachable objects by type * Total disk size of reachable objects by type @@ -70,6 +73,12 @@ supported: + By default every reference enumerated by `for-each-ref` contributes to the counts and object walk. Use `--ref-filter` to narrow the scope. ++ +Symbolic references also contribute to their usual reference-type counts. +Reference-name lengths are measured in bytes, including the full `refs/` +prefix. Remote references are those under `refs/remotes/`; all other +references, including tags, are considered local for these measurements. +Loose and packed counts are zero for reference backends other than `files`. `--ref-filter=`;; Only count references whose full name matches one of the diff --git a/builtin/repo.c b/builtin/repo.c index 1d3a4a79523202..1e5328efa82783 100644 --- a/builtin/repo.c +++ b/builtin/repo.c @@ -357,6 +357,13 @@ struct ref_stats { size_t tags; size_t annotated_tags; size_t others; + size_t symbolic; + size_t loose; + size_t packed; + size_t max_local_refname_length; + size_t total_local_refname_length; + size_t max_remote_refname_length; + size_t total_remote_refname_length; }; struct object_values { @@ -547,6 +554,21 @@ static void stats_table_setup_structure(struct stats_table *table, " * %s", _("Annotated")); stats_table_count_addf(table, refs->remotes, " * %s", _("Remotes")); stats_table_count_addf(table, refs->others, " * %s", _("Others")); + stats_table_count_addf(table, refs->symbolic, + " * %s", _("Symbolic refs")); + stats_table_count_addf(table, refs->loose, " * %s", _("Loose refs")); + stats_table_count_addf(table, refs->packed, " * %s", _("Packed refs")); + stats_table_addf(table, " * %s", _("Refname length")); + stats_table_addf(table, " * %s", _("Local")); + stats_table_count_addf(table, refs->max_local_refname_length, + " * %s", _("Maximum")); + stats_table_count_addf(table, refs->total_local_refname_length, + " * %s", _("Total")); + stats_table_addf(table, " * %s", _("Remote")); + stats_table_count_addf(table, refs->max_remote_refname_length, + " * %s", _("Maximum")); + stats_table_count_addf(table, refs->total_remote_refname_length, + " * %s", _("Total")); object_count_total = get_total_object_values(&objects->type_counts); stats_table_addf(table, ""); @@ -817,6 +839,20 @@ static void structure_keyvalue_print(struct repo_structure *stats, stats->refs.remotes, value_delim); print_keyvalue("references.others.count", key_delim, stats->refs.others, value_delim); + print_keyvalue("references.symbolic.count", key_delim, + stats->refs.symbolic, value_delim); + print_keyvalue("references.loose.count", key_delim, + stats->refs.loose, value_delim); + print_keyvalue("references.packed.count", key_delim, + stats->refs.packed, value_delim); + print_keyvalue("references.local.max_length", key_delim, + stats->refs.max_local_refname_length, value_delim); + print_keyvalue("references.local.total_length", key_delim, + stats->refs.total_local_refname_length, value_delim); + print_keyvalue("references.remotes.max_length", key_delim, + stats->refs.max_remote_refname_length, value_delim); + print_keyvalue("references.remotes.total_length", key_delim, + stats->refs.total_remote_refname_length, value_delim); print_keyvalue("objects.commits.count", key_delim, stats->objects.type_counts.commits, value_delim); @@ -898,12 +934,14 @@ static int count_references(const struct reference *ref, void *cb_data) { struct count_references_data *data = cb_data; struct ref_stats *stats = data->stats; - size_t ref_count; + size_t ref_count, refname_length; + unsigned int ref_kind; if (!ref_matches_any_filter(ref->name, data->filters)) return 0; - switch (ref_kind_from_refname(ref->name)) { + ref_kind = ref_kind_from_refname(ref->name); + switch (ref_kind) { case FILTER_REFS_BRANCHES: stats->branches++; break; @@ -923,6 +961,29 @@ static int count_references(const struct reference *ref, void *cb_data) BUG("unexpected reference type"); } + if (ref->flags & REF_ISSYMREF) + stats->symbolic++; + + if (data->repo->ref_storage_format == REF_STORAGE_FORMAT_FILES) { + /* A symref can inherit REF_ISPACKED from its target. */ + if ((ref->flags & REF_ISPACKED) && + !(ref->flags & REF_ISSYMREF)) + stats->packed++; + else + stats->loose++; + } + + refname_length = strlen(ref->name); + if (ref_kind == FILTER_REFS_REMOTES) { + stats->total_remote_refname_length += refname_length; + if (refname_length > stats->max_remote_refname_length) + stats->max_remote_refname_length = refname_length; + } else { + stats->total_local_refname_length += refname_length; + if (refname_length > stats->max_local_refname_length) + stats->max_local_refname_length = refname_length; + } + /* * While iterating through references for counting, also add OIDs in * preparation for the path walk. diff --git a/t/t1901-repo-structure.sh b/t/t1901-repo-structure.sh index c2c198c43e9922..23b51e33251056 100755 --- a/t/t1901-repo-structure.sh +++ b/t/t1901-repo-structure.sh @@ -36,6 +36,16 @@ test_expect_success 'empty repository' ' | * Annotated | 0 | | * Remotes | 0 | | * Others | 0 | + | * Symbolic refs | 0 | + | * Loose refs | 0 | + | * Packed refs | 0 | + | * Refname length | | + | * Local | | + | * Maximum | 0 | + | * Total | 0 | + | * Remote | | + | * Maximum | 0 | + | * Total | 0 | | | | | * Reachable objects | | | * Count | 0 | @@ -76,7 +86,7 @@ test_expect_success 'empty repository' ' test_expect_success SHA1 'repository with references and objects' ' test_when_finished "rm -rf repo" && - git init repo && + git init --initial-branch=main --ref-format=files repo && ( cd repo && test_commit_bulk 1005 && @@ -84,6 +94,8 @@ test_expect_success SHA1 'repository with references and objects' ' oid="$(git rev-parse HEAD)" && git update-ref refs/remotes/origin/foo "$oid" && + git symbolic-ref refs/remotes/origin/HEAD \ + refs/remotes/origin/foo && # Also creates a commit, tree, and blob. git notes add -m foo && @@ -95,12 +107,22 @@ test_expect_success SHA1 'repository with references and objects' ' | Repository structure | Value | | ------------------------- | ---------- | | * References | | - | * Count | 4 | + | * Count | 5 | | * Branches | 1 | | * Tags | 1 | | * Annotated | 1 | - | * Remotes | 1 | + | * Remotes | 2 | | * Others | 1 | + | * Symbolic refs | 1 | + | * Loose refs | 5 | + | * Packed refs | 0 | + | * Refname length | | + | * Local | | + | * Maximum | 18 | + | * Total | 46 | + | * Remote | | + | * Maximum | 24 | + | * Total | 47 | | | | | * Reachable objects | | | * Count | 3.02 k | @@ -148,7 +170,7 @@ test_expect_success SHA1 'repository with references and objects' ' test_expect_success SHA1 'lines and nul format' ' test_when_finished "rm -rf repo" && - git init repo && + git init --initial-branch=main --ref-format=files repo && ( cd repo && test_commit_bulk 42 && @@ -161,6 +183,13 @@ test_expect_success SHA1 'lines and nul format' ' references.tags.annotated.count=1 references.remotes.count=0 references.others.count=0 + references.symbolic.count=0 + references.loose.count=3 + references.packed.count=0 + references.local.max_length=53 + references.local.total_length=81 + references.remotes.max_length=0 + references.remotes.total_length=0 objects.commits.count=42 objects.trees.count=42 objects.blobs.count=42 @@ -228,6 +257,84 @@ test_expect_success 'progress meter option' ' ) ' +for ref_format in files reftable +do + test_expect_success "$ref_format reference statistics" ' + test_when_finished "rm -rf repo" && + git init --initial-branch=main \ + --ref-format="$ref_format" repo && + ( + cd repo && + test_commit --no-tag one && + git tag v1 && + git update-ref refs/notes/commits HEAD && + git update-ref refs/remotes/origin/long-branch HEAD && + git symbolic-ref refs/heads/alias refs/heads/main && + git symbolic-ref refs/remotes/origin/HEAD \ + refs/remotes/origin/long-branch && + git pack-refs --all && + test_commit --no-tag two && + + if test "$ref_format" = files + then + loose=3 && + packed=3 + else + loose=0 && + packed=0 + fi && + cat >expect <<-EOF && + references.branches.count=2 + references.tags.count=1 + references.tags.annotated.count=0 + references.remotes.count=2 + references.others.count=1 + references.symbolic.count=2 + references.loose.count=$loose + references.packed.count=$packed + references.local.max_length=18 + references.local.total_length=61 + references.remotes.max_length=31 + references.remotes.total_length=55 + EOF + git repo structure --format=lines >out && + sed -n "/^references\./p" out >actual && + test_cmp expect actual && + + if test "$ref_format" = files + then + loose=1 && + packed=1 + fi && + cat >expect <<-EOF && + references.branches.count=0 + references.tags.count=0 + references.tags.annotated.count=0 + references.remotes.count=2 + references.others.count=0 + references.symbolic.count=1 + references.loose.count=$loose + references.packed.count=$packed + references.local.max_length=0 + references.local.total_length=0 + references.remotes.max_length=31 + references.remotes.total_length=55 + EOF + git repo structure --format=nul \ + --ref-filter="refs/remotes/origin/*" >out && + tr "\012\000" "=\012" decoded && + sed -n "/^references\./p" decoded >actual && + test_cmp expect actual && + + sed "s/=[0-9]*$/=0/" expect >expect-empty && + git repo structure --format=lines \ + --ref-filter="refs/does-not-exist/" >out && + sed -n "/^references\./p" out >actual && + test_cmp expect-empty actual + ) + ' +done + test_expect_success '--ref-filter narrows the set of refs' ' test_when_finished "rm -rf repo" && git init repo && From d2d8b2b84fa55e0827127c0a365d0778c273606a Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 29 Apr 2024 15:40:00 -0400 Subject: [PATCH 003/289] survey: show some commits/trees/blobs histograms With this commit, we gather statistics about the sizes of commits, trees, and blobs in the repository, and then present them in the form of "hexbins", i.e. log(16) histograms that show how many objects fall into the 0..15 bytes range, the 16..255 range, the 256..4095 range, etc. For commits, we also show the total count grouped by the number of parents, and for trees we additionally show the total count grouped by number of entries in the form of "qbins", i.e. log(4) histograms. Signed-off-by: Jeff Hostetler Signed-off-by: Johannes Schindelin --- Documentation/git-repo.adoc | 23 +++++ builtin/repo.c | 169 ++++++++++++++++++++++++++++++-- t/t1901-repo-structure.sh | 189 +++++++++++++++++++++++++++++++++++- 3 files changed, 371 insertions(+), 10 deletions(-) diff --git a/Documentation/git-repo.adoc b/Documentation/git-repo.adoc index ff64a5f67d614c..624e716e7753d2 100644 --- a/Documentation/git-repo.adoc +++ b/Documentation/git-repo.adoc @@ -68,6 +68,8 @@ supported: * Total inflated size of reachable objects by type * Total disk size of reachable objects by type * Largest reachable objects in the repository by type +* Histograms of commit, tree, and blob sizes, tree entry counts, and + commit parent counts * Optionally, the top-_N_ largest paths by count, on-disk size, and inflated size (see `--top` below) + @@ -124,6 +126,27 @@ supported: + `-z` is an alias for `--format=nul`. +STRUCTURE HISTOGRAMS +-------------------- +Size histograms group commits, trees, and blobs by inflated size in bytes: +`0..15`, `16..255`, `256..4095`, and so on. Tree entry histograms use the +ranges `0..3`, `4..15`, `16..63`, and so on. Each bucket reports the number +of objects and their total inflated and on-disk sizes. Empty buckets are +omitted. + +The commit parent histogram reports exact parent counts from 0 through 30. +Its final bucket, shown as `31+` in tables, includes all commits with at +least 31 parents. + +In `lines` and `nul` output, size histogram keys have the form +`objects..histogram.size..`, where `` is +`commits`, `trees`, or `blobs`, and `` is `count`, `inflated_size`, +or `disk_size`. Tree entry histograms use +`objects.trees.histogram.entries..`. Bucket numbers start +at zero for the first range listed above. Parent counts use +`objects.commits.histogram.parents..count`, with `31` identifying +the final bucket. All histograms honor `--ref-filter`. + INFO KEYS --------- In order to obtain a set of values from `git repo info`, you should provide diff --git a/builtin/repo.c b/builtin/repo.c index 1e5328efa82783..19b738f4c88eef 100644 --- a/builtin/repo.c +++ b/builtin/repo.c @@ -373,6 +373,19 @@ struct object_values { size_t blobs; }; +/* Log16 size buckets and log4 entry buckets cover the full size_t range. */ +#define HBIN_SHIFT 4 +#define HBIN_LEN (sizeof(size_t) * CHAR_BIT / HBIN_SHIFT) +#define QBIN_SHIFT 2 +#define QBIN_LEN (sizeof(size_t) * CHAR_BIT / QBIN_SHIFT) +#define PBIN_VEC_LEN 32 + +struct object_histogram_bin { + size_t count; + size_t inflated_size; + size_t disk_size; +}; + struct object_stats { struct object_values type_counts; struct object_values inflated_sizes; @@ -380,6 +393,11 @@ struct object_stats { struct largest_objects largest; struct top_paths top_trees; struct top_paths top_blobs; + struct object_histogram_bin commit_sizes[HBIN_LEN]; + struct object_histogram_bin tree_sizes[HBIN_LEN]; + struct object_histogram_bin blob_sizes[HBIN_LEN]; + struct object_histogram_bin tree_entries[QBIN_LEN]; + size_t commit_parents[PBIN_VEC_LEN]; }; struct repo_structure { @@ -676,9 +694,9 @@ static void stats_table_setup_top_paths(struct stats_table *table, #define INDEX_WIDTH 4 -static void stats_table_print_structure(const struct stats_table *table) +static void stats_table_print(const struct stats_table *table, + const char *name_col_title) { - const char *name_col_title = _("Repository structure"); const char *value_col_title = _("Value"); int title_name_width = utf8_strwidth(name_col_title); int title_value_width = utf8_strwidth(value_col_title); @@ -763,6 +781,72 @@ static void stats_table_clear(struct stats_table *table) string_list_clear(&table->annotations, 1); } +static void histogram_table_print(const char *title, + const struct object_histogram_bin *bins, + size_t nr, unsigned int shift) +{ + struct stats_table table = { + .rows = STRING_LIST_INIT_DUP, + .annotations = STRING_LIST_INIT_DUP, + }; + + for (size_t i = 0; i < nr; i++) { + size_t lower, upper; + + if (!bins[i].count) + continue; + + lower = i ? (size_t)1 << (i * shift) : 0; + upper = SIZE_MAX >> (sizeof(size_t) * CHAR_BIT - + (i + 1) * shift); + stats_table_addf(&table, "* %" PRIuMAX "..%" PRIuMAX, + (uintmax_t)lower, (uintmax_t)upper); + stats_table_count_addf(&table, bins[i].count, + " * %s", _("Count")); + stats_table_size_addf(&table, bins[i].inflated_size, + " * %s", _("Inflated size")); + stats_table_size_addf(&table, bins[i].disk_size, + " * %s", _("Disk size")); + } + + if (table.rows.nr) { + putchar('\n'); + stats_table_print(&table, title); + } + stats_table_clear(&table); +} + +static void structure_histograms_table_print(struct object_stats *stats) +{ + struct stats_table table = { + .rows = STRING_LIST_INIT_DUP, + .annotations = STRING_LIST_INIT_DUP, + }; + + for (size_t i = 0; i < ARRAY_SIZE(stats->commit_parents); i++) { + if (!stats->commit_parents[i]) + continue; + stats_table_count_addf(&table, stats->commit_parents[i], + "%" PRIuMAX "%s", (uintmax_t)i, + i == PBIN_VEC_LEN - 1 ? "+" : ""); + } + + if (table.rows.nr) { + putchar('\n'); + stats_table_print(&table, _("Commit parent histogram")); + } + stats_table_clear(&table); + + histogram_table_print(_("Commit size histogram"), stats->commit_sizes, + ARRAY_SIZE(stats->commit_sizes), HBIN_SHIFT); + histogram_table_print(_("Tree entry histogram"), stats->tree_entries, + ARRAY_SIZE(stats->tree_entries), QBIN_SHIFT); + histogram_table_print(_("Tree size histogram"), stats->tree_sizes, + ARRAY_SIZE(stats->tree_sizes), HBIN_SHIFT); + histogram_table_print(_("Blob size histogram"), stats->blob_sizes, + ARRAY_SIZE(stats->blob_sizes), HBIN_SHIFT); +} + static inline void print_keyvalue(const char *key, char key_delim, size_t value, char value_delim) { @@ -826,6 +910,26 @@ static void top_paths_keyvalue_print(const char *prefix, } } +static void histogram_keyvalue_print(const char *prefix, + const struct object_histogram_bin *bins, + size_t nr, char key_delim, + char value_delim) +{ + for (size_t i = 0; i < nr; i++) { + if (!bins[i].count) + continue; + + printf("%s.%" PRIuMAX ".", prefix, (uintmax_t)i); + print_keyvalue("count", key_delim, bins[i].count, value_delim); + printf("%s.%" PRIuMAX ".", prefix, (uintmax_t)i); + print_keyvalue("inflated_size", key_delim, + bins[i].inflated_size, value_delim); + printf("%s.%" PRIuMAX ".", prefix, (uintmax_t)i); + print_keyvalue("disk_size", key_delim, + bins[i].disk_size, value_delim); + } +} + static void structure_keyvalue_print(struct repo_structure *stats, char key_delim, char value_delim) { @@ -900,6 +1004,32 @@ static void structure_keyvalue_print(struct repo_structure *stats, top_paths_keyvalue_print("objects.blobs.top", &stats->objects.top_blobs, key_delim, value_delim); + for (size_t i = 0; i < ARRAY_SIZE(stats->objects.commit_parents); i++) { + if (!stats->objects.commit_parents[i]) + continue; + printf("objects.commits.histogram.parents.%" PRIuMAX ".", + (uintmax_t)i); + print_keyvalue("count", key_delim, + stats->objects.commit_parents[i], value_delim); + } + + histogram_keyvalue_print("objects.commits.histogram.size", + stats->objects.commit_sizes, + ARRAY_SIZE(stats->objects.commit_sizes), + key_delim, value_delim); + histogram_keyvalue_print("objects.trees.histogram.entries", + stats->objects.tree_entries, + ARRAY_SIZE(stats->objects.tree_entries), + key_delim, value_delim); + histogram_keyvalue_print("objects.trees.histogram.size", + stats->objects.tree_sizes, + ARRAY_SIZE(stats->objects.tree_sizes), + key_delim, value_delim); + histogram_keyvalue_print("objects.blobs.histogram.size", + stats->objects.blob_sizes, + ARRAY_SIZE(stats->objects.blob_sizes), + key_delim, value_delim); + fflush(stdout); } @@ -1126,6 +1256,19 @@ static size_t count_tree_entries(struct object *obj) return count; } +static void increment_histogram(struct object_histogram_bin *bins, + unsigned int shift, size_t value, + size_t inflated, off_t disk) +{ + size_t bin = 0; + + while (value >>= shift) + bin++; + bins[bin].count++; + bins[bin].inflated_size += inflated; + bins[bin].disk_size += disk; +} + static int count_objects(const char *path, struct oid_array *oids, enum object_type type, void *cb_data) { @@ -1136,7 +1279,7 @@ static int count_objects(const char *path, struct oid_array *oids, for (size_t i = 0; i < oids->nr; i++) { struct object_info oi = OBJECT_INFO_INIT; - size_t inflated; + size_t inflated, count; struct commit *commit; struct object *obj; void *content; @@ -1169,22 +1312,33 @@ static int count_objects(const char *path, struct oid_array *oids, break; case OBJ_COMMIT: commit = object_as_type(obj, OBJ_COMMIT, 0); + count = commit_list_count(commit->parents); stats->type_counts.commits++; stats->inflated_sizes.commits += inflated; stats->disk_sizes.commits += disk; check_largest(&stats->largest.commit_size, &oids->oid[i], inflated); check_largest(&stats->largest.parent_count, &oids->oid[i], - commit_list_count(commit->parents)); + count); + if (count >= PBIN_VEC_LEN) + count = PBIN_VEC_LEN - 1; + stats->commit_parents[count]++; + increment_histogram(stats->commit_sizes, HBIN_SHIFT, + inflated, inflated, disk); break; case OBJ_TREE: + count = count_tree_entries(obj); stats->type_counts.trees++; stats->inflated_sizes.trees += inflated; stats->disk_sizes.trees += disk; check_largest(&stats->largest.tree_size, &oids->oid[i], inflated); check_largest(&stats->largest.tree_entries, &oids->oid[i], - count_tree_entries(obj)); + count); + increment_histogram(stats->tree_entries, QBIN_SHIFT, + count, inflated, disk); + increment_histogram(stats->tree_sizes, HBIN_SHIFT, + inflated, inflated, disk); break; case OBJ_BLOB: stats->type_counts.blobs++; @@ -1192,6 +1346,8 @@ static int count_objects(const char *path, struct oid_array *oids, stats->disk_sizes.blobs += disk; check_largest(&stats->largest.blob_size, &oids->oid[i], inflated); + increment_histogram(stats->blob_sizes, HBIN_SHIFT, + inflated, inflated, disk); break; default: BUG("invalid object type"); @@ -1321,7 +1477,8 @@ static int cmd_repo_structure(int argc, const char **argv, const char *prefix, case FORMAT_TABLE: stats_table_setup_structure(&table, &stats); stats_table_setup_top_paths(&table, &stats.objects); - stats_table_print_structure(&table); + stats_table_print(&table, _("Repository structure")); + structure_histograms_table_print(&stats.objects); break; case FORMAT_NEWLINE_TERMINATED: structure_keyvalue_print(&stats, '=', '\n'); diff --git a/t/t1901-repo-structure.sh b/t/t1901-repo-structure.sh index 23b51e33251056..fb33556b6dac75 100755 --- a/t/t1901-repo-structure.sh +++ b/t/t1901-repo-structure.sh @@ -21,6 +21,38 @@ object_type_disk_usage() { fi } +check_structure_summary() { + sed -n "1,$(wc -l summary && + test_cmp expect summary +} + +expected_size_histograms() { + git rev-list --all --objects --no-object-names >oids && + git cat-file \ + --batch-check="%(objecttype) %(objectsize) %(objectsize:disk)" \ + sizes && + awk ' + $1 == "tag" { next } + { + bin = 0 + for (size = $2; size >= 16; size = int(size / 16)) + bin++ + key = "objects." $1 "s.histogram.size." bin + count[key]++ + inflated[key] += $2 + disk[key] += $3 + } + END { + for (key in count) { + printf "%s.count=%.0f\n", key, count[key] + printf "%s.inflated_size=%.0f\n", + key, inflated[key] + printf "%s.disk_size=%.0f\n", key, disk[key] + } + } + ' sizes +} + test_expect_success 'empty repository' ' test_when_finished "rm -rf repo" && git init repo && @@ -163,7 +195,7 @@ test_expect_success SHA1 'repository with references and objects' ' git repo structure >out 2>err && - test_cmp expect out && + check_structure_summary out && test_line_count = 0 err ) ' @@ -218,13 +250,13 @@ test_expect_success SHA1 'lines and nul format' ' git repo structure --format=lines >out 2>err && - test_cmp expect out && + check_structure_summary out && test_line_count = 0 err && git repo structure --format=nul >out 2>err && tr "\012\000" "=\012" actual && - test_cmp expect actual && + check_structure_summary actual && test_line_count = 0 err && # "-z", as a synonym to "--format=nul", participates in the @@ -232,7 +264,7 @@ test_expect_success SHA1 'lines and nul format' ' git repo structure --format=table -z >out 2>err && tr "\012\000" "=\012" actual && - test_cmp expect actual && + check_structure_summary actual && test_line_count = 0 err ) ' @@ -257,6 +289,155 @@ test_expect_success 'progress meter option' ' ) ' +test_expect_success 'object histograms cover size and entry boundaries' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + for size in 0 1 15 16 255 256 4095 4096 + do + test-tool genzeros "$size" >blob-$size && + git add blob-$size || return 1 + done && + tree=$(git write-tree) && + commit=$(git commit-tree "$tree" -m blobs) && + git update-ref refs/heads/blobs "$commit" && + empty_blob=$(git hash-object -w --stdin tree-input && + tree=$(git mktree expect-unsorted && + sort expect-unsorted >expect-sizes && + git repo structure --format=lines >out && + sed -n "/^objects\..*\.histogram\.size\./p" \ + out >actual-unsorted && + sort actual-unsorted >actual-sizes && + test_cmp expect-sizes actual-sizes && + + cat >expect <<-\EOF && + objects.blobs.histogram.size.0.count=3 + objects.blobs.histogram.size.1.count=2 + objects.blobs.histogram.size.2.count=2 + objects.blobs.histogram.size.3.count=1 + EOF + sed -n "/^objects\.blobs\.histogram\..*\.count=/p" \ + out >actual && + test_cmp expect actual && + + key=objects.trees.histogram.entries && + cat >expect <<-EOF && + $key.0.count=2 + $key.1.count=3 + $key.2.count=2 + $key.3.count=1 + EOF + sed -n "/^$key\..*\.count=/p" out >actual && + test_cmp expect actual && + + git repo structure --format=nul >nul && + tr "\012\000" "=\012" decoded && + test_cmp out decoded && + + git repo structure >table && + for kind in "Commit parent" "Commit size" \ + "Tree entry" "Tree size" "Blob size" + do + test_grep "| $kind histogram " table || + return 1 + done && + sed -n "/^| Blob size histogram /,\$p" table >blobs && + for range in 0..15 16..255 256..4095 4096..65535 + do + test_grep -F "| * $range " blobs || return 1 + done && + + tree=$(git rev-parse refs/heads/tree-16^{tree}) && + format="%(objectsize) %(objectsize:disk)" && + echo "$tree" | + git cat-file --batch-check="$format" \ + >tree-size && + read inflated disk expect <<-EOF && + $key.2.count=1 + $key.2.inflated_size=$inflated + $key.2.disk_size=$disk + EOF + git repo structure --format=lines \ + --ref-filter=refs/heads/tree-16 >filtered && + sed -n "/^objects\.trees\.histogram\.entries\./p" \ + filtered >actual && + test_cmp expect actual || return 1 + done && + + git repo structure --format=lines \ + --ref-filter=refs/heads/missing >out && + test_grep ! "\.histogram\." out + ) +' + +test_expect_success 'commit parent histogram groups 31 or more parents' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + tree=$(git mktree expect <<-\EOF && + objects.commits.histogram.parents.0.count=32 + objects.commits.histogram.parents.1.count=1 + objects.commits.histogram.parents.2.count=1 + objects.commits.histogram.parents.31.count=2 + EOF + git repo structure --format=lines >out && + sed -n "/^objects\.commits\.histogram\.parents\./p" \ + out >actual && + test_cmp expect actual && + test_grep "^objects.commits.max_parents=32$" out && + git repo structure >table && + test_grep "^| 31+ *| *2 *|$" table + ) +' + for ref_format in files reftable do test_expect_success "$ref_format reference statistics" ' From 4a763a51f3afa8b9f7282a905b2ec1e1d3d62959 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 1 May 2024 12:56:38 -0400 Subject: [PATCH 004/289] survey: add vector of largest objects for various scaling dimensions Create `struct large_item` and `struct large_item_vec` to capture the n largest commits, trees, and blobs under various scaling dimensions, such as size in bytes, number of commit parents, or number of entries in a tree. Each of these have a command line option to set them independently. Signed-off-by: Jeff Hostetler --- Documentation/config/repo.adoc | 22 ++++ Documentation/git-repo.adoc | 41 ++++++ builtin/repo.c | 231 ++++++++++++++++++++++++++++++--- t/t1901-repo-structure.sh | 226 +++++++++++++++++++++++++++++++- 4 files changed, 499 insertions(+), 21 deletions(-) diff --git a/Documentation/config/repo.adoc b/Documentation/config/repo.adoc index 7f8cd632965aab..f7c34f398682b6 100644 --- a/Documentation/config/repo.adoc +++ b/Documentation/config/repo.adoc @@ -8,4 +8,26 @@ repo.structure.*:: number of largest paths to report in each detail table. Must be non-negative; defaults to `0`, which disables the detail tables. + showBlobSizes:: + A non-negative integer specifying the default for + `--blob-sizes=` in linkgit:git-repo[1]. Defaults to `0`, + which disables the list of largest blobs. + showCommitParents:: + A non-negative integer specifying the default for + `--commit-parents=` in linkgit:git-repo[1]. Defaults to + `0`, which disables the list of commits with the most + parents. + showCommitSizes:: + A non-negative integer specifying the default for + `--commit-sizes=` in linkgit:git-repo[1]. Defaults to + `0`, which disables the list of largest commits. + showTreeEntries:: + A non-negative integer specifying the default for + `--tree-entries=` in linkgit:git-repo[1]. Defaults to + `0`, which disables the list of trees with the most + entries. + showTreeSizes:: + A non-negative integer specifying the default for + `--tree-sizes=` in linkgit:git-repo[1]. Defaults to `0`, + which disables the list of largest trees. -- diff --git a/Documentation/git-repo.adoc b/Documentation/git-repo.adoc index 624e716e7753d2..ff840e5105ccb9 100644 --- a/Documentation/git-repo.adoc +++ b/Documentation/git-repo.adoc @@ -102,6 +102,33 @@ Loose and packed counts are zero for reference backends other than `files`. can also be set via the `repo.structure.top` configuration variable; an explicit `--top=` on the command line overrides the configured value. + + `--commit-parents=`;; + Report the _n_ commits with the most parents. + + `--commit-sizes=`;; + Report the _n_ largest commits by inflated size in bytes, + usually those with the largest commit messages. + + `--tree-entries=`;; + Report the _n_ trees with the most entries (files and + subdirectories). + + `--tree-sizes=`;; + Report the _n_ largest trees by inflated size in bytes. + This ranking can differ from the entry-count ranking + when entry names have very different lengths. + + `--blob-sizes=`;; + Report the _n_ largest blobs by inflated size in bytes. ++ +These five object-list limits are independent of each other and of `--top`, +which ranks paths rather than individual objects. Each limit must be +non-negative and defaults to `0`, which disables that list. Configuration +variables under `repo.structure.show*` provide defaults (see "CONFIGURATION" +below); an explicit command-line limit overrides the corresponding default. +Lists show the measured values and OIDs in descending order, with at most +_n_ entries, and honor `--ref-filter`. + The output format can be chosen through the flag `--format`. Three formats are supported: @@ -147,6 +174,16 @@ at zero for the first range listed above. Parent counts use `objects.commits.histogram.parents..count`, with `31` identifying the final bucket. All histograms honor `--ref-filter`. +STRUCTURE OBJECT LISTS +---------------------- +In `lines` and `nul` output, the optional object lists use keys of the form +`objects..largest...`. Ranks start at 1. +For commits, the dimensions are `by_parents` and `by_size`; for trees they +are `by_entries` and `by_size`; blobs support `by_size`. Each ranked object +has an `oid` field. Its value field is named `parents` for `by_parents`, +`entries` for `by_entries`, or `inflated_size` for `by_size`. Parent counts +in these lists are not capped by the histogram's final bucket. + INFO KEYS --------- In order to obtain a set of values from `git repo info`, you should provide @@ -199,6 +236,10 @@ using the `nul` format: git repo info --format=nul layout.bare layout.shallow ------------ +CONFIGURATION +------------- +include::config/repo.adoc[] + SEE ALSO -------- linkgit:git-rev-parse[1] diff --git a/builtin/repo.c b/builtin/repo.c index 19b738f4c88eef..60ca7231cd15a6 100644 --- a/builtin/repo.c +++ b/builtin/repo.c @@ -307,6 +307,12 @@ struct object_data { size_t value; }; +struct top_objects { + size_t nr; + size_t alloc; + struct object_data *data; +}; + struct largest_objects { struct object_data tag_size; struct object_data commit_size; @@ -398,6 +404,11 @@ struct object_stats { struct object_histogram_bin blob_sizes[HBIN_LEN]; struct object_histogram_bin tree_entries[QBIN_LEN]; size_t commit_parents[PBIN_VEC_LEN]; + struct top_objects top_commit_parents; + struct top_objects top_commit_sizes; + struct top_objects top_tree_entries; + struct top_objects top_tree_sizes; + struct top_objects top_blob_sizes; }; struct repo_structure { @@ -703,16 +714,20 @@ static void stats_table_print(const struct stats_table *table, int name_col_width = table->name_col_width; int value_col_width = table->value_col_width; int unit_col_width = table->unit_col_width; + int index_width = INDEX_WIDTH; struct string_list_item *item; struct strbuf buf = STRBUF_INIT; + for (size_t n = table->annotations.nr; n >= 10; n /= 10) + index_width++; + if (title_name_width > name_col_width) name_col_width = title_name_width; if (title_value_width > value_col_width + unit_col_width + 1) value_col_width = title_value_width - unit_col_width; strbuf_addstr(&buf, "| "); - strbuf_utf8_align(&buf, ALIGN_LEFT, name_col_width + INDEX_WIDTH, + strbuf_utf8_align(&buf, ALIGN_LEFT, name_col_width + index_width, name_col_title); strbuf_addstr(&buf, " | "); strbuf_utf8_align(&buf, ALIGN_LEFT, @@ -721,7 +736,7 @@ static void stats_table_print(const struct stats_table *table, printf("%s\n", buf.buf); printf("| "); - for (int i = 0; i < name_col_width + INDEX_WIDTH; i++) + for (int i = 0; i < name_col_width + index_width; i++) putchar('-'); printf(" | "); for (int i = 0; i < value_col_width + unit_col_width + 1; i++) @@ -743,11 +758,16 @@ static void stats_table_print(const struct stats_table *table, strbuf_addstr(&buf, "| "); strbuf_utf8_align(&buf, ALIGN_LEFT, name_col_width, item->string); - if (entry && entry->oid) + if (entry && entry->oid) { + size_t len = buf.len; + strbuf_addf(&buf, " [%" PRIuMAX "]", (uintmax_t)entry->index); - else - strbuf_addchars(&buf, ' ', INDEX_WIDTH); + strbuf_addchars(&buf, ' ', + index_width - (buf.len - len)); + } else { + strbuf_addchars(&buf, ' ', index_width); + } strbuf_addstr(&buf, " | "); strbuf_utf8_align(&buf, ALIGN_RIGHT, value_col_width, value); @@ -847,6 +867,50 @@ static void structure_histograms_table_print(struct object_stats *stats) ARRAY_SIZE(stats->blob_sizes), HBIN_SHIFT); } +static void top_objects_table_print(const char *title, struct top_objects *top, + int by_size) +{ + struct stats_table table = { + .rows = STRING_LIST_INIT_DUP, + .annotations = STRING_LIST_INIT_DUP, + }; + + for (size_t i = 0; i < top->nr; i++) { + struct object_data *item = &top->data[i]; + + if (by_size) + stats_table_object_size_addf(&table, &item->oid, + item->value, + "%" PRIuMAX, + (uintmax_t)(i + 1)); + else + stats_table_object_count_addf(&table, &item->oid, + item->value, + "%" PRIuMAX, + (uintmax_t)(i + 1)); + } + + if (table.rows.nr) { + putchar('\n'); + stats_table_print(&table, title); + } + stats_table_clear(&table); +} + +static void structure_top_objects_table_print(struct object_stats *stats) +{ + top_objects_table_print(_("Largest commits by parent count"), + &stats->top_commit_parents, 0); + top_objects_table_print(_("Largest commits by size"), + &stats->top_commit_sizes, 1); + top_objects_table_print(_("Largest trees by entry count"), + &stats->top_tree_entries, 0); + top_objects_table_print(_("Largest trees by size"), + &stats->top_tree_sizes, 1); + top_objects_table_print(_("Largest blobs by size"), + &stats->top_blob_sizes, 1); +} + static inline void print_keyvalue(const char *key, char key_delim, size_t value, char value_delim) { @@ -930,6 +994,20 @@ static void histogram_keyvalue_print(const char *prefix, } } +static void top_objects_keyvalue_print(const char *prefix, const char *metric, + const struct top_objects *top, + char key_delim, char value_delim) +{ + for (size_t i = 0; i < top->nr; i++) { + printf("%s.%" PRIuMAX ".", prefix, (uintmax_t)(i + 1)); + print_keyvalue(metric, key_delim, top->data[i].value, + value_delim); + printf("%s.%" PRIuMAX ".oid%c%s%c", prefix, + (uintmax_t)(i + 1), key_delim, + oid_to_hex(&top->data[i].oid), value_delim); + } +} + static void structure_keyvalue_print(struct repo_structure *stats, char key_delim, char value_delim) { @@ -1030,6 +1108,26 @@ static void structure_keyvalue_print(struct repo_structure *stats, ARRAY_SIZE(stats->objects.blob_sizes), key_delim, value_delim); + top_objects_keyvalue_print("objects.commits.largest.by_parents", + "parents", + &stats->objects.top_commit_parents, + key_delim, value_delim); + top_objects_keyvalue_print("objects.commits.largest.by_size", + "inflated_size", + &stats->objects.top_commit_sizes, + key_delim, value_delim); + top_objects_keyvalue_print("objects.trees.largest.by_entries", + "entries", &stats->objects.top_tree_entries, + key_delim, value_delim); + top_objects_keyvalue_print("objects.trees.largest.by_size", + "inflated_size", + &stats->objects.top_tree_sizes, + key_delim, value_delim); + top_objects_keyvalue_print("objects.blobs.largest.by_size", + "inflated_size", + &stats->objects.top_blob_sizes, + key_delim, value_delim); + fflush(stdout); } @@ -1242,6 +1340,34 @@ static void check_largest(struct object_data *data, struct object_id *oid, } } +static void init_top_objects(struct top_objects *top, int limit, + const char *option) +{ + if (limit < 0) + die(_("--%s= must be non-negative"), option); + top->alloc = limit; + if (limit) + ALLOC_ARRAY(top->data, top->alloc); +} + +static void maybe_insert_top_object(struct top_objects *top, + const struct object_id *oid, size_t value) +{ + size_t pos = top->nr; + + while (pos > 0 && value >= top->data[pos - 1].value) + pos--; + if (pos >= top->alloc) + return; + if (top->nr < top->alloc) + top->nr++; + for (size_t i = top->nr - 1; i > pos; i--) + top->data[i] = top->data[i - 1]; + + oidcpy(&top->data[pos].oid, oid); + top->data[pos].value = value; +} + static size_t count_tree_entries(struct object *obj) { struct tree *t = object_as_type(obj, OBJ_TREE, 0); @@ -1320,6 +1446,10 @@ static int count_objects(const char *path, struct oid_array *oids, inflated); check_largest(&stats->largest.parent_count, &oids->oid[i], count); + maybe_insert_top_object(&stats->top_commit_parents, + &oids->oid[i], count); + maybe_insert_top_object(&stats->top_commit_sizes, + &oids->oid[i], inflated); if (count >= PBIN_VEC_LEN) count = PBIN_VEC_LEN - 1; stats->commit_parents[count]++; @@ -1335,6 +1465,10 @@ static int count_objects(const char *path, struct oid_array *oids, inflated); check_largest(&stats->largest.tree_entries, &oids->oid[i], count); + maybe_insert_top_object(&stats->top_tree_entries, + &oids->oid[i], count); + maybe_insert_top_object(&stats->top_tree_sizes, + &oids->oid[i], inflated); increment_histogram(stats->tree_entries, QBIN_SHIFT, count, inflated, disk); increment_histogram(stats->tree_sizes, HBIN_SHIFT, @@ -1346,6 +1480,8 @@ static int count_objects(const char *path, struct oid_array *oids, stats->disk_sizes.blobs += disk; check_largest(&stats->largest.blob_size, &oids->oid[i], inflated); + maybe_insert_top_object(&stats->top_blob_sizes, + &oids->oid[i], inflated); increment_histogram(stats->blob_sizes, HBIN_SHIFT, inflated, inflated, disk); break; @@ -1403,16 +1539,39 @@ static void structure_count_objects(struct object_stats *stats, stop_progress(&data.progress); } +struct repo_structure_opts { + int top_nr; + int commit_parents; + int commit_sizes; + int tree_entries; + int tree_sizes; + int blob_sizes; +}; + static int repo_structure_config_cb(const char *var, const char *value, const struct config_context *cctx, void *cb) { - int *top_nr = cb; - - if (!strcmp(var, "repo.structure.top")) { - *top_nr = git_config_int(var, value, cctx->kvi); - if (*top_nr < 0) - die(_("repo.structure.top must be non-negative")); + struct repo_structure_opts *opts = cb; + int *limit = NULL; + + if (!strcmp(var, "repo.structure.top")) + limit = &opts->top_nr; + else if (!strcmp(var, "repo.structure.showcommitparents")) + limit = &opts->commit_parents; + else if (!strcmp(var, "repo.structure.showcommitsizes")) + limit = &opts->commit_sizes; + else if (!strcmp(var, "repo.structure.showtreeentries")) + limit = &opts->tree_entries; + else if (!strcmp(var, "repo.structure.showtreesizes")) + limit = &opts->tree_sizes; + else if (!strcmp(var, "repo.structure.showblobsizes")) + limit = &opts->blob_sizes; + + if (limit) { + *limit = git_config_int(var, value, cctx->kvi); + if (*limit < 0) + die(_("%s must be non-negative"), var); return 0; } @@ -1428,9 +1587,9 @@ static int cmd_repo_structure(int argc, const char **argv, const char *prefix, }; enum output_format format = FORMAT_TABLE; struct repo_structure stats = { 0 }; + struct repo_structure_opts opts = { 0 }; struct rev_info revs; int show_progress = -1; - int top_nr = 0; struct string_list ref_filters = STRING_LIST_INIT_DUP; struct option options[] = { OPT_CALLBACK_F(0, "format", &format, N_("format"), @@ -1444,33 +1603,59 @@ static int cmd_repo_structure(int argc, const char **argv, const char *prefix, OPT_STRING_LIST(0, "ref-filter", &ref_filters, N_("pattern"), N_("only count refs matching ; " "repeat to union multiple patterns")), - OPT_INTEGER(0, "top", &top_nr, + OPT_INTEGER(0, "top", &opts.top_nr, N_("report the top largest paths " "per category")), + OPT_INTEGER_F(0, "commit-parents", &opts.commit_parents, + N_("show commits with the most parents"), + PARSE_OPT_NONEG), + OPT_INTEGER_F(0, "commit-sizes", &opts.commit_sizes, + N_("show largest commits by size in bytes"), + PARSE_OPT_NONEG), + OPT_INTEGER_F(0, "tree-entries", &opts.tree_entries, + N_("show trees with the most entries"), + PARSE_OPT_NONEG), + OPT_INTEGER_F(0, "tree-sizes", &opts.tree_sizes, + N_("show largest trees by size in bytes"), + PARSE_OPT_NONEG), + OPT_INTEGER_F(0, "blob-sizes", &opts.blob_sizes, + N_("show largest blobs by size in bytes"), + PARSE_OPT_NONEG), OPT_END() }; - repo_config(repo, repo_structure_config_cb, &top_nr); + repo_config(repo, repo_structure_config_cb, &opts); argc = parse_options(argc, argv, prefix, options, repo_structure_usage, 0); if (argc) usage(_("too many arguments")); - if (top_nr < 0) + if (opts.top_nr < 0) die(_("--top= must be non-negative")); + init_top_objects(&stats.objects.top_commit_parents, + opts.commit_parents, "commit-parents"); + init_top_objects(&stats.objects.top_commit_sizes, + opts.commit_sizes, "commit-sizes"); + init_top_objects(&stats.objects.top_tree_entries, + opts.tree_entries, "tree-entries"); + init_top_objects(&stats.objects.top_tree_sizes, + opts.tree_sizes, "tree-sizes"); + init_top_objects(&stats.objects.top_blob_sizes, + opts.blob_sizes, "blob-sizes"); + repo_init_revisions(repo, &revs, prefix); if (show_progress < 0) show_progress = isatty(2); - if (top_nr) { - init_top_paths(&stats.objects.top_trees, top_nr); - init_top_paths(&stats.objects.top_blobs, top_nr); + if (opts.top_nr) { + init_top_paths(&stats.objects.top_trees, opts.top_nr); + init_top_paths(&stats.objects.top_blobs, opts.top_nr); } structure_count_references(&stats.refs, &revs, repo, &ref_filters, show_progress); - structure_count_objects(&stats.objects, &revs, repo, top_nr, + structure_count_objects(&stats.objects, &revs, repo, opts.top_nr, show_progress); switch (format) { @@ -1479,6 +1664,7 @@ static int cmd_repo_structure(int argc, const char **argv, const char *prefix, stats_table_setup_top_paths(&table, &stats.objects); stats_table_print(&table, _("Repository structure")); structure_histograms_table_print(&stats.objects); + structure_top_objects_table_print(&stats.objects); break; case FORMAT_NEWLINE_TERMINATED: structure_keyvalue_print(&stats, '=', '\n'); @@ -1492,10 +1678,15 @@ static int cmd_repo_structure(int argc, const char **argv, const char *prefix, stats_table_clear(&table); string_list_clear(&ref_filters, 0); - if (top_nr) { + if (opts.top_nr) { clear_top_paths(&stats.objects.top_trees); clear_top_paths(&stats.objects.top_blobs); } + free(stats.objects.top_commit_parents.data); + free(stats.objects.top_commit_sizes.data); + free(stats.objects.top_tree_entries.data); + free(stats.objects.top_tree_sizes.data); + free(stats.objects.top_blob_sizes.data); release_revisions(&revs); return 0; diff --git a/t/t1901-repo-structure.sh b/t/t1901-repo-structure.sh index fb33556b6dac75..9e7dc69bcc5002 100755 --- a/t/t1901-repo-structure.sh +++ b/t/t1901-repo-structure.sh @@ -434,7 +434,231 @@ test_expect_success 'commit parent histogram groups 31 or more parents' ' test_cmp expect actual && test_grep "^objects.commits.max_parents=32$" out && git repo structure >table && - test_grep "^| 31+ *| *2 *|$" table + test_grep "^| 31+ *| *2 *|$" table && + + key=objects.commits.largest.by_parents && + boundary=$(git rev-parse refs/heads/boundary) && + cat >expect <<-EOF && + $key.1.parents=32 + $key.1.oid=$commit + $key.2.parents=31 + $key.2.oid=$boundary + $key.3.parents=2 + $key.3.oid=$two + EOF + git repo structure --format=lines --commit-parents=3 >out && + sed -n "/^$key\./p" out >actual && + test_cmp expect actual && + + for limit in 9 10 12 + do + git repo structure --commit-parents=$limit >table && + sed -n "/^| Largest commits by parent count /,/^$/p" \ + table >parents && + test_line_count = $((limit + 3)) parents && + awk " + !NF { next } + !width { width = length } + length != width { + print length, width + exit 1 + } + " parents || return 1 + done + ) +' + +test_expect_success 'largest object lists have independent sorted limits' ' + test_when_finished "rm -rf repo" && + git init --initial-branch=main repo && + ( + cd repo && + empty_blob=$(git hash-object -w --stdin big && + big=$(git hash-object -w big) && + printf abcdefghijklmnop >tied && + tied=$(git hash-object -w tied) && + empty_tree=$(git mktree tree-input <<-EOF && + 100644 blob $empty_blob a + 100644 blob $big b + 100644 blob $tied c + EOF + wide=$(git mktree tree-input <<-EOF && + 100644 blob $empty_blob $long-a + 100644 blob $big $long-b + EOF + narrow=$(git mktree message && + root=$(git commit-tree "$empty_tree" -F message) && + other=$(git commit-tree "$narrow" -m other) && + child=$(git commit-tree "$wide" -p "$root" -m child) && + merge=$(git commit-tree "$wide" -p "$child" -p "$other" \ + -m merge) && + git update-ref refs/heads/main "$merge" && + git update-ref refs/heads/empty "$root" && + root_size=$(git cat-file -s "$root") && + narrow_size=$(git cat-file -s "$narrow") && + commit_parents=objects.commits.largest.by_parents && + commit_sizes=objects.commits.largest.by_size && + tree_entries=objects.trees.largest.by_entries && + tree_sizes=objects.trees.largest.by_size && + blob_sizes=objects.blobs.largest.by_size && + + set -- --commit-parents=2 --commit-sizes=1 \ + --tree-entries=4 --tree-sizes=1 --blob-sizes=4 && + for storage in loose packed + do + if test "$storage" = packed + then + git repack -ad + fi && + git repo structure --format=lines "$@" >out && + cat >expect <<-EOF && + $commit_parents.1.parents=2 + $commit_parents.1.oid=$merge + $commit_parents.2.parents=1 + $commit_parents.2.oid=$child + $commit_sizes.1.inflated_size=$root_size + $commit_sizes.1.oid=$root + $tree_entries.1.entries=3 + $tree_entries.1.oid=$wide + $tree_entries.2.entries=2 + $tree_entries.2.oid=$narrow + $tree_entries.3.entries=0 + $tree_entries.3.oid=$empty_tree + $tree_sizes.1.inflated_size=$narrow_size + $tree_sizes.1.oid=$narrow + EOF + sed -n "/^$blob_sizes\./d; /\.largest\./p" \ + out >actual && + test_cmp expect actual && + cat >expect <<-EOF && + $blob_sizes.1.inflated_size=16 + $blob_sizes.2.inflated_size=16 + $blob_sizes.3.inflated_size=0 + EOF + sed -n "/^$blob_sizes\..*\.inflated_size=/p" \ + out >actual && + test_cmp expect actual && + printf "%s\n" "$big" "$tied" >expect-unsorted && + sort expect-unsorted >expect && + sed -n "s/^$blob_sizes\.[12]\.oid=//p" \ + out >actual-unsorted && + sort actual-unsorted >actual && + test_cmp expect actual && + test_grep "^$blob_sizes.3.oid=$empty_blob$" out && + test_grep ! "^$blob_sizes.4." out && + + git repo structure --format=nul "$@" >nul && + tr "\012\000" "=\012" decoded && + test_cmp out decoded && + git repo structure "$@" >table && + sed -n "/^| Largest commits by parent count /,/^$/p" \ + table >parents && + test_grep "^| 1 *\[1\] | *2 *|$" parents && + sed -n "/^| Largest blobs by size /,/^$/p" \ + table >blobs && + test_grep "^| 1 *\[1\] | *16 B *|$" blobs || + return 1 + done && + + git repo structure --format=lines "$@" \ + --ref-filter=refs/heads/empty >out && + test_grep "^$commit_parents.1.parents=0$" out && + test_grep ! "^$commit_parents.2." out && + test_grep "^$tree_entries.1.entries=0$" out && + test_grep ! "^$tree_entries.2." out && + test_grep ! "^$blob_sizes." out && + git repo structure --format=lines "$@" \ + --ref-filter=refs/heads/missing >out && + test_grep ! "\.largest\." out + ) +' + +for spec in \ + "commit-parents showCommitParents commits by_parents parents" \ + "commit-sizes showCommitSizes commits by_size inflated_size" \ + "tree-entries showTreeEntries trees by_entries entries" \ + "tree-sizes showTreeSizes trees by_size inflated_size" \ + "blob-sizes showBlobSizes blobs by_size inflated_size" +do + set -- $spec + option=$1 config=$2 type=$3 dimension=$4 metric=$5 + + test_expect_success "--$option is opt-in and overrides its config" ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit --no-tag one file && + case "$type" in + commits) oid=$(git rev-parse HEAD) ;; + trees) oid=$(git rev-parse HEAD^{tree}) ;; + blobs) oid=$(git rev-parse HEAD:file) ;; + esac && + case "$metric" in + parents) value=0 ;; + entries) value=1 ;; + inflated_size) value=$(git cat-file -s "$oid") ;; + esac && + key=objects.$type.largest.$dimension && + cat >expect <<-EOF && + $key.1.$metric=$value + $key.1.oid=$oid + EOF + git repo structure --format=lines >default && + test_grep ! "\.largest\." default && + git repo structure --format=lines "--$option=3" >out && + sed -n "/\.largest\./p" out >actual && + test_cmp expect actual && + git -c repo.structure.$config=3 repo structure \ + --format=lines >configured && + test_cmp out configured && + git -c repo.structure.$config=3 repo structure \ + --format=lines "--$option=0" >disabled && + test_cmp default disabled && + git -c repo.structure.$config=0 repo structure \ + --format=lines "--$option=3" >override && + test_cmp out override && + + test_commit --no-tag two other "larger content" && + git -c repo.structure.$config=1 repo structure \ + --format=lines >limit-one && + git -c repo.structure.$config=2 repo structure \ + --format=lines >limit-two && + git repo structure --format=lines "--$option=2" \ + >expected-two && + test_cmp expected-two limit-two && + sed -n "/\.largest\..*\.oid=/p" limit-one >oids && + test_line_count = 1 oids && + sed -n "/\.largest\..*\.oid=/p" limit-two >oids && + test_line_count = 2 oids && + + test_must_fail git repo structure "--$option=-1" \ + 2>err && + test_grep "must be non-negative" err && + test_must_fail git -c repo.structure.$config=-1 \ + repo structure 2>err && + test_grep "must be non-negative" err && + test_must_fail git repo structure "--$option=bad" \ + 2>err + ) + ' +done + +test_expect_success 'largest object lists are empty in an empty repository' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + git repo structure >expect && + git repo structure --commit-parents=3 --commit-sizes=3 \ + --tree-entries=3 --tree-sizes=3 --blob-sizes=3 \ + >actual && + test_cmp expect actual ) ' From 48f7678231ca7c1b498983b6bc0ac649d55bd94c Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 15 May 2024 15:56:36 -0400 Subject: [PATCH 005/289] survey: add pathname of blob or tree to large_item_vec Include the pathname of each blob or tree in the large_item_vec to help identify the file or directory associated with the OID and size information. This pathname is computed during the path walk, so it reflects the first observed pathname seen for that OID during the traversal over all of the refs. Since the file or directory could have moved (without being modified), there may be multiple "correct" pathnames for a particular OID. Since we do not control the ref traversal order, we should consider it to be a "suggested pathname" for the OID. Signed-off-by: Jeff Hostetler --- Documentation/git-repo.adoc | 7 ++ builtin/repo.c | 77 ++++++++++++++------- t/t1901-repo-structure.sh | 134 +++++++++++++++++++++++++++++++++++- 3 files changed, 194 insertions(+), 24 deletions(-) diff --git a/Documentation/git-repo.adoc b/Documentation/git-repo.adoc index ff840e5105ccb9..dec160769d2381 100644 --- a/Documentation/git-repo.adoc +++ b/Documentation/git-repo.adoc @@ -184,6 +184,13 @@ has an `oid` field. Its value field is named `parents` for `by_parents`, `entries` for `by_entries`, or `inflated_size` for `by_size`. Parent counts in these lists are not capped by the histogram's final bucket. +Tree and blob entries also have a `path` field containing the first path +associated with that object by the path walk. This is a suggested location, +not necessarily a path in `HEAD`: an unchanged object may occur at multiple +paths. An empty path denotes a root tree. Table and `lines` output quote +unusual paths according to `core.quotePath`; `nul` output preserves the +path bytes without quoting. + INFO KEYS --------- In order to obtain a set of values from `git repo info`, you should provide diff --git a/builtin/repo.c b/builtin/repo.c index 60ca7231cd15a6..21951fe00f36a6 100644 --- a/builtin/repo.c +++ b/builtin/repo.c @@ -307,10 +307,15 @@ struct object_data { size_t value; }; +struct top_object { + struct object_data object; + char *path; +}; + struct top_objects { size_t nr; size_t alloc; - struct object_data *data; + struct top_object *data; }; struct largest_objects { @@ -874,26 +879,34 @@ static void top_objects_table_print(const char *title, struct top_objects *top, .rows = STRING_LIST_INIT_DUP, .annotations = STRING_LIST_INIT_DUP, }; + struct strbuf label = STRBUF_INIT; for (size_t i = 0; i < top->nr; i++) { - struct object_data *item = &top->data[i]; + struct object_data *item = &top->data[i].object; + const char *path = top->data[i].path; + + strbuf_reset(&label); + strbuf_addf(&label, "%" PRIuMAX, (uintmax_t)(i + 1)); + if (path && *path) { + strbuf_addstr(&label, ": "); + quote_c_style(path, &label, NULL, 0); + } if (by_size) stats_table_object_size_addf(&table, &item->oid, item->value, - "%" PRIuMAX, - (uintmax_t)(i + 1)); + "%s", label.buf); else stats_table_object_count_addf(&table, &item->oid, item->value, - "%" PRIuMAX, - (uintmax_t)(i + 1)); + "%s", label.buf); } if (table.rows.nr) { putchar('\n'); stats_table_print(&table, title); } + strbuf_release(&label); stats_table_clear(&table); } @@ -999,12 +1012,19 @@ static void top_objects_keyvalue_print(const char *prefix, const char *metric, char key_delim, char value_delim) { for (size_t i = 0; i < top->nr; i++) { + const struct top_object *item = &top->data[i]; + printf("%s.%" PRIuMAX ".", prefix, (uintmax_t)(i + 1)); - print_keyvalue(metric, key_delim, top->data[i].value, + print_keyvalue(metric, key_delim, item->object.value, value_delim); printf("%s.%" PRIuMAX ".oid%c%s%c", prefix, (uintmax_t)(i + 1), key_delim, - oid_to_hex(&top->data[i].oid), value_delim); + oid_to_hex(&item->object.oid), value_delim); + if (item->path) { + printf("%s.%" PRIuMAX ".", prefix, (uintmax_t)(i + 1)); + print_keyvalue_path("path", key_delim, item->path, + value_delim); + } } } @@ -1350,22 +1370,33 @@ static void init_top_objects(struct top_objects *top, int limit, ALLOC_ARRAY(top->data, top->alloc); } +static void clear_top_objects(struct top_objects *top) +{ + for (size_t i = 0; i < top->nr; i++) + free(top->data[i].path); + free(top->data); +} + static void maybe_insert_top_object(struct top_objects *top, - const struct object_id *oid, size_t value) + const struct object_id *oid, size_t value, + const char *path) { size_t pos = top->nr; - while (pos > 0 && value >= top->data[pos - 1].value) + while (pos > 0 && value >= top->data[pos - 1].object.value) pos--; if (pos >= top->alloc) return; - if (top->nr < top->alloc) + if (top->nr == top->alloc) + free(top->data[top->nr - 1].path); + else top->nr++; for (size_t i = top->nr - 1; i > pos; i--) top->data[i] = top->data[i - 1]; - oidcpy(&top->data[pos].oid, oid); - top->data[pos].value = value; + oidcpy(&top->data[pos].object.oid, oid); + top->data[pos].object.value = value; + top->data[pos].path = xstrdup_or_null(path); } static size_t count_tree_entries(struct object *obj) @@ -1447,9 +1478,9 @@ static int count_objects(const char *path, struct oid_array *oids, check_largest(&stats->largest.parent_count, &oids->oid[i], count); maybe_insert_top_object(&stats->top_commit_parents, - &oids->oid[i], count); + &oids->oid[i], count, NULL); maybe_insert_top_object(&stats->top_commit_sizes, - &oids->oid[i], inflated); + &oids->oid[i], inflated, NULL); if (count >= PBIN_VEC_LEN) count = PBIN_VEC_LEN - 1; stats->commit_parents[count]++; @@ -1466,9 +1497,9 @@ static int count_objects(const char *path, struct oid_array *oids, check_largest(&stats->largest.tree_entries, &oids->oid[i], count); maybe_insert_top_object(&stats->top_tree_entries, - &oids->oid[i], count); + &oids->oid[i], count, path); maybe_insert_top_object(&stats->top_tree_sizes, - &oids->oid[i], inflated); + &oids->oid[i], inflated, path); increment_histogram(stats->tree_entries, QBIN_SHIFT, count, inflated, disk); increment_histogram(stats->tree_sizes, HBIN_SHIFT, @@ -1481,7 +1512,7 @@ static int count_objects(const char *path, struct oid_array *oids, check_largest(&stats->largest.blob_size, &oids->oid[i], inflated); maybe_insert_top_object(&stats->top_blob_sizes, - &oids->oid[i], inflated); + &oids->oid[i], inflated, path); increment_histogram(stats->blob_sizes, HBIN_SHIFT, inflated, inflated, disk); break; @@ -1682,11 +1713,11 @@ static int cmd_repo_structure(int argc, const char **argv, const char *prefix, clear_top_paths(&stats.objects.top_trees); clear_top_paths(&stats.objects.top_blobs); } - free(stats.objects.top_commit_parents.data); - free(stats.objects.top_commit_sizes.data); - free(stats.objects.top_tree_entries.data); - free(stats.objects.top_tree_sizes.data); - free(stats.objects.top_blob_sizes.data); + clear_top_objects(&stats.objects.top_commit_parents); + clear_top_objects(&stats.objects.top_commit_sizes); + clear_top_objects(&stats.objects.top_tree_entries); + clear_top_objects(&stats.objects.top_tree_sizes); + clear_top_objects(&stats.objects.top_blob_sizes); release_revisions(&revs); return 0; diff --git a/t/t1901-repo-structure.sh b/t/t1901-repo-structure.sh index 9e7dc69bcc5002..5bfb30fc207728 100755 --- a/t/t1901-repo-structure.sh +++ b/t/t1901-repo-structure.sh @@ -506,6 +506,11 @@ test_expect_success 'largest object lists have independent sorted limits' ' tree_entries=objects.trees.largest.by_entries && tree_sizes=objects.trees.largest.by_size && blob_sizes=objects.blobs.largest.by_size && + printf "%s %s\n" \ + "$big" b "$big" "$long-b" \ + "$tied" c \ + "$empty_blob" a "$empty_blob" "$long-a" \ + >allowed-paths && set -- --commit-parents=2 --commit-sizes=1 \ --tree-entries=4 --tree-sizes=1 --blob-sizes=4 && @@ -525,12 +530,16 @@ test_expect_success 'largest object lists have independent sorted limits' ' $commit_sizes.1.oid=$root $tree_entries.1.entries=3 $tree_entries.1.oid=$wide + $tree_entries.1.path= $tree_entries.2.entries=2 $tree_entries.2.oid=$narrow + $tree_entries.2.path= $tree_entries.3.entries=0 $tree_entries.3.oid=$empty_tree + $tree_entries.3.path= $tree_sizes.1.inflated_size=$narrow_size $tree_sizes.1.oid=$narrow + $tree_sizes.1.path= EOF sed -n "/^$blob_sizes\./d; /\.largest\./p" \ out >actual && @@ -551,6 +560,15 @@ test_expect_success 'largest object lists have independent sorted limits' ' test_cmp expect actual && test_grep "^$blob_sizes.3.oid=$empty_blob$" out && test_grep ! "^$blob_sizes.4." out && + for rank in 1 2 3 + do + oid=$(sed -n \ + "s/^$blob_sizes.$rank.oid=//p" out) && + path=$(sed -n \ + "s/^$blob_sizes.$rank.path=//p" out) && + test_grep -F -x "$oid $path" allowed-paths || + return 1 + done && git repo structure --format=nul "$@" >nul && tr "\012\000" "=\012" decoded && @@ -561,7 +579,7 @@ test_expect_success 'largest object lists have independent sorted limits' ' test_grep "^| 1 *\[1\] | *2 *|$" parents && sed -n "/^| Largest blobs by size /,/^$/p" \ table >blobs && - test_grep "^| 1 *\[1\] | *16 B *|$" blobs || + test_grep "^| 1: .* \[1\] | *16 B *|$" blobs || return 1 done && @@ -578,6 +596,116 @@ test_expect_success 'largest object lists have independent sorted limits' ' ) ' +test_expect_success 'largest object paths stay with their ranked objects' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + mkdir dir && + i=0 && + >objects && + for size in 1 9 2 8 3 7 + do + i=$((i + 1)) && + test-tool genzeros "$size" >"dir/$i" && + oid=$(git hash-object -w "dir/$i") && + printf "%s %s %s\n" "$size" "$oid" "dir/$i" \ + >>objects || return 1 + done && + git add dir && + test_tick && + git commit -m paths && + dir=$(git rev-parse HEAD:dir) && + dir_size=$(git cat-file -s "$dir") && + entries=objects.trees.largest.by_entries && + sizes=objects.trees.largest.by_size && + blobs=objects.blobs.largest.by_size && + sort -rn objects >sorted && + for limit in 1 2 4 + do + cat >expect <<-EOF && + $entries.1.entries=6 + $entries.1.oid=$dir + $entries.1.path=dir/ + $sizes.1.inflated_size=$dir_size + $sizes.1.oid=$dir + $sizes.1.path=dir/ + EOF + sed -n "1,${limit}p" sorted >selected && + rank=0 && + while read -r size oid path + do + rank=$((rank + 1)) && + key=$blobs.$rank && + printf "%s=%s\n" \ + "$key.inflated_size" "$size" \ + "$key.oid" "$oid" \ + "$key.path" "$path" >>expect || + return 1 + done out && + sed -n "/\.largest\./p" out >actual && + test_cmp expect actual || return 1 + done + ) +' + +test_expect_success 'largest object paths are quoted except in NUL output' ' + test_when_finished "rm -rf repo" && + git init --initial-branch=main repo && + ( + cd repo && + test-tool genzeros 16 >blob && + blob=$(git hash-object -w blob) && + printf "100644 blob %s\tfile\0" "$blob" >tree-input && + dir=$(git mktree -z tree-input && + root=$(git mktree -z expect <<-\EOF && + objects.trees.largest.by_size.1.path= + objects.trees.largest.by_size.2.path="d\t\n\"\\x/" + objects.blobs.largest.by_size.1.path="d\t\n\"\\x/file" + EOF + git repo structure --format=lines "$@" >out && + sed -n "/\.largest\..*\.path=/p" out >actual && + test_cmp expect actual && + + cat >patterns <<-\EOF && + 2: "d\t\n\"\\x/" + 1: "d\t\n\"\\x/file" + EOF + git repo structure "$@" >table && + while IFS= read -r pattern + do + test_grep -F "$pattern" table || return 1 + done expect && + printf "%s\n%s\0" \ + "$trees.1.inflated_size" "$root_size" \ + "$trees.1.oid" "$root" \ + "$trees.1.path" "" \ + "$trees.2.inflated_size" "$dir_size" \ + "$trees.2.oid" "$dir" \ + "$trees.2.path" "$name/" \ + "$blobs.1.inflated_size" 16 \ + "$blobs.1.oid" "$blob" \ + "$blobs.1.path" "$name/file" >>expect && + git repo structure --format=nul "$@" >actual && + test_cmp expect actual + ) +' + for spec in \ "commit-parents showCommitParents commits by_parents parents" \ "commit-sizes showCommitSizes commits by_size inflated_size" \ @@ -609,6 +737,10 @@ do $key.1.$metric=$value $key.1.oid=$oid EOF + case "$type" in + trees) echo "$key.1.path=" >>expect ;; + blobs) echo "$key.1.path=file" >>expect ;; + esac && git repo structure --format=lines >default && test_grep ! "\.largest\." default && git repo structure --format=lines "--$option=3" >out && From 84ee7a35f9e87c809dd21634bf45cec4a79bd014 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 15 May 2024 17:44:41 -0400 Subject: [PATCH 006/289] survey: add commit-oid to large_item detail Signed-off-by: Jeff Hostetler --- Documentation/git-repo.adoc | 4 ++++ builtin/repo.c | 31 +++++++++++++++++++++++++------ t/t1901-repo-structure.sh | 24 +++++++++++++++++++++++- 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/Documentation/git-repo.adoc b/Documentation/git-repo.adoc index dec160769d2381..b825d7bcc84b2c 100644 --- a/Documentation/git-repo.adoc +++ b/Documentation/git-repo.adoc @@ -191,6 +191,10 @@ paths. An empty path denotes a root tree. Table and `lines` output quote unusual paths according to `core.quotePath`; `nul` output preserves the path bytes without quoting. +Entries also have a `commit_oid` field when an associated commit is known. +Currently this is available only for commit objects, where it equals +`oid`. Table output shows this ID as `(commit )`. + INFO KEYS --------- In order to obtain a set of values from `git repo info`, you should provide diff --git a/builtin/repo.c b/builtin/repo.c index 21951fe00f36a6..4ef7cb09ce3018 100644 --- a/builtin/repo.c +++ b/builtin/repo.c @@ -310,6 +310,7 @@ struct object_data { struct top_object { struct object_data object; char *path; + struct object_id containing_commit_oid; }; struct top_objects { @@ -884,6 +885,8 @@ static void top_objects_table_print(const char *title, struct top_objects *top, for (size_t i = 0; i < top->nr; i++) { struct object_data *item = &top->data[i].object; const char *path = top->data[i].path; + const struct object_id *commit_oid = + &top->data[i].containing_commit_oid; strbuf_reset(&label); strbuf_addf(&label, "%" PRIuMAX, (uintmax_t)(i + 1)); @@ -891,6 +894,9 @@ static void top_objects_table_print(const char *title, struct top_objects *top, strbuf_addstr(&label, ": "); quote_c_style(path, &label, NULL, 0); } + if (!is_null_oid(commit_oid)) + strbuf_addf(&label, _(" (commit %s)"), + oid_to_hex(commit_oid)); if (by_size) stats_table_object_size_addf(&table, &item->oid, @@ -1025,6 +1031,11 @@ static void top_objects_keyvalue_print(const char *prefix, const char *metric, print_keyvalue_path("path", key_delim, item->path, value_delim); } + if (!is_null_oid(&item->containing_commit_oid)) + printf("%s.%" PRIuMAX ".commit_oid%c%s%c", + prefix, (uintmax_t)(i + 1), key_delim, + oid_to_hex(&item->containing_commit_oid), + value_delim); } } @@ -1379,7 +1390,8 @@ static void clear_top_objects(struct top_objects *top) static void maybe_insert_top_object(struct top_objects *top, const struct object_id *oid, size_t value, - const char *path) + const char *path, + const struct object_id *commit_oid) { size_t pos = top->nr; @@ -1397,6 +1409,8 @@ static void maybe_insert_top_object(struct top_objects *top, oidcpy(&top->data[pos].object.oid, oid); top->data[pos].object.value = value; top->data[pos].path = xstrdup_or_null(path); + oidcpy(&top->data[pos].containing_commit_oid, + commit_oid ? commit_oid : null_oid(the_repository->hash_algo)); } static size_t count_tree_entries(struct object *obj) @@ -1478,9 +1492,11 @@ static int count_objects(const char *path, struct oid_array *oids, check_largest(&stats->largest.parent_count, &oids->oid[i], count); maybe_insert_top_object(&stats->top_commit_parents, - &oids->oid[i], count, NULL); + &oids->oid[i], count, NULL, + &oids->oid[i]); maybe_insert_top_object(&stats->top_commit_sizes, - &oids->oid[i], inflated, NULL); + &oids->oid[i], inflated, NULL, + &oids->oid[i]); if (count >= PBIN_VEC_LEN) count = PBIN_VEC_LEN - 1; stats->commit_parents[count]++; @@ -1497,9 +1513,11 @@ static int count_objects(const char *path, struct oid_array *oids, check_largest(&stats->largest.tree_entries, &oids->oid[i], count); maybe_insert_top_object(&stats->top_tree_entries, - &oids->oid[i], count, path); + &oids->oid[i], count, + path, NULL); maybe_insert_top_object(&stats->top_tree_sizes, - &oids->oid[i], inflated, path); + &oids->oid[i], inflated, + path, NULL); increment_histogram(stats->tree_entries, QBIN_SHIFT, count, inflated, disk); increment_histogram(stats->tree_sizes, HBIN_SHIFT, @@ -1512,7 +1530,8 @@ static int count_objects(const char *path, struct oid_array *oids, check_largest(&stats->largest.blob_size, &oids->oid[i], inflated); maybe_insert_top_object(&stats->top_blob_sizes, - &oids->oid[i], inflated, path); + &oids->oid[i], inflated, + path, NULL); increment_histogram(stats->blob_sizes, HBIN_SHIFT, inflated, inflated, disk); break; diff --git a/t/t1901-repo-structure.sh b/t/t1901-repo-structure.sh index 5bfb30fc207728..9f1d79adb2297e 100755 --- a/t/t1901-repo-structure.sh +++ b/t/t1901-repo-structure.sh @@ -441,10 +441,13 @@ test_expect_success 'commit parent histogram groups 31 or more parents' ' cat >expect <<-EOF && $key.1.parents=32 $key.1.oid=$commit + $key.1.commit_oid=$commit $key.2.parents=31 $key.2.oid=$boundary + $key.2.commit_oid=$boundary $key.3.parents=2 $key.3.oid=$two + $key.3.commit_oid=$two EOF git repo structure --format=lines --commit-parents=3 >out && sed -n "/^$key\./p" out >actual && @@ -452,6 +455,12 @@ test_expect_success 'commit parent histogram groups 31 or more parents' ' for limit in 9 10 12 do + git repo structure --format=lines \ + --commit-parents=$limit >out && + sed -n "s/\.oid=/.commit_oid=/p" out >expect && + sed -n "/\.commit_oid=/p" out >actual && + test_line_count = $limit actual && + test_cmp expect actual && git repo structure --commit-parents=$limit >table && sed -n "/^| Largest commits by parent count /,/^$/p" \ table >parents && @@ -524,10 +533,13 @@ test_expect_success 'largest object lists have independent sorted limits' ' cat >expect <<-EOF && $commit_parents.1.parents=2 $commit_parents.1.oid=$merge + $commit_parents.1.commit_oid=$merge $commit_parents.2.parents=1 $commit_parents.2.oid=$child + $commit_parents.2.commit_oid=$child $commit_sizes.1.inflated_size=$root_size $commit_sizes.1.oid=$root + $commit_sizes.1.commit_oid=$root $tree_entries.1.entries=3 $tree_entries.1.oid=$wide $tree_entries.1.path= @@ -576,7 +588,8 @@ test_expect_success 'largest object lists have independent sorted limits' ' git repo structure "$@" >table && sed -n "/^| Largest commits by parent count /,/^$/p" \ table >parents && - test_grep "^| 1 *\[1\] | *2 *|$" parents && + test_grep "^| 1 (commit $merge) *\[1\] | *2 *|$" \ + parents && sed -n "/^| Largest blobs by size /,/^$/p" \ table >blobs && test_grep "^| 1: .* \[1\] | *16 B *|$" blobs || @@ -738,6 +751,7 @@ do $key.1.oid=$oid EOF case "$type" in + commits) echo "$key.1.commit_oid=$oid" >>expect ;; trees) echo "$key.1.path=" >>expect ;; blobs) echo "$key.1.path=file" >>expect ;; esac && @@ -746,6 +760,14 @@ do git repo structure --format=lines "--$option=3" >out && sed -n "/\.largest\./p" out >actual && test_cmp expect actual && + git repo structure --format=nul "--$option=3" >nul && + tr "\012\000" "=\012" decoded && + test_cmp out decoded && + git repo structure "--$option=3" >table && + case "$type" in + commits) test_grep -F "(commit $oid)" table ;; + *) test_grep ! -F "(commit " table ;; + esac && git -c repo.structure.$config=3 repo structure \ --format=lines >configured && test_cmp out configured && From bac7b8ae46cca02f8b2b7e08e9d2dc628017ed8e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 17 Sep 2026 10:35:08 +0200 Subject: [PATCH 007/289] fixup! ci(dockerized): do show the result of failing tests again The test reporter fails with "Permission denied" when appending `FAILED_TEST_ARTIFACTS` to `GITHUB_ENV` in linux-TEST-vars, linux32 and debian-12: https://github.com/microsoft/git/actions/runs/35114876023 Each Actions step receives a distinct environment file, so the earlier build/test permission adjustment does not cover the reporter. Give this step the access it needs to record failure artifacts while retaining rootless execution and unchanged reporting logic. Assisted-by: GPT-6 Signed-off-by: Johannes Schindelin --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 71cabd013fb4f1..0453ccf46383f0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -479,7 +479,9 @@ jobs: - run: chmod a+w $GITHUB_ENV && sudo --preserve-env --set-home --user=builder ci/run-build-and-tests.sh - name: print test failures if: failure() && env.FAILED_TEST_ARTIFACTS != '' - run: sudo --preserve-env --set-home --user=builder ci/print-test-failures.sh + run: | + chmod a+w "$GITHUB_ENV" && + sudo --preserve-env --set-home --user=builder ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' uses: actions/upload-artifact@v7 From 679d30d5c81456b3549349ffa1d437d6c43e71cb Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 20 May 2024 17:23:39 -0400 Subject: [PATCH 008/289] survey: add commit name-rev lookup to each large_item Signed-off-by: Jeff Hostetler --- Documentation/git-repo.adoc | 10 +++ builtin/repo.c | 101 +++++++++++++++++++++- t/t1901-repo-structure.sh | 166 +++++++++++++++++++++++++++++++++++- 3 files changed, 272 insertions(+), 5 deletions(-) diff --git a/Documentation/git-repo.adoc b/Documentation/git-repo.adoc index b825d7bcc84b2c..c75af89fb5c70b 100644 --- a/Documentation/git-repo.adoc +++ b/Documentation/git-repo.adoc @@ -195,6 +195,16 @@ Entries also have a `commit_oid` field when an associated commit is known. Currently this is available only for commit objects, where it equals `oid`. Table output shows this ID as `(commit )`. +The `name_rev` field contains the result of passing `commit_oid` to +`git name-rev --name-only --annotate-stdin`. This uses all refs for naming, +even when `--ref-filter` limits the objects counted. An object ID is kept +unchanged if no name is found. Names appear in parentheses in tables and +use the same quoting rules as paths in all output formats. + +No name lookup is performed for entries without a `commit_oid`. If lookup +fails, a warning is printed and `name_rev` is omitted; the other statistics +are still reported. + INFO KEYS --------- In order to obtain a set of values from `git repo info`, you should provide diff --git a/builtin/repo.c b/builtin/repo.c index 4ef7cb09ce3018..cc641b8abe45cf 100644 --- a/builtin/repo.c +++ b/builtin/repo.c @@ -15,10 +15,13 @@ #include "ref-filter.h" #include "refs.h" #include "revision.h" +#include "run-command.h" #include "setup.h" #include "strbuf.h" #include "string-list.h" +#include "strvec.h" #include "shallow.h" +#include "trace2.h" #include "tree.h" #include "tree-walk.h" #include "utf8.h" @@ -311,6 +314,7 @@ struct top_object { struct object_data object; char *path; struct object_id containing_commit_oid; + char *name_rev; }; struct top_objects { @@ -885,6 +889,7 @@ static void top_objects_table_print(const char *title, struct top_objects *top, for (size_t i = 0; i < top->nr; i++) { struct object_data *item = &top->data[i].object; const char *path = top->data[i].path; + const char *name_rev = top->data[i].name_rev; const struct object_id *commit_oid = &top->data[i].containing_commit_oid; @@ -897,6 +902,11 @@ static void top_objects_table_print(const char *title, struct top_objects *top, if (!is_null_oid(commit_oid)) strbuf_addf(&label, _(" (commit %s)"), oid_to_hex(commit_oid)); + if (name_rev) { + strbuf_addstr(&label, " ("); + quote_c_style(name_rev, &label, NULL, 0); + strbuf_addch(&label, ')'); + } if (by_size) stats_table_object_size_addf(&table, &item->oid, @@ -1036,6 +1046,11 @@ static void top_objects_keyvalue_print(const char *prefix, const char *metric, prefix, (uintmax_t)(i + 1), key_delim, oid_to_hex(&item->containing_commit_oid), value_delim); + if (item->name_rev) { + printf("%s.%" PRIuMAX ".", prefix, (uintmax_t)(i + 1)); + print_keyvalue_path("name_rev", key_delim, + item->name_rev, value_delim); + } } } @@ -1383,8 +1398,10 @@ static void init_top_objects(struct top_objects *top, int limit, static void clear_top_objects(struct top_objects *top) { - for (size_t i = 0; i < top->nr; i++) + for (size_t i = 0; i < top->nr; i++) { free(top->data[i].path); + free(top->data[i].name_rev); + } free(top->data); } @@ -1411,6 +1428,8 @@ static void maybe_insert_top_object(struct top_objects *top, top->data[pos].path = xstrdup_or_null(path); oidcpy(&top->data[pos].containing_commit_oid, commit_oid ? commit_oid : null_oid(the_repository->hash_algo)); + /* Revision names are resolved only after ranking is complete. */ + top->data[pos].name_rev = NULL; } static size_t count_tree_entries(struct object *obj) @@ -1589,6 +1608,85 @@ static void structure_count_objects(struct object_stats *stats, stop_progress(&data.progress); } +static void structure_lookup_name_revs(struct object_stats *stats, + struct repository *repo, + int show_progress) +{ + struct top_objects *lists[] = { + &stats->top_commit_parents, + &stats->top_commit_sizes, + &stats->top_tree_entries, + &stats->top_tree_sizes, + &stats->top_blob_sizes, + }; + struct child_process cp = CHILD_PROCESS_INIT; + struct strbuf in = STRBUF_INIT, out = STRBUF_INIT; + struct string_list names = STRING_LIST_INIT_NODUP; + struct progress *progress = NULL; + size_t nr = 0, k = 0; + int failed = 1; + + for (size_t i = 0; i < ARRAY_SIZE(lists); i++) { + for (size_t j = 0; j < lists[i]->nr; j++) { + struct top_object *item = &lists[i]->data[j]; + + if (is_null_oid(&item->containing_commit_oid)) + continue; + strbuf_addf(&in, "%s\n", + oid_to_hex(&item->containing_commit_oid)); + nr++; + } + } + if (!nr) + return; + + trace2_region_enter("repo", "name-rev", repo); + if (show_progress) + progress = start_progress(repo, + _("Resolving revision names"), nr); + + cp.git_cmd = 1; + strvec_pushl(&cp.args, "name-rev", "--name-only", + "--annotate-stdin", NULL); + if (pipe_command(&cp, in.buf, in.len, &out, 0, NULL, 0)) { + warning(_("could not resolve revision names")); + goto cleanup; + } + if (!out.len || out.buf[out.len - 1] != '\n' || + memchr(out.buf, '\0', out.len)) + goto invalid_output; + strbuf_trim_trailing_newline(&out); + string_list_split_in_place_f(&names, out.buf, "\n", -1, + STRING_LIST_SPLIT_TRIM); + if (names.nr != nr) + goto invalid_output; + for (size_t i = 0; i < names.nr; i++) + if (!*names.items[i].string) + goto invalid_output; + + for (size_t i = 0; i < ARRAY_SIZE(lists); i++) { + for (size_t j = 0; j < lists[i]->nr; j++) { + struct top_object *item = &lists[i]->data[j]; + + if (is_null_oid(&item->containing_commit_oid)) + continue; + item->name_rev = xstrdup(names.items[k++].string); + display_progress(progress, k); + } + } + failed = 0; + goto cleanup; + +invalid_output: + warning(_("unexpected output from 'git name-rev'")); +cleanup: + stop_progress_msg(&progress, failed ? _("failed") : _("done")); + trace2_region_leave("repo", "name-rev", repo); + string_list_clear(&names, 0); + strbuf_release(&in); + strbuf_release(&out); +} + struct repo_structure_opts { int top_nr; int commit_parents; @@ -1707,6 +1805,7 @@ static int cmd_repo_structure(int argc, const char **argv, const char *prefix, show_progress); structure_count_objects(&stats.objects, &revs, repo, opts.top_nr, show_progress); + structure_lookup_name_revs(&stats.objects, repo, show_progress); switch (format) { case FORMAT_TABLE: diff --git a/t/t1901-repo-structure.sh b/t/t1901-repo-structure.sh index 9f1d79adb2297e..f336f72f4cc8b5 100755 --- a/t/t1901-repo-structure.sh +++ b/t/t1901-repo-structure.sh @@ -442,12 +442,15 @@ test_expect_success 'commit parent histogram groups 31 or more parents' ' $key.1.parents=32 $key.1.oid=$commit $key.1.commit_oid=$commit + $key.1.name_rev=overflow $key.2.parents=31 $key.2.oid=$boundary $key.2.commit_oid=$boundary + $key.2.name_rev=boundary $key.3.parents=2 $key.3.oid=$two $key.3.commit_oid=$two + $key.3.name_rev=two EOF git repo structure --format=lines --commit-parents=3 >out && sed -n "/^$key\./p" out >actual && @@ -461,6 +464,11 @@ test_expect_success 'commit parent histogram groups 31 or more parents' ' sed -n "/\.commit_oid=/p" out >actual && test_line_count = $limit actual && test_cmp expect actual && + sed -n "s/^.*\.commit_oid=//p" out >oids && + git name-rev --name-only --annotate-stdin \ + expect && + sed -n "s/^.*\.name_rev=//p" out >actual && + test_cmp expect actual && git repo structure --commit-parents=$limit >table && sed -n "/^| Largest commits by parent count /,/^$/p" \ table >parents && @@ -534,12 +542,15 @@ test_expect_success 'largest object lists have independent sorted limits' ' $commit_parents.1.parents=2 $commit_parents.1.oid=$merge $commit_parents.1.commit_oid=$merge + $commit_parents.1.name_rev=main $commit_parents.2.parents=1 $commit_parents.2.oid=$child $commit_parents.2.commit_oid=$child + $commit_parents.2.name_rev=main~1 $commit_sizes.1.inflated_size=$root_size $commit_sizes.1.oid=$root $commit_sizes.1.commit_oid=$root + $commit_sizes.1.name_rev=empty $tree_entries.1.entries=3 $tree_entries.1.oid=$wide $tree_entries.1.path= @@ -588,7 +599,7 @@ test_expect_success 'largest object lists have independent sorted limits' ' git repo structure "$@" >table && sed -n "/^| Largest commits by parent count /,/^$/p" \ table >parents && - test_grep "^| 1 (commit $merge) *\[1\] | *2 *|$" \ + test_grep "(commit $merge) (main) *\[1\] | *2 *|$" \ parents && sed -n "/^| Largest blobs by size /,/^$/p" \ table >blobs && @@ -719,6 +730,150 @@ test_expect_success 'largest object paths are quoted except in NUL output' ' ) ' +test_expect_success 'ranked revision names are batched and use all refs' ' + test_when_finished "rm -rf repo" && + git init --initial-branch=main repo && + ( + cd repo && + test_commit --no-tag one file && + git tag -a -m tag v1 && + name=tags/v1^0 && + key=objects.commits.largest && + set -- --commit-parents=4 --commit-sizes=4 \ + --tree-entries=4 --tree-sizes=4 --blob-sizes=4 && + GIT_TRACE2_EVENT="$PWD/trace" git repo structure \ + --format=lines --progress \ + --ref-filter=refs/heads/main "$@" >out 2>err && + printf "%s=%s\n" \ + "$key.by_parents.1.name_rev" "$name" \ + "$key.by_size.1.name_rev" "$name" >expect && + sed -n "/\.name_rev=/p" out >actual && + test_cmp expect actual && + test_grep "^references.tags.count=0$" out && + test_grep "Resolving revision names" err && + grep "child_start.*\"name-rev\"" trace >children && + test_line_count = 1 children && + + GIT_TRACE2_EVENT="$PWD/default-trace" \ + git repo structure >out && + test_grep ! "child_start.*\"name-rev\"" default-trace && + GIT_TRACE2_EVENT="$PWD/tree-trace" git repo structure \ + --tree-entries=4 --tree-sizes=4 --blob-sizes=4 >out && + test_grep ! "child_start.*\"name-rev\"" tree-trace + ) +' + +test_expect_success 'ranked revision names quote text but preserve NUL data' ' + test_when_finished "rm -rf repo" && + name=$(printf "q\042\303\251") && + git init --ref-format=reftable --initial-branch="$name" repo && + ( + cd repo && + test_commit --no-tag one file && + oid=$(git rev-parse HEAD) && + key=objects.commits.largest.by_parents && + cat >expect <<-\EOF && + objects.commits.largest.by_parents.1.name_rev="q\"\303\251" + EOF + git -c core.quotePath=true repo structure \ + --commit-parents=2 --format=lines >out && + sed -n "/\.name_rev=/p" out >actual && + test_cmp expect actual && + cat >pattern <<-\EOF && + ("q\"\303\251") + EOF + git -c core.quotePath=true repo structure \ + --commit-parents=2 >table && + grep -F -f pattern table >found && + test_line_count = 1 found && + + printf "%s=\042q\134\042\303\251\042\n" \ + "$key.1.name_rev" >expect && + git -c core.quotePath=false repo structure \ + --commit-parents=2 --format=lines >out && + sed -n "/\.name_rev=/p" out >actual && + test_cmp expect actual && + + git repo structure --format=nul >expect && + printf "%s\n%s\0" \ + "$key.1.parents" 0 \ + "$key.1.oid" "$oid" \ + "$key.1.commit_oid" "$oid" \ + "$key.1.name_rev" "$name" >>expect && + git repo structure --commit-parents=2 --format=nul >actual && + test_cmp expect actual + ) +' + +test_expect_success 'revision-name failures leave other statistics intact' ' + test_when_finished "rm -rf repo" && + git init --initial-branch=main repo && + ( + cd repo && + test_commit --no-tag one file && + oid=$(git rev-parse HEAD) && + set -- --commit-parents=3 --commit-sizes=3 --format=lines && + git repo structure "$@" >original && + sed "/\.name_rev=/d" original >expect && + printf "%s\n" "$oid" "$oid" >expect-input && + mkdir mock && + write_script mock/git <<-\EOF && + if test "$*" != "name-rev --name-only --annotate-stdin" + then + echo "unexpected command: $*" >&2 + exit 1 + fi + cat >name-rev-input || exit 1 + case "$NAME_REV_MODE" in + fail) exit 1 ;; + short) printf "first\n" ;; + extra) printf "first\nsecond\nthird\n" ;; + unterminated) printf "first\nsecond" ;; + nul) printf "first\0ignored\nsecond\n" ;; + empty) printf "first\n\n" ;; + raw) cat name-rev-input ;; + crlf) printf "first\r\nsecond\r\n" ;; + esac + EOF + for mode in fail short extra unterminated nul empty + do + NAME_REV_MODE=$mode git --exec-path="$PWD/mock" \ + repo structure "$@" >out 2>err && + test_cmp expect-input name-rev-input && + test_cmp expect out && + case "$mode" in + fail) + test_grep "could not resolve revision names" err + ;; + *) + test_grep "unexpected output.*name-rev" err + ;; + esac || return 1 + done && + + key=objects.commits.largest && + for mode in raw crlf + do + NAME_REV_MODE=$mode git --exec-path="$PWD/mock" \ + repo structure "$@" >out 2>err && + test_must_be_empty err && + test_cmp expect-input name-rev-input && + sed "/\.name_rev=/d" out >actual && + test_cmp expect actual && + case "$mode" in + raw) first=$oid second=$oid ;; + crlf) first=first second=second ;; + esac && + printf "%s=%s\n" \ + "$key.by_parents.1.name_rev" "$first" \ + "$key.by_size.1.name_rev" "$second" \ + >expect-names && + sed -n "/\.name_rev=/p" out >actual && + test_cmp expect-names actual || return 1 + done + ) +' + for spec in \ "commit-parents showCommitParents commits by_parents parents" \ "commit-sizes showCommitSizes commits by_size inflated_size" \ @@ -731,7 +886,7 @@ do test_expect_success "--$option is opt-in and overrides its config" ' test_when_finished "rm -rf repo" && - git init repo && + git init --initial-branch=main repo && ( cd repo && test_commit --no-tag one file && @@ -751,7 +906,10 @@ do $key.1.oid=$oid EOF case "$type" in - commits) echo "$key.1.commit_oid=$oid" >>expect ;; + commits) + printf "%s=%s\n" "$key.1.commit_oid" "$oid" \ + "$key.1.name_rev" main >>expect + ;; trees) echo "$key.1.path=" >>expect ;; blobs) echo "$key.1.path=file" >>expect ;; esac && @@ -765,7 +923,7 @@ do test_cmp out decoded && git repo structure "--$option=3" >table && case "$type" in - commits) test_grep -F "(commit $oid)" table ;; + commits) test_grep -F "(commit $oid) (main)" table ;; *) test_grep ! -F "(commit " table ;; esac && git -c repo.structure.$config=3 repo structure \ From 7ce6a1237d7be7d902ebfe70a902ea9ac8e1b7ea Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 18 Sep 2026 16:41:41 +0200 Subject: [PATCH 009/289] fixup! Add a GitHub workflow to verify that Git/Scalar work in Nano Server Nano Server needs runtime DLLs from the SDK used to build Git, but the current x64 environment no longer contains `/mingw64/bin`. Honor the setup action's `MSYSTEM` selection while keeping the container-side directory independent of the toolchain name. Assisted-by: GPT-6 Signed-off-by: Johannes Schindelin --- .github/workflows/nano-server.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nano-server.yml b/.github/workflows/nano-server.yml index 2e6da3ceea6f90..60e056a44bf8cc 100644 --- a/.github/workflows/nano-server.yml +++ b/.github/workflows/nano-server.yml @@ -28,11 +28,11 @@ jobs: docker run \ --user "ContainerAdministrator" \ -v "$WINDBG_DIR:C:/dbg" \ - -v "$(cygpath -aw /mingw64/bin):C:/mingw64-bin" \ + -v "$(cygpath -aw "/${MSYSTEM,,}/bin"):C:/mingw-bin" \ -v "$(cygpath -aw .):C:/test" \ $IMAGE pwsh.exe -Command ' - # Extend the PATH to include the `.dll` files in /mingw64/bin/ - $env:PATH += ";C:\mingw64-bin" + # Add runtime DLLs from the active SDK to PATH. + $env:PATH += ";C:\mingw-bin" # For each executable to test pick some no-operation set of # flags/subcommands or something that should quickly result in an From 4b3f05fd72c4ec674e28144c768d37335517dae4 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 5 Apr 2017 10:58:09 -0600 Subject: [PATCH 010/289] reset --stdin: trim carriage return from the paths While using the reset --stdin feature on windows path added may have a \r at the end of the path that wasn't getting removed so didn't match the path in the index and wasn't reset. Signed-off-by: Kevin Willford --- t/t7108-reset-stdin.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/t/t7108-reset-stdin.sh b/t/t7108-reset-stdin.sh index b7cbcbf869296c..db5483b8f10052 100755 --- a/t/t7108-reset-stdin.sh +++ b/t/t7108-reset-stdin.sh @@ -29,4 +29,13 @@ test_expect_success '--stdin requires --mixed' ' git reset --mixed --stdin list && + git reset --stdin Date: Tue, 4 Jun 2024 10:37:23 -0400 Subject: [PATCH 011/289] survey: add --no-name-rev option Computing `git name-rev` on each commit, tree, and blob in each of the various large_item_vec can be very expensive if there are too many refs, especially if the user doesn't need the result. Lets make it optional. The `--no-name-rev` option can save 50 calls to `git name-rev` since we have 5 large_item_vec's and each defaults to 10 items. Signed-off-by: Jeff Hostetler --- Documentation/config/repo.adoc | 5 +++ Documentation/git-repo.adoc | 8 ++++ builtin/repo.c | 12 +++++- t/t1901-repo-structure.sh | 70 ++++++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 2 deletions(-) diff --git a/Documentation/config/repo.adoc b/Documentation/config/repo.adoc index f7c34f398682b6..5705280940f93f 100644 --- a/Documentation/config/repo.adoc +++ b/Documentation/config/repo.adoc @@ -3,6 +3,11 @@ repo.structure.*:: `git repo structure` command. + -- + nameRev:: + A boolean controlling whether revision names for ranked + objects are resolved and reported. Defaults to `true`. + The `--[no-]name-rev` option in linkgit:git-repo[1] + overrides this setting. top:: This integer value implies `--top=`, specifying the number of largest paths to report in each detail table. diff --git a/Documentation/git-repo.adoc b/Documentation/git-repo.adoc index c75af89fb5c70b..1c5c37be124cf8 100644 --- a/Documentation/git-repo.adoc +++ b/Documentation/git-repo.adoc @@ -94,6 +94,14 @@ Loose and packed counts are zero for reference backends other than `files`. `--ref-filter='refs/remotes/origin/*'` for a single remote's branches. + `--name-rev`;; + `--no-name-rev`;; + Resolve revision names for the commits associated with + ranked objects. Enabled by default. `--no-name-rev` skips + the lookup and omits `name_rev` from every output format. + The default can be set with `repo.structure.nameRev`; + explicit command-line options override this setting. + `--top=`;; Also report the _n_ largest paths in the repository, separately for trees and blobs and separately ranked by diff --git a/builtin/repo.c b/builtin/repo.c index cc641b8abe45cf..ef87405832353d 100644 --- a/builtin/repo.c +++ b/builtin/repo.c @@ -1688,6 +1688,7 @@ static void structure_lookup_name_revs(struct object_stats *stats, } struct repo_structure_opts { + int name_rev; int top_nr; int commit_parents; int commit_sizes; @@ -1715,6 +1716,10 @@ static int repo_structure_config_cb(const char *var, const char *value, limit = &opts->tree_sizes; else if (!strcmp(var, "repo.structure.showblobsizes")) limit = &opts->blob_sizes; + else if (!strcmp(var, "repo.structure.namerev")) { + opts->name_rev = git_config_bool(var, value); + return 0; + } if (limit) { *limit = git_config_int(var, value, cctx->kvi); @@ -1735,7 +1740,7 @@ static int cmd_repo_structure(int argc, const char **argv, const char *prefix, }; enum output_format format = FORMAT_TABLE; struct repo_structure stats = { 0 }; - struct repo_structure_opts opts = { 0 }; + struct repo_structure_opts opts = { .name_rev = 1 }; struct rev_info revs; int show_progress = -1; struct string_list ref_filters = STRING_LIST_INIT_DUP; @@ -1748,6 +1753,8 @@ static int cmd_repo_structure(int argc, const char **argv, const char *prefix, PARSE_OPT_NONEG | PARSE_OPT_NOARG, parse_format_cb), OPT_BOOL(0, "progress", &show_progress, N_("show progress")), + OPT_BOOL(0, "name-rev", &opts.name_rev, + N_("resolve revision names for reported commits")), OPT_STRING_LIST(0, "ref-filter", &ref_filters, N_("pattern"), N_("only count refs matching ; " "repeat to union multiple patterns")), @@ -1805,7 +1812,8 @@ static int cmd_repo_structure(int argc, const char **argv, const char *prefix, show_progress); structure_count_objects(&stats.objects, &revs, repo, opts.top_nr, show_progress); - structure_lookup_name_revs(&stats.objects, repo, show_progress); + if (opts.name_rev) + structure_lookup_name_revs(&stats.objects, repo, show_progress); switch (format) { case FORMAT_TABLE: diff --git a/t/t1901-repo-structure.sh b/t/t1901-repo-structure.sh index f336f72f4cc8b5..de519f5857ed3d 100755 --- a/t/t1901-repo-structure.sh +++ b/t/t1901-repo-structure.sh @@ -763,6 +763,76 @@ test_expect_success 'ranked revision names are batched and use all refs' ' ) ' +test_expect_success 'revision-name lookup can be disabled by option or config' ' + test_when_finished "rm -rf repo" && + git init --initial-branch=main repo && + ( + cd repo && + test_commit --no-tag one file && + oid=$(git rev-parse HEAD) && + name_rev_report () { + git "$@" --commit-parents=2 --commit-sizes=2 \ + --tree-entries=2 --tree-sizes=2 --blob-sizes=2 + } && + name_rev_report repo structure --format=lines >enabled && + test_grep "\.name_rev=" enabled && + sed "/\.name_rev=/d" enabled >disabled && + child_pattern="child_start.*\"name-rev\"" && + for spec in \ + "default enabled" \ + "default enabled --name-rev" \ + "default disabled --no-name-rev" \ + "true enabled" \ + "false disabled" \ + "bare enabled" \ + "true disabled --no-name-rev" \ + "false enabled --name-rev" \ + "true disabled --name-rev --no-name-rev" \ + "false enabled --no-name-rev --name-rev" + do + set -- $spec && + config=$1 expected=$2 && + shift 2 && + set -- repo structure "$@" && + case "$config" in + default) ;; + bare) set -- -c repo.structure.nameRev "$@" ;; + *) set -- -c repo.structure.nameRev=$config "$@" ;; + esac && + >trace && + test_env GIT_TRACE2_EVENT="$PWD/trace" \ + git "$@" --format=lines --commit-parents=2 \ + --commit-sizes=2 --tree-entries=2 \ + --tree-sizes=2 --blob-sizes=2 >actual 2>err && + test_cmp "$expected" actual && + test_must_be_empty err && + case "$expected" in + enabled) + grep "$child_pattern" trace >children && + test_line_count = 1 children + ;; + disabled) + test_grep ! "\"name-rev\"" trace + ;; + esac || return 1 + done && + + name_rev_report repo structure --no-name-rev \ + --format=nul >nul && + tr "\012\000" "=\012" actual && + test_cmp disabled actual && + name_rev_report repo structure --no-name-rev \ + --progress >table 2>err && + test_grep -F "(commit $oid)" table && + test_grep ! -F "(main)" table && + test_grep ! "Resolving revision names" err && + + test_must_fail git -c repo.structure.nameRev=invalid \ + repo structure --commit-parents=1 2>err && + test_grep "bad boolean config value" err + ) +' + test_expect_success 'ranked revision names quote text but preserve NUL data' ' test_when_finished "rm -rf repo" && name=$(printf "q\042\303\251") && From 91ae0a03eb68bebbe6f82fbd1536ec6783b0bbef Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 18 Sep 2026 18:11:58 +0200 Subject: [PATCH 012/289] fixup! mingw: allow `git.exe` to be used instead of the "Git wrapper" The pretend SDK layout should reflect the current x64 UCRT64 defaults when neither `MINGW_PREFIX` nor `MSYSTEM` is specified. Explicit selections remain honored; only the fixture's fallback changes. Assisted-by: GPT-6 Signed-off-by: Johannes Schindelin --- t/t0060-path-utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/t0060-path-utils.sh b/t/t0060-path-utils.sh index 5abfa202c19dca..2c805f85140c24 100755 --- a/t/t0060-path-utils.sh +++ b/t/t0060-path-utils.sh @@ -623,7 +623,7 @@ test_expect_success !VALGRIND,RUNTIME_PREFIX,CAN_EXEC_IN_PWD '%(prefix)/ works' test_expect_success MINGW,RUNTIME_PREFIX 'MSYSTEM/PATH is adjusted if necessary' ' if test -z "$MINGW_PREFIX" then - MINGW_PREFIX="/$(echo "${MSYSTEM:-MINGW64}" | tr A-Z a-z)" + MINGW_PREFIX="/$(echo "${MSYSTEM:-UCRT64}" | tr A-Z a-z)" fi && mkdir -p "$HOME"/bin pretend"$MINGW_PREFIX"/bin \ pretend"$MINGW_PREFIX"/libexec/git-core pretend/usr/bin && From 2a72bc296950ea5a659dc5d94653e92af1f70185 Mon Sep 17 00:00:00 2001 From: Victoria Dye Date: Tue, 19 Sep 2023 18:20:32 -0700 Subject: [PATCH 013/289] sparse-index.c: fix use of index hashes in expand_index In ac8acb4f2c7 (sparse-index: complete partial expansion, 2022-05-23), 'expand_index()' was updated to expand the index to a given pathspec. However, the 'path_matches_pattern_list()' method used to facilitate this has the side effect of initializing or updating the index hash variables ('name_hash', 'dir_hash', and 'name_hash_initialized'). This operation is performed on 'istate', though, not 'full'; as a result, the initialized hashes are later overwritten when copied from 'full'. To ensure the correct hashes are in 'istate' after the index expansion, change the arg used in 'path_matches_pattern_list()' from 'istate' to 'full'. Note that this does not fully solve the problem. If 'istate' does not have an initialized 'name_hash' when its contents are copied to 'full', initialized hashes will be copied back into 'istate' but 'name_hash_initialized' will be 0. Therefore, we also need to copy 'full->name_hash_initialized' back to 'istate' after the index expansion is complete. Signed-off-by: Victoria Dye --- sparse-index.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sparse-index.c b/sparse-index.c index 3d77dadae56d08..a3d32dab2b816a 100644 --- a/sparse-index.c +++ b/sparse-index.c @@ -416,7 +416,7 @@ void expand_index(struct index_state *istate, struct pattern_list *pl) if (pl && path_matches_pattern_list(ce->name, ce->ce_namelen, NULL, &dtype, - pl, istate) == NOT_MATCHED) { + pl, full) == NOT_MATCHED) { set_index_entry(full, full->cache_nr++, ce); continue; } @@ -444,6 +444,7 @@ void expand_index(struct index_state *istate, struct pattern_list *pl) } /* Copy back into original index. */ + istate->name_hash_initialized = full->name_hash_initialized; memcpy(&istate->name_hash, &full->name_hash, sizeof(full->name_hash)); memcpy(&istate->dir_hash, &full->dir_hash, sizeof(full->dir_hash)); istate->sparse_index = pl ? INDEX_PARTIALLY_SPARSE : INDEX_EXPANDED; From 2315267cb6a72657d7aaf25a6cf3b02896a9f0e1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 23 Apr 2024 12:21:01 +0200 Subject: [PATCH 014/289] Identify microsoft/git via a distinct version suffix It has been a long-standing practice in Git for Windows to append `.windows.`, and in microsoft/git to append `.vfs.0.0`. Let's keep doing that. Signed-off-by: Johannes Schindelin --- GIT-VERSION-GEN | 3 +++ 1 file changed, 3 insertions(+) diff --git a/GIT-VERSION-GEN b/GIT-VERSION-GEN index e1ba5497648c07..18bc599a78404e 100755 --- a/GIT-VERSION-GEN +++ b/GIT-VERSION-GEN @@ -2,6 +2,9 @@ DEF_VER=v2.56.0-rc1 +# Identify microsoft/git via a distinct version suffix +DEF_VER=$DEF_VER.vfs.0.0 + LF=' ' From 8545e8f985f070e315277065c08bbcdb1b43b78a Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 17 Jun 2024 15:20:05 -0400 Subject: [PATCH 015/289] survey: started TODO list at bottom of source file --- builtin/survey.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/builtin/survey.c b/builtin/survey.c index d3b36b891404cd..8f8602e9d57710 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -111,3 +111,49 @@ int cmd_survey(int argc, const char **argv, const char *prefix, strvec_clear(&child_argv); return 1; } + +/* + * NEEDSWORK: The following is a bit of a laundry list of things + * that I'd like to add. + * + * [] Dump stats on all of the packfiles. The number and size of each. + * Whether each is in the .git directory or in an alternate. The state + * of the IDX or MIDX files and etc. Delta chain stats. All of this + * data is relative to the "lived-in" state of the repository. Stuff + * that may change after a GC or repack. + * + * [] Dump stats on each remote. When we fetch from a remote the size + * of the response is related to the set of haves on the server. You + * can see this in `GIT_TRACE_CURL=1 git fetch`. We get a `ls-refs` + * payload that lists all of the branches and tags on the server, so + * at a minimum the RefName and SHA for each. But for annotated tags + * we also get the peeled SHA. The size of this overhead on every + * fetch is proporational to the size of the `git ls-remote` response + * (roughly, although the latter repeats the RefName of the peeled + * tag). If, for example, you have 500K refs on a remote, you're + * going to have a long "haves" message, so every fetch will be slow + * just because of that overhead (not counting new objects to be + * downloaded). + * + * Note that the local set of tags in "refs/tags/" is a union over all + * remotes. However, since most people only have one remote, we can + * probaly estimate the overhead value directly from the size of the + * set of "refs/tags/" that we visited while building the `ref_info` + * and `ref_array` and not need to ask the remote. + * + * [] Dump info on the complexity of the DAG. Criss-cross merges. + * The number of edges that must be touched to compute merge bases. + * Edge length. The number of parallel lanes in the history that must + * be navigated to get to the merge base. What affects the cost of + * the Ahead/Behind computation? How often do criss-crosses occur and + * do they cause various operations to slow down? + * + * [] If there are primary branches (like "main" or "master") are they + * always on the left side of merges? Does the graph have a clean + * left edge? Or are there normal and "backwards" merges? Do these + * cause problems at scale? + * + * [] If we have a hierarchy of FI/RI branches like "L1", "L2, ..., + * can we learn anything about the shape of the repo around these FI + * and RI integrations? + */ From c558d239b81fae0d9489c1738fed5a95a5e90538 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 18 Sep 2026 18:12:34 +0200 Subject: [PATCH 016/289] fixup! Add an AGENTS.md file to help with AI-assisted debugging/development The PowerShell example still assumes `/mingw64`, which is absent from the current x64 SDK. Keep its environment and search path consistent with UCRT64 while preserving the non-login `bash` invocation and access to `/usr/bin`. Assisted-by: GPT-6 Signed-off-by: Johannes Schindelin --- AGENTS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7273095017731a..a387a425110291 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,8 +48,8 @@ to `PATH`, then invoke a non-login `bash -c` (replace `C:\git-sdk-64` with your SDK root): ```powershell -$env:MSYSTEM = "MINGW64" -$env:PATH = "C:\git-sdk-64\mingw64\bin;C:\git-sdk-64\usr\bin;" + $env:PATH +$env:MSYSTEM = "UCRT64" +$env:PATH = "C:\git-sdk-64\ucrt64\bin;C:\git-sdk-64\usr\bin;" + $env:PATH & C:\git-sdk-64\usr\bin\bash.exe -c "make -j15" ``` From 62dfe306baa2977dc5c68cd0844d026bd5eabcb6 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 25 Aug 2023 09:58:27 -0400 Subject: [PATCH 017/289] t5300: confirm failure of git index-pack when non-idx suffix requested Add test case to demonstrate that `git index-pack -o pack-path` fails if does not end in ".idx" when `--rev-index` is enabled. In e37d0b8730b (builtin/index-pack.c: write reverse indexes, 2021-01-25) we learned to create `.rev` reverse indexes in addition to `.idx` index files. The `.rev` file pathname is constructed by replacing the suffix on the `.idx` file. The code assumes a hard-coded "idx" suffix. In a8dd7e05b1c (config: enable `pack.writeReverseIndex` by default, 2023-04-12) reverse indexes were enabled by default. If the `-o ` argument is used, the index file may have a different suffix. This causes an error when it tries to create the reverse index pathname. The test here demonstrates the failure. (The test forces `--rev-index` to avoid interaction with `GIT_TEST_NO_WRITE_REV_INDEX` during CI runs.) Signed-off-by: Jeff Hostetler --- t/t5300-pack-object.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/t/t5300-pack-object.sh b/t/t5300-pack-object.sh index aac139e6a096eb..a9612c51b02993 100755 --- a/t/t5300-pack-object.sh +++ b/t/t5300-pack-object.sh @@ -381,6 +381,31 @@ test_expect_success 'build pack index for an existing pack' ' : ' +# The `--rev-index` option of `git index-pack` is now the default, so +# a `foo.rev` REV file will be created when a `foo.idx` IDX file is +# created. Normally, these pathnames are based upon the `foo.pack` +# PACK file pathname. +# +# However, the `-o` option lets you set the pathname of the IDX file +# indepdent of the PACK file. +# +# Verify what happens if these suffixes are changed. +# +test_expect_success 'complain about index name' ' + # Normal case { .pack, .idx, .rev } + cat test-1-${packname_1}.pack >test-complain-0.pack && + git index-pack -o test-complain-0.idx --rev-index test-complain-0.pack && + test -f test-complain-0.idx && + test -f test-complain-0.rev && + + # Non .idx suffix + cat test-1-${packname_1}.pack >test-complain-1.pack && + test_must_fail git index-pack -o test-complain-1.idx-suffix --rev-index test-complain-1.pack 2>err && + grep "does not end" err && + ! test -f test-complain-1.idx-suffix && + ! test -f test-complain-1.rev +' + test_expect_success 'unpacking with --strict' ' for j in a b c d e f g From 4b6a06262704b3ed344b588b19635eb7d8beaf76 Mon Sep 17 00:00:00 2001 From: Victoria Dye Date: Wed, 20 Sep 2023 13:12:30 -0700 Subject: [PATCH 018/289] t1092: add test for untracked files and directories Add a test verifying that sparse-checkout (with and without sparse index enabled) treat untracked files & directories correctly when changing sparse patterns. Specifically, it ensures that 'git sparse-checkout set' * deletes empty directories outside the sparse cone * does _not_ delete untracked files outside the sparse cone Signed-off-by: Victoria Dye --- t/t1092-sparse-checkout-compatibility.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh index 05b54062b3bc85..9899e87cb0d241 100755 --- a/t/t1092-sparse-checkout-compatibility.sh +++ b/t/t1092-sparse-checkout-compatibility.sh @@ -317,6 +317,22 @@ test_expect_success 'root directory cannot be sparse' ' test_cmp expect actual ' +test_expect_success 'sparse-checkout with untracked files and dirs' ' + init_repos && + + # Empty directories outside sparse cone are deleted + run_on_sparse mkdir -p deep/empty && + test_sparse_match git sparse-checkout set folder1 && + test_must_be_empty sparse-checkout-err && + run_on_sparse test_path_is_missing deep && + + # Untracked files outside sparse cone are not deleted + run_on_sparse touch folder1/another && + test_sparse_match git sparse-checkout set folder2 && + test_grep "directory ${SQ}folder1/${SQ} contains untracked files" sparse-checkout-err && + run_on_sparse test_path_exists folder1/another +' + test_expect_success 'status with options' ' init_repos && test_sparse_match ls && From 387d42e8b3cc60d31690db37621a187c933410e1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 4 Apr 2017 12:04:11 +0200 Subject: [PATCH 019/289] gvfs: ensure that the version is based on a GVFS tag Since we really want to be based on a `.vfs.*` tag, let's make sure that there was a new-enough one, i.e. one that agrees with the first three version numbers of the recorded default version. This prevents e.g. v2.22.0.vfs.0.. from being used when the current release train was not yet tagged. It is important to get the first three numbers of the version right because e.g. Scalar makes decisions depending on those (such as assuming that the `git maintenance` built-in is not available, even though it actually _is_ available). Signed-off-by: Johannes Schindelin --- GIT-VERSION-GEN | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/GIT-VERSION-GEN b/GIT-VERSION-GEN index 18bc599a78404e..e7540a11ed0155 100755 --- a/GIT-VERSION-GEN +++ b/GIT-VERSION-GEN @@ -50,9 +50,15 @@ then test -d "${GIT_DIR:-.git}" || test -f "$SOURCE_DIR"/.git; } && - VN=$(git -C "$SOURCE_DIR" describe --dirty --match="v[0-9]*" 2>/dev/null) && + VN=$(git -C "$SOURCE_DIR" describe --dirty --match="v[0-9]*vfs*" 2>/dev/null) && case "$VN" in *$LF*) (exit 1) ;; + v[0-9]*) + if test "${VN%%.vfs.*}" != "${DEF_VER%%.vfs.*}" + then + echo "Found version $VN, which is not based on $DEF_VER" >&2 + exit 1 + fi ;; esac then VN=$(echo "$VN" | sed -e 's/-/./g'); From 5d6b31e4f1df4f8ce2d6a6d51cd6c6060afe2285 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 28 Jun 2024 15:22:46 -0400 Subject: [PATCH 020/289] survey: expanded TODO list at the bottom of the source file Signed-off-by: Jeff Hostetler --- builtin/survey.c | 150 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 117 insertions(+), 33 deletions(-) diff --git a/builtin/survey.c b/builtin/survey.c index 8f8602e9d57710..a6635b09e3aa00 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -113,47 +113,131 @@ int cmd_survey(int argc, const char **argv, const char *prefix, } /* - * NEEDSWORK: The following is a bit of a laundry list of things - * that I'd like to add. + * NEEDSWORK: So far, I only have iteration on the requested set of + * refs and treewalk/reachable objects on that set of refs. The + * following is a bit of a laundry list of things that I'd like to + * add. * * [] Dump stats on all of the packfiles. The number and size of each. - * Whether each is in the .git directory or in an alternate. The state - * of the IDX or MIDX files and etc. Delta chain stats. All of this - * data is relative to the "lived-in" state of the repository. Stuff - * that may change after a GC or repack. + * Whether each is in the .git directory or in an alternate. The + * state of the IDX or MIDX files and etc. Delta chain stats. All + * of this data is relative to the "lived-in" state of the + * repository. Stuff that may change after a GC or repack. + * + * [] Clone and Index stats. partial, shallow, sparse-checkout, + * sparse-index, etc. Hydration stats. * * [] Dump stats on each remote. When we fetch from a remote the size - * of the response is related to the set of haves on the server. You - * can see this in `GIT_TRACE_CURL=1 git fetch`. We get a `ls-refs` - * payload that lists all of the branches and tags on the server, so - * at a minimum the RefName and SHA for each. But for annotated tags - * we also get the peeled SHA. The size of this overhead on every - * fetch is proporational to the size of the `git ls-remote` response - * (roughly, although the latter repeats the RefName of the peeled - * tag). If, for example, you have 500K refs on a remote, you're - * going to have a long "haves" message, so every fetch will be slow - * just because of that overhead (not counting new objects to be - * downloaded). - * - * Note that the local set of tags in "refs/tags/" is a union over all - * remotes. However, since most people only have one remote, we can - * probaly estimate the overhead value directly from the size of the - * set of "refs/tags/" that we visited while building the `ref_info` - * and `ref_array` and not need to ask the remote. + * of the response is related to the set of haves on the server. + * You can see this in `GIT_TRACE_CURL=1 git fetch`. We get a + * `ls-refs` payload that lists all of the branches and tags on the + * server, so at a minimum the RefName and SHA for each. But for + * annotated tags we also get the peeled SHA. The size of this + * overhead on every fetch is proporational to the size of the `git + * ls-remote` response (roughly, although the latter repeats the + * RefName of the peeled tag). If, for example, you have 500K refs + * on a remote, you're going to have a long "haves" message, so + * every fetch will be slow just because of that overhead (not + * counting new objects to be downloaded). + * + * Note that the local set of tags in "refs/tags/" is a union over + * all remotes. However, since most people only have one remote, + * we can probaly estimate the overhead value directly from the + * size of the set of "refs/tags/" that we visited while building + * the `ref_info` and `ref_array` and not need to ask the remote. * * [] Dump info on the complexity of the DAG. Criss-cross merges. - * The number of edges that must be touched to compute merge bases. - * Edge length. The number of parallel lanes in the history that must - * be navigated to get to the merge base. What affects the cost of - * the Ahead/Behind computation? How often do criss-crosses occur and - * do they cause various operations to slow down? + * The number of edges that must be touched to compute merge bases. + * Edge length. The number of parallel lanes in the history that + * must be navigated to get to the merge base. What affects the + * cost of the Ahead/Behind computation? How often do + * criss-crosses occur and do they cause various operations to slow + * down? * * [] If there are primary branches (like "main" or "master") are they - * always on the left side of merges? Does the graph have a clean - * left edge? Or are there normal and "backwards" merges? Do these - * cause problems at scale? + * always on the left side of merges? Does the graph have a clean + * left edge? Or are there normal and "backwards" merges? Do + * these cause problems at scale? * * [] If we have a hierarchy of FI/RI branches like "L1", "L2, ..., - * can we learn anything about the shape of the repo around these FI - * and RI integrations? + * can we learn anything about the shape of the repo around these + * FI and RI integrations? + * + * [] Do we need a no-PII flag to omit pathnames or branch/tag names + * in the various histograms? (This would turn off --name-rev + * too.) + * + * [] I have so far avoided adding opinions about individual fields + * (such as the way `git-sizer` prints a row of stars or bangs in + * the last column). + * + * I'm wondering if that is a job of this executable or if it + * should be done in a post-processing step using the JSON output. + * + * My problem with the `git-sizer` approach is that it doesn't give + * the (casual) user any information on why it has stars or bangs. + * And there isn't a good way to print detailed information in the + * ASCII-art tables that would be easy to understand. + * + * [] For example, a large number of refs does not define a cliff. + * Performance will drop off (linearly, quadratically, ... ??). + * The tool should refer them to article(s) talking about the + * different problems that it could cause. So should `git + * survey` just print the number and (implicitly) refer them to + * the man page (chapter/verse) or to a tool that will interpret + * the number and explain it? + * + * [] Alternatively, should `git survey` do that analysis too and + * just print footnotes for each large number? + * + * [] The computation of the raw survey JSON data can take HOURS on + * a very large repo (like Windows), so I'm wondering if we + * want to keep the opinion portion separate. + * + * [] In addition to opinions based on the static data, I would like + * to dump the JSON results (or the Trace2 telemetry) into a DB and + * aggregate it with other users. + * + * Granted, they should all see the same DAG and the same set of + * reachable objects, but we could average across all datasets + * generated on a particular date and detect outlier users. + * + * [] Maybe someone cloned from the `_full` endpoint rather than + * the limited refs endpoint. + * + * [] Maybe that user is having problems with repacking / GC / + * maintenance without knowing it. + * + * [] I'd also like to dump use the DB to compare survey datasets over + * a time. How fast is their repository growing and in what ways? + * + * [] I'd rather have the delta analysis NOT be inside `git + * survey`, so it makes sense to consider having all of it in a + * post-process step. + * + * [] Another reason to put the opinion analysis in a post-process + * is that it would be easier to generate plots on the data tables. + * Granted, we can get plots from telemetry, but a stand-alone user + * could run the JSON thru python or jq or something and generate + * something nicer than ASCII-art and it could handle cross-referencing + * and hyperlinking to helpful information on each issue. + * + * [] I think there are several classes of data that we can report on: + * + * [] The "inherit repo properties", such as the shape and size of + * the DAG -- these should be universal in each enlistment. + * + * [] The "ODB lived in properties", such as the efficiency + * of the repack and things like partial and shallow clone. + * These will vary, but indicate health of the ODB. + * + * [] The "index related properties", such as sparse-checkout, + * sparse-index, cache-tree, untracked-cache, fsmonitor, and + * etc. These will also vary, but are more like knobs for + * the user to adjust. + * + * [] I want to compare these with Matt's "dimensions of scale" + * notes and see if there are other pieces of data that we + * could compute/consider. + * */ From 3e3cda8287389f3b473ffa0587ee499c4cf99809 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 21 Sep 2026 20:45:58 +0200 Subject: [PATCH 021/289] fixup! survey: turn into a thin shim over `git repo structure` The deprecation note still promises behavior intentionally removed by the shim. Explain the migration to `git repo structure` so users can choose their ref scope and output, including opting out of name lookup, without relying on ignored configuration or unsupported options. Assisted-by: GPT-6 Signed-off-by: Johannes Schindelin --- Documentation/config/survey.adoc | 15 ++---------- Documentation/git-survey.adoc | 39 +++++++++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/Documentation/config/survey.adoc b/Documentation/config/survey.adoc index 9e594a2092f225..0bee888d1b9d74 100644 --- a/Documentation/config/survey.adoc +++ b/Documentation/config/survey.adoc @@ -1,14 +1,3 @@ survey.*:: - These variables adjust the default behavior of the `git survey` - command. The intention is that this command could be run in the - background with these options. -+ --- - verbose:: - This boolean value implies the `--[no-]verbose` option. - progress:: - This boolean value implies the `--[no-]progress` option. - top:: - This integer value implies `--top=`, specifying the - number of entries in the detail tables. --- + Deprecated and ignored by `git survey`. See the "MIGRATION" section + of linkgit:git-survey[1] for replacements and changed defaults. diff --git a/Documentation/git-survey.adoc b/Documentation/git-survey.adoc index e97656b49dee75..4d5ca497a8e411 100644 --- a/Documentation/git-survey.adoc +++ b/Documentation/git-survey.adoc @@ -44,8 +44,8 @@ Ref Selection The following options control the set of refs that `git survey` will examine. By default, `git survey` will look at tags, local branches, and remote refs. -If any of the following options are given, the default set is cleared and -only refs for the given options are added. +If any ref-selection option other than `--detached` is given, the default +set is cleared and only refs for the given options are added. --all-refs:: Use all refs. This includes local branches, tags, remote refs, @@ -61,11 +61,44 @@ only refs for the given options are added. Add remote branches (`refs/remote/`) to the set. --detached:: - Add HEAD to the set. + Ignored with a warning. This option does not affect ref selection. --other:: Add notes (`refs/notes/`) and stashes (`refs/stash/`) to the set. +MIGRATION +--------- +`git repo structure` selects all refs by default. To retain the survey +default scope, pass `--ref-filter='refs/heads/*'`, +`--ref-filter='refs/tags/*'`, and `--ref-filter='refs/remotes/*'`. + +Neither command separately enumerates detached `HEAD`, even with +`git survey --all-refs`. To include history reachable only from `HEAD`, +first create a branch or tag pointing to `HEAD` and include that ref in +the selection. The new ref is also counted. + +For path detail tables, `git repo structure` defaults to `--top=0`; use +`--top=10` or `repo.structure.top=10` for the survey default. The shim +always supplies `--top` (10 unless overridden on its command line), so +`repo.structure.top` does not change `git survey`'s limit. + +`--commit-parents=`, `--commit-sizes=`, `--tree-entries=`, +`--tree-sizes=`, `--blob-sizes=`, and `--[no-]name-rev` retain their +names in `git repo structure`, but are not accepted by the shim. Each +object-list limit defaults to 0. + +Name lookup for ranked commits is now enabled by default. Use +`git repo structure --no-name-rev` or set `repo.structure.nameRev=false` +to opt out; the configuration also applies through `git survey`. + +All `survey.*` settings are ignored. For `top`, `nameRev`, +`showCommitParents`, `showCommitSizes`, `showTreeEntries`, `showTreeSizes`, +and `showBlobSizes`, replace the `survey.` prefix with `repo.structure.` +when invoking `git repo structure` directly. Use `--progress` or +`--no-progress` instead of `survey.progress`; no `repo.structure.progress` +exists. `survey.verbose` has no replacement; the shim ignores `--verbose` +with a warning. + OUTPUT ------ From 21a20d75ec854f6ff88025db57ca5963ce31e965 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 25 Aug 2023 11:06:28 -0400 Subject: [PATCH 022/289] index-pack: disable rev-index if index file has non .idx suffix Teach index-pack to silently omit the reverse index if the index file does not have the standard ".idx" suffix. In e37d0b8730b (builtin/index-pack.c: write reverse indexes, 2021-01-25) we learned to create `.rev` reverse indexes in addition to `.idx` index files. The `.rev` file pathname is constructed by replacing the suffix on the `.idx` file. The code assumes a hard-coded "idx" suffix. In a8dd7e05b1c (config: enable `pack.writeReverseIndex` by default, 2023-04-12) reverse indexes were enabled by default. If the `-o ` argument is used, the index file may have a different suffix. This causes an error when it tries to create the reverse index pathname. Since we do not know why the user requested a non-standard suffix for the index, we cannot guess what the proper corresponding suffix should be for the reverse index. So we disable it. The t5300 test has been updated to verify that we no longer error out and that the .rev file is not created. TODO We could warn the user that we skipped it (perhaps only if they TODO explicitly requested `--rev-index` on the command line). TODO TODO Ideally, we should add an `--rev-index-path=` argument TODO or change `--rev-index` to take a pathname. TODO TODO I'll leave these questions for a future series. Signed-off-by: Jeff Hostetler --- builtin/index-pack.c | 4 ++++ t/t5300-pack-object.sh | 7 +++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/builtin/index-pack.c b/builtin/index-pack.c index 6b2a87e2d39355..53a6decadfb5ae 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -1902,6 +1902,7 @@ int cmd_index_pack(int argc, unsigned foreign_nr = 1; /* zero is a "good" value, assume bad */ int report_end_of_input = 0; int hash_algo = 0; + int dash_o = 0; show_usage_if_asked(argc, argv, index_pack_usage); @@ -1988,6 +1989,7 @@ int cmd_index_pack(int argc, if (index_name || (i+1) >= argc) usage(index_pack_usage); index_name = argv[++i]; + dash_o = 1; } else if (starts_with(arg, "--index-version=")) { char *c; opts.version = strtoul(arg + 16, &c, 10); @@ -2041,6 +2043,8 @@ int cmd_index_pack(int argc, repo_set_hash_algo(the_repository, GIT_HASH_DEFAULT); opts.flags &= ~(WRITE_REV | WRITE_REV_VERIFY); + if (rev_index && dash_o && !ends_with(index_name, ".idx")) + rev_index = 0; if (rev_index) { opts.flags |= verify ? WRITE_REV_VERIFY : WRITE_REV; if (index_name) diff --git a/t/t5300-pack-object.sh b/t/t5300-pack-object.sh index a9612c51b02993..26f5c3d78d6c35 100755 --- a/t/t5300-pack-object.sh +++ b/t/t5300-pack-object.sh @@ -398,11 +398,10 @@ test_expect_success 'complain about index name' ' test -f test-complain-0.idx && test -f test-complain-0.rev && - # Non .idx suffix + # Non .idx suffix -- implicitly omits the .rev cat test-1-${packname_1}.pack >test-complain-1.pack && - test_must_fail git index-pack -o test-complain-1.idx-suffix --rev-index test-complain-1.pack 2>err && - grep "does not end" err && - ! test -f test-complain-1.idx-suffix && + git index-pack -o test-complain-1.idx-suffix --rev-index test-complain-1.pack && + test -f test-complain-1.idx-suffix && ! test -f test-complain-1.rev ' From 8fe03189bbfc51f81436acaac3a2cb24bc42a7ba Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 24 Jan 2017 17:30:59 +0100 Subject: [PATCH 023/289] gvfs: add a GVFS-specific header file This header file will accumulate GVFS-specific definitions. Signed-off-by: Kevin Willford --- gvfs.h | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 gvfs.h diff --git a/gvfs.h b/gvfs.h new file mode 100644 index 00000000000000..b6dbe85eae4071 --- /dev/null +++ b/gvfs.h @@ -0,0 +1,9 @@ +#ifndef GVFS_H +#define GVFS_H + +/* + * This file is for the specific settings and methods + * used for GVFS functionality + */ + +#endif /* GVFS_H */ From 6e71e20e59d5ad9bc61ab5628491505f78ae6017 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 1 Jul 2024 12:07:01 -0400 Subject: [PATCH 024/289] survey: expanded TODO with more notes Signed-off-by: Jeff Hostetler --- builtin/survey.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/builtin/survey.c b/builtin/survey.c index a6635b09e3aa00..637dde74eb7c8b 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -146,6 +146,16 @@ int cmd_survey(int argc, const char **argv, const char *prefix, * size of the set of "refs/tags/" that we visited while building * the `ref_info` and `ref_array` and not need to ask the remote. * + * [] Should the "string length of refnames / remote refs", for + * example, be sub-divided by remote so we can project the + * cost of the haves/wants overhead a fetch. + * + * [] Can we examine the merge commits and classify them as clean or + * dirty? (ie. ones with merge conflicts that needed to be + * addressed during the merge itself.) + * + * [] Do dirty merges affect performance of later operations? + * * [] Dump info on the complexity of the DAG. Criss-cross merges. * The number of edges that must be touched to compute merge bases. * Edge length. The number of parallel lanes in the history that From b6e9a10c164c72156cc0674f7c2a6425542c9d03 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 26 Jun 2024 12:27:41 -0400 Subject: [PATCH 025/289] trace2: prefetch value of GIT_TRACE2_DST_DEBUG at startup Prefetch the value of GIT_TRACE2_DST_DEBUG during startup and before we try to open any Trace2 destination pathnames. Normally, Trace2 always silently fails if a destination target cannot be opened so that it doesn't affect the execution of a Git command. The command should run normally, but just not generate any trace data. This can make it difficult to debug a telemetry setup, since the user doesn't know why telemetry isn't being generated. If the environment variable GIT_TRACE2_DST_DEBUG is true, the Trace2 startup will print a warning message with the `errno` to make debugging easier. However, on Windows, looking up the env variable resets `errno` so the warning message always ends with `...tracing: No error` which is not very helpful. Prefetch the env variable at startup. This avoids the need to update each call-site to capture `errno` in the usual `saved-errno` variable. Signed-off-by: Jeff Hostetler --- trace2.c | 10 ++++++++++ trace2/tr2_dst.c | 2 +- trace2/tr2_dst.h | 12 ++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/trace2.c b/trace2.c index c23c0a227b7032..24dd7b1be9fbbe 100644 --- a/trace2.c +++ b/trace2.c @@ -227,6 +227,16 @@ void trace2_initialize_fl(const char *file, int line) if (!tr2_tgt_want_builtins()) return; trace2_enabled = 1; + + /* + * getenv() on Windows stomps on `errno` and the code in + * tr2_dst.c verifies that warnings are enabled before + * formatting the warning message (and calling strerror()). + * So prefetch the value from the environment before we need + * it. + */ + tr2_dst_want_warning(); + if (!git_env_bool("GIT_TRACE2_REDACT", 1)) trace2_redact = 0; diff --git a/trace2/tr2_dst.c b/trace2/tr2_dst.c index 5be892cd5cdefa..61579f24bdbde3 100644 --- a/trace2/tr2_dst.c +++ b/trace2/tr2_dst.c @@ -24,7 +24,7 @@ */ static int tr2env_max_files = 0; -static int tr2_dst_want_warning(void) +int tr2_dst_want_warning(void) { static int tr2env_dst_debug = -1; diff --git a/trace2/tr2_dst.h b/trace2/tr2_dst.h index b1a8c144e073ba..4166539eb9e100 100644 --- a/trace2/tr2_dst.h +++ b/trace2/tr2_dst.h @@ -35,4 +35,16 @@ int tr2_dst_trace_want(struct tr2_dst *dst); */ void tr2_dst_write_line(struct tr2_dst *dst, struct strbuf *buf_line); +/* + * Return true if we want warning messages when trying to open a + * destination. + * + * (Trace2 always silently fails if a target cannot be opened so that + * we don't affect the execution of the Git command, but it is helpful + * for debugging telemetry configuration if we log warning messages + * when trying to open a target. This is controlled by another config + * value.) + */ +int tr2_dst_want_warning(void); + #endif /* TR2_DST_H */ From 6bb6ceae9a11034a5db6d1a7e6c6350b6a848268 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 24 Jan 2017 17:34:12 +0100 Subject: [PATCH 026/289] gvfs: add the core.gvfs config setting This does not do anything yet. The next patches will add various values for that config setting that correspond to the various features offered/required by GVFS. Signed-off-by: Kevin Willford gvfs: refactor loading the core.gvfs config value This code change makes sure that the config value for core_gvfs is always loaded before checking it. Signed-off-by: Kevin Willford --- Documentation/config/core.adoc | 3 +++ Makefile | 1 + config.c | 1 + gvfs.c | 48 ++++++++++++++++++++++++++++++++++ gvfs.h | 4 +++ meson.build | 1 + 6 files changed, 58 insertions(+) create mode 100644 gvfs.c diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index 4d40c16ede9cfe..7697cfb038c3fe 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -781,6 +781,9 @@ core.multiPackIndex:: single index. See linkgit:git-multi-pack-index[1] for more information. Defaults to true. +core.gvfs:: + Enable the features needed for GVFS. + core.sparseCheckout:: Enable "sparse checkout" feature. See linkgit:git-sparse-checkout[1] for more information. diff --git a/Makefile b/Makefile index c98be9155c54ae..46da13672b1a88 100644 --- a/Makefile +++ b/Makefile @@ -1187,6 +1187,7 @@ LIB_OBJS += git-zlib.o LIB_OBJS += gpg-interface.o LIB_OBJS += graph.o LIB_OBJS += grep.o +LIB_OBJS += gvfs.o LIB_OBJS += hash-lookup.o LIB_OBJS += hash.o LIB_OBJS += hashmap.o diff --git a/config.c b/config.c index d9019e7e6c34b0..105700b1368030 100644 --- a/config.c +++ b/config.c @@ -10,6 +10,7 @@ #include "abspath.h" #include "advice.h" #include "date.h" +#include "gvfs.h" #include "branch.h" #include "config.h" #include "dir.h" diff --git a/gvfs.c b/gvfs.c new file mode 100644 index 00000000000000..0c43639bcb21de --- /dev/null +++ b/gvfs.c @@ -0,0 +1,48 @@ +#define USE_THE_REPOSITORY_VARIABLE +#include "git-compat-util.h" +#include "environment.h" +#include "gvfs.h" +#include "setup.h" +#include "config.h" + +static int gvfs_config_loaded; +static struct repository *gvfs_repo; +static int core_gvfs; +static int core_gvfs_is_bool; + +static int early_core_gvfs_config(const char *var, const char *value, + const struct config_context *ctx, void *cb UNUSED) +{ + if (!strcmp(var, "core.gvfs")) + core_gvfs = git_config_bool_or_int("core.gvfs", value, ctx->kvi, + &core_gvfs_is_bool); + return 0; +} + +static void gvfs_load_config_value(struct repository *r) +{ + if (gvfs_config_loaded && gvfs_repo == r) + return; + + if (r) { + repo_config_get_bool_or_int(r, "core.gvfs", + &core_gvfs_is_bool, &core_gvfs); + } else if (startup_info->have_repository == 0) + read_early_config(the_repository, early_core_gvfs_config, NULL); + else + repo_config_get_bool_or_int(the_repository, "core.gvfs", + &core_gvfs_is_bool, &core_gvfs); + + /* Turn on all bits if a bool was set in the settings */ + if (core_gvfs_is_bool && core_gvfs) + core_gvfs = -1; + + gvfs_config_loaded = 1; + gvfs_repo = r; +} + +int gvfs_config_is_set(struct repository *r, int mask) +{ + gvfs_load_config_value(r); + return (core_gvfs & mask) == mask; +} diff --git a/gvfs.h b/gvfs.h index b6dbe85eae4071..01b748ebcecc99 100644 --- a/gvfs.h +++ b/gvfs.h @@ -1,9 +1,13 @@ #ifndef GVFS_H #define GVFS_H +struct repository; + /* * This file is for the specific settings and methods * used for GVFS functionality */ +int gvfs_config_is_set(struct repository *r, int mask); + #endif /* GVFS_H */ diff --git a/meson.build b/meson.build index 79b6af6eaf9641..4dbacf2cbcf7f4 100644 --- a/meson.build +++ b/meson.build @@ -386,6 +386,7 @@ libgit_sources = [ 'gpg-interface.c', 'graph.c', 'grep.c', + 'gvfs.c', 'hash-lookup.c', 'hash.c', 'hashmap.c', From 31045006ff7ddeaf86461ebe6be27a0080422026 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 24 Jan 2017 17:38:59 +0100 Subject: [PATCH 027/289] gvfs: add the feature to skip writing the index' SHA-1 This takes a substantial amount of time, and if the user is reasonably sure that the files' integrity is not compromised, that time can be saved. Git no longer verifies the SHA-1 by default, anyway. Signed-off-by: Kevin Willford Update for 2023-02-27: This feature was upstreamed as the index.skipHash config option. This resulted in some changes to the struct and some of the setup code. In particular, the config reading was moved to prepare_repo_settings(), so the core.gvfs bit check was moved there, too. Signed-off-by: Kevin Willford Signed-off-by: Derrick Stolee Signed-off-by: Johannes Schindelin --- Documentation/config/core.adoc | 10 +++++++++- gvfs.h | 5 +++++ repo-settings.c | 8 ++++++++ t/meson.build | 1 + t/t1018-read-tree-skip-sha-on-read.sh | 22 ++++++++++++++++++++++ 5 files changed, 45 insertions(+), 1 deletion(-) create mode 100755 t/t1018-read-tree-skip-sha-on-read.sh diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index 7697cfb038c3fe..72275a2d46514b 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -782,7 +782,15 @@ core.multiPackIndex:: information. Defaults to true. core.gvfs:: - Enable the features needed for GVFS. + Enable the features needed for GVFS. This value can be set to true + to indicate all features should be turned on or the bit values listed + below can be used to turn on specific features. ++ +-- + GVFS_SKIP_SHA_ON_INDEX:: + Bit value 1 + Disables the calculation of the sha when writing the index +-- core.sparseCheckout:: Enable "sparse checkout" feature. See linkgit:git-sparse-checkout[1] diff --git a/gvfs.h b/gvfs.h index 01b748ebcecc99..9bf5d00b365d90 100644 --- a/gvfs.h +++ b/gvfs.h @@ -8,6 +8,11 @@ struct repository; * used for GVFS functionality */ +/* + * The list of bits in the core_gvfs setting + */ +#define GVFS_SKIP_SHA_ON_INDEX (1 << 0) + int gvfs_config_is_set(struct repository *r, int mask); #endif /* GVFS_H */ diff --git a/repo-settings.c b/repo-settings.c index f3be3b8c5a3d09..11b0eb8adc571f 100644 --- a/repo-settings.c +++ b/repo-settings.c @@ -5,6 +5,7 @@ #include "midx.h" #include "pack-objects.h" #include "setup.h" +#include "gvfs.h" static void repo_cfg_bool(struct repository *r, const char *key, int *dest, int def) @@ -85,6 +86,13 @@ void prepare_repo_settings(struct repository *r) r->settings.pack_use_bitmap_boundary_traversal); repo_cfg_bool(r, "core.usereplacerefs", &r->settings.read_replace_refs, 1); + /* + * For historical compatibility reasons, enable index.skipHash based + * on a bit in core.gvfs. + */ + if (gvfs_config_is_set(r, GVFS_SKIP_SHA_ON_INDEX)) + r->settings.index_skip_hash = 1; + /* * The GIT_TEST_MULTI_PACK_INDEX variable is special in that * either it *or* the config sets diff --git a/t/meson.build b/t/meson.build index 729de1177e76af..0bc88a047fe223 100644 --- a/t/meson.build +++ b/t/meson.build @@ -174,6 +174,7 @@ integration_tests = [ 't1015-read-index-unmerged.sh', 't1016-compatObjectFormat.sh', 't1017-cat-file-remote-object-info.sh', + 't1018-read-tree-skip-sha-on-read.sh', 't1020-subdirectory.sh', 't1022-read-tree-partial-clone.sh', 't1050-large.sh', diff --git a/t/t1018-read-tree-skip-sha-on-read.sh b/t/t1018-read-tree-skip-sha-on-read.sh new file mode 100755 index 00000000000000..5b76a80a0020dc --- /dev/null +++ b/t/t1018-read-tree-skip-sha-on-read.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +test_description='check that read-tree works with core.gvfs config value' + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-read-tree.sh + +test_expect_success setup ' + echo one >a && + git add a && + git commit -m initial +' +test_expect_success 'read-tree without core.gvsf' ' + read_tree_u_must_succeed -m -u HEAD +' + +test_expect_success 'read-tree with core.gvfs set to 1' ' + git config core.gvfs 1 && + read_tree_u_must_succeed -m -u HEAD +' + +test_done From dad67c0a51c18127ec4b93a1898493cb27e53a3d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 18 May 2021 23:22:56 +0200 Subject: [PATCH 028/289] scalar: set the config write-lock timeout to 150ms By default, Git fails immediately when locking a config file for writing fails due to an existing lock. With this change, Scalar-registered repositories will fall back to trying a couple times within a 150ms timeout. Signed-off-by: Johannes Schindelin --- Documentation/scalar.adoc | 3 +++ scalar.c | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/Documentation/scalar.adoc b/Documentation/scalar.adoc index 5252fb134a47ab..397c4bde35e473 100644 --- a/Documentation/scalar.adoc +++ b/Documentation/scalar.adoc @@ -347,6 +347,9 @@ status.aheadBehind=false:: message that can be disabled by disabling the `advice.statusAheadBehind` config. +core.configWriteLockTimeoutMS:: + Sets a timeout to work gracefully around Git config write contention. + The following settings are different based on which platform is in use: core.untrackedCache=(true|false):: diff --git a/scalar.c b/scalar.c index a80d8ee3ff54a7..edcbceb4011159 100644 --- a/scalar.c +++ b/scalar.c @@ -141,6 +141,7 @@ static int set_recommended_config(int reconfigure) { "commitGraph.changedPaths", "true" }, { "commitGraph.generationVersion", "1" }, { "core.autoCRLF", "false" }, + { "core.configWriteLockTimeoutMS", "150" }, { "core.logAllRefUpdates", "true" }, { "core.safeCRLF", "false" }, { "credential.https://dev.azure.com.useHttpPath", "true" }, @@ -228,6 +229,11 @@ static int set_recommended_config(int reconfigure) */ static int toggle_maintenance(int enable) { + unsigned long ul; + + if (repo_config_get_ulong(the_repository, "core.configWriteLockTimeoutMS", &ul)) + git_config_push_parameter("core.configWriteLockTimeoutMS=150"); + return run_git("maintenance", enable ? "start" : "unregister", enable ? NULL : "--force", @@ -237,10 +243,14 @@ static int toggle_maintenance(int enable) static int add_or_remove_enlistment(int add) { int res; + unsigned long ul; if (!the_repository->worktree) die(_("Scalar enlistments require a worktree")); + if (repo_config_get_ulong(the_repository, "core.configWriteLockTimeoutMS", &ul)) + git_config_push_parameter("core.configWriteLockTimeoutMS=150"); + res = run_git("config", "--global", "--get", "--fixed-value", "scalar.repo", the_repository->worktree, NULL); From 0e97b919b2fa07bf2e3fe98dd35efd9704564d4a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 18 May 2021 23:22:56 +0200 Subject: [PATCH 029/289] scalar: set the config write-lock timeout to 150ms By default, Git fails immediately when locking a config file for writing fails due to an existing lock. With this change, Scalar-registered repositories will fall back to trying a couple times within a 150ms timeout. Signed-off-by: Johannes Schindelin --- Documentation/scalar.adoc | 2 +- scalar.c | 11 +---------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/Documentation/scalar.adoc b/Documentation/scalar.adoc index 397c4bde35e473..0e0811b35dc8b6 100644 --- a/Documentation/scalar.adoc +++ b/Documentation/scalar.adoc @@ -347,7 +347,7 @@ status.aheadBehind=false:: message that can be disabled by disabling the `advice.statusAheadBehind` config. -core.configWriteLockTimeoutMS:: +core.configLockTimeout:: Sets a timeout to work gracefully around Git config write contention. The following settings are different based on which platform is in use: diff --git a/scalar.c b/scalar.c index edcbceb4011159..dd0cf3a260bfed 100644 --- a/scalar.c +++ b/scalar.c @@ -141,7 +141,7 @@ static int set_recommended_config(int reconfigure) { "commitGraph.changedPaths", "true" }, { "commitGraph.generationVersion", "1" }, { "core.autoCRLF", "false" }, - { "core.configWriteLockTimeoutMS", "150" }, + { "core.configLockTimeout", "150" }, { "core.logAllRefUpdates", "true" }, { "core.safeCRLF", "false" }, { "credential.https://dev.azure.com.useHttpPath", "true" }, @@ -229,11 +229,6 @@ static int set_recommended_config(int reconfigure) */ static int toggle_maintenance(int enable) { - unsigned long ul; - - if (repo_config_get_ulong(the_repository, "core.configWriteLockTimeoutMS", &ul)) - git_config_push_parameter("core.configWriteLockTimeoutMS=150"); - return run_git("maintenance", enable ? "start" : "unregister", enable ? NULL : "--force", @@ -243,14 +238,10 @@ static int toggle_maintenance(int enable) static int add_or_remove_enlistment(int add) { int res; - unsigned long ul; if (!the_repository->worktree) die(_("Scalar enlistments require a worktree")); - if (repo_config_get_ulong(the_repository, "core.configWriteLockTimeoutMS", &ul)) - git_config_push_parameter("core.configWriteLockTimeoutMS=150"); - res = run_git("config", "--global", "--get", "--fixed-value", "scalar.repo", the_repository->worktree, NULL); From 6e5b9ce78d74ffb87700d2db3dae6132dbffeeb5 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 16 Sep 2026 11:50:44 +0200 Subject: [PATCH 030/289] scalar: upgrade the config lock timeout setting automagically In Git v2.56.0, a different name was chosen for that setting, and also a different default value. Let's automagically upgrade the name of the config setting, if present. Signed-off-by: Johannes Schindelin --- scalar.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/scalar.c b/scalar.c index dd0cf3a260bfed..0333ed5e30c620 100644 --- a/scalar.c +++ b/scalar.c @@ -188,6 +188,28 @@ static int set_recommended_config(int reconfigure) int i; char *value; + /* + * If a user has "core.configWriteLockTimeoutMS" set, try to switch to + * the new (non-deprecated) setting (core.configLockTimeout). + */ + if (!repo_config_get_string(the_repository, "core.configwritelocktimeoutms", + &value)) { + char *dummy = NULL; + if (repo_config_get_string(the_repository, "core.configlocktimeout", + &dummy) && + repo_config_set_gently(the_repository, "core.configlocktimeout", + value)) + return error(_("could not configure %s=%s"), + "core.configLockTimeout", value); + if (repo_config_set_gently(the_repository, + "core.configwritelocktimeoutms", + NULL)) + return error(_("could not configure %s=%s"), + "core.configWriteLockTimeoutMS", "NULL"); + free(value); + free(dummy); + } + for (i = 0; config[i].key; i++) { if (set_config_if_missing(config + i, reconfigure)) return error(_("could not configure %s=%s"), From e3d9c2c6880e2b2ddcd96975b933045bb3d42d6a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 16 Sep 2026 12:01:59 +0200 Subject: [PATCH 031/289] config: (handle and) warn about deprecated lock timeout setting This commit reintroduces the 'core.configWriteLockTimeoutMS' config setting (originally added in 6333d1ad39a5 (git_config_set_multivar_in_file_gently(): add a lock timeout, 2021-05-18) as part of Microsoft v2.32.0). Upstream, this setting was superseded by the `core.configLockTimeout` setting that was added in df67d73ca326 (config: retry acquiring config.lock, configurable via core.configLockTimeout, 2026-05-17). However, several applications (e.g., 'scalar') utilize the original config setting, so it should be preserved for a deprecation period before complete removal: Additionally, for this deprecation period, advise users to switch to using 'core.configLockTimeout' to specify their preferred timeout. Signed-off-by: Johannes Schindelin --- Documentation/config/advice.adoc | 4 +++- advice.c | 1 + advice.h | 1 + config.c | 15 ++++++++++++++- 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/Documentation/config/advice.adoc b/Documentation/config/advice.adoc index e182cc9838a4c0..1c3048f7e50742 100644 --- a/Documentation/config/advice.adoc +++ b/Documentation/config/advice.adoc @@ -178,7 +178,9 @@ all advice messages. Shown when the user tries to create a worktree from an invalid reference, to tell the user how to create a new unborn branch instead. - + useCoreConfigWriteLockTimeoutMSConfig:: + Advice shown if the deprecated 'core.configWriteLockTimeoutMS' + config setting is in use. useCoreFSMonitorConfig:: Advice shown if the deprecated 'core.useBuiltinFSMonitor' config setting is in use. diff --git a/advice.c b/advice.c index 3fa240c521494c..fd99f6fba0477d 100644 --- a/advice.c +++ b/advice.c @@ -91,6 +91,7 @@ static struct { [ADVICE_SUBMODULE_MERGE_CONFLICT] = { "submoduleMergeConflict" }, [ADVICE_SUGGEST_DETACHING_HEAD] = { "suggestDetachingHead" }, [ADVICE_UPDATE_SPARSE_PATH] = { "updateSparsePath" }, + [ADVICE_USE_CORE_CONFIG_WRITE_LOCK_TIMEOUT_MS_CONFIG] = { "useCoreConfigWriteLockTimeoutMSConfig" }, [ADVICE_USE_CORE_FSMONITOR_CONFIG] = { "useCoreFSMonitorConfig" }, [ADVICE_WAITING_FOR_EDITOR] = { "waitingForEditor" }, [ADVICE_WORKTREE_ADD_ORPHAN] = { "worktreeAddOrphan" }, diff --git a/advice.h b/advice.h index ba02040196dbeb..caa3d6d03397cd 100644 --- a/advice.h +++ b/advice.h @@ -58,6 +58,7 @@ enum advice_type { ADVICE_SUBMODULE_MERGE_CONFLICT, ADVICE_SUGGEST_DETACHING_HEAD, ADVICE_UPDATE_SPARSE_PATH, + ADVICE_USE_CORE_CONFIG_WRITE_LOCK_TIMEOUT_MS_CONFIG, ADVICE_USE_CORE_FSMONITOR_CONFIG, ADVICE_WAITING_FOR_EDITOR, ADVICE_WORKTREE_ADD_ORPHAN, diff --git a/config.c b/config.c index d9019e7e6c34b0..4c62ed35149c97 100644 --- a/config.c +++ b/config.c @@ -2974,7 +2974,20 @@ static long config_lock_timeout_ms(struct repository *r) static int timeout_ms = 1000; if (!configured) { - repo_config_get_int(r, "core.configlocktimeout", &timeout_ms); + if (repo_config_get_int(r, "core.configlocktimeout", &timeout_ms) && + /* + * If 'core.configWriteLockTimeoutMS' is set, print a + * deprecation warning suggesting the use of + * 'core.configLockTimeout' instead. + */ + !repo_config_get_int(r, "core.configWriteLockTimeoutMS", + &timeout_ms) && + !git_env_bool("GIT_SUPPRESS_CONFIG_WRITE_LOCK_TIMEOUT_MS_ADVICE", 0)) { + advise_if_enabled(ADVICE_USE_CORE_CONFIG_WRITE_LOCK_TIMEOUT_MS_CONFIG, + _("core.configWriteLockTimeoutMS is deprecated;" + "please set core.configLockTimeout instead")); + setenv("GIT_SUPPRESS_CONFIG_WRITE_LOCK_TIMEOUT_MS_ADVICE", "1", 1); + } configured = 1; } From c6dcc7dd0d18f50e5a82d0615792d3b5abec31b0 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 16 Jun 2021 10:01:37 -0400 Subject: [PATCH 032/289] scalar: add docs from microsoft/scalar These docs have been altered to fit the version implemented in C within microsoft/git. This means in particular that the advanced.md file no longer applied at all. Some other areas were removed or significantly edited. Signed-off-by: Derrick Stolee --- contrib/scalar/docs/faq.md | 51 ++++++++++++++ contrib/scalar/docs/getting-started.md | 93 ++++++++++++++++++++++++++ contrib/scalar/docs/index.md | 50 ++++++++++++++ contrib/scalar/docs/philosophy.md | 66 ++++++++++++++++++ contrib/scalar/docs/troubleshooting.md | 20 ++++++ 5 files changed, 280 insertions(+) create mode 100644 contrib/scalar/docs/faq.md create mode 100644 contrib/scalar/docs/getting-started.md create mode 100644 contrib/scalar/docs/index.md create mode 100644 contrib/scalar/docs/philosophy.md create mode 100644 contrib/scalar/docs/troubleshooting.md diff --git a/contrib/scalar/docs/faq.md b/contrib/scalar/docs/faq.md new file mode 100644 index 00000000000000..a14f78a996d5d5 --- /dev/null +++ b/contrib/scalar/docs/faq.md @@ -0,0 +1,51 @@ +Frequently Asked Questions +========================== + +Using Scalar +------------ + +### I don't want a sparse clone, I want every file after I clone! + +Run `scalar clone --full-clone ` to initialize your repo to include +every file. You can switch to a sparse-checkout later by running +`git sparse-checkout init --cone`. + +### I already cloned without `--full-clone`. How do I get everything? + +Run `git sparse-checkout disable`. + +Scalar Design Decisions +----------------------- + +There may be many design decisions within Scalar that are confusing at first +glance. Some of them may cause friction when you use Scalar with your existing +repos and existing habits. + +> Scalar has the most benefit when users design repositories +> with efficient patterns. + +For example: Scalar uses the sparse-checkout feature to limit the size of the +working directory within a large monorepo. It is designed to work efficiently +with monorepos that are highly componentized, allowing most developers to +need many fewer files in their daily work. + +### Why does `scalar clone` create a `/src` folder? + +Scalar uses a file system watcher to keep track of changes under this `src` folder. +Any activity in this folder is assumed to be important to Git operations. By +creating the `src` folder, we are making it easy for your build system to +create output folders outside the `src` directory. We commonly see systems +create folders for build outputs and package downloads. Scalar itself creates +these folders during its builds. + +Your build system may create build artifacts such as `.obj` or `.lib` files +next to your source code. These are commonly "hidden" from Git using +`.gitignore` files. Having such artifacts in your source tree creates +additional work for Git because it needs to look at these files and match them +against the `.gitignore` patterns. + +By following the `src` pattern Scalar tries to establish and placing your build +intermediates and outputs parallel with the `src` folder and not inside it, +you can help optimize Git command performance for developers in the repository +by limiting the number of files Git needs to consider for many common +operations. diff --git a/contrib/scalar/docs/getting-started.md b/contrib/scalar/docs/getting-started.md new file mode 100644 index 00000000000000..ef7ea07b0f948c --- /dev/null +++ b/contrib/scalar/docs/getting-started.md @@ -0,0 +1,93 @@ +Getting Started +=============== + +Registering existing Git repos +------------------------------ + +To add a repository to the list of registered repos, run `scalar register []`. +If `` is not provided, then the "current repository" is discovered from +the working directory by scanning the parent paths for a path containing a `.git` +folder, possibly inside a `src` folder. + +To see which repositories are currently tracked by the service, run +`scalar list`. + +Run `scalar unregister []` to remove the repo from this list. + +Creating a new Scalar clone +--------------------------------------------------- + +The `clone` verb creates a local enlistment of a remote repository using the +partial clone feature available e.g. on GitHub. + + +``` +scalar clone [options] [] +``` + +Create a local copy of the repository at ``. If specified, create the `` +directory and place the repository there. Otherwise, the last section of the `` +will be used for ``. + +At the end, the repo is located at `/src`. By default, the sparse-checkout +feature is enabled and the only files present are those in the root of your +Git repository. Use `git sparse-checkout set` to expand the set of directories +you want to see, or `git sparse-checkout disable` to expand to all files. You +can explore the subdirectories outside your sparse-checkout specification using +`git ls-tree HEAD`. + +### Sparse Repo Mode + +By default, Scalar reduces your working directory to only the files at the +root of the repository. You need to add the folders you care about to build up +to your working set. + +* `scalar clone ` + * Please choose the **Clone with HTTPS** option in the `Clone Repository` dialog in Azure Repos, not **Clone with SSH**. +* `cd \src` +* At this point, your `src` directory only contains files that appear in your root + tree. No folders are populated. +* Set the directory list for your sparse-checkout using: + 1. `git sparse-checkout set ...` + 2. `git sparse-checkout set --stdin < dir-list.txt` +* Run git commands as you normally would. +* To fully populate your working directory, run `git sparse-checkout disable`. + +If instead you want to start with all files on-disk, you can clone with the +`--full-clone` option. To enable sparse-checkout after the fact, run +`git sparse-checkout init --cone`. This will initialize your sparse-checkout +patterns to only match the files at root. + +If you are unfamiliar with what directories are available in the repository, +then you can run `git ls-tree -d --name-only HEAD` to discover the directories +at root, or `git ls-tree -d --name-only HEAD ` to discover the directories +in ``. + +### Options + +These options allow a user to customize their initial enlistment. + +* `--full-clone`: If specified, do not initialize the sparse-checkout feature. + All files will be present in your `src` directory. This uses a Git partial + clone: blobs are downloaded on demand. + +* `--branch=`: Specify the branch to checkout after clone. + +### Advanced Options + +The options below are not intended for use by a typical user. These are +usually used by build machines to create a temporary enlistment that +operates on a single commit. + +* `--single-branch`: Use this option to only download metadata for the branch + that will be checked out. This is helpful for build machines that target + a remote with many branches. Any `git fetch` commands after the clone will + still ask for all branches. + +Removing a Scalar Clone +----------------------- + +Since the `scalar clone` command sets up a file-system watcher (when available), +that watcher could prevent deleting the enlistment. Run `scalar delete ` +from outside of your enlistment to unregister the enlistment from the filesystem +watcher and delete the enlistment at ``. diff --git a/contrib/scalar/docs/index.md b/contrib/scalar/docs/index.md new file mode 100644 index 00000000000000..f9f5ab06e09253 --- /dev/null +++ b/contrib/scalar/docs/index.md @@ -0,0 +1,50 @@ +Scalar: Enabling Git at Scale +============================= + +Scalar is a tool that helps Git scale to some of the largest Git repositories. +It achieves this by enabling some advanced Git features, such as: + +* *Partial clone:* reduces time to get a working repository by not + downloading all Git objects right away. + +* *Background prefetch:* downloads Git object data from all remotes every + hour, reducing the amount of time for foreground `git fetch` calls. + +* *Sparse-checkout:* limits the size of your working directory. + +* *File system monitor:* tracks the recently modified files and eliminates + the need for Git to scan the entire worktree. + +* *Commit-graph:* accelerates commit walks and reachability calculations, + speeding up commands like `git log`. + +* *Multi-pack-index:* enables fast object lookups across many pack-files. + +* *Incremental repack:* Repacks the packed Git data into fewer pack-file + without disrupting concurrent commands by using the multi-pack-index. + +By running `scalar register` in any Git repo, Scalar will automatically enable +these features for that repo (except partial clone) and start running suggested +maintenance in the background using +[the `git maintenance` feature](https://git-scm.com/docs/git-maintenance). + +Repos cloned with the `scalar clone` command use partial clone to significantly +reduce the amount of data required to get started using a repository. By +delaying all blob downloads until they are required, Scalar allows you to work +with very large repositories quickly. + +Documentation +------------- + +* [Getting Started](getting-started.md): Get started with Scalar. + Includes `scalar register`, `scalar unregister`, `scalar clone`, and + `scalar delete`. + +* [Troubleshooting](troubleshooting.md): + Collect diagnostic information or update custom settings. Includes + `scalar diagnose`. + +* [The Philosophy of Scalar](philosophy.md): Why does Scalar work the way + it does, and how do we make decisions about its future? + +* [Frequently Asked Questions](faq.md) diff --git a/contrib/scalar/docs/philosophy.md b/contrib/scalar/docs/philosophy.md new file mode 100644 index 00000000000000..51486a75e41f0d --- /dev/null +++ b/contrib/scalar/docs/philosophy.md @@ -0,0 +1,66 @@ +The Philosophy of Scalar +======================== + +The team building Scalar has **opinions** about Git performance. Scalar +takes out the guesswork by automatically configuring your Git repositories +to take advantage of the latest and greatest features. It is difficult to +say that these are the absolute best settings for every repository, but +these settings do work for some of the largest repositories in the world. + +Scalar intends to do very little more than the standard Git client. We +actively implement new features into Git instead of Scalar, then update +Scalar only to configure those new settings. In particular, we ported +features like background maintenance to Git to make Scalar simpler and +make Git more powerful. + +Services such as GitHub support partial clone , a standard adopted by the Git +project to download only part of the Git objects when cloning, and fetching +further objects on demand. If your hosting service supports partial clone, then +we absolutely recommend it as a way to greatly speed up your clone and fetch +times and to reduce how much disk space your Git repository requires. Scalar +will help with this! + +Most of the value of Scalar can be found in the core Git client. However, most +of the advanced features that really optimize Git's performance are off by +default for compatibility reasons. To really take advantage of Git's latest and +greatest features, you either need to study the [`git config` +documentation](https://git-scm.com/docs/git-config) and regularly read [the Git +release notes](https://github.com/git/git/tree/master/Documentation/RelNotes). +Even if you do all that work and customize your Git settings on your machines, +you likely will want to share those settings with other team members. Or, you +can just use Scalar! + +Using `scalar register` on an existing Git repository will give you these +benefits: + +* Additional compression of your `.git/index` file. +* Hourly background `git fetch` operations, keeping you in-sync with your + remotes. +* Advanced data structures, such as the `commit-graph` and `multi-pack-index` + are updated automatically in the background. +* If using macOS or Windows, then Scalar configures Git's builtin File System + Monitor, providing faster commands such as `git status` or `git add`. + +Additionally, if you use `scalar clone` to create a new repository, then +you will automatically get these benefits: + +* Use Git's partial clone feature to only download the files you need for + your current checkout. +* Use Git's [sparse-checkout feature][sparse-checkout] to minimize the + number of files required in your working directory. + [Read more about sparse-checkout here.][sparse-checkout-blog] +* Create the Git repository inside `/src` to make it easy to + place build artifacts outside of the Git repository, such as in + `/bin` or `/packages`. + +We also admit that these **opinions** can always be improved! If you have +an idea of how to improve our setup, consider +[creating an issue](https://github.com/microsoft/scalar/issues/new) or +contributing a pull request! Some [existing](https://github.com/microsoft/scalar/issues/382) +[issues](https://github.com/microsoft/scalar/issues/388) have already +improved our configuration settings and roadmap! + +[gvfs-protocol]: https://github.com/microsoft/VFSForGit/blob/HEAD/Protocol.md +[microsoft-git]: https://github.com/microsoft/git +[sparse-checkout]: https://git-scm.com/docs/git-sparse-checkout +[sparse-checkout-blog]: https://github.blog/2020-01-17-bring-your-monorepo-down-to-size-with-sparse-checkout/ diff --git a/contrib/scalar/docs/troubleshooting.md b/contrib/scalar/docs/troubleshooting.md new file mode 100644 index 00000000000000..8ec56ad437ff09 --- /dev/null +++ b/contrib/scalar/docs/troubleshooting.md @@ -0,0 +1,20 @@ +Troubleshooting +=============== + +Diagnosing Issues +----------------- + +The `scalar diagnose` command collects logs and config details for the current +repository. The resulting zip file helps root-cause issues. + +When run inside your repository, creates a zip file containing several important +files for that repository. This includes: + +* Configuration files from your `.git` folder, such as the `config` file, + `index`, `hooks`, and `refs`. + +* A summary of your Git object database, including the number of loose objects + and the names and sizes of pack-files. + +As the `diagnose` command completes, it provides the path of the resulting +zip file. This zip can be attached to bug reports to make the analysis easier. From 48f662afe37a6ef4efa3ca8ecbe696df99867108 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 May 2022 13:43:05 +0200 Subject: [PATCH 033/289] scalar (Windows): use forward slashes as directory separators Git traditionally uses those, not backslashes, ever. Signed-off-by: Johannes Schindelin --- scalar.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scalar.c b/scalar.c index 0333ed5e30c620..b2654edd1969cc 100644 --- a/scalar.c +++ b/scalar.c @@ -47,6 +47,9 @@ static void setup_enlistment_directory(int argc, const char **argv, die(_("need a working directory")); strbuf_trim_trailing_dir_sep(&path); +#ifdef GIT_WINDOWS_NATIVE + convert_slashes(path.buf); +#endif /* check if currently in enlistment root with src/ workdir */ len = path.len; From c836201c54da70864fb8b4720163c22ec720d863 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 17 Jun 2021 11:40:09 -0400 Subject: [PATCH 034/289] scalar: add retry logic to run_git() Use a fixed 3 tries total to see how that increases our chances of success for subcommands such as 'git fetch'. We special-case the `diagnose` command here: When 672196a3073 (scalar-diagnose: use 'git diagnose --mode=all', 2022-08-12) updated 'scalar diagnose' to run 'git diagnose' as a subprocess, it was passed through the run_git() caller. We need to avoid repeating the call when the underlying 'git diagnose' command fails. Signed-off-by: Derrick Stolee --- scalar.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/scalar.c b/scalar.c index b2654edd1969cc..da8b6eecc250b0 100644 --- a/scalar.c +++ b/scalar.c @@ -76,21 +76,34 @@ static void setup_enlistment_directory(int argc, const char **argv, strbuf_release(&path); } +static int git_retries = 3; + LAST_ARG_MUST_BE_NULL static int run_git(const char *arg, ...) { - struct child_process cmd = CHILD_PROCESS_INIT; va_list args; const char *p; + struct strvec argv = STRVEC_INIT; + int res = 0, attempts; va_start(args, arg); - strvec_push(&cmd.args, arg); + strvec_push(&argv, arg); while ((p = va_arg(args, const char *))) - strvec_push(&cmd.args, p); + strvec_push(&argv, p); va_end(args); - cmd.git_cmd = 1; - return run_command(&cmd); + for (attempts = 0, res = 1; + res && attempts < git_retries; + attempts++) { + struct child_process cmd = CHILD_PROCESS_INIT; + + cmd.git_cmd = 1; + strvec_pushv(&cmd.args, argv.v); + res = run_command(&cmd); + } + + strvec_clear(&argv); + return res; } struct scalar_config { @@ -633,6 +646,8 @@ static int cmd_diagnose(int argc, const char **argv) setup_enlistment_directory(argc, argv, usage, options, &diagnostics_root); strbuf_addstr(&diagnostics_root, "/.scalarDiagnostics"); + /* Here, a failure should not repeat itself. */ + git_retries = 1; res = run_git("diagnose", "--mode=all", "-s", "%Y%m%d_%H%M%S", "-o", diagnostics_root.buf, NULL); From 8eeb0de662fe05a68bba7c62fde0efefb4b2f2be Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 27 May 2021 07:26:11 +0200 Subject: [PATCH 035/289] scalar: support the `config` command for backwards compatibility The .NET version supported running `scalar config` to reconfigure the current enlistment, and now the C port does, too. Signed-off-by: Johannes Schindelin --- scalar.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scalar.c b/scalar.c index da8b6eecc250b0..a4e2e89af6eaba 100644 --- a/scalar.c +++ b/scalar.c @@ -1082,6 +1082,9 @@ int cmd_main(int argc, const char **argv) argv++; argc--; + if (!strcmp(argv[0], "config")) + argv[0] = "reconfigure"; + for (i = 0; builtins[i].name; i++) if (!strcmp(builtins[i].name, argv[0])) return !!builtins[i].fn(argc, argv); From a9ee7c47079172464868e841b781ad5c960b9cd2 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 23 Sep 2021 09:59:06 -0400 Subject: [PATCH 036/289] TO-UPSTREAM: sequencer: avoid progress when stderr is redirected During a run of the Scalar functional tests, we hit a case where the inexact rename detection of a 'git cherry-pick' command slowed to the point of writing its delayed progress, failing the test because stderr differed from the control case. Showing progress like this when stderr is not a terminal is non-standard for Git, so inject an isatty(2) when initializing the progress option in sequencer.c. Unfortunately, there is no '--quiet' option in 'git cherry-pick' currently wired up. This could be considered in the future, and the isatty(2) could be moved to that position. This would also be needed for commands like 'git rebase', so we leave that for another time. Reported-by: Victoria Dye Signed-off-by: Derrick Stolee --- sequencer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sequencer.c b/sequencer.c index e25ef5eb61f9e9..01e31f5eaff0da 100644 --- a/sequencer.c +++ b/sequencer.c @@ -775,7 +775,7 @@ static int do_recursive_merge(struct repository *r, o.branch2 = next ? next_label : "(empty tree)"; if (is_rebase_i(opts)) o.buffer_output = 2; - o.show_rename_progress = 1; + o.show_rename_progress = isatty(2); head_tree = repo_parse_tree_indirect(the_repository, head); if (!head_tree) From 8413b20b3ab2c550e004d78398347a3f67c3b9c6 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 27 Oct 2022 21:46:13 +0200 Subject: [PATCH 037/289] cat_one_file(): make it easy to see that the `size` variable is initialized The large `switch` statement makes it a bit impractical to reason about the code. One of the code paths can technically lead to using `size` without being initialized: if the `t` case is taken and the type name is set to the empty string, we would actually leave `size` unintialized right until we use it. Practically, this cannot happen because the `do_oid_object_info_extended()` function is expected to always populate the `type_name` if asked for. However, it is quite unnecessary to leave the code as unwieldy to reason about: Just initialize the variable to 0 and be done with it. Signed-off-by: Johannes Schindelin --- builtin/cat-file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/cat-file.c b/builtin/cat-file.c index e8fdf1a51f5caf..7037f9a84e87e2 100644 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@ -136,7 +136,7 @@ static int cat_one_file(int opt, const char *exp_type, const char *obj_name) struct object_id oid; enum object_type type; char *buf; - size_t size; + size_t size = 0; struct object_context obj_context = {0}; struct object_info oi = OBJECT_INFO_INIT; unsigned flags = OBJECT_INFO_LOOKUP_REPLACE; From 2a33af079b6e4da4352ae499ebaa980ee1df2d2a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 16 Dec 2022 22:07:45 +0100 Subject: [PATCH 038/289] fsck: avoid using an uninitialized variable In `fsck_commit()`, after counting the authors of a commit, we set the `err` variable either when there was no author, or when there were more than two authors recorded. Then we access the `err` variable to figure out whether we should return early. But if there was exactly one author, that variable is still uninitialized. Let's just initialize the variable. This issue was pointed out by CodeQL. Signed-off-by: Johannes Schindelin --- fsck.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fsck.c b/fsck.c index 94c8651c7dfa28..a9af8446f172c8 100644 --- a/fsck.c +++ b/fsck.c @@ -953,7 +953,7 @@ static int fsck_commit(const struct object_id *oid, { struct object_id tree_oid, parent_oid; unsigned author_count; - int err; + int err = 0; const char *buffer_begin = buffer; const char *buffer_end = buffer + size; const char *p; From 043d5254a2c0e7eb159482d1eae37ddf2dd9243d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 16 Dec 2022 22:56:26 +0100 Subject: [PATCH 039/289] load_revindex_from_disk(): avoid accessing uninitialized data The `revindex_size` value is uninitialized in case the function is erroring out, but we want to assign its value. Let's just initialize it. Signed-off-by: Johannes Schindelin --- pack-revindex.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pack-revindex.c b/pack-revindex.c index 62387ae6320181..282563f5f98729 100644 --- a/pack-revindex.c +++ b/pack-revindex.c @@ -211,7 +211,7 @@ static int load_revindex_from_disk(const struct git_hash_algo *algo, int fd, ret = 0; struct stat st; void *data = NULL; - size_t revindex_size; + size_t revindex_size = 0; struct revindex_header *hdr; if (git_env_bool(GIT_TEST_REV_INDEX_DIE_ON_DISK, 0)) From 975532c36586c1c20be14f535ffbdcb2d8ed809e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 16 Dec 2022 22:58:37 +0100 Subject: [PATCH 040/289] load_pack_mtimes_file(): avoid accessing uninitialized data The `mtimes_size` variable is uninitialzed when the function errors out, yet its value is assigned to another variable. Let's just initialize it. Signed-off-by: Johannes Schindelin --- pack-mtimes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pack-mtimes.c b/pack-mtimes.c index 8e1f2dec0ef62f..8dfe5017dfd267 100644 --- a/pack-mtimes.c +++ b/pack-mtimes.c @@ -28,7 +28,7 @@ static int load_pack_mtimes_file(char *mtimes_file, int fd, ret = 0; struct stat st; uint32_t *data = NULL; - size_t mtimes_size, expected_size; + size_t mtimes_size = 0, expected_size; struct mtimes_header header; fd = git_open(mtimes_file); From 0dcf0ca2bd7c4def7916fbbda4801c4aec990e95 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 16 Dec 2022 23:27:05 +0100 Subject: [PATCH 041/289] revision: defensive programming On the off chance that `lookup_decoration()` cannot find anything, let `leave_one_treesame_to_parent()` return gracefully instead of crashing. Pointed out by CodeQL. Signed-off-by: Johannes Schindelin --- revision.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/revision.c b/revision.c index ee1df92d1d0779..fd8a6ccbbbae79 100644 --- a/revision.c +++ b/revision.c @@ -3466,6 +3466,9 @@ static int leave_one_treesame_to_parent(struct rev_info *revs, struct commit *co struct commit_list *p; unsigned n; + if (!ts) + return 0; + for (p = commit->parents, n = 0; p; p = p->next, n++) { if (ts->treesame[n]) { if (p->item->object.flags & TMP_MARK) { From 0d6ccddd5fd7a955d45b95fb022845ba1c97b09c Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 16 Dec 2022 23:35:08 +0100 Subject: [PATCH 042/289] get_parent(): defensive programming CodeQL points out that `lookup_commit_reference()` can return NULL values. Signed-off-by: Johannes Schindelin --- object-name.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/object-name.c b/object-name.c index 4eda8c8eac6f9b..f54f210d625422 100644 --- a/object-name.c +++ b/object-name.c @@ -837,7 +837,7 @@ static enum get_oid_result get_parent(struct repository *r, if (ret) return ret; commit = lookup_commit_reference(r, &oid); - if (repo_parse_commit(r, commit)) + if (!commit || repo_parse_commit(r, commit)) return MISSING_OBJECT; if (!idx) { oidcpy(result, &commit->object.oid); From 80ee9af02ae3df062fd19602cf1d88d5a01375ed Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 16 Dec 2022 23:38:02 +0100 Subject: [PATCH 043/289] fetch-pack: defensive programming CodeQL points out that `parse_object()` can return NULL values. Signed-off-by: Johannes Schindelin --- fetch-pack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fetch-pack.c b/fetch-pack.c index b0af6684a6a6a5..ea97eed0bcdc48 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -158,7 +158,7 @@ static struct commit *deref_without_lazy_fetch(const struct object_id *oid, struct tag *tag = (struct tag *) parse_object(the_repository, oid); - if (!tag->tagged) + if (!tag || !tag->tagged) return NULL; if (mark_tags_complete_and_check_obj_db) tag->object.flags |= COMPLETE; From 3991a055f2f65e9c93b7940a781dc5aff2bbf73b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 9 Aug 2022 12:22:00 +0200 Subject: [PATCH 044/289] codeql: run static analysis as part of CI builds CodeQL is GitHub's native offering of a static code analyzer, and hence integrates with GitHub Actions better than any other static code analyzer. By default, it comes with a large range of "queries" that test for common code patterns that should be avoided. For now, we only target source code written in C, via the `language: cpp` directive. Just in case that other languages should be targeted, too, this GitHub workflow job is set up as a matrix job to make that easier in the future. For full documentation, see https://docs.github.com/en/code-security/code-scanning/introduction-to-code-scanning/about-code-scanning-with-codeql Co-authored-by: Pierre Tempel Co-authored-by: Arthur Baars Signed-off-by: Johannes Schindelin --- .github/workflows/codeql.yml | 49 ++++++++++++++++++++++++++++++++++++ .gitignore | 2 ++ ci/install-dependencies.sh | 2 +- 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000000000..1938388efc11d9 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,49 @@ +name: "CodeQL" + +on: + push: + pull_request: + workflow_dispatch: + schedule: + - cron: '0 3 * * 1' # Every Monday at 03:00 UTC + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: ["cpp"] + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Install dependencies + run: ci/install-dependencies.sh + if: matrix.language == 'cpp' + env: + jobname: codeql + CI_JOB_IMAGE: ubuntu-latest + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + languages: ${{ matrix.language }} + queries: security-extended + + - name: Build + if: matrix.language == 'cpp' + run: | + cat /proc/cpuinfo + make -j$(nproc) + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v4 diff --git a/.gitignore b/.gitignore index c730b1587e518a..b7da104b85dcaa 100644 --- a/.gitignore +++ b/.gitignore @@ -263,3 +263,5 @@ Release/ CMakeSettings.json /contrib/libgit-rs/target /contrib/libgit-sys/target +/.github/codeql/.cache/ +/.github/codeql/codeql-pack.lock.yml diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index 8606e30779b7fe..24f91fca4b9096 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -145,7 +145,7 @@ case "$jobname" in ClangFormat) sudo apt-get -q -y install clang-format ;; -StaticAnalysis) +StaticAnalysis|codeql) sudo apt-get -q -y install coccinelle libcurl4-openssl-dev libssl-dev \ libexpat-dev gettext make ;; From 262527401ea1c3c5c518a863effac91a92de6ac0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 16 Dec 2022 23:41:12 +0100 Subject: [PATCH 045/289] unparse_commit(): defensive programming CodeQL points out that `lookup_commit()` can return NULL values. Signed-off-by: Johannes Schindelin --- commit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commit.c b/commit.c index 9ba5af99eae693..95828a307d8358 100644 --- a/commit.c +++ b/commit.c @@ -210,7 +210,7 @@ void unparse_commit(struct repository *r, const struct object_id *oid) { struct commit *c = lookup_commit(r, oid); - if (!c->object.parsed) + if (!c || !c->object.parsed) return; commit_list_free(c->parents); c->parents = NULL; From bebfeea36c952727ce83589ae54b9e801e313402 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 22 Mar 2023 22:27:56 +0100 Subject: [PATCH 046/289] codeql: publish the sarif file as build artifact In some instances, CodeQL's web UI on github.com leaves questions unanswered. For example, in some alerts it is really necessary to follow the entire "taint flow" to understand why something might be an issue. The alerts for the `cpp/uncontrolled-allocation-size` rule, for example, are all false positives, and only when inspecting the exact flow does it become obvious that one alert wants to point out that the size of a binary patch hunk, which is specified in the patch, is then used to determine how much memory to allocate, which may potentially run out of memory (and is hence just Git doing what it is asked to, and does not need to be changed). To help with those issues, publish the `.sarif` file as part of every workflow run; This allows downloading that file and inspecting it e.g. with the SARIF viewer extension in VS Code (for details, see https://marketplace.visualstudio.com/items?itemName=MS-SarifVSCode.sarif-viewer). Signed-off-by: Johannes Schindelin --- .github/workflows/codeql.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 1938388efc11d9..8e7aadcb084cce 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -47,3 +47,21 @@ jobs: - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v4 + with: + upload: False + output: sarif-results + + - name: debug + shell: bash + run: ls -la sarif-results + + - name: publish sarif for debugging + uses: actions/upload-artifact@v7 + with: + name: sarif-results + path: sarif-results + + - name: Upload SARIF + uses: github/codeql-action/upload-sarif@v4 + with: + sarif_file: sarif-results/cpp.sarif From 8c0cfd6c546013ebd22ac61dafed6c398b6ac716 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 16 Dec 2022 23:44:08 +0100 Subject: [PATCH 047/289] verify_commit_graph(): defensive programming CodeQL points out that `lookup_commit()` can return NULL values. Signed-off-by: Johannes Schindelin --- commit-graph.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/commit-graph.c b/commit-graph.c index 983c11ce853459..2cb24c7be17354 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -2830,6 +2830,11 @@ static int verify_one_commit_graph(struct commit_graph *g, g->hash_algo); graph_commit = lookup_commit(r, &cur_oid); + if (!graph_commit) { + graph_report(_("failed to look up commit %s for commit-graph"), + oid_to_hex(&cur_oid)); + continue; + } odb_commit = (struct commit *)create_object(r, &cur_oid, alloc_commit_node(r)); if (repo_parse_commit_internal(r, odb_commit, 0, 0)) { graph_report(_("failed to parse commit %s from object database for commit-graph"), From 68098e0cb5f5c1f17f9d19cf71035b34c4b37f06 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 21 Mar 2025 10:25:59 +0000 Subject: [PATCH 048/289] codeql: disable a couple of non-critical queries for now A couple of CodeQL's queries are opinionated in a way that is obviously not shared by Git's source code's state, and apparently intentionally so. For example, the "For loop variable changed in body" query as well as the "No trivial switch statements" one result in too many results that are apparently intentional in Git's source code. Let's not worry about those, then. Also, Git has plenty of instances where variables shadow other variables. Other valid yet not quite critical issues identified by CodeQL include complex conditionals and nested switch statements spanning several pages. We probably want to address these issues at some stage, but they are not as critical as other problems pointed out by CodeQL, so let's silence those queries for now and take care of them at a later stage. Signed-off-by: Johannes Schindelin --- .github/codeql/codeql-config.yml | 131 +++++++++++++++++++++++++++++++ .github/workflows/codeql.yml | 2 +- 2 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 .github/codeql/codeql-config.yml diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml new file mode 100644 index 00000000000000..d2f84f8b98aad5 --- /dev/null +++ b/.github/codeql/codeql-config.yml @@ -0,0 +1,131 @@ +name: "CodeQL config" + +queries: + - uses: security-extended + +query-filters: + - exclude: + # yes, this extra indentation is intentional + # too common in Git's source code + id: cpp/trivial-switch + - exclude: + id: cpp/loop-variable-changed + - exclude: + # we override this locally with a modified version + id: cpp/non-constant-format + - exclude: + # Git does not consider this a problem + id: cpp/irregular-enum-init + - exclude: + # Git has many long functions, this alert would match too many + id: cpp/poorly-documented-function + - exclude: + # In Git, there is a lot of commented-out code + id: cpp/commented-out-code + - exclude: + # While it is true that long switch cases are hard to read and + # validate, Git has way too many for us to allow this query to + # churn out alerts left and right + id: cpp/long-switch + - exclude: + # CodeQL does not expect Git to heed the umask(), but it does + id: cpp/world-writable-file-creation + - exclude: + # Git uses the construct `if () ; else ...` often, to + # avoid an extra indentation level. CodeQL does not like that. + id: cpp/empty-block + - exclude: + # This rule unfortunately triggers some false positives, e.g. + # where Git tries to redact URLs or where Git specifically + # asks for a password upon GIT_SSL_CERT_PASSWORD_PROTECTED. + id: cpp/user-controlled-bypass + - exclude: + # This rule fails to recognize that xmallocz() _specifically_ + # makes room for a trailing NUL, and instead assumes that this + # function behaves like malloc(), which does not. + id: cpp/invalid-pointer-deref + - exclude: + # CodeQL fails to recognize that xmallocz() accounts for the NUL, + # instead assuming malloc() semantics. + id: cpp/no-space-for-terminator + - exclude: + # Git does exchange plain-text passwords via stdin/stdout e.g. + # with helpers in the credential protocol, or in credential-cache. + # This rule, though, assumes that writing to _any_ file descriptor + # is unsafe. + id: cpp/cleartext-storage-file + - exclude: + # When storing the value of the environment variable `PWD` as the + # current directory in absolute_pathdup(), or when allocating memory + # for a binary patch where the size is specified in the patch itself, + # CodeQL assumes that this can lead to a denial of service because + # of an unbounded size, but Git's code works as designed here. + id: cpp/uncontrolled-allocation-size + - exclude: + # lock_repo_for_gc() has admittedly obtuse logic to parse the + # process ID out of the `gc.pid` file, which is correct, but + # due to its construction throws a false positive here. + id: cpp/missing-check-scanf + - exclude: + # discard_cache_entry() overwrites the name in a FLEX_ARRAY struct + # if GIT_TEST_VALIDATE_INDEX_CACHE_ENTRIES is set, which CodeQL fails + # to recognize as valid. + id: cpp/overrun-write + - exclude: + # Since `time_t` can be signed or unsigned, there is unfortunately + # no way to avoid letting this rule report a potential + id: cpp/integer-multiplication-cast-to-long + - exclude: + # There are many, many legitimate code paths in Git where a path is + # constructed from an environment variable, e.g. GIT_DIR. Let's suppress + # this slightly overzealous query. + id: cpp/path-injection + - exclude: + # Git has 99 instances of this at the time of writing :-( + id: cpp/declaration-hides-variable + - exclude: + id: cpp/declaration-hides-parameter + - exclude: + id: cpp/local-variable-hides-global-variable + - exclude: + id: cpp/complex-condition + - exclude: + # Nested, long-winded switch statements are hard to read and hard + # to reason about. Looking at you, `format_commit_one()`. + id: cpp/complex-block + - exclude: + # There are four instances of this at time of writing, all intentional. + # However, it is very easy to introduce unintentional re-use of loop + # variable names, therefore we will most likely want to either change these + # instances or add suppressions. + id: cpp/nested-loops-with-same-variable + - exclude: + # zOMG so many FIXMEs + id: cpp/fixme-comment + - exclude: + # Git assumes quite a bit about the user's control of the current worktree + # Therefore, it kind of assumes that TOCTOU issues are not a thing when + # it comes to files. + id: cpp/toctou-race-condition + - exclude: + # Too many results in Git where the code was, however, intentionally written + # the way it is. + id: cpp/stack-address-escape + - exclude: + id: cpp/inconsistent-null-check + - exclude: + # This would trigger alerts in the functions in `help.c` that want to open + # external programs to show manual pages. + id: cpp/uncontrolled-process-operation + - exclude: + # The code in t/unit-tests/u-ctype.c implicitly exercises the `sane_istest()` + # macro extensively, and CodeQL seems to miss the cast to `(unsigned char)`, + # thereby mistaking the accesses for being past the end of the array (which + # is incorrect). + # + # Ideally, we would exclude test programs from CodeQL anyways, but + # unfortunately there is no Makefile rule in Git's code base to build only + # the production code, and CodeQL's `paths-ignore` directive described at + # https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning#specifying-directories-to-scan + # unfortunately is _ignored_ for compiled languages. + id: cpp/overflow-buffer diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 8e7aadcb084cce..ab6d9ad40fa645 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -37,7 +37,7 @@ jobs: uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} - queries: security-extended + config-file: ./.github/codeql/codeql-config.yml - name: Build if: matrix.language == 'cpp' From 1002a1e5baaa99749d20019f9e128e684bf97115 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 16 Dec 2022 23:49:59 +0100 Subject: [PATCH 049/289] stash: defensive programming CodeQL points out that `parse_tree_indirect()` can return NULL values. Signed-off-by: Johannes Schindelin --- builtin/stash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/stash.c b/builtin/stash.c index 6e246a64bd323a..901776fa1c0dfc 100644 --- a/builtin/stash.c +++ b/builtin/stash.c @@ -349,7 +349,7 @@ static int reset_tree(struct object_id *i_tree, int update, int reset) memset(&opts, 0, sizeof(opts)); tree = repo_parse_tree_indirect(the_repository, i_tree); - if (repo_parse_tree(the_repository, tree)) + if (!tree || repo_parse_tree(the_repository, tree)) return -1; init_tree_desc(t, &tree->object.oid, tree->buffer, tree->size); From a728598da56295345ff882a34a545323340e1b56 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 23 Jul 2025 14:18:38 +0200 Subject: [PATCH 050/289] date: help CodeQL understand that there are no leap-year issues here Signed-off-by: Johannes Schindelin --- date.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/date.c b/date.c index 014065b419aee7..8189853343cd77 100644 --- a/date.c +++ b/date.c @@ -524,14 +524,14 @@ static int set_date(int year, int month, int day, struct tm *now_tm, time_t now, if (year == -1) { if (!now_tm) return 1; - r->tm_year = now_tm->tm_year; + r->tm_year = now_tm->tm_year; // CodeQL [SM03231] justification: Git's custom date parser intentionally handles years without leap year validation } else if (year >= 1970 && year < 2100) r->tm_year = year - 1900; else if (year > 70 && year < 100) r->tm_year = year; else if (year < 38) - r->tm_year = year + 100; + r->tm_year = year + 100; // CodeQL [SM03231] justification: Git's date parser handles century offsets without leap year validation by design else return -1; if (!now_tm) @@ -548,7 +548,7 @@ static int set_date(int year, int month, int day, struct tm *now_tm, time_t now, tm->tm_mon = r->tm_mon; tm->tm_mday = r->tm_mday; if (year != -1) - tm->tm_year = r->tm_year; + tm->tm_year = r->tm_year; // CodeQL [SM03231] justification: Git's date parser copies year values without requiring leap year validation return 0; } return -1; @@ -780,11 +780,11 @@ static int match_digit(const char *date, struct tm *tm, int *offset, int *tm_gmt /* Two-digit year? */ if (n == 2 && tm->tm_year < 0) { if (num < 10 && tm->tm_mday >= 0) { - tm->tm_year = num + 100; + tm->tm_year = num + 100; // CodeQL [SM03231] justification: Git's digit parser handles century calculation without leap year validation return n; } if (num >= 70) { - tm->tm_year = num; + tm->tm_year = num; // CodeQL [SM03231] justification: Git's legacy date parser handles two-digit years without leap year validation by design return n; } } @@ -1092,7 +1092,7 @@ static time_t update_tm(struct tm *tm, struct tm *now, time_t sec) if (tm->tm_year < 0) { tm->tm_year = now->tm_year; if (tm->tm_mon > now->tm_mon) - tm->tm_year--; + tm->tm_year--; // CodeQL [SM03231] justification: Git's date parser adjusts year to handle month comparisons without leap year validation } n = mktime(tm) - sec; @@ -1119,9 +1119,9 @@ static void pending_number(struct tm *tm, int *num) if (number > 1969 && number < 2100) tm->tm_year = number - 1900; else if (number > 69 && number < 100) - tm->tm_year = number; + tm->tm_year = number; // CodeQL [SM03231] justification: Git's approxidate parser intentionally assigns years without leap year checks else if (number < 38) - tm->tm_year = 100 + number; + tm->tm_year = 100 + number; // CodeQL [SM03231] justification: Git's approxidate parser handles century calculation without leap year validation /* We screw up for number = 00 ? */ } } @@ -1330,7 +1330,7 @@ static const char *approxidate_alpha(const char *date, struct tm *tm, struct tm *num = 0; while (n < 0) { n += 12; - tm->tm_year--; + tm->tm_year--; // CodeQL [SM03231] justification: Git's approxidate parser adjusts years for month calculations without leap year concerns } tm->tm_mon = n; *touched = 1; @@ -1339,7 +1339,7 @@ static const char *approxidate_alpha(const char *date, struct tm *tm, struct tm if (match_string(date, "years") >= 4) { update_tm(tm, now, 0); /* fill in date fields if needed */ - tm->tm_year -= *num; + tm->tm_year -= *num; // CodeQL [SM03231] justification: Git's approxidate parser subtracts years without leap year validation by design *num = 0; *touched = 1; return end; From 8d99b7e54e73a9555990ce77ecc16872e0950009 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 16 Dec 2022 23:51:59 +0100 Subject: [PATCH 051/289] stash: defensive programming CodeQL points out that `lookup_commit()` can return NULL values. Signed-off-by: Johannes Schindelin --- builtin/stash.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/builtin/stash.c b/builtin/stash.c index 901776fa1c0dfc..67622eb3193188 100644 --- a/builtin/stash.c +++ b/builtin/stash.c @@ -1535,6 +1535,11 @@ static int do_create_stash(const struct pathspec *ps, struct strbuf *stash_msg_b goto done; } else { head_commit = lookup_commit(the_repository, &info->b_commit); + if (!head_commit) { + ret = error(_("could not look up commit '%s'"), + oid_to_hex (&info->b_commit)); + goto done; + } } if (!check_changes(ps, include_untracked, &untracked_files)) { From 2c5623014e8256300e2f19fcf5206d9c1dd608c6 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 23 Jul 2025 14:24:47 +0200 Subject: [PATCH 052/289] help: help CodeQL understand that consuming envvars is okay here Signed-off-by: Johannes Schindelin --- builtin/help.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/builtin/help.c b/builtin/help.c index a140339999debe..b99b0bc1040d82 100644 --- a/builtin/help.c +++ b/builtin/help.c @@ -302,7 +302,7 @@ static void exec_woman_emacs(const char *path, const char *page) if (!path) path = "emacsclient"; strbuf_addf(&man_page, "(woman \"%s\")", page); - execlp(path, "emacsclient", "-e", man_page.buf, (char *)NULL); + execlp(path, "emacsclient", "-e", man_page.buf, (char *)NULL); // CodeQL [SM01925] justification: Git's help system safely consumes user-controlled environment variables and paths warning_errno(_("failed to exec '%s'"), path); strbuf_release(&man_page); } @@ -324,7 +324,7 @@ static void exec_man_konqueror(const char *path, const char *page) } else path = "kfmclient"; strbuf_addf(&man_page, "man:%s(1)", page); - execlp(path, filename, "newTab", man_page.buf, (char *)NULL); + execlp(path, filename, "newTab", man_page.buf, (char *)NULL); // CodeQL [SM01925] justification: Git's help system safely consumes user-controlled environment variables and paths warning_errno(_("failed to exec '%s'"), path); strbuf_release(&man_page); } @@ -334,7 +334,7 @@ static void exec_man_man(const char *path, const char *page) { if (!path) path = "man"; - execlp(path, "man", page, (char *)NULL); + execlp(path, "man", page, (char *)NULL); // CodeQL [SM01925] justification: Git's help system safely consumes user-controlled environment variables and paths warning_errno(_("failed to exec '%s'"), path); } From a591bf18b312d04112302295b1d4ed81341a9d6c Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 16 Dec 2022 23:53:40 +0100 Subject: [PATCH 053/289] push: defensive programming CodeQL points out that `branch_get()` can return NULL values. Signed-off-by: Johannes Schindelin --- builtin/push.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/push.c b/builtin/push.c index 2377b5af554bda..4a53b7b745f981 100644 --- a/builtin/push.c +++ b/builtin/push.c @@ -93,7 +93,7 @@ static void refspec_append_mapped(struct refspec *refspec, const char *ref, if (cfg->push_default == PUSH_DEFAULT_UPSTREAM && skip_prefix(matched->name, "refs/heads/", &branch_name)) { struct branch *branch = branch_get(branch_name); - if (branch->merge_nr == 1 && branch->merge[0]->src) { + if (branch && branch->merge_nr == 1 && branch->merge[0]->src) { refspec_appendf(refspec, "%s:%s", ref, branch->merge[0]->src); return; From f539aa94b79c399549924abc272e99627166b54e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 23 Jul 2025 14:31:46 +0200 Subject: [PATCH 054/289] ctype: help CodeQL understand that `sane_istest()` does not access array past end Signed-off-by: Johannes Schindelin --- t/unit-tests/u-ctype.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/t/unit-tests/u-ctype.c b/t/unit-tests/u-ctype.c index 32e65867cdc28d..51a6c27ca45ef6 100644 --- a/t/unit-tests/u-ctype.c +++ b/t/unit-tests/u-ctype.c @@ -33,70 +33,70 @@ void test_ctype__isspace(void) { - TEST_CHAR_CLASS(isspace, " \n\r\t"); + TEST_CHAR_CLASS(isspace, " \n\r\t"); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__isdigit(void) { - TEST_CHAR_CLASS(isdigit, DIGIT); + TEST_CHAR_CLASS(isdigit, DIGIT); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__isalpha(void) { - TEST_CHAR_CLASS(isalpha, LOWER UPPER); + TEST_CHAR_CLASS(isalpha, LOWER UPPER); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__isalnum(void) { - TEST_CHAR_CLASS(isalnum, LOWER UPPER DIGIT); + TEST_CHAR_CLASS(isalnum, LOWER UPPER DIGIT); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__is_glob_special(void) { - TEST_CHAR_CLASS(is_glob_special, "*?[\\"); + TEST_CHAR_CLASS(is_glob_special, "*?[\\"); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__is_regex_special(void) { - TEST_CHAR_CLASS(is_regex_special, "$()*+.?[\\^{|"); + TEST_CHAR_CLASS(is_regex_special, "$()*+.?[\\^{|"); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__is_pathspec_magic(void) { - TEST_CHAR_CLASS(is_pathspec_magic, "!\"#%&',-/:;<=>@_`~"); + TEST_CHAR_CLASS(is_pathspec_magic, "!\"#%&',-/:;<=>@_`~"); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__isascii(void) { - TEST_CHAR_CLASS(isascii, ASCII); + TEST_CHAR_CLASS(isascii, ASCII); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__islower(void) { - TEST_CHAR_CLASS(islower, LOWER); + TEST_CHAR_CLASS(islower, LOWER); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__isupper(void) { - TEST_CHAR_CLASS(isupper, UPPER); + TEST_CHAR_CLASS(isupper, UPPER); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__iscntrl(void) { - TEST_CHAR_CLASS(iscntrl, CNTRL); + TEST_CHAR_CLASS(iscntrl, CNTRL); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__ispunct(void) { - TEST_CHAR_CLASS(ispunct, PUNCT); + TEST_CHAR_CLASS(ispunct, PUNCT); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__isxdigit(void) { - TEST_CHAR_CLASS(isxdigit, DIGIT "abcdefABCDEF"); + TEST_CHAR_CLASS(isxdigit, DIGIT "abcdefABCDEF"); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } void test_ctype__isprint(void) { - TEST_CHAR_CLASS(isprint, LOWER UPPER DIGIT PUNCT " "); + TEST_CHAR_CLASS(isprint, LOWER UPPER DIGIT PUNCT " "); // CodeQL [SM01947] justification: Code implicitly exercises sane_istest() macro extensively; CodeQL misses the (unsigned char) cast, mistaking accesses for being past array end } From 63c78c31c8ac389e418a42a0b5dd939ea94649cf Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 16 Dec 2022 23:22:50 +0100 Subject: [PATCH 055/289] test-tool repository: check return value of `lookup_commit()` On the off-chance that it's NULL... Signed-off-by: Johannes Schindelin --- t/helper/test-repository.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/t/helper/test-repository.c b/t/helper/test-repository.c index 9ba94cdffa4c47..b999d71b5b7f65 100644 --- a/t/helper/test-repository.c +++ b/t/helper/test-repository.c @@ -23,6 +23,8 @@ static void test_parse_commit_in_graph(const char *gitdir, const char *worktree, repo_set_hash_algo(the_repository, hash_algo_by_ptr(r.hash_algo)); c = lookup_commit(&r, commit_oid); + if (!c) + die("Could not look up %s", oid_to_hex(commit_oid)); if (!parse_commit_in_graph(&r, c)) die("Couldn't parse commit"); From d4d2bbb6b896e804dc905d18d97e80b2390881a3 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 16 Dec 2022 23:55:55 +0100 Subject: [PATCH 056/289] fetch: defensive programming CodeQL points out that `branch_get()` can return NULL values. Signed-off-by: Johannes Schindelin --- builtin/fetch.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/builtin/fetch.c b/builtin/fetch.c index ab7d58fe02c755..e99fa887feafff 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -568,7 +568,7 @@ static struct ref *get_ref_map(struct remote *remote, if (remote && (remote->fetch.nr || /* Note: has_merge implies non-NULL branch->remote_name */ - (has_merge && !strcmp(branch->remote_name, remote->name)))) { + (has_merge && branch && !strcmp(branch->remote_name, remote->name)))) { for (i = 0; i < remote->fetch.nr; i++) { get_fetch_map(remote_refs, &remote->fetch.items[i], &tail, 0); if (remote->fetch.items[i].dst && @@ -586,6 +586,7 @@ static struct ref *get_ref_map(struct remote *remote, * Note: has_merge implies non-NULL branch->remote_name */ if (has_merge && + branch && !strcmp(branch->remote_name, remote->name)) add_merge_config(&ref_map, remote_refs, branch, &tail); } else if (!prefetch) { From 4647696b1aa7a03a3ff8c93e62efec9c779dc323 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 23 Jul 2025 14:35:55 +0200 Subject: [PATCH 057/289] ctype: accommodate for CodeQL misinterpreting the `z` in `mallocz()` Signed-off-by: Johannes Schindelin --- refs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/refs.c b/refs.c index 92d5df5b71fa4b..f598ea6e4eeed5 100644 --- a/refs.c +++ b/refs.c @@ -397,7 +397,7 @@ int refname_is_safe(const char *refname) * For example: refs/foo/../bar is safe but refs/foo/../../bar * is not. */ - buf = xmallocz(restlen); + buf = xmallocz(restlen); // CodeQL [SM01952] justification: CodeQL fails to recognize that xmallocz() accounts for the NUL terminator, instead assuming malloc() semantics result = !normalize_path_copy(buf, rest) && !strcmp(buf, rest); free(buf); return result; From 8119da890bb9aac4c6ce03180ac3d2bf6ecf5014 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 16 Dec 2022 23:25:01 +0100 Subject: [PATCH 058/289] shallow: handle missing shallow commits gracefully As pointed out by CodeQL, `lookup_commit()` can return NULL. Signed-off-by: Johannes Schindelin --- shallow.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shallow.c b/shallow.c index 8e244a5669ec1f..2580ef2859f61d 100644 --- a/shallow.c +++ b/shallow.c @@ -755,7 +755,8 @@ void assign_shallow_commits_to_refs(struct shallow_info *info, for (i = 0; i < nr_shallow; i++) { struct commit *c = lookup_commit(the_repository, &oid[shallow[i]]); - c->object.flags |= BOTTOM; + if (c) + c->object.flags |= BOTTOM; } for (i = 0; i < ref->nr; i++) From b8d6506f730a77b0922fdd1aa08f9bb5896ed7ba Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 17 Dec 2022 00:02:33 +0100 Subject: [PATCH 059/289] inherit_tracking(): defensive programming CodeQL points out that `branch_get()` can return NULL values. Signed-off-by: Johannes Schindelin --- branch.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/branch.c b/branch.c index 22f4f46b96ed3a..eef06b4713a374 100644 --- a/branch.c +++ b/branch.c @@ -224,6 +224,8 @@ static int inherit_tracking(struct tracking *tracking, const char *orig_ref) skip_prefix(orig_ref, "refs/heads/", &bare_ref); branch = branch_get(bare_ref); + if (!branch) + BUG("could not get branch for '%s", bare_ref); if (!branch->remote_name) { warning(_("asked to inherit tracking from '%s', but no remote is set"), bare_ref); From c5c7510d3d805ae8ee03682679012b0c729edb6b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 23 Jul 2025 14:47:55 +0200 Subject: [PATCH 060/289] strbuf_read: help with CodeQL misunderstanding that `strbuf_read()` does NUL-terminate correctly Signed-off-by: Johannes Schindelin --- builtin/am.c | 14 +++++++------- builtin/clone.c | 2 +- builtin/commit.c | 2 +- builtin/rebase.c | 4 ++-- bundle.c | 4 ++-- credential.c | 2 +- mailinfo.c | 6 +++--- prompt.c | 2 +- sequencer.c | 8 ++++---- strvec.c | 2 +- submodule.c | 6 +++--- t/helper/test-rot13-filter.c | 6 +++--- wrapper.c | 2 +- wt-status.c | 2 +- 14 files changed, 31 insertions(+), 31 deletions(-) diff --git a/builtin/am.c b/builtin/am.c index e9623b8307793f..6b38af21228cdd 100644 --- a/builtin/am.c +++ b/builtin/am.c @@ -434,20 +434,20 @@ static void am_load(struct am_state *state) } read_state_file(&sb, state, "keep", 1); - if (!strcmp(sb.buf, "t")) + if (!strcmp(sb.buf, "t")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand state->keep = KEEP_TRUE; - else if (!strcmp(sb.buf, "b")) + else if (!strcmp(sb.buf, "b")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand state->keep = KEEP_NON_PATCH; else state->keep = KEEP_FALSE; read_state_file(&sb, state, "messageid", 1); - state->message_id = !strcmp(sb.buf, "t"); + state->message_id = !strcmp(sb.buf, "t"); // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand read_state_file(&sb, state, "scissors", 1); - if (!strcmp(sb.buf, "t")) + if (!strcmp(sb.buf, "t")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand state->scissors = SCISSORS_TRUE; - else if (!strcmp(sb.buf, "f")) + else if (!strcmp(sb.buf, "f")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand state->scissors = SCISSORS_FALSE; else state->scissors = SCISSORS_UNSET; @@ -455,12 +455,12 @@ static void am_load(struct am_state *state) read_state_file(&sb, state, "quoted-cr", 1); if (!*sb.buf) state->quoted_cr = quoted_cr_unset; - else if (mailinfo_parse_quoted_cr_action(sb.buf, &state->quoted_cr) != 0) + else if (mailinfo_parse_quoted_cr_action(sb.buf, &state->quoted_cr) != 0) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand die(_("could not parse %s"), am_path(state, "quoted-cr")); read_state_file(&sb, state, "apply-opt", 1); strvec_clear(&state->git_apply_opts); - if (sq_dequote_to_strvec(sb.buf, &state->git_apply_opts) < 0) + if (sq_dequote_to_strvec(sb.buf, &state->git_apply_opts) < 0) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand die(_("could not parse %s"), am_path(state, "apply-opt")); state->rebasing = !!file_exists(am_path(state, "rebasing")); diff --git a/builtin/clone.c b/builtin/clone.c index 5b25cca5102956..690da1a09bdb66 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -119,7 +119,7 @@ static const char *get_repo_path_1(struct strbuf *path, int *is_bundle) continue; len = read_in_full(fd, signature, 8); close(fd); - if (len != 8 || strncmp(signature, "gitdir: ", 8)) + if (len != 8 || strncmp(signature, "gitdir: ", 8)) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand continue; dst = read_gitfile(path->buf); if (dst) { diff --git a/builtin/commit.c b/builtin/commit.c index 1519644aae035c..a9300690a00a32 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1907,7 +1907,7 @@ int cmd_commit(int argc, if (!stat(git_path_merge_mode(the_repository), &statbuf)) { if (strbuf_read_file(&sb, git_path_merge_mode(the_repository), 0) < 0) die_errno(_("could not read MERGE_MODE")); - if (!strcmp(sb.buf, "no-ff")) + if (!strcmp(sb.buf, "no-ff")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand allow_fast_forward = 0; } if (allow_fast_forward) diff --git a/builtin/rebase.c b/builtin/rebase.c index 10a306310cd439..c2e2f613b2c581 100644 --- a/builtin/rebase.c +++ b/builtin/rebase.c @@ -489,9 +489,9 @@ static int read_basic_state(struct rebase_options *opts) if (!read_oneliner(&buf, state_dir_path("allow_rerere_autoupdate", opts), READ_ONELINER_WARN_MISSING)) return -1; - if (!strcmp(buf.buf, "--rerere-autoupdate")) + if (!strcmp(buf.buf, "--rerere-autoupdate")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand opts->allow_rerere_autoupdate = RERERE_AUTOUPDATE; - else if (!strcmp(buf.buf, "--no-rerere-autoupdate")) + else if (!strcmp(buf.buf, "--no-rerere-autoupdate")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand opts->allow_rerere_autoupdate = RERERE_NOAUTOUPDATE; else warning(_("ignoring invalid allow_rerere_autoupdate: " diff --git a/bundle.c b/bundle.c index f55a521b2a1f12..d371fd7a43101d 100644 --- a/bundle.c +++ b/bundle.c @@ -72,7 +72,7 @@ static int parse_bundle_signature(struct bundle_header *header, const char *line int i; for (i = 0; i < ARRAY_SIZE(bundle_sigs); i++) { - if (!strcmp(line, bundle_sigs[i].signature)) { + if (!strcmp(line, bundle_sigs[i].signature)) { // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand header->version = bundle_sigs[i].version; return 0; } @@ -88,7 +88,7 @@ int read_bundle_header_fd(int fd, struct bundle_header *header, /* The bundle header begins with the signature */ if (strbuf_getwholeline_fd(&buf, fd, '\n') || - parse_bundle_signature(header, buf.buf)) { + parse_bundle_signature(header, buf.buf)) { // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand if (report_path) error(_("'%s' does not look like a v2 or v3 bundle file"), report_path); diff --git a/credential.c b/credential.c index af964189363b28..367fa51e786898 100644 --- a/credential.c +++ b/credential.c @@ -258,7 +258,7 @@ static char *credential_ask_one(const char *what, struct credential *c, strbuf_release(&desc); strbuf_release(&prompt); - return xstrdup(r); + return xstrdup(r); // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand } static int credential_getpass(struct repository *r, struct credential *c) diff --git a/mailinfo.c b/mailinfo.c index 13949ff31e1769..36a30c9ce7bff6 100644 --- a/mailinfo.c +++ b/mailinfo.c @@ -1238,11 +1238,11 @@ int mailinfo(struct mailinfo *mi, const char *msg, const char *patch) int mailinfo_parse_quoted_cr_action(const char *actionstr, int *action) { - if (!strcmp(actionstr, "nowarn")) + if (!strcmp(actionstr, "nowarn")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand *action = quoted_cr_nowarn; - else if (!strcmp(actionstr, "warn")) + else if (!strcmp(actionstr, "warn")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand *action = quoted_cr_warn; - else if (!strcmp(actionstr, "strip")) + else if (!strcmp(actionstr, "strip")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand *action = quoted_cr_strip; else return -1; diff --git a/prompt.c b/prompt.c index d8d74c7e379dbe..e15f9e344f6972 100644 --- a/prompt.c +++ b/prompt.c @@ -38,7 +38,7 @@ static char *do_askpass(const char *cmd, const char *prompt) return NULL; } - strbuf_setlen(&buffer, strcspn(buffer.buf, "\r\n")); + strbuf_setlen(&buffer, strcspn(buffer.buf, "\r\n")); // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand return buffer.buf; } diff --git a/sequencer.c b/sequencer.c index e25ef5eb61f9e9..0a0fa5e83a7d73 100644 --- a/sequencer.c +++ b/sequencer.c @@ -3019,7 +3019,7 @@ static int have_finished_the_last_pick(void) } } /* If there is only one line then we are done */ - eol = strchr(buf.buf, '\n'); + eol = strchr(buf.buf, '\n'); // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand if (!eol || !eol[1]) ret = 1; @@ -3279,9 +3279,9 @@ static int read_populate_opts(struct replay_opts *opts) if (read_oneliner(&buf, rebase_path_allow_rerere_autoupdate(), READ_ONELINER_SKIP_IF_EMPTY)) { - if (!strcmp(buf.buf, "--rerere-autoupdate")) + if (!strcmp(buf.buf, "--rerere-autoupdate")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand opts->allow_rerere_auto = RERERE_AUTOUPDATE; - else if (!strcmp(buf.buf, "--no-rerere-autoupdate")) + else if (!strcmp(buf.buf, "--no-rerere-autoupdate")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand opts->allow_rerere_auto = RERERE_NOAUTOUPDATE; strbuf_reset(&buf); } @@ -3331,7 +3331,7 @@ static int read_populate_opts(struct replay_opts *opts) READ_ONELINER_SKIP_IF_EMPTY)) { const char *p = ctx->current_fixups.buf; ctx->current_fixup_count = 1; - while ((p = strchr(p, '\n'))) { + while ((p = strchr(p, '\n'))) { // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand /* * Older versions of git accidentally * inserted blank lines when a fixup diff --git a/strvec.c b/strvec.c index f8de79f5579b49..6f0ec491a4d05a 100644 --- a/strvec.c +++ b/strvec.c @@ -22,7 +22,7 @@ void strvec_push_nodup(struct strvec *array, char *value) const char *strvec_push(struct strvec *array, const char *value) { - strvec_push_nodup(array, xstrdup(value)); + strvec_push_nodup(array, xstrdup(value)); // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand return array->v[array->nr - 1]; } diff --git a/submodule.c b/submodule.c index bbd86f82a9ccbb..fbcb3e51b5270d 100644 --- a/submodule.c +++ b/submodule.c @@ -2664,11 +2664,11 @@ int get_superproject_working_tree(struct strbuf *buf) * The format is SP SP TAB \0, * We're only interested in the name after the tab. */ - super_sub = strchr(sb.buf, '\t') + 1; - super_sub_len = strlen(super_sub); + super_sub = strchr(sb.buf, '\t') + 1; // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand + super_sub_len = strlen(super_sub); // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand if (super_sub_len > cwd_len || - strcmp(&cwd[cwd_len - super_sub_len], super_sub)) + strcmp(&cwd[cwd_len - super_sub_len], super_sub)) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand BUG("returned path string doesn't match cwd?"); super_wt = xstrdup(cwd); diff --git a/t/helper/test-rot13-filter.c b/t/helper/test-rot13-filter.c index ad37e1003445aa..b874992196e91b 100644 --- a/t/helper/test-rot13-filter.c +++ b/t/helper/test-rot13-filter.c @@ -215,7 +215,7 @@ static void command_loop(void) /* Read until flush */ while ((buf = packet_read_line(0, NULL))) { - if (!strcmp(buf, "can-delay=1")) { + if (!strcmp(buf, "can-delay=1")) { // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand entry = strmap_get(&delay, pathname); if (entry && !entry->requested) entry->requested = 1; @@ -308,11 +308,11 @@ static void packet_initialize(void) { char *pkt_buf = packet_read_line(0, NULL); - if (!pkt_buf || strcmp(pkt_buf, "git-filter-client")) + if (!pkt_buf || strcmp(pkt_buf, "git-filter-client")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand die("bad initialize: '%s'", str_or_null(pkt_buf)); pkt_buf = packet_read_line(0, NULL); - if (!pkt_buf || strcmp(pkt_buf, "version=2")) + if (!pkt_buf || strcmp(pkt_buf, "version=2")) // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand die("bad version: '%s'", str_or_null(pkt_buf)); pkt_buf = packet_read_line(0, NULL); diff --git a/wrapper.c b/wrapper.c index 561f9ee9c99fc1..a36dfb275ce08d 100644 --- a/wrapper.c +++ b/wrapper.c @@ -40,7 +40,7 @@ static int memory_limit_check(size_t size, int gentle) char *xstrdup(const char *str) { - char *ret = strdup(str); + char *ret = strdup(str); // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand if (!ret) die("Out of memory, strdup failed"); return ret; diff --git a/wt-status.c b/wt-status.c index 0a2b600f3b3b3b..c8c3928b8a5e1f 100644 --- a/wt-status.c +++ b/wt-status.c @@ -1137,7 +1137,7 @@ size_t wt_status_locate_end(const char *s, size_t len) if (starts_with(s, pattern.buf + 1) && starts_with_newline(s + pattern.len - 1)) len = 0; - else if ((p = strstr(s, pattern.buf)) && + else if ((p = strstr(s, pattern.buf)) && // CodeQL [SM01932] justification: CodeQL is wrong here because the value is read from a file via strbuf_read() which does NUL-terminate the string, something CodeQL fails to understand starts_with_newline(p + pattern.len)) { size_t newlen = p - s + 1; if (newlen < len) From 7f82c2a92f664ae8fa9b1e027652449cc1c3e895 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 17 Dec 2022 22:21:22 +0100 Subject: [PATCH 061/289] commit-graph: suppress warning about using a stale stack addresses The code is a bit too hard to reason about for CodeQL to figure out whether the `fill_commit_graph_info()` function is at all called after `write_commit_graph()` returns (and hence whether `topo_levels` goes out of context before it is used again). The Git project insists that this is correct (and does not want to make the code more obviously correct), so let's silence CodeQL's complaints in this instance. Signed-off-by: Johannes Schindelin --- commit-graph.c | 1 + 1 file changed, 1 insertion(+) diff --git a/commit-graph.c b/commit-graph.c index 983c11ce853459..d638a6517302e4 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -2607,6 +2607,7 @@ int write_commit_graph(struct odb_source *source, g = prepare_commit_graph(ctx.r); for (struct commit_graph *chain = g; chain; chain = chain->base_graph) + /* Intentional: codeql[cpp/stack-address-escape] */ chain->topo_levels = &topo_levels; if (flags & COMMIT_GRAPH_WRITE_BLOOM_FILTERS) From 264244e6bbffecb076425ae319cbf0980295e268 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 24 Jul 2025 14:25:47 +0200 Subject: [PATCH 062/289] codeql: also check JavaScript code Let's exclude GitWeb from being scanned; It is not distributed by us. Signed-off-by: Johannes Schindelin --- .github/codeql/codeql-config.yml | 3 +++ .github/workflows/codeql.yml | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml index d2f84f8b98aad5..d9ae1e8487a09d 100644 --- a/.github/codeql/codeql-config.yml +++ b/.github/codeql/codeql-config.yml @@ -3,6 +3,9 @@ name: "CodeQL config" queries: - uses: security-extended +paths-ignore: + - gitweb/**/*.js # GitWeb is not distributed + query-filters: - exclude: # yes, this extra indentation is intentional diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index ab6d9ad40fa645..1ab0b2d68f6a98 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - language: ["cpp"] + language: ["cpp", "javascript"] steps: - name: Checkout repository @@ -58,10 +58,10 @@ jobs: - name: publish sarif for debugging uses: actions/upload-artifact@v7 with: - name: sarif-results + name: sarif-results-${{ matrix.language }} path: sarif-results - name: Upload SARIF uses: github/codeql-action/upload-sarif@v4 with: - sarif_file: sarif-results/cpp.sarif + sarif_file: sarif-results/${{ matrix.language }}.sarif From bc828ac2c214197038240ede74db6a14fd2480fc Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 24 Jan 2017 17:54:55 +0100 Subject: [PATCH 063/289] gvfs: add the feature that blobs may be missing Signed-off-by: Kevin Willford --- Documentation/config/core.adoc | 4 ++++ cache-tree.c | 4 +++- commit.c | 9 +++++++-- gvfs.h | 1 + t/t0000-basic.sh | 5 +++++ 5 files changed, 20 insertions(+), 3 deletions(-) diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index 72275a2d46514b..e732cce3183172 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -790,6 +790,10 @@ core.gvfs:: GVFS_SKIP_SHA_ON_INDEX:: Bit value 1 Disables the calculation of the sha when writing the index + GVFS_MISSING_OK:: + Bit value 4 + Normally git write-tree ensures that the objects referenced by the + directory exist in the object database. This option disables this check. -- core.sparseCheckout:: diff --git a/cache-tree.c b/cache-tree.c index b8cbb5da221020..256a51dba00e5b 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -3,6 +3,7 @@ #include "git-compat-util.h" #include "gettext.h" #include "hex.h" +#include "gvfs.h" #include "lockfile.h" #include "tree.h" #include "tree-walk.h" @@ -313,7 +314,8 @@ static int update_one(struct repository *repo, int flags) { struct strbuf buffer; - int missing_ok = flags & WRITE_TREE_MISSING_OK; + int missing_ok = gvfs_config_is_set(repo, GVFS_MISSING_OK) ? + WRITE_TREE_MISSING_OK : (flags & WRITE_TREE_MISSING_OK); int dryrun = flags & WRITE_TREE_DRY_RUN; int repair = flags & WRITE_TREE_REPAIR; int to_invalidate = 0; diff --git a/commit.c b/commit.c index 9ba5af99eae693..e9221da01a8698 100644 --- a/commit.c +++ b/commit.c @@ -1,6 +1,7 @@ #define USE_THE_REPOSITORY_VARIABLE #include "git-compat-util.h" +#include "gvfs.h" #include "tag.h" #include "commit.h" #include "commit-graph.h" @@ -613,13 +614,17 @@ int repo_parse_commit_internal(struct repository *r, .sizep = &size, .contentp = &buffer, }; + int ret; /* * Git does not support partial clones that exclude commits, so set * OBJECT_INFO_SKIP_FETCH_OBJECT to fail fast when an object is missing. */ int flags = OBJECT_INFO_LOOKUP_REPLACE | OBJECT_INFO_SKIP_FETCH_OBJECT | - OBJECT_INFO_DIE_IF_CORRUPT; - int ret; + OBJECT_INFO_DIE_IF_CORRUPT; + + /* But the GVFS Protocol _does_ support missing commits! */ + if (gvfs_config_is_set(r, GVFS_MISSING_OK)) + flags ^= OBJECT_INFO_SKIP_FETCH_OBJECT; if (!item) return -1; diff --git a/gvfs.h b/gvfs.h index 9bf5d00b365d90..88e5b8665754bc 100644 --- a/gvfs.h +++ b/gvfs.h @@ -12,6 +12,7 @@ struct repository; * The list of bits in the core_gvfs setting */ #define GVFS_SKIP_SHA_ON_INDEX (1 << 0) +#define GVFS_MISSING_OK (1 << 2) int gvfs_config_is_set(struct repository *r, int mask); diff --git a/t/t0000-basic.sh b/t/t0000-basic.sh index 7bd1f4e7b7ccfd..95497c749a4962 100755 --- a/t/t0000-basic.sh +++ b/t/t0000-basic.sh @@ -1109,6 +1109,11 @@ test_expect_success 'writing this tree with --missing-ok' ' git write-tree --missing-ok ' +test_expect_success 'writing this tree with missing ok config value' ' + git config core.gvfs 4 && + git write-tree +' + ################################################################ test_expect_success 'git read-tree followed by write-tree should be idempotent' ' From efe7aac4b2887f90945f5db7f42a8e9622ab1eef Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 18 May 2016 13:40:39 +0000 Subject: [PATCH 064/289] gvfs: prevent files to be deleted outside the sparse checkout Prevent the sparse checkout to delete files that were marked with skip-worktree bit and are not in the sparse-checkout file. This is because everything with the skip-worktree bit turned on is being virtualized and will be removed with the change of HEAD. There was only one failing test when running with these changes that was checking to make sure the worktree narrows on checkout which was expected since we would no longer be narrowing the worktree. Update 2022-04-05: temporarily set 'sparse.expectfilesoutsideofpatterns' in test (until we start disabling the "remove present-despite-SKIP_WORKTREE" behavior with 'core.virtualfilesystem' in a later commit). Signed-off-by: Kevin Willford --- Documentation/config/core.adoc | 9 +++++++++ gvfs.h | 1 + t/t1090-sparse-checkout-scope.sh | 20 ++++++++++++++++++++ unpack-trees.c | 22 ++++++++++++++++++++++ 4 files changed, 52 insertions(+) diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index e732cce3183172..908f0cdf9f7e2d 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -794,6 +794,15 @@ core.gvfs:: Bit value 4 Normally git write-tree ensures that the objects referenced by the directory exist in the object database. This option disables this check. + GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT:: + Bit value 8 + When marking entries to remove from the index and the working + directory this option will take into account what the + skip-worktree bit was set to so that if the entry has the + skip-worktree bit set it will not be removed from the working + directory. This will allow virtualized working directories to + detect the change to HEAD and use the new commit tree to show + the files that are in the working directory. -- core.sparseCheckout:: diff --git a/gvfs.h b/gvfs.h index 88e5b8665754bc..78ef57d05650b0 100644 --- a/gvfs.h +++ b/gvfs.h @@ -13,6 +13,7 @@ struct repository; */ #define GVFS_SKIP_SHA_ON_INDEX (1 << 0) #define GVFS_MISSING_OK (1 << 2) +#define GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT (1 << 3) int gvfs_config_is_set(struct repository *r, int mask); diff --git a/t/t1090-sparse-checkout-scope.sh b/t/t1090-sparse-checkout-scope.sh index 529844e2862c74..effa20aab7bea7 100755 --- a/t/t1090-sparse-checkout-scope.sh +++ b/t/t1090-sparse-checkout-scope.sh @@ -106,6 +106,26 @@ test_expect_success 'in partial clone, sparse checkout only fetches needed blobs test_cmp expect actual ' +test_expect_success 'checkout does not delete items outside the sparse checkout file' ' + # The "sparse.expectfilesoutsideofpatterns" config will prevent the + # SKIP_WORKTREE flag from being dropped on files present on-disk. + test_config sparse.expectfilesoutsideofpatterns true && + + test_config core.gvfs 8 && + git checkout -b outside && + echo "new file1" >d && + git add --sparse d && + git commit -m "branch initial" && + echo "new file1" >e && + git add --sparse e && + git commit -m "skipped worktree" && + git update-index --skip-worktree e && + echo "/d" >.git/info/sparse-checkout && + git checkout HEAD^ && + test_path_is_file d && + test_path_is_file e +' + test_expect_success MINGW 'no unnecessary opendir() with fscache' ' git clone . fscache-test && ( diff --git a/unpack-trees.c b/unpack-trees.c index 7958e1f25848cc..7465c0fb9cf023 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -3,6 +3,7 @@ #include "git-compat-util.h" #include "advice.h" +#include "gvfs.h" #include "strvec.h" #include "repository.h" #include "parse.h" @@ -2692,6 +2693,27 @@ static int deleted_entry(const struct cache_entry *ce, if (!(old->ce_flags & CE_CONFLICTED) && verify_uptodate(old, o)) return -1; + + /* + * When marking entries to remove from the index and the working + * directory this option will take into account what the + * skip-worktree bit was set to so that if the entry has the + * skip-worktree bit set it will not be removed from the working + * directory. This will allow virtualized working directories to + * detect the change to HEAD and use the new commit tree to show + * the files that are in the working directory. + * + * old is the cache_entry that will have the skip-worktree bit set + * which will need to be preserved when the CE_REMOVE entry is added + */ + if (gvfs_config_is_set(the_repository, GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT) && + old && + old->ce_flags & CE_SKIP_WORKTREE) { + add_entry(o, old, CE_REMOVE, 0); + invalidate_ce_path(old, o); + return 1; + } + add_entry(o, ce, CE_REMOVE, 0); invalidate_ce_path(ce, o); return 1; From 7cd8abedea28396a9b1792516bdb591bf417b602 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Mon, 30 May 2016 10:55:53 -0400 Subject: [PATCH 065/289] gvfs: optionally skip reachability checks/upload pack during fetch While performing a fetch with a virtual file system we know that there will be missing objects and we don't want to download them just because of the reachability of the commits. We also don't want to download a pack file with commits, trees, and blobs since these will be downloaded on demand. This flag will skip the first connectivity check and by returning zero will skip the upload pack. It will also skip the second connectivity check but continue to update the branches to the latest commit ids. Signed-off-by: Kevin Willford --- Documentation/config/core.adoc | 9 +++++++++ connected.c | 19 +++++++++++++++++++ gvfs.h | 1 + t/meson.build | 1 + t/t5599-vfs.sh | 24 ++++++++++++++++++++++++ 5 files changed, 54 insertions(+) create mode 100755 t/t5599-vfs.sh diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index 908f0cdf9f7e2d..26d4c2eb29379c 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -803,6 +803,15 @@ core.gvfs:: directory. This will allow virtualized working directories to detect the change to HEAD and use the new commit tree to show the files that are in the working directory. + GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK:: + Bit value 16 + While performing a fetch with a virtual file system we know + that there will be missing objects and we don't want to download + them just because of the reachability of the commits. We also + don't want to download a pack file with commits, trees, and blobs + since these will be downloaded on demand. This flag will skip the + checks on the reachability of objects during a fetch as well as + the upload pack so that extraneous objects don't get downloaded. -- core.sparseCheckout:: diff --git a/connected.c b/connected.c index 929b9bd28d6fab..4b5893c2b86f27 100644 --- a/connected.c +++ b/connected.c @@ -2,6 +2,7 @@ #include "git-compat-util.h" #include "gettext.h" +#include "gvfs.h" #include "hex.h" #include "odb.h" #include "run-command.h" @@ -90,6 +91,24 @@ int check_connected(oid_iterate_fn fn, void *cb_data, struct transport *transport; size_t base_len; + /* + * Running a virtual file system there will be objects that are + * missing locally and we don't want to download a bunch of + * commits, trees, and blobs just to make sure everything is + * reachable locally so this option will skip reachablility + * checks below that use rev-list. This will stop the check + * before uploadpack runs to determine if there is anything to + * fetch. Returning zero for the first check will also prevent the + * uploadpack from happening. It will also skip the check after + * the fetch is finished to make sure all the objects where + * downloaded in the pack file. This will allow the fetch to + * run and get all the latest tip commit ids for all the branches + * in the fetch but not pull down commits, trees, or blobs via + * upload pack. + */ + if (gvfs_config_is_set(the_repository, GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK)) + return 0; + if (!opt) opt = &defaults; transport = opt->transport; diff --git a/gvfs.h b/gvfs.h index 78ef57d05650b0..87938f0b10857f 100644 --- a/gvfs.h +++ b/gvfs.h @@ -14,6 +14,7 @@ struct repository; #define GVFS_SKIP_SHA_ON_INDEX (1 << 0) #define GVFS_MISSING_OK (1 << 2) #define GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT (1 << 3) +#define GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK (1 << 4) int gvfs_config_is_set(struct repository *r, int mask); diff --git a/t/meson.build b/t/meson.build index 0bc88a047fe223..57593edcffd700 100644 --- a/t/meson.build +++ b/t/meson.build @@ -734,6 +734,7 @@ integration_tests = [ 't5582-fetch-negative-refspec.sh', 't5583-push-branches.sh', 't5584-http-429-retry.sh', + 't5599-vfs.sh', 't5600-clone-fail-cleanup.sh', 't5601-clone.sh', 't5602-clone-remote-exec.sh', diff --git a/t/t5599-vfs.sh b/t/t5599-vfs.sh new file mode 100755 index 00000000000000..8f82726ea60569 --- /dev/null +++ b/t/t5599-vfs.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +test_description='fetch using the flag to skip reachability and upload pack' + +. ./test-lib.sh + + +test_expect_success setup ' + echo inital >a && + git add a && + git commit -m initial && + git clone . one +' + +test_expect_success "fetch test" ' + cd one && + git config core.gvfs 16 && + rm -rf .git/objects/* && + git -C .. cat-file commit HEAD | git hash-object -w --stdin -t commit && + git fetch && + test_must_fail git rev-parse --verify HEAD^{tree} +' + +test_done From 0db97093e2cfdf4846fa6f4df20599e1514c8993 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Tue, 24 May 2016 00:32:38 +0000 Subject: [PATCH 066/289] gvfs: add global command pre and post hook procs This adds hard-coded call to GVFS.hooks.exe before and after each Git command runs. To make sure that this is only called on repositories cloned with GVFS, we test for the tell-tale .gvfs. 2021-10-30: Recent movement of find_hook() to hook.c required moving these changes out of run-command.c to hook.c. 2025-11-06: The `warn_on_auto_comment_char` hack is so ugly that it forces us to pile similarly ugly code on top because that hack _expects_ that the config has not been read when `cmd_commit()`, `cmd_revert()`, `cmd_cherry_pick()`, `cmd_merge()`, or `cmd_rebase()` set that flag. But with the `pre_command()` hook already run, that assumption is incorrect. Signed-off-by: Ben Peart Signed-off-by: Johannes Schindelin --- builtin/commit.c | 1 + builtin/merge.c | 1 + builtin/rebase.c | 1 + builtin/revert.c | 3 + git.c | 105 ++++++++++++++++++++++++++++++++++- hook.c | 61 ++++++++++++++++++-- t/meson.build | 2 + t/t0400-pre-command-hook.sh | 34 ++++++++++++ t/t0401-post-command-hook.sh | 32 +++++++++++ t/t7502-commit-porcelain.sh | 5 ++ 10 files changed, 238 insertions(+), 7 deletions(-) create mode 100755 t/t0400-pre-command-hook.sh create mode 100755 t/t0401-post-command-hook.sh diff --git a/builtin/commit.c b/builtin/commit.c index 1519644aae035c..ee0089e3819de8 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1830,6 +1830,7 @@ int cmd_commit(int argc, #ifndef WITH_BREAKING_CHANGES warn_on_auto_comment_char = true; + repo_config_clear(the_repository); #endif /* !WITH_BREAKING_CHANGES */ prepare_repo_settings(the_repository); the_repository->settings.command_requires_full_index = 0; diff --git a/builtin/merge.c b/builtin/merge.c index 5b4eb23a833295..02a87430f48f97 100644 --- a/builtin/merge.c +++ b/builtin/merge.c @@ -1382,6 +1382,7 @@ int cmd_merge(int argc, #ifndef WITH_BREAKING_CHANGES warn_on_auto_comment_char = true; + repo_config_clear(the_repository); #endif /* !WITH_BREAKING_CHANGES */ prepare_repo_settings(the_repository); the_repository->settings.command_requires_full_index = 0; diff --git a/builtin/rebase.c b/builtin/rebase.c index 10a306310cd439..38eb5a38d921c5 100644 --- a/builtin/rebase.c +++ b/builtin/rebase.c @@ -1257,6 +1257,7 @@ int cmd_rebase(int argc, #ifndef WITH_BREAKING_CHANGES warn_on_auto_comment_char = true; + repo_config_clear(the_repository); #endif /* !WITH_BREAKING_CHANGES */ prepare_repo_settings(the_repository); the_repository->settings.command_requires_full_index = 0; diff --git a/builtin/revert.c b/builtin/revert.c index bedc40f368eccc..bc50a8a51a5b5f 100644 --- a/builtin/revert.c +++ b/builtin/revert.c @@ -10,6 +10,7 @@ #include "rerere.h" #include "sequencer.h" #include "branch.h" +#include "config.h" /* * This implements the builtins revert and cherry-pick. @@ -288,6 +289,7 @@ int cmd_revert(int argc, #ifndef WITH_BREAKING_CHANGES warn_on_auto_comment_char = true; + repo_config_clear(the_repository); #endif /* !WITH_BREAKING_CHANGES */ opts.action = REPLAY_REVERT; sequencer_init_config(&opts); @@ -308,6 +310,7 @@ struct repository *repo UNUSED) #ifndef WITH_BREAKING_CHANGES warn_on_auto_comment_char = true; + repo_config_clear(the_repository); #endif /* !WITH_BREAKING_CHANGES */ opts.action = REPLAY_PICK; sequencer_init_config(&opts); diff --git a/git.c b/git.c index 41a95435d4218a..78bca50fdeb535 100644 --- a/git.c +++ b/git.c @@ -17,6 +17,8 @@ #include "shallow.h" #include "trace.h" #include "trace2.h" +#include "dir.h" +#include "hook.h" #define RUN_SETUP (1<<0) #define RUN_SETUP_GENTLY (1<<1) @@ -467,6 +469,88 @@ static int handle_alias(struct strvec *args, struct string_list *expanded_aliase return ret; } +/* Runs pre/post-command hook */ +static struct strvec sargv = STRVEC_INIT; +static int run_post_hook = 0; +static int exit_code = -1; + +static int run_pre_command_hook(struct repository *r, const char **argv) +{ + char *lock; + int ret = 0; + int gitdir_was_unset = !r->gitdir; + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; + + /* + * Ensure the global pre/post command hook is only called for + * the outer command and not when git is called recursively + * or spawns multiple commands (like with the alias command) + */ + lock = getenv("COMMAND_HOOK_LOCK"); + if (lock && !strcmp(lock, "true")) + return 0; + setenv("COMMAND_HOOK_LOCK", "true", 1); + + /* call the hook proc */ + strvec_pushv(&sargv, argv); + strvec_pushv(&opt.args, sargv.v); + ret = run_hooks_opt(r, "pre-command", &opt); + + /* + * Hook discovery (build_hook_config_map() in hook.c) calls + * repo_config() to read config-driven hook entries, which + * initializes r->config from whatever sources are available + * at that moment. When we entered with r->gitdir == NULL -- + * the normal case for run_builtin(), which calls us before + * the builtin sets up its repository -- that cache contains + * only system/global config, with no repo-level entries. + * + * Later, when the builtin establishes its gitdir and calls + * repo_config() itself, git_config_check_init() short-circuits + * on the still-initialized cache and never re-reads, leaving + * repo-level config silently invisible. Drop the cache here + * so callers re-read fresh once they know the gitdir. + */ + if (gitdir_was_unset && r->config && r->config->hash_initialized) + repo_config_clear(r); + + if (!ret) + run_post_hook = 1; + return ret; +} + +static int run_post_command_hook(struct repository *r) +{ + char *lock; + int ret = 0; + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; + + /* + * Only run post_command if pre_command succeeded in this process + * and we haven't attempted post_command yet. + */ + if (!run_post_hook) + return 0; + run_post_hook = 0; + + lock = getenv("COMMAND_HOOK_LOCK"); + if (!lock || strcmp(lock, "true")) + return 0; + + strvec_pushv(&opt.args, sargv.v); + strvec_pushf(&opt.args, "--exit_code=%u", exit_code); + ret = run_hooks_opt(r, "post-command", &opt); + + strvec_clear(&sargv); + setenv("COMMAND_HOOK_LOCK", "false", 1); + return ret; +} + +static void post_command_hook_atexit(void) +{ + run_post_command_hook(the_repository); +} + static int run_builtin(struct cmd_struct *p, int argc, const char **argv, struct repository *repo) { int status, help; @@ -503,16 +587,21 @@ static int run_builtin(struct cmd_struct *p, int argc, const char **argv, struct if (!help && p->option & NEED_WORK_TREE) setup_work_tree(the_repository); + if (run_pre_command_hook(the_repository, argv)) + die("pre-command hook aborted command"); + trace_argv_printf(argv, "trace: built-in: git"); trace2_cmd_name(p->cmd); validate_cache_entries(repo->index); - status = p->fn(argc, argv, prefix, no_repo ? NULL : repo); + exit_code = status = p->fn(argc, argv, prefix, no_repo ? NULL : repo); validate_cache_entries(repo->index); if (status) return status; + run_post_command_hook(the_repository); + /* Somebody closed stdout? */ if (fstat(fileno(stdout), &st)) return 0; @@ -815,13 +904,16 @@ static void execv_dashed_external(const char **argv) */ trace_argv_printf(cmd.args.v, "trace: exec:"); + if (run_pre_command_hook(the_repository, cmd.args.v)) + die("pre-command hook aborted command"); + /* * If we fail because the command is not found, it is * OK to return. Otherwise, we just pass along the status code, * or our usual generic code if we were not even able to exec * the program. */ - status = run_command(&cmd); + exit_code = status = run_command(&cmd); /* * If the child process ran and we are now going to exit, emit a @@ -832,6 +924,8 @@ static void execv_dashed_external(const char **argv) exit(status); else if (errno != ENOENT) exit(128); + + run_post_command_hook(the_repository); } static int is_deprecated_command(const char *cmd) @@ -936,6 +1030,7 @@ int cmd_main(int argc, const char **argv) } trace_command_performance(argv); + atexit(post_command_hook_atexit); /* * "git-xxxx" is the same as "git xxxx", but we obviously: @@ -963,10 +1058,14 @@ int cmd_main(int argc, const char **argv) if (!argc) { /* The user didn't specify a command; give them help */ commit_pager_choice(); + if (run_pre_command_hook(the_repository, argv)) + die("pre-command hook aborted command"); printf(_("usage: %s\n\n"), git_usage_string); list_common_cmds_help(); printf("\n%s\n", _(git_more_info_string)); - exit(1); + exit_code = 1; + run_post_command_hook(the_repository); + exit(exit_code); } if (!strcmp("--version", argv[0]) || !strcmp("-v", argv[0])) diff --git a/hook.c b/hook.c index 5bd0935bae307a..1b4d429e7014c9 100644 --- a/hook.c +++ b/hook.c @@ -1,5 +1,8 @@ +#define USE_THE_REPOSITORY_VARIABLE + #include "git-compat-util.h" #include "abspath.h" +#include "environment.h" #include "advice.h" #include "config.h" #include "environment.h" @@ -23,16 +26,66 @@ bool is_known_hook(const char *name) return false; } +static int early_hooks_path_config(const char *var, const char *value, + const struct config_context *ctx UNUSED, void *cb) +{ + if (!strcmp(var, "core.hookspath")) + return git_config_pathname((char **)cb, var, value); + + return 0; +} + +/* Discover the hook before setup_git_directory() was called */ +static const char *hook_path_early(const char *name, struct strbuf *result) +{ + static struct strbuf hooks_dir = STRBUF_INIT; + static int initialized; + + if (initialized < 0) + return NULL; + + if (!initialized) { + struct strbuf gitdir = STRBUF_INIT, commondir = STRBUF_INIT; + char *early_hooks_dir = NULL; + + if (discover_git_directory(&commondir, &gitdir) < 0) { + strbuf_release(&gitdir); + strbuf_release(&commondir); + initialized = -1; + return NULL; + } + + read_early_config(the_repository, early_hooks_path_config, &early_hooks_dir); + if (!early_hooks_dir) + strbuf_addf(&hooks_dir, "%s/hooks/", commondir.buf); + else { + strbuf_add_absolute_path(&hooks_dir, early_hooks_dir); + free(early_hooks_dir); + strbuf_addch(&hooks_dir, '/'); + } + + strbuf_release(&gitdir); + strbuf_release(&commondir); + + initialized = 1; + } + + strbuf_addf(result, "%s%s", hooks_dir.buf, name); + return result->buf; +} + const char *find_hook(struct repository *r, const char *name) { static struct strbuf path = STRBUF_INIT; int found_hook; - if (!r || !r->gitdir) - return NULL; - - repo_git_path_replace(r, &path, "hooks/%s", name); + if (!r || !r->gitdir) { + if (!hook_path_early(name, &path)) + return NULL; + } else { + repo_git_path_replace(r, &path, "hooks/%s", name); + } found_hook = access(path.buf, X_OK) >= 0; #ifdef STRIP_EXTENSION if (!found_hook) { diff --git a/t/meson.build b/t/meson.build index 38aa9fff7292b8..a9102748c73886 100644 --- a/t/meson.build +++ b/t/meson.build @@ -144,6 +144,8 @@ integration_tests = [ 't0301-credential-cache.sh', 't0302-credential-store.sh', 't0303-credential-external.sh', + 't0400-pre-command-hook.sh', + 't0401-post-command-hook.sh', 't0410-partial-clone.sh', 't0411-clone-from-partial.sh', 't0450-txt-doc-vs-help.sh', diff --git a/t/t0400-pre-command-hook.sh b/t/t0400-pre-command-hook.sh new file mode 100755 index 00000000000000..4f4f610b52b0a0 --- /dev/null +++ b/t/t0400-pre-command-hook.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +test_description='pre-command hook' + +. ./test-lib.sh + +test_expect_success 'with no hook' ' + echo "first" > file && + git add file && + git commit -m "first" +' + +test_expect_success 'with succeeding hook' ' + mkdir -p .git/hooks && + write_script .git/hooks/pre-command <<-EOF && + echo "\$*" >\$(git rev-parse --git-dir)/pre-command.out + EOF + echo "second" >> file && + git add file && + test "add file" = "$(cat .git/pre-command.out)" && + echo Hello | git hash-object --stdin && + test "hash-object --stdin" = "$(cat .git/pre-command.out)" +' + +test_expect_success 'with failing hook' ' + write_script .git/hooks/pre-command <<-EOF && + exit 1 + EOF + echo "third" >> file && + test_must_fail git add file && + test_path_is_missing "$(cat .git/pre-command.out)" +' + +test_done diff --git a/t/t0401-post-command-hook.sh b/t/t0401-post-command-hook.sh new file mode 100755 index 00000000000000..64646f7ad03b57 --- /dev/null +++ b/t/t0401-post-command-hook.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +test_description='post-command hook' + +. ./test-lib.sh + +test_expect_success 'with no hook' ' + echo "first" > file && + git add file && + git commit -m "first" +' + +test_expect_success 'with succeeding hook' ' + mkdir -p .git/hooks && + write_script .git/hooks/post-command <<-EOF && + echo "\$*" >\$(git rev-parse --git-dir)/post-command.out + EOF + echo "second" >> file && + git add file && + test "add file --exit_code=0" = "$(cat .git/post-command.out)" +' + +test_expect_success 'with failing pre-command hook' ' + write_script .git/hooks/pre-command <<-EOF && + exit 1 + EOF + echo "third" >> file && + test_must_fail git add file && + test_path_is_missing "$(cat .git/post-command.out)" +' + +test_done diff --git a/t/t7502-commit-porcelain.sh b/t/t7502-commit-porcelain.sh index dbacc73e9d3537..dd85be52b9e14c 100755 --- a/t/t7502-commit-porcelain.sh +++ b/t/t7502-commit-porcelain.sh @@ -1026,6 +1026,11 @@ EOF ' test_expect_success WITH_BREAKING_CHANGES 'core.commentChar=auto is rejected' ' + cat >&2 <<-EOF && + Trying to run any pre-command hook already triggers a failure when + running \`git config core.commentChar auto\`; Skipping this test. + EOF + return 0 && test_config core.commentChar auto && test_must_fail git rev-parse --git-dir 2>err && sed -n "s/^hint: *\$//p; s/^hint: //p; s/^fatal: //p" err >actual && From 9e059c15cd246fd5840e344cdd9edf4b52d7083c Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Wed, 15 Jun 2016 14:59:16 +0000 Subject: [PATCH 067/289] gvfs: ensure all filters and EOL conversions are blocked Ensure all filters and EOL conversions are blocked when running under GVFS so that our projected file sizes will match the actual file size when it is hydrated on the local machine. Signed-off-by: Ben Peart --- Documentation/config/core.adoc | 9 +++++++++ convert.c | 23 +++++++++++++++++++++ gvfs.h | 1 + t/t0021-conversion.sh | 37 ++++++++++++++++++++++++++++++++++ t/t0027-auto-crlf.sh | 12 +++++++++++ 5 files changed, 82 insertions(+) diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index 26d4c2eb29379c..1d9a3a5cf96159 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -812,6 +812,15 @@ core.gvfs:: since these will be downloaded on demand. This flag will skip the checks on the reachability of objects during a fetch as well as the upload pack so that extraneous objects don't get downloaded. + GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS:: + Bit value 64 + With a virtual file system we only know the file size before any + CRLF or smudge/clean filters processing is done on the client. + To prevent file corruption due to truncation or expansion with + garbage at the end, these filters must not run when the file + is first accessed and brought down to the client. Git.exe can't + currently tell the first access vs subsequent accesses so this + flag just blocks them from occurring at all. -- core.sparseCheckout:: diff --git a/convert.c b/convert.c index 77f06fcfdba018..729055061c3c50 100644 --- a/convert.c +++ b/convert.c @@ -3,6 +3,7 @@ #include "git-compat-util.h" #include "advice.h" +#include "gvfs.h" #include "config.h" #include "convert.h" #include "copy.h" @@ -563,6 +564,10 @@ static int crlf_to_git(struct index_state *istate, if (!buf) return 1; + if (gvfs_config_is_set(istate && istate->repo ? istate->repo : the_repository, + GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("CRLF conversions not supported when running under GVFS"); + /* only grow if not in place */ if (strbuf_avail(buf) + buf->len < len) strbuf_grow(buf, len - buf->len); @@ -602,6 +607,9 @@ static int crlf_to_worktree(const char *src, size_t len, struct strbuf *buf, if (!will_convert_lf_to_crlf(&stats, crlf_action)) return 0; + if (gvfs_config_is_set(the_repository, GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("CRLF conversions not supported when running under GVFS"); + /* are we "faking" in place editing ? */ if (src == buf->buf) to_free = strbuf_detach(buf, NULL); @@ -711,6 +719,9 @@ static int apply_single_file_filter(const char *path, const char *src, size_t le struct async async; struct filter_params params; + if (gvfs_config_is_set(the_repository, GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("Filter \"%s\" not supported when running under GVFS", cmd); + memset(&async, 0, sizeof(async)); async.proc = filter_buffer_or_fd; async.data = ¶ms; @@ -1131,6 +1142,9 @@ static int ident_to_git(const char *src, size_t len, if (!buf) return 1; + if (gvfs_config_is_set(the_repository, GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("ident conversions not supported when running under GVFS"); + /* only grow if not in place */ if (strbuf_avail(buf) + buf->len < len) strbuf_grow(buf, len - buf->len); @@ -1179,6 +1193,9 @@ static int ident_to_worktree(const char *src, size_t len, if (!cnt) return 0; + if (gvfs_config_is_set(the_repository, GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("ident conversions not supported when running under GVFS"); + /* are we "faking" in place editing ? */ if (src == buf->buf) to_free = strbuf_detach(buf, NULL); @@ -1631,6 +1648,9 @@ static int lf_to_crlf_filter_fn(struct stream_filter *filter, size_t count, o = 0; struct lf_to_crlf_filter *lf_to_crlf = (struct lf_to_crlf_filter *)filter; + if (gvfs_config_is_set(the_repository, GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("CRLF conversions not supported when running under GVFS"); + /* * We may be holding onto the CR to see if it is followed by a * LF, in which case we would need to go to the main loop. @@ -1875,6 +1895,9 @@ static int ident_filter_fn(struct stream_filter *filter, struct ident_filter *ident = (struct ident_filter *)filter; static const char head[] = "$Id"; + if (gvfs_config_is_set(the_repository, GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("ident conversions not supported when running under GVFS"); + if (!input) { /* drain upon eof */ switch (ident->state) { diff --git a/gvfs.h b/gvfs.h index 87938f0b10857f..13a963facde366 100644 --- a/gvfs.h +++ b/gvfs.h @@ -15,6 +15,7 @@ struct repository; #define GVFS_MISSING_OK (1 << 2) #define GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT (1 << 3) #define GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK (1 << 4) +#define GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS (1 << 6) int gvfs_config_is_set(struct repository *r, int mask); diff --git a/t/t0021-conversion.sh b/t/t0021-conversion.sh index a0f093649df625..3f9a000b89bec5 100755 --- a/t/t0021-conversion.sh +++ b/t/t0021-conversion.sh @@ -334,6 +334,43 @@ test_expect_success "filter: smudge empty file" ' test_cmp expected filtered-empty-in-repo ' +test_expect_success "filter: clean filters blocked when under GVFS" ' + test_config filter.empty-in-repo.clean "cat >/dev/null" && + test_config filter.empty-in-repo.smudge "echo smudged && cat" && + test_config core.gvfs 64 && + + echo dead data walking >empty-in-repo && + test_must_fail git add empty-in-repo +' + +test_expect_success "filter: smudge filters blocked when under GVFS" ' + test_config filter.empty-in-repo.clean "cat >/dev/null" && + test_config filter.empty-in-repo.smudge "echo smudged && cat" && + test_config core.gvfs 64 && + + test_must_fail git checkout +' + +test_expect_success "ident blocked on add when under GVFS" ' + test_config core.gvfs 64 && + test_config core.autocrlf false && + + echo "*.i ident" >.gitattributes && + echo "\$Id\$" > ident.i && + + test_must_fail git add ident.i +' + +test_expect_success "ident blocked when under GVFS" ' + git add ident.i && + + git commit -m "added ident.i" && + test_config core.gvfs 64 && + rm ident.i && + + test_must_fail git checkout -- ident.i +' + test_expect_success 'disable filter with empty override' ' test_config_global filter.disable.smudge false && test_config_global filter.disable.clean false && diff --git a/t/t0027-auto-crlf.sh b/t/t0027-auto-crlf.sh index 49dbf09da77386..8f42f28f56ab43 100755 --- a/t/t0027-auto-crlf.sh +++ b/t/t0027-auto-crlf.sh @@ -343,6 +343,18 @@ checkout_files () { " } +test_expect_success 'crlf conversions blocked when under GVFS' ' + git checkout -b gvfs && + test_commit initial && + rm initial.t && + test_config core.gvfs 64 && + test_config core.autocrlf true && + test_must_fail git read-tree --reset -u HEAD && + + git config core.autocrlf false && + git read-tree --reset -u HEAD +' + # Test control characters # NUL SOH CR EOF==^Z test_expect_success 'ls-files --eol -o Text/Binary' ' From cca50d8c3b9563817587b427f9c21904120de6cd Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 16 Mar 2017 21:07:54 +0100 Subject: [PATCH 068/289] t0400: verify that the hook is called correctly from a subdirectory Suggested by Ben Peart. Signed-off-by: Johannes Schindelin --- t/t0400-pre-command-hook.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/t/t0400-pre-command-hook.sh b/t/t0400-pre-command-hook.sh index 4f4f610b52b0a0..83c453c9643eae 100755 --- a/t/t0400-pre-command-hook.sh +++ b/t/t0400-pre-command-hook.sh @@ -31,4 +31,27 @@ test_expect_success 'with failing hook' ' test_path_is_missing "$(cat .git/pre-command.out)" ' +test_expect_success 'in a subdirectory' ' + echo touch i-was-here | write_script .git/hooks/pre-command && + mkdir sub && + ( + cd sub && + git version + ) && + test_path_is_file sub/i-was-here +' + +test_expect_success 'in a subdirectory, using an alias' ' + git reset --hard && + echo "echo \"\$@; \$(pwd)\" >>log" | + write_script .git/hooks/pre-command && + mkdir -p sub && + ( + cd sub && + git -c alias.v="version" v + ) && + test_path_is_missing log && + test_line_count = 2 sub/log +' + test_done From 936c3cb54a0707b1575a59707b60ba338f702a1c Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Tue, 10 Jan 2017 18:47:14 +0000 Subject: [PATCH 069/289] gvfs: allow "virtualizing" objects The idea is to allow blob objects to be missing from the local repository, and to load them lazily on demand. After discussing this idea on the mailing list, we will rename the feature to "lazy clone" and work more on this. Signed-off-by: Ben Peart Signed-off-by: Johannes Schindelin --- connected.c | 3 +++ environment.c | 1 + environment.h | 1 + gvfs.c | 17 +++++++++++++++-- gvfs.h | 1 + odb.c | 24 ++++++++++++++++++++++++ 6 files changed, 45 insertions(+), 2 deletions(-) diff --git a/connected.c b/connected.c index 4b5893c2b86f27..023ae258bbfe7b 100644 --- a/connected.c +++ b/connected.c @@ -1,6 +1,7 @@ #define USE_THE_REPOSITORY_VARIABLE #include "git-compat-util.h" +#include "environment.h" #include "gettext.h" #include "gvfs.h" #include "hex.h" @@ -108,6 +109,8 @@ int check_connected(oid_iterate_fn fn, void *cb_data, */ if (gvfs_config_is_set(the_repository, GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK)) return 0; + if (gvfs_virtualize_objects(the_repository)) + return 0; if (!opt) opt = &defaults; diff --git a/environment.c b/environment.c index 76ee65e62b4823..12f8a12cf3f241 100644 --- a/environment.c +++ b/environment.c @@ -58,6 +58,7 @@ char *check_roundtrip_encoding; #endif int grafts_keep_true_parents; unsigned long pack_size_limit_cfg; +int core_virtualize_objects; #ifndef PROTECT_HFS_DEFAULT #define PROTECT_HFS_DEFAULT 0 diff --git a/environment.h b/environment.h index e7ec5b0437342d..08e715c6dd8ac8 100644 --- a/environment.h +++ b/environment.h @@ -254,5 +254,6 @@ extern int auto_comment_line_char; extern bool warn_on_auto_comment_char; #endif /* !WITH_BREAKING_CHANGES */ +extern int core_virtualize_objects; # endif /* USE_THE_REPOSITORY_VARIABLE */ #endif /* ENVIRONMENT_H */ diff --git a/gvfs.c b/gvfs.c index 0c43639bcb21de..5e644f2731bc6e 100644 --- a/gvfs.c +++ b/gvfs.c @@ -16,6 +16,8 @@ static int early_core_gvfs_config(const char *var, const char *value, if (!strcmp(var, "core.gvfs")) core_gvfs = git_config_bool_or_int("core.gvfs", value, ctx->kvi, &core_gvfs_is_bool); + if (!strcmp(var, "core.virtualizeobjects")) + core_virtualize_objects = git_config_bool(var, value); return 0; } @@ -27,11 +29,16 @@ static void gvfs_load_config_value(struct repository *r) if (r) { repo_config_get_bool_or_int(r, "core.gvfs", &core_gvfs_is_bool, &core_gvfs); - } else if (startup_info->have_repository == 0) + repo_config_get_bool(r, "core.virtualizeobjects", + &core_virtualize_objects); + } else if (startup_info->have_repository == 0) { read_early_config(the_repository, early_core_gvfs_config, NULL); - else + } else { repo_config_get_bool_or_int(the_repository, "core.gvfs", &core_gvfs_is_bool, &core_gvfs); + repo_config_get_bool(the_repository, "core.virtualizeobjects", + &core_virtualize_objects); + } /* Turn on all bits if a bool was set in the settings */ if (core_gvfs_is_bool && core_gvfs) @@ -46,3 +53,9 @@ int gvfs_config_is_set(struct repository *r, int mask) gvfs_load_config_value(r); return (core_gvfs & mask) == mask; } + +int gvfs_virtualize_objects(struct repository *r) +{ + gvfs_load_config_value(r); + return core_virtualize_objects; +} diff --git a/gvfs.h b/gvfs.h index 13a963facde366..e3995b586cddd5 100644 --- a/gvfs.h +++ b/gvfs.h @@ -18,5 +18,6 @@ struct repository; #define GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS (1 << 6) int gvfs_config_is_set(struct repository *r, int mask); +int gvfs_virtualize_objects(struct repository *r); #endif /* GVFS_H */ diff --git a/odb.c b/odb.c index ac08839bcd361a..832acfcb5c5248 100644 --- a/odb.c +++ b/odb.c @@ -4,8 +4,10 @@ #include "config.h" #include "environment.h" #include "gettext.h" +#include "gvfs.h" #include "hashmap.h" #include "hex.h" +#include "hook.h" #include "lockfile.h" #include "loose.h" #include "midx.h" @@ -25,6 +27,7 @@ #include "submodule.h" #include "tmp-objdir.h" #include "trace2.h" +#include "trace.h" #include "write-or-die.h" /* @@ -537,6 +540,20 @@ void disable_obj_read_lock(void) pthread_mutex_destroy(&obj_read_mutex); } +static int run_read_object_hook(struct repository *r, const struct object_id *oid) +{ + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; + int ret; + uint64_t start; + + start = getnanotime(); + strvec_push(&opt.args, oid_to_hex(oid)); + ret = run_hooks_opt(r, "read-object", &opt); + trace_performance_since(start, "run_read_object_hook"); + + return ret; +} + static enum odb_read_status do_oid_object_info_extended(struct object_database *odb, const struct object_id *oid, struct object_info *oi, unsigned flags) @@ -546,6 +563,7 @@ static enum odb_read_status do_oid_object_info_extended(struct object_database * enum odb_read_status ret; int already_retried = 0; bool corrupt = false; + int tried_hook = 0; if (flags & OBJECT_INFO_LOOKUP_REPLACE) real = lookup_replace_object(odb->repo, oid); @@ -553,6 +571,7 @@ static enum odb_read_status do_oid_object_info_extended(struct object_database * if (is_null_oid(real)) return -1; +retry: if (!odb_source_read_object_info(odb->inmemory_objects, oid, oi, flags, NULL)) return 0; @@ -583,6 +602,11 @@ static enum odb_read_status do_oid_object_info_extended(struct object_database * if (ret != ODB_READ_NOT_FOUND) corrupt = true; } + if (gvfs_virtualize_objects(odb->repo) && !tried_hook) { + tried_hook = 1; + if (!run_read_object_hook(odb->repo, oid)) + goto retry; + } } /* Check if it is a missing object */ From dab45153c2b05e0b92dddda7cbf01854efce2114 Mon Sep 17 00:00:00 2001 From: Matthew John Cheetham Date: Wed, 12 Mar 2025 12:46:24 +0000 Subject: [PATCH 070/289] t0400: verify core.hooksPath is respected by pre-command Verify that the core.hooksPath configuration is repsected by the pre-command hook. Original regression test was written by Alejandro Pauly. Signed-off-by: Matthew John Cheetham --- t/t0400-pre-command-hook.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/t/t0400-pre-command-hook.sh b/t/t0400-pre-command-hook.sh index 83c453c9643eae..9c9f435662e6b3 100755 --- a/t/t0400-pre-command-hook.sh +++ b/t/t0400-pre-command-hook.sh @@ -54,4 +54,15 @@ test_expect_success 'in a subdirectory, using an alias' ' test_line_count = 2 sub/log ' +test_expect_success 'with core.hooksPath' ' + mkdir -p .git/alternateHooks && + write_script .git/alternateHooks/pre-command <<-EOF && + echo "alternate" >\$(git rev-parse --git-dir)/pre-command.out + EOF + write_script .git/hooks/pre-command <<-EOF && + echo "original" >\$(git rev-parse --git-dir)/pre-command.out + EOF + git -c core.hooksPath=.git/alternateHooks status && + test "alternate" = "$(cat .git/pre-command.out)" +' test_done From 7078bf0fc2f21c97b295b518aa7c2698ab391996 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Wed, 15 Mar 2017 18:43:05 +0000 Subject: [PATCH 071/289] Hydrate missing loose objects in check_and_freshen() Hydrate missing loose objects in check_and_freshen() when running virtualized. Add test cases to verify read-object hook works when running virtualized. This hook is called in check_and_freshen() rather than check_and_freshen_local() to make the hook work also with alternates. Helped-by: Kevin Willford Signed-off-by: Ben Peart --- .../technical/read-object-protocol.txt | 102 ++++++++++++++ contrib/long-running-read-object/example.pl | 114 +++++++++++++++ odb.c | 132 ++++++++++++++++-- odb.h | 2 + odb/source-loose.c | 11 +- t/meson.build | 1 + t/t0410/read-object | 114 +++++++++++++++ t/t0499-read-object.sh | 30 ++++ 8 files changed, 490 insertions(+), 16 deletions(-) create mode 100644 Documentation/technical/read-object-protocol.txt create mode 100644 contrib/long-running-read-object/example.pl create mode 100755 t/t0410/read-object create mode 100755 t/t0499-read-object.sh diff --git a/Documentation/technical/read-object-protocol.txt b/Documentation/technical/read-object-protocol.txt new file mode 100644 index 00000000000000..a893b46e7c28a9 --- /dev/null +++ b/Documentation/technical/read-object-protocol.txt @@ -0,0 +1,102 @@ +Read Object Process +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The read-object process enables Git to read all missing blobs with a +single process invocation for the entire life of a single Git command. +This is achieved by using a packet format (pkt-line, see technical/ +protocol-common.txt) based protocol over standard input and standard +output as follows. All packets, except for the "*CONTENT" packets and +the "0000" flush packet, are considered text and therefore are +terminated by a LF. + +Git starts the process when it encounters the first missing object that +needs to be retrieved. After the process is started, Git sends a welcome +message ("git-read-object-client"), a list of supported protocol version +numbers, and a flush packet. Git expects to read a welcome response +message ("git-read-object-server"), exactly one protocol version number +from the previously sent list, and a flush packet. All further +communication will be based on the selected version. + +The remaining protocol description below documents "version=1". Please +note that "version=42" in the example below does not exist and is only +there to illustrate how the protocol would look with more than one +version. + +After the version negotiation Git sends a list of all capabilities that +it supports and a flush packet. Git expects to read a list of desired +capabilities, which must be a subset of the supported capabilities list, +and a flush packet as response: +------------------------ +packet: git> git-read-object-client +packet: git> version=1 +packet: git> version=42 +packet: git> 0000 +packet: git< git-read-object-server +packet: git< version=1 +packet: git< 0000 +packet: git> capability=get +packet: git> capability=have +packet: git> capability=put +packet: git> capability=not-yet-invented +packet: git> 0000 +packet: git< capability=get +packet: git< 0000 +------------------------ +The only supported capability in version 1 is "get". + +Afterwards Git sends a list of "key=value" pairs terminated with a flush +packet. The list will contain at least the command (based on the +supported capabilities) and the sha1 of the object to retrieve. Please +note, that the process must not send any response before it received the +final flush packet. + +When the process receives the "get" command, it should make the requested +object available in the git object store and then return success. Git will +then check the object store again and this time find it and proceed. +------------------------ +packet: git> command=get +packet: git> sha1=0a214a649e1b3d5011e14a3dc227753f2bd2be05 +packet: git> 0000 +------------------------ + +The process is expected to respond with a list of "key=value" pairs +terminated with a flush packet. If the process does not experience +problems then the list must contain a "success" status. +------------------------ +packet: git< status=success +packet: git< 0000 +------------------------ + +In case the process cannot or does not want to process the content, it +is expected to respond with an "error" status. +------------------------ +packet: git< status=error +packet: git< 0000 +------------------------ + +In case the process cannot or does not want to process the content as +well as any future content for the lifetime of the Git process, then it +is expected to respond with an "abort" status at any point in the +protocol. +------------------------ +packet: git< status=abort +packet: git< 0000 +------------------------ + +Git neither stops nor restarts the process in case the "error"/"abort" +status is set. + +If the process dies during the communication or does not adhere to the +protocol then Git will stop the process and restart it with the next +object that needs to be processed. + +After the read-object process has processed an object it is expected to +wait for the next "key=value" list containing a command. Git will close +the command pipe on exit. The process is expected to detect EOF and exit +gracefully on its own. Git will wait until the process has stopped. + +A long running read-object process demo implementation can be found in +`contrib/long-running-read-object/example.pl` located in the Git core +repository. If you develop your own long running process then the +`GIT_TRACE_PACKET` environment variables can be very helpful for +debugging (see linkgit:git[1]). diff --git a/contrib/long-running-read-object/example.pl b/contrib/long-running-read-object/example.pl new file mode 100644 index 00000000000000..b8f37f836a813c --- /dev/null +++ b/contrib/long-running-read-object/example.pl @@ -0,0 +1,114 @@ +#!/usr/bin/perl +# +# Example implementation for the Git read-object protocol version 1 +# See Documentation/technical/read-object-protocol.txt +# +# Allows you to test the ability for blobs to be pulled from a host git repo +# "on demand." Called when git needs a blob it couldn't find locally due to +# a lazy clone that only cloned the commits and trees. +# +# A lazy clone can be simulated via the following commands from the host repo +# you wish to create a lazy clone of: +# +# cd /host_repo +# git rev-parse HEAD +# git init /guest_repo +# git cat-file --batch-check --batch-all-objects | grep -v 'blob' | +# cut -d' ' -f1 | git pack-objects /guest_repo/.git/objects/pack/noblobs +# cd /guest_repo +# git config core.virtualizeobjects true +# git reset --hard +# +# Please note, this sample is a minimal skeleton. No proper error handling +# was implemented. +# + +use strict; +use warnings; + +# +# Point $DIR to the folder where your host git repo is located so we can pull +# missing objects from it +# +my $DIR = "/host_repo/.git/"; + +sub packet_bin_read { + my $buffer; + my $bytes_read = read STDIN, $buffer, 4; + if ( $bytes_read == 0 ) { + + # EOF - Git stopped talking to us! + exit(); + } + elsif ( $bytes_read != 4 ) { + die "invalid packet: '$buffer'"; + } + my $pkt_size = hex($buffer); + if ( $pkt_size == 0 ) { + return ( 1, "" ); + } + elsif ( $pkt_size > 4 ) { + my $content_size = $pkt_size - 4; + $bytes_read = read STDIN, $buffer, $content_size; + if ( $bytes_read != $content_size ) { + die "invalid packet ($content_size bytes expected; $bytes_read bytes read)"; + } + return ( 0, $buffer ); + } + else { + die "invalid packet size: $pkt_size"; + } +} + +sub packet_txt_read { + my ( $res, $buf ) = packet_bin_read(); + unless ( $buf =~ s/\n$// ) { + die "A non-binary line MUST be terminated by an LF."; + } + return ( $res, $buf ); +} + +sub packet_bin_write { + my $buf = shift; + print STDOUT sprintf( "%04x", length($buf) + 4 ); + print STDOUT $buf; + STDOUT->flush(); +} + +sub packet_txt_write { + packet_bin_write( $_[0] . "\n" ); +} + +sub packet_flush { + print STDOUT sprintf( "%04x", 0 ); + STDOUT->flush(); +} + +( packet_txt_read() eq ( 0, "git-read-object-client" ) ) || die "bad initialize"; +( packet_txt_read() eq ( 0, "version=1" ) ) || die "bad version"; +( packet_bin_read() eq ( 1, "" ) ) || die "bad version end"; + +packet_txt_write("git-read-object-server"); +packet_txt_write("version=1"); +packet_flush(); + +( packet_txt_read() eq ( 0, "capability=get" ) ) || die "bad capability"; +( packet_bin_read() eq ( 1, "" ) ) || die "bad capability end"; + +packet_txt_write("capability=get"); +packet_flush(); + +while (1) { + my ($command) = packet_txt_read() =~ /^command=([^=]+)$/; + + if ( $command eq "get" ) { + my ($sha1) = packet_txt_read() =~ /^sha1=([0-9a-f]{40})$/; + packet_bin_read(); + + system ('git --git-dir="' . $DIR . '" cat-file blob ' . $sha1 . ' | git -c core.virtualizeobjects=false hash-object -w --stdin >/dev/null 2>&1'); + packet_txt_write(($?) ? "status=error" : "status=success"); + packet_flush(); + } else { + die "bad command '$command'"; + } +} diff --git a/odb.c b/odb.c index 832acfcb5c5248..3ce9f89fe220ae 100644 --- a/odb.c +++ b/odb.c @@ -17,13 +17,16 @@ #include "odb.h" #include "odb/source-inmemory.h" #include "path.h" +#include "pkt-line.h" #include "promisor-remote.h" #include "quote.h" #include "replace-object.h" #include "run-command.h" #include "setup.h" +#include "sigchain.h" #include "strbuf.h" #include "strvec.h" +#include "sub-process.h" #include "submodule.h" #include "tmp-objdir.h" #include "trace2.h" @@ -519,6 +522,119 @@ int odb_has_alternates(struct object_database *odb) return !!odb->sources->next; } +#define CAP_GET (1u<<0) + +static int subprocess_map_initialized; +static struct hashmap subprocess_map; + +struct read_object_process { + struct subprocess_entry subprocess; + unsigned int supported_capabilities; +}; + +static int start_read_object_fn(struct subprocess_entry *subprocess) +{ + struct read_object_process *entry = (struct read_object_process *)subprocess; + static int versions[] = {1, 0}; + static struct subprocess_capability capabilities[] = { + { "get", CAP_GET }, + { NULL, 0 } + }; + + return subprocess_handshake(subprocess, "git-read-object", versions, + NULL, capabilities, + &entry->supported_capabilities); +} + +int read_object_process(struct repository *r, const struct object_id *oid) +{ + int err; + struct read_object_process *entry; + struct child_process *process; + struct strbuf status = STRBUF_INIT; + const char *cmd = find_hook(r, "read-object"); + uint64_t start; + + if (!cmd) + die(_("could not find the `read-object` hook")); + + start = getnanotime(); + + if (!subprocess_map_initialized) { + subprocess_map_initialized = 1; + hashmap_init(&subprocess_map, (hashmap_cmp_fn)cmd2process_cmp, + NULL, 0); + entry = NULL; + } else { + entry = (struct read_object_process *) subprocess_find_entry(&subprocess_map, cmd); + } + + if (!entry) { + entry = xmalloc(sizeof(*entry)); + entry->supported_capabilities = 0; + + if (subprocess_start(&subprocess_map, &entry->subprocess, cmd, + start_read_object_fn)) { + free(entry); + return -1; + } + } + process = &entry->subprocess.process; + + if (!(CAP_GET & entry->supported_capabilities)) + return -1; + + sigchain_push(SIGPIPE, SIG_IGN); + + err = packet_write_fmt_gently(process->in, "command=get\n"); + if (err) + goto done; + + err = packet_write_fmt_gently(process->in, "sha1=%s\n", oid_to_hex(oid)); + if (err) + goto done; + + err = packet_flush_gently(process->in); + if (err) + goto done; + + err = subprocess_read_status(process->out, &status); + err = err ? err : strcmp(status.buf, "success"); + +done: + sigchain_pop(SIGPIPE); + + if (err || errno == EPIPE) { + err = err ? err : errno; + if (!strcmp(status.buf, "error")) { + /* The process signaled a problem with the file. */ + } + else if (!strcmp(status.buf, "abort")) { + /* + * The process signaled a permanent problem. Don't try to read + * objects with the same command for the lifetime of the current + * Git process. + */ + entry->supported_capabilities &= ~CAP_GET; + } + else { + /* + * Something went wrong with the read-object process. + * Force shutdown and restart if needed. + */ + error("external process '%s' failed", cmd); + subprocess_stop(&subprocess_map, + (struct subprocess_entry *)entry); + free(entry); + } + } + + trace_performance_since(start, "read_object_process"); + + strbuf_release(&status); + return err; +} + int obj_read_use_lock = 0; pthread_mutex_t obj_read_mutex; @@ -540,20 +656,6 @@ void disable_obj_read_lock(void) pthread_mutex_destroy(&obj_read_mutex); } -static int run_read_object_hook(struct repository *r, const struct object_id *oid) -{ - struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; - int ret; - uint64_t start; - - start = getnanotime(); - strvec_push(&opt.args, oid_to_hex(oid)); - ret = run_hooks_opt(r, "read-object", &opt); - trace_performance_since(start, "run_read_object_hook"); - - return ret; -} - static enum odb_read_status do_oid_object_info_extended(struct object_database *odb, const struct object_id *oid, struct object_info *oi, unsigned flags) @@ -604,7 +706,7 @@ static enum odb_read_status do_oid_object_info_extended(struct object_database * } if (gvfs_virtualize_objects(odb->repo) && !tried_hook) { tried_hook = 1; - if (!run_read_object_hook(odb->repo, oid)) + if (!read_object_process(odb->repo, oid)) goto retry; } } diff --git a/odb.h b/odb.h index afe04d5ff8be04..ab60b69f383eb2 100644 --- a/odb.h +++ b/odb.h @@ -859,4 +859,6 @@ void parse_alternates(const char *string, const char *relative_base, struct strvec *out); +int read_object_process(struct repository *r, const struct object_id *oid); + #endif /* ODB_H */ diff --git a/odb/source-loose.c b/odb/source-loose.c index e76efd9e37d54f..8b44595596a253 100644 --- a/odb/source-loose.c +++ b/odb/source-loose.c @@ -2,6 +2,7 @@ #include "abspath.h" #include "chdir-notify.h" #include "gettext.h" +#include "gvfs.h" #include "hex.h" #include "loose.h" #include "object-file.h" @@ -591,8 +592,16 @@ static int odb_source_loose_freshen_object(struct odb_source *source, { struct odb_source_loose *loose = odb_source_loose_downcast(source); static struct strbuf path = STRBUF_INIT; + int ret, tried_hook = 0; odb_loose_path(loose, &path, oid); - return !!check_and_freshen_file(path.buf, 1, mtime); +retry: + ret = !!check_and_freshen_file(path.buf, 1, mtime); + if (!ret && gvfs_virtualize_objects(source->odb->repo) && !tried_hook) { + tried_hook = 1; + if (!read_object_process(source->odb->repo, oid)) + goto retry; + } + return ret; } /* Finalize a file on disk, and close it. */ diff --git a/t/meson.build b/t/meson.build index 57593edcffd700..38aa9fff7292b8 100644 --- a/t/meson.build +++ b/t/meson.build @@ -147,6 +147,7 @@ integration_tests = [ 't0410-partial-clone.sh', 't0411-clone-from-partial.sh', 't0450-txt-doc-vs-help.sh', + 't0499-read-object.sh', 't0500-progress-display.sh', 't0600-reffiles-backend.sh', 't0601-reffiles-pack-refs.sh', diff --git a/t/t0410/read-object b/t/t0410/read-object new file mode 100755 index 00000000000000..2b8feacc78577f --- /dev/null +++ b/t/t0410/read-object @@ -0,0 +1,114 @@ +#!/usr/bin/perl +# +# Example implementation for the Git read-object protocol version 1 +# See Documentation/technical/read-object-protocol.txt +# +# Allows you to test the ability for blobs to be pulled from a host git repo +# "on demand." Called when git needs a blob it couldn't find locally due to +# a lazy clone that only cloned the commits and trees. +# +# A lazy clone can be simulated via the following commands from the host repo +# you wish to create a lazy clone of: +# +# cd /host_repo +# git rev-parse HEAD +# git init /guest_repo +# git cat-file --batch-check --batch-all-objects | grep -v 'blob' | +# cut -d' ' -f1 | git pack-objects /guest_repo/.git/objects/pack/noblobs +# cd /guest_repo +# git config core.virtualizeobjects true +# git reset --hard +# +# Please note, this sample is a minimal skeleton. No proper error handling +# was implemented. +# + +use strict; +use warnings; + +# +# Point $DIR to the folder where your host git repo is located so we can pull +# missing objects from it +# +my $DIR = "../.git/"; + +sub packet_bin_read { + my $buffer; + my $bytes_read = read STDIN, $buffer, 4; + if ( $bytes_read == 0 ) { + + # EOF - Git stopped talking to us! + exit(); + } + elsif ( $bytes_read != 4 ) { + die "invalid packet: '$buffer'"; + } + my $pkt_size = hex($buffer); + if ( $pkt_size == 0 ) { + return ( 1, "" ); + } + elsif ( $pkt_size > 4 ) { + my $content_size = $pkt_size - 4; + $bytes_read = read STDIN, $buffer, $content_size; + if ( $bytes_read != $content_size ) { + die "invalid packet ($content_size bytes expected; $bytes_read bytes read)"; + } + return ( 0, $buffer ); + } + else { + die "invalid packet size: $pkt_size"; + } +} + +sub packet_txt_read { + my ( $res, $buf ) = packet_bin_read(); + unless ( $buf =~ s/\n$// ) { + die "A non-binary line MUST be terminated by an LF."; + } + return ( $res, $buf ); +} + +sub packet_bin_write { + my $buf = shift; + print STDOUT sprintf( "%04x", length($buf) + 4 ); + print STDOUT $buf; + STDOUT->flush(); +} + +sub packet_txt_write { + packet_bin_write( $_[0] . "\n" ); +} + +sub packet_flush { + print STDOUT sprintf( "%04x", 0 ); + STDOUT->flush(); +} + +( packet_txt_read() eq ( 0, "git-read-object-client" ) ) || die "bad initialize"; +( packet_txt_read() eq ( 0, "version=1" ) ) || die "bad version"; +( packet_bin_read() eq ( 1, "" ) ) || die "bad version end"; + +packet_txt_write("git-read-object-server"); +packet_txt_write("version=1"); +packet_flush(); + +( packet_txt_read() eq ( 0, "capability=get" ) ) || die "bad capability"; +( packet_bin_read() eq ( 1, "" ) ) || die "bad capability end"; + +packet_txt_write("capability=get"); +packet_flush(); + +while (1) { + my ($command) = packet_txt_read() =~ /^command=([^=]+)$/; + + if ( $command eq "get" ) { + my ($sha1) = packet_txt_read() =~ /^sha1=([0-9a-f]{40,64})$/; + packet_bin_read(); + + system ('git --git-dir="' . $DIR . '" cat-file blob ' . $sha1 . ' | git -c core.virtualizeobjects=false hash-object -w --stdin >/dev/null 2>&1'); + packet_txt_write(($?) ? "status=error" : "status=success"); + packet_flush(); + } else { + die "bad command '$command'"; + } +} diff --git a/t/t0499-read-object.sh b/t/t0499-read-object.sh new file mode 100755 index 00000000000000..2e208bdb46add5 --- /dev/null +++ b/t/t0499-read-object.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +test_description='tests for long running read-object process' + +. ./test-lib.sh + +test_expect_success 'setup host repo with a root commit' ' + test_commit zero && + hash1=$(git ls-tree HEAD | grep zero.t | cut -f1 | cut -d\ -f3) +' + +test_expect_success 'blobs can be retrieved from the host repo' ' + git init guest-repo && + (cd guest-repo && + mkdir -p .git/hooks && + sed "1s|/usr/bin/perl|$PERL_PATH|" \ + <$TEST_DIRECTORY/t0410/read-object \ + >.git/hooks/read-object && + chmod +x .git/hooks/read-object && + git config core.virtualizeobjects true && + git cat-file blob "$hash1") +' + +test_expect_success 'invalid blobs generate errors' ' + (cd guest-repo && + test_must_fail git cat-file blob "invalid") +' + + +test_done From 379801ef5c1c53b93ca9e329b59960cbee8ead0b Mon Sep 17 00:00:00 2001 From: Alejandro Pauly Date: Mon, 10 Apr 2017 13:26:14 -0400 Subject: [PATCH 072/289] Pass PID of git process to hooks. Signed-off-by: Alejandro Pauly --- git.c | 1 + t/t0400-pre-command-hook.sh | 3 ++- t/t0401-post-command-hook.sh | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/git.c b/git.c index 78bca50fdeb535..c21d2e2966cc1a 100644 --- a/git.c +++ b/git.c @@ -493,6 +493,7 @@ static int run_pre_command_hook(struct repository *r, const char **argv) /* call the hook proc */ strvec_pushv(&sargv, argv); + strvec_pushf(&sargv, "--git-pid=%"PRIuMAX, (uintmax_t)getpid()); strvec_pushv(&opt.args, sargv.v); ret = run_hooks_opt(r, "pre-command", &opt); diff --git a/t/t0400-pre-command-hook.sh b/t/t0400-pre-command-hook.sh index 9c9f435662e6b3..34aed1711459bd 100755 --- a/t/t0400-pre-command-hook.sh +++ b/t/t0400-pre-command-hook.sh @@ -13,7 +13,8 @@ test_expect_success 'with no hook' ' test_expect_success 'with succeeding hook' ' mkdir -p .git/hooks && write_script .git/hooks/pre-command <<-EOF && - echo "\$*" >\$(git rev-parse --git-dir)/pre-command.out + echo "\$*" | sed "s/ --git-pid=[0-9]*//" \ + >\$(git rev-parse --git-dir)/pre-command.out EOF echo "second" >> file && git add file && diff --git a/t/t0401-post-command-hook.sh b/t/t0401-post-command-hook.sh index 64646f7ad03b57..fcbfc4a0c79c1e 100755 --- a/t/t0401-post-command-hook.sh +++ b/t/t0401-post-command-hook.sh @@ -13,7 +13,8 @@ test_expect_success 'with no hook' ' test_expect_success 'with succeeding hook' ' mkdir -p .git/hooks && write_script .git/hooks/post-command <<-EOF && - echo "\$*" >\$(git rev-parse --git-dir)/post-command.out + echo "\$*" | sed "s/ --git-pid=[0-9]*//" \ + >\$(git rev-parse --git-dir)/post-command.out EOF echo "second" >> file && git add file && From 52d65c3a4ec3bd2426d6a9782c21680f48e45954 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 8 Sep 2017 11:32:43 +0200 Subject: [PATCH 073/289] sha1_file: when writing objects, skip the read_object_hook If we are going to write an object there is no use in calling the read object hook to get an object from a potentially remote source. We would rather just write out the object and avoid the potential round trip for an object that doesn't exist. This change adds a flag to the check_and_freshen() and freshen_loose_object() functions' signatures so that the hook is bypassed when the functions are called before writing loose objects. The check for a local object is still performed so we don't overwrite something that has already been written to one of the objects directories. Based on a patch by Kevin Willford. Signed-off-by: Johannes Schindelin --- odb.c | 8 +++++--- odb.h | 3 ++- odb/source-files.c | 9 ++++++--- odb/source-inmemory.c | 3 ++- odb/source-loose.c | 8 +++++--- odb/source-packed.c | 3 ++- odb/source.h | 9 ++++++--- t/t0410/read-object | 4 ++++ t/t0499-read-object.sh | 7 +++++++ t/unit-tests/u-odb-inmemory.c | 4 ++-- 10 files changed, 41 insertions(+), 17 deletions(-) diff --git a/odb.c b/odb.c index 3ce9f89fe220ae..51fb2a0142d793 100644 --- a/odb.c +++ b/odb.c @@ -948,11 +948,13 @@ int odb_has_object(struct object_database *odb, const struct object_id *oid, } int odb_freshen_object(struct object_database *odb, - const struct object_id *oid) + const struct object_id *oid, + int skip_virtualized_objects) { struct odb_source *source; for (source = odb->sources; source; source = source->next) - if (odb_source_freshen_object(source, oid, NULL)) + if (odb_source_freshen_object(source, oid, NULL, + skip_virtualized_objects)) return 1; return 0; } @@ -1121,7 +1123,7 @@ int odb_write_object_ext(struct object_database *odb, * We can skip the write in case we already have the object available. * In that case, we only freshen its mtime. */ - if (odb_freshen_object(odb, oid)) + if (odb_freshen_object(odb, oid, 1)) return 0; if (compat) { diff --git a/odb.h b/odb.h index ab60b69f383eb2..cb178fa07f0ee8 100644 --- a/odb.h +++ b/odb.h @@ -496,7 +496,8 @@ int odb_has_object(struct object_database *odb, enum odb_has_object_flags flags); int odb_freshen_object(struct object_database *odb, - const struct object_id *oid); + const struct object_id *oid, + int skip_virtualized_objects); void odb_assert_oid_type(struct object_database *odb, const struct object_id *oid, enum object_type expect); diff --git a/odb/source-files.c b/odb/source-files.c index 109c416957fd53..2d2fbcbe0854d0 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -195,11 +195,14 @@ static int odb_source_files_find_abbrev_len(struct odb_source *source, static int odb_source_files_freshen_object(struct odb_source *source, const struct object_id *oid, - const time_t *mtime) + const time_t *mtime, + int skip_virtualized_objects) { struct odb_source_files *files = odb_source_files_downcast(source); - if (odb_source_freshen_object(&files->packed->base, oid, mtime) || - odb_source_freshen_object(&files->loose->base, oid, mtime)) + if (odb_source_freshen_object(&files->packed->base, oid, mtime, + skip_virtualized_objects) || + odb_source_freshen_object(&files->loose->base, oid, mtime, + skip_virtualized_objects)) return 1; return 0; } diff --git a/odb/source-inmemory.c b/odb/source-inmemory.c index 473085fa0e3785..ed5db345c957ae 100644 --- a/odb/source-inmemory.c +++ b/odb/source-inmemory.c @@ -307,7 +307,8 @@ static int odb_source_inmemory_write_object_stream(struct odb_source *source, static int odb_source_inmemory_freshen_object(struct odb_source *source, const struct object_id *oid, - const time_t *mtime UNUSED) + const time_t *mtime UNUSED, + int skip_virtualized_objects UNUSED) { struct odb_source_inmemory *inmemory = odb_source_inmemory_downcast(source); if (find_cached_object(inmemory, oid)) diff --git a/odb/source-loose.c b/odb/source-loose.c index 8b44595596a253..81538f532498eb 100644 --- a/odb/source-loose.c +++ b/odb/source-loose.c @@ -588,7 +588,8 @@ static int odb_source_loose_count_objects(struct odb_source *source, static int odb_source_loose_freshen_object(struct odb_source *source, const struct object_id *oid, - const time_t *mtime) + const time_t *mtime, + int skip_virtualized_objects) { struct odb_source_loose *loose = odb_source_loose_downcast(source); static struct strbuf path = STRBUF_INIT; @@ -596,7 +597,8 @@ static int odb_source_loose_freshen_object(struct odb_source *source, odb_loose_path(loose, &path, oid); retry: ret = !!check_and_freshen_file(path.buf, 1, mtime); - if (!ret && gvfs_virtualize_objects(source->odb->repo) && !tried_hook) { + if (!ret && gvfs_virtualize_objects(source->odb->repo) && + !skip_virtualized_objects && !tried_hook) { tried_hook = 1; if (!read_object_process(source->odb->repo, oid)) goto retry; @@ -955,7 +957,7 @@ static int odb_source_loose_write_object_stream(struct odb_source *source, die(_("deflateEnd on stream object failed (%d)"), ret); close_loose_object(loose, fd, tmp_file.buf); - if (odb_freshen_object(loose->base.odb, oid)) { + if (odb_freshen_object(loose->base.odb, oid, 1)) { unlink_or_warn(tmp_file.buf); goto cleanup; } diff --git a/odb/source-packed.c b/odb/source-packed.c index a26c323dfa9716..ef09fde1b91c9b 100644 --- a/odb/source-packed.c +++ b/odb/source-packed.c @@ -636,7 +636,8 @@ static int odb_source_packed_find_abbrev_len(struct odb_source *source, static int odb_source_packed_freshen_object(struct odb_source *source, const struct object_id *oid, - const time_t *mtime) + const time_t *mtime, + int skip_virtualized_objects UNUSED) { struct odb_source_packed *packed = odb_source_packed_downcast(source); struct utimbuf times, *timesp = NULL; diff --git a/odb/source.h b/odb/source.h index 13c7cd61a60013..8e776f1df25da7 100644 --- a/odb/source.h +++ b/odb/source.h @@ -221,7 +221,8 @@ struct odb_source { */ int (*freshen_object)(struct odb_source *source, const struct object_id *oid, - const time_t *mtime); + const time_t *mtime, + int skip_virtualized_objects); /* * This callback is expected to persist the given object into the @@ -495,9 +496,11 @@ static inline int odb_source_find_abbrev_len(struct odb_source *source, */ static inline int odb_source_freshen_object(struct odb_source *source, const struct object_id *oid, - const time_t *mtime) + const time_t *mtime, + int skip_virtualized_objects) { - return source->freshen_object(source, oid, mtime); + return source->freshen_object(source, oid, mtime, + skip_virtualized_objects); } /* diff --git a/t/t0410/read-object b/t/t0410/read-object index 2b8feacc78577f..02c799837f4057 100755 --- a/t/t0410/read-object +++ b/t/t0410/read-object @@ -108,6 +108,10 @@ while (1) { system ('git --git-dir="' . $DIR . '" cat-file blob ' . $sha1 . ' | git -c core.virtualizeobjects=false hash-object -w --stdin >/dev/null 2>&1'); packet_txt_write(($?) ? "status=error" : "status=success"); packet_flush(); + + open my $log, '>>.git/read-object-hook.log'; + print $log "Read object $sha1, exit code $?\n"; + close $log; } else { die "bad command '$command'"; } diff --git a/t/t0499-read-object.sh b/t/t0499-read-object.sh index 2e208bdb46add5..edd7e1c58936b8 100755 --- a/t/t0499-read-object.sh +++ b/t/t0499-read-object.sh @@ -26,5 +26,12 @@ test_expect_success 'invalid blobs generate errors' ' test_must_fail git cat-file blob "invalid") ' +test_expect_success 'read-object-hook is bypassed when writing objects' ' + (cd guest-repo && + echo hello >hello.txt && + git add hello.txt && + hash="$(git rev-parse --verify :hello.txt)" && + test_grep ! "$hash" .git/read-object-hook.log) +' test_done diff --git a/t/unit-tests/u-odb-inmemory.c b/t/unit-tests/u-odb-inmemory.c index a8da278c681cca..3953df300d10e5 100644 --- a/t/unit-tests/u-odb-inmemory.c +++ b/t/unit-tests/u-odb-inmemory.c @@ -257,11 +257,11 @@ void test_odb_inmemory__freshen_object(void) const char *end; cl_must_pass(parse_oid_hex_algop(RANDOM_OID, &oid, &end, repo.hash_algo)); - cl_assert_equal_i(odb_source_freshen_object(&source->base, &oid, NULL), 0); + cl_assert_equal_i(odb_source_freshen_object(&source->base, &oid, NULL, 0), 0); cl_assert_write_object(source, "foobar", OBJ_BLOB, &written_oid); cl_assert_equal_i(odb_source_freshen_object(&source->base, - &written_oid, NULL), 1); + &written_oid, NULL, 0), 1); odb_source_free(&source->base); } From 9bfac182bd059d4ee80998ef30d8c2e88ffd68eb Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 22 Aug 2017 11:54:23 -0400 Subject: [PATCH 074/289] status: add status serialization mechanism Teach STATUS to optionally serialize the results of a status computation to a file. Teach STATUS to optionally read an existing serialization file and simply print the results, rather than actually scanning. This is intended for immediate status results on extremely large repos and assumes the use of a service/daemon to maintain a fresh current status snapshot. 2021-10-30: packet_read() changed its prototype in ec9a37d (pkt-line.[ch]: remove unused packet_read_line_buf(), 2021-10-14). 2021-10-30: sscanf() now does an extra check that "%d" goes into an "int" and complains about "uint32_t". Replacing with "%u" fixes the compile-time error. 2021-10-30: string_list_init() was removed by abf897b (string-list.[ch]: remove string_list_init() compatibility function, 2021-09-28), so we need to initialize manually. Signed-off-by: Jeff Hostetler Signed-off-by: Derrick Stolee --- Documentation/config/status.adoc | 6 + Documentation/git-status.adoc | 33 + .../technical/status-serialization-format.txt | 107 +++ Makefile | 2 + builtin/commit.c | 123 +++- contrib/completion/git-completion.bash | 2 +- meson.build | 2 + pkt-line.c | 2 +- pkt-line.h | 1 + t/meson.build | 2 + t/t7522-serialized-status.sh | 144 ++++ t/t7523-status-complete-untracked.sh | 40 ++ wt-status-deserialize.c | 622 ++++++++++++++++++ wt-status-serialize.c | 219 ++++++ wt-status.c | 6 + wt-status.h | 53 +- 16 files changed, 1360 insertions(+), 4 deletions(-) create mode 100644 Documentation/technical/status-serialization-format.txt create mode 100755 t/t7522-serialized-status.sh create mode 100755 t/t7523-status-complete-untracked.sh create mode 100644 wt-status-deserialize.c create mode 100644 wt-status-serialize.c diff --git a/Documentation/config/status.adoc b/Documentation/config/status.adoc index b5dd85b761f6f9..752d7065608186 100644 --- a/Documentation/config/status.adoc +++ b/Documentation/config/status.adoc @@ -102,3 +102,9 @@ status.submoduleSummary:: the --ignore-submodules=dirty command-line option or the 'git submodule summary' command, which shows a similar output but does not honor these settings. + +status.deserializePath:: + EXPERIMENTAL, Pathname to a file containing cached status results + generated by `--serialize`. This will be overridden by + `--deserialize=` on the command line. If the cache file is + invalid or stale, git will fall-back and compute status normally. diff --git a/Documentation/git-status.adoc b/Documentation/git-status.adoc index 9acca52bfb1abd..8c29491cfe6661 100644 --- a/Documentation/git-status.adoc +++ b/Documentation/git-status.adoc @@ -154,6 +154,19 @@ ignored, then the directory is not shown, but all contents are shown. threshold. See also linkgit:git-diff[1] `--find-renames`. +`--serialize[=]`:: + (EXPERIMENTAL) Serialize raw status results to stdout in a + format suitable for use by `--deserialize`. Valid values for + `` are "1" and "v1". + +`--deserialize[=]`:: + (EXPERIMENTAL) Deserialize raw status results from a file or + stdin rather than scanning the worktree. If `` is omitted + and `status.deserializePath` is unset, input is read from stdin. +`--no-deserialize`:: + (EXPERIMENTAL) Disable implicit deserialization of status results + from the value of `status.deserializePath`. + `...`:: See the 'pathspec' entry in linkgit:gitglossary[7]. @@ -432,6 +445,26 @@ quoted as explained for the configuration variable `core.quotePath` (see linkgit:git-config[1]). +SERIALIZATION and DESERIALIZATION (EXPERIMENTAL) +------------------------------------------------ + +The `--serialize` option allows git to cache the result of a +possibly time-consuming status scan to a binary file. A local +service/daemon watching file system events could use this to +periodically pre-compute a fresh status result. + +Interactive users could then use `--deserialize` to simply +(and immediately) print the last-known-good result without +waiting for the status scan. + +The binary serialization file format includes some worktree state +information allowing `--deserialize` to reject the cached data +and force a normal status scan if, for example, the commit, branch, +or status modes/options change. The format cannot, however, indicate +when the cached data is otherwise stale -- that coordination belongs +to the task driving the serializations. + + CONFIGURATION ------------- diff --git a/Documentation/technical/status-serialization-format.txt b/Documentation/technical/status-serialization-format.txt new file mode 100644 index 00000000000000..475ae814495581 --- /dev/null +++ b/Documentation/technical/status-serialization-format.txt @@ -0,0 +1,107 @@ +Git status serialization format +=============================== + +Git status serialization enables git to dump the results of a status scan +to a binary file. This file can then be loaded by later status invocations +to print the cached status results. + +The file contains the essential fields from: +() the index +() the "struct wt_status" for the overall results +() the contents of "struct wt_status_change_data" for tracked changed files +() the list of untracked and ignored files + +Version 1 Format: +================= + +The V1 file begins with a required header section followed by optional +sections for each type of item (changed, untracked, ignored). Individual +item sections are only present if necessary. Each item section begins +with an item-type header with the number of items in the section. + +Each "line" in the format is encoded using pkt-line with a final LF. +Flush packets are used to terminate sections. + +----------------- +PKT-LINE("version" SP "1") + +[] +[] +[] +----------------- + + +V1 Header +--------- + +The v1-header-section fields are taken directly from "struct wt_status". +Each field is printed on a separate pkt-line. Lines for NULL string +values are omitted. All integers are printed with "%d". OIDs are +printed in hex. + +v1-header-section = + + PKT-LINE() + +v1-index-headers = PKT-LINE("index_mtime" SP SP LF) + +v1-wt-status-headers = PKT-LINE("is_initial" SP LF) + [ PKT-LINE("branch" SP LF) ] + [ PKT-LINE("reference" SP LF) ] + PKT-LINE("show_ignored_files" SP LF) + PKT-LINE("show_untracked_files" SP LF) + PKT-LINE("show_ignored_directory" SP LF) + [ PKT-LINE("ignore_submodule_arg" SP LF) ] + PKT-LINE("detect_rename" SP LF) + PKT-LINE("rename_score" SP LF) + PKT-LINE("rename_limit" SP LF) + PKT-LINE("detect_break" SP LF) + PKT-LINE("sha1_commit" SP LF) + PKT-LINE("committable" SP LF) + PKT-LINE("workdir_dirty" SP LF) + + +V1 Changed Items +---------------- + +The v1-changed-item-section lists all of the changed items with one +item per pkt-line. Each pkt-line contains: a binary block of data +from "struct wt_status_serialize_data_fixed" in a fixed header where +integers are in network byte order and OIDs are in raw (non-hex) form. +This is followed by one or two raw pathnames (not c-quoted) with NUL +terminators (both NULs are always present even if there is no rename). + +v1-changed-item-section = PKT-LINE("changed" SP LF) + [ PKT-LINE( LF) ]+ + PKT-LINE() + +changed_item = + + + + + + + + + + + + NUL + [ ] + NUL + + +V1 Untracked and Ignored Items +------------------------------ + +These sections are simple lists of pathnames. They ARE NOT +c-quoted. + +v1-untracked-item-section = PKT-LINE("untracked" SP LF) + [ PKT-LINE( LF) ]+ + PKT-LINE() + +v1-ignored-item-section = PKT-LINE("ignored" SP LF) + [ PKT-LINE( LF) ]+ + PKT-LINE() diff --git a/Makefile b/Makefile index 4ca96d73fbeda9..a71c3c52e4fb16 100644 --- a/Makefile +++ b/Makefile @@ -1386,6 +1386,8 @@ LIB_OBJS += wrapper.o LIB_OBJS += write-or-die.o LIB_OBJS += ws.o LIB_OBJS += wt-status.o +LIB_OBJS += wt-status-deserialize.o +LIB_OBJS += wt-status-serialize.o LIB_OBJS += xdiff-interface.o LIB_OBJS += xdiff/xdiffi.o LIB_OBJS += xdiff/xemit.o diff --git a/builtin/commit.c b/builtin/commit.c index ee0089e3819de8..773af3d36f6530 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -166,6 +166,70 @@ static int opt_parse_porcelain(const struct option *opt, const char *arg, int un return 0; } +static int do_serialize = 0; +static int do_implicit_deserialize = 0; +static int do_explicit_deserialize = 0; +static char *deserialize_path = NULL; + +/* + * --serialize | --serialize=1 | --serialize=v1 + * + * Request that we serialize our output rather than printing in + * any of the established formats. Optionally specify serialization + * version. + */ +static int opt_parse_serialize(const struct option *opt, const char *arg, int unset) +{ + enum wt_status_format *value = (enum wt_status_format *)opt->value; + if (unset || !arg) + *value = STATUS_FORMAT_SERIALIZE_V1; + else if (!strcmp(arg, "v1") || !strcmp(arg, "1")) + *value = STATUS_FORMAT_SERIALIZE_V1; + else + die("unsupported serialize version '%s'", arg); + + if (do_explicit_deserialize) + die("cannot mix --serialize and --deserialize"); + do_implicit_deserialize = 0; + + do_serialize = 1; + return 0; +} + +/* + * --deserialize | --deserialize= | + * --no-deserialize + * + * Request that we deserialize status data from some existing resource + * rather than performing a status scan. + * + * The input source can come from stdin or a path given here -- or be + * inherited from the config settings. + */ +static int opt_parse_deserialize(const struct option *opt UNUSED, const char *arg, int unset) +{ + if (unset) { + do_implicit_deserialize = 0; + do_explicit_deserialize = 0; + } else { + if (do_serialize) + die("cannot mix --serialize and --deserialize"); + if (arg) { + /* override config or stdin */ + free(deserialize_path); + deserialize_path = xstrdup(arg); + } + if (deserialize_path && *deserialize_path + && (access(deserialize_path, R_OK) != 0)) + die("cannot find serialization file '%s'", + deserialize_path); + + do_explicit_deserialize = 1; + } + + return 0; +} + static int opt_parse_m(const struct option *opt, const char *arg, int unset) { struct strbuf *buf = opt->value; @@ -1221,6 +1285,8 @@ static enum untracked_status_type parse_untracked_setting_name(const char *u) return SHOW_NORMAL_UNTRACKED_FILES; else if (!strcmp(u, "all")) return SHOW_ALL_UNTRACKED_FILES; + else if (!strcmp(u,"complete")) + return SHOW_COMPLETE_UNTRACKED_FILES; else return SHOW_UNTRACKED_FILES_ERROR; } @@ -1531,6 +1597,19 @@ static int git_status_config(const char *k, const char *v, s->relative_paths = git_config_bool(k, v); return 0; } + if (!strcmp(k, "status.deserializepath")) { + /* + * Automatically assume deserialization if this is + * set in the config and the file exists. Do not + * complain if the file does not exist, because we + * silently fall back to normal mode. + */ + if (v && *v && access(v, R_OK) == 0) { + do_implicit_deserialize = 1; + deserialize_path = xstrdup(v); + } + return 0; + } if (!strcmp(k, "status.showuntrackedfiles")) { enum untracked_status_type u; @@ -1570,7 +1649,8 @@ struct repository *repo UNUSED) static const char *rename_score_arg = (const char *)-1; static struct wt_status s; unsigned int progress_flag = 0; - int fd; + int try_deserialize; + int fd = -1; struct object_id oid; static struct option builtin_status_options[] = { OPT__VERBOSE(&verbose, N_("be verbose")), @@ -1585,6 +1665,12 @@ struct repository *repo UNUSED) OPT_CALLBACK_F(0, "porcelain", &status_format, N_("version"), N_("machine-readable output"), PARSE_OPT_OPTARG, opt_parse_porcelain), + OPT_CALLBACK_F(0, "serialize", &status_format, + N_("version"), N_("serialize raw status data to stdout"), + PARSE_OPT_OPTARG | PARSE_OPT_NONEG, opt_parse_serialize), + OPT_CALLBACK_F(0, "deserialize", NULL, + N_("path"), N_("deserialize raw status data from file"), + PARSE_OPT_OPTARG, opt_parse_deserialize), OPT_SET_INT(0, "long", &status_format, N_("show status in long format (default)"), STATUS_FORMAT_LONG), @@ -1646,10 +1732,26 @@ struct repository *repo UNUSED) s.show_untracked_files == SHOW_NO_UNTRACKED_FILES) die(_("Unsupported combination of ignored and untracked-files arguments")); + if (s.show_untracked_files == SHOW_COMPLETE_UNTRACKED_FILES && + s.show_ignored_mode == SHOW_NO_IGNORED) + die(_("Complete Untracked only supported with ignored files")); + parse_pathspec(&s.pathspec, 0, PATHSPEC_PREFER_FULL, prefix, argv); + /* + * If we want to try to deserialize status data from a cache file, + * we need to re-order the initialization code. The problem is that + * this makes for a very nasty diff and causes merge conflicts as we + * carry it forward. And it easy to mess up the merge, so we + * duplicate some code here to hopefully reduce conflicts. + */ + try_deserialize = (!do_serialize && + (do_implicit_deserialize || do_explicit_deserialize)); + if (try_deserialize) + goto skip_init; + enable_fscache(0); if (status_format != STATUS_FORMAT_PORCELAIN && status_format != STATUS_FORMAT_PORCELAIN_V2) @@ -1664,6 +1766,7 @@ struct repository *repo UNUSED) else fd = -1; +skip_init: s.is_initial = repo_get_oid(the_repository, s.reference, &oid) ? 1 : 0; if (!s.is_initial) oidcpy(&s.oid_commit, &oid); @@ -1680,6 +1783,24 @@ struct repository *repo UNUSED) s.rename_score = parse_rename_score(&rename_score_arg); } + if (try_deserialize) { + if (s.relative_paths) + s.prefix = prefix; + + if (wt_status_deserialize(&s, deserialize_path) == DESERIALIZE_OK) + return 0; + + /* deserialize failed, so force the initialization we skipped above. */ + enable_fscache(1); + repo_read_index_preload(the_repository, &s.pathspec, 0); + refresh_index(the_repository->index, REFRESH_QUIET|REFRESH_UNMERGED, &s.pathspec, NULL, NULL); + + if (use_optional_locks()) + fd = repo_hold_locked_index(the_repository, &index_lock, 0); + else + fd = -1; + } + wt_status_collect(&s); if (0 <= fd) diff --git a/contrib/completion/git-completion.bash b/contrib/completion/git-completion.bash index 9f8b9b50ff834c..ee64f26e5f1ee5 100644 --- a/contrib/completion/git-completion.bash +++ b/contrib/completion/git-completion.bash @@ -1848,7 +1848,7 @@ _git_clone () esac } -__git_untracked_file_modes="all no normal" +__git_untracked_file_modes="all no normal complete" __git_trailer_tokens () { diff --git a/meson.build b/meson.build index 215171f61e15d1..a2cf533b4c2f40 100644 --- a/meson.build +++ b/meson.build @@ -582,6 +582,8 @@ libgit_sources = [ 'write-or-die.c', 'ws.c', 'wt-status.c', + 'wt-status-deserialize.c', + 'wt-status-serialize.c', 'xdiff-interface.c', 'xdiff/xdiffi.c', 'xdiff/xemit.c', diff --git a/pkt-line.c b/pkt-line.c index 3fc3e9ea7059be..e479a4c3e13157 100644 --- a/pkt-line.c +++ b/pkt-line.c @@ -230,7 +230,7 @@ static int do_packet_write(const int fd_out, const char *buf, size_t size, return 0; } -static int packet_write_gently(const int fd_out, const char *buf, size_t size) +int packet_write_gently(const int fd_out, const char *buf, size_t size) { struct strbuf err = STRBUF_INIT; if (do_packet_write(fd_out, buf, size, &err)) { diff --git a/pkt-line.h b/pkt-line.h index e6cf85e34ee3c4..5c43e253993983 100644 --- a/pkt-line.h +++ b/pkt-line.h @@ -29,6 +29,7 @@ void packet_write(int fd_out, const char *buf, size_t size); void packet_buf_write(struct strbuf *buf, const char *fmt, ...) __attribute__((format (printf, 2, 3))); int packet_flush_gently(int fd); int packet_write_fmt_gently(int fd, const char *fmt, ...) __attribute__((format (printf, 2, 3))); +int packet_write_gently(const int fd_out, const char *buf, size_t size); int write_packetized_from_fd_no_flush(int fd_in, int fd_out); int write_packetized_from_buf_no_flush_count(const char *src_in, size_t len, int fd_out, int *packet_counter); diff --git a/t/meson.build b/t/meson.build index d02bcef6af1947..e455d247f31977 100644 --- a/t/meson.build +++ b/t/meson.build @@ -955,6 +955,8 @@ integration_tests = [ 't7519-status-fsmonitor.sh', 't7520-ignored-hook-warning.sh', 't7521-ignored-mode.sh', + 't7522-serialized-status.sh', + 't7523-status-complete-untracked.sh', 't7524-commit-summary.sh', 't7525-status-rename.sh', 't7526-commit-pathspec-file.sh', diff --git a/t/t7522-serialized-status.sh b/t/t7522-serialized-status.sh new file mode 100755 index 00000000000000..f88e03f591aabd --- /dev/null +++ b/t/t7522-serialized-status.sh @@ -0,0 +1,144 @@ +#!/bin/sh + +test_description='git serialized status tests' + +. ./test-lib.sh + +# This file includes tests for serializing / deserializing +# status data. These tests cover two basic features: +# +# [1] Because users can request different types of untracked-file +# and ignored file reporting, the cache data generated by +# serialize must use either the same untracked and ignored +# parameters as the later deserialize invocation; otherwise, +# the deserialize invocation must disregard the cached data +# and run a full scan itself. +# +# To increase the number of cases where the cached status can +# be used, we have added a "--untracked-file=complete" option +# that reports a superset or union of the results from the +# "-u normal" and "-u all". We combine this with a filter in +# deserialize to filter the results. +# +# Ignored file reporting is simpler in that is an all or +# nothing; there are no subsets. +# +# The tests here (in addition to confirming that a cache +# file can be generated and used by a subsequent status +# command) need to test this untracked-file filtering. +# +# [2] ensuring the status calls are using data from the status +# cache as expected. This includes verifying cached data +# is used when appropriate as well as falling back to +# performing a new status scan when the data in the cache +# is insufficient/known stale. + +test_expect_success 'setup' ' + git branch -M main && + cat >.gitignore <<-\EOF && + *.ign + ignored_dir/ + EOF + + mkdir tracked ignored_dir && + touch tracked_1.txt tracked/tracked_1.txt && + git add . && + test_tick && + git commit -m"Adding original file." && + mkdir untracked && + touch ignored.ign ignored_dir/ignored_2.txt \ + untracked_1.txt untracked/untracked_2.txt untracked/untracked_3.txt +' + +test_expect_success 'verify untracked-files=complete with no conversion' ' + test_when_finished "rm serialized_status.dat new_change.txt output" && + cat >expect <<-\EOF && + ? expect + ? serialized_status.dat + ? untracked/ + ? untracked/untracked_2.txt + ? untracked/untracked_3.txt + ? untracked_1.txt + ! ignored.ign + ! ignored_dir/ + EOF + + git status --untracked-files=complete --ignored=matching --serialize >serialized_status.dat && + touch new_change.txt && + + git status --porcelain=v2 --untracked-files=complete --ignored=matching --deserialize=serialized_status.dat >output && + test_filter_gitconfig output && + test_cmp expect output +' + +test_expect_success 'verify untracked-files=complete to untracked-files=normal conversion' ' + test_when_finished "rm serialized_status.dat new_change.txt output" && + cat >expect <<-\EOF && + ? expect + ? serialized_status.dat + ? untracked/ + ? untracked_1.txt + EOF + + git status --untracked-files=complete --ignored=matching --serialize >serialized_status.dat && + touch new_change.txt && + + git status --porcelain=v2 --deserialize=serialized_status.dat >output && + test_cmp expect output +' + +test_expect_success 'verify untracked-files=complete to untracked-files=all conversion' ' + test_when_finished "rm serialized_status.dat new_change.txt output" && + cat >expect <<-\EOF && + ? expect + ? serialized_status.dat + ? untracked/untracked_2.txt + ? untracked/untracked_3.txt + ? untracked_1.txt + ! ignored.ign + ! ignored_dir/ + EOF + + git status --untracked-files=complete --ignored=matching --serialize >serialized_status.dat && + touch new_change.txt && + + git status --porcelain=v2 --untracked-files=all --ignored=matching --deserialize=serialized_status.dat >output && + test_filter_gitconfig output && + test_cmp expect output +' + +test_expect_success 'verify serialized status with non-convertible ignore mode does new scan' ' + test_when_finished "rm serialized_status.dat new_change.txt output" && + cat >expect <<-\EOF && + ? expect + ? new_change.txt + ? output + ? serialized_status.dat + ? untracked/ + ? untracked_1.txt + ! ignored.ign + ! ignored_dir/ + EOF + + git status --untracked-files=complete --ignored=matching --serialize >serialized_status.dat && + touch new_change.txt && + + git status --porcelain=v2 --ignored --deserialize=serialized_status.dat >output && + test_filter_gitconfig output && + test_cmp expect output +' + +test_expect_success 'verify serialized status handles path scopes' ' + test_when_finished "rm serialized_status.dat new_change.txt output" && + cat >expect <<-\EOF && + ? untracked/ + EOF + + git status --untracked-files=complete --ignored=matching --serialize >serialized_status.dat && + touch new_change.txt && + + git status --porcelain=v2 --deserialize=serialized_status.dat untracked >output && + test_cmp expect output +' + +test_done diff --git a/t/t7523-status-complete-untracked.sh b/t/t7523-status-complete-untracked.sh new file mode 100755 index 00000000000000..c5c03d76350f9f --- /dev/null +++ b/t/t7523-status-complete-untracked.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +test_description='git status untracked complete tests' + +. ./test-lib.sh + +test_expect_success 'setup' ' + cat >.gitignore <<-\EOF && + *.ign + ignored_dir/ + EOF + + mkdir tracked ignored_dir && + touch tracked_1.txt tracked/tracked_1.txt && + git add . && + test_tick && + git commit -m"Adding original file." && + mkdir untracked && + touch ignored.ign ignored_dir/ignored_2.txt \ + untracked_1.txt untracked/untracked_2.txt untracked/untracked_3.txt +' + +test_expect_success 'verify untracked-files=complete' ' + cat >expect <<-\EOF && + ? expect + ? output + ? untracked/ + ? untracked/untracked_2.txt + ? untracked/untracked_3.txt + ? untracked_1.txt + ! ignored.ign + ! ignored_dir/ + EOF + + git status --porcelain=v2 --untracked-files=complete --ignored >output && + test_filter_gitconfig output && + test_cmp expect output +' + +test_done diff --git a/wt-status-deserialize.c b/wt-status-deserialize.c new file mode 100644 index 00000000000000..90174bb3d1fc51 --- /dev/null +++ b/wt-status-deserialize.c @@ -0,0 +1,622 @@ +#define USE_THE_REPOSITORY_VARIABLE +#include "git-compat-util.h" +#include "environment.h" +#include "hex.h" +#include "hash.h" +#include "wt-status.h" +#include "pkt-line.h" +#include "trace.h" +#include "statinfo.h" +#include "hex.h" + +static struct trace_key trace_deserialize = TRACE_KEY_INIT(DESERIALIZE); + +enum deserialize_parse_strategy { + DESERIALIZE_STRATEGY_AS_IS, + DESERIALIZE_STRATEGY_SKIP, + DESERIALIZE_STRATEGY_NORMAL, + DESERIALIZE_STRATEGY_ALL +}; + +static int check_path_contains(const char *out, int out_len, const char *in, int in_len) +{ + return (out_len > 0 && + out_len < in_len && + (out[out_len - 1] == '/') && + !memcmp(out, in, out_len)); +} + +static const char *my_packet_read_line(int fd, int *line_len) +{ + static char buf[LARGE_PACKET_MAX]; + + *line_len = packet_read(fd, buf, sizeof(buf), + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + return (*line_len > 0) ? buf : NULL; +} + +/* + * mtime_reported contains the mtime of the index when the + * serialization snapshot was computed. + * + * mtime_observed_on_disk contains the mtime of the index now. + * + * If these 2 times are different, then the .git/index has + * changed since the serialization cache was created and we + * must reject the cache because anything could have changed. + * + * If they are the same, we continue trying to use the cache. + */ +static int my_validate_index(const char *path, const struct cache_time *mtime_reported) +{ + struct stat st; + struct cache_time mtime_observed_on_disk; + + if (lstat(path, &st)) { + trace_printf_key(&trace_deserialize, "could not stat index"); + return DESERIALIZE_ERR; + } + mtime_observed_on_disk.sec = st.st_mtime; + mtime_observed_on_disk.nsec = ST_MTIME_NSEC(st); + if ((mtime_observed_on_disk.sec != mtime_reported->sec) || + (mtime_observed_on_disk.nsec != mtime_reported->nsec)) { + trace_printf_key(&trace_deserialize, "index mtime changed [des %d.%d][obs %d.%d]", + mtime_reported->sec, mtime_reported->nsec, + mtime_observed_on_disk.sec, mtime_observed_on_disk.nsec); + return DESERIALIZE_ERR; + } + + return DESERIALIZE_OK; +} + +static int wt_deserialize_v1_header(struct wt_status *s, int fd) +{ + struct cache_time index_mtime; + int line_len, nr_fields; + const char *line; + const char *arg; + + /* + * parse header lines up to the first flush packet. + */ + while ((line = my_packet_read_line(fd, &line_len))) { + + if (skip_prefix(line, "index_mtime ", &arg)) { + nr_fields = sscanf(arg, "%u %u", + &index_mtime.sec, + &index_mtime.nsec); + if (nr_fields != 2) { + trace_printf_key(&trace_deserialize, "invalid index_mtime (%d) '%s'", + nr_fields, line); + return DESERIALIZE_ERR; + } + continue; + } + + if (skip_prefix(line, "is_initial ", &arg)) { + s->is_initial = (int)strtol(arg, NULL, 10); + continue; + } + if (skip_prefix(line, "branch ", &arg)) { + s->branch = xstrdup(arg); + continue; + } + if (skip_prefix(line, "reference ", &arg)) { + s->reference = xstrdup(arg); + continue; + } + /* pathspec */ + /* verbose */ + /* amend */ + if (skip_prefix(line, "whence ", &arg)) { + s->whence = (int)strtol(arg, NULL, 10); + continue; + } + /* nowarn */ + /* use_color */ + /* no_gettext */ + /* display_comment_prefix */ + /* relative_paths */ + /* submodule_summary */ + if (skip_prefix(line, "show_ignored_mode ", &arg)) { + s->show_ignored_mode = (int)strtol(arg, NULL, 10); + continue; + } + if (skip_prefix(line, "show_untracked_files ", &arg)) { + s->show_untracked_files = (int)strtol(arg, NULL, 10); + continue; + } + if (skip_prefix(line, "ignore_submodule_arg ", &arg)) { + s->ignore_submodule_arg = xstrdup(arg); + continue; + } + /* color_palette */ + /* colopts */ + /* null_termination */ + /* commit_template */ + /* show_branch */ + /* show_stash */ + if (skip_prefix(line, "hints ", &arg)) { + s->hints = (int)strtol(arg, NULL, 10); + continue; + } + if (skip_prefix(line, "detect_rename ", &arg)) { + s->detect_rename = (int)strtol(arg, NULL, 10); + continue; + } + if (skip_prefix(line, "rename_score ", &arg)) { + s->rename_score = (int)strtol(arg, NULL, 10); + continue; + } + if (skip_prefix(line, "rename_limit ", &arg)) { + s->rename_limit = (int)strtol(arg, NULL, 10); + continue; + } + /* status_format */ + if (skip_prefix(line, "sha1_commit ", &arg)) { + if (get_oid_hex(arg, &s->oid_commit)) { + trace_printf_key(&trace_deserialize, "invalid sha1_commit"); + return DESERIALIZE_ERR; + } + continue; + } + if (skip_prefix(line, "committable ", &arg)) { + s->committable = (int)strtol(arg, NULL, 10); + continue; + } + if (skip_prefix(line, "workdir_dirty ", &arg)) { + s->workdir_dirty = (int)strtol(arg, NULL, 10); + continue; + } + /* prefix */ + + trace_printf_key(&trace_deserialize, "unexpected line '%s'", line); + return DESERIALIZE_ERR; + } + + return my_validate_index(s->index_file, &index_mtime); +} + +/* + * Build a string-list of (count) lines from the input. + */ +static int wt_deserialize_v1_changed_items(struct wt_status *s, int fd, int count UNUSED) +{ + struct wt_status_serialize_data *sd; + char *p; + int line_len; + const char *line; + struct string_list_item *item; + + memset(&s->change, 0, sizeof(s->change)); + s->change.strdup_strings = 1; + + /* + * + + * + * + * NUL [] NUL + */ + while ((line = my_packet_read_line(fd, &line_len))) { + struct wt_status_change_data *d = xcalloc(1, sizeof(*d)); + sd = (struct wt_status_serialize_data *)line; + + d->worktree_status = ntohl(sd->fixed.worktree_status); + d->index_status = ntohl(sd->fixed.index_status); + d->stagemask = ntohl(sd->fixed.stagemask); + d->rename_status = ntohl(sd->fixed.rename_status); + d->rename_score = ntohl(sd->fixed.rename_score); + d->mode_head = ntohl(sd->fixed.mode_head); + d->mode_index = ntohl(sd->fixed.mode_index); + d->mode_worktree = ntohl(sd->fixed.mode_worktree); + d->dirty_submodule = ntohl(sd->fixed.dirty_submodule); + d->new_submodule_commits = ntohl(sd->fixed.new_submodule_commits); + oidcpy(&d->oid_head, &sd->fixed.oid_head); + oidcpy(&d->oid_index, &sd->fixed.oid_index); + + p = sd->variant; + item = string_list_append(&s->change, p); + p += strlen(p) + 1; + if (*p) + d->rename_source = xstrdup(p); + item->util = d; + + trace_printf_key( + &trace_deserialize, + "change: %d %d %d %d %d %o %o %o %d %d %s %s '%s' '%s'", + d->worktree_status, + d->index_status, + d->stagemask, + d->rename_status, + d->rename_score, + d->mode_head, + d->mode_index, + d->mode_worktree, + d->dirty_submodule, + d->new_submodule_commits, + oid_to_hex(&d->oid_head), + oid_to_hex(&d->oid_index), + item->string, + (d->rename_source ? d->rename_source : "")); + } + + return DESERIALIZE_OK; +} + +static int wt_deserialize_v1_untracked_items(struct wt_status *s, + int fd, + int count UNUSED, + enum deserialize_parse_strategy strategy) +{ + int line_len; + const char *line; + char *out = NULL; + int out_len = 0; + + memset(&s->untracked, 0, sizeof(s->untracked)); + s->untracked.strdup_strings = 1; + + /* + * + + * + */ + while ((line = my_packet_read_line(fd, &line_len))) { + if (strategy == DESERIALIZE_STRATEGY_AS_IS) + string_list_append(&s->untracked, line); + if (strategy == DESERIALIZE_STRATEGY_SKIP) + continue; + if (strategy == DESERIALIZE_STRATEGY_NORMAL) { + + /* Only add "normal" entries to list */ + if (out && + check_path_contains(out, out_len, line, line_len)) { + continue; + } + else { + out = string_list_append(&s->untracked, line)->string; + out_len = line_len; + } + } + if (strategy == DESERIALIZE_STRATEGY_ALL) { + /* Only add "all" entries to list */ + if (line[line_len - 1] != '/') + string_list_append(&s->untracked, line); + } + } + + return DESERIALIZE_OK; +} + +static int wt_deserialize_v1_ignored_items(struct wt_status *s, + int fd, + int count UNUSED, + enum deserialize_parse_strategy strategy) +{ + int line_len; + const char *line; + + memset(&s->ignored, 0, sizeof(s->ignored)); + s->ignored.strdup_strings = 1; + + /* + * + + * + */ + while ((line = my_packet_read_line(fd, &line_len))) { + if (strategy == DESERIALIZE_STRATEGY_AS_IS) + string_list_append(&s->ignored, line); + else + continue; + } + + return DESERIALIZE_OK; +} + +static int validate_untracked_files_arg(enum untracked_status_type cmd, + enum untracked_status_type des, + enum deserialize_parse_strategy *strategy) +{ + *strategy = DESERIALIZE_STRATEGY_AS_IS; + + if (cmd == des) { + *strategy = DESERIALIZE_STRATEGY_AS_IS; + } else if (cmd == SHOW_NO_UNTRACKED_FILES) { + *strategy = DESERIALIZE_STRATEGY_SKIP; + } else if (des == SHOW_COMPLETE_UNTRACKED_FILES) { + if (cmd == SHOW_ALL_UNTRACKED_FILES) + *strategy = DESERIALIZE_STRATEGY_ALL; + else if (cmd == SHOW_NORMAL_UNTRACKED_FILES) + *strategy = DESERIALIZE_STRATEGY_NORMAL; + } else { + return DESERIALIZE_ERR; + } + + return DESERIALIZE_OK; +} + +static int validate_ignored_files_arg(enum show_ignored_type cmd, + enum show_ignored_type des, + enum deserialize_parse_strategy *strategy) +{ + *strategy = DESERIALIZE_STRATEGY_AS_IS; + + if (cmd == SHOW_NO_IGNORED) { + *strategy = DESERIALIZE_STRATEGY_SKIP; + } + else if (cmd != des) { + return DESERIALIZE_ERR; + } + + return DESERIALIZE_OK; +} + +static int wt_deserialize_v1(const struct wt_status *cmd_s, struct wt_status *s, int fd) +{ + int line_len; + const char *line; + const char *arg; + int nr_changed = 0; + int nr_untracked = 0; + int nr_ignored = 0; + + enum deserialize_parse_strategy ignored_strategy = DESERIALIZE_STRATEGY_AS_IS, untracked_strategy = DESERIALIZE_STRATEGY_AS_IS; + + if (wt_deserialize_v1_header(s, fd) == DESERIALIZE_ERR) + return DESERIALIZE_ERR; + + /* + * We now have the header parsed. Look at the command args (as passed in), and see how to parse + * the serialized data + */ + if (validate_untracked_files_arg(cmd_s->show_untracked_files, s->show_untracked_files, &untracked_strategy)) { + trace_printf_key(&trace_deserialize, "reject: show_untracked_file: command: %d, serialized : %d", + cmd_s->show_untracked_files, + s->show_untracked_files); + return DESERIALIZE_ERR; + } + + if (validate_ignored_files_arg(cmd_s->show_ignored_mode, s->show_ignored_mode, &ignored_strategy)) { + trace_printf_key(&trace_deserialize, "reject: show_ignored_mode: command: %d, serialized: %d", + cmd_s->show_ignored_mode, + s->show_ignored_mode); + return DESERIALIZE_ERR; + } + + /* + * [ [+] ] + * [ [+] ] + * [ [+] ] + */ + while ((line = my_packet_read_line(fd, &line_len))) { + if (skip_prefix(line, "changed ", &arg)) { + nr_changed = (int)strtol(arg, NULL, 10); + if (wt_deserialize_v1_changed_items(s, fd, nr_changed) + == DESERIALIZE_ERR) + return DESERIALIZE_ERR; + continue; + } + if (skip_prefix(line, "untracked ", &arg)) { + nr_untracked = (int)strtol(arg, NULL, 10); + if (wt_deserialize_v1_untracked_items(s, fd, nr_untracked, untracked_strategy) + == DESERIALIZE_ERR) + return DESERIALIZE_ERR; + continue; + } + if (skip_prefix(line, "ignored ", &arg)) { + nr_ignored = (int)strtol(arg, NULL, 10); + if (wt_deserialize_v1_ignored_items(s, fd, nr_ignored, ignored_strategy) + == DESERIALIZE_ERR) + return DESERIALIZE_ERR; + continue; + } + trace_printf_key(&trace_deserialize, "unexpected line '%s'", line); + return DESERIALIZE_ERR; + } + + return DESERIALIZE_OK; +} + +static int wt_deserialize_parse(const struct wt_status *cmd_s, struct wt_status *s, int fd) +{ + int line_len; + const char *line; + const char *arg; + + if ((line = my_packet_read_line(fd, &line_len)) && + (skip_prefix(line, "version ", &arg))) { + int version = (int)strtol(arg, NULL, 10); + if (version == 1) + return wt_deserialize_v1(cmd_s, s, fd); + } + trace_printf_key(&trace_deserialize, "missing/unsupported version"); + return DESERIALIZE_ERR; +} + +static inline int my_strcmp_null(const char *a, const char *b) +{ + const char *alt_a = (a) ? a : ""; + const char *alt_b = (b) ? b : ""; + + return strcmp(alt_a, alt_b); +} + +static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *des_s, int fd) +{ + /* + * Check the path spec on the current command + */ + if (cmd_s->pathspec.nr > 1) { + trace_printf_key(&trace_deserialize, "reject: multiple pathspecs"); + return DESERIALIZE_ERR; + } + + /* + * If we have a pathspec, but it maches the root (e.g. no filtering) + * then this is OK. + */ + if (cmd_s->pathspec.nr == 1 && + my_strcmp_null(cmd_s->pathspec.items[0].match, "")) { + trace_printf_key(&trace_deserialize, "reject: pathspec"); + return DESERIALIZE_ERR; + } + + /* + * Copy over some required fields from the current command. + */ + des_s->repo = cmd_s->repo; + des_s->index_file = cmd_s->index_file; + + /* + * Deserialize cached status + */ + if (wt_deserialize_parse(cmd_s, des_s, fd) == DESERIALIZE_ERR) + return DESERIALIZE_ERR; + + /* + * Compare fields in cmd_s with those observed in des_s and + * complain if they are incompatible (such as different "-u" + * or "--ignored" settings). + */ + if (cmd_s->is_initial != des_s->is_initial) { + trace_printf_key(&trace_deserialize, "reject: is_initial"); + return DESERIALIZE_ERR; + } + if (my_strcmp_null(cmd_s->branch, des_s->branch)) { + trace_printf_key(&trace_deserialize, "reject: branch"); + return DESERIALIZE_ERR; + } + if (my_strcmp_null(cmd_s->reference, des_s->reference)) { + trace_printf_key(&trace_deserialize, "reject: reference"); + return DESERIALIZE_ERR; + } + /* verbose */ + /* amend */ + if (cmd_s->whence != des_s->whence) { + trace_printf_key(&trace_deserialize, "reject: whence"); + return DESERIALIZE_ERR; + } + /* nowarn */ + /* use_color */ + /* no_gettext */ + /* display_comment_prefix */ + /* relative_paths */ + /* submodule_summary */ + + /* show_ignored_files - already validated */ + /* show_untrackes_files - already validated */ + + /* + * Submodules are not supported by status serialization. + * The status will not be serialized if it contains submodules, + * and so this check is not needed. + * + * if (my_strcmp_null(cmd_s->ignore_submodule_arg, des_s->ignore_submodule_arg)) { + * trace_printf_key(&trace_deserialize, "reject: ignore_submodule_arg"); + * return DESERIALIZE_ERR; + * } + */ + + /* color_palette */ + /* colopts */ + /* null_termination */ + /* commit_template */ + /* show_branch */ + /* show_stash */ + /* hints */ + if (cmd_s->detect_rename != des_s->detect_rename) { + trace_printf_key(&trace_deserialize, "reject: detect_rename"); + return DESERIALIZE_ERR; + } + if (cmd_s->rename_score != des_s->rename_score) { + trace_printf_key(&trace_deserialize, "reject: rename_score"); + return DESERIALIZE_ERR; + } + if (cmd_s->rename_limit != des_s->rename_limit) { + trace_printf_key(&trace_deserialize, "reject: rename_limit"); + return DESERIALIZE_ERR; + } + /* status_format */ + if (!oideq(&cmd_s->oid_commit, &des_s->oid_commit)) { + trace_printf_key(&trace_deserialize, "reject: sha1_commit"); + return DESERIALIZE_ERR; + } + + /* + * Copy over display-related fields from the current command. + */ + des_s->verbose = cmd_s->verbose; + /* amend */ + /* whence */ + des_s->nowarn = cmd_s->nowarn; + des_s->use_color = cmd_s->use_color; + des_s->no_gettext = cmd_s->no_gettext; + des_s->display_comment_prefix = cmd_s->display_comment_prefix; + des_s->relative_paths = cmd_s->relative_paths; + des_s->submodule_summary = cmd_s->submodule_summary; + memcpy(des_s->color_palette, cmd_s->color_palette, + sizeof(char)*WT_STATUS_MAXSLOT*COLOR_MAXLEN); + des_s->colopts = cmd_s->colopts; + des_s->null_termination = cmd_s->null_termination; + /* commit_template */ + des_s->show_branch = cmd_s->show_branch; + des_s->show_stash = cmd_s->show_stash; + /* hints */ + des_s->status_format = cmd_s->status_format; + des_s->fp = cmd_s->fp; + if (cmd_s->prefix && *cmd_s->prefix) + des_s->prefix = xstrdup(cmd_s->prefix); + + return DESERIALIZE_OK; +} + + +/* + * Read raw serialized status data from the given file + * + * Verify that the args specified in the current command + * are compatible with the deserialized data (such as "-uno"). + * + * Copy display-related fields from the current command + * into the deserialized data (so that the user can request + * long or short as they please). + */ +int wt_status_deserialize(const struct wt_status *cmd_s, + const char *path) +{ + struct wt_status des_s; + int result; + struct string_list_item *change; + + if (path && *path && strcmp(path, "0")) { + int fd = xopen(path, O_RDONLY); + if (fd == -1) { + trace_printf_key(&trace_deserialize, "could not read '%s'", path); + return DESERIALIZE_ERR; + } + trace_printf_key(&trace_deserialize, "reading serialization file '%s'", path); + result = wt_deserialize_fd(cmd_s, &des_s, fd); + close(fd); + } else { + trace_printf_key(&trace_deserialize, "reading stdin"); + result = wt_deserialize_fd(cmd_s, &des_s, 0); + } + + if (result == DESERIALIZE_OK) { + wt_status_get_state(cmd_s->repo, &des_s.state, des_s.branch && + !strcmp(des_s.branch, "HEAD")); + wt_status_print(&des_s); + } + + free(des_s.branch); + free((char *)des_s.reference); + for_each_string_list_item(change, &des_s.change) { + struct wt_status_change_data *d = change->util; + if (d) + free(d->rename_source); + } + string_list_clear(&des_s.change, 1); + string_list_clear(&des_s.ignored, 1); + string_list_clear(&des_s.untracked, 1); + return result; +} diff --git a/wt-status-serialize.c b/wt-status-serialize.c new file mode 100644 index 00000000000000..a0dcf8e65ed44e --- /dev/null +++ b/wt-status-serialize.c @@ -0,0 +1,219 @@ +#include "git-compat-util.h" +#include "hex.h" +#include "repository.h" +#include "wt-status.h" +#include "pkt-line.h" +#include "trace.h" +#include "read-cache-ll.h" + +static struct trace_key trace_serialize = TRACE_KEY_INIT(SERIALIZE); + +/* + * Write V1 header fields. + */ +static void wt_serialize_v1_header(struct wt_status *s, int fd) +{ + /* + * Write select fields from the current index to help + * the deserializer recognize a stale data set. + */ + packet_write_fmt(fd, "index_mtime %d %d\n", + s->repo->index->timestamp.sec, + s->repo->index->timestamp.nsec); + + /* + * Write data from wt_status to qualify this status report. + * That is, if this run specified "-uno", the consumer of + * our serialization should know that. + */ + packet_write_fmt(fd, "is_initial %d\n", s->is_initial); + if (s->branch) + packet_write_fmt(fd, "branch %s\n", s->branch); + if (s->reference) + packet_write_fmt(fd, "reference %s\n", s->reference); + /* pathspec */ + /* verbose */ + /* amend */ + packet_write_fmt(fd, "whence %d\n", s->whence); + /* nowarn */ + /* use_color */ + /* no_gettext */ + /* display_comment_prefix */ + /* relative_paths */ + /* submodule_summary */ + packet_write_fmt(fd, "show_ignored_mode %d\n", s->show_ignored_mode); + packet_write_fmt(fd, "show_untracked_files %d\n", s->show_untracked_files); + if (s->ignore_submodule_arg) + packet_write_fmt(fd, "ignore_submodule_arg %s\n", s->ignore_submodule_arg); + /* color_palette */ + /* colopts */ + /* null_termination */ + /* commit_template */ + /* show_branch */ + /* show_stash */ + packet_write_fmt(fd, "hints %d\n", s->hints); + packet_write_fmt(fd, "detect_rename %d\n", s->detect_rename); + packet_write_fmt(fd, "rename_score %d\n", s->rename_score); + packet_write_fmt(fd, "rename_limit %d\n", s->rename_limit); + /* status_format */ + packet_write_fmt(fd, "sha1_commit %s\n", oid_to_hex(&s->oid_commit)); + packet_write_fmt(fd, "committable %d\n", s->committable); + packet_write_fmt(fd, "workdir_dirty %d\n", s->workdir_dirty); + /* prefix */ + packet_flush(fd); +} + +/* + * Print changed/unmerged items. + * We write raw (not c-quoted) pathname(s). The rename_source is only + * set when status computed a rename/copy. + * + * We ALWAYS write a final LF to the packet-line (for debugging) + * even though Linux pathnames allow LFs. + */ +static inline void wt_serialize_v1_changed(struct wt_status *s UNUSED, int fd, + struct string_list_item *item) +{ + struct wt_status_change_data *d = item->util; + struct wt_status_serialize_data sd; + char *begin; + char *end; + char *p; + int len_path, len_rename_source; + + trace_printf_key(&trace_serialize, + "change: %d %d %d %d %d %o %o %o %d %d %s %s '%s' '%s'", + d->worktree_status, + d->index_status, + d->stagemask, + d->rename_status, + d->rename_score, + d->mode_head, + d->mode_index, + d->mode_worktree, + d->dirty_submodule, + d->new_submodule_commits, + oid_to_hex(&d->oid_head), + oid_to_hex(&d->oid_index), + item->string, + (d->rename_source ? d->rename_source : "")); + + sd.fixed.worktree_status = htonl(d->worktree_status); + sd.fixed.index_status = htonl(d->index_status); + sd.fixed.stagemask = htonl(d->stagemask); + sd.fixed.rename_status = htonl(d->rename_status); + sd.fixed.rename_score = htonl(d->rename_score); + sd.fixed.mode_head = htonl(d->mode_head); + sd.fixed.mode_index = htonl(d->mode_index); + sd.fixed.mode_worktree = htonl(d->mode_worktree); + sd.fixed.dirty_submodule = htonl(d->dirty_submodule); + sd.fixed.new_submodule_commits = htonl(d->new_submodule_commits); + oidcpy(&sd.fixed.oid_head, &d->oid_head); + oidcpy(&sd.fixed.oid_index, &d->oid_index); + + begin = (char *)&sd; + end = begin + sizeof(sd); + + p = sd.variant; + + /* + * Write NUL [] NUL LF at the end of the buffer. + */ + len_path = strlen(item->string); + len_rename_source = d->rename_source ? strlen(d->rename_source) : 0; + + /* + * This is a bit of a hack, but I don't want to split the + * status detail record across multiple pkt-lines. + */ + if (p + len_path + 1 + len_rename_source + 1 + 1 >= end) + BUG("path to long to serialize '%s'", item->string); + + memcpy(p, item->string, len_path); + p += len_path; + *p++ = '\0'; + + if (len_rename_source) { + memcpy(p, d->rename_source, len_rename_source); + p += len_rename_source; + } + *p++ = '\0'; + *p++ = '\n'; + + if (packet_write_gently(fd, begin, (p - begin))) + BUG("cannot serialize '%s'", item->string); +} + +/* + * Write raw (not c-quoted) pathname for an untracked item. + * We ALWAYS write a final LF to the packet-line (for debugging) + * even though Linux pathnames allows LFs. That is, deserialization + * should use the packet-line length and omit the final LF. + */ +static inline void wt_serialize_v1_untracked(struct wt_status *s UNUSED, int fd, + struct string_list_item *item) +{ + packet_write_fmt(fd, "%s\n", item->string); +} + +/* + * Write raw (not c-quoted) pathname for an ignored item. + * We ALWAYS write a final LF to the packet-line (for debugging) + * even though Linux pathnames allows LFs. + */ +static inline void wt_serialize_v1_ignored(struct wt_status *s UNUSED, int fd, + struct string_list_item *item) +{ + packet_write_fmt(fd, "%s\n", item->string); +} + +/* + * Serialize the list of changes to stdout. The goal of this + * is to just serialize the key fields in wt_status so that a + * later command can rebuilt it and do the printing. + * + * We DO NOT include the contents of wt_status_state NOR + * current branch info. This info easily gets stale and + * is relatively quick for the status consumer to compute + * as necessary. + */ +void wt_status_serialize_v1(struct wt_status *s) +{ + int fd = 1; /* we always write to stdout */ + struct string_list_item *iter; + size_t k; + + /* + * version header must be first line. + */ + packet_write_fmt(fd, "version 1\n"); + + wt_serialize_v1_header(s, fd); + + if (s->change.nr > 0) { + packet_write_fmt(fd, "changed %"PRIuMAX"\n", (uintmax_t)s->change.nr); + for (k = 0; k < s->change.nr; k++) { + iter = &(s->change.items[k]); + wt_serialize_v1_changed(s, fd, iter); + } + packet_flush(fd); + } + + if (s->untracked.nr > 0) { + packet_write_fmt(fd, "untracked %"PRIuMAX"\n", (uintmax_t)s->untracked.nr); + for (k = 0; k < s->untracked.nr; k++) { + iter = &(s->untracked.items[k]); + wt_serialize_v1_untracked(s, fd, iter); + } + packet_flush(fd); + } + + if (s->ignored.nr > 0) { + packet_write_fmt(fd, "ignored %"PRIuMAX"\n", (uintmax_t)s->ignored.nr); + for (k = 0; k < s->ignored.nr; k++) { + iter = &(s->ignored.items[k]); + wt_serialize_v1_ignored(s, fd, iter); + } + packet_flush(fd); + } +} diff --git a/wt-status.c b/wt-status.c index d4983ceb68541b..7f15ebf4eeb93c 100644 --- a/wt-status.c +++ b/wt-status.c @@ -816,6 +816,9 @@ static void wt_status_collect_untracked(struct wt_status *s) if (s->show_untracked_files != SHOW_ALL_UNTRACKED_FILES) dir.flags |= DIR_SHOW_OTHER_DIRECTORIES | DIR_HIDE_EMPTY_DIRECTORIES; + if (s->show_untracked_files == SHOW_COMPLETE_UNTRACKED_FILES) + dir.flags |= DIR_KEEP_UNTRACKED_CONTENTS; + if (s->show_ignored_mode) { dir.flags |= DIR_SHOW_IGNORED_TOO; @@ -2757,6 +2760,9 @@ void wt_status_print(struct wt_status *s) case STATUS_FORMAT_LONG: wt_longstatus_print(s); break; + case STATUS_FORMAT_SERIALIZE_V1: + wt_status_serialize_v1(s); + break; } trace2_region_leave("status", "print", s->repo); diff --git a/wt-status.h b/wt-status.h index 2143f50b49208c..f5ac9e5c656f8f 100644 --- a/wt-status.h +++ b/wt-status.h @@ -4,6 +4,7 @@ #include "string-list.h" #include "color.h" #include "pathspec.h" +#include "pkt-line.h" #include "remote.h" struct repository; @@ -26,7 +27,8 @@ enum untracked_status_type { SHOW_UNTRACKED_FILES_ERROR = -1, SHOW_NO_UNTRACKED_FILES = 0, SHOW_NORMAL_UNTRACKED_FILES, - SHOW_ALL_UNTRACKED_FILES + SHOW_ALL_UNTRACKED_FILES, + SHOW_COMPLETE_UNTRACKED_FILES, }; enum show_ignored_type { @@ -74,6 +76,7 @@ enum wt_status_format { STATUS_FORMAT_SHORT, STATUS_FORMAT_PORCELAIN, STATUS_FORMAT_PORCELAIN_V2, + STATUS_FORMAT_SERIALIZE_V1, STATUS_FORMAT_UNSPECIFIED }; @@ -194,4 +197,52 @@ int require_clean_work_tree(struct repository *repo, int ignore_submodules, int gently); +#define DESERIALIZE_OK 0 +#define DESERIALIZE_ERR 1 + +struct wt_status_serialize_data_fixed +{ + uint32_t worktree_status; + uint32_t index_status; + uint32_t stagemask; + uint32_t rename_status; + uint32_t rename_score; + uint32_t mode_head; + uint32_t mode_index; + uint32_t mode_worktree; + uint32_t dirty_submodule; + uint32_t new_submodule_commits; + struct object_id oid_head; + struct object_id oid_index; +}; + +/* + * Consume the maximum amount of data possible in a + * packet-line record. This is overkill because we + * have at most 2 relative pathnames, but means we + * don't need to allocate a variable length structure. + */ +struct wt_status_serialize_data +{ + struct wt_status_serialize_data_fixed fixed; + char variant[LARGE_PACKET_DATA_MAX + - sizeof(struct wt_status_serialize_data_fixed)]; +}; + +/* + * Serialize computed status scan results using "version 1" format + * to the given file. + */ +void wt_status_serialize_v1(struct wt_status *s); + +/* + * Deserialize existing status results from the given file and + * populate a (new) "struct wt_status". Use the contents of "cmd_s" + * (computed from the command line arguments) to verify that the + * cached data is compatible and overlay various display-related + * fields. + */ +int wt_status_deserialize(const struct wt_status *cmd_s, + const char *path); + #endif /* STATUS_H */ From 115a64e31138605c9412fc089db49c4ff6cdad0f Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 26 Sep 2018 12:29:26 -0400 Subject: [PATCH 075/289] gvfs:trace2:data: add trace2 tracing around read_object_process Add trace2 region around read_object_process to collect time spent waiting for missing objects to be dynamically fetched. Signed-off-by: Jeff Hostetler --- odb.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/odb.c b/odb.c index c8588040440ac1..11b55d5a450b0d 100644 --- a/odb.c +++ b/odb.c @@ -31,6 +31,7 @@ #include "tmp-objdir.h" #include "trace2.h" #include "trace.h" +#include "trace2.h" #include "write-or-die.h" /* @@ -560,6 +561,8 @@ int read_object_process(struct repository *r, const struct object_id *oid) start = getnanotime(); + trace2_region_enter("subprocess", "read_object",r); + if (!subprocess_map_initialized) { subprocess_map_initialized = 1; hashmap_init(&subprocess_map, (hashmap_cmp_fn)cmd2process_cmp, @@ -576,13 +579,16 @@ int read_object_process(struct repository *r, const struct object_id *oid) if (subprocess_start(&subprocess_map, &entry->subprocess, cmd, start_read_object_fn)) { free(entry); - return -1; + err = -1; + goto leave_region; } } process = &entry->subprocess.process; - if (!(CAP_GET & entry->supported_capabilities)) - return -1; + if (!(CAP_GET & entry->supported_capabilities)) { + err = -1; + goto leave_region; + } sigchain_push(SIGPIPE, SIG_IGN); @@ -631,6 +637,10 @@ int read_object_process(struct repository *r, const struct object_id *oid) trace_performance_since(start, "read_object_process"); +leave_region: + trace2_region_leave_printf("subprocess", "read_object", r, + "result %d", err); + strbuf_release(&status); return err; } From 193e7a2ee34b6793ac7829e1da5c75b76b3411da Mon Sep 17 00:00:00 2001 From: Jameson Miller Date: Wed, 10 Jan 2018 11:56:26 -0500 Subject: [PATCH 076/289] Teach ahead-behind and serialized status to play nicely together --- t/t7522-serialized-status.sh | 34 +++++++++++++++++++++++++++++++++- wt-status-deserialize.c | 3 ++- wt-status-serialize.c | 1 + 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/t/t7522-serialized-status.sh b/t/t7522-serialized-status.sh index f88e03f591aabd..c4c14fe2f71d79 100755 --- a/t/t7522-serialized-status.sh +++ b/t/t7522-serialized-status.sh @@ -47,7 +47,13 @@ test_expect_success 'setup' ' git commit -m"Adding original file." && mkdir untracked && touch ignored.ign ignored_dir/ignored_2.txt \ - untracked_1.txt untracked/untracked_2.txt untracked/untracked_3.txt + untracked_1.txt untracked/untracked_2.txt untracked/untracked_3.txt && + + test_oid_cache <<-EOF + branch_oid sha1:68d4a437ea4c2de65800f48c053d4d543b55c410 + + branch_oid sha256:6b95e4b1ea911dad213f2020840f5e92d3066cf9e38cf35f79412ec58d409ce4 + EOF ' test_expect_success 'verify untracked-files=complete with no conversion' ' @@ -141,4 +147,30 @@ test_expect_success 'verify serialized status handles path scopes' ' test_cmp expect output ' +test_expect_success 'verify no-ahead-behind and serialized status integration' ' + test_when_finished "rm serialized_status.dat new_change.txt output" && + cat >expect <<-EOF && + # branch.oid $(test_oid branch_oid) + # branch.head alt_branch + # branch.upstream main + # branch.ab +1 -0 + ? expect + ? serialized_status.dat + ? untracked/ + ? untracked_1.txt + EOF + + git checkout -b alt_branch main --track >/dev/null && + touch alt_branch_changes.txt && + git add alt_branch_changes.txt && + test_tick && + git commit -m"New commit on alt branch" && + + git status --untracked-files=complete --ignored=matching --serialize >serialized_status.dat && + touch new_change.txt && + + git -c status.aheadBehind=false status --porcelain=v2 --branch --ahead-behind --deserialize=serialized_status.dat >output && + test_cmp expect output +' + test_done diff --git a/wt-status-deserialize.c b/wt-status-deserialize.c index 90174bb3d1fc51..52e3958bcf2eba 100644 --- a/wt-status-deserialize.c +++ b/wt-status-deserialize.c @@ -524,6 +524,7 @@ static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *de /* show_branch */ /* show_stash */ /* hints */ + /* ahead_behind_flags */ if (cmd_s->detect_rename != des_s->detect_rename) { trace_printf_key(&trace_deserialize, "reject: detect_rename"); return DESERIALIZE_ERR; @@ -562,6 +563,7 @@ static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *de des_s->show_branch = cmd_s->show_branch; des_s->show_stash = cmd_s->show_stash; /* hints */ + des_s->ahead_behind_flags = cmd_s->ahead_behind_flags; des_s->status_format = cmd_s->status_format; des_s->fp = cmd_s->fp; if (cmd_s->prefix && *cmd_s->prefix) @@ -570,7 +572,6 @@ static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *de return DESERIALIZE_OK; } - /* * Read raw serialized status data from the given file * diff --git a/wt-status-serialize.c b/wt-status-serialize.c index a0dcf8e65ed44e..b7102ce80baf50 100644 --- a/wt-status-serialize.c +++ b/wt-status-serialize.c @@ -52,6 +52,7 @@ static void wt_serialize_v1_header(struct wt_status *s, int fd) /* show_branch */ /* show_stash */ packet_write_fmt(fd, "hints %d\n", s->hints); + /* ahead_behind_flags */ packet_write_fmt(fd, "detect_rename %d\n", s->detect_rename); packet_write_fmt(fd, "rename_score %d\n", s->rename_score); packet_write_fmt(fd, "rename_limit %d\n", s->rename_limit); From ec1b027506d70feb5a3b457f0271702633ad3812 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 26 Sep 2018 11:21:22 -0400 Subject: [PATCH 077/289] gvfs:trace2:data: status deserialization information Add trace2 region and data events describing attempts to deserialize status data using a status cache. A category:status, label:deserialize region is pushed around the deserialize code. Deserialization results when reading from a file are: category:status, path = category:status, polled = category:status, result = "ok" | "reject" When reading from STDIN are: category:status, path = "STDIN" category:status, result = "ok" | "reject" Status will fallback and run a normal status scan when a "reject" is reported (unless "--deserialize-wait=fail"). If "ok" is reported, status was able to use the status cache and avoid scanning the workdir. Additionally, a cmd_mode is emitted for each step: collection, deserialization, and serialization. For example, if deserialization is attempted and fails and status falls back to actually computing the status, a cmd_mode message containing "deserialize" is issued and then a cmd_mode for "collect" is issued. Also, if deserialization fails, a data message containing the rejection reason is emitted. Signed-off-by: Jeff Hostetler --- builtin/commit.c | 19 +++++++++++- wt-status-deserialize.c | 67 ++++++++++++++++++++++++++++++++++++++--- wt-status.h | 2 ++ 3 files changed, 83 insertions(+), 5 deletions(-) diff --git a/builtin/commit.c b/builtin/commit.c index 5ea4002e752f4e..87b11d0dcacb57 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -170,6 +170,7 @@ static int opt_parse_porcelain(const struct option *opt, const char *arg, int un static int do_serialize = 0; static char *serialize_path = NULL; +static int reject_implicit = 0; static int do_implicit_deserialize = 0; static int do_explicit_deserialize = 0; static char *deserialize_path = NULL; @@ -233,7 +234,7 @@ static int opt_parse_deserialize(const struct option *opt UNUSED, const char *ar } if (!deserialize_path || !*deserialize_path) do_explicit_deserialize = 1; /* read stdin */ - else if (access(deserialize_path, R_OK) == 0) + else if (wt_status_deserialize_access(deserialize_path, R_OK) == 0) do_explicit_deserialize = 1; /* can read from this file */ else { /* @@ -1659,6 +1660,8 @@ static int git_status_config(const char *k, const char *v, if (v && *v && access(v, R_OK) == 0) { do_implicit_deserialize = 1; deserialize_path = xstrdup(v); + } else { + reject_implicit = 1; } return 0; } @@ -1829,6 +1832,17 @@ struct repository *repo UNUSED) if (try_deserialize) goto skip_init; + /* + * If we implicitly received a status cache pathname from the config + * and the file does not exist, we silently reject it and do the normal + * status "collect". Fake up some trace2 messages to reflect this and + * assist post-processors know this case is different. + */ + if (!do_serialize && reject_implicit) { + trace2_cmd_mode("implicit-deserialize"); + trace2_data_string("status", the_repository, "deserialize/reject", + "status-cache/access"); + } enable_fscache(0); if (status_format != STATUS_FORMAT_PORCELAIN && @@ -1872,6 +1886,7 @@ struct repository *repo UNUSED) if (s.relative_paths) s.prefix = prefix; + trace2_cmd_mode("deserialize"); result = wt_status_deserialize(&s, deserialize_path, dw); if (result == DESERIALIZE_OK) return 0; @@ -1889,6 +1904,7 @@ struct repository *repo UNUSED) fd = -1; } + trace2_cmd_mode("collect"); wt_status_collect(&s); if (0 <= fd) @@ -1903,6 +1919,7 @@ struct repository *repo UNUSED) if (fd_serialize < 0) die_errno(_("could not serialize to '%s'"), serialize_path); + trace2_cmd_mode("serialize"); wt_status_serialize_v1(fd_serialize, &s); close(fd_serialize); } diff --git a/wt-status-deserialize.c b/wt-status-deserialize.c index 2224cdbfcfde6c..3f6b28b69f4a1c 100644 --- a/wt-status-deserialize.c +++ b/wt-status-deserialize.c @@ -9,6 +9,23 @@ #include "statinfo.h" #include "hex.h" #include "path.h" +#include "trace2.h" + +static void set_deserialize_reject_reason(const char *reason) +{ + trace2_data_string("status", the_repository, "deserialize/reject", + reason); +} + +int wt_status_deserialize_access(const char *path, int mode) +{ + int a = access(path, mode); + + if (a != 0) + set_deserialize_reject_reason("status-cache/access"); + + return a; +} static struct trace_key trace_deserialize = TRACE_KEY_INIT(DESERIALIZE); @@ -55,6 +72,7 @@ static int my_validate_index(const char *path, const struct cache_time *mtime_re struct cache_time mtime_observed_on_disk; if (lstat(path, &st)) { + set_deserialize_reject_reason("index/not-found"); trace_printf_key(&trace_deserialize, "could not stat index"); return DESERIALIZE_ERR; } @@ -62,6 +80,7 @@ static int my_validate_index(const char *path, const struct cache_time *mtime_re mtime_observed_on_disk.nsec = ST_MTIME_NSEC(st); if ((mtime_observed_on_disk.sec != mtime_reported->sec) || (mtime_observed_on_disk.nsec != mtime_reported->nsec)) { + set_deserialize_reject_reason("index/mtime-changed"); trace_printf_key(&trace_deserialize, "index mtime changed [des %d %d][obs %d %d]", mtime_reported->sec, mtime_reported->nsec, @@ -87,10 +106,12 @@ static int my_validate_excludes(const char *path, const char *key, const char *l r = (strcmp(line, sb.buf) ? DESERIALIZE_ERR : DESERIALIZE_OK); - if (r == DESERIALIZE_ERR) + if (r == DESERIALIZE_ERR) { + set_deserialize_reject_reason("excludes/changed"); trace_printf_key(&trace_deserialize, "%s changed [cached '%s'][observed '%s']", key, line, sb.buf); + } strbuf_release(&sb); return r; @@ -147,6 +168,7 @@ static int wt_deserialize_v1_header(struct wt_status *s, int fd) &index_mtime.sec, &index_mtime.nsec); if (nr_fields != 2) { + set_deserialize_reject_reason("v1-header/invalid-index-mtime"); trace_printf_key(&trace_deserialize, "invalid index_mtime (%d) '%s'", nr_fields, line); return DESERIALIZE_ERR; @@ -230,6 +252,7 @@ static int wt_deserialize_v1_header(struct wt_status *s, int fd) /* status_format */ if (skip_prefix(line, "sha1_commit ", &arg)) { if (get_oid_hex(arg, &s->oid_commit)) { + set_deserialize_reject_reason("v1-header/invalid-commit-sha"); trace_printf_key(&trace_deserialize, "invalid sha1_commit"); return DESERIALIZE_ERR; } @@ -245,19 +268,23 @@ static int wt_deserialize_v1_header(struct wt_status *s, int fd) } /* prefix */ + set_deserialize_reject_reason("v1-header/unexpected-line"); trace_printf_key(&trace_deserialize, "unexpected line '%s'", line); return DESERIALIZE_ERR; } if (!have_required_index_mtime) { + set_deserialize_reject_reason("v1-header/missing-index-mtime"); trace_printf_key(&trace_deserialize, "missing '%s'", "index_mtime"); return DESERIALIZE_ERR; } if (!have_required_core_excludes) { + set_deserialize_reject_reason("v1-header/missing-core-excludes"); trace_printf_key(&trace_deserialize, "missing '%s'", "core_excludes"); return DESERIALIZE_ERR; } if (!have_required_repo_excludes) { + set_deserialize_reject_reason("v1-header/missing-repo-excludes"); trace_printf_key(&trace_deserialize, "missing '%s'", "repo_excludes"); return DESERIALIZE_ERR; } @@ -344,6 +371,7 @@ static int wt_deserialize_v1_changed_items(const struct wt_status *cmd_s, * So we reject the status cache and let the fallback * code run. */ + set_deserialize_reject_reason("v1-data/unmerged"); trace_printf_key( &trace_deserialize, "reject: V2 format and umerged file: %s", @@ -485,6 +513,7 @@ static int wt_deserialize_v1(const struct wt_status *cmd_s, struct wt_status *s, * the serialized data */ if (validate_untracked_files_arg(cmd_s->show_untracked_files, &s->show_untracked_files, &untracked_strategy)) { + set_deserialize_reject_reason("args/untracked-files"); trace_printf_key(&trace_deserialize, "reject: show_untracked_file: command: %d, serialized : %d", cmd_s->show_untracked_files, s->show_untracked_files); @@ -492,6 +521,7 @@ static int wt_deserialize_v1(const struct wt_status *cmd_s, struct wt_status *s, } if (validate_ignored_files_arg(cmd_s->show_ignored_mode, s->show_ignored_mode, &ignored_strategy)) { + set_deserialize_reject_reason("args/ignored-mode"); trace_printf_key(&trace_deserialize, "reject: show_ignored_mode: command: %d, serialized: %d", cmd_s->show_ignored_mode, s->show_ignored_mode); @@ -525,6 +555,7 @@ static int wt_deserialize_v1(const struct wt_status *cmd_s, struct wt_status *s, return DESERIALIZE_ERR; continue; } + set_deserialize_reject_reason("v1-data/unexpected-line"); trace_printf_key(&trace_deserialize, "unexpected line '%s'", line); return DESERIALIZE_ERR; } @@ -544,6 +575,7 @@ static int wt_deserialize_parse(const struct wt_status *cmd_s, struct wt_status if (version == 1) return wt_deserialize_v1(cmd_s, s, fd); } + set_deserialize_reject_reason("status-cache/unsupported-version"); trace_printf_key(&trace_deserialize, "missing/unsupported version"); return DESERIALIZE_ERR; } @@ -564,6 +596,7 @@ static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *de * Check the path spec on the current command */ if (cmd_s->pathspec.nr > 1) { + set_deserialize_reject_reason("args/multiple-pathspecs"); trace_printf_key(&trace_deserialize, "reject: multiple pathspecs"); return DESERIALIZE_ERR; } @@ -574,6 +607,7 @@ static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *de */ if (cmd_s->pathspec.nr == 1 && my_strcmp_null(cmd_s->pathspec.items[0].match, "")) { + set_deserialize_reject_reason("args/root-pathspec"); trace_printf_key(&trace_deserialize, "reject: pathspec"); return DESERIALIZE_ERR; } @@ -596,20 +630,24 @@ static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *de * or "--ignored" settings). */ if (cmd_s->is_initial != des_s->is_initial) { + set_deserialize_reject_reason("args/is-initial-changed"); trace_printf_key(&trace_deserialize, "reject: is_initial"); return DESERIALIZE_ERR; } if (my_strcmp_null(cmd_s->branch, des_s->branch)) { + set_deserialize_reject_reason("args/branch-changed"); trace_printf_key(&trace_deserialize, "reject: branch"); return DESERIALIZE_ERR; } if (my_strcmp_null(cmd_s->reference, des_s->reference)) { + set_deserialize_reject_reason("args/reference-changed"); trace_printf_key(&trace_deserialize, "reject: reference"); return DESERIALIZE_ERR; } /* verbose */ /* amend */ if (cmd_s->whence != des_s->whence) { + set_deserialize_reject_reason("args/whence-changed"); trace_printf_key(&trace_deserialize, "reject: whence"); return DESERIALIZE_ERR; } @@ -643,19 +681,23 @@ static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *de /* hints */ /* ahead_behind_flags */ if (cmd_s->detect_rename != des_s->detect_rename) { + set_deserialize_reject_reason("args/detect-rename-changed"); trace_printf_key(&trace_deserialize, "reject: detect_rename"); return DESERIALIZE_ERR; } if (cmd_s->rename_score != des_s->rename_score) { + set_deserialize_reject_reason("args/rename-score-changed"); trace_printf_key(&trace_deserialize, "reject: rename_score"); return DESERIALIZE_ERR; } if (cmd_s->rename_limit != des_s->rename_limit) { + set_deserialize_reject_reason("args/rename-limit-changed"); trace_printf_key(&trace_deserialize, "reject: rename_limit"); return DESERIALIZE_ERR; } /* status_format */ if (!oideq(&cmd_s->oid_commit, &des_s->oid_commit)) { + set_deserialize_reject_reason("args/commit-changed"); trace_printf_key(&trace_deserialize, "reject: sha1_commit"); return DESERIALIZE_ERR; } @@ -743,15 +785,18 @@ static int try_deserialize_read_from_file(const struct wt_status *cmd_s, enum wt_status_deserialize_wait dw, struct wt_status *des_s) { - int k, limit; + int k = 0; + int limit; int result = DESERIALIZE_ERR; /* * For "fail" or "no", try exactly once to read the status cache. * Return an error if the file is stale. */ - if (dw == DESERIALIZE_WAIT__FAIL || dw == DESERIALIZE_WAIT__NO) - return try_deserialize_read_from_file_1(cmd_s, path, des_s); + if (dw == DESERIALIZE_WAIT__FAIL || dw == DESERIALIZE_WAIT__NO) { + result = try_deserialize_read_from_file_1(cmd_s, path, des_s); + goto done; + } /* * Wait for the status cache file to refresh. Wait duration can @@ -776,6 +821,12 @@ static int try_deserialize_read_from_file(const struct wt_status *cmd_s, sleep_millisec(100); } +done: + trace2_data_string("status", the_repository, "deserialize/path", path); + trace2_data_intmax("status", the_repository, "deserialize/polled", k); + trace2_data_string("status", the_repository, "deserialize/result", + ((result == DESERIALIZE_OK) ? "ok" : "reject")); + trace_printf_key(&trace_deserialize, "wait polled=%d result=%d '%s'", k, result, path); @@ -802,6 +853,8 @@ int wt_status_deserialize(const struct wt_status *cmd_s, int result; struct string_list_item *change; + trace2_region_enter("status", "deserialize", the_repository); + if (path && *path && strcmp(path, "0")) { result = try_deserialize_read_from_file(cmd_s, path, dw, &des_s); } else { @@ -812,8 +865,14 @@ int wt_status_deserialize(const struct wt_status *cmd_s, * term, since we cannot read stdin multiple times. */ result = wt_deserialize_fd(cmd_s, &des_s, 0); + + trace2_data_string("status", the_repository, "deserialize/path", "STDIN"); + trace2_data_string("status", the_repository, "deserialize/result", + ((result == DESERIALIZE_OK) ? "ok" : "reject")); } + trace2_region_leave("status", "deserialize", the_repository); + if (result == DESERIALIZE_OK) { wt_status_get_state(cmd_s->repo, &des_s.state, des_s.branch && !strcmp(des_s.branch, "HEAD")); diff --git a/wt-status.h b/wt-status.h index 95154c79994bc4..eae408eff54912 100644 --- a/wt-status.h +++ b/wt-status.h @@ -255,6 +255,8 @@ int wt_status_deserialize(const struct wt_status *cmd_s, const char *path, enum wt_status_deserialize_wait dw); +int wt_status_deserialize_access(const char *path, int mode); + /* * A helper routine for serialize and deserialize to compute * metadata for the user-global and repo-local excludes files. From f27b3084a938c38860394352ac8903fff316326e Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 22 Feb 2017 12:50:43 -0700 Subject: [PATCH 078/289] sparse-checkout: make sure to update files with a modify/delete conflict When using the sparse-checkout feature, the file might not be on disk because the skip-worktree bit is on. This used to be a bug in the (hence deleted) `recursive` strategy. Let's ensure that this bug does not resurface. Signed-off-by: Kevin Willford Signed-off-by: Johannes Schindelin --- t/meson.build | 1 + t/t7616-merge-sparse-checkout.sh | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100755 t/t7616-merge-sparse-checkout.sh diff --git a/t/meson.build b/t/meson.build index a9102748c73886..392809fd278025 100644 --- a/t/meson.build +++ b/t/meson.build @@ -973,6 +973,7 @@ integration_tests = [ 't7612-merge-verify-signatures.sh', 't7614-merge-signoff.sh', 't7615-diff-algo-with-mergy-operations.sh', + 't7616-merge-sparse-checkout.sh', 't7700-repack.sh', 't7701-repack-unpack-unreachable.sh', 't7702-repack-cyclic-alternate.sh', diff --git a/t/t7616-merge-sparse-checkout.sh b/t/t7616-merge-sparse-checkout.sh new file mode 100755 index 00000000000000..5ce12431f62ad1 --- /dev/null +++ b/t/t7616-merge-sparse-checkout.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +test_description='merge can handle sparse-checkout' + +. ./test-lib.sh + +# merges with conflicts + +test_expect_success 'setup' ' + git branch -M main && + test_commit a && + test_commit file && + git checkout -b delete-file && + git rm file.t && + test_tick && + git commit -m "remove file" && + git checkout main && + test_commit modify file.t changed +' + +test_expect_success 'merge conflict deleted file and modified' ' + echo "/a.t" >.git/info/sparse-checkout && + test_config core.sparsecheckout true && + git checkout -f && + test_path_is_missing file.t && + test_must_fail git merge delete-file && + test_path_is_file file.t && + test "changed" = "$(cat file.t)" +' + +test_done From 3ab1d758886fbf058a94bffd225614b837342e02 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 2 Feb 2018 14:17:05 -0500 Subject: [PATCH 079/289] status: serialize to path Teach status serialization to take an optional pathname on the command line to direct that cache data be written there rather than to stdout. When used this way, normal status results will still be written to stdout. When no path is given, only binary serialization data is written to stdout. Usage: git status --serialize[=] Signed-off-by: Jeff Hostetler --- Documentation/git-status.adoc | 10 ++++++---- builtin/commit.c | 36 ++++++++++++++++++++++++++--------- t/t7522-serialized-status.sh | 35 ++++++++++++++++++++++++++++++++++ wt-status-serialize.c | 5 ++--- wt-status.c | 2 +- wt-status.h | 2 +- 6 files changed, 72 insertions(+), 18 deletions(-) diff --git a/Documentation/git-status.adoc b/Documentation/git-status.adoc index 8c29491cfe6661..c24e2edcc7f76e 100644 --- a/Documentation/git-status.adoc +++ b/Documentation/git-status.adoc @@ -154,10 +154,12 @@ ignored, then the directory is not shown, but all contents are shown. threshold. See also linkgit:git-diff[1] `--find-renames`. -`--serialize[=]`:: - (EXPERIMENTAL) Serialize raw status results to stdout in a - format suitable for use by `--deserialize`. Valid values for - `` are "1" and "v1". +`--serialize[=]`:: + (EXPERIMENTAL) Serialize raw status results to a file or stdout + in a format suitable for use by `--deserialize`. If a path is + given, serialize data will be written to that path *and* normal + status output will be written to stdout. If path is omitted, + only binary serialization data will be written to stdout. `--deserialize[=]`:: (EXPERIMENTAL) Deserialize raw status results from a file or diff --git a/builtin/commit.c b/builtin/commit.c index 773af3d36f6530..84668f1d7c708f 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -167,26 +167,34 @@ static int opt_parse_porcelain(const struct option *opt, const char *arg, int un } static int do_serialize = 0; +static char *serialize_path = NULL; + static int do_implicit_deserialize = 0; static int do_explicit_deserialize = 0; static char *deserialize_path = NULL; /* - * --serialize | --serialize=1 | --serialize=v1 + * --serialize | --serialize= + * + * Request that we serialize status output rather than or in addition to + * printing in any of the established formats. + * + * Without a path, we write binary serialization data to stdout (and omit + * the normal status output). * - * Request that we serialize our output rather than printing in - * any of the established formats. Optionally specify serialization - * version. + * With a path, we write binary serialization data to the and then + * write normal status output. */ static int opt_parse_serialize(const struct option *opt, const char *arg, int unset) { enum wt_status_format *value = (enum wt_status_format *)opt->value; if (unset || !arg) *value = STATUS_FORMAT_SERIALIZE_V1; - else if (!strcmp(arg, "v1") || !strcmp(arg, "1")) - *value = STATUS_FORMAT_SERIALIZE_V1; - else - die("unsupported serialize version '%s'", arg); + + if (arg) { + free(serialize_path); + serialize_path = xstrdup(arg); + } if (do_explicit_deserialize) die("cannot mix --serialize and --deserialize"); @@ -1666,7 +1674,7 @@ struct repository *repo UNUSED) N_("version"), N_("machine-readable output"), PARSE_OPT_OPTARG, opt_parse_porcelain), OPT_CALLBACK_F(0, "serialize", &status_format, - N_("version"), N_("serialize raw status data to stdout"), + N_("path"), N_("serialize raw status data to path or stdout"), PARSE_OPT_OPTARG | PARSE_OPT_NONEG, opt_parse_serialize), OPT_CALLBACK_F(0, "deserialize", NULL, N_("path"), N_("deserialize raw status data from file"), @@ -1809,6 +1817,16 @@ struct repository *repo UNUSED) if (s.relative_paths) s.prefix = prefix; + if (serialize_path) { + int fd_serialize = xopen(serialize_path, + O_WRONLY | O_CREAT | O_TRUNC, 0666); + if (fd_serialize < 0) + die_errno(_("could not serialize to '%s'"), + serialize_path); + wt_status_serialize_v1(fd_serialize, &s); + close(fd_serialize); + } + wt_status_print(&s); wt_status_collect_free_buffers(&s); diff --git a/t/t7522-serialized-status.sh b/t/t7522-serialized-status.sh index c4c14fe2f71d79..2a95c84b770e1f 100755 --- a/t/t7522-serialized-status.sh +++ b/t/t7522-serialized-status.sh @@ -173,4 +173,39 @@ test_expect_success 'verify no-ahead-behind and serialized status integration' ' test_cmp expect output ' +test_expect_success 'verify new --serialize=path mode' ' + #test_when_finished "rm serialized_status.dat expect new_change.txt output.1 output.2" && + cat >expect <<-\EOF && + ? expect + ? output.1 + ? untracked/ + ? untracked_1.txt + EOF + + git checkout -b serialize_path_branch main --track >/dev/null && + touch alt_branch_changes.txt && + git add alt_branch_changes.txt && + test_tick && + git commit -m"New commit on serialize_path_branch" && + + git status --porcelain=v2 --serialize=serialized_status.dat >output.1 && + touch new_change.txt && + + git status --porcelain=v2 --deserialize=serialized_status.dat >output.2 && + test_cmp expect output.1 && + test_cmp expect output.2 +' + +test_expect_success 'renames' ' + git init -b main rename_test && + echo OLDNAME >rename_test/OLDNAME && + git -C rename_test add OLDNAME && + git -C rename_test commit -m OLDNAME && + git -C rename_test mv OLDNAME NEWNAME && + git -C rename_test status --serialize=renamed.dat >output.1 && + echo DIRT >rename_test/DIRT && + git -C rename_test status --deserialize=renamed.dat >output.2 && + test_cmp output.1 output.2 +' + test_done diff --git a/wt-status-serialize.c b/wt-status-serialize.c index b7102ce80baf50..179e76f8081fdf 100644 --- a/wt-status-serialize.c +++ b/wt-status-serialize.c @@ -169,7 +169,7 @@ static inline void wt_serialize_v1_ignored(struct wt_status *s UNUSED, int fd, } /* - * Serialize the list of changes to stdout. The goal of this + * Serialize the list of changes to the given file. The goal of this * is to just serialize the key fields in wt_status so that a * later command can rebuilt it and do the printing. * @@ -178,9 +178,8 @@ static inline void wt_serialize_v1_ignored(struct wt_status *s UNUSED, int fd, * is relatively quick for the status consumer to compute * as necessary. */ -void wt_status_serialize_v1(struct wt_status *s) +void wt_status_serialize_v1(int fd, struct wt_status *s) { - int fd = 1; /* we always write to stdout */ struct string_list_item *iter; size_t k; diff --git a/wt-status.c b/wt-status.c index 7f15ebf4eeb93c..aae1e9e8055dc4 100644 --- a/wt-status.c +++ b/wt-status.c @@ -2761,7 +2761,7 @@ void wt_status_print(struct wt_status *s) wt_longstatus_print(s); break; case STATUS_FORMAT_SERIALIZE_V1: - wt_status_serialize_v1(s); + wt_status_serialize_v1(1, s); break; } diff --git a/wt-status.h b/wt-status.h index f5ac9e5c656f8f..e7a19f0eb3f993 100644 --- a/wt-status.h +++ b/wt-status.h @@ -233,7 +233,7 @@ struct wt_status_serialize_data * Serialize computed status scan results using "version 1" format * to the given file. */ -void wt_status_serialize_v1(struct wt_status *s); +void wt_status_serialize_v1(int fd, struct wt_status *s); /* * Deserialize existing status results from the given file and From 6c3dfbdd93dd77167c8c9c99850e350d53918f4b Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 7 Jan 2019 12:45:48 -0500 Subject: [PATCH 080/289] gvfs:trace2:data: status serialization Add trace information around status serialization. Signed-off-by: Jeff Hostetler --- wt-status-serialize.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/wt-status-serialize.c b/wt-status-serialize.c index 4a763725a2e685..c26cd49a213b9c 100644 --- a/wt-status-serialize.c +++ b/wt-status-serialize.c @@ -9,6 +9,7 @@ #include "trace.h" #include "read-cache-ll.h" #include "path.h" +#include "trace2.h" static struct trace_key trace_serialize = TRACE_KEY_INIT(SERIALIZE); @@ -305,6 +306,8 @@ void wt_status_serialize_v1(int fd, struct wt_status *s) struct string_list_item *iter; size_t k; + trace2_region_enter("status", "serialize", the_repository); + /* * version header must be first line. */ @@ -338,4 +341,6 @@ void wt_status_serialize_v1(int fd, struct wt_status *s) } packet_flush(fd); } + + trace2_region_leave("status", "serialize", the_repository); } From 5f08216ab8d5fed6da41395f96a904715c3075ec Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 30 Sep 2022 12:59:40 -0400 Subject: [PATCH 081/289] worktree: allow in Scalar repositories The 'git worktree' command was marked as BLOCK_ON_GVFS_REPO because it does not interact well with the virtual filesystem of VFS for Git. When a Scalar clone uses the GVFS protocol, it enables the GVFS_BLOCK_COMMANDS flag, since commands like 'git gc' do not work well with the GVFS protocol. However, 'git worktree' works just fine with the GVFS protocol since it isn't doing anything special. It copies the sparse-checkout from the current worktree, so it does not have performance issues. This is a highly requested option. The solution is to stop using the BLOCK_ON_GVFS_REPO option and instead add a special-case check in cmd_worktree() specifically for a particular bit of the 'core_gvfs' global variable (loaded by very early config reading) that corresponds to the virtual filesystem. The bit that most closely resembled this behavior was non-obviously named, but does provide a signal that we are in a Scalar clone and not a VFS for Git clone. The error message is copied from git.c, so it will have the same output as before if a user runs this in a VFS for Git clone. Signed-off-by: Derrick Stolee --- builtin/worktree.c | 8 ++++++++ gvfs.h | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/builtin/worktree.c b/builtin/worktree.c index 77ecd0f71f4247..b1ab1190d88449 100644 --- a/builtin/worktree.c +++ b/builtin/worktree.c @@ -4,6 +4,7 @@ #include "builtin.h" #include "abspath.h" #include "advice.h" +#include "gvfs.h" #include "checkout.h" #include "config.h" #include "copy.h" @@ -1529,6 +1530,13 @@ int cmd_worktree(int ac, repo_config(the_repository, git_worktree_config, NULL); + /* + * git-worktree is special-cased to work in Scalar repositories + * even when they use the GVFS Protocol. + */ + if (gvfs_config_is_set(repo, GVFS_USE_VIRTUAL_FILESYSTEM)) + die("'git %s' is not supported on a GVFS repo", "worktree"); + if (!prefix) prefix = ""; diff --git a/gvfs.h b/gvfs.h index e3995b586cddd5..59ee3893b1efa6 100644 --- a/gvfs.h +++ b/gvfs.h @@ -13,7 +13,18 @@ struct repository; */ #define GVFS_SKIP_SHA_ON_INDEX (1 << 0) #define GVFS_MISSING_OK (1 << 2) + +/* + * This behavior of not deleting outside of the sparse-checkout + * is specific to the virtual filesystem support. It is only + * enabled by VFS for Git, and so can be used as an indicator + * that we are in a virtualized filesystem environment and not + * in a Scalar environment. This bit has two names to reflect + * that. + */ #define GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT (1 << 3) +#define GVFS_USE_VIRTUAL_FILESYSTEM (1 << 3) + #define GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK (1 << 4) #define GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS (1 << 6) From 9be55c95140d9d30088180b26f6a08939ab69186 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 7 Feb 2018 10:59:03 -0500 Subject: [PATCH 082/289] status: reject deserialize in V2 and conflicts Teach status deserialize code to reject status cache when printing in porcelain V2 and there are unresolved conflicts in the cache file. A follow-on task might extend the cache format to include this additiona data. See code for longer explanation. Signed-off-by: Jeff Hostetler --- t/t7522-serialized-status.sh | 90 +++++++++++++++++++++++++++++++++++- wt-status-deserialize.c | 28 ++++++++++- 2 files changed, 115 insertions(+), 3 deletions(-) diff --git a/t/t7522-serialized-status.sh b/t/t7522-serialized-status.sh index 2a95c84b770e1f..06be656dccf475 100755 --- a/t/t7522-serialized-status.sh +++ b/t/t7522-serialized-status.sh @@ -51,8 +51,14 @@ test_expect_success 'setup' ' test_oid_cache <<-EOF branch_oid sha1:68d4a437ea4c2de65800f48c053d4d543b55c410 + x_base sha1:587be6b4c3f93f93c489c0111bba5596147a26cb + x_ours sha1:b68025345d5301abad4d9ec9166f455243a0d746 + x_theirs sha1:975fbec8256d3e8a3797e7a3611380f27c49f4ac branch_oid sha256:6b95e4b1ea911dad213f2020840f5e92d3066cf9e38cf35f79412ec58d409ce4 + x_base sha256:14f5162e2fe3d240d0d37aaab0f90e4af9a7cfa79639f3bab005b5bfb4174d9f + x_ours sha256:3a404ba030a4afa912155c476a48a253d4b3a43d0098431b6d6ca6e554bd78fb + x_theirs sha256:44dc634218adec09e34f37839b3840bad8c6103693e9216626b32d00e093fa35 EOF ' @@ -174,7 +180,7 @@ test_expect_success 'verify no-ahead-behind and serialized status integration' ' ' test_expect_success 'verify new --serialize=path mode' ' - #test_when_finished "rm serialized_status.dat expect new_change.txt output.1 output.2" && + test_when_finished "rm serialized_status.dat expect new_change.txt output.1 output.2" && cat >expect <<-\EOF && ? expect ? output.1 @@ -196,6 +202,88 @@ test_expect_success 'verify new --serialize=path mode' ' test_cmp expect output.2 ' +test_expect_success 'merge conflicts' ' + + # create a merge conflict. + + git init -b main conflicts && + echo x >conflicts/x.txt && + git -C conflicts add x.txt && + git -C conflicts commit -m x && + git -C conflicts branch a && + git -C conflicts branch b && + git -C conflicts checkout a && + echo y >conflicts/x.txt && + git -C conflicts add x.txt && + git -C conflicts commit -m a && + git -C conflicts checkout b && + echo z >conflicts/x.txt && + git -C conflicts add x.txt && + git -C conflicts commit -m b && + test_must_fail git -C conflicts merge --no-commit a && + + # verify that regular status correctly identifies it + # in each format. + + cat >expect.v2 <observed.v2 && + test_cmp expect.v2 observed.v2 && + + cat >expect.long <..." to mark resolution) + both modified: x.txt + +no changes added to commit (use "git add" and/or "git commit -a") +EOF + git -C conflicts status --long >observed.long && + test_cmp expect.long observed.long && + + cat >expect.short <observed.short && + test_cmp expect.short observed.short && + + # save status data in serialized cache. + + git -C conflicts status --serialize >serialized && + + # make some dirt in the worktree so we can tell whether subsequent + # status commands used the cached data or did a fresh status. + + echo dirt >conflicts/dirt.txt && + + # run status using the cached data. + + git -C conflicts status --long --deserialize=../serialized >observed.long && + test_cmp expect.long observed.long && + + git -C conflicts status --short --deserialize=../serialized >observed.short && + test_cmp expect.short observed.short && + + # currently, the cached data does not have enough information about + # merge conflicts for porcelain V2 format. (And V2 format looks at + # the index to get that data, but the whole point of the serialization + # is to avoid reading the index unnecessarily.) So V2 always rejects + # the cached data when there is an unresolved conflict. + + cat >expect.v2.dirty <observed.v2 && + test_cmp expect.v2.dirty observed.v2 + +' + test_expect_success 'renames' ' git init -b main rename_test && echo OLDNAME >rename_test/OLDNAME && diff --git a/wt-status-deserialize.c b/wt-status-deserialize.c index 52e3958bcf2eba..293b9a71ecb99d 100644 --- a/wt-status-deserialize.c +++ b/wt-status-deserialize.c @@ -181,7 +181,8 @@ static int wt_deserialize_v1_header(struct wt_status *s, int fd) /* * Build a string-list of (count) lines from the input. */ -static int wt_deserialize_v1_changed_items(struct wt_status *s, int fd, int count UNUSED) +static int wt_deserialize_v1_changed_items(const struct wt_status *cmd_s, + struct wt_status *s, int fd, int count UNUSED) { struct wt_status_serialize_data *sd; char *p; @@ -239,6 +240,29 @@ static int wt_deserialize_v1_changed_items(struct wt_status *s, int fd, int coun oid_to_hex(&d->oid_index), item->string, (d->rename_source ? d->rename_source : "")); + + if (d->stagemask && + cmd_s->status_format == STATUS_FORMAT_PORCELAIN_V2) { + /* + * We have an unresolved conflict and the user wants + * to see porcelain V2 output. The cached status data + * does not contain enough information for V2 (because + * the main status computation does not capture it). + * We only get a single change record for the file with + * a single SHA -- we don't get the stage [123] mode + * and SHA data. The V2 detail-line print code looks + * up this information directly from the index. The + * whole point of this serialization cache is to avoid + * reading the index, so the V2 print code gets zeros. + * So we reject the status cache and let the fallback + * code run. + */ + trace_printf_key( + &trace_deserialize, + "reject: V2 format and umerged file: %s", + item->string); + return DESERIALIZE_ERR; + } } return DESERIALIZE_OK; @@ -391,7 +415,7 @@ static int wt_deserialize_v1(const struct wt_status *cmd_s, struct wt_status *s, while ((line = my_packet_read_line(fd, &line_len))) { if (skip_prefix(line, "changed ", &arg)) { nr_changed = (int)strtol(arg, NULL, 10); - if (wt_deserialize_v1_changed_items(s, fd, nr_changed) + if (wt_deserialize_v1_changed_items(cmd_s, s, fd, nr_changed) == DESERIALIZE_ERR) return DESERIALIZE_ERR; continue; From b5d3600d7a29484623edbcf8d67a421bc2781baa Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 19 Nov 2018 16:26:37 -0500 Subject: [PATCH 083/289] gvfs:trace2:data: add vfs stats Report virtual filesystem summary data. Signed-off-by: Jeff Hostetler --- virtualfilesystem.c | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/virtualfilesystem.c b/virtualfilesystem.c index 450584b106fb7b..947b1516417fb3 100644 --- a/virtualfilesystem.c +++ b/virtualfilesystem.c @@ -3,6 +3,7 @@ #include "git-compat-util.h" #include "environment.h" #include "gettext.h" +#include "trace2.h" #include "config.h" #include "dir.h" #include "hashmap.h" @@ -258,6 +259,11 @@ void apply_virtualfilesystem(struct index_state *istate) { char *buf, *entry; size_t i; + int nr_unknown = 0; + int nr_vfs_dirs = 0; + int nr_vfs_rows = 0; + int nr_bulk_skip = 0; + int nr_explicit_skip = 0; if (!repo_config_get_virtualfilesystem(istate->repo)) return; @@ -275,16 +281,21 @@ void apply_virtualfilesystem(struct index_state *istate) if (buf[i] == '\0') { ssize_t pos, len; + nr_vfs_rows++; + len = buf + i - entry; /* look for a directory wild card (ie "dir1/") */ if (buf[i - 1] == '/') { + nr_vfs_dirs++; if (repo_ignore_case(the_repository)) adjust_dirname_case(istate, entry); pos = index_name_pos(istate, entry, len); if (pos < 0) { pos = -pos - 1; while ((size_t)pos < istate->cache_nr && !fspathncmp(istate->cache[pos]->name, entry, len)) { + if (istate->cache[pos]->ce_flags & CE_SKIP_WORKTREE) + nr_bulk_skip++; istate->cache[pos]->ce_flags &= ~CE_SKIP_WORKTREE; pos++; } @@ -292,18 +303,41 @@ void apply_virtualfilesystem(struct index_state *istate) } else { if (repo_ignore_case(the_repository)) { struct cache_entry *ce = index_file_exists(istate, entry, len, repo_ignore_case(the_repository)); - if (ce) + if (ce) { + if (ce->ce_flags & CE_SKIP_WORKTREE) + nr_explicit_skip++; ce->ce_flags &= ~CE_SKIP_WORKTREE; + } + else { + nr_unknown++; + } } else { int pos = index_name_pos(istate, entry, len); - if (pos >= 0) + if (pos >= 0) { + if (istate->cache[pos]->ce_flags & CE_SKIP_WORKTREE) + nr_explicit_skip++; istate->cache[pos]->ce_flags &= ~CE_SKIP_WORKTREE; + } + else { + nr_unknown++; + } } } entry += len + 1; } } + + if (nr_vfs_rows > 0) { + trace2_data_intmax("vfs", the_repository, "apply/tracked", nr_bulk_skip + nr_explicit_skip); + + trace2_data_intmax("vfs", the_repository, "apply/vfs_rows", nr_vfs_rows); + trace2_data_intmax("vfs", the_repository, "apply/vfs_dirs", nr_vfs_dirs); + + trace2_data_intmax("vfs", the_repository, "apply/nr_unknown", nr_unknown); + trace2_data_intmax("vfs", the_repository, "apply/nr_bulk_skip", nr_bulk_skip); + trace2_data_intmax("vfs", the_repository, "apply/nr_explicit_skip", nr_explicit_skip); + } } /* From e638b16e48e15184697285e43c0a8a6bb34d7fde Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 1 Mar 2017 15:17:12 -0800 Subject: [PATCH 084/289] sparse-checkout: avoid writing entries with the skip-worktree bit When using the sparse-checkout feature git should not write to the working directory for files with the skip-worktree bit on. With the skip-worktree bit on the file may or may not be in the working directory and if it is not we don't want or need to create it by calling checkout_entry. There are two callers of checkout_target. Both of which check that the file does not exist before calling checkout_target. load_current which make a call to lstat right before calling checkout_target and check_preimage which will only run checkout_taret it stat_ret is less than zero. It sets stat_ret to zero and only if !stat->cached will it lstat the file and set stat_ret to something other than zero. This patch checks if skip-worktree bit is on in checkout_target and just returns so that the entry doesn't not end up in the working directory. This is so that apply will not create a file in the working directory, then update the index but not keep the working directory up to date with the changes that happened in the index. Signed-off-by: Kevin Willford --- apply.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/apply.c b/apply.c index f7f9f21809df5f..19a7ab55af0900 100644 --- a/apply.c +++ b/apply.c @@ -20,6 +20,7 @@ #include "dir.h" #include "environment.h" #include "gettext.h" +#include "gvfs.h" #include "hex.h" #include "xdiff-interface.h" #include "merge-ll.h" @@ -3499,6 +3500,25 @@ static int checkout_target(struct index_state *istate, { struct checkout costate = CHECKOUT_INIT; + /* + * Do not checkout the entry if the skipworktree bit is set + * + * Both callers of this method (check_preimage and load_current) + * check for the existance of the file before calling this + * method so we know that the file doesn't exist at this point + * and we don't need to perform that check again here. + * We just need to check the skip-worktree and return. + * + * This is to prevent git from creating a file in the + * working directory that has the skip-worktree bit on, + * then updating the index from the patch and not keeping + * the working directory version up to date with what it + * changed the index version to be. + */ + if (gvfs_config_is_set(istate->repo, GVFS_USE_VIRTUAL_FILESYSTEM) && + ce_skip_worktree(ce)) + return 0; + costate.refresh_cache = 1; costate.istate = istate; if (checkout_entry(ce, &costate, NULL, NULL) || From 0b0ec59cb1f62e1ff6ae64a42af39d677eaddc47 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 20 Jul 2018 12:08:50 -0400 Subject: [PATCH 085/289] serialize-status: serialize global and repo-local exclude file metadata Changes to the global or repo-local excludes files can change the results returned by "git status" for untracked files. Therefore, it is important that the exclude-file values used during serialization are still current at the time of deserialization. Teach "git status --serialize" to report metadata on the user's global exclude file (which defaults to "$XDG_HOME/git/ignore") and for the repo-local excludes file (which is in ".git/info/excludes"). Serialize will record the pathnames and mtimes for these files in the serialization header (next to the mtime data for the .git/index file). Teach "git status --deserialize" to validate this new metadata. If either exclude file has changed since the serialization-cache-file was written, then deserialize will reject the cache file and force a full/normal status run. Signed-off-by: Jeff Hostetler --- wt-status-deserialize.c | 86 ++++++++++++++++++++++++++++ wt-status-serialize.c | 122 ++++++++++++++++++++++++++++++++++++++++ wt-status.h | 8 +++ 3 files changed, 216 insertions(+) diff --git a/wt-status-deserialize.c b/wt-status-deserialize.c index 293b9a71ecb99d..e1715fdce5b0b0 100644 --- a/wt-status-deserialize.c +++ b/wt-status-deserialize.c @@ -8,6 +8,7 @@ #include "trace.h" #include "statinfo.h" #include "hex.h" +#include "path.h" static struct trace_key trace_deserialize = TRACE_KEY_INIT(DESERIALIZE); @@ -70,12 +71,70 @@ static int my_validate_index(const char *path, const struct cache_time *mtime_re return DESERIALIZE_OK; } +/* + * Use the given key and exclude pathname to compute a serialization header + * reflecting the current contents on disk. See if that matches the value + * computed for this key when the cache was written. Reject the cache if + * anything has changed. + */ +static int my_validate_excludes(const char *path, const char *key, const char *line) +{ + struct strbuf sb = STRBUF_INIT; + int r; + + wt_serialize_compute_exclude_header(&sb, key, path); + + r = (strcmp(line, sb.buf) ? DESERIALIZE_ERR : DESERIALIZE_OK); + + if (r == DESERIALIZE_ERR) + trace_printf_key(&trace_deserialize, + "%s changed [cached '%s'][observed '%s']", + key, line, sb.buf); + + strbuf_release(&sb); + return r; +} + +static int my_parse_core_excludes(const char *line) +{ + /* + * In dir.c:setup_standard_excludes() they use either the value of + * the "core.excludefile" variable (stored in the global "excludes_file" + * variable) -or- the default value "$XDG_HOME/git/ignore". This is done + * during wt_status_collect_untracked() which we are hoping to not call. + * + * Fake the setup here. + */ + + if (repo_excludes_file(the_repository)) { + return my_validate_excludes(repo_excludes_file(the_repository), + "core_excludes", line); + } else { + char *path = xdg_config_home("ignore"); + int r = my_validate_excludes(path, "core_excludes", line); + free(path); + return r; + } +} + +static int my_parse_repo_excludes(const char *line) +{ + char *path = repo_git_path(the_repository, "info/exclude"); + int r = my_validate_excludes(path, "repo_excludes", line); + free(path); + + return r; +} + static int wt_deserialize_v1_header(struct wt_status *s, int fd) { struct cache_time index_mtime; int line_len, nr_fields; const char *line; const char *arg; + int have_required_index_mtime = 0; + int have_required_core_excludes = 0; + int have_required_repo_excludes = 0; /* * parse header lines up to the first flush packet. @@ -91,6 +150,20 @@ static int wt_deserialize_v1_header(struct wt_status *s, int fd) nr_fields, line); return DESERIALIZE_ERR; } + have_required_index_mtime = 1; + continue; + } + + if (skip_prefix(line, "core_excludes ", &arg)) { + if (my_parse_core_excludes(line) != DESERIALIZE_OK) + return DESERIALIZE_ERR; + have_required_core_excludes = 1; + continue; + } + if (skip_prefix(line, "repo_excludes ", &arg)) { + if (my_parse_repo_excludes(line) != DESERIALIZE_OK) + return DESERIALIZE_ERR; + have_required_repo_excludes = 1; continue; } @@ -175,6 +248,19 @@ static int wt_deserialize_v1_header(struct wt_status *s, int fd) return DESERIALIZE_ERR; } + if (!have_required_index_mtime) { + trace_printf_key(&trace_deserialize, "missing '%s'", "index_mtime"); + return DESERIALIZE_ERR; + } + if (!have_required_core_excludes) { + trace_printf_key(&trace_deserialize, "missing '%s'", "core_excludes"); + return DESERIALIZE_ERR; + } + if (!have_required_repo_excludes) { + trace_printf_key(&trace_deserialize, "missing '%s'", "repo_excludes"); + return DESERIALIZE_ERR; + } + return my_validate_index(s->index_file, &index_mtime); } diff --git a/wt-status-serialize.c b/wt-status-serialize.c index 179e76f8081fdf..4a763725a2e685 100644 --- a/wt-status-serialize.c +++ b/wt-status-serialize.c @@ -1,13 +1,133 @@ +#define USE_THE_REPOSITORY_VARIABLE + #include "git-compat-util.h" +#include "environment.h" #include "hex.h" #include "repository.h" #include "wt-status.h" #include "pkt-line.h" #include "trace.h" #include "read-cache-ll.h" +#include "path.h" static struct trace_key trace_serialize = TRACE_KEY_INIT(SERIALIZE); +/* + * Compute header record for exclude file using format: + * SP SP LF + */ +void wt_serialize_compute_exclude_header(struct strbuf *sb, + const char *key, + const char *path) +{ + struct stat st; + struct stat_data sd; + + memset(&sd, 0, sizeof(sd)); + + strbuf_setlen(sb, 0); + + if (!path || !*path) { + strbuf_addf(sb, "%s U (unset)", key); + } else if (lstat(path, &st) == -1) { + if (is_missing_file_error(errno)) + strbuf_addf(sb, "%s E (not-found) %s", key, path); + else + strbuf_addf(sb, "%s E (other) %s", key, path); + } else { + fill_stat_data(&sd, &st); + strbuf_addf(sb, "%s F %d %d %s", + key, sd.sd_mtime.sec, sd.sd_mtime.nsec, path); + } +} + +static void append_exclude_info(int fd, const char *path, const char *key) +{ + struct strbuf sb = STRBUF_INIT; + + wt_serialize_compute_exclude_header(&sb, key, path); + + packet_write_fmt(fd, "%s\n", sb.buf); + + strbuf_release(&sb); +} + +static void append_core_excludes_file_info(int fd) +{ + /* + * Write pathname and mtime of the core/global excludes file to + * the status cache header. Since a change in the global excludes + * will/may change the results reported by status, the deserialize + * code should be able to reject the status cache if the excludes + * file changes since when the cache was written. + * + * The "core.excludefile" setting defaults to $XDG_HOME/git/ignore + * and uses a global variable which should have been set during + * wt_status_collect_untracked(). + * + * See dir.c:setup_standard_excludes() + */ + append_exclude_info(fd, repo_excludes_file(the_repository), "core_excludes"); +} + +static void append_repo_excludes_file_info(int fd) +{ + /* + * Likewise, there is a per-repo excludes file in .git/info/excludes + * that can change the results reported by status. And the deserialize + * code needs to be able to reject the status cache if this file + * changes. + * + * See dir.c:setup_standard_excludes() and git_path_info_excludes(). + * We replicate the pathname construction here because of the static + * variables/functions used in dir.c. + */ + char *path = repo_git_path(the_repository, "info/exclude"); + + append_exclude_info(fd, path, "repo_excludes"); + + free(path); +} + +/* + * WARNING: The status cache attempts to preserve the essential in-memory + * status data after a status scan into a "serialization" (aka "status cache") + * file. It allows later "git status --deserialize=" instances to + * just print the cached status results without scanning the workdir (and + * without reading the index). + * + * The status cache file is valid as long as: + * [1] the set of functional command line options are the same (think "-u"). + * [2] repo-local and user-global configuration settings are compatible. + * [3] nothing in the workdir has changed. + * + * We rely on: + * [1.a] We remember the relevant (functional, non-display) command line + * arguments in the status cache header. + * [2.a] We use the mtime of the .git/index to detect staging changes. + * [2.b] We use the mtimes of the excludes files to detect changes that + * might affect untracked file reporting. + * + * But we need external help to verify [3]. + * [] This includes changes to tracked files. + * [] This includes changes to tracked .gitignore files that might change + * untracked file reporting. + * [] This includes the creation of new, untracked per-directory .gitignore + * files that might change untracked file reporting. + * + * [3.a] On GVFS repos, we rely on the GVFS service (mount) daemon to + * watch the filesystem and invalidate (delete) the status cache + * when anything changes inside the workdir. + * + * [3.b] TODO This problem is not solved for non-GVFS repos. + * [] It is possible that the untracked-cache index extension + * could help with this but that requires status to read the + * index to load the extension. + * [] It is possible that the new fsmonitor facility could also + * provide this information, but that to requires reading the + * index. + */ + /* * Write V1 header fields. */ @@ -20,6 +140,8 @@ static void wt_serialize_v1_header(struct wt_status *s, int fd) packet_write_fmt(fd, "index_mtime %d %d\n", s->repo->index->timestamp.sec, s->repo->index->timestamp.nsec); + append_core_excludes_file_info(fd); + append_repo_excludes_file_info(fd); /* * Write data from wt_status to qualify this status report. diff --git a/wt-status.h b/wt-status.h index e7a19f0eb3f993..b0908234ad7d6d 100644 --- a/wt-status.h +++ b/wt-status.h @@ -245,4 +245,12 @@ void wt_status_serialize_v1(int fd, struct wt_status *s); int wt_status_deserialize(const struct wt_status *cmd_s, const char *path); +/* + * A helper routine for serialize and deserialize to compute + * metadata for the user-global and repo-local excludes files. + */ +void wt_serialize_compute_exclude_header(struct strbuf *sb, + const char *key, + const char *path); + #endif /* STATUS_H */ From 9ae932bd8622e53429a4e4851c35ceca28dbcc8a Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 15 Apr 2019 13:39:43 -0700 Subject: [PATCH 086/289] trace2: refactor setting process starting time Create trace2_initialize_clock() and call from main() to capture process start time in isolation and before other sub-systems are ready. Signed-off-by: Jeff Hostetler Signed-off-by: Junio C Hamano --- compat/mingw.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index fd014eef498e55..fbd118b5937a35 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -4422,6 +4422,8 @@ int wmain(int argc, const wchar_t **wargv) SetConsoleCtrlHandler(handle_ctrl_c, TRUE); + trace2_initialize_clock(); + maybe_redirect_std_handles(); adjust_symlink_flags(); fsync_object_files = 1; From 919aede5668593033199fcb6d0714903184db9af Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 5 Apr 2017 10:55:32 -0600 Subject: [PATCH 087/289] Do not remove files outside the sparse-checkout Signed-off-by: Kevin Willford --- unpack-trees.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/unpack-trees.c b/unpack-trees.c index 7465c0fb9cf023..6e9ab9edd9a7a5 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -575,7 +575,9 @@ static int apply_sparse_checkout(struct index_state *istate, ce->ce_flags &= ~CE_SKIP_WORKTREE; return -1; } - ce->ce_flags |= CE_WT_REMOVE; + if (!gvfs_config_is_set(istate->repo, GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT)) + ce->ce_flags |= CE_WT_REMOVE; + ce->ce_flags &= ~CE_UPDATE; } if (was_skip_worktree && !ce_skip_worktree(ce)) { From af8635afc180e7536d3c11ab53156f86ee9410e6 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 25 Jul 2018 14:49:37 -0400 Subject: [PATCH 088/289] status: deserialization wait Teach `git status --deserialize` to either wait indefintely or immediately fail if the status serialization cache file is stale. Signed-off-by: Jeff Hostetler --- Documentation/config/status.adoc | 16 +++++ builtin/commit.c | 59 ++++++++++++++- t/t7522-serialized-status.sh | 52 ++++++++++++++ wt-status-deserialize.c | 119 ++++++++++++++++++++++++++++--- wt-status.h | 12 +++- 5 files changed, 245 insertions(+), 13 deletions(-) diff --git a/Documentation/config/status.adoc b/Documentation/config/status.adoc index 752d7065608186..9663e890774059 100644 --- a/Documentation/config/status.adoc +++ b/Documentation/config/status.adoc @@ -108,3 +108,19 @@ status.deserializePath:: generated by `--serialize`. This will be overridden by `--deserialize=` on the command line. If the cache file is invalid or stale, git will fall-back and compute status normally. + +status.deserializeWait:: + EXPERIMENTAL, Specifies what `git status --deserialize` should do + if the serialization cache file is stale and whether it should + fall-back and compute status normally. This will be overridden by + `--deserialize-wait=` on the command line. ++ +-- +* `fail` - cause git to exit with an error when the status cache file +is stale; this is intended for testing and debugging. +* `block` - cause git to spin and periodically retry the cache file +every 100 ms; this is intended to help coordinate with another git +instance concurrently computing the cache file. +* `no` - to immediately fall-back if cache file is stale. This is the default. +* `` - time (in tenths of a second) to spin and retry. +-- diff --git a/builtin/commit.c b/builtin/commit.c index 84668f1d7c708f..676c3c8f7dfbe7 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -173,6 +173,9 @@ static int do_implicit_deserialize = 0; static int do_explicit_deserialize = 0; static char *deserialize_path = NULL; +static enum wt_status_deserialize_wait implicit_deserialize_wait = DESERIALIZE_WAIT__UNSET; +static enum wt_status_deserialize_wait explicit_deserialize_wait = DESERIALIZE_WAIT__UNSET; + /* * --serialize | --serialize= * @@ -238,6 +241,40 @@ static int opt_parse_deserialize(const struct option *opt UNUSED, const char *ar return 0; } +static enum wt_status_deserialize_wait parse_dw(const char *arg) +{ + int tenths; + + if (!strcmp(arg, "fail")) + return DESERIALIZE_WAIT__FAIL; + else if (!strcmp(arg, "block")) + return DESERIALIZE_WAIT__BLOCK; + else if (!strcmp(arg, "no")) + return DESERIALIZE_WAIT__NO; + + /* + * Otherwise, assume it is a timeout in tenths of a second. + * If it contains a bogus value, atol() will return zero + * which is OK. + */ + tenths = atol(arg); + if (tenths < 0) + tenths = DESERIALIZE_WAIT__NO; + return tenths; +} + +static int opt_parse_deserialize_wait(const struct option *opt UNUSED, + const char *arg, + int unset) +{ + if (unset) + explicit_deserialize_wait = DESERIALIZE_WAIT__UNSET; + else + explicit_deserialize_wait = parse_dw(arg); + + return 0; +} + static int opt_parse_m(const struct option *opt, const char *arg, int unset) { struct strbuf *buf = opt->value; @@ -1618,6 +1655,13 @@ static int git_status_config(const char *k, const char *v, } return 0; } + if (!strcmp(k, "status.deserializewait")) { + if (!v || !*v) + implicit_deserialize_wait = DESERIALIZE_WAIT__UNSET; + else + implicit_deserialize_wait = parse_dw(v); + return 0; + } if (!strcmp(k, "status.showuntrackedfiles")) { enum untracked_status_type u; @@ -1679,6 +1723,9 @@ struct repository *repo UNUSED) OPT_CALLBACK_F(0, "deserialize", NULL, N_("path"), N_("deserialize raw status data from file"), PARSE_OPT_OPTARG, opt_parse_deserialize), + OPT_CALLBACK_F(0, "deserialize-wait", NULL, + N_("fail|block|no"), N_("how to wait if status cache file is invalid"), + PARSE_OPT_OPTARG, opt_parse_deserialize_wait), OPT_SET_INT(0, "long", &status_format, N_("show status in long format (default)"), STATUS_FORMAT_LONG), @@ -1792,11 +1839,21 @@ struct repository *repo UNUSED) } if (try_deserialize) { + int result; + enum wt_status_deserialize_wait dw = implicit_deserialize_wait; + if (explicit_deserialize_wait != DESERIALIZE_WAIT__UNSET) + dw = explicit_deserialize_wait; + if (dw == DESERIALIZE_WAIT__UNSET) + dw = DESERIALIZE_WAIT__NO; + if (s.relative_paths) s.prefix = prefix; - if (wt_status_deserialize(&s, deserialize_path) == DESERIALIZE_OK) + result = wt_status_deserialize(&s, deserialize_path, dw); + if (result == DESERIALIZE_OK) return 0; + if (dw == DESERIALIZE_WAIT__FAIL) + die(_("Rejected status serialization cache")); /* deserialize failed, so force the initialization we skipped above. */ enable_fscache(1); diff --git a/t/t7522-serialized-status.sh b/t/t7522-serialized-status.sh index 06be656dccf475..c8240dc3de1ad7 100755 --- a/t/t7522-serialized-status.sh +++ b/t/t7522-serialized-status.sh @@ -202,6 +202,58 @@ test_expect_success 'verify new --serialize=path mode' ' test_cmp expect output.2 ' +test_expect_success 'try deserialize-wait feature' ' + test_when_finished "rm -f serialized_status.dat dirt expect.* output.* trace.*" && + + git status --serialize=serialized_status.dat >output.1 && + + # make status cache stale by updating the mtime on the index. confirm that + # deserialize fails when requested. + sleep 1 && + touch .git/index && + test_must_fail git status --deserialize=serialized_status.dat --deserialize-wait=fail && + test_must_fail git -c status.deserializeWait=fail status --deserialize=serialized_status.dat && + + cat >expect.1 <<-\EOF && + ? expect.1 + ? output.1 + ? serialized_status.dat + ? untracked/ + ? untracked_1.txt + EOF + + # refresh the status cache. + git status --porcelain=v2 --serialize=serialized_status.dat >output.1 && + test_cmp expect.1 output.1 && + + # create some dirt. confirm deserialize used the existing status cache. + echo x >dirt && + git status --porcelain=v2 --deserialize=serialized_status.dat >output.2 && + test_cmp output.1 output.2 && + + # make the cache stale and try the timeout feature and wait upto + # 2 tenths of a second. confirm deserialize timed out and rejected + # the status cache and did a normal scan. + + cat >expect.2 <<-\EOF && + ? dirt + ? expect.1 + ? expect.2 + ? output.1 + ? output.2 + ? serialized_status.dat + ? trace.2 + ? untracked/ + ? untracked_1.txt + EOF + + sleep 1 && + touch .git/index && + GIT_TRACE_DESERIALIZE=1 git status --porcelain=v2 --deserialize=serialized_status.dat --deserialize-wait=2 >output.2 2>trace.2 && + test_cmp expect.2 output.2 && + grep "wait polled=2 result=1" trace.2 >trace.2g +' + test_expect_success 'merge conflicts' ' # create a merge conflict. diff --git a/wt-status-deserialize.c b/wt-status-deserialize.c index e1715fdce5b0b0..ff906bc23100e4 100644 --- a/wt-status-deserialize.c +++ b/wt-status-deserialize.c @@ -62,7 +62,8 @@ static int my_validate_index(const char *path, const struct cache_time *mtime_re mtime_observed_on_disk.nsec = ST_MTIME_NSEC(st); if ((mtime_observed_on_disk.sec != mtime_reported->sec) || (mtime_observed_on_disk.nsec != mtime_reported->nsec)) { - trace_printf_key(&trace_deserialize, "index mtime changed [des %d.%d][obs %d.%d]", + trace_printf_key(&trace_deserialize, + "index mtime changed [des %d %d][obs %d %d]", mtime_reported->sec, mtime_reported->nsec, mtime_observed_on_disk.sec, mtime_observed_on_disk.nsec); return DESERIALIZE_ERR; @@ -553,6 +554,8 @@ static inline int my_strcmp_null(const char *a, const char *b) static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *des_s, int fd) { + memset(des_s, 0, sizeof(*des_s)); + /* * Check the path spec on the current command */ @@ -682,8 +685,101 @@ static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *de return DESERIALIZE_OK; } +static struct cache_time deserialize_prev_mtime = { 0, 0 }; + +static int try_deserialize_read_from_file_1(const struct wt_status *cmd_s, + const char *path, + struct wt_status *des_s) +{ + struct stat st; + int result; + int fd; + + /* + * If we are spinning waiting for the status cache to become + * valid, skip re-reading it if the mtime has not changed + * since the last time we read it. + */ + if (lstat(path, &st)) { + trace_printf_key(&trace_deserialize, + "could not lstat '%s'", path); + return DESERIALIZE_ERR; + } + if ((uint32_t)st.st_mtime == deserialize_prev_mtime.sec && + ST_MTIME_NSEC(st) == deserialize_prev_mtime.nsec) { + trace_printf_key(&trace_deserialize, + "mtime has not changed '%s'", path); + return DESERIALIZE_ERR; + } + + fd = xopen(path, O_RDONLY); + if (fd == -1) { + trace_printf_key(&trace_deserialize, + "could not read '%s'", path); + return DESERIALIZE_ERR; + } + + deserialize_prev_mtime.sec = st.st_mtime; + deserialize_prev_mtime.nsec = ST_MTIME_NSEC(st); + + trace_printf_key(&trace_deserialize, + "reading serialization file (%d %d) '%s'", + deserialize_prev_mtime.sec, + deserialize_prev_mtime.nsec, + path); + + result = wt_deserialize_fd(cmd_s, des_s, fd); + close(fd); + + return result; +} + +static int try_deserialize_read_from_file(const struct wt_status *cmd_s, + const char *path, + enum wt_status_deserialize_wait dw, + struct wt_status *des_s) +{ + int k, limit; + int result = DESERIALIZE_ERR; + + /* + * For "fail" or "no", try exactly once to read the status cache. + * Return an error if the file is stale. + */ + if (dw == DESERIALIZE_WAIT__FAIL || dw == DESERIALIZE_WAIT__NO) + return try_deserialize_read_from_file_1(cmd_s, path, des_s); + + /* + * Wait for the status cache file to refresh. Wait duration can + * be in tenths of a second or unlimited. Poll every 100ms. + */ + if (dw == DESERIALIZE_WAIT__BLOCK) { + /* + * Convert "unlimited" to 1 day. + */ + limit = 10 * 60 * 60 * 24; + } else { + /* spin for dw tenths of a second */ + limit = dw; + } + for (k = 0; k < limit; k++) { + result = try_deserialize_read_from_file_1( + cmd_s, path, des_s); + + if (result == DESERIALIZE_OK) + break; + + sleep_millisec(100); + } + + trace_printf_key(&trace_deserialize, + "wait polled=%d result=%d '%s'", + k, result, path); + return result; +} + /* - * Read raw serialized status data from the given file + * Read raw serialized status data from the given file (or STDIN). * * Verify that the args specified in the current command * are compatible with the deserialized data (such as "-uno"). @@ -691,25 +787,26 @@ static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *de * Copy display-related fields from the current command * into the deserialized data (so that the user can request * long or short as they please). + * + * Print status report using cached data. */ int wt_status_deserialize(const struct wt_status *cmd_s, - const char *path) + const char *path, + enum wt_status_deserialize_wait dw) { struct wt_status des_s; int result; struct string_list_item *change; if (path && *path && strcmp(path, "0")) { - int fd = xopen(path, O_RDONLY); - if (fd == -1) { - trace_printf_key(&trace_deserialize, "could not read '%s'", path); - return DESERIALIZE_ERR; - } - trace_printf_key(&trace_deserialize, "reading serialization file '%s'", path); - result = wt_deserialize_fd(cmd_s, &des_s, fd); - close(fd); + result = try_deserialize_read_from_file(cmd_s, path, dw, &des_s); } else { trace_printf_key(&trace_deserialize, "reading stdin"); + + /* + * Read status cache data from stdin. Ignore the deserialize-wait + * term, since we cannot read stdin multiple times. + */ result = wt_deserialize_fd(cmd_s, &des_s, 0); } diff --git a/wt-status.h b/wt-status.h index b0908234ad7d6d..95154c79994bc4 100644 --- a/wt-status.h +++ b/wt-status.h @@ -229,6 +229,15 @@ struct wt_status_serialize_data - sizeof(struct wt_status_serialize_data_fixed)]; }; +enum wt_status_deserialize_wait +{ + DESERIALIZE_WAIT__UNSET = -3, + DESERIALIZE_WAIT__FAIL = -2, /* return error, do not fallback */ + DESERIALIZE_WAIT__BLOCK = -1, /* unlimited timeout */ + DESERIALIZE_WAIT__NO = 0, /* immediately fallback */ + /* any positive value is a timeout in tenths of a second */ +}; + /* * Serialize computed status scan results using "version 1" format * to the given file. @@ -243,7 +252,8 @@ void wt_status_serialize_v1(int fd, struct wt_status *s); * fields. */ int wt_status_deserialize(const struct wt_status *cmd_s, - const char *path); + const char *path, + enum wt_status_deserialize_wait dw); /* * A helper routine for serialize and deserialize to compute From 3f1d7c19d85bfa1ca957028c9a02a65dda8010b4 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 30 Apr 2019 16:02:39 -0400 Subject: [PATCH 089/289] trace2:gvfs:experiment: report_tracking Signed-off-by: Jeff Hostetler --- builtin/checkout.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/builtin/checkout.c b/builtin/checkout.c index f2e5a3308e37f7..eab0e93a96a818 100644 --- a/builtin/checkout.c +++ b/builtin/checkout.c @@ -1050,8 +1050,11 @@ static void update_refs_for_switch(const struct checkout_opts *opts, strbuf_release(&msg); if (!opts->quiet && !opts->force_detach && - (new_branch_info->path || !strcmp(new_branch_info->name, "HEAD"))) + (new_branch_info->path || !strcmp(new_branch_info->name, "HEAD"))) { + trace2_region_enter("exp", "report_tracking", the_repository); report_tracking(new_branch_info); + trace2_region_leave("exp", "report_tracking", the_repository); + } } static int add_pending_uninteresting_ref(const struct reference *ref, void *cb_data) From d4d2f19bc066b86c0b17640333afdc3095a69a04 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Fri, 16 Nov 2018 11:28:59 -0700 Subject: [PATCH 090/289] send-pack: do not check for sha1 file when GVFS_MISSING_OK set --- send-pack.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/send-pack.c b/send-pack.c index c6711b0a02509c..3d4954ac5dd301 100644 --- a/send-pack.c +++ b/send-pack.c @@ -3,6 +3,7 @@ #include "commit.h" #include "date.h" #include "gettext.h" +#include "gvfs.h" #include "hex.h" #include "odb.h" #include "pkt-line.h" @@ -50,7 +51,7 @@ static void append_negative_object(struct repository *r, * The remote end may have advertised objects that we do not have in * our object database. Skip those, as we cannot use them as boundary. */ - if (!odb_has_object(r->objects, oid, 0)) + if (!gvfs_config_is_set(r, GVFS_MISSING_OK) && !odb_has_object(r->objects, oid, 0)) return; oid_array_append(haves, oid); } From 8718e265511a8e732464c7ef9cbc300aaf717112 Mon Sep 17 00:00:00 2001 From: Matthew John Cheetham Date: Thu, 6 Mar 2025 11:12:35 +0000 Subject: [PATCH 091/289] git.c: add VFS enabled cmd blocking Add the ability to block built-in commands based on if the `core.gvfs` setting has the `GVFS_USE_VIRTUAL_FILESYSTEM` bit set. This allows us to selectively block commands that use the GVFS protocol, but don't use VFS for Git (for example repos cloned via `scalar clone` against Azure DevOps). Signed-off-by: Matthew John Cheetham --- git.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/git.c b/git.c index de4aef3cbc8dbe..82e5568641bc29 100644 --- a/git.c +++ b/git.c @@ -33,6 +33,7 @@ #define NO_PARSEOPT (1<<5) /* parse-options is not used */ #define DEPRECATED (1<<6) #define BLOCK_ON_GVFS_REPO (1<<7) /* command not allowed in GVFS repos */ +#define BLOCK_ON_VFS_ENABLED (1<<8) /* command not allowed when virtual file system is used */ struct cmd_struct { const char *cmd; @@ -593,6 +594,9 @@ static int run_builtin(struct cmd_struct *p, int argc, const char **argv, struct if (!help && p->option & BLOCK_ON_GVFS_REPO && gvfs_config_is_set(repo, GVFS_BLOCK_COMMANDS)) die("'git %s' is not supported on a GVFS repo", p->cmd); + if (!help && p->option & BLOCK_ON_VFS_ENABLED && gvfs_config_is_set(repo, GVFS_USE_VIRTUAL_FILESYSTEM)) + die("'git %s' is not supported when using the virtual file system", p->cmd); + if (run_pre_command_hook(the_repository, argv)) die("pre-command hook aborted command"); From e3dce25b4aa23a3bd5da7cfb3c512ed5274b35c7 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Thu, 11 Jan 2018 16:25:08 -0500 Subject: [PATCH 092/289] Add virtual file system settings and hook proc On index load, clear/set the skip worktree bits based on the virtual file system data. Use virtual file system data to update skip-worktree bit in unpack-trees. Use virtual file system data to exclude files and folders not explicitly requested. Update 2022-04-05: disable the "present-despite-SKIP_WORKTREE" file removal behavior when 'core.virtualfilesystem' is enabled. Signed-off-by: Ben Peart --- Documentation/config/core.adoc | 8 + Documentation/githooks.adoc | 20 ++ Makefile | 1 + config.c | 26 +++ config.h | 2 + dir.c | 43 +++- environment.c | 7 +- environment.h | 2 + meson.build | 1 + read-cache.c | 2 + sparse-index.c | 5 +- t/meson.build | 1 + t/t1090-sparse-checkout-scope.sh | 4 +- t/t1093-virtualfilesystem.sh | 369 +++++++++++++++++++++++++++++++ unpack-trees.c | 14 +- virtualfilesystem.c | 312 ++++++++++++++++++++++++++ virtualfilesystem.h | 25 +++ wt-status.c | 2 + 18 files changed, 836 insertions(+), 8 deletions(-) create mode 100755 t/t1093-virtualfilesystem.sh create mode 100644 virtualfilesystem.c create mode 100644 virtualfilesystem.h diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index 1d9a3a5cf96159..0161b882ee57f5 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -111,6 +111,14 @@ Version 2 uses an opaque string so that the monitor can return something that can be used to determine what files have changed without race conditions. +core.virtualFilesystem:: + If set, the value of this variable is used as a command which + will identify all files and directories that are present in + the working directory. Git will only track and update files + listed in the virtual file system. Using the virtual file system + will supersede the sparse-checkout settings which will be ignored. + See the "virtual file system" section of linkgit:githooks[5]. + core.trustctime:: If false, the ctime differences between the index and the working tree are ignored; useful when the inode change time diff --git a/Documentation/githooks.adoc b/Documentation/githooks.adoc index ed045940d18ba5..29d63c7ed2c318 100644 --- a/Documentation/githooks.adoc +++ b/Documentation/githooks.adoc @@ -760,6 +760,26 @@ and "0" meaning they were not. Only one parameter should be set to "1" when the hook runs. The hook running passing "1", "1" should not be possible. +virtualFilesystem +~~~~~~~~~~~~~~~~~~ + +"Virtual File System" allows populating the working directory sparsely. +The projection data is typically automatically generated by an external +process. Git will limit what files it checks for changes as well as which +directories are checked for untracked files based on the path names given. +Git will also only update those files listed in the projection. + +The hook is invoked when the configuration option core.virtualFilesystem +is set. It takes one argument, a version (currently 1). + +The hook should output to stdout the list of all files in the working +directory that git should track. The paths are relative to the root +of the working directory and are separated by a single NUL. Full paths +('dir1/a.txt') as well as directories are supported (ie 'dir1/'). + +The exit status determines whether git will use the data from the +hook. On error, git will abort the command with an error message. + SEE ALSO -------- linkgit:git-hook[1] diff --git a/Makefile b/Makefile index 46da13672b1a88..4ca96d73fbeda9 100644 --- a/Makefile +++ b/Makefile @@ -1378,6 +1378,7 @@ LIB_OBJS += varint.o endif LIB_OBJS += version.o LIB_OBJS += versioncmp.o +LIB_OBJS += virtualfilesystem.o LIB_OBJS += walker.o LIB_OBJS += wildmatch.o LIB_OBJS += worktree.o diff --git a/config.c b/config.c index 105700b1368030..7bded08a21d308 100644 --- a/config.c +++ b/config.c @@ -2540,6 +2540,32 @@ int repo_config_get_max_percent_split_change(struct repository *r) return -1; /* default value */ } +int repo_config_get_virtualfilesystem(struct repository *r) +{ + /* Run only once. */ + static int virtual_filesystem_result = -1; + struct repo_config_values *cfg = repo_config_values(r); + extern char *core_virtualfilesystem; + if (virtual_filesystem_result >= 0) + return virtual_filesystem_result; + + if (repo_config_get_pathname(r, "core.virtualfilesystem", &core_virtualfilesystem)) + core_virtualfilesystem = xstrdup_or_null(getenv("GIT_VIRTUALFILESYSTEM_TEST")); + + if (core_virtualfilesystem && !*core_virtualfilesystem) + FREE_AND_NULL(core_virtualfilesystem); + + /* virtual file system relies on the sparse checkout logic so force it on */ + if (core_virtualfilesystem) { + cfg->apply_sparse_checkout = 1; + virtual_filesystem_result = 1; + return 1; + } + + virtual_filesystem_result = 0; + return 0; +} + int repo_config_get_index_threads(struct repository *r, int *dest) { int is_bool, val; diff --git a/config.h b/config.h index b66dd08007c97a..c6075701e728be 100644 --- a/config.h +++ b/config.h @@ -703,6 +703,8 @@ int repo_config_get_index_threads(struct repository *r, int *dest); int repo_config_get_split_index(struct repository *r); int repo_config_get_max_percent_split_change(struct repository *r); +int repo_config_get_virtualfilesystem(struct repository *r); + /* This dies if the configured or default date is in the future */ int repo_config_get_expiry(struct repository *r, const char *key, char **output); diff --git a/dir.c b/dir.c index 4224179ac0f494..d5cda158459677 100644 --- a/dir.c +++ b/dir.c @@ -11,6 +11,7 @@ #include "git-compat-util.h" #include "abspath.h" +#include "virtualfilesystem.h" #include "config.h" #include "convert.h" #include "dir.h" @@ -1533,6 +1534,19 @@ enum pattern_match_result path_matches_pattern_list( int result = NOT_MATCHED; size_t slash_pos; + if (core_virtualfilesystem) { + /* + * The virtual file system data is used to prevent git from traversing + * any part of the tree that is not in the virtual file system. Return + * 1 to exclude the entry if it is not found in the virtual file system, + * else fall through to the regular excludes logic as it may further exclude. + */ + if (*dtype == DT_UNKNOWN) + *dtype = resolve_dtype(DT_UNKNOWN, istate, pathname, pathlen); + if (is_excluded_from_virtualfilesystem(pathname, pathlen, *dtype) > 0) + return 1; + } + if (!pl->use_cone_patterns) { pattern = last_matching_pattern_from_list(pathname, pathlen, basename, dtype, pl, istate); @@ -1624,6 +1638,13 @@ static int path_in_sparse_checkout_1(const char *path, enum pattern_match_result match = UNDECIDED; const char *end, *slash; + /* + * When using a virtual filesystem, there aren't really patterns + * to follow, but be extra careful to skip this check. + */ + if (core_virtualfilesystem) + return 1; + /* * We default to accepting a path if the path is empty, there are no * patterns, or the patterns are of the wrong type. @@ -1879,8 +1900,22 @@ struct path_pattern *last_matching_pattern(struct dir_struct *dir, int is_excluded(struct dir_struct *dir, struct index_state *istate, const char *pathname, int *dtype_p) { - struct path_pattern *pattern = - last_matching_pattern(dir, istate, pathname, dtype_p); + struct path_pattern *pattern; + + if (core_virtualfilesystem) { + /* + * The virtual file system data is used to prevent git from traversing + * any part of the tree that is not in the virtual file system. Return + * 1 to exclude the entry if it is not found in the virtual file system, + * else fall through to the regular excludes logic as it may further exclude. + */ + if (*dtype_p == DT_UNKNOWN) + *dtype_p = resolve_dtype(DT_UNKNOWN, istate, pathname, strlen(pathname)); + if (is_excluded_from_virtualfilesystem(pathname, strlen(pathname), *dtype_p) > 0) + return 1; + } + + pattern = last_matching_pattern(dir, istate, pathname, dtype_p); if (pattern) return pattern->flags & PATTERN_FLAG_NEGATIVE ? 0 : 1; return 0; @@ -2498,6 +2533,8 @@ static enum path_treatment treat_path(struct dir_struct *dir, repo_ignore_case(the_repository)); if (dtype != DT_DIR && has_path_in_index) return path_none; + if (is_excluded_from_virtualfilesystem(path->buf, path->len, dtype) > 0) + return path_excluded; /* * When we are looking at a directory P in the working tree, @@ -2702,6 +2739,8 @@ static void add_path_to_appropriate_result_list(struct dir_struct *dir, /* add the path to the appropriate result list */ switch (state) { case path_excluded: + if (is_excluded_from_virtualfilesystem(path->buf, path->len, DT_DIR) > 0) + break; if (dir->flags & DIR_SHOW_IGNORED) dir_add_name(dir, istate, path->buf, path->len); else if ((dir->flags & DIR_SHOW_IGNORED_TOO) || diff --git a/environment.c b/environment.c index 12f8a12cf3f241..9e276c8eb7ec98 100644 --- a/environment.c +++ b/environment.c @@ -57,6 +57,7 @@ char *check_roundtrip_encoding; #define OBJECT_CREATION_MODE OBJECT_CREATION_USES_HARDLINKS #endif int grafts_keep_true_parents; +char *core_virtualfilesystem; unsigned long pack_size_limit_cfg; int core_virtualize_objects; @@ -548,7 +549,11 @@ int git_default_core_config(const char *var, const char *value, } if (!strcmp(var, "core.sparsecheckout")) { - cfg->apply_sparse_checkout = git_config_bool(var, value); + /* virtual file system relies on the sparse checkout logic so force it on */ + if (core_virtualfilesystem) + cfg->apply_sparse_checkout = 1; + else + cfg->apply_sparse_checkout = git_config_bool(var, value); return 0; } diff --git a/environment.h b/environment.h index 08e715c6dd8ac8..22261414b6df72 100644 --- a/environment.h +++ b/environment.h @@ -235,6 +235,8 @@ extern int minimum_abbrev, default_abbrev; extern int assume_unchanged; extern unsigned long pack_size_limit_cfg; +extern char *core_virtualfilesystem; + extern int grafts_keep_true_parents; const char *get_log_output_encoding(void); diff --git a/meson.build b/meson.build index 4dbacf2cbcf7f4..215171f61e15d1 100644 --- a/meson.build +++ b/meson.build @@ -574,6 +574,7 @@ libgit_sources = [ 'utf8.c', 'version.c', 'versioncmp.c', + 'virtualfilesystem.c', 'walker.c', 'wildmatch.c', 'worktree.c', diff --git a/read-cache.c b/read-cache.c index 4cf60d6776e5a2..7ffd3d1d38c509 100644 --- a/read-cache.c +++ b/read-cache.c @@ -8,6 +8,7 @@ #define DISABLE_SIGN_COMPARE_WARNINGS #include "git-compat-util.h" +#include "virtualfilesystem.h" #include "config.h" #include "date.h" #include "diff.h" @@ -1967,6 +1968,7 @@ static void post_read_index_from(struct index_state *istate) tweak_untracked_cache(istate); tweak_split_index(istate); tweak_fsmonitor(istate); + apply_virtualfilesystem(istate); } static size_t estimate_cache_size_from_compressed(unsigned int entries) diff --git a/sparse-index.c b/sparse-index.c index a3d32dab2b816a..b8fd03ca79f2d4 100644 --- a/sparse-index.c +++ b/sparse-index.c @@ -281,7 +281,7 @@ static int add_path_to_index(const struct object_id *oid, size_t len = base->len; if (S_ISDIR(mode)) { - int dtype; + int dtype = DT_DIR; size_t baselen = base->len; if (!ctx->pl) return READ_TREE_RECURSIVE; @@ -405,7 +405,7 @@ void expand_index(struct index_state *istate, struct pattern_list *pl) struct cache_entry *ce = istate->cache[i]; struct tree *tree; struct pathspec ps; - int dtype; + int dtype = DT_UNKNOWN; if (!S_ISSPARSEDIR(ce->ce_mode)) { set_index_entry(full, full->cache_nr++, ce); @@ -683,6 +683,7 @@ void clear_skip_worktree_from_present_files(struct index_state *istate) struct repo_config_values *cfg = repo_config_values(the_repository); if (!cfg->apply_sparse_checkout || + core_virtualfilesystem || cfg->sparse_expect_files_outside_of_patterns) return; diff --git a/t/meson.build b/t/meson.build index 4ba6e06f27eaef..d02bcef6af1947 100644 --- a/t/meson.build +++ b/t/meson.build @@ -187,6 +187,7 @@ integration_tests = [ 't1090-sparse-checkout-scope.sh', 't1091-sparse-checkout-builtin.sh', 't1092-sparse-checkout-compatibility.sh', + 't1093-virtualfilesystem.sh', 't1100-commit-tree-options.sh', 't1300-config.sh', 't1301-shared-repo.sh', diff --git a/t/t1090-sparse-checkout-scope.sh b/t/t1090-sparse-checkout-scope.sh index effa20aab7bea7..02b393e36a7d28 100755 --- a/t/t1090-sparse-checkout-scope.sh +++ b/t/t1090-sparse-checkout-scope.sh @@ -107,9 +107,9 @@ test_expect_success 'in partial clone, sparse checkout only fetches needed blobs ' test_expect_success 'checkout does not delete items outside the sparse checkout file' ' - # The "sparse.expectfilesoutsideofpatterns" config will prevent the + # The "core.virtualfilesystem" config will prevent the # SKIP_WORKTREE flag from being dropped on files present on-disk. - test_config sparse.expectfilesoutsideofpatterns true && + test_config core.virtualfilesystem true && test_config core.gvfs 8 && git checkout -b outside && diff --git a/t/t1093-virtualfilesystem.sh b/t/t1093-virtualfilesystem.sh new file mode 100755 index 00000000000000..bd0c9f72ba3c4a --- /dev/null +++ b/t/t1093-virtualfilesystem.sh @@ -0,0 +1,369 @@ +#!/bin/sh + +test_description='virtual file system tests' + +. ./test-lib.sh + +clean_repo () { + rm .git/index && + git -c core.virtualfilesystem= reset --hard HEAD && + git -c core.virtualfilesystem= clean -fd && + touch untracked.txt && + touch dir1/untracked.txt && + touch dir2/untracked.txt +} + +test_expect_success 'setup' ' + git branch -M main && + mkdir -p .git/hooks/ && + cat > .gitignore <<-\EOF && + .gitignore + expect* + actual* + EOF + mkdir -p dir1 && + touch dir1/file1.txt && + touch dir1/file2.txt && + mkdir -p dir2 && + touch dir2/file1.txt && + touch dir2/file2.txt && + git add . && + git commit -m "initial" && + git config --local core.virtualfilesystem .git/hooks/virtualfilesystem +' + +test_expect_success 'test hook parameters and version' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + if test "$#" -ne 1 + then + echo "$0: Exactly 1 argument expected" >&2 + exit 2 + fi + + if test "$1" != 1 + then + echo "$0: Unsupported hook version." >&2 + exit 1 + fi + EOF + git status && + write_script .git/hooks/virtualfilesystem <<-\EOF && + exit 3 + EOF + test_must_fail git status +' + +test_expect_success 'verify status is clean' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir2/file1.txt\0" + EOF + rm -f .git/index && + git checkout -f && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir2/file1.txt\0" + printf "dir1/file1.txt\0" + printf "dir1/file2.txt\0" + EOF + git status > actual && + cat > expected <<-\EOF && + On branch main + nothing to commit, working tree clean + EOF + test_cmp expected actual +' + +test_expect_success 'verify skip-worktree bit is set for absolute path' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file1.txt\0" + EOF + git ls-files -v > actual && + cat > expected <<-\EOF && + H dir1/file1.txt + S dir1/file2.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'verify skip-worktree bit is cleared for absolute path' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file2.txt\0" + EOF + git ls-files -v > actual && + cat > expected <<-\EOF && + S dir1/file1.txt + H dir1/file2.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'verify folder wild cards' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/\0" + EOF + git ls-files -v > actual && + cat > expected <<-\EOF && + H dir1/file1.txt + H dir1/file2.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'verify folders not included are ignored' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file1.txt\0" + printf "dir1/file2.txt\0" + EOF + mkdir -p dir1/dir2 && + touch dir1/a && + touch dir1/b && + touch dir1/dir2/a && + touch dir1/dir2/b && + git add . && + git ls-files -v > actual && + cat > expected <<-\EOF && + H dir1/file1.txt + H dir1/file2.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'verify including one file doesnt include the rest' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file1.txt\0" + printf "dir1/file2.txt\0" + printf "dir1/dir2/a\0" + EOF + mkdir -p dir1/dir2 && + touch dir1/a && + touch dir1/b && + touch dir1/dir2/a && + touch dir1/dir2/b && + git add . && + git ls-files -v > actual && + cat > expected <<-\EOF && + H dir1/dir2/a + H dir1/file1.txt + H dir1/file2.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'verify files not listed are ignored by git clean -f -x' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "untracked.txt\0" + printf "dir1/\0" + EOF + mkdir -p dir3 && + touch dir3/untracked.txt && + git clean -f -x && + test ! -f untracked.txt && + test -d dir1 && + test -f dir1/file1.txt && + test -f dir1/file2.txt && + test ! -f dir1/untracked.txt && + test -f dir2/file1.txt && + test -f dir2/file2.txt && + test -f dir2/untracked.txt && + test -d dir3 && + test -f dir3/untracked.txt +' + +test_expect_success 'verify files not listed are ignored by git clean -f -d -x' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "untracked.txt\0" + printf "dir1/\0" + printf "dir3/\0" + EOF + mkdir -p dir3 && + touch dir3/untracked.txt && + git clean -f -d -x && + test ! -f untracked.txt && + test -d dir1 && + test -f dir1/file1.txt && + test -f dir1/file2.txt && + test ! -f dir1/untracked.txt && + test -f dir2/file1.txt && + test -f dir2/file2.txt && + test -f dir2/untracked.txt && + test ! -d dir3 && + test ! -f dir3/untracked.txt +' + +test_expect_success 'verify folder entries include all files' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/\0" + EOF + mkdir -p dir1/dir2 && + touch dir1/a && + touch dir1/b && + touch dir1/dir2/a && + touch dir1/dir2/b && + git status -su > actual && + cat > expected <<-\EOF && + ?? dir1/a + ?? dir1/b + ?? dir1/untracked.txt + EOF + test_cmp expected actual +' + +test_expect_success 'verify case insensitivity of virtual file system entries' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/a\0" + printf "Dir1/Dir2/a\0" + printf "DIR2/\0" + EOF + mkdir -p dir1/dir2 && + touch dir1/a && + touch dir1/b && + touch dir1/dir2/a && + touch dir1/dir2/b && + git -c core.ignorecase=false status -su > actual && + cat > expected <<-\EOF && + ?? dir1/a + EOF + test_cmp expected actual && + git -c core.ignorecase=true status -su > actual && + cat > expected <<-\EOF && + ?? dir1/a + ?? dir1/dir2/a + ?? dir2/untracked.txt + EOF + test_cmp expected actual +' + +test_expect_success 'on file created' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file3.txt\0" + EOF + touch dir1/file3.txt && + git add . && + git ls-files -v > actual && + cat > expected <<-\EOF && + S dir1/file1.txt + S dir1/file2.txt + H dir1/file3.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'on file renamed' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file1.txt\0" + printf "dir1/file3.txt\0" + EOF + mv dir1/file1.txt dir1/file3.txt && + git status -su > actual && + cat > expected <<-\EOF && + D dir1/file1.txt + ?? dir1/file3.txt + EOF + test_cmp expected actual +' + +test_expect_success 'on file deleted' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file1.txt\0" + EOF + rm dir1/file1.txt && + git status -su > actual && + cat > expected <<-\EOF && + D dir1/file1.txt + EOF + test_cmp expected actual +' + +test_expect_success 'on file overwritten' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file1.txt\0" + EOF + echo "overwritten" > dir1/file1.txt && + git status -su > actual && + cat > expected <<-\EOF && + M dir1/file1.txt + EOF + test_cmp expected actual +' + +test_expect_success 'on folder created' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/dir1/\0" + EOF + mkdir -p dir1/dir1 && + git status -su > actual && + cat > expected <<-\EOF && + EOF + test_cmp expected actual && + git clean -fd && + test ! -d "/dir1/dir1" +' + +test_expect_success 'on folder renamed' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir3/\0" + printf "dir1/file1.txt\0" + printf "dir1/file2.txt\0" + printf "dir3/file1.txt\0" + printf "dir3/file2.txt\0" + EOF + mv dir1 dir3 && + git status -su > actual && + cat > expected <<-\EOF && + D dir1/file1.txt + D dir1/file2.txt + ?? dir3/file1.txt + ?? dir3/file2.txt + ?? dir3/untracked.txt + EOF + test_cmp expected actual +' + +test_expect_success 'folder with same prefix as file' ' + clean_repo && + touch dir1.sln && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/\0" + printf "dir1.sln\0" + EOF + git add dir1.sln && + git ls-files -v > actual && + cat > expected <<-\EOF && + H dir1.sln + H dir1/file1.txt + H dir1/file2.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_done diff --git a/unpack-trees.c b/unpack-trees.c index 6e9ab9edd9a7a5..29917231249344 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -4,6 +4,7 @@ #include "git-compat-util.h" #include "advice.h" #include "gvfs.h" +#include "virtualfilesystem.h" #include "strvec.h" #include "repository.h" #include "parse.h" @@ -1710,6 +1711,14 @@ static int clear_ce_flags_1(struct index_state *istate, continue; } + /* if it's not in the virtual file system, exit early */ + if (core_virtualfilesystem) { + if (is_included_in_virtualfilesystem(ce->name, ce->ce_namelen) > 0) + ce->ce_flags &= ~clear_mask; + cache++; + continue; + } + if (prefix->len && strncmp(ce->name, prefix->buf, prefix->len)) break; @@ -1938,7 +1947,10 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options if (!o->skip_sparse_checkout) { memset(&pl, 0, sizeof(pl)); free_pattern_list = 1; - populate_from_existing_patterns(o, &pl); + if (core_virtualfilesystem) + o->internal.pl = &pl; + else + populate_from_existing_patterns(o, &pl); } index_state_init(&o->internal.result, o->src_index->repo); diff --git a/virtualfilesystem.c b/virtualfilesystem.c new file mode 100644 index 00000000000000..acb0963d14ea2b --- /dev/null +++ b/virtualfilesystem.c @@ -0,0 +1,312 @@ +#define USE_THE_REPOSITORY_VARIABLE + +#include "git-compat-util.h" +#include "environment.h" +#include "gettext.h" +#include "config.h" +#include "dir.h" +#include "hashmap.h" +#include "run-command.h" +#include "name-hash.h" +#include "read-cache-ll.h" +#include "virtualfilesystem.h" + +#define HOOK_INTERFACE_VERSION (1) + +static struct strbuf virtual_filesystem_data = STRBUF_INIT; +static struct hashmap virtual_filesystem_hashmap; +static struct hashmap parent_directory_hashmap; + +struct virtualfilesystem { + struct hashmap_entry ent; /* must be the first member! */ + const char *pattern; + int patternlen; +}; + +static unsigned int(*vfshash)(const void *buf, size_t len); +static int(*vfscmp)(const char *a, const char *b, size_t len); + +static int vfs_hashmap_cmp(const void *cmp_data UNUSED, + const struct hashmap_entry *he1, + const struct hashmap_entry *he2, + const void *key UNUSED) +{ + const struct virtualfilesystem *vfs1 = + container_of(he1, const struct virtualfilesystem, ent); + const struct virtualfilesystem *vfs2 = + container_of(he2, const struct virtualfilesystem, ent); + + return vfscmp(vfs1->pattern, vfs2->pattern, vfs1->patternlen); +} + +static void get_virtual_filesystem_data(struct repository *r, struct strbuf *vfs_data) +{ + struct child_process cp = CHILD_PROCESS_INIT; + int err; + + strbuf_init(vfs_data, 0); + + strvec_push(&cp.args, core_virtualfilesystem); + strvec_pushf(&cp.args, "%d", HOOK_INTERFACE_VERSION); + cp.use_shell = 1; + cp.dir = repo_get_work_tree(r); + + err = capture_command(&cp, vfs_data, 1024); + if (err) + die("unable to load virtual file system"); +} + +static int check_includes_hashmap(struct hashmap *map, const char *pattern, int patternlen) +{ + struct strbuf sb = STRBUF_INIT; + struct virtualfilesystem vfs; + char *slash; + + /* Check straight mapping */ + strbuf_reset(&sb); + strbuf_add(&sb, pattern, patternlen); + vfs.pattern = sb.buf; + vfs.patternlen = sb.len; + hashmap_entry_init(&vfs.ent, vfshash(vfs.pattern, vfs.patternlen)); + if (hashmap_get_entry(map, &vfs, ent, NULL)) { + strbuf_release(&sb); + return 1; + } + + /* + * Check to see if it matches a directory or any path + * underneath it. In other words, 'a/b/foo.txt' will match + * '/', 'a/', and 'a/b/'. + */ + slash = strchr(sb.buf, '/'); + while (slash) { + vfs.pattern = sb.buf; + vfs.patternlen = slash - sb.buf + 1; + hashmap_entry_init(&vfs.ent, vfshash(vfs.pattern, vfs.patternlen)); + if (hashmap_get_entry(map, &vfs, ent, NULL)) { + strbuf_release(&sb); + return 1; + } + slash = strchr(slash + 1, '/'); + } + + strbuf_release(&sb); + return 0; +} + +static void includes_hashmap_add(struct hashmap *map, const char *pattern, const int patternlen) +{ + struct virtualfilesystem *vfs; + + vfs = xmalloc(sizeof(struct virtualfilesystem)); + vfs->pattern = pattern; + vfs->patternlen = patternlen; + hashmap_entry_init(&vfs->ent, vfshash(vfs->pattern, vfs->patternlen)); + hashmap_add(map, &vfs->ent); +} + +static void initialize_includes_hashmap(struct hashmap *map, struct strbuf *vfs_data) +{ + char *buf, *entry; + size_t len, i; + + /* + * Build a hashmap of the virtual file system data we can use to look + * for cache entry matches quickly + */ + vfshash = repo_ignore_case(the_repository) ? memihash : memhash; + vfscmp = repo_ignore_case(the_repository) ? strncasecmp : strncmp; + hashmap_init(map, vfs_hashmap_cmp, NULL, 0); + + entry = buf = vfs_data->buf; + len = vfs_data->len; + for (i = 0; i < len; i++) { + if (buf[i] == '\0') { + includes_hashmap_add(map, entry, buf + i - entry); + entry = buf + i + 1; + } + } +} + +/* + * Return 1 if the requested item is found in the virtual file system, + * 0 for not found and -1 for undecided. + */ +int is_included_in_virtualfilesystem(const char *pathname, int pathlen) +{ + if (!core_virtualfilesystem) + return -1; + + if (!virtual_filesystem_hashmap.tablesize && virtual_filesystem_data.len) + initialize_includes_hashmap(&virtual_filesystem_hashmap, &virtual_filesystem_data); + if (!virtual_filesystem_hashmap.tablesize) + return -1; + + return check_includes_hashmap(&virtual_filesystem_hashmap, pathname, pathlen); +} + +static void parent_directory_hashmap_add(struct hashmap *map, const char *pattern, const int patternlen) +{ + const char *slash; + struct virtualfilesystem *vfs; + + /* + * Add any directories leading up to the file as the excludes logic + * needs to match directories leading up to the files as well. Detect + * and prevent unnecessary duplicate entries which will be common. + */ + if (patternlen > 1) { + slash = strchr(pattern + 1, '/'); + while (slash) { + vfs = xmalloc(sizeof(struct virtualfilesystem)); + vfs->pattern = pattern; + vfs->patternlen = slash - pattern + 1; + hashmap_entry_init(&vfs->ent, vfshash(vfs->pattern, vfs->patternlen)); + if (hashmap_get_entry(map, vfs, ent, NULL)) + free(vfs); + else + hashmap_add(map, &vfs->ent); + slash = strchr(slash + 1, '/'); + } + } +} + +static void initialize_parent_directory_hashmap(struct hashmap *map, struct strbuf *vfs_data) +{ + char *buf, *entry; + size_t len, i; + + /* + * Build a hashmap of the parent directories contained in the virtual + * file system data we can use to look for matches quickly + */ + vfshash = repo_ignore_case(the_repository) ? memihash : memhash; + vfscmp = repo_ignore_case(the_repository) ? strncasecmp : strncmp; + hashmap_init(map, vfs_hashmap_cmp, NULL, 0); + + entry = buf = vfs_data->buf; + len = vfs_data->len; + for (i = 0; i < len; i++) { + if (buf[i] == '\0') { + parent_directory_hashmap_add(map, entry, buf + i - entry); + entry = buf + i + 1; + } + } +} + +static int check_directory_hashmap(struct hashmap *map, const char *pathname, int pathlen) +{ + struct strbuf sb = STRBUF_INIT; + struct virtualfilesystem vfs; + + /* Check for directory */ + strbuf_reset(&sb); + strbuf_add(&sb, pathname, pathlen); + strbuf_addch(&sb, '/'); + vfs.pattern = sb.buf; + vfs.patternlen = sb.len; + hashmap_entry_init(&vfs.ent, vfshash(vfs.pattern, vfs.patternlen)); + if (hashmap_get_entry(map, &vfs, ent, NULL)) { + strbuf_release(&sb); + return 0; + } + + strbuf_release(&sb); + return 1; +} + +/* + * Return 1 for exclude, 0 for include and -1 for undecided. + */ +int is_excluded_from_virtualfilesystem(const char *pathname, int pathlen, int dtype) +{ + if (!core_virtualfilesystem) + return -1; + + if (dtype != DT_REG && dtype != DT_DIR && dtype != DT_LNK) + die(_("is_excluded_from_virtualfilesystem passed unhandled dtype")); + + if (dtype == DT_REG || dtype == DT_LNK) { + int ret = is_included_in_virtualfilesystem(pathname, pathlen); + if (ret > 0) + return 0; + if (ret == 0) + return 1; + return ret; + } + + if (dtype == DT_DIR) { + if (!parent_directory_hashmap.tablesize && virtual_filesystem_data.len) + initialize_parent_directory_hashmap(&parent_directory_hashmap, &virtual_filesystem_data); + if (!parent_directory_hashmap.tablesize) + return -1; + + return check_directory_hashmap(&parent_directory_hashmap, pathname, pathlen); + } + + return -1; +} + +/* + * Update the CE_SKIP_WORKTREE bits based on the virtual file system. + */ +void apply_virtualfilesystem(struct index_state *istate) +{ + char *buf, *entry; + size_t i; + + if (!repo_config_get_virtualfilesystem(istate->repo)) + return; + + if (!virtual_filesystem_data.len) + get_virtual_filesystem_data(istate->repo, &virtual_filesystem_data); + + /* set CE_SKIP_WORKTREE bit on all entries */ + for (i = 0; i < istate->cache_nr; i++) + istate->cache[i]->ce_flags |= CE_SKIP_WORKTREE; + + /* clear CE_SKIP_WORKTREE bit for everything in the virtual file system */ + entry = buf = virtual_filesystem_data.buf; + for (i = 0; i < virtual_filesystem_data.len; i++) { + if (buf[i] == '\0') { + ssize_t pos, len; + + len = buf + i - entry; + + /* look for a directory wild card (ie "dir1/") */ + if (buf[i - 1] == '/') { + if (repo_ignore_case(the_repository)) + adjust_dirname_case(istate, entry); + pos = index_name_pos(istate, entry, len); + if (pos < 0) { + pos = -pos - 1; + while ((size_t)pos < istate->cache_nr && !fspathncmp(istate->cache[pos]->name, entry, len)) { + istate->cache[pos]->ce_flags &= ~CE_SKIP_WORKTREE; + pos++; + } + } + } else { + if (repo_ignore_case(the_repository)) { + struct cache_entry *ce = index_file_exists(istate, entry, len, repo_ignore_case(the_repository)); + if (ce) + ce->ce_flags &= ~CE_SKIP_WORKTREE; + } else { + int pos = index_name_pos(istate, entry, len); + if (pos >= 0) + istate->cache[pos]->ce_flags &= ~CE_SKIP_WORKTREE; + } + } + + entry += len + 1; + } + } +} + +/* + * Free the virtual file system data structures. + */ +void free_virtualfilesystem(void) { + hashmap_clear_and_free(&virtual_filesystem_hashmap, struct virtualfilesystem, ent); + hashmap_clear_and_free(&parent_directory_hashmap, struct virtualfilesystem, ent); + strbuf_release(&virtual_filesystem_data); +} diff --git a/virtualfilesystem.h b/virtualfilesystem.h new file mode 100644 index 00000000000000..5e8c5b096df09a --- /dev/null +++ b/virtualfilesystem.h @@ -0,0 +1,25 @@ +#ifndef VIRTUALFILESYSTEM_H +#define VIRTUALFILESYSTEM_H + +/* + * Update the CE_SKIP_WORKTREE bits based on the virtual file system. + */ +void apply_virtualfilesystem(struct index_state *istate); + +/* + * Return 1 if the requested item is found in the virtual file system, + * 0 for not found and -1 for undecided. + */ +int is_included_in_virtualfilesystem(const char *pathname, int pathlen); + +/* + * Return 1 for exclude, 0 for include and -1 for undecided. + */ +int is_excluded_from_virtualfilesystem(const char *pathname, int pathlen, int dtype); + +/* + * Free the virtual file system data structures. + */ +void free_virtualfilesystem(void); + +#endif diff --git a/wt-status.c b/wt-status.c index 0a2b600f3b3b3b..0643907c1f32af 100644 --- a/wt-status.c +++ b/wt-status.c @@ -1738,6 +1738,8 @@ static void show_sparse_checkout_in_use(struct wt_status *s, { if (s->state.sparse_checkout_percentage == SPARSE_CHECKOUT_DISABLED) return; + if (core_virtualfilesystem) + return; if (s->state.sparse_checkout_percentage == SPARSE_CHECKOUT_SPARSE_INDEX) status_printf_ln(s, color, _("You are in a sparse checkout.")); From 787969cfa6d323cab632085a487ad9b3f0bb14e1 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 25 Jun 2019 16:38:50 -0400 Subject: [PATCH 093/289] status: deserialize with -uno does not print correct hint With the "--untracked-files=complete" option status computes a superset of the untracked files. We use this when writing the status cache. If subsequent deserialize commands ask for either the complete set or one of the "no", "normal", or "all" subsets, it can still use the cache file because of filtering in the deserialize parser. When running status with the "-uno" option, the long format status would print a "(use -u to show untracked files)" hint. When deserializing with the "-uno" option and using a cache computed with "-ucomplete", the "nothing to commit, working tree clean" message would be printed instead of the hint. It was easy to miss because the correct hint message was printed if the cache was rejected for any reason (and status did the full fallback). The "struct wt_status des" structure was initialized with the content of the status cache (and thus defaulted to "complete"). This change sets "des.show_untracked_files" to the requested subset from the command-line or config. This allows the long format to print the hint. Signed-off-by: Jeff Hostetler --- t/t7522-serialized-status.sh | 55 ++++++++++++++++++++++++++++++++++++ wt-status-deserialize.c | 16 +++++++---- 2 files changed, 65 insertions(+), 6 deletions(-) diff --git a/t/t7522-serialized-status.sh b/t/t7522-serialized-status.sh index c8240dc3de1ad7..5b3133b6316309 100755 --- a/t/t7522-serialized-status.sh +++ b/t/t7522-serialized-status.sh @@ -348,4 +348,59 @@ test_expect_success 'renames' ' test_cmp output.1 output.2 ' +test_expect_success 'hint message when cached with u=complete' ' + git init -b main hint && + echo xxx >hint/xxx && + git -C hint add xxx && + git -C hint commit -m xxx && + + cat >expect.clean <expect.use_u <hint.output_normal && + test_cmp expect.clean hint.output_normal && + + git -C hint status --untracked-files=all >hint.output_all && + test_cmp expect.clean hint.output_all && + + git -C hint status --untracked-files=no >hint.output_no && + test_cmp expect.use_u hint.output_no && + + # Create long format output for "complete" and create status cache. + + git -C hint status --untracked-files=complete --ignored=matching --serialize=../hint.dat >hint.output_complete && + test_cmp expect.clean hint.output_complete && + + # Capture long format output using the status cache and verify + # that the output matches the non-cached version. There are 2 + # ways to specify untracked-files, so do them both. + + git -C hint status --deserialize=../hint.dat -unormal >hint.d1_normal && + test_cmp expect.clean hint.d1_normal && + git -C hint -c status.showuntrackedfiles=normal status --deserialize=../hint.dat >hint.d2_normal && + test_cmp expect.clean hint.d2_normal && + + git -C hint status --deserialize=../hint.dat -uall >hint.d1_all && + test_cmp expect.clean hint.d1_all && + git -C hint -c status.showuntrackedfiles=all status --deserialize=../hint.dat >hint.d2_all && + test_cmp expect.clean hint.d2_all && + + git -C hint status --deserialize=../hint.dat -uno >hint.d1_no && + test_cmp expect.use_u hint.d1_no && + git -C hint -c status.showuntrackedfiles=no status --deserialize=../hint.dat >hint.d2_no && + test_cmp expect.use_u hint.d2_no + +' + test_done diff --git a/wt-status-deserialize.c b/wt-status-deserialize.c index ff906bc23100e4..2224cdbfcfde6c 100644 --- a/wt-status-deserialize.c +++ b/wt-status-deserialize.c @@ -425,20 +425,24 @@ static int wt_deserialize_v1_ignored_items(struct wt_status *s, } static int validate_untracked_files_arg(enum untracked_status_type cmd, - enum untracked_status_type des, + enum untracked_status_type *des, enum deserialize_parse_strategy *strategy) { *strategy = DESERIALIZE_STRATEGY_AS_IS; - if (cmd == des) { + if (cmd == *des) { *strategy = DESERIALIZE_STRATEGY_AS_IS; } else if (cmd == SHOW_NO_UNTRACKED_FILES) { *strategy = DESERIALIZE_STRATEGY_SKIP; - } else if (des == SHOW_COMPLETE_UNTRACKED_FILES) { - if (cmd == SHOW_ALL_UNTRACKED_FILES) + *des = cmd; + } else if (*des == SHOW_COMPLETE_UNTRACKED_FILES) { + if (cmd == SHOW_ALL_UNTRACKED_FILES) { *strategy = DESERIALIZE_STRATEGY_ALL; - else if (cmd == SHOW_NORMAL_UNTRACKED_FILES) + *des = cmd; + } else if (cmd == SHOW_NORMAL_UNTRACKED_FILES) { *strategy = DESERIALIZE_STRATEGY_NORMAL; + *des = cmd; + } } else { return DESERIALIZE_ERR; } @@ -480,7 +484,7 @@ static int wt_deserialize_v1(const struct wt_status *cmd_s, struct wt_status *s, * We now have the header parsed. Look at the command args (as passed in), and see how to parse * the serialized data */ - if (validate_untracked_files_arg(cmd_s->show_untracked_files, s->show_untracked_files, &untracked_strategy)) { + if (validate_untracked_files_arg(cmd_s->show_untracked_files, &s->show_untracked_files, &untracked_strategy)) { trace_printf_key(&trace_deserialize, "reject: show_untracked_file: command: %d, serialized : %d", cmd_s->show_untracked_files, s->show_untracked_files); From f0e4207e5412ade809d92dad1cd743c2bc3d1f36 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 14 Jun 2019 12:38:31 -0400 Subject: [PATCH 094/289] trace2:gvfs:experiment: read_cache: annotate thread usage in read-cache Add trace2_thread_start() and trace2_thread_exit() events to the worker threads used to read the index. This gives per-thread perf data. These workers were introduced in: abb4bb83845 read-cache: load cache extensions on a worker thread 77ff1127a4c read-cache: load cache entries on worker threads Signed-off-by: Jeff Hostetler --- read-cache.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/read-cache.c b/read-cache.c index 7ffd3d1d38c509..5b99ecaff90795 100644 --- a/read-cache.c +++ b/read-cache.c @@ -2041,6 +2041,17 @@ static void *load_index_extensions(void *_data) return NULL; } +static void *load_index_extensions_threadproc(void *_data) +{ + void *result; + + trace2_thread_start("load_index_extensions"); + result = load_index_extensions(_data); + trace2_thread_exit(); + + return result; +} + /* * A helper function that will load the specified range of cache entries * from the memory mapped file and add them to the given index. @@ -2117,12 +2128,17 @@ static void *load_cache_entries_thread(void *_data) struct load_cache_entries_thread_data *p = _data; int i; + trace2_thread_start("load_cache_entries"); + /* iterate across all ieot blocks assigned to this thread */ for (i = p->ieot_start; i < p->ieot_start + p->ieot_blocks; i++) { p->consumed += load_cache_entry_block(p->istate, p->ce_mem_pool, p->offset, p->ieot->entries[i].nr, p->mmap, p->ieot->entries[i].offset, NULL); p->offset += p->ieot->entries[i].nr; } + + trace2_thread_exit(); + return NULL; } @@ -2292,7 +2308,7 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist) int err; p.src_offset = extension_offset; - err = pthread_create(&p.pthread, NULL, load_index_extensions, &p); + err = pthread_create(&p.pthread, NULL, load_index_extensions_threadproc, &p); if (err) die(_("unable to create load_index_extensions thread: %s"), strerror(err)); From 14afb0c1f74cadee44fa85571314e70d947020d3 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 13 Aug 2025 12:47:31 +0200 Subject: [PATCH 095/289] gvfs: allow corrupt objects to be re-downloaded As of 9e59b38c88c (object-file: emit corruption errors when detected, 2022-12-14), Git will loudly complain about corrupt objects. That is fine, as long as the idea isn't to re-download locally-corrupted objects. But that's exactly what we want to do in VFS for Git via the `read-object` hook, as per the `GitCorruptObjectTests` code added in https://github.com/microsoft/VFSForGit/commit/2db0c030eb25 (New features: [...] - GVFS can now recover from corrupted git object files [...] , 2018-02-16). So let's support precisely that, and add a regression test that ensures that re-downloading corrupt objects via the `read-object` hook works. While at it, avoid the XOR operator to flip the bits, when we actually want to make sure that they are turned off: Use the AND-NOT operator for that purpose. Helped-by: Matthew John Cheetham Helped-by: Derrick Stolee Signed-off-by: Johannes Schindelin --- commit.c | 9 +++++++-- odb.c | 3 +++ t/t1060-object-corruption.sh | 30 ++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/commit.c b/commit.c index e9221da01a8698..fc904cfa14562d 100644 --- a/commit.c +++ b/commit.c @@ -622,9 +622,14 @@ int repo_parse_commit_internal(struct repository *r, int flags = OBJECT_INFO_LOOKUP_REPLACE | OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_DIE_IF_CORRUPT; - /* But the GVFS Protocol _does_ support missing commits! */ + /* + * But the GVFS Protocol _does_ support missing commits! + * And the idea with VFS for Git is to re-download corrupted objects, + * not to fail! + */ if (gvfs_config_is_set(r, GVFS_MISSING_OK)) - flags ^= OBJECT_INFO_SKIP_FETCH_OBJECT; + flags &= ~(OBJECT_INFO_SKIP_FETCH_OBJECT | + OBJECT_INFO_DIE_IF_CORRUPT); if (!item) return -1; diff --git a/odb.c b/odb.c index 51fb2a0142d793..c8588040440ac1 100644 --- a/odb.c +++ b/odb.c @@ -874,6 +874,9 @@ void *odb_read_object(struct object_database *odb, unsigned flags = OBJECT_INFO_DIE_IF_CORRUPT | OBJECT_INFO_LOOKUP_REPLACE; void *data; + if (gvfs_config_is_set(odb->repo, GVFS_MISSING_OK)) + flags &= ~OBJECT_INFO_DIE_IF_CORRUPT; + oi.typep = type; oi.sizep = size; oi.contentp = &data; diff --git a/t/t1060-object-corruption.sh b/t/t1060-object-corruption.sh index d2ef468b4528ea..3bb56bb7ba8171 100755 --- a/t/t1060-object-corruption.sh +++ b/t/t1060-object-corruption.sh @@ -3,6 +3,7 @@ test_description='see how we handle various forms of corruption' . ./test-lib.sh +. "$TEST_DIRECTORY"/lib-diff-data.sh # convert "1234abcd" to ".git/objects/12/34abcd" obj_to_file() { @@ -62,6 +63,35 @@ test_expect_success 'streaming a corrupt blob fails' ' ) ' +test_expect_success PERL 'truncated objects can be re-retrieved via GVFS' ' + git init truncated && + COPYING_test_data >truncated/COPYING && + git -C truncated add COPYING && + test_tick && + git -C truncated commit -m initial COPYING && + + # set up the `read-object` hook so that it overwrites the corrupt object + mkdir -p truncated/.git/hooks && + sed -e "1s|/usr/bin/perl|$PERL_PATH|" \ + -e "s/system/unlink \".git\/objects\/\" . substr(\$sha1, 0, 2) . \"\/\" . substr(\$sha1, 2); &/" \ + <$TEST_DIRECTORY/t0410/read-object \ + >truncated/.git/hooks/read-object && + chmod +x truncated/.git/hooks/read-object && + + # ensure that the parent repository has a copy of the object, from + # where the `read-object` can read it + sha="$(git hash-object -w truncated/COPYING)" && + file=$(obj_to_file $sha) && + size=$(test_file_size $file) && + chmod u+w truncated/$file && + test-tool truncate truncated/$file $(($size-8)) && + + rm truncated/COPYING && + test_must_fail git -C truncated reset --hard && + git -C truncated -c core.gvfs=4 -c core.virtualizeObjects \ + reset --hard +' + test_expect_success 'getting type of a corrupt blob fails' ' ( cd bit-error && From be11956e6669facd2f9056549ac5b1a21d106aa7 Mon Sep 17 00:00:00 2001 From: Matthew John Cheetham Date: Thu, 6 Mar 2025 11:14:37 +0000 Subject: [PATCH 096/289] git.c: permit repack cmd in Scalar repos Loosen the blocking of the `repack` command from all "GVFS repos" (those that have `core.gvfs` set) to only those that actually use the virtual file system (VFS for Git only). This allows for `repack` to be used in Scalar clones. Signed-off-by: Matthew John Cheetham --- git.c | 2 +- t/t0402-block-command-on-gvfs.sh | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/git.c b/git.c index 82e5568641bc29..75caf592bb73a9 100644 --- a/git.c +++ b/git.c @@ -740,7 +740,7 @@ static struct cmd_struct commands[] = { { "remote", cmd_remote, RUN_SETUP }, { "remote-ext", cmd_remote_ext, NO_PARSEOPT }, { "remote-fd", cmd_remote_fd, NO_PARSEOPT }, - { "repack", cmd_repack, RUN_SETUP | BLOCK_ON_GVFS_REPO }, + { "repack", cmd_repack, RUN_SETUP | BLOCK_ON_VFS_ENABLED }, { "replace", cmd_replace, RUN_SETUP }, { "replay", cmd_replay, RUN_SETUP }, { "repo", cmd_repo, RUN_SETUP }, diff --git a/t/t0402-block-command-on-gvfs.sh b/t/t0402-block-command-on-gvfs.sh index 3ec7620ce6194d..a8f7fb07a603e2 100755 --- a/t/t0402-block-command-on-gvfs.sh +++ b/t/t0402-block-command-on-gvfs.sh @@ -22,7 +22,6 @@ not_with_gvfs fsck not_with_gvfs gc not_with_gvfs gc --auto not_with_gvfs prune -not_with_gvfs repack not_with_gvfs submodule status not_with_gvfs update-index --index-version 2 not_with_gvfs update-index --skip-worktree @@ -36,4 +35,14 @@ test_expect_success 'test gc --auto succeeds when disabled via config' ' git gc --auto ' +test_expect_success 'test repack fails with VFS bit enabled' ' + test_config core.gvfs true && + test_must_fail git repack +' + +test_expect_success 'test repack succeeds with VFS bit disabled' ' + test_config core.gvfs 150 && + git repack +' + test_done From 155b7d17adbbc093bd19401d8f25ded20ba2d2dd Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Wed, 1 Aug 2018 13:26:22 -0400 Subject: [PATCH 097/289] virtualfilesystem: don't run the virtual file system hook if the index has been redirected Fixes #13 Some git commands spawn helpers and redirect the index to a different location. These include "difftool -d" and the sequencer (i.e. `git rebase -i`, `git cherry-pick` and `git revert`) and others. In those instances we don't want to update their temporary index with our virtualization data. Helped-by: Johannes Schindelin Signed-off-by: Ben Peart --- config.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/config.c b/config.c index 7bded08a21d308..0abf5c1cde1c8a 100644 --- a/config.c +++ b/config.c @@ -2555,11 +2555,25 @@ int repo_config_get_virtualfilesystem(struct repository *r) if (core_virtualfilesystem && !*core_virtualfilesystem) FREE_AND_NULL(core_virtualfilesystem); - /* virtual file system relies on the sparse checkout logic so force it on */ if (core_virtualfilesystem) { - cfg->apply_sparse_checkout = 1; - virtual_filesystem_result = 1; - return 1; + /* + * Some git commands spawn helpers and redirect the index to a different + * location. These include "difftool -d" and the sequencer + * (i.e. `git rebase -i`, `git cherry-pick` and `git revert`) and others. + * In those instances we don't want to update their temporary index with + * our virtualization data. + */ + char *default_index_file = xstrfmt("%s/%s", r->gitdir, "index"); + int should_run_hook = !strcmp(default_index_file, r->index_file); + + free(default_index_file); + if (should_run_hook) { + /* virtual file system relies on the sparse checkout logic so force it on */ + cfg->apply_sparse_checkout = 1; + virtual_filesystem_result = 1; + return 1; + } + FREE_AND_NULL(core_virtualfilesystem); } virtual_filesystem_result = 0; From f29d65eae717dd3882f454845df6cd9aa0d06b8a Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Thu, 21 Nov 2019 12:01:04 -0700 Subject: [PATCH 098/289] fsmonitor: check CE_FSMONITOR_VALID in ce_uptodate When using fsmonitor the CE_FSMONITOR_VALID flag should be checked when wanting to know if the entry has been updated. If the flag is set the entry should be considered up to date and the same as if the CE_UPTODATE is set. In order to trust the CE_FSMONITOR_VALID flag, the fsmonitor data needs to be refreshed when the fsmonitor bitmap is applied to the index in tweak_fsmonitor. Since the fsmonitor data is kept up to date for every command, some tests needed to be updated to take that into account. istate->untracked->use_fsmonitor was set in tweak_fsmonitor when the fsmonitor bitmap data was loaded and is now in refresh_fsmonitor since that is being called in tweak_fsmonitor. refresh_fsmonitor will only be called once and any other callers should be setting it when refreshing the fsmonitor data so that code can use the fsmonitor data when checking untracked files. When writing the index, fsmonitor_last_update is used to determine if the fsmonitor bitmap should be created and the extension data written to the index. When running through unpack-trees this is not copied to the result index. This makes the next time a git command is ran do all the work of lstating all files to determine what is clean since all entries in the index are marked as dirty since there wasn't any fsmonitor data saved in the index extension. Copying the fsmonitor_last_update to the result index will cause the extension data for fsmonitor to be in the index for the next git command to use. Signed-off-by: Kevin Willford --- read-cache-ll.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/read-cache-ll.h b/read-cache-ll.h index d279059b5fa428..31fa370e43d379 100644 --- a/read-cache-ll.h +++ b/read-cache-ll.h @@ -118,7 +118,7 @@ static inline unsigned create_ce_flags(unsigned stage) #define ce_namelen(ce) ((ce)->ce_namelen) #define ce_size(ce) cache_entry_size(ce_namelen(ce)) #define ce_stage(ce) ((CE_STAGEMASK & (ce)->ce_flags) >> CE_STAGESHIFT) -#define ce_uptodate(ce) ((ce)->ce_flags & CE_UPTODATE) +#define ce_uptodate(ce) (((ce)->ce_flags & CE_UPTODATE) || ((ce)->ce_flags & CE_FSMONITOR_VALID)) #define ce_skip_worktree(ce) ((ce)->ce_flags & CE_SKIP_WORKTREE) #define ce_mark_uptodate(ce) ((ce)->ce_flags |= CE_UPTODATE) #define ce_intent_to_add(ce) ((ce)->ce_flags & CE_INTENT_TO_ADD) From dc929de52c35da4b0a38d2016233f99b003a4443 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 9 Jul 2019 14:43:47 -0400 Subject: [PATCH 099/289] trace2:gvfs:experiment: read-cache: time read/write of cache-tree extension Add regions around code to read and write the cache-tree extension when the index is read or written. This is an experiment and may be dropped in future releases if we don't need it anymore. This experiment demonstrates that it takes more time to parse and deserialize the cache-tree extension than it does to read the cache-entries. Commits [1] and [2] spreads cache-entry reading across N-1 cores and dedicates a single core to simultaneously read the index extensions. Local testing (on my machine) shows that reading the cache-tree extension takes ~0.28 seconds. The 11 cache-entry threads take ~0.08 seconds. The main thread is blocked for 0.15 to 0.20 seconds waiting for the extension thread to finish. Let's use this commit to gather some telemetry and confirm this. My point is that improvements, such as index V5 which makes the cache entries smaller, may improve performance, but the gains may be limited because of this extension. And that we may need to look inside the cache-tree extension to truly improve do_read_index() performance. [1] abb4bb83845 read-cache: load cache extensions on a worker thread [2] 77ff1127a4c read-cache: load cache entries on worker threads Signed-off-by: Jeff Hostetler --- read-cache.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/read-cache.c b/read-cache.c index 5b99ecaff90795..b808401099a7e7 100644 --- a/read-cache.c +++ b/read-cache.c @@ -1751,7 +1751,10 @@ static int read_index_extension(struct index_state *istate, { switch (CACHE_EXT(ext)) { case CACHE_EXT_TREE: + trace2_region_enter("index", "read/extension/cache_tree", NULL); istate->cache_tree = cache_tree_read(istate->repo, data, sz); + trace2_data_intmax("index", NULL, "read/extension/cache_tree/bytes", (intmax_t)sz); + trace2_region_leave("index", "read/extension/cache_tree", NULL); break; case CACHE_EXT_RESOLVE_UNDO: istate->resolve_undo = resolve_undo_read(data, sz, the_hash_algo); @@ -3032,9 +3035,13 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, !drop_cache_tree && istate->cache_tree) { strbuf_reset(&sb); + trace2_region_enter("index", "write/extension/cache_tree", NULL); cache_tree_write(istate->repo, &sb, istate->cache_tree); err = write_index_ext_header(f, eoie_c, CACHE_EXT_TREE, sb.len) < 0; hashwrite(f, sb.buf, sb.len); + trace2_data_intmax("index", NULL, "write/extension/cache_tree/bytes", (intmax_t)sb.len); + trace2_region_leave("index", "write/extension/cache_tree", NULL); + if (err) { ret = -1; goto out; From df9d8552248c9d73aa4531901fdaca3a1f392a98 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Mon, 3 Jul 2017 13:39:45 -0600 Subject: [PATCH 100/289] cache-tree: remove use of strbuf_addf in update_one String formatting can be a performance issue when there are hundreds of thousands of trees. Change to stop using the strbuf_addf and just add the strings or characters individually. There are a limited number of modes so added a switch for the known ones and a default case if something comes through that are not a known one for git. In one scenario regarding a huge worktree, this reduces the time required for a `git checkout ` from 44 seconds to 38 seconds, i.e. it is a non-negligible performance improvement. Signed-off-by: Kevin Willford --- cache-tree.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/cache-tree.c b/cache-tree.c index 256a51dba00e5b..afab0a60990ea1 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -489,7 +489,29 @@ static int update_one(struct repository *repo, continue; strbuf_grow(&buffer, entlen + 100); - strbuf_addf(&buffer, "%o %.*s%c", mode, entlen, path + baselen, '\0'); + + switch (mode) { + case 0100644: + strbuf_add(&buffer, "100644 ", 7); + break; + case 0100664: + strbuf_add(&buffer, "100664 ", 7); + break; + case 0100755: + strbuf_add(&buffer, "100755 ", 7); + break; + case 0120000: + strbuf_add(&buffer, "120000 ", 7); + break; + case 0160000: + strbuf_add(&buffer, "160000 ", 7); + break; + default: + strbuf_addf(&buffer, "%o ", mode); + break; + } + strbuf_add(&buffer, path + baselen, entlen); + strbuf_addch(&buffer, '\0'); strbuf_add(&buffer, oid->hash, repo->hash_algo->rawsz); #if DEBUG_CACHE_TREE From 491ddfa026279b10e7bd4385a5b803c579af1346 Mon Sep 17 00:00:00 2001 From: Matthew John Cheetham Date: Thu, 13 Mar 2025 13:31:14 +0000 Subject: [PATCH 101/289] git.c: permit fsck cmd in Scalar repos Loosen the blocking of the `fsck` command from all "GVFS repos" (those that have `core.gvfs` set) to only those that actually use the virtual file system (VFS for Git only). This allows for `fsck` to be used in Scalar clones. Signed-off-by: Matthew John Cheetham --- git.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git.c b/git.c index 75caf592bb73a9..928eb1413aee04 100644 --- a/git.c +++ b/git.c @@ -682,7 +682,7 @@ static struct cmd_struct commands[] = { { "for-each-repo", cmd_for_each_repo, RUN_SETUP_GENTLY }, { "format-patch", cmd_format_patch, RUN_SETUP }, { "format-rev", cmd_format_rev, RUN_SETUP }, - { "fsck", cmd_fsck, RUN_SETUP | BLOCK_ON_GVFS_REPO}, + { "fsck", cmd_fsck, RUN_SETUP | BLOCK_ON_VFS_ENABLED }, { "fsck-objects", cmd_fsck, RUN_SETUP }, { "fsmonitor--daemon", cmd_fsmonitor__daemon, RUN_SETUP }, { "gc", cmd_gc, RUN_SETUP }, From 71d3838ed70fd40e308a09a8a83471497392ea8e Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 9 Oct 2018 10:19:14 -0600 Subject: [PATCH 102/289] virtualfilesystem: check if directory is included Add check to see if a directory is included in the virtualfilesystem before checking the directory hashmap. This allows a directory entry like foo/ to find all untracked files in subdirectories. --- t/t1093-virtualfilesystem.sh | 2 ++ virtualfilesystem.c | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/t/t1093-virtualfilesystem.sh b/t/t1093-virtualfilesystem.sh index bd0c9f72ba3c4a..8ba9a2a75e093a 100755 --- a/t/t1093-virtualfilesystem.sh +++ b/t/t1093-virtualfilesystem.sh @@ -222,6 +222,8 @@ test_expect_success 'verify folder entries include all files' ' cat > expected <<-\EOF && ?? dir1/a ?? dir1/b + ?? dir1/dir2/a + ?? dir1/dir2/b ?? dir1/untracked.txt EOF test_cmp expected actual diff --git a/virtualfilesystem.c b/virtualfilesystem.c index acb0963d14ea2b..450584b106fb7b 100644 --- a/virtualfilesystem.c +++ b/virtualfilesystem.c @@ -236,6 +236,10 @@ int is_excluded_from_virtualfilesystem(const char *pathname, int pathlen, int dt } if (dtype == DT_DIR) { + int ret = is_included_in_virtualfilesystem(pathname, pathlen); + if (ret > 0) + return 0; + if (!parent_directory_hashmap.tablesize && virtual_filesystem_data.len) initialize_parent_directory_hashmap(&parent_directory_hashmap, &virtual_filesystem_data); if (!parent_directory_hashmap.tablesize) From a9167736c9db7892eb2cfbfce220c9496576212f Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Thu, 21 Nov 2019 09:24:36 -0700 Subject: [PATCH 103/289] fsmonitor: add script for debugging and update script for tests The fsmonitor script that can be used for running all the git tests using watchman was causing some of the tests to fail because it wrote to stderr and created some files for debugging purposes. Add a new debug script to use with debugging and modify the other script to remove the code that would cause tests to fail. Signed-off-by: Kevin Willford --- t/t7519/fsmonitor-watchman | 22 +----- t/t7519/fsmonitor-watchman-debug | 128 +++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 21 deletions(-) create mode 100755 t/t7519/fsmonitor-watchman-debug diff --git a/t/t7519/fsmonitor-watchman b/t/t7519/fsmonitor-watchman index bcc055c1e0945c..9bddc61d863160 100755 --- a/t/t7519/fsmonitor-watchman +++ b/t/t7519/fsmonitor-watchman @@ -17,7 +17,6 @@ use IPC::Open2; # 'git config core.fsmonitor .git/hooks/query-watchman' # my ($version, $time) = @ARGV; -#print STDERR "$0 $version $time\n"; # Check the hook interface version @@ -42,7 +41,7 @@ launch_watchman(); sub launch_watchman { - my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j') + my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j --no-pretty') or die "open2() failed: $!\n" . "Falling back to scanning...\n"; @@ -60,19 +59,11 @@ sub launch_watchman { "fields": ["name"] }] END - - open (my $fh, ">", ".git/watchman-query.json"); - print $fh $query; - close $fh; print CHLD_IN $query; close CHLD_IN; my $response = do {local $/; }; - open ($fh, ">", ".git/watchman-response.json"); - print $fh $response; - close $fh; - die "Watchman: command returned no output.\n" . "Falling back to scanning...\n" if $response eq ""; die "Watchman: command returned invalid output: $response\n" . @@ -91,7 +82,6 @@ sub launch_watchman { my $o = $json_pkg->new->utf8->decode($response); if ($o->{error} and $o->{error} =~ m/unable to resolve root .* directory (.*) is not watched/) { - print STDERR "Adding '$git_work_tree' to watchman's watch list.\n"; qx/watchman watch "$git_work_tree"/; die "Failed to make watchman watch '$git_work_tree'.\n" . "Falling back to scanning...\n" if $? != 0; @@ -100,11 +90,6 @@ sub launch_watchman { # return the fast "everything is dirty" flag to git and do the # Watchman query just to get it over with now so we won't pay # the cost in git to look up each individual file. - - open ($fh, ">", ".git/watchman-output.out"); - print "/\0"; - close $fh; - print "/\0"; exit 0; } @@ -112,11 +97,6 @@ sub launch_watchman { die "Watchman: $o->{error}.\n" . "Falling back to scanning...\n" if $o->{error}; - open ($fh, ">", ".git/watchman-output.out"); - binmode $fh, ":utf8"; - print $fh @{$o->{files}}; - close $fh; - binmode STDOUT, ":utf8"; local $, = "\0"; print @{$o->{files}}; diff --git a/t/t7519/fsmonitor-watchman-debug b/t/t7519/fsmonitor-watchman-debug new file mode 100755 index 00000000000000..d8e7a1e5ba85c0 --- /dev/null +++ b/t/t7519/fsmonitor-watchman-debug @@ -0,0 +1,128 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use IPC::Open2; + +# An example hook script to integrate Watchman +# (https://facebook.github.io/watchman/) with git to speed up detecting +# new and modified files. +# +# The hook is passed a version (currently 1) and a time in nanoseconds +# formatted as a string and outputs to stdout all files that have been +# modified since the given time. Paths must be relative to the root of +# the working tree and separated by a single NUL. +# +# To enable this hook, rename this file to "query-watchman" and set +# 'git config core.fsmonitor .git/hooks/query-watchman' +# +my ($version, $time) = @ARGV; +#print STDERR "$0 $version $time\n"; + +# Check the hook interface version + +if ($version == 1) { + # convert nanoseconds to seconds + # subtract one second to make sure watchman will return all changes + $time = int ($time / 1000000000) - 1; +} else { + die "Unsupported query-fsmonitor hook version '$version'.\n" . + "Falling back to scanning...\n"; +} + +my $git_work_tree; +if ($^O =~ 'msys' || $^O =~ 'cygwin') { + $git_work_tree = Win32::GetCwd(); + $git_work_tree =~ tr/\\/\//; +} else { + require Cwd; + $git_work_tree = Cwd::cwd(); +} + +my $retry = 1; + +launch_watchman(); + +sub launch_watchman { + + my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j') + or die "open2() failed: $!\n" . + "Falling back to scanning...\n"; + + # In the query expression below we're asking for names of files that + # changed since $time but were not transient (ie created after + # $time but no longer exist). + # + # To accomplish this, we're using the "since" generator to use the + # recency index to select candidate nodes and "fields" to limit the + # output to file names only. + + my $query = <<" END"; + ["query", "$git_work_tree", { + "since": $time, + "fields": ["name"] + }] + END + + open (my $fh, ">", ".git/watchman-query.json"); + print $fh $query; + close $fh; + + print CHLD_IN $query; + close CHLD_IN; + my $response = do {local $/; }; + + open ($fh, ">", ".git/watchman-response.json"); + print $fh $response; + close $fh; + + die "Watchman: command returned no output.\n" . + "Falling back to scanning...\n" if $response eq ""; + die "Watchman: command returned invalid output: $response\n" . + "Falling back to scanning...\n" unless $response =~ /^\{/; + + my $json_pkg; + eval { + require JSON::XS; + $json_pkg = "JSON::XS"; + 1; + } or do { + require JSON::PP; + $json_pkg = "JSON::PP"; + }; + + my $o = $json_pkg->new->utf8->decode($response); + + if ($retry > 0 and $o->{error} and $o->{error} =~ m/unable to resolve root .* directory (.*) is not watched/) { + print STDERR "Adding '$git_work_tree' to watchman's watch list.\n"; + $retry--; + qx/watchman watch "$git_work_tree"/; + die "Failed to make watchman watch '$git_work_tree'.\n" . + "Falling back to scanning...\n" if $? != 0; + + # Watchman will always return all files on the first query so + # return the fast "everything is dirty" flag to git and do the + # Watchman query just to get it over with now so we won't pay + # the cost in git to look up each individual file. + + open ($fh, ">", ".git/watchman-output.out"); + print "/\0"; + close $fh; + + print "/\0"; + eval { launch_watchman() }; + exit 0; + } + + die "Watchman: $o->{error}.\n" . + "Falling back to scanning...\n" if $o->{error}; + + open ($fh, ">", ".git/watchman-output.out"); + binmode $fh, ":utf8"; + print $fh @{$o->{files}}; + close $fh; + + binmode STDOUT, ":utf8"; + local $, = "\0"; + print @{$o->{files}}; +} From b407c07aa3ef39e2c31186b238072fe4926cf47d Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 16 Jul 2019 09:09:53 -0400 Subject: [PATCH 104/289] trace2:gvfs:experiment: add region to apply_virtualfilesystem() Signed-off-by: Jeff Hostetler --- virtualfilesystem.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/virtualfilesystem.c b/virtualfilesystem.c index 947b1516417fb3..b079ddfa1d03ac 100644 --- a/virtualfilesystem.c +++ b/virtualfilesystem.c @@ -268,6 +268,8 @@ void apply_virtualfilesystem(struct index_state *istate) if (!repo_config_get_virtualfilesystem(istate->repo)) return; + trace2_region_enter("vfs", "apply", the_repository); + if (!virtual_filesystem_data.len) get_virtual_filesystem_data(istate->repo, &virtual_filesystem_data); @@ -338,6 +340,8 @@ void apply_virtualfilesystem(struct index_state *istate) trace2_data_intmax("vfs", the_repository, "apply/nr_bulk_skip", nr_bulk_skip); trace2_data_intmax("vfs", the_repository, "apply/nr_explicit_skip", nr_explicit_skip); } + + trace2_region_leave("vfs", "apply", the_repository); } /* From d744a50fffb77f9692aea47c72a868f6e5e8c928 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Thu, 6 Dec 2018 11:09:19 -0500 Subject: [PATCH 105/289] gvfs: block unsupported commands when running in a GVFS repo The following commands and options are not currently supported when working in a GVFS repo. Add code to detect and block these commands from executing. 1) fsck 2) gc 4) prune 5) repack 6) submodule 8) update-index --split-index 9) update-index --index-version (other than 4) 10) update-index --[no-]skip-worktree 11) worktree Signed-off-by: Ben Peart Signed-off-by: Johannes Schindelin --- builtin/gc.c | 11 ++++++++- builtin/update-index.c | 12 +++++++++- git.c | 15 ++++++++---- gvfs.h | 1 + t/meson.build | 1 + t/t0402-block-command-on-gvfs.sh | 39 ++++++++++++++++++++++++++++++++ 6 files changed, 72 insertions(+), 7 deletions(-) create mode 100755 t/t0402-block-command-on-gvfs.sh diff --git a/builtin/gc.c b/builtin/gc.c index 57a3520263d7be..b8b6ac6eb5031a 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -19,6 +19,7 @@ #include "dir.h" #include "environment.h" #include "hex.h" +#include "gvfs.h" #include "config.h" #include "tempfile.h" #include "lockfile.h" @@ -566,7 +567,7 @@ static int maintenance_task_odb(struct maintenance_run_opts *opts, int cmd_gc(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { int aggressive = 0; int force = 0; @@ -635,6 +636,14 @@ int cmd_gc(int argc, if (cfg.prune_expire && parse_expiry_date(cfg.prune_expire, &dummy)) die(_("failed to parse prune expiry value %s"), cfg.prune_expire); + if (gvfs_config_is_set(repo, GVFS_BLOCK_COMMANDS)) { + int gc_auto_threshold = 6700; + if (!opts.auto_flag || + repo_config_get_int(repo, "gc.auto", &gc_auto_threshold) || + gc_auto_threshold > 0) + die(_("'git gc' is not supported on a GVFS repo")); + } + if (opts.auto_flag) { struct odb_optimize_options optimize_opts = { .strategy = ODB_OPTIMIZE_INCREMENTAL, diff --git a/builtin/update-index.c b/builtin/update-index.c index b25d4ecb1091ba..f84553b0e51b46 100644 --- a/builtin/update-index.c +++ b/builtin/update-index.c @@ -8,6 +8,7 @@ #define DISABLE_SIGN_COMPARE_WARNINGS #include "builtin.h" +#include "gvfs.h" #include "config.h" #include "environment.h" #include "gettext.h" @@ -915,7 +916,7 @@ static enum parse_opt_result reupdate_callback( int cmd_update_index(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { int newfd, entries, has_errors = 0, nul_term_line = 0; enum uc_mode untracked_cache = UC_UNSPECIFIED; @@ -1181,7 +1182,13 @@ int cmd_update_index(int argc, argc = parse_options_end(&ctx); getline_fn = nul_term_line ? strbuf_getline_nul : strbuf_getline_lf; + if (mark_skip_worktree_only && gvfs_config_is_set(repo, GVFS_BLOCK_COMMANDS)) + die(_("modifying the skip worktree bit is not supported on a GVFS repo")); + if (preferred_index_format) { + if (preferred_index_format != 4 && gvfs_config_is_set(repo, GVFS_BLOCK_COMMANDS)) + die(_("changing the index version is not supported on a GVFS repo")); + if (preferred_index_format < 0) { printf(_("%d\n"), the_repository->index->version); } else if (preferred_index_format < INDEX_FORMAT_LB || @@ -1227,6 +1234,9 @@ int cmd_update_index(int argc, odb_transaction_commit_and_finalize_or_die(transaction); if (split_index > 0) { + if (gvfs_config_is_set(repo, GVFS_BLOCK_COMMANDS)) + die(_("split index is not supported on a GVFS repo")); + if (repo_config_get_split_index(the_repository) == 0) warning(_("core.splitIndex is set to false; " "remove or change it, if you really want to " diff --git a/git.c b/git.c index c21d2e2966cc1a..de4aef3cbc8dbe 100644 --- a/git.c +++ b/git.c @@ -1,6 +1,7 @@ #define USE_THE_REPOSITORY_VARIABLE #include "builtin.h" +#include "gvfs.h" #include "config.h" #include "environment.h" #include "exec-cmd.h" @@ -31,6 +32,7 @@ #define DELAY_PAGER_CONFIG (1<<4) #define NO_PARSEOPT (1<<5) /* parse-options is not used */ #define DEPRECATED (1<<6) +#define BLOCK_ON_GVFS_REPO (1<<7) /* command not allowed in GVFS repos */ struct cmd_struct { const char *cmd; @@ -588,6 +590,9 @@ static int run_builtin(struct cmd_struct *p, int argc, const char **argv, struct if (!help && p->option & NEED_WORK_TREE) setup_work_tree(the_repository); + if (!help && p->option & BLOCK_ON_GVFS_REPO && gvfs_config_is_set(repo, GVFS_BLOCK_COMMANDS)) + die("'git %s' is not supported on a GVFS repo", p->cmd); + if (run_pre_command_hook(the_repository, argv)) die("pre-command hook aborted command"); @@ -673,7 +678,7 @@ static struct cmd_struct commands[] = { { "for-each-repo", cmd_for_each_repo, RUN_SETUP_GENTLY }, { "format-patch", cmd_format_patch, RUN_SETUP }, { "format-rev", cmd_format_rev, RUN_SETUP }, - { "fsck", cmd_fsck, RUN_SETUP }, + { "fsck", cmd_fsck, RUN_SETUP | BLOCK_ON_GVFS_REPO}, { "fsck-objects", cmd_fsck, RUN_SETUP }, { "fsmonitor--daemon", cmd_fsmonitor__daemon, RUN_SETUP }, { "gc", cmd_gc, RUN_SETUP }, @@ -718,7 +723,7 @@ static struct cmd_struct commands[] = { { "pack-refs", cmd_pack_refs, RUN_SETUP }, { "patch-id", cmd_patch_id, RUN_SETUP_GENTLY | NO_PARSEOPT }, { "pickaxe", cmd_blame, RUN_SETUP }, - { "prune", cmd_prune, RUN_SETUP }, + { "prune", cmd_prune, RUN_SETUP | BLOCK_ON_GVFS_REPO}, { "prune-packed", cmd_prune_packed, RUN_SETUP }, { "pull", cmd_pull, RUN_SETUP | NEED_WORK_TREE }, { "push", cmd_push, RUN_SETUP }, @@ -731,7 +736,7 @@ static struct cmd_struct commands[] = { { "remote", cmd_remote, RUN_SETUP }, { "remote-ext", cmd_remote_ext, NO_PARSEOPT }, { "remote-fd", cmd_remote_fd, NO_PARSEOPT }, - { "repack", cmd_repack, RUN_SETUP }, + { "repack", cmd_repack, RUN_SETUP | BLOCK_ON_GVFS_REPO }, { "replace", cmd_replace, RUN_SETUP }, { "replay", cmd_replay, RUN_SETUP }, { "repo", cmd_repo, RUN_SETUP }, @@ -753,7 +758,7 @@ static struct cmd_struct commands[] = { { "stash", cmd_stash, RUN_SETUP | NEED_WORK_TREE }, { "status", cmd_status, RUN_SETUP | NEED_WORK_TREE }, { "stripspace", cmd_stripspace }, - { "submodule--helper", cmd_submodule__helper, RUN_SETUP }, + { "submodule--helper", cmd_submodule__helper, RUN_SETUP | BLOCK_ON_GVFS_REPO }, { "survey", cmd_survey, RUN_SETUP }, { "switch", cmd_switch, RUN_SETUP | NEED_WORK_TREE }, { "symbolic-ref", cmd_symbolic_ref, RUN_SETUP }, @@ -775,7 +780,7 @@ static struct cmd_struct commands[] = { #ifndef WITH_BREAKING_CHANGES { "whatchanged", cmd_whatchanged, RUN_SETUP | DEPRECATED }, #endif - { "worktree", cmd_worktree, RUN_SETUP }, + { "worktree", cmd_worktree, RUN_SETUP | BLOCK_ON_GVFS_REPO }, { "write-tree", cmd_write_tree, RUN_SETUP }, }; diff --git a/gvfs.h b/gvfs.h index 59ee3893b1efa6..8368f950243c63 100644 --- a/gvfs.h +++ b/gvfs.h @@ -12,6 +12,7 @@ struct repository; * The list of bits in the core_gvfs setting */ #define GVFS_SKIP_SHA_ON_INDEX (1 << 0) +#define GVFS_BLOCK_COMMANDS (1 << 1) #define GVFS_MISSING_OK (1 << 2) /* diff --git a/t/meson.build b/t/meson.build index 392809fd278025..4ba6e06f27eaef 100644 --- a/t/meson.build +++ b/t/meson.build @@ -146,6 +146,7 @@ integration_tests = [ 't0303-credential-external.sh', 't0400-pre-command-hook.sh', 't0401-post-command-hook.sh', + 't0402-block-command-on-gvfs.sh', 't0410-partial-clone.sh', 't0411-clone-from-partial.sh', 't0450-txt-doc-vs-help.sh', diff --git a/t/t0402-block-command-on-gvfs.sh b/t/t0402-block-command-on-gvfs.sh new file mode 100755 index 00000000000000..3ec7620ce6194d --- /dev/null +++ b/t/t0402-block-command-on-gvfs.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +test_description='block commands in GVFS repo' + +. ./test-lib.sh + +not_with_gvfs () { + command=$1 && + shift && + test_expect_success "test $command $*" " + test_config alias.g4rbled $command && + test_config core.gvfs true && + test_must_fail git $command $* && + test_must_fail git g4rbled $* && + test_unconfig core.gvfs && + test_must_fail git -c core.gvfs=true $command $* && + test_must_fail git -c core.gvfs=true g4rbled $* + " +} + +not_with_gvfs fsck +not_with_gvfs gc +not_with_gvfs gc --auto +not_with_gvfs prune +not_with_gvfs repack +not_with_gvfs submodule status +not_with_gvfs update-index --index-version 2 +not_with_gvfs update-index --skip-worktree +not_with_gvfs update-index --no-skip-worktree +not_with_gvfs update-index --split-index +not_with_gvfs worktree list + +test_expect_success 'test gc --auto succeeds when disabled via config' ' + test_config core.gvfs true && + test_config gc.auto 0 && + git gc --auto +' + +test_done From 76b2053be7590b3a0a1a72020f5046376d5df49e Mon Sep 17 00:00:00 2001 From: Matthew John Cheetham Date: Thu, 13 Mar 2025 13:31:50 +0000 Subject: [PATCH 106/289] git.c: permit prune cmd in Scalar repos Loosen the blocking of the `prune` command from all "GVFS repos" (those that have `core.gvfs` set) to only those that actually use the virtual file system (VFS for Git only). This allows for `prune` to be used in Scalar clones. Signed-off-by: Matthew John Cheetham --- git.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git.c b/git.c index 928eb1413aee04..cfa4d78e66b7a3 100644 --- a/git.c +++ b/git.c @@ -727,7 +727,7 @@ static struct cmd_struct commands[] = { { "pack-refs", cmd_pack_refs, RUN_SETUP }, { "patch-id", cmd_patch_id, RUN_SETUP_GENTLY | NO_PARSEOPT }, { "pickaxe", cmd_blame, RUN_SETUP }, - { "prune", cmd_prune, RUN_SETUP | BLOCK_ON_GVFS_REPO}, + { "prune", cmd_prune, RUN_SETUP | BLOCK_ON_VFS_ENABLED }, { "prune-packed", cmd_prune_packed, RUN_SETUP }, { "pull", cmd_pull, RUN_SETUP | NEED_WORK_TREE }, { "push", cmd_push, RUN_SETUP }, From 3b7e478757e9268e14158d1133e1daa48ebc6502 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 28 May 2019 21:48:08 +0200 Subject: [PATCH 107/289] backwards-compatibility: support the post-indexchanged hook When our patches to support that hook were upstreamed, the hook's name was eliciting some reviewer suggestions, and it was renamed to `post-index-change`. These patches (with the new name) made it into v2.22.0. However, VFSforGit users may very well have checkouts with that hook installed under the original name. To support this, let's just introduce a hack where we look a bit more closely when we just failed to find the `post-index-change` hook, and allow any `post-indexchanged` hook to run instead (if it exists). --- hook.c | 12 ++++++++++++ t/t7113-post-index-change-hook.sh | 30 ++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/hook.c b/hook.c index 1b4d429e7014c9..83cde2c5a784ab 100644 --- a/hook.c +++ b/hook.c @@ -144,6 +144,18 @@ static void list_hooks_add_default(struct repository *r, const char *hookname, const char *hook_path = find_hook(r, hookname); struct hook *h; + /* + * Backwards compatibility hack in VFS for Git: when originally + * introduced (and used!), it was called `post-indexchanged`, but this + * name was changed during the review on the Git mailing list. + * + * Therefore, when the `post-index-change` hook is not found, let's + * look for a hook with the old name (which would be found in case of + * already-existing checkouts). + */ + if (!hook_path && !strcmp(hookname, "post-index-change")) + hook_path = find_hook(r, "post-indexchanged"); + if (!hook_path) return; diff --git a/t/t7113-post-index-change-hook.sh b/t/t7113-post-index-change-hook.sh index c10d94fe3d3b01..22b81a67cc32b6 100755 --- a/t/t7113-post-index-change-hook.sh +++ b/t/t7113-post-index-change-hook.sh @@ -15,6 +15,36 @@ test_expect_success 'setup' ' git commit -m "initial" ' +test_expect_success 'post-indexchanged' ' + mkdir -p .git/hooks && + test_when_finished "rm -f .git/hooks/post-indexchanged marker" && + write_script .git/hooks/post-indexchanged <<-\EOF && + : >marker + EOF + + : make sure -changed is called if -change does not exist && + test_when_finished "echo testing >dir1/file2.txt && git status" && + echo changed >dir1/file2.txt && + : force index to be dirty && + test-tool chmtime -60 .git/index && + git status && + test_path_is_file marker && + + test_when_finished "rm -f .git/hooks/post-index-change marker2" && + write_script .git/hooks/post-index-change <<-\EOF && + : >marker2 + EOF + + : make sure -changed is not called if -change exists && + rm -f marker marker2 && + echo testing >dir1/file2.txt && + : force index to be dirty && + test-tool chmtime -60 .git/index && + git status && + test_path_is_missing marker && + test_path_is_file marker2 +' + test_expect_success 'test status, add, commit, others trigger hook without flags set' ' test_hook post-index-change <<-\EOF && if test "$1" -eq 1; then From b9e929c700ef3f2499f2759d16b9d66b32ab8bae Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 10 Apr 2020 21:14:44 -0400 Subject: [PATCH 108/289] status: disable deserialize when verbose output requested. Disable deserialization when verbose output requested. Verbose mode causes Git to print diffs for modified files. This requires the index to be loaded to have the currently staged OID values. Without loading the index, verbose output make it look like everything was deleted. Signed-off-by: Jeff Hostetler --- builtin/commit.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/builtin/commit.c b/builtin/commit.c index 676c3c8f7dfbe7..b9a17f671cdf60 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -43,6 +43,7 @@ #include "commit-graph.h" #include "pretty.h" #include "trailer.h" +#include "trace2.h" static const char * const builtin_commit_usage[] = { N_("git commit [-a | --interactive | --patch] [-s] [-v] [-u[]] [--amend]\n" @@ -1804,6 +1805,22 @@ struct repository *repo UNUSED) */ try_deserialize = (!do_serialize && (do_implicit_deserialize || do_explicit_deserialize)); + + /* + * Disable deserialize when verbose is set because it causes us to + * print diffs for each modified file, but that requires us to have + * the index loaded and we don't want to do that (at least not now for + * this seldom used feature). My fear is that would further tangle + * the merge conflict with upstream. + * + * TODO Reconsider this in the future. + */ + if (try_deserialize && verbose) { + trace2_data_string("status", the_repository, "deserialize/reject", + "args/verbose"); + try_deserialize = 0; + } + if (try_deserialize) goto skip_init; From 6dfe362aa0239a8395c317434e31f1ae3e3883a6 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 16 Jul 2019 10:08:08 -0400 Subject: [PATCH 109/289] trace2:gvfs:experiment: add region around unpack_trees() Signed-off-by: Jeff Hostetler --- unpack-trees.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/unpack-trees.c b/unpack-trees.c index 29917231249344..bd700c87314526 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -1919,6 +1919,8 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options if (o->df_conflict_entry) BUG("o->df_conflict_entry is an output only field"); + trace2_region_enter("exp", "unpack_trees", NULL); + trace_performance_enter(); trace2_region_enter("unpack_trees", "unpack_trees", repo); @@ -2128,6 +2130,7 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options } trace2_region_leave("unpack_trees", "unpack_trees", repo); trace_performance_leave("unpack_trees"); + trace2_region_leave("exp", "unpack_trees", NULL); return ret; return_failed: From 5293057492ffebe018c6cf80017e51d4f8220c18 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 15 Apr 2020 16:19:31 +0000 Subject: [PATCH 110/289] gvfs: allow overriding core.gvfs In earlier versions of `microsoft/git`, we found a user who had set `core.gvfs = false` in their global config. This should not have been necessary, but it also should not have caused a problem. However, it did. The reason was that `gvfs_load_config_value()` was called from `config.c` when reading config key/value pairs from all the config files. The local config should override the global config, and this is done by `config.c` reading the global config first then reading the local config. However, our logic only allowed writing the `core_gvfs` variable once. In v2.51.0, we had to adapt to upstream changes that changed way the `core.gvfs` config value is read, and the special handling is no longer necessary, yet we still want the test case that ensures that this bug does not experience a regression. Signed-off-by: Derrick Stolee Signed-off-by: Johannes Schindelin --- t/t0021-conversion.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/t/t0021-conversion.sh b/t/t0021-conversion.sh index 3f9a000b89bec5..3fe5673825434d 100755 --- a/t/t0021-conversion.sh +++ b/t/t0021-conversion.sh @@ -348,6 +348,10 @@ test_expect_success "filter: smudge filters blocked when under GVFS" ' test_config filter.empty-in-repo.smudge "echo smudged && cat" && test_config core.gvfs 64 && + test_must_fail git checkout && + + # ensure the local core.gvfs setting overwrites the global setting + git config --global core.gvfs false && test_must_fail git checkout ' From af6ff28cd87a0cd0bc96f107fa1593d598718e54 Mon Sep 17 00:00:00 2001 From: Matthew John Cheetham Date: Thu, 6 Mar 2025 11:16:04 +0000 Subject: [PATCH 111/289] worktree: remove special case GVFS cmd blocking Replace the special casing of the `worktree` command being blocked on VFS-enabled repos with the new `BLOCK_ON_VFS_ENABLED` flag. Signed-off-by: Matthew John Cheetham --- builtin/worktree.c | 7 ------- git.c | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/builtin/worktree.c b/builtin/worktree.c index b1ab1190d88449..993266a4a9a5ab 100644 --- a/builtin/worktree.c +++ b/builtin/worktree.c @@ -1530,13 +1530,6 @@ int cmd_worktree(int ac, repo_config(the_repository, git_worktree_config, NULL); - /* - * git-worktree is special-cased to work in Scalar repositories - * even when they use the GVFS Protocol. - */ - if (gvfs_config_is_set(repo, GVFS_USE_VIRTUAL_FILESYSTEM)) - die("'git %s' is not supported on a GVFS repo", "worktree"); - if (!prefix) prefix = ""; diff --git a/git.c b/git.c index cfa4d78e66b7a3..98670fbd7aaf47 100644 --- a/git.c +++ b/git.c @@ -784,7 +784,7 @@ static struct cmd_struct commands[] = { #ifndef WITH_BREAKING_CHANGES { "whatchanged", cmd_whatchanged, RUN_SETUP | DEPRECATED }, #endif - { "worktree", cmd_worktree, RUN_SETUP | BLOCK_ON_GVFS_REPO }, + { "worktree", cmd_worktree, RUN_SETUP | BLOCK_ON_VFS_ENABLED }, { "write-tree", cmd_write_tree, RUN_SETUP }, }; From 6aa05b598b07df30d66d4bd5e4a20ae5021e2c5e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 18 Jun 2021 14:45:20 +0200 Subject: [PATCH 112/289] gvfs: verify that the built-in FSMonitor is disabled When using a virtual file system layer, the FSMonitor does not make sense. Signed-off-by: Johannes Schindelin --- t/t1093-virtualfilesystem.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/t/t1093-virtualfilesystem.sh b/t/t1093-virtualfilesystem.sh index 8ba9a2a75e093a..cad13d680cb199 100755 --- a/t/t1093-virtualfilesystem.sh +++ b/t/t1093-virtualfilesystem.sh @@ -368,4 +368,15 @@ test_expect_success 'folder with same prefix as file' ' test_cmp expected actual ' +test_expect_success MINGW,FSMONITOR_DAEMON 'virtualfilesystem hook disables built-in FSMonitor' ' + clean_repo && + test_config core.usebuiltinfsmonitor true && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/\0" + EOF + git config core.virtualfilesystem .git/hooks/virtualfilesystem && + git status && + test_must_fail git fsmonitor--daemon status +' + test_done From 7503d9991897356b460bfcce9c8d402d9fd3acda Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 10 Apr 2020 21:18:41 -0400 Subject: [PATCH 113/289] t7524: add test for verbose status deserialzation Verify that `git status --deserialize=x -v` does not crash and generates the same output as a normal (scanning) status command. These issues are described in the previous 2 commits. Signed-off-by: Jeff Hostetler --- t/t7522-serialized-status.sh | 39 ++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/t/t7522-serialized-status.sh b/t/t7522-serialized-status.sh index 5b3133b6316309..8f32f6cfe70556 100755 --- a/t/t7522-serialized-status.sh +++ b/t/t7522-serialized-status.sh @@ -403,4 +403,43 @@ EOF ' +test_expect_success 'ensure deserialize -v does not crash' ' + + git init -b main verbose_test && + touch verbose_test/a && + touch verbose_test/b && + touch verbose_test/c && + git -C verbose_test add a b c && + git -C verbose_test commit -m abc && + + echo green >>verbose_test/a && + git -C verbose_test add a && + echo red_1 >>verbose_test/b && + echo red_2 >verbose_test/dirt && + + git -C verbose_test status >output.ref && + git -C verbose_test status -v >output.ref_v && + + git -C verbose_test --no-optional-locks status --serialize=../verbose_test.dat >output.ser.long && + git -C verbose_test --no-optional-locks status --serialize=../verbose_test.dat_v -v >output.ser.long_v && + + # Verify that serialization does not affect the status output itself. + test_cmp output.ref output.ser.long && + test_cmp output.ref_v output.ser.long_v && + + GIT_TRACE2_PERF="$(pwd)"/verbose_test.log \ + git -C verbose_test status --deserialize=../verbose_test.dat >output.des.long && + + # Verify that normal deserialize was actually used and produces the same result. + test_cmp output.ser.long output.des.long && + test_grep -q "deserialize/result:ok" verbose_test.log && + + GIT_TRACE2_PERF="$(pwd)"/verbose_test.log_v \ + git -C verbose_test status --deserialize=../verbose_test.dat_v -v >output.des.long_v && + + # Verify that vebose mode produces the same result because verbose was rejected. + test_cmp output.ser.long_v output.des.long_v && + test_grep -q "deserialize/reject:args/verbose" verbose_test.log_v +' + test_done From 755435784313591d87f89c92205145fdaa5d28ba Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 16 Jul 2019 10:16:37 -0400 Subject: [PATCH 114/289] trace2:gvfs:experiment: add region to cache_tree_fully_valid() Signed-off-by: Jeff Hostetler --- cache-tree.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/cache-tree.c b/cache-tree.c index afab0a60990ea1..defdda314b0a0c 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -275,8 +275,8 @@ static void discard_unused_subtrees(struct cache_tree *it) } } -static int cache_tree_fully_valid_recursive(struct object_database *odb, - struct cache_tree *it) +static int cache_tree_fully_valid_recursive_1(struct object_database *odb, + struct cache_tree *it) { int i; if (!it) @@ -286,12 +286,24 @@ static int cache_tree_fully_valid_recursive(struct object_database *odb, ODB_HAS_OBJECT_RECHECK_PACKED | ODB_HAS_OBJECT_FETCH_PROMISOR)) return 0; for (i = 0; i < it->subtree_nr; i++) { - if (!cache_tree_fully_valid_recursive(odb, it->down[i]->cache_tree)) + if (!cache_tree_fully_valid_recursive_1(odb, it->down[i]->cache_tree)) return 0; } return 1; } +static int cache_tree_fully_valid_recursive(struct object_database *odb, + struct cache_tree *it) +{ + int result; + + trace2_region_enter("cache_tree", "fully_valid", NULL); + result = cache_tree_fully_valid_recursive_1(odb, it); + trace2_region_leave("cache_tree", "fully_valid", NULL); + + return result; +} + int cache_tree_fully_valid(struct index_state *istate) { return cache_tree_fully_valid_recursive(istate->repo->objects, From fe70f934a0f827659ac3c042778be4ff4330e31d Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Fri, 27 Jul 2018 12:00:44 -0600 Subject: [PATCH 115/289] BRANCHES.md: Add explanation of branches and using forks --- BRANCHES.md | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 BRANCHES.md diff --git a/BRANCHES.md b/BRANCHES.md new file mode 100644 index 00000000000000..364158375e7d55 --- /dev/null +++ b/BRANCHES.md @@ -0,0 +1,59 @@ +Branches used in this repo +========================== + +The document explains the branching structure that we are using in the VFSForGit repository as well as the forking strategy that we have adopted for contributing. + +Repo Branches +------------- + +1. `vfs-#` + + These branches are used to track the specific version that match Git for Windows with the VFSForGit specific patches on top. When a new version of Git for Windows is released, the VFSForGit patches will be rebased on that windows version and a new gvfs-# branch created to create pull requests against. + + #### Examples + + ``` + vfs-2.27.0 + vfs-2.30.0 + ``` + + The versions of git for VFSForGit are based on the Git for Windows versions. v2.20.0.vfs.1 will correspond with the v2.20.0.windows.1 with the VFSForGit specific patches applied to the windows version. + +2. `vfs-#-exp` + + These branches are for releasing experimental features to early adopters. They + should contain everything within the corresponding `vfs-#` branch; if the base + branch updates, then merge into the `vfs-#-exp` branch as well. + +Tags +---- + +We are using annotated tags to build the version number for git. The build will look back through the commit history to find the first tag matching `v[0-9]*vfs*` and build the git version number using that tag. + +Full releases are of the form `v2.XX.Y.vfs.Z.W` where `v2.XX.Y` comes from the +upstream version and `Z.W` are custom updates within our fork. Specifically, +the `.Z` value represents the "compatibility level" with VFS for Git. Only +increase this version when making a breaking change with a released version +of VFS for Git. The `.W` version is used for minor updates between major +versions. + +Experimental releases are of the form `v2.XX.Y.vfs.Z.W.exp`. The `.exp` +suffix indicates that experimental features are available. The rest of the +version string comes from the full release tag. These versions will only +be made available as pre-releases on the releases page, never a full release. + +Forking +------- + +A personal fork of this repository and a branch in that repository should be used for development. + +These branches should be based on the latest vfs-# branch. If there are work in progress pull requests that you have based on a previous version branch when a new version branch is created, you will need to move your patches to the new branch to get them in that latest version. + +#### Example + +``` +git clone +git remote add ms https://github.com/Microsoft/git.git +git checkout -b my-changes ms/vfs-2.20.0 --no-track +git push -fu origin HEAD +``` From b51cffb90dc16903db469021aeaf5a8f3907dca1 Mon Sep 17 00:00:00 2001 From: Matthew John Cheetham Date: Wed, 26 Mar 2025 10:31:38 +0000 Subject: [PATCH 116/289] builtin/repack.c: emit warning when shared cache is present Emit a warning message when the `gvfs.sharedCache` option is set that the `repack` command will not perform repacking on the shared cache. In the future we can teach `repack` to operate on the shared cache, at which point we can drop this commit. Signed-off-by: Matthew John Cheetham --- builtin/repack.c | 6 ++++++ gvfs.h | 2 ++ 2 files changed, 8 insertions(+) diff --git a/builtin/repack.c b/builtin/repack.c index c4360382c1fce2..a8652e2d6d447e 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -19,6 +19,7 @@ #include "hex.h" #include "wt-status.h" #include "read-cache-ll.h" +#include "gvfs.h" #define ALL_INTO_ONE 1 #define LOOSEN_UNREACHABLE 2 @@ -169,6 +170,7 @@ int cmd_repack(int argc, struct tempfile *refs_snapshot = NULL; int i, ret; int show_progress; + const char *tmp_obj_dir = NULL; /* variables to be filled by option parsing */ struct repack_config_ctx config_ctx; @@ -430,6 +432,10 @@ int cmd_repack(int argc, write_bitmaps = 0; } + if (gvfs_config_is_set(repo, GVFS_ANY_MASK) && + !repo_config_get_value(repo, "gvfs.sharedcache", &tmp_obj_dir)) + warning(_("shared object cache is configured but will not be repacked")); + if (config_ctx.midx_split_factor < 2) die(_("invalid value for %s: %d"), "--midx-split-factor", config_ctx.midx_split_factor); diff --git a/gvfs.h b/gvfs.h index 8368f950243c63..5d3159425c742e 100644 --- a/gvfs.h +++ b/gvfs.h @@ -29,6 +29,8 @@ struct repository; #define GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK (1 << 4) #define GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS (1 << 6) +#define GVFS_ANY_MASK 0xFFFFFFFF + int gvfs_config_is_set(struct repository *r, int mask); int gvfs_virtualize_objects(struct repository *r); From 32f6ed5de3b3ed291fd36a6b67fdf795e1f88441 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 21 Jun 2024 17:18:59 -0400 Subject: [PATCH 117/289] wt-status: add trace2 data for sparse-checkout percentage When sparse-checkout is enabled, add the sparse-checkout percentage to the Trace2 data stream. This number was already computed and printed on the console in the "You are in a sparse checkout..." message. It would be helpful to log it too for performance monitoring. Signed-off-by: Jeff Hostetler --- wt-status.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/wt-status.c b/wt-status.c index 0643907c1f32af..d4983ceb68541b 100644 --- a/wt-status.c +++ b/wt-status.c @@ -2708,6 +2708,36 @@ void wt_status_print(struct wt_status *s) s->untracked.nr); trace2_data_intmax("status", s->repo, "count/ignored", s->ignored.nr); + switch (s->state.sparse_checkout_percentage) { + case SPARSE_CHECKOUT_DISABLED: + break; + case SPARSE_CHECKOUT_SPARSE_INDEX: + /* + * Log just the observed size of the sparse-index. + * + * When sparse-index is enabled we can have + * sparse-directory entries in addition to individual + * sparse-file entries, so we don't know the complete + * size of the index. And we do not want to force + * expand it just to emit some telemetry data. So we + * cannot report a percentage for the space savings. + * + * It is possible that if the telemetry data is + * aggregated, someone will have a good estimate for + * the size of a fully populated index and can compute + * a percentage after the fact. + */ + trace2_data_intmax("status", s->repo, + "sparse-index/size", + s->repo->index->cache_nr); + break; + default: + trace2_data_intmax("status", s->repo, + "sparse-checkout/percentage", + s->state.sparse_checkout_percentage); + break; + } + trace2_region_enter("status", "print", s->repo); switch (s->status_format) { From 5aa1c05c9bf8b056bb787090df596560274a3359 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 13 May 2020 17:38:50 -0400 Subject: [PATCH 118/289] deserialize-status: silently fallback if we cannot read cache file Teach Git to not throw a fatal error when an explicitly-specified status-cache file (`git status --deserialize=`) could not be found or opened for reading and silently fallback to a traditional scan. This matches the behavior when the status-cache file is implicitly given via a config setting. Note: the current version causes a test to start failing. Mark this as an expected result for now. Signed-off-by: Jeff Hostetler Signed-off-by: Derrick Stolee --- builtin/commit.c | 18 ++++++++++++------ t/t7522-serialized-status.sh | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/builtin/commit.c b/builtin/commit.c index b9a17f671cdf60..5ea4002e752f4e 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -231,12 +231,18 @@ static int opt_parse_deserialize(const struct option *opt UNUSED, const char *ar free(deserialize_path); deserialize_path = xstrdup(arg); } - if (deserialize_path && *deserialize_path - && (access(deserialize_path, R_OK) != 0)) - die("cannot find serialization file '%s'", - deserialize_path); - - do_explicit_deserialize = 1; + if (!deserialize_path || !*deserialize_path) + do_explicit_deserialize = 1; /* read stdin */ + else if (access(deserialize_path, R_OK) == 0) + do_explicit_deserialize = 1; /* can read from this file */ + else { + /* + * otherwise, silently fallback to the normal + * collection scan + */ + do_implicit_deserialize = 0; + do_explicit_deserialize = 0; + } } return 0; diff --git a/t/t7522-serialized-status.sh b/t/t7522-serialized-status.sh index 8f32f6cfe70556..b5c5906187ab7d 100755 --- a/t/t7522-serialized-status.sh +++ b/t/t7522-serialized-status.sh @@ -442,4 +442,20 @@ test_expect_success 'ensure deserialize -v does not crash' ' test_grep -q "deserialize/reject:args/verbose" verbose_test.log_v ' +test_expect_success 'fallback when implicit' ' + git init -b main implicit_fallback_test && + git -C implicit_fallback_test -c status.deserializepath=foobar status +' + +test_expect_success 'fallback when explicit' ' + git init -b main explicit_fallback_test && + git -C explicit_fallback_test status --deserialize=foobar +' + +test_expect_success 'deserialize from stdin' ' + git init -b main stdin_test && + git -C stdin_test status --serialize >serialized_status.dat && + cat serialize_status.dat | git -C stdin_test status --deserialize +' + test_done From 3fa557bd5c026566614b54b4cfd1d43e73030146 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 3 Oct 2019 13:21:26 -0400 Subject: [PATCH 119/289] credential: set trace2_child_class for credential manager children Signed-off-by: Jeff Hostetler --- credential.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/credential.c b/credential.c index af964189363b28..b392280b29f2af 100644 --- a/credential.c +++ b/credential.c @@ -461,6 +461,8 @@ static int run_credential_helper(struct credential *c, else helper.no_stdout = 1; + helper.trace2_child_class = "cred"; + if (start_command(&helper) < 0) return -1; From bc332b650c5ed188bbbc8486b510a8de415783db Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 18 Sep 2019 10:35:45 -0400 Subject: [PATCH 120/289] sub-process: do not borrow cmd pointer from caller Teach subprocess_start() to use a copy of the passed `cmd` string rather than borrowing the buffer from the caller. Some callers of subprocess_start() pass the value returned from find_hook() which points to a static buffer and therefore is only good until the next call to find_hook(). This could cause problems for the long-running background processes managed by sub-process.c where later calls to subprocess_find_entry() to get an existing process will fail. This could cause more than 1 long-running process to be created. TODO Need to confirm, but if only read_object_hook() uses TODO subprocess_start() in this manner, we could drop this TODO commit when we drop support for read_object_hook(). Signed-off-by: Jeff Hostetler --- sub-process.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/sub-process.c b/sub-process.c index 2d5c965169727b..9020428366f129 100644 --- a/sub-process.c +++ b/sub-process.c @@ -78,7 +78,12 @@ int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, co int err; struct child_process *process; - entry->cmd = cmd; + // BUGBUG most callers to subprocess_start() pass in "cmd" the value + // BUGBUG of find_hook() which returns a static buffer (that's only + // BUGBUG good until the next call to find_hook()). + // BUGFIX Defer assignment until we copy the string in our argv. + // entry->cmd = cmd; + process = &entry->process; child_process_init(process); @@ -90,6 +95,8 @@ int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, co process->clean_on_exit_handler = subprocess_exit_handler; process->trace2_child_class = "subprocess"; + entry->cmd = process->args.v[0]; + err = start_command(process); if (err) { error("cannot fork to run subprocess '%s'", cmd); From 09086b74613fa6ca737d7488a5e310a0a078ab1d Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 18 Sep 2019 10:45:58 -0400 Subject: [PATCH 121/289] sub-process: add subprocess_start_argv() Add function to start a subprocess with an argv. Signed-off-by: Jeff Hostetler --- sub-process.c | 45 +++++++++++++++++++++++++++++++++++++++++++++ sub-process.h | 6 ++++++ 2 files changed, 51 insertions(+) diff --git a/sub-process.c b/sub-process.c index 9020428366f129..611ba221eb4c83 100644 --- a/sub-process.c +++ b/sub-process.c @@ -5,6 +5,7 @@ #include "sub-process.h" #include "sigchain.h" #include "pkt-line.h" +#include "quote.h" int cmd2process_cmp(const void *cmp_data UNUSED, const struct hashmap_entry *eptr, @@ -116,6 +117,50 @@ int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, co return 0; } +int subprocess_start_strvec(struct hashmap *hashmap, + struct subprocess_entry *entry, + int is_git_cmd, + const struct strvec *argv, + subprocess_start_fn startfn) +{ + int err; + struct child_process *process; + struct strbuf quoted = STRBUF_INIT; + + process = &entry->process; + + child_process_init(process); + strvec_pushv(&process->args, argv->v); + process->use_shell = 1; + process->in = -1; + process->out = -1; + process->git_cmd = is_git_cmd; + process->clean_on_exit = 1; + process->clean_on_exit_handler = subprocess_exit_handler; + process->trace2_child_class = "subprocess"; + + sq_quote_argv_pretty("ed, argv->v); + entry->cmd = strbuf_detach("ed, NULL); + + err = start_command(process); + if (err) { + error("cannot fork to run subprocess '%s'", entry->cmd); + return err; + } + + hashmap_entry_init(&entry->ent, strhash(entry->cmd)); + + err = startfn(entry); + if (err) { + error("initialization for subprocess '%s' failed", entry->cmd); + subprocess_stop(hashmap, entry); + return err; + } + + hashmap_add(hashmap, &entry->ent); + return 0; +} + static int handshake_version(struct child_process *process, const char *welcome_prefix, int *versions, int *chosen_version) diff --git a/sub-process.h b/sub-process.h index bfc3959a1b4894..3bd658c5dc8e3e 100644 --- a/sub-process.h +++ b/sub-process.h @@ -56,6 +56,12 @@ typedef int(*subprocess_start_fn)(struct subprocess_entry *entry); int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, const char *cmd, subprocess_start_fn startfn); +int subprocess_start_strvec(struct hashmap *hashmap, + struct subprocess_entry *entry, + int is_git_cmd, + const struct strvec *argv, + subprocess_start_fn startfn); + /* Kill a subprocess and remove it from the subprocess hashmap. */ void subprocess_stop(struct hashmap *hashmap, struct subprocess_entry *entry); From fa9f21e528174d67519e3c00f72ec226c8cd2e85 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 24 Sep 2019 14:31:10 -0400 Subject: [PATCH 122/289] sha1-file: add function to update existing loose object cache Create a function to add a new object to the loose object cache after the existing odb/xx/ directory was scanned. This will be used in a later commit to keep the loose object cache fresh after dynamically fetching an individual object and without requiring the odb/xx/ directory to be rescanned. Signed-off-by: Jeff Hostetler --- object-file.h | 8 ++++++++ odb/source-loose.c | 21 +++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/object-file.h b/object-file.h index ee3aac7965a527..861f4cb75add42 100644 --- a/object-file.h +++ b/object-file.h @@ -24,6 +24,14 @@ int index_path(struct index_state *istate, struct object_id *oid, const char *pa struct object_info; struct odb_source; +/* + * Add a new object to the loose object cache (possibly after the + * cache was populated). This might be used after dynamically + * fetching a missing object. + */ +void odb_source_loose_cache_add_new_oid(struct odb_source *source, + const struct object_id *oid); + /* * Put in `buf` the name of the file in the local object database that * would be used to store a loose object with the specified oid. diff --git a/odb/source-loose.c b/odb/source-loose.c index 81538f532498eb..94d2372669aea0 100644 --- a/odb/source-loose.c +++ b/odb/source-loose.c @@ -1008,6 +1008,27 @@ static int odb_source_loose_write_alternate(struct odb_source *source UNUSED, return error("loose source does not support alternates"); } +void odb_source_loose_cache_add_new_oid(struct odb_source *source, + const struct object_id *oid) +{ + struct odb_source_loose *loose; + struct oidtree *cache; + + switch (source->type) { + case ODB_SOURCE_FILES: + loose = odb_source_files_downcast(source)->loose; + break; + case ODB_SOURCE_LOOSE: + loose = odb_source_loose_downcast(source); + break; + default: + BUG("source of type '%d' has no loose cache", source->type); + } + + cache = odb_source_loose_cache(loose, oid); + append_loose_object(oid, NULL, cache); +} + static void odb_source_loose_clear_cache(struct odb_source_loose *loose) { oidtree_clear(loose->cache); From 26176b81684074efcaa06e6f87af8026f3616063 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 16 Jul 2019 10:40:56 -0400 Subject: [PATCH 123/289] trace2:gvfs:experiment: add unpack_entry() counter to unpack_trees() and report_tracking() Signed-off-by: Jeff Hostetler Signed-off-by: Johannes Schindelin --- builtin/checkout.c | 6 ++++++ packfile.c | 9 +++++++++ packfile.h | 5 +++++ unpack-trees.c | 5 +++++ 4 files changed, 25 insertions(+) diff --git a/builtin/checkout.c b/builtin/checkout.c index eab0e93a96a818..866fdcffeb7b60 100644 --- a/builtin/checkout.c +++ b/builtin/checkout.c @@ -20,6 +20,7 @@ #include "object-file.h" #include "object-name.h" #include "odb.h" +#include "packfile.h" #include "parse-options.h" #include "path.h" #include "preload-index.h" @@ -1051,8 +1052,13 @@ static void update_refs_for_switch(const struct checkout_opts *opts, if (!opts->quiet && !opts->force_detach && (new_branch_info->path || !strcmp(new_branch_info->name, "HEAD"))) { + unsigned long nr_unpack_entry_at_start; + trace2_region_enter("exp", "report_tracking", the_repository); + nr_unpack_entry_at_start = get_nr_unpack_entry(); report_tracking(new_branch_info); + trace2_data_intmax("exp", NULL, "report_tracking/nr_unpack_entries", + (intmax_t)(get_nr_unpack_entry() - nr_unpack_entry_at_start)); trace2_region_leave("exp", "report_tracking", the_repository); } } diff --git a/packfile.c b/packfile.c index 4fa5fd67c8497f..702cf33622054a 100644 --- a/packfile.c +++ b/packfile.c @@ -1501,6 +1501,13 @@ struct unpack_entry_stack_ent { size_t size; }; +static unsigned long g_nr_unpack_entry; + +unsigned long get_nr_unpack_entry(void) +{ + return g_nr_unpack_entry; +} + void *unpack_entry(struct repository *r, struct packed_git *p, off_t obj_offset, enum object_type *final_type, size_t *final_size) { @@ -1514,6 +1521,8 @@ void *unpack_entry(struct repository *r, struct packed_git *p, off_t obj_offset, int delta_stack_nr = 0, delta_stack_alloc = UNPACK_ENTRY_STACK_PREALLOC; int base_from_cache = 0; + g_nr_unpack_entry++; + prepare_repo_settings(p->repo); write_pack_access_log(p, obj_offset); diff --git a/packfile.h b/packfile.h index 6d30d15a0053b3..b3c66273f6272c 100644 --- a/packfile.h +++ b/packfile.h @@ -359,4 +359,9 @@ int load_idx(const char *path, const unsigned int hashsz, void *idx_map, */ int parse_pack_header_option(const char *in, unsigned char *out, unsigned int *len); +/* + * Return the number of objects fetched from a packfile. + */ +unsigned long get_nr_unpack_entry(void); + #endif diff --git a/unpack-trees.c b/unpack-trees.c index bd700c87314526..2cd5932a142e54 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -17,6 +17,7 @@ #include "tree-walk.h" #include "cache-tree.h" #include "unpack-trees.h" +#include "packfile.h" #include "progress.h" #include "refs.h" #include "attr.h" @@ -1906,6 +1907,7 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options int free_pattern_list = 0; struct dir_struct dir = DIR_INIT; struct repo_config_values *cfg = repo_config_values(the_repository); + unsigned long nr_unpack_entry_at_start; if (o->reset == UNPACK_RESET_INVALID) BUG("o->reset had a value of 1; should be UNPACK_TREES_*_UNTRACKED"); @@ -1920,6 +1922,7 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options BUG("o->df_conflict_entry is an output only field"); trace2_region_enter("exp", "unpack_trees", NULL); + nr_unpack_entry_at_start = get_nr_unpack_entry(); trace_performance_enter(); trace2_region_enter("unpack_trees", "unpack_trees", repo); @@ -2130,6 +2133,8 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options } trace2_region_leave("unpack_trees", "unpack_trees", repo); trace_performance_leave("unpack_trees"); + trace2_data_intmax("unpack_trees", NULL, "unpack_trees/nr_unpack_entries", + (intmax_t)(get_nr_unpack_entry() - nr_unpack_entry_at_start)); trace2_region_leave("exp", "unpack_trees", NULL); return ret; From 502918c1a01659b8c01422272ec2e34338f4e465 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 17 Apr 2026 14:17:47 +0200 Subject: [PATCH 124/289] index-pack: avoid immediate object fetch while parsing packfile Prevent packfile parsing from accidentally dynamically fetching each individual object found in the packfile. When index-pack parses the input packfile, it does a lookup in the ODB to test for conflicts/collisions. This can accidentally cause the object to be individually fetched when gvfs-helper (or read-object-hook or partial-clone) is enabled. The call site was migrated to odb_has_object() as part of the upstream refactoring, but odb_has_object(odb, oid, HAS_OBJECT_FETCH_PROMISOR) sets only OBJECT_INFO_QUICK without OBJECT_INFO_SKIP_FETCH_OBJECT, which means it WILL trigger remote fetches via gvfs-helper. But we want to prevent index-pack from individually fetching every object it encounters during the collision check. Passing 0 instead gives us both OBJECT_INFO_QUICK and OBJECT_INFO_SKIP_FETCH_OBJECT, which is the correct equivalent of the original OBJECT_INFO_FOR_PREFETCH behavior. Signed-off-by: Jeff Hostetler Signed-off-by: Johannes Schindelin --- builtin/index-pack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/index-pack.c b/builtin/index-pack.c index 53a6decadfb5ae..5028a92f069877 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -892,7 +892,7 @@ static void sha1_object(const void *data, struct object_entry *obj_entry, if (startup_info->have_repository) { read_lock(); collision_test_needed = odb_has_object(the_repository->objects, oid, - ODB_HAS_OBJECT_FETCH_PROMISOR); + 0); read_unlock(); } From f09b67d1b400e7e6907d0622e325860cfaa0c155 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 25 Jul 2019 15:43:50 -0400 Subject: [PATCH 125/289] trace2:gvfs:experiment: increase default event depth for unpack-tree data --- trace2/tr2_tgt_event.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trace2/tr2_tgt_event.c b/trace2/tr2_tgt_event.c index 5a0381791f7eb4..5ecc0b920e3694 100644 --- a/trace2/tr2_tgt_event.c +++ b/trace2/tr2_tgt_event.c @@ -39,7 +39,7 @@ static struct tr2_dst tr2dst_event = { * event target. Use the TR2_SYSENV_EVENT_NESTING setting to increase * region details in the event target. */ -static int tr2env_event_max_nesting_levels = 2; +static int tr2env_event_max_nesting_levels = 4; /* * Use the TR2_SYSENV_EVENT_BRIEF to omit the