More work supporting objects larger than 4GB on Windows

This patch series tries to address the problems pointed out by the expensive tests that now run in CI: t5608 and t7508 verify various aspects about objects larger than 4GB, which Git does not currently handle correctly when run on a platform where size_t is 64-bit and unsigned long is 32-bit.

Changes vs v1:

* Rebased onto master, which merged ps/odb-source-loose (with which these patches previously conflicted rather badly).
* Removed superfluous size_t s variables (thanks, Patrick!).

Johannes Schindelin (7):
compat/msvc: use _chsize_s for ftruncate
patch-delta: use size_t for sizes pack-objects(check_pack_inflate()): use size_t instead of unsigned
long
packfile: widen unpack_entry()'s size out-parameter to size_t
pack-objects: use size_t for in-core object sizes packfile,delta: drop the cast_size_t_to_ulong() wrappers
odb: use size_t for object_info.sizep and the size APIs

 apply.c                       |  8 ++--
 archive.c                     |  4 +-
 attr.c                        |  2 +-
 bisect.c                      |  2 +-
 blame.c                       | 15 +++++--
 builtin/cat-file.c            | 61 ++++++++++++++---------------
 builtin/difftool.c            |  2 +-
 builtin/fast-export.c         |  7 +++-
 builtin/fast-import.c         | 29 ++++++++++----
 builtin/fsck.c                |  2 +-
 builtin/grep.c                | 12 +++---
 builtin/index-pack.c          | 10 ++---
 builtin/log.c                 |  2 +-
 builtin/ls-files.c            |  2 +-
 builtin/ls-tree.c             |  4 +-
 builtin/merge-tree.c          |  6 +--
 builtin/mktag.c               |  2 +-
 builtin/notes.c               |  6 +--
 builtin/pack-objects.c        | 73 +++++++++++++++++++++--------------
 builtin/repo.c                |  4 +-
 builtin/tag.c                 |  4 +-
 builtin/unpack-file.c         |  2 +-
 builtin/unpack-objects.c      |  8 ++--
 bundle.c                      |  2 +-
 combine-diff.c                |  4 +-
 commit.c                      | 10 ++---
 compat/msvc-posix.h           | 24 +++++++++++-
 config.c                      |  2 +-
 delta.h                       | 20 +++-------
 diff.c                        |  5 ++-
 dir.c                         |  2 +-
 entry.c                       |  4 +-
 fmt-merge-msg.c               |  4 +-
 fsck.c                        |  2 +-
 grep.c                        |  4 +-
 http-push.c                   |  2 +-
 list-objects-filter.c         |  2 +-
 mailmap.c                     |  2 +-
 match-trees.c                 |  4 +-
 merge-blobs.c                 |  6 +--
 merge-blobs.h                 |  2 +-
 merge-ort.c                   |  2 +-
 notes-cache.c                 |  2 +-
 notes-merge.c                 |  2 +-
 notes.c                       |  8 ++--
 object-file.c                 |  6 +--
 object.c                      |  2 +-
 odb.c                         | 12 +++---
 odb.h                         | 10 ++---
 odb/source-loose.c            | 12 +-----
 odb/streaming.c               | 13 +------
 pack-bitmap.c                 |  4 +-
 pack-check.c                  |  5 +--
 pack-objects.h                |  2 +-
 packfile.c                    | 54 ++++++++++----------------
 packfile.h                    |  5 ++-
 patch-delta.c                 |  8 ++--
 path-walk.c                   |  2 +-
 protocol-caps.c               |  5 ++-
 read-cache.c                  |  6 +--
 ref-filter.c                  |  2 +-
 reflog.c                      |  2 +-
 rerere.c                      |  2 +-
 submodule-config.c            |  2 +-
 t/helper/test-delta.c         | 10 +++--
 t/helper/test-pack-deltas.c   |  3 +-
 t/helper/test-partial-clone.c |  2 +-
 t/unit-tests/u-odb-inmemory.c |  2 +-
 tag.c                         |  4 +-
 tree-walk.c                   | 10 +++--
 tree.c                        |  2 +-
 xdiff-interface.c             |  2 +-
 72 files changed, 300 insertions(+), 271 deletions(-)

base-commit: ea97ad8d017de0c9037451a78008a0fd60abea0c
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-2137%2Fdscho%2Fobjects-larger-than-4gb-on-windows-pt2-v2
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-2137/dscho/objects-larger-than-4gb-on-windows-pt2-v2
Pull-Request: https://github.com/gitgitgadget/git/pull/2137

Range-diff vs v1:

 1:  de9fc5c455 = 1:  531bca775c compat/msvc: use _chsize_s for ftruncate
 2:  1fd7646ca1 = 2:  66a642c39e patch-delta: use size_t for sizes
 3:  ddb75326cd = 3:  271a5299e3 pack-objects(check_pack_inflate()): use size_t instead of unsigned long
 4:  bdebc36f21 = 4:  5c329535df packfile: widen unpack_entry()'s size out-parameter to size_t
 5:  68750ba2d1 = 5:  01b9209b26 pack-objects: use size_t for in-core object sizes
 6:  460d733fee = 6:  12c142f8ab packfile,delta: drop the `cast_size_t_to_ulong()` wrappers
 7:  f3aeae983a ! 7:  37d030d867 odb: use size_t for object_info.sizep and the size APIs

@@ builtin/cat-file.c: static int cat_one_file(int opt, const char *exp_type, const
struct object_info oi = OBJECT_INFO_INIT;
unsigned flags = OBJECT_INFO_LOOKUP_REPLACE;
@@ builtin/cat-file.c: static int cat_one_file(int opt, const char *exp_type, const char *obj_name)
- if (use_mailmap && (type == OBJ_COMMIT || type == OBJ_TAG)) {
- size_t s = size;
- buf = replace_idents_using_mailmap(buf, &s);
+ if (odb_read_object_info_extended(the_repository->objects, &oid, &oi, flags) < 0)
+ die("git cat-file: could not get object info");
+
+- if (use_mailmap && (type == OBJ_COMMIT || type == OBJ_TAG)) {
+- size_t s = size;
+- buf = replace_idents_using_mailmap(buf, &s);
- size = cast_size_t_to_ulong(s);
-+ size = s;
- }
+- }
++ if (use_mailmap && (type == OBJ_COMMIT || type == OBJ_TAG))
++ buf = replace_idents_using_mailmap(buf, &size);

printf("%"PRIuMAX"\n", (uintmax_t)size);
+ ret = 0;
@@ builtin/cat-file.c: static int cat_one_file(int opt, const char *exp_type, const char *obj_name)
break;

@@ builtin/cat-file.c: static int cat_one_file(int opt, const char *exp_type, const

case 'p':
@@ builtin/cat-file.c: static int cat_one_file(int opt, const char *exp_type, const char *obj_name)
- if (use_mailmap) {
- size_t s = size;
- buf = replace_idents_using_mailmap(buf, &s);
+ if (!buf)
+ die("Cannot read object %s", obj_name);
+
+- if (use_mailmap) {
+- size_t s = size;
+- buf = replace_idents_using_mailmap(buf, &s);
- size = cast_size_t_to_ulong(s);
-+ size = s;
- }
+- }
++ if (use_mailmap)
++ buf = replace_idents_using_mailmap(buf, &size);

/* otherwise just spit out the data */
+ break;
@@ builtin/cat-file.c: static int cat_one_file(int opt, const char *exp_type, const char *obj_name)
- if (use_mailmap) {
- size_t s = size;
- buf = replace_idents_using_mailmap(buf, &s);
+ buf = odb_read_object_peeled(the_repository->objects, &oid,
+ exp_type_id, &size, NULL);
+
+- if (use_mailmap) {
+- size_t s = size;
+- buf = replace_idents_using_mailmap(buf, &s);
- size = cast_size_t_to_ulong(s);
-+ size = s;
- }
+- }
++ if (use_mailmap)
++ buf = replace_idents_using_mailmap(buf, &size);
break;
}
+ default:
@@ builtin/cat-file.c: cleanup:
struct expand_data {
struct object_id oid;
@@ builtin/cat-file.c: static void print_object_or_die(struct batch_options *opt, s

contents = odb_read_object(the_repository->objects, oid,
@@ builtin/cat-file.c: static void print_object_or_die(struct batch_options *opt, struct expand_data *d
- if (use_mailmap) {
- size_t s = size;
- contents = replace_idents_using_mailmap(contents, &s);
+ if (!contents)
+ die("object %s disappeared", oid_to_hex(oid));
+
+- if (use_mailmap) {
+- size_t s = size;
+- contents = replace_idents_using_mailmap(contents, &s);
- size = cast_size_t_to_ulong(s);
-+ size = s;
- }
+- }
++ if (use_mailmap)
++ contents = replace_idents_using_mailmap(contents, &size);

if (type != data->type)
+ die("object %s changed type!?", oid_to_hex(oid));
@@ builtin/cat-file.c: static void batch_object_write(const char *obj_name,
+ }
+
+ if (use_mailmap && (data->type == OBJ_COMMIT || data->type == OBJ_TAG)) {
+- size_t s = data->size;
+ char *buf = NULL;
+
+ buf = odb_read_object(the_repository->objects, &data->oid,
+ &data->type, &data->size);
if (!buf)
die(_("unable to read %s"), oid_to_hex(&data->oid));
- buf = replace_idents_using_mailmap(buf, &s);
+- buf = replace_idents_using_mailmap(buf, &s);
- data->size = cast_size_t_to_ulong(s);
-+ data->size = s;
++ buf = replace_idents_using_mailmap(buf, &data->size);

free(buf);
}
@@ builtin/log.c: static int show_blob_object(const struct object_id *oid, struct r

## builtin/ls-files.c ##
@@ builtin/ls-files.c: static void expand_objectsize(struct repository *repo, struct strbuf *line,
- const enum object_type type, unsigned int padded)
- {
+ size_t len;
+
if (type == OBJ_BLOB) {
- unsigned long size;
+ size_t size;
@@ builtin/ls-files.c: static void expand_objectsize(struct repository *repo, struc

## builtin/ls-tree.c ##
@@ builtin/ls-tree.c: static void expand_objectsize(struct strbuf *line, const struct object_id *oid,
- const enum object_type type, unsigned int padded)
- {
+ size_t len;
+
if (type == OBJ_BLOB) {
- unsigned long size;
+ size_t size;
@@ notes.c: static void format_note(struct notes_tree *t, const struct object_id *o
if (!t)

## object-file.c ##
-@@ object-file.c: static int parse_loose_header(const char *hdr, struct object_info *oi) +@@ object-file.c: int parse_loose_header(const char *hdr, struct object_info *oi)
}

if (oi->sizep)
@@ object-file.c: static int parse_loose_header(const char *hdr, struct object_info

/*
* The length must be followed by a zero byte
-@@ object-file.c: static int read_object_info_from_path(struct odb_source *source,
- void *map = NULL;
- git_zstream stream, *stream_to_end = NULL;
- char hdr[MAX_HEADER_LEN];
-- unsigned long size_scratch;
-+ size_t size_scratch;
- enum object_type type_scratch;
- struct stat st;
-
@@ object-file.c: int force_object_loose(struct odb_source *source,
- {
+ struct odb_source_files *files = odb_source_files_downcast(source);
const struct git_hash_algo *compat = source->odb->repo->compat_hash_algo; void *buf;
- unsigned long len;
@@ object-file.c: int read_loose_object(struct repository *repo,

fd = git_open(path);
if (fd >= 0)
-@@ object-file.c: int odb_source_loose_read_object_stream(struct odb_read_stream **out,
- struct object_info oi = OBJECT_INFO_INIT;
- struct odb_loose_read_stream *st;
- unsigned long mapsize;
-- unsigned long size_ul;
- void *mapped;
-
- mapped = odb_source_loose_map_object(source, oid, &mapsize);
-@@ object-file.c: int odb_source_loose_read_object_stream(struct odb_read_stream **out,
- goto error;
- }
-
-- /*
-- * object_info.sizep is unsigned long* (32-bit on Windows), but
-- * st->base.size is size_t (64-bit). Use temporary variable.
-- * Note: loose objects >4GB would still truncate here, but such
-- * large loose objects are uncommon (they'd normally be packed).
-- */
-- oi.sizep = &size_ul;
-+ oi.sizep = &st->base.size;
- oi.typep = &st->base.type;
-
- if (parse_loose_header(st->hdr, &oi) < 0 || st->base.type < 0)
- goto error;
-- st->base.size = size_ul;
-
- st->mapped = mapped;
- st->mapsize = mapsize;

## object.c ##
@@ object.c: struct object *parse_object_with_flags(struct repository *r,
@@ odb.h: int odb_read_object_info_extended(struct object_database *odb,
enum odb_has_object_flags {
/* Retry packed storage after checking packed and loose storage */

+ ## odb/source-loose.c ##
+@@ odb/source-loose.c: static int read_object_info_from_path(struct odb_source_loose *loose,
+ void *map = NULL;
+ git_zstream stream, *stream_to_end = NULL;
+ char hdr[MAX_HEADER_LEN];
+- unsigned long size_scratch;
++ size_t size_scratch;
+ enum object_type type_scratch;
+ struct stat st;
+
+@@ odb/source-loose.c: static int odb_source_loose_read_object_stream(struct odb_read_stream **out,
+ struct object_info oi = OBJECT_INFO_INIT;
+ struct odb_loose_read_stream *st;
+ unsigned long mapsize;
+- unsigned long size_ul;
+ void *mapped;
+
+ mapped = odb_source_loose_map_object(loose, oid, &mapsize);
+@@ odb/source-loose.c: static int odb_source_loose_read_object_stream(struct odb_read_stream **out,
+ goto error;
+ }
+
+- /*
+- * object_info.sizep is unsigned long* (32-bit on Windows), but
+- * st->base.size is size_t (64-bit). Use temporary variable.
+- * Note: loose objects >4GB would still truncate here, but such
+- * large loose objects are uncommon (they'd normally be packed).
+- */
+- oi.sizep = &size_ul;
++ oi.sizep = &st->base.size;
+ oi.typep = &st->base.type;
+
+ if (parse_loose_header(st->hdr, &oi) < 0 || st->base.type < 0)
+ goto error;
+- st->base.size = size_ul;
+
+ st->mapped = mapped;
+ st->mapsize = mapsize;
+
## odb/streaming.c ##
@@ odb/streaming.c: static int open_istream_incore(struct odb_read_stream **out,
.base.read = read_istream_incore,

gitgitgadget