Support hashing objects larger than 4GB on Windows

Philip Oakley has contributed these patches ~4.5 years ago, and they have been carried in Git for Windows ever since.

Now that there are already other patch series flying around that try to address various aspects about >4GB objects (which aren't handled well by Git until it stops forcing unsigned long to do size_t's job), it seems a good time to upstream these patches, too, at long last.

Changes since v1:

* Rebased to current master to resolve the conflicts with ps/odb-source-loose
* Dropped the !LONG_IS_64BIT prereq from the added/touched tests, as it is now no longer needed

Philip Oakley (6):
hash-object: demonstrate a >4GB/LLP64 problem object-file.c: use size_t for header lengths
hash algorithms: use size_t for section lengths hash-object --stdin: verify that it works with >4GB/LLP64
hash-object: add another >4GB/LLP64 test case
hash-object: add a >4GB/LLP64 test case using filtered input

 object-file.c          | 14 +++++++-------
 object-file.h          |  6 +++---
 odb/source-files.c     |  2 +-
 odb/source-inmemory.c  |  2 +-
 odb/source-loose.c     |  4 ++--
 odb/source.h           |  2 +-
 sha1dc_git.c           |  3 +--
 sha1dc_git.h           |  2 +-
 t/t1007-hash-object.sh | 39 +++++++++++++++++++++++++++++++++++++++
 9 files changed, 56 insertions(+), 18 deletions(-)

base-commit: 700432b2ba22603a0bcb71475c9c333d17c9b0d1
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-2138%2Fdscho%2FPhilipOakley%2Fhashliteral_t-v2
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-2138/dscho/PhilipOakley/hashliteral_t-v2
Pull-Request: https://github.com/gitgitgadget/git/pull/2138

Range-diff vs v1:

 1:  84e1cd0aa0 = 1:  9c01bac407 hash-object: demonstrate a >4GB/LLP64 problem
 2:  809d83e46f ! 2:  aa5859c14f object-file.c: use size_t for header lengths

@@ Commit message

## object-file.c ##
-@@ object-file.c: int odb_source_loose_read_object_info(struct odb_source *source, +@@ object-file.c: int parse_loose_header(const char *hdr, struct object_info *oi)
static void hash_object_body(const struct git_hash_algo *algo, struct git_hash_ctx *c,
const void *buf, unsigned long len,
struct object_id *oid,
@@ object-file.c: int odb_source_loose_read_object_info(struct odb_source *source,
@@ object-file.c: static void hash_object_body(const struct git_hash_algo *algo, struct git_hash_c
}

- static void write_object_file_prepare(const struct git_hash_algo *algo,
-- const void *buf, unsigned long len,
-+ const void *buf, size_t len,
- enum object_type type, struct object_id *oid,
-- char *hdr, int *hdrlen)
-+ char *hdr, size_t *hdrlen)
+ void write_object_file_prepare(const struct git_hash_algo *algo,
+- const void *buf, unsigned long len,
++ const void *buf, size_t len,
+ enum object_type type, struct object_id *oid,
+- char *hdr, int *hdrlen)
++ char *hdr, size_t *hdrlen)
{
struct git_hash_ctx c;

@@ object-file.c: out:

write_object_file_prepare(algo, buf, len, type, oid, hdr, &hdrlen);
}
-@@ object-file.c: cleanup:
+
+ ## object-file.h ##
+@@ object-file.h: int finalize_object_file_flags(struct repository *repo,
+ enum finalize_object_file_flags flags);
+
+ void hash_object_file(const struct git_hash_algo *algo, const void *buf,
+- unsigned long len, enum object_type type,
++ size_t len, enum object_type type,
+ struct object_id *oid);
+ void write_object_file_prepare(const struct git_hash_algo *algo,
+- const void *buf, unsigned long len,
++ const void *buf, size_t len,
+ enum object_type type, struct object_id *oid,
+- char *hdr, int *hdrlen);
++ char *hdr, size_t *hdrlen);
+ int write_loose_object(struct odb_source_loose *loose,
+ const struct object_id *oid, char *hdr,
+ int hdrlen, const void *buf, unsigned long len,
+
+ ## odb/source-files.c ##
+@@ odb/source-files.c: static int odb_source_files_freshen_object(struct odb_source *source,
+ }
+
+ static int odb_source_files_write_object(struct odb_source *source,
+- const void *buf, unsigned long len,
++ const void *buf, size_t len,
+ enum object_type type,
+ struct object_id *oid,
+ struct object_id *compat_oid,
+
+ ## odb/source-inmemory.c ##
+@@ odb/source-inmemory.c: static int odb_source_inmemory_count_objects(struct odb_source *source,
}

- int odb_source_loose_write_object(struct odb_source *source,
-- const void *buf, unsigned long len,
-+ const void *buf, size_t len,
- enum object_type type, struct object_id *oid,
- struct object_id *compat_oid_in,
- enum odb_write_object_flags flags)
-@@ object-file.c: int odb_source_loose_write_object(struct odb_source *source,
+ static int odb_source_inmemory_write_object(struct odb_source *source,
+- const void *buf, unsigned long len,
++ const void *buf, size_t len,
+ enum object_type type,
+ struct object_id *oid,
+ struct object_id *compat_oid UNUSED,
+
+ ## odb/source-loose.c ##
+@@ odb/source-loose.c: static int odb_source_loose_freshen_object(struct odb_source *source,
+ }
+
+ static int odb_source_loose_write_object(struct odb_source *source,
+- const void *buf, unsigned long len,
++ const void *buf, size_t len,
+ enum object_type type, struct object_id *oid,
+ struct object_id *compat_oid_in,
+ enum odb_write_object_flags flags)
+@@ odb/source-loose.c: static int odb_source_loose_write_object(struct odb_source *source,
const struct git_hash_algo *compat = source->odb->repo->compat_hash_algo; struct object_id compat_oid;
char hdr[MAX_HEADER_LEN];
@@ object-file.c: int odb_source_loose_write_object(struct odb_source *source,
/* Generate compat_oid */
if (compat) {

- ## object-file.h ##
-@@ object-file.h: int odb_source_loose_freshen_object(struct odb_source *source,
- const struct object_id *oid);
-
- int odb_source_loose_write_object(struct odb_source *source,
-- const void *buf, unsigned long len,
-+ const void *buf, size_t len,
- enum object_type type, struct object_id *oid,
- struct object_id *compat_oid_in,
- enum odb_write_object_flags flags);
-@@ object-file.h: int finalize_object_file_flags(struct repository *repo,
- enum finalize_object_file_flags flags);
-
- void hash_object_file(const struct git_hash_algo *algo, const void *buf,
-- unsigned long len, enum object_type type,
-+ size_t len, enum object_type type,
- struct object_id *oid);
-
- /* Helper to check and "touch" a file */
+ ## odb/source.h ##
+@@ odb/source.h: struct odb_source {
+ * return 0 on success, a negative error code otherwise.
+ */
+ int (*write_object)(struct odb_source *source,
+- const void *buf, unsigned long len,
++ const void *buf, size_t len,
+ enum object_type type,
+ struct object_id *oid,
+ struct object_id *compat_oid,
3: 253d6f8004 ! 3: b401eb490f hash algorithms: use size_t for section lengths
@@ Commit message

## object-file.c ##
-@@ object-file.c: int odb_source_loose_read_object_info(struct odb_source *source, +@@ object-file.c: int parse_loose_header(const char *hdr, struct object_info *oi)
}

static void hash_object_body(const struct git_hash_algo *algo, struct git_hash_ctx *c,
@@ object-file.c: int odb_source_loose_read_object_info(struct odb_source *source,
struct object_id *oid,
char *hdr, size_t *hdrlen)
{
-@@ object-file.c: static void write_object_file_prepare(const struct git_hash_algo *algo, +@@ object-file.c: void write_object_file_prepare(const struct git_hash_algo *algo,
/* Generate the header */
*hdrlen = format_object_header(hdr, *hdrlen, type, len);

@@ t/t1007-hash-object.sh: test_expect_success '--stdin outside of repository (uses
'

-test_expect_failure EXPENSIVE,SIZE_T_IS_64BIT,!LONG_IS_64BIT \
-+test_expect_success EXPENSIVE,SIZE_T_IS_64BIT,!LONG_IS_64BIT \
++test_expect_success EXPENSIVE,SIZE_T_IS_64BIT \
'files over 4GB hash literally' '
test-tool genzeros $((5*1024*1024*1024)) >big &&
test_oid large5GB >expect &&
4: ba629a3f03 ! 4: 411727336a hash-object --stdin: verify that it works with >4GB/LLP64

     @@ Commit message
          Signed-off-by: Johannes Schindelin <[email protected]>
      
       ## t/t1007-hash-object.sh ##
     -@@ t/t1007-hash-object.sh: test_expect_success EXPENSIVE,SIZE_T_IS_64BIT,!LONG_IS_64BIT \
     +@@ t/t1007-hash-object.sh: test_expect_success EXPENSIVE,SIZE_T_IS_64BIT \
       	test_cmp expect actual
       '
       
     -+test_expect_success EXPENSIVE,SIZE_T_IS_64BIT,!LONG_IS_64BIT \
     ++test_expect_success EXPENSIVE,SIZE_T_IS_64BIT \
      +		'files over 4GB hash correctly via --stdin' '
      +	{ test -f big || test-tool genzeros $((5*1024*1024*1024)) >big; } &&
      +	test_oid large5GB >expect &&
 5:  f48d570bba ! 5:  e6bb4e6228 hash-object: add another >4GB/LLP64 test case
     @@ Commit message
          Signed-off-by: Johannes Schindelin <[email protected]>
      
       ## t/t1007-hash-object.sh ##
     -@@ t/t1007-hash-object.sh: test_expect_success EXPENSIVE,SIZE_T_IS_64BIT,!LONG_IS_64BIT \
     +@@ t/t1007-hash-object.sh: test_expect_success EXPENSIVE,SIZE_T_IS_64BIT \
       	test_cmp expect actual
       '
       
     -+test_expect_success EXPENSIVE,SIZE_T_IS_64BIT,!LONG_IS_64BIT \
     ++test_expect_success EXPENSIVE,SIZE_T_IS_64BIT \
      +		'files over 4GB hash correctly' '
      +	{ test -f big || test-tool genzeros $((5*1024*1024*1024)) >big; } &&
      +	test_oid large5GB >expect &&
 6:  8a6beeb16d ! 6:  568807ac34 hash-object: add a >4GB/LLP64 test case using filtered input
     @@ Commit message
          Signed-off-by: Johannes Schindelin <[email protected]>
      
       ## t/t1007-hash-object.sh ##
     -@@ t/t1007-hash-object.sh: test_expect_success EXPENSIVE,SIZE_T_IS_64BIT,!LONG_IS_64BIT \
     +@@ t/t1007-hash-object.sh: test_expect_success EXPENSIVE,SIZE_T_IS_64BIT \
       	test_cmp expect actual
       '
       
      +# This clean filter does nothing, other than excercising the interface.
      +# We ensure that cleaning doesn't mangle large files on 64-bit Windows.
     -+test_expect_success EXPENSIVE,SIZE_T_IS_64BIT,!LONG_IS_64BIT \
     ++test_expect_success EXPENSIVE,SIZE_T_IS_64BIT \
      +		'hash filtered files over 4GB correctly' '
      +	{ test -f big || test-tool genzeros $((5*1024*1024*1024)) >big; } &&
      +	test_oid large5GB >expect &&

gitgitgadget