object-file: move logic related to the loose backend

index 54f680d6d78..e92da0d8b2e 100644
@@ -81,7 +81,7 @@ static int for_each_file_in_obj_subdir(unsigned int subdir_nr,
 		strbuf_add(path, de->d_name, namelen);
 		if (namelen == algop->hexsz - 2 &&
 		    !hex_to_bytes(oid.hash + 1, de->d_name,
-				  algop->rawsz - 1)) {
+				  algop->rawsz - 1, HEX_KIND_OID)) {
 			oid_set_algo(&oid, algop);
 			memset(oid.hash + algop->rawsz, 0,
 			       GIT_MAX_RAWSZ - algop->rawsz);

index 132aaab6f84..c1b78a25c9f 100644
@@ -151,179 +151,6 @@ int stream_object_signature(struct repository *r,
 	return !oideq(oid, &real_oid) ? -1 : 0;
 }
 
-/*
- * Map and close the given loose object fd. The path argument is used for
- * error reporting.
- */
-static void *map_fd(int fd, const char *path, unsigned long *size)
-{
-	void *map = NULL;
-	struct stat st;
-
-	if (!fstat(fd, &st)) {
-		*size = xsize_t(st.st_size);
-		if (!*size) {
-			/* mmap() is forbidden on empty files */
-			error(_("object file %s is empty"), path);
-			close(fd);
-			return NULL;
-		}
-		map = xmmap(NULL, *size, PROT_READ, MAP_PRIVATE, fd, 0);
-	}
-	close(fd);
-	return map;
-}
-
-enum unpack_loose_header_result unpack_loose_header(git_zstream *stream,
-						    unsigned char *map,
-						    unsigned long mapsize,
-						    void *buffer,
-						    unsigned long bufsiz)
-{
-	int status;
-
-	/* Get the data stream */
-	memset(stream, 0, sizeof(*stream));
-	stream->next_in = map;
-	stream->avail_in = mapsize;
-	stream->next_out = buffer;
-	stream->avail_out = bufsiz;
-
-	git_inflate_init(stream);
-	obj_read_unlock();
-	status = git_inflate(stream, 0);
-	obj_read_lock();
-	if (status != Z_OK && status != Z_STREAM_END)
-		return ULHR_BAD;
-
-	/*
-	 * Check if entire header is unpacked in the first iteration.
-	 */
-	if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
-		return ULHR_OK;
-
-	/*
-	 * We have a header longer than MAX_HEADER_LEN.
-	 */
-	return ULHR_TOO_LONG;
-}
-
-void *unpack_loose_rest(git_zstream *stream,
-			void *buffer, unsigned long size,
-			const struct object_id *oid)
-{
-	size_t bytes = strlen(buffer) + 1, n;
-	unsigned char *buf = xmallocz(size);
-	int status = Z_OK;
-
-	n = stream->total_out - bytes;
-	if (n > size)
-		n = size;
-	memcpy(buf, (char *) buffer + bytes, n);
-	bytes = n;
-	if (bytes <= size) {
-		/*
-		 * The above condition must be (bytes <= size), not
-		 * (bytes < size).  In other words, even though we
-		 * expect no more output and set avail_out to zero,
-		 * the input zlib stream may have bytes that express
-		 * "this concludes the stream", and we *do* want to
-		 * eat that input.
-		 *
-		 * Otherwise we would not be able to test that we
-		 * consumed all the input to reach the expected size;
-		 * we also want to check that zlib tells us that all
-		 * went well with status == Z_STREAM_END at the end.
-		 */
-		stream->next_out = buf + bytes;
-		stream->avail_out = size - bytes;
-		while (status == Z_OK) {
-			obj_read_unlock();
-			status = git_inflate(stream, Z_FINISH);
-			obj_read_lock();
-		}
-	}
-
-	if (status != Z_STREAM_END) {
-		error(_("corrupt loose object '%s'"), oid_to_hex(oid));
-		FREE_AND_NULL(buf);
-	} else if (stream->avail_in) {
-		error(_("garbage at end of loose object '%s'"),
-		      oid_to_hex(oid));
-		FREE_AND_NULL(buf);
-	}
-
-	return buf;
-}
-
-/*
- * parse_loose_header() parses the starting "<type> <len>\0" of an
- * object. If it doesn't follow that format -1 is returned. To check
- * the validity of the <type> populate the "typep" in the "struct
- * object_info". It will be OBJ_BAD if the object type is unknown. The
- * parsed <len> can be retrieved via "oi->sizep", and from there
- * passed to unpack_loose_rest().
- *
- * We used to just use "sscanf()", but that's actually way
- * too permissive for what we want to check. So do an anal
- * object header parse by hand.
- */
-int parse_loose_header(const char *hdr, struct object_info *oi)
-{
-	const char *type_buf = hdr;
-	size_t size;
-	int type, type_len = 0;
-
-	/*
-	 * The type can be of any size but is followed by
-	 * a space.
-	 */
-	for (;;) {
-		char c = *hdr++;
-		if (!c)
-			return -1;
-		if (c == ' ')
-			break;
-		type_len++;
-	}
-
-	type = type_from_string_gently(type_buf, type_len, 1);
-	if (oi->typep)
-		*oi->typep = type;
-
-	/*
-	 * The length must follow immediately, and be in canonical
-	 * decimal format (ie "010" is not valid).
-	 */
-	size = *hdr++ - '0';
-	if (size > 9)
-		return -1;
-	if (size) {
-		for (;;) {
-			unsigned long c = *hdr - '0';
-			if (c > 9)
-				break;
-			hdr++;
-			size = st_add(st_mult(size, 10), c);
-		}
-	}
-
-	if (oi->sizep)
-		*oi->sizep = size;
-
-	/*
-	 * The length must be followed by a zero byte
-	 */
-	if (*hdr)
-		return -1;
-
-	/*
-	 * The format is valid, but the type may still be bogus. The
-	 * Caller needs to check its oi->typep.
-	 */
-	return 0;
-}
-
 #define CHECK_COLLISION_DEST_VANISHED -2
 
 static int check_collision(const char *source, const char *dest)

@@ -1127,133 +954,6 @@ int for_each_loose_file_in_source(struct odb_source *source,
 	return r;
 }
 
-static int check_stream_oid(git_zstream *stream,
-			    const char *hdr,
-			    unsigned long size,
-			    const char *path,
-			    const struct object_id *expected_oid,
-			    const struct git_hash_algo *algop)
-{
-	struct git_hash_ctx c;
-	struct object_id real_oid;
-	unsigned char buf[4096];
-	unsigned long total_read;
-	int status = Z_OK;
-
-	git_hash_init(&c, algop);
-	git_hash_update(&c, hdr, stream->total_out);
-
-	/*
-	 * We already read some bytes into hdr, but the ones up to the NUL
-	 * do not count against the object's content size.
-	 */
-	total_read = stream->total_out - strlen(hdr) - 1;
-
-	/*
-	 * This size comparison must be "<=" to read the final zlib packets;
-	 * see the comment in unpack_loose_rest for details.
-	 */
-	while (total_read <= size &&
-	       (status == Z_OK ||
-		(status == Z_BUF_ERROR && !stream->avail_out))) {
-		stream->next_out = buf;
-		stream->avail_out = sizeof(buf);
-		if (size - total_read < stream->avail_out)
-			stream->avail_out = size - total_read;
-		status = git_inflate(stream, Z_FINISH);
-		git_hash_update(&c, buf, stream->next_out - buf);
-		total_read += stream->next_out - buf;
-	}
-
-	if (status != Z_STREAM_END) {
-		error(_("corrupt loose object '%s'"), oid_to_hex(expected_oid));
-		git_hash_discard(&c);
-		return -1;
-	}
-	if (stream->avail_in) {
-		error(_("garbage at end of loose object '%s'"),
-		      oid_to_hex(expected_oid));
-		git_hash_discard(&c);
-		return -1;
-	}
-
-	git_hash_final_oid(&real_oid, &c);
-	if (!oideq(expected_oid, &real_oid)) {
-		error(_("hash mismatch for %s (expected %s)"), path,
-		      oid_to_hex(expected_oid));
-		return -1;
-	}
-
-	return 0;
-}
-
-int read_loose_object(struct repository *repo,
-		      const char *path,
-		      const struct object_id *expected_oid,
-		      struct object_id *real_oid,
-		      void **contents,
-		      struct object_info *oi)
-{
-	int ret = -1;
-	int fd;
-	void *map = NULL;
-	unsigned long mapsize;
-	git_zstream stream;
-	char hdr[MAX_HEADER_LEN];
-	size_t *size = oi->sizep;
-
-	fd = git_open(path);
-	if (fd >= 0)
-		map = map_fd(fd, path, &mapsize);
-	if (!map) {
-		error_errno(_("unable to mmap %s"), path);
-		goto out;
-	}
-
-	if (unpack_loose_header(&stream, map, mapsize, hdr, sizeof(hdr)) != ULHR_OK) {
-		error(_("unable to unpack header of %s"), path);
-		goto out_inflate;
-	}
-
-	if (parse_loose_header(hdr, oi) < 0) {
-		error(_("unable to parse header of %s"), path);
-		goto out_inflate;
-	}
-
-	if (*oi->typep < 0) {
-		error(_("unable to parse type from header '%s' of %s"),
-		      hdr, path);
-		goto out_inflate;
-	}
-
-	if (*oi->typep == OBJ_BLOB &&
-	    *size > repo_settings_get_big_file_threshold(repo)) {
-		if (check_stream_oid(&stream, hdr, *size, path, expected_oid,
-				     repo->hash_algo) < 0)
-			goto out_inflate;
-	} else {
-		*contents = unpack_loose_rest(&stream, hdr, *size, expected_oid);
-		if (!*contents) {
-			error(_("unable to unpack contents of %s"), path);
-			goto out_inflate;
-		}
-		hash_object_file(repo->hash_algo,
-				 *contents, *size,
-				 *oi->typep, real_oid);
-		if (!oideq(expected_oid, real_oid))
-			goto out_inflate;
-	}
-
-	ret = 0; /* everything checks out */
-
-out_inflate:
-	git_inflate_end(&stream);
-out:
-	if (map)
-		munmap(map, mapsize);
-	return ret;
-}
-
 static int write_pack_lockfile(struct odb_source *source, const char *pack_name,
 			       const char *keep_msg, struct tempfile **out)
 {

57 lines not on this page · raw message

324 lines not on this page · raw message

index c1b78a25c9f..ab919f03fef 100644
@@ -43,31 +43,6 @@ static int get_conv_flags(unsigned flags)
 		return 0;
 }
 
-static void fill_loose_path(struct strbuf *buf,
-			    const struct object_id *oid,
-			    const struct git_hash_algo *algop)
-{
-	for (size_t i = 0; i < algop->rawsz; i++) {
-		static char hex[] = "0123456789abcdef";
-		unsigned int val = oid->hash[i];
-		strbuf_addch(buf, hex[val >> 4]);
-		strbuf_addch(buf, hex[val & 0xf]);
-		if (!i)
-			strbuf_addch(buf, '/');
-	}
-}
-
-const char *odb_loose_path(struct odb_source_loose *loose,
-			   struct strbuf *buf,
-			   const struct object_id *oid)
-{
-	strbuf_reset(buf);
-	strbuf_addstr(buf, loose->base.path);
-	strbuf_addch(buf, '/');
-	fill_loose_path(buf, oid, loose->base.odb->repo->hash_algo);
-	return buf->buf;
-}
-
 /* Returns 1 if we have successfully freshened the file, 0 otherwise. */
 static int freshen_file(const char *fn, const time_t *mtime)
 {

index 65ab44542b0..3cfcaefa420 100644
@@ -23,14 +23,6 @@ int index_path(struct index_state *istate, struct object_id *oid, const char *pa
 
 struct odb_source;
 
-/*
- * Put in `buf` the name of the file in the local object database that
- * would be used to store a loose object with the specified oid.
- */
-const char *odb_loose_path(struct odb_source_loose *source,
-			   struct strbuf *buf,
-			   const struct object_id *oid);
-
 /*
  * Iterate over the files in the loose-object parts of the object
  * directory "path", triggering the following callbacks:

index 96fc2ed213f..0f29162d752 100644
@@ -18,6 +18,27 @@
 #include "tempfile.h"
 #include "write-or-die.h"
 
+const char *odb_loose_path(struct odb_source_loose *loose,
+			   struct strbuf *buf,
+			   const struct object_id *oid)
+{
+	static const char hex[] = "0123456789abcdef";
+
+	strbuf_reset(buf);
+	strbuf_addstr(buf, loose->base.path);
+	strbuf_addch(buf, '/');
+
+	for (size_t i = 0; i < loose->base.odb->repo->hash_algo->rawsz; i++) {
+		unsigned int val = oid->hash[i];
+		strbuf_addch(buf, hex[val >> 4]);
+		strbuf_addch(buf, hex[val & 0xf]);
+		if (!i)
+			strbuf_addch(buf, '/');
+	}
+
+	return buf->buf;
+}
+
 static int append_loose_object(const struct object_id *oid,
 			       const char *path UNUSED,
 			       void *data)

index a6ae808662b..8d4cdac0d50 100644
@@ -49,4 +49,12 @@ static inline struct odb_source_loose *odb_source_loose_downcast(struct odb_sour
 	return container_of(source, struct odb_source_loose, base);
 }
 
+/*
+ * Put in `buf` the name of the file in the local object database that
+ * would be used to store a loose object with the specified oid.
+ */
+const char *odb_loose_path(struct odb_source_loose *source,
+			   struct strbuf *buf,
+			   const struct object_id *oid);
+
 #endif

index 3f7b20fde53..aae040567e3 100644
@@ -14,6 +14,7 @@
 #include "quote.h"
 #include "packfile.h"
 #include "object-file.h"
+#include "odb/source-loose.h"
 
 static unsigned long garbage;
 static off_t size_garbage;

index 81cecf6b6c3..d3ba6f0c9e9 100644
@@ -31,6 +31,7 @@
 #include "packfile.h"
 #include "object-file.h"
 #include "odb.h"
+#include "odb/source-loose.h"
 #include "path.h"
 #include "reflog.h"
 #include "rerere.h"

index 97f4c1418bc..af2a547d931 100644
@@ -17,6 +17,7 @@
 #include "object-file.h"
 #include "object-name.h"
 #include "odb.h"
+#include "odb/source-loose.h"
 #include "shallow.h"
 
 static const char * const prune_usage[] = {

104 lines not on this page · raw message

50 lines not on this page · raw message

104 lines not on this page · raw message

index 8d4cdac0d50..20f6ec7fa08 100644
@@ -57,4 +57,37 @@ const char *odb_loose_path(struct odb_source_loose *source,
 			   struct strbuf *buf,
 			   const struct object_id *oid);
 
+/*
+ * Iterate over the files in the loose-object parts of the object
+ * directory "path", triggering the following callbacks:
+ *
+ *  - loose_object is called for each loose object we find.
+ *
+ *  - loose_cruft is called for any files that do not appear to be
+ *    loose objects. Note that we only look in the loose object
+ *    directories "objects/[0-9a-f]{2}/", so we will not report
+ *    "objects/foobar" as cruft.
+ *
+ *  - loose_subdir is called for each top-level hashed subdirectory
+ *    of the object directory (e.g., "$OBJDIR/f0"). It is called
+ *    after the objects in the directory are processed.
+ *
+ * Any callback that is NULL will be ignored. Callbacks returning non-zero
+ * will end the iteration.
+ */
+typedef int each_loose_object_fn(const struct object_id *oid,
+				 const char *path,
+				 void *data);
+typedef int each_loose_cruft_fn(const char *basename,
+				const char *path,
+				void *data);
+typedef int each_loose_subdir_fn(unsigned int nr,
+				 const char *path,
+				 void *data);
+int for_each_loose_file_in_source(struct odb_source *source,
+				  each_loose_object_fn obj_cb,
+				  each_loose_cruft_fn cruft_cb,
+				  each_loose_subdir_fn subdir_cb,
+				  void *data);
+
 #endif

index b6749b8d132..d0a8a062b97 100644
@@ -3,6 +3,7 @@
 #include "git-compat-util.h"
 #include "gettext.h"
 #include "object-file.h"
+#include "odb/source-loose.h"
 #include "packfile.h"
 #include "progress.h"
 #include "prune-packed.h"

index a90268e200a..5cd679e802c 100644
@@ -544,28 +544,11 @@ static void *odb_source_loose_map_object(struct odb_source_loose *loose,
 					 const struct object_id *oid,
 					 unsigned long *size)
 {
-	const char *p;
-	int fd = open_loose_object(loose, oid, &p);
-	void *map = NULL;
-	struct stat st;
-
+	const char *path;
+	int fd = open_loose_object(loose, oid, &path);
 	if (fd < 0)
 		return NULL;
-
-	if (!fstat(fd, &st)) {
-		*size = xsize_t(st.st_size);
-		if (!*size) {
-			/* mmap() is forbidden on empty files */
-			error(_("object file %s is empty"), p);
-			goto out;
-		}
-
-		map = xmmap(NULL, *size, PROT_READ, MAP_PRIVATE, fd, 0);
-	}
-
-out:
-	close(fd);
-	return map;
+	return map_fd(fd, path, size);
 }
 
 struct odb_loose_read_stream {

index ca3a891dfbc..e53c69af31f 100644
@@ -34,6 +34,7 @@
 #include "object-file.h"
 #include "object-file-convert.h"
 #include "odb.h"
+#include "odb/source-loose.h"
 #include "odb/streaming.h"
 #include "replace-object.h"
 #include "dir.h"

index 4f50dd9b8d0..13730a27847 100644
@@ -11,6 +11,7 @@
 #include "packfile.h"
 #include "object-file.h"
 #include "odb.h"
+#include "odb/source-loose.h"
 
 struct alt_base {
 	char *base;

index d79678d01f0..25b5fc3b3b7 100644
@@ -21,6 +21,7 @@
 #include "string-list.h"
 #include "object-file.h"
 #include "odb.h"
+#include "odb/source-loose.h"
 #include "tempfile.h"
 #include "date.h"
 #include "trace2.h"

index d6a2bb5ba3c..64cd6f1c3c1 100644
@@ -4,6 +4,7 @@
 #include "object-file.h"
 #include "odb.h"
 #include "odb/source-files.h"
+#include "odb/source-loose.h"
 #include "hex.h"
 #include "repository.h"
 #include "wrapper.h"

index d3ddcbf309c..917a65d5b33 100644
@@ -20,6 +20,7 @@
 #include "object-file-convert.h"
 #include "object-file.h"
 #include "odb.h"
+#include "odb/source.h"
 #include "odb/streaming.h"
 #include "odb/transaction.h"
 #include "pack.h"

index 83d3f38a540..746356ceb51 100644
@@ -4,7 +4,7 @@
 #include "git-zlib.h"
 #include "object.h"
 #include "odb.h"
-#include "odb/source-loose.h"
+#include "odb/streaming.h"
 #include "odb/transaction.h"
 
 /* The maximum size for an object header. */

index 61bcdafe82a..bc94463a239 100644
@@ -9,6 +9,7 @@
 #include "hex.h"
 #include "object-file.h"
 #include "object-name.h"
+#include "odb/source.h"
 #include "refs.h"
 #include "repository.h"
 #include "config.h"

index a0b7119c639..c705ce06adb 100644
@@ -21,6 +21,7 @@
 #include "dir.h"
 #include "environment.h"
 #include "gettext.h"
+#include "git-zlib.h"
 #include "hex.h"
 #include "xdiff-interface.h"
 #include "merge-ll.h"

index 9db3f83eea8..c4d955b61f1 100644
@@ -5,6 +5,7 @@
 #include "abspath.h"
 #include "environment.h"
 #include "gettext.h"
+#include "git-zlib.h"
 #include "hex.h"
 #include "config.h"
 #include "lockfile.h"

index 0264c18c5f8..6d758a04419 100644
@@ -6,6 +6,7 @@
 #include "delta.h"
 #include "environment.h"
 #include "gettext.h"
+#include "git-zlib.h"
 #include "hex.h"
 #include "pack.h"
 #include "csum-file.h"

index e53c69af31f..f13c49090a1 100644
@@ -4,6 +4,7 @@
 #include "builtin.h"
 #include "environment.h"
 #include "gettext.h"
+#include "git-zlib.h"
 #include "hex.h"
 #include "config.h"
 #include "attr.h"

index b4ac17f8efc..0309f91f5ab 100644
@@ -18,6 +18,7 @@
 #include "diff.h"
 #include "diffcore.h"
 #include "delta.h"
+#include "git-zlib.h"
 #include "hex.h"
 #include "xdiff-interface.h"
 #include "color.h"

index 917a65d5b33..cf859520949 100644
@@ -15,6 +15,7 @@
 #include "environment.h"
 #include "fsck.h"
 #include "gettext.h"
+#include "git-zlib.h"
 #include "hex.h"
 #include "loose.h"
 #include "object-file-convert.h"

index 746356ceb51..70138a358de 100644
@@ -1,7 +1,6 @@
 #ifndef OBJECT_FILE_H
 #define OBJECT_FILE_H
 
-#include "git-zlib.h"
 #include "object.h"
 #include "odb.h"
 #include "odb/streaming.h"

index 5cd679e802c..737d583f033 100644
@@ -3,6 +3,7 @@
 #include "chdir-notify.h"
 #include "dir.h"
 #include "gettext.h"
+#include "git-zlib.h"
 #include "hex.h"
 #include "loose.h"
 #include "object-file.h"

index 605811b7b09..c6c3d7c9c0c 100644
@@ -2,6 +2,7 @@
 
 #include "git-compat-util.h"
 #include "environment.h"
+#include "git-zlib.h"
 #include "hex.h"
 #include "repository.h"
 #include "pack.h"

index b32b4cd75da..59e0e0c1c91 100644
@@ -3,6 +3,7 @@
 #include "git-compat-util.h"
 #include "environment.h"
 #include "gettext.h"
+#include "git-zlib.h"
 #include "hex.h"
 #include "list.h"
 #include "pack.h"