Introduce rust: In xdiff

This is a continuation of
https://lore.kernel.org/git/[email protected]/,
but I am removing the RFC label.

Changes since the last RFC patch series (range-diff added below):

* Addressed feedback:
* Use self instead of this
* Update ivec to use Git code style
* Remove environment variable CI_IS_DOCKER from github workflows
* Remove PHONY target interop-objs
* Removed left over asserts in Rust and fix near by code
* Other changes:
* Avoid name mangling of ivec if c++ compiler used
* Fixed a typo in patch 11 commit message

High level overview:

* patch 1: add a policy for using Rust (brian's patch, with a small tweak)
* patch 2: introduce Rust to the codebase
* patches 3-5: adapt CI (github workflows) to build Git with Rust
* patch 6: introduce the ivec type
* patches 7-14: xdiff code cleanup in preparation for translating to Rust
* patch 15: translate a C function into Rust and call it from C

Build results for these changes:
https://github.com/git/git/actions/runs/17330190596

Range-Diff against last RFC version:

 1:  6d065f550f =  1:  6d065f550f doc: add a policy for using Rust
 2:  0393995125 =  2:  cc9cf87775 xdiff: introduce rust
 3:  a98d9e4d21 !  3:  6c47401ba0 github workflows: install rust
    @@ .github/workflows/main.yml: jobs:
                image: fedora:latest
              # A RHEL 8 compatible distro.  Supported until 2029-05-31.
     @@ .github/workflows/main.yml: jobs:
    -       jobname: ${{matrix.vector.jobname}}
            CC: ${{matrix.vector.cc}}
            CI_JOB_IMAGE: ${{matrix.vector.image}}
    -+      CI_IS_DOCKER: "true"
            CUSTOM_PATH: /custom
     +      CARGO_HOME: /home/builder/.cargo
          runs-on: ubuntu-latest
 4:  0d2b39c3e0 =  4:  8e350700bc win+Meson: do allow linking with the Rust-built xdiff
 5:  e65488ab99 =  5:  a00cbf7bcb github workflows: upload Cargo.lock
 6:  db5d22b188 !  6:  922d506ed6 ivec: create a vector type that is interoperable between C and Rust
    @@ Makefile: reconfigure config.mak.autogen: config.status
      endif
      
     +INTEROP_OBJS += interop/ivec.o
    -+.PHONY: interop-objs
    -+interop-objs: $(INTEROP_OBJS)
     +
      XDIFF_OBJS += xdiff/xdiffi.o
      XDIFF_OBJS += xdiff/xemit.o
    @@ interop/ivec.c (new)
     @@
     +#include "ivec.h"
     +
    -+static void ivec_set_capacity(void* self, usize new_capacity) {
    -+	struct rawivec *this = self;
    ++#ifdef __cplusplus
    ++extern "C" {
    ++#endif
    ++
    ++static void ivec_set_capacity(void *self_, usize new_capacity)
    ++{
    ++	struct rawivec *self = self_;
    ++
     +	if (new_capacity == 0)
    -+		FREE_AND_NULL(this->ptr);
    ++		FREE_AND_NULL(self->ptr);
     +	else
    -+		this->ptr = xrealloc(this->ptr, new_capacity * this->element_size);
    -+	this->capacity = new_capacity;
    ++		self->ptr = xrealloc(self->ptr, new_capacity * self->element_size);
    ++	self->capacity = new_capacity;
     +}
     +
    -+void ivec_init(void* self, usize element_size) {
    -+	struct rawivec *this = self;
    -+	this->ptr = NULL;
    -+	this->length = 0;
    -+	this->capacity = 0;
    -+	this->element_size = element_size;
    ++void ivec_init(void *self_, usize element_size)
    ++{
    ++	struct rawivec *self = self_;
    ++
    ++	self->ptr = NULL;
    ++	self->length = 0;
    ++	self->capacity = 0;
    ++	self->element_size = element_size;
     +}
     +
     +/*
    @@ interop/ivec.c (new)
     + * to the specified capacity, and then calloc self.capacity number of
     + * elements.
     + */
    -+void ivec_zero(void* self, usize capacity) {
    -+	struct rawivec *this = self;
    -+	if (this->ptr)
    -+		FREE_AND_NULL(this->ptr);
    -+	this->capacity = this->length = capacity;
    -+	this->ptr = xcalloc(this->capacity, this->element_size);
    ++void ivec_zero(void *self_, usize capacity)
    ++{
    ++	struct rawivec *self = self_;
    ++
    ++	if (self->ptr)
    ++		FREE_AND_NULL(self->ptr);
    ++	self->capacity = self->length = capacity;
    ++	self->ptr = xcalloc(self->capacity, self->element_size);
     +}
     +
    -+void ivec_clear(void* self) {
    -+	struct rawivec *this = self;
    -+	this->length = 0;
    ++void ivec_clear(void *self_)
    ++{
    ++	struct rawivec *self = self_;
    ++
    ++	self->length = 0;
     +}
     +
    -+void ivec_reserve_exact(void* self, usize additional) {
    -+	struct rawivec *this = self;
    -+	usize new_capacity = this->capacity + additional;
    ++void ivec_reserve_exact(void *self_, usize additional)
    ++{
    ++	struct rawivec *self = self_;
    ++	usize new_capacity = self->capacity + additional;
    ++
     +	ivec_set_capacity(self, new_capacity);
     +}
     +
    -+void ivec_reserve(void* self, usize additional) {
    -+	struct rawivec *this = self;
    ++void ivec_reserve(void *self_, usize additional)
    ++{
    ++	struct rawivec *self = self_;
     +	usize growby = 128;
    -+	if (this->capacity > growby) {
    -+		growby = this->capacity;
    ++
    ++	if (self->capacity > growby) {
    ++		growby = self->capacity;
     +	}
     +	if (additional > growby) {
     +		growby = additional;
    @@ interop/ivec.c (new)
     +	ivec_reserve_exact(self, growby);
     +}
     +
    -+void ivec_shrink_to_fit(void* self) {
    -+	struct rawivec *this = self;
    -+	ivec_set_capacity(self, this->length);
    ++void ivec_shrink_to_fit(void *self_)
    ++{
    ++	struct rawivec *self = self_;
    ++
    ++	ivec_set_capacity(self_, self->length);
     +}
     +
    -+void ivec_resize(void* self, usize new_length, void* default_value) {
    -+	struct rawivec *this = self;
    -+	isize additional = (isize) (new_length - this->capacity);
    ++void ivec_resize(void *self_, usize new_length, void *default_value)
    ++{
    ++	struct rawivec *self = self_;
    ++	isize additional = (isize) (new_length - self->capacity);
    ++
     +	if (additional > 0) {
    -+		ivec_reserve(self, additional);
    ++		ivec_reserve(self_, additional);
     +	}
     +
    -+	for (usize i = this->length; i < new_length; i++) {
    -+		void* dst = (u8*) this->ptr + (this->length + i) * this->element_size;
    -+		memcpy(dst, default_value, this->element_size);
    ++	for (usize i = self->length; i < new_length; i++) {
    ++		void *dst = (u8 *)self->ptr + (self->length + i) * self->element_size;
    ++		memcpy(dst, default_value, self->element_size);
     +	}
    -+	this->length = new_length;
    ++	self->length = new_length;
     +}
     +
    -+void ivec_push(void* self, void* value) {
    -+	struct rawivec *this = self;
    -+	u8* dst;
    ++void ivec_push(void *self_, void *value)
    ++{
    ++	struct rawivec *self = self_;
    ++	u8 *dst;
     +
    -+	if (this->length == this->capacity) {
    -+		ivec_reserve(self, 1);
    ++	if (self->length == self->capacity) {
    ++		ivec_reserve(self_, 1);
     +	}
    -+	dst = (u8*) this->ptr + this->length * this->element_size;
    -+	memcpy(dst, value, this->element_size);
    -+	this->length++;
    ++	dst = (u8 *)self->ptr + self->length * self->element_size;
    ++	memcpy(dst, value, self->element_size);
    ++	self->length++;
     +}
     +
    -+void ivec_extend_from_slice(void* self, void const* ptr, usize size) {
    -+	struct rawivec *this = self;
    -+	u8* dst;
    ++void ivec_extend_from_slice(void *self_, void const *ptr, usize size)
    ++{
    ++	struct rawivec *self = self_;
    ++	u8 *dst;
     +
     +	if (size == 0)
     +		return;
     +
    -+	if (this->length + size > this->capacity) {
    -+		ivec_reserve(self, this->capacity - this->length + size);
    ++	if (self->length + size > self->capacity) {
    ++		ivec_reserve(self_, self->capacity - self->length + size);
     +	}
    -+	dst = (u8*) this->ptr + this->length * this->element_size;
    -+	memcpy(dst, ptr, size * this->element_size);
    -+	this->length += size;
    ++	dst = (u8 *)self->ptr + self->length * self->element_size;
    ++	memcpy(dst, ptr, size * self->element_size);
    ++	self->length += size;
     +}
     +
    -+bool ivec_equal(void* self, void* other) {
    -+	struct rawivec *lhs = self;
    ++bool ivec_equal(void *self_, void *other)
    ++{
    ++	struct rawivec *lhs = self_;
     +	struct rawivec *rhs = other;
     +
     +	if (lhs->element_size != rhs->element_size) {
    @@ interop/ivec.c (new)
     +		return false;
     +	}
     +
    -+
     +	for (usize i = 0; i < lhs->length; i++) {
    -+		void* left = (u8 *) lhs->ptr + i * lhs->element_size;
    -+		void* right = (u8 *) rhs->ptr + i * rhs->element_size;
    ++		void *left = (u8 *)lhs->ptr + i * lhs->element_size;
    ++		void *right = (u8 *)rhs->ptr + i * rhs->element_size;
     +		if (memcmp(left, right, lhs->element_size) != 0) {
     +			return false;
     +		}
    @@ interop/ivec.c (new)
     +}
     +
     +
    -+void ivec_free(void* self) {
    -+	struct rawivec *this = self;
    -+	FREE_AND_NULL(this->ptr);
    -+	this->length = 0;
    -+	this->capacity = 0;
    ++void ivec_free(void *self_)
    ++{
    ++	struct rawivec *self = self_;
    ++
    ++	FREE_AND_NULL(self->ptr);
    ++	self->length = 0;
    ++	self->capacity = 0;
     +	/* don't modify self->element_size */
     +}
     +
    -+void ivec_move(void* source, void* destination) {
    -+	struct rawivec *this = source;
    -+	struct rawivec *that = destination;
    ++void ivec_move(void *source, void *destination)
    ++{
    ++	struct rawivec *src = source;
    ++	struct rawivec *dst = destination;
     +
    -+	if (this->element_size != that->element_size)
    ++	if (src->element_size != dst->element_size)
     +		BUG("mismatched element_size");
     +
     +	ivec_free(destination);
    -+	that->ptr = this->ptr;
    -+	that->length = this->length;
    -+	that->capacity = this->capacity;
    ++	dst->ptr = src->ptr;
    ++	dst->length = src->length;
    ++	dst->capacity = src->capacity;
    ++
    ++	src->ptr = NULL;
    ++	src->length = 0;
    ++	src->capacity = 0;
    ++}
     +
    -+	this->ptr = NULL;
    -+	this->length = 0;
    -+	this->capacity = 0;
    ++#ifdef __cplusplus
     +}
    ++#endif
     
      ## interop/ivec.h (new) ##
     @@
     +#ifndef IVEC_H
     +#define IVEC_H
     +
    -+#include "../git-compat-util.h"
    ++#include <git-compat-util.h>
    ++
    ++#ifdef __cplusplus
    ++extern "C" {
    ++#endif
     +
     +struct rawivec {
    -+	void* ptr;
    ++	void *ptr;
     +	usize length;
     +	usize capacity;
     +	usize element_size;
    @@ interop/ivec.h (new)
     +
     +#define DEFINE_IVEC_TYPE(type, suffix) \
     +struct ivec_##suffix { \
    -+	type* ptr; \
    ++	type *ptr; \
     +	size_t length; \
     +	size_t capacity; \
     +	size_t element_size; \
    @@ interop/ivec.h (new)
     +DEFINE_IVEC_TYPE(usize, usize);
     +DEFINE_IVEC_TYPE(isize, isize);
     +
    -+void ivec_init(void* self, usize element_size);
    -+void ivec_zero(void* self, usize capacity);
    -+void ivec_clear(void* self);
    -+void ivec_reserve_exact(void* self, usize additional);
    -+void ivec_reserve(void* self, usize additional);
    -+void ivec_shrink_to_fit(void* self);
    -+void ivec_resize(void* self, usize new_length, void* default_value);
    -+void ivec_push(void* self, void* value);
    -+void ivec_extend_from_slice(void* self, void const* ptr, usize size);
    -+bool ivec_equal(void* self, void* other);
    -+void ivec_free(void* self);
    -+void ivec_move(void* source, void* destination);
    ++void ivec_init(void *self_, usize element_size);
    ++void ivec_zero(void *self_, usize capacity);
    ++void ivec_clear(void *self_);
    ++void ivec_reserve_exact(void *self_, usize additional);
    ++void ivec_reserve(void *self_, usize additional);
    ++void ivec_shrink_to_fit(void *self_);
    ++void ivec_resize(void *self_, usize new_length, void *default_value);
    ++void ivec_push(void *self_, void *value);
    ++void ivec_extend_from_slice(void *self_, void const *ptr, usize size);
    ++bool ivec_equal(void *self_, void *other);
    ++void ivec_free(void *self_);
    ++void ivec_move(void *source, void *destination);
    ++
    ++#ifdef __cplusplus
    ++}
    ++#endif
     +
     +#endif //IVEC_H
     
    @@ rust/interop/src/ivec.rs (new)
     +
     +impl<T> Drop for IVec<T> {
     +    fn drop(&mut self) {
    -+        unsafe {
    -+            self._free();
    -+        }
    ++        self._set_capacity(0);
     +    }
     +}
     +
    @@ rust/interop/src/ivec.rs (new)
     +        }
     +    }
     +
    -+    fn _zero(&mut self) {
    -+        self.ptr = std::ptr::null_mut();
    -+        self.length = 0;
    -+        self.capacity = 0;
    -+        // DO NOT MODIFY element_size!!!
    -+    }
    -+
    -+    unsafe fn _free(&mut self) {
    -+        free(self.ptr as *mut std::ffi::c_void);
    -+        self._zero();
    -+    }
    -+
     +    fn _set_capacity(&mut self, new_capacity: usize) {
     +        unsafe {
     +            if new_capacity == self.capacity {
     +                return;
     +            }
    ++            if new_capacity < self.length {
    ++                self.truncate(new_capacity);
    ++            }
     +            if new_capacity == 0 {
    -+                self._free();
    ++                free(self.ptr as *mut c_void);
    ++                self.ptr = std::ptr::null_mut();
    ++                self.length = 0;
    ++                self.capacity = 0;
    ++                // DO NOT MODIFY element_size!!!
     +            } else {
    -+                let t = realloc(
    -+                    self.ptr as *mut std::ffi::c_void,
    -+                    new_capacity * size_of::<T>(),
    -+                );
    ++                let t = realloc(self.ptr as *mut c_void, new_capacity * size_of::<T>());
     +                if t.is_null() {
     +                    panic!("out of memory");
     +                }
    @@ rust/interop/src/ivec.rs (new)
     +            /* capacity does not need to be changed */
     +        }
     +
    ++        /* IVec grows */
     +        if new_length > self.length {
     +            let range = self.length..new_length;
     +            self._buffer_mut()[range].fill(default_value);
    ++            self.length = new_length;
     +        }
     +
    -+        self.length = new_length;
    ++        /* IVec shrinks */
    ++        if new_length < self.length {
    ++            self.truncate(new_length);
    ++        }
     +    }
     +
     +    fn _buffer_mut(&mut self) -> &mut [T] {
    @@ rust/interop/src/ivec.rs (new)
     +        self._resize(new_length, default_value, true);
     +    }
     +
    ++    pub fn truncate(&mut self, new_length: usize) {
    ++        if new_length >= self.length {
    ++            return;
    ++        }
    ++
    ++        if std::mem::needs_drop::<T>() {
    ++            let range = new_length..self.length;
    ++            for v in &mut self.as_mut_slice()[range] {
    ++                unsafe {
    ++                    std::ptr::drop_in_place(v);
    ++                }
    ++            }
    ++        }
    ++
    ++        self.length = new_length;
    ++    }
    ++
     +    pub fn insert(&mut self, index: usize, value: T) {
     +        if self.length == self.capacity {
     +            self.reserve(1);
    @@ rust/interop/src/ivec.rs (new)
     +#[cfg(test)]
     +mod tests {
     +    use crate::ivec::IVec;
    ++    use std::cell::RefCell;
    ++    use std::mem::size_of;
    ++    use std::ops::{Deref, DerefMut};
     +    use std::panic;
    ++    use std::rc::Rc;
    ++
    ++    struct DropTest {
    ++        value: Rc<RefCell<u64>>,
    ++    }
    ++
    ++    impl DropTest {
    ++        fn new(value: Rc<RefCell<u64>>) -> Self {
    ++            Self { value }
    ++        }
    ++    }
    ++
    ++    impl Drop for DropTest {
    ++        fn drop(&mut self) {
    ++            *self.value.borrow_mut() -= 1;
    ++        }
    ++    }
    ++
    ++    #[test]
    ++    fn test_drop_elements() {
    ++        let counter = Rc::new(RefCell::new(0u64));
    ++        let size = 5;
    ++
    ++        /* drop whole IVec */
    ++        let mut vec = IVec::new();
    ++        for _ in 0..size {
    ++            let tmp = DropTest::new(counter.clone());
    ++            *tmp.value.borrow_mut() += 1;
    ++            vec.push(tmp);
    ++        }
    ++        let cur = *counter.borrow();
    ++        assert_eq!(size, cur);
    ++        drop(vec);
    ++
    ++        let cur = *counter.borrow();
    ++        assert_eq!(0u64, cur);
    ++
    ++        /* drop some elements */
    ++        let mut vec = IVec::new();
    ++        for i in 0..size {
    ++            let tmp = DropTest::new(counter.clone());
    ++            *tmp.value.borrow_mut() += 1;
    ++            vec.push(tmp);
    ++        }
    ++        let cur = *counter.borrow().deref();
    ++        assert_eq!(size, cur);
    ++
    ++        let expected = 2u64;
    ++        vec.truncate(expected as usize);
    ++
    ++        let cur = *counter.borrow();
    ++        assert_eq!(expected, cur);
    ++        drop(vec);
    ++    }
     +
     +    #[test]
     +    fn test_panic_on_out_of_bounds() {
    @@ rust/interop/src/ivec.rs (new)
     +        let expected = vec.capacity + 10;
     +        let default_value = 19;
     +        vec.resize(expected, default_value);
    -+        // assert_eq!(vec.capacity, vec.slice.len());
     +        assert_eq!(expected, vec.length);
     +        assert!(vec.capacity >= expected);
     +        for i in 0..vec.length {
    @@ rust/interop/src/ivec.rs (new)
     +        }
     +
     +        vec.reserve(10);
    -+        // assert_eq!(vec.capacity, vec.slice.len());
     +        assert!(vec.capacity > vec.length);
     +        let length_before = vec.length;
     +        vec.shrink_to_fit();
     +        assert_eq!(length_before, vec.length);
     +        assert_eq!(vec.length, vec.capacity);
    -+        // assert_eq!(vec.capacity, vec.slice.len());
     +    }
     +
     +    #[test]
    @@ rust/interop/src/ivec.rs (new)
     +        assert_eq!(16, vec.element_size);
     +        assert_eq!(size_of::<usize>() * 4, size_of::<IVec<u128>>());
     +    }
    -+
    -+    #[test]
    -+    fn test_manual_free() {
    -+        type TestType = i16;
    -+        let mut vec = IVec::<TestType>::new();
    -+
    -+        unsafe { vec._free() };
    -+        assert!(vec.ptr.is_null());
    -+        assert_eq!(0, vec.length);
    -+        assert_eq!(0, vec.capacity);
    -+        assert_eq!(size_of::<TestType>(), vec.element_size);
    -+    }
     +}
     
      ## rust/interop/src/lib.rs ##
    @@ rust/interop/src/lib.rs
     +    pub fn realloc(ptr: *mut c_void, size: usize) -> *mut c_void;
     +    pub fn free(ptr: *mut c_void);
     +}
    -
    - ## rust/xdiff/src/lib.rs ##
    -@@
    -+
 7:  d4bed95463 =  7:  3a6f822e2a xdiff/xprepare: remove superfluous forward declarations
 8:  7c68ce5349 =  8:  c7cd71dae0 xdiff: delete unnecessary fields from xrecord_t and xdfile_t
 9:  e516ccc8c0 =  9:  1096c8f0a4 xdiff: make fields of xrecord_t Rust friendly
10:  21bfb9f088 = 10:  0cd2168512 xdiff: use one definition for freeing xdfile_t
11:  6ce0e252b3 ! 11:  7ea2dccd71 xdiff: replace chastore with an ivec in xdfile_t

@@ Commit message
xdiff: replace chastore with an ivec in xdfile_t

xdfile_t currently uses a chastore which functions as a memory pool and
- a vector which maps to the alocations created by the chastore. It seems
+ a vector which maps to the allocations created by the chastore. It seems
like xrecord_t used to be a linked list until the recs and nrec fields were added. I think that xrecord_t.next was meant to be removed, but was overlooked. This dual data structure setup make the code somewhat

12:  0cfc6cf26b = 12:  d8f561e173 xdiff: delete nrec field from xdfile_t
13:  cf0387d851 = 13:  15c2854a32 xdiff: delete recs field from xdfile_t
14:  ea699135f9 = 14:  3dacb2c09d xdiff: make xdfile_t more rust friendly
15:  b18544b74f ! 15:  9fdd23acf6 xdiff: implement xdl_trim_ends() in Rust
    @@ rust/xdiff/src/lib.rs
     @@
     +pub mod xprepare;
     +pub mod xtypes;
    - 
    ++
     +use crate::xprepare::trim_ends;
     +use crate::xtypes::xdfile;
     +

Ezekiel Newren (13):
xdiff: introduce rust
github workflows: install rust
github workflows: upload Cargo.lock
ivec: create a vector type that is interoperable between C and Rust xdiff/xprepare: remove superfluous forward declarations
xdiff: delete unnecessary fields from xrecord_t and xdfile_t
xdiff: make fields of xrecord_t Rust friendly
xdiff: use one definition for freeing xdfile_t
xdiff: replace chastore with an ivec in xdfile_t
xdiff: delete nrec field from xdfile_t
xdiff: delete recs field from xdfile_t
xdiff: make xdfile_t more rust friendly
xdiff: implement xdl_trim_ends() in Rust

Johannes Schindelin (1):
win+Meson: do allow linking with the Rust-built xdiff

brian m. carlson (1):
doc: add a policy for using Rust

 .github/workflows/main.yml                    |  88 ++-
 .gitignore                                    |   3 +
 Documentation/Makefile                        |   1 +
 Documentation/technical/platform-support.adoc |   2 +
 Documentation/technical/rust-support.adoc     | 142 +++++
 Makefile                                      |  67 ++-
 build_rust.sh                                 |  57 ++
 ci/install-dependencies.sh                    |  14 +-
 ci/install-rust-toolchain.sh                  |  30 +
 ci/install-rustup.sh                          |  25 +
 ci/lib.sh                                     |   1 +
 ci/make-test-artifacts.sh                     |   9 +
 ci/run-build-and-tests.sh                     |  13 +
 config.mak.uname                              |   4 +
 git-compat-util.h                             |  17 +
 interop/ivec.c                                | 180 ++++++
 interop/ivec.h                                |  60 ++
 meson.build                                   |  54 +-
 rust/Cargo.toml                               |   6 +
 rust/interop/Cargo.toml                       |  14 +
 rust/interop/src/ivec.rs                      | 516 ++++++++++++++++++
 rust/interop/src/lib.rs                       |  10 +
 rust/xdiff/Cargo.toml                         |  15 +
 rust/xdiff/src/lib.rs                         |  15 +
 rust/xdiff/src/xprepare.rs                    |  27 +
 rust/xdiff/src/xtypes.rs                      |  19 +
 xdiff/xdiffi.c                                |  60 +-
 xdiff/xdiffi.h                                |   8 +-
 xdiff/xemit.c                                 |  24 +-
 xdiff/xhistogram.c                            |   2 +-
 xdiff/xmerge.c                                |  72 +--
 xdiff/xpatience.c                             |  16 +-
 xdiff/xprepare.c                              | 271 ++++-----
 xdiff/xtypes.h                                |  27 +-
 xdiff/xutils.c                                |  12 +-
 35 files changed, 1562 insertions(+), 319 deletions(-)
 create mode 100644 Documentation/technical/rust-support.adoc
 create mode 100755 build_rust.sh
 create mode 100755 ci/install-rust-toolchain.sh
 create mode 100755 ci/install-rustup.sh
 create mode 100644 interop/ivec.c
 create mode 100644 interop/ivec.h
 create mode 100644 rust/Cargo.toml
 create mode 100644 rust/interop/Cargo.toml
 create mode 100644 rust/interop/src/ivec.rs
 create mode 100644 rust/interop/src/lib.rs
 create mode 100644 rust/xdiff/Cargo.toml
 create mode 100644 rust/xdiff/src/lib.rs
 create mode 100644 rust/xdiff/src/xprepare.rs
 create mode 100644 rust/xdiff/src/xtypes.rs

base-commit: 16bd9f20a403117f2e0d9bcda6c6e621d3763e77
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-git-2043%2Fezekielnewren%2Fintroduce_rust-v1
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-git-2043/ezekielnewren/introduce_rust-v1
Pull-Request: https://github.com/git/git/pull/2043

gitgitgadget

From: "brian m. carlson" <[email protected]>

Git has historically been written primarily in C, with some shell and Perl. However, C is not memory safe, which makes it more likely that security vulnerabilities or other bugs will be introduced, and it is also more verbose and less ergonomic than other, more modern languages.

One of the most common modern compiled languages which is easily interoperable with C is Rust. It is popular (the most admired language on the 2024 Stack Overflow Developer Survey), efficient, portable, and robust.

Introduce a document laying out the incremental introduction of Rust to Git and provide a detailed rationale for doing so, including the points above. Propose a design for this approach that addresses the needs of downstreams and distributors, as well as contributors.

Since we don't want to carry both a C and Rust version of code and want to be able to add new features only in Rust, mention that Rust is a required part of our platform support policy.

It should be noted that a recent discussion at the Berlin Git Merge Contributor Summit found widespread support for the addition of Rust to Git. While of course not all contributors were represented, the proposal appeared to have the support of a majority of active contributors.

[en: Added some comments about types, and changed the recommondations
about cbindgen, bindgen, rustix, libc.]

---
 Documentation/Makefile                        |   1 +
 Documentation/technical/platform-support.adoc |   2 +
 Documentation/technical/rust-support.adoc     | 142 ++++++++++++++++++
 3 files changed, 145 insertions(+)
 create mode 100644 Documentation/technical/rust-support.adoc

index b109d25e9c..066b761c01 100644
@@ -127,6 +127,7 @@ TECH_DOCS += technical/parallel-checkout
 TECH_DOCS += technical/partial-clone
 TECH_DOCS += technical/platform-support
 TECH_DOCS += technical/racy-git
+TECH_DOCS += technical/rust-support
 TECH_DOCS += technical/reftable
 TECH_DOCS += technical/scalar
 TECH_DOCS += technical/send-pack-pipeline

index 0a2fb28d62..dc71672dcb 100644
@@ -33,6 +33,8 @@ meet the following minimum requirements:
 
 * Has active security support (taking security releases of dependencies, etc)
 
+* Supports Rust and the toolchain version specified in link:rust-support.adoc[].
+
 These requirements are a starting point, and not sufficient on their own for the
 Git community to be enthusiastic about supporting your platform. Maintainers of
 platforms which do meet these requirements can follow the steps below to make it

gitgitgadget

Could you create a patch with your preferred wording about bindgen and cbindgen for me? I'd prefer to introduce bindgen/cbindgen in a different patch series because this one is already doing a lot just to make Rust exist in Git.

I think it's fine to introduce it in a different series. I'll plan to do that myself if it doesn't get done sooner.

My change to the text is the following:

Tools like bindgen and cbindgen create C-styled unsafe Rust code rather than idiomatic Rust; where possible, we prefer to switch to idiomatic Rust. However, we may use bindgen and cbindgen to share existing Git types as an interim step.

You can find the patch at https://github.com/bk2204/git.git in the rust-policy branch.

brian m. carlson (they/them)
Toronto, Ontario, CA

Actually now that I'm knee deep in adding cbindgen to Rust it'll make less sense to add it later. I'm currently working on refactoring my entire patch series to include cbindgen from the beginning. I haven't looked into cbindgen until now because I wanted to understand at a deep level how C <-> Rust ffi worked rather than using an automagical tool like cbindgen. I now think cbindgen should be part of the introduction of Rust.

I've been able to get cbindgen to work with 1.63.0, but I'm not ready to release my next patch series yet.

From: Ezekiel Newren <[email protected]>

Upcoming patches will simplify xdiff, while also porting parts of it to Rust. In preparation, add some stubs and setup the Rust build. For now, it is easier to let cargo build rust and have make or meson merely link against the static library that cargo builds. In line with ongoing libification efforts, use multiple crates to allow more modularity on the Rust side. xdiff is the crate that this series will focus on, but we also introduce the interop crate for future patch series.

In order to facilitate interoperability between C and Rust, introduce C definitions for Rust primitive types in git-compat-util.h.

---
 .gitignore              |  3 +++
 Makefile                | 53 ++++++++++++++++++++++++++++----------
 build_rust.sh           | 57 +++++++++++++++++++++++++++++++++++++++++
 git-compat-util.h       | 17 ++++++++++++
 meson.build             | 52 +++++++++++++++++++++++++++++++------
 rust/Cargo.toml         |  6 +++++
 rust/interop/Cargo.toml | 14 ++++++++++
 rust/interop/src/lib.rs |  0
 rust/xdiff/Cargo.toml   | 15 +++++++++++
 rust/xdiff/src/lib.rs   |  0
 10 files changed, 196 insertions(+), 21 deletions(-)
 create mode 100755 build_rust.sh
 create mode 100644 rust/Cargo.toml
 create mode 100644 rust/interop/Cargo.toml
 create mode 100644 rust/interop/src/lib.rs
 create mode 100644 rust/xdiff/Cargo.toml
 create mode 100644 rust/xdiff/src/lib.rs

index 04c444404e..ff81e3580c 100644
@@ -254,3 +254,6 @@ Release/
 /contrib/buildsystems/out
 /contrib/libgit-rs/target
 /contrib/libgit-sys/target
+/.idea/
+/rust/target/
+/rust/Cargo.lock

index 4678e21c4c..82dc99764a 100644
@@ -196,6 +196,23 @@ static inline int is_xplatform_dir_sep(int c)
 #include "compat/msvc.h"
 #endif
 
+/* rust types */
+typedef uint8_t   u8;
+typedef uint16_t  u16;
+typedef uint32_t  u32;
+typedef uint64_t  u64;
+
+typedef int8_t    i8;
+typedef int16_t   i16;
+typedef int32_t   i32;
+typedef int64_t   i64;
+
+typedef float     f32;
+typedef double    f64;
+
+typedef size_t    usize;
+typedef ptrdiff_t isize;
+
 /* used on Mac OS X */
 #ifdef PRECOMPOSE_UNICODE
 #include "compat/precompose_utf8.h"

new file mode 100644
index 0000000000..ed3d79d7f8
@@ -0,0 +1,6 @@
+[workspace]
+members = [
+    "xdiff",
+    "interop",
+]
+resolver = "2"

new file mode 100644
index 0000000000..045e3b01cf
@@ -0,0 +1,14 @@
+[package]
+name = "interop"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+name = "interop"
+path = "src/lib.rs"
+## staticlib to generate xdiff.a for use by gcc
+## cdylib (optional) to generate xdiff.so for use by gcc
+## rlib is required by the rust unit tests
+crate-type = ["staticlib", "rlib"]
+
+[dependencies]

new file mode 100644
index 0000000000..e69de29bb2

new file mode 100644
index 0000000000..eb7966aada
@@ -0,0 +1,15 @@
+[package]
+name = "xdiff"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+name = "xdiff"
+path = "src/lib.rs"
+## staticlib to generate xdiff.a for use by gcc
+## cdylib (optional) to generate xdiff.so for use by gcc
+## rlib is required by the rust unit tests
+crate-type = ["staticlib", "rlib"]
+
+[dependencies]
+interop = { path = "../interop" }

new file mode 100644
index 0000000000..e69de29bb2

gitgitgadget

From: Ezekiel Newren <[email protected]>

Prefer using actions-rs/toolchain@v1 where possible to install rustup, but for docker targets use a script to install rustup. Consolidate the Rust toolchain definitions in main.yaml. Use install-rust-toolchain.sh to ensure the correct toolchain is used. Five overrides are used in main.yaml:

* On Windows: Rust didn't resolve the bcrypt library on Windows correctly until version 1.78.0. Also since rustup mis-identifies the Rust toolchain, the Rust target triple must be set to x86_64-pc-windows-gnu for make (win build), and x86_64-pc-windows-msvc for meson (win+Meson build).
* On musl: libc differences, such as ftruncate64 vs ftruncate, were not accounted for until Rust version 1.72.0. No older version of Rust will work on musl for our needs.
* In a 32-bit docker container running on a 64-bit host, we need to override the Rust target triple. This is because rustup asks the kernel for the bitness of the system and it says 64, even though the container is 32-bit. This also allows us to remove the BITNESS environment variable in ci/lib.sh.

The logic for selecting library names was initially provided in a patch from Johannes, but was reworked and squashed into this commit.

---
 .github/workflows/main.yml   | 60 +++++++++++++++++++++++++++++++++++-
 ci/install-dependencies.sh   | 14 ++++-----
 ci/install-rust-toolchain.sh | 30 ++++++++++++++++++
 ci/install-rustup.sh         | 25 +++++++++++++++
 ci/lib.sh                    |  1 +
 ci/make-test-artifacts.sh    |  9 ++++++
 ci/run-build-and-tests.sh    | 13 ++++++++
 7 files changed, 144 insertions(+), 8 deletions(-)
 create mode 100755 ci/install-rust-toolchain.sh
 create mode 100755 ci/install-rustup.sh

index 7dbf9f7f12..ac1d583ab2 100644
@@ -26,6 +26,13 @@ jobs:
     outputs:
       enabled: ${{ steps.check-ref.outputs.enabled }}${{ steps.skip-if-redundant.outputs.enabled }}
       skip_concurrent: ${{ steps.check-ref.outputs.skip_concurrent }}
+      rust_version_minimum: 1.61.0
+      rust_version_windows: 1.78.0
+      rust_version_musl: 1.72.0
+      ## the rust target is inferred by rustup unless specified
+      rust_target_windows_make: x86_64-pc-windows-gnu
+      rust_target_windows_meson: x86_64-pc-windows-msvc
+      rust_target_32bit_linux: i686-unknown-linux-gnu
     steps:
       - name: try to clone ci-config branch
         run: |

@@ -108,12 +115,26 @@ jobs:
     needs: ci-config
     if: needs.ci-config.outputs.enabled == 'yes'
     runs-on: windows-latest
+    env:
+      CARGO_HOME: "/c/Users/runneradmin/.cargo"
     concurrency:
       group: windows-build-${{ github.ref }}
       cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }}
     steps:
     - uses: actions/checkout@v4
     - uses: git-for-windows/setup-git-for-windows-sdk@v1
+    - name: Install rustup via github actions
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        profile: minimal
+        override: false
+    - name: Install Rust toolchain
+      shell: bash
+      env:
+        RUST_VERSION: ${{ needs.ci-config.outputs.rust_version_windows }}
+        RUST_TARGET: ${{ needs.ci-config.outputs.rust_target_windows_make }}
+      run: ci/install-rust-toolchain.sh
     - name: build
       shell: bash
       env:

@@ -254,12 +275,26 @@ jobs:
     needs: ci-config
     if: needs.ci-config.outputs.enabled == 'yes'
     runs-on: windows-latest
+    env:
+      CARGO_HOME: "/c/Users/runneradmin/.cargo"
     concurrency:
       group: windows-meson-build-${{ github.ref }}
       cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }}
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
+    - name: Install rustup via github actions
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        profile: minimal
+        override: false
+    - name: Install Rust toolchain
+      shell: bash
+      env:
+        RUST_VERSION: ${{ needs.ci-config.outputs.rust_version_windows }}
+        RUST_TARGET: ${{ needs.ci-config.outputs.rust_target_windows_meson }}
+      run: ci/install-rust-toolchain.sh
     - name: Set up dependencies
       shell: pwsh
       run: pip install meson ninja

@@ -329,11 +364,24 @@ jobs:
       jobname: ${{matrix.vector.jobname}}
       CI_JOB_IMAGE: ${{matrix.vector.pool}}
       TEST_OUTPUT_DIRECTORY: ${{github.workspace}}/t
+      CARGO_HOME: "/Users/runner/.cargo"
     runs-on: ${{matrix.vector.pool}}
     steps:
     - uses: actions/checkout@v4
     - run: ci/install-dependencies.sh
-    - run: ci/run-build-and-tests.sh
+    - name: Install rustup via github actions
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        profile: minimal
+        override: false
+    - name: Install Rust toolchain
+      shell: bash
+      env:
+        RUST_VERSION: ${{ needs.ci-config.outputs.rust_version_minimum }}
+      run: ci/install-rust-toolchain.sh
+    - name: Run build and tests
+      run: ci/run-build-and-tests.sh
     - name: print test failures
       if: failure() && env.FAILED_TEST_ARTIFACTS != ''
       run: ci/print-test-failures.sh

@@ -393,9 +441,11 @@ jobs:
           cc: gcc
         - jobname: linux-musl-meson
           image: alpine:latest
+          rust_version_override: ${{ needs.ci-config.outputs.rust_version_musl }}
         # Supported until 2025-04-02.
         - jobname: linux32
           image: i386/ubuntu:focal
+          rust_target_override: ${{ needs.ci-config.outputs.rust_target_32bit_linux }}
         - jobname: pedantic
           image: fedora:latest
         # A RHEL 8 compatible distro.  Supported until 2029-05-31.

@@ -409,6 +459,7 @@ jobs:
       CC: ${{matrix.vector.cc}}
       CI_JOB_IMAGE: ${{matrix.vector.image}}
       CUSTOM_PATH: /custom
+      CARGO_HOME: /home/builder/.cargo
     runs-on: ubuntu-latest
     container: ${{matrix.vector.image}}
     steps:

@@ -433,6 +484,13 @@ jobs:
     - run: ci/install-dependencies.sh
     - run: useradd builder --create-home
     - run: chown -R builder .
+    - name: Install rustup via script
+      run: sudo --preserve-env --set-home --user=builder ci/install-rustup.sh
+    - name: Install Rust toolchain
+      env:
+        RUST_VERSION: ${{ matrix.vector.rust_version_override || needs.ci-config.outputs.rust_version_minimum }}
+        RUST_TARGET: ${{ matrix.vector.rust_target_override || '' }}
+      run: sudo --preserve-env --set-home --user=builder ci/install-rust-toolchain.sh
     - run: sudo --preserve-env --set-home --user=builder ci/run-build-and-tests.sh
     - name: print test failures
       if: failure() && env.FAILED_TEST_ARTIFACTS != ''

index d061a47293..7801075821 100755
@@ -24,14 +24,14 @@ fi
 
 case "$distro" in
 alpine-*)
-	apk add --update shadow sudo meson ninja-build gcc libc-dev curl-dev openssl-dev expat-dev gettext \
+	apk add --update shadow sudo meson ninja-build gcc libc-dev curl curl-dev openssl-dev expat-dev gettext \
 		zlib-ng-dev pcre2-dev python3 musl-libintl perl-utils ncurses \
 		apache2 apache2-http2 apache2-proxy apache2-ssl apache2-webdav apr-util-dbd_sqlite3 \
 		bash cvs gnupg perl-cgi perl-dbd-sqlite perl-io-tty >/dev/null
 	;;
 fedora-*|almalinux-*)
 	dnf -yq update >/dev/null &&
-	dnf -yq install shadow-utils sudo make gcc findutils diffutils perl python3 gawk gettext zlib-devel expat-devel openssl-devel curl-devel pcre2-devel >/dev/null
+	dnf -yq install shadow-utils sudo make gcc findutils diffutils perl python3 gawk gettext zlib-devel expat-devel openssl-devel curl curl-devel pcre2-devel >/dev/null
 	;;
 ubuntu-*|i386/ubuntu-*|debian-*)
 	# Required so that apt doesn't wait for user input on certain packages.

@@ -55,8 +55,8 @@ ubuntu-*|i386/ubuntu-*|debian-*)
 	sudo apt-get -q update
 	sudo apt-get -q -y install \
 		$LANGUAGES apache2 cvs cvsps git gnupg $SVN \
-		make libssl-dev libcurl4-openssl-dev libexpat-dev wget sudo default-jre \
-		tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl \
+		make libssl-dev curl libcurl4-openssl-dev libexpat-dev wget sudo default-jre \
+		tcl tk gettext zlib1g zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl \
 		libemail-valid-perl libio-pty-perl libio-socket-ssl-perl libnet-smtp-ssl-perl libdbd-sqlite3-perl libcgi-pm-perl \
 		libsecret-1-dev libpcre2-dev meson ninja-build pkg-config \
 		${CC_PACKAGE:-${CC:-gcc}} $PYTHON_PACKAGE

@@ -121,13 +121,13 @@ ClangFormat)
 	;;
 StaticAnalysis)
 	sudo apt-get -q update
-	sudo apt-get -q -y install coccinelle libcurl4-openssl-dev libssl-dev \
+	sudo apt-get -q -y install coccinelle curl libcurl4-openssl-dev libssl-dev \
 		libexpat-dev gettext make
 	;;
 sparse)
 	sudo apt-get -q update -q
-	sudo apt-get -q -y install libssl-dev libcurl4-openssl-dev \
-		libexpat-dev gettext zlib1g-dev sparse
+	sudo apt-get -q -y install libssl-dev curl libcurl4-openssl-dev \
+		libexpat-dev gettext zlib1g zlib1g-dev sparse
 	;;
 Documentation)
 	sudo apt-get -q update

new file mode 100755
index 0000000000..06a29c4cfa
@@ -0,0 +1,30 @@
+#!/bin/sh
+
+if [ "$CARGO_HOME" = "" ]; then
+  echo >&2 "::error:: CARGO_HOME is not set"
+  exit 2
+fi
+export PATH="$CARGO_HOME/bin:$PATH"
+rustup -vV || exit $?
+
+## Enforce the correct Rust toolchain
+rustup override unset || true
+
+## install a specific version of rust
+if [ "$RUST_TARGET" != "" ]; then
+  rustup default --force-non-host "$RUST_VERSION-$RUST_TARGET" || exit $?
+else
+  rustup default "$RUST_VERSION" || exit $?
+fi
+
+rustc -vV || exit $?
+
+RE_RUST_TARGET="$RUST_TARGET"
+if [ "$RUST_TARGET" = "" ]; then
+  RE_RUST_TARGET="[^ ]+"
+fi
+
+if ! rustup show active-toolchain | grep -E "^$RUST_VERSION-$RE_RUST_TARGET \(default\)$"; then
+  echo >&2 "::error:: wrong Rust toolchain, active-toolchain: $(rustup show active-toolchain)"
+  exit 3
+fi

new file mode 100755
index 0000000000..0036231aee
@@ -0,0 +1,25 @@
+#!/bin/sh
+
+## github workflows actions-rs/toolchain@v1 doesn't work for docker
+## targets. This script should only be used if the ci pipeline
+## doesn't support installing rust on a particular target.
+
+if [ "$(id -u)" -eq 0 ]; then
+  echo >&2 "::warning:: installing rust as root"
+fi
+
+if [ "$CARGO_HOME" = "" ]; then
+  echo >&2 "::error:: CARGO_HOME is not set"
+  exit 2
+fi
+
+export RUSTUP_HOME=$CARGO_HOME
+
+## install rustup
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain none -y
+if [ ! -f $CARGO_HOME/env ]; then
+  echo "PATH=$CARGO_HOME/bin:\$PATH" > $CARGO_HOME/env
+fi
+. $CARGO_HOME/env
+
+rustup -vV

index f561884d40..a7992b22fd 100755
@@ -1,5 +1,6 @@
 # Library of functions shared by all CI scripts
 
+
 if test true = "$GITHUB_ACTIONS"
 then
 	begin_group () {

index 74141af0cc..e37ed7030c 100755
@@ -7,6 +7,15 @@ mkdir -p "$1" # in case ci/lib.sh decides to quit early
 
 . ${0%/*}/lib.sh
 
+## ensure rustup is in the PATH variable
+if [ "$CARGO_HOME" = "" ]; then
+  echo >&2 "::error:: CARGO_HOME is not set"
+  exit 2
+fi
+export PATH="$CARGO_HOME/bin:$PATH"
+
+rustc -vV
+
 group Build make artifacts-tar ARTIFACTS_DIRECTORY="$1"
 
 check_unignored_build_artifacts

index 01823fd0f1..22b61e2812 100755
@@ -5,6 +5,15 @@
 
 . ${0%/*}/lib.sh
 
+## ensure rustup is in the PATH variable
+if [ "$CARGO_HOME" = "" ]; then
+  echo >&2 "::error:: CARGO_HOME is not set"
+  exit 2
+fi
+. $CARGO_HOME/env
+
+rustc -vV || exit $?
+
 run_tests=t
 
 case "$jobname" in

@@ -72,5 +81,9 @@ case "$jobname" in
 	;;
 esac
 
+if [ -d "$CARGO_HOME" ]; then
+  rm -rf $CARGO_HOME
+fi
+
 check_unignored_build_artifacts
 save_good_tree

gitgitgadget

From: Johannes Schindelin <[email protected]>

When linking against the Rust-built xdiff, there is now a new required
dependency: Without _also_ linking to the system library userenv, the compile would fail with this error message:

xdiff.lib(std-c85e9beb7923f636.std.df32d1bc89881d89-cgu.0.rcgu.o) :
error LNK2019: unresolved external symbol __imp_GetUserProfileDirectoryW referenced in function _ZN3std3env8home_dir17hfd1c3b6676cd78f6E

Therefore, just like we do in case of Makefile-based builds on Windows, we now also link to that library when building with Meson.

Note that if we only have Rust depend upon libuserenv then at link time GCC would complain about:

undefined reference to `GetUserProfileDirectoryW'

Apparently there is _some_ closure that gets compiled in that requires this function, and that in turn forces Git to link to libuserenv.

This is a new requirement, and therefore has not been made part of the "minimal Git for Windows SDK".

In the near future, I intend to include it, but for now let's just ensure that the file is added manually if it is missing.

[en: Squashed a few of Johannes's patches, and moved lib userenv
handling from an earlier patch]

---
 .github/workflows/main.yml | 8 ++++++++
 config.mak.uname           | 4 ++++
 meson.build                | 1 +
 3 files changed, 13 insertions(+)

index ac1d583ab2..ff2f90d122 100644
@@ -123,6 +123,14 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - uses: git-for-windows/setup-git-for-windows-sdk@v1
+    - name: ensure that libuserenv.a is present
+      shell: bash
+      run: |
+        cd /mingw64/lib && {
+          test -f libuserenv.a ||
+          /c/Program\ Files/Git/mingw64/bin/curl -Lo libuserenv.a \
+            https://github.com/git-for-windows/git-sdk-64/raw/HEAD/mingw64/lib/libuserenv.a
+        }
     - name: Install rustup via github actions
       uses: actions-rs/toolchain@v1
       with:

index 3e26bb074a..6805e3778a 100644
@@ -740,6 +740,10 @@ ifeq ($(uname_S),MINGW)
 		COMPAT_CFLAGS += -D_USE_32BIT_TIME_T
 		BASIC_LDFLAGS += -Wl,--large-address-aware
         endif
+
+	# Unfortunately now needed because of Rust
+	EXTLIBS += -luserenv
+
 	CC = gcc
 	COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 -DDETECT_MSYS_TTY \
 		-fstack-protector-strong

index 324f968338..5aa9901bfc 100644
@@ -1267,6 +1267,7 @@ elif host_machine.system() == 'windows'
   ]
 
   libgit_dependencies += compiler.find_library('ntdll')
+  libgit_dependencies += compiler.find_library('userenv')
   libgit_include_directories += 'compat/win32'
   if compiler.get_id() == 'msvc'
     libgit_include_directories += 'compat/vcbuild/include'

gitgitgadget

From: Ezekiel Newren <[email protected]>

Make each ci workflow upload its Cargo.lock file as a build artifact so that we can audit build dependencies.

---
 .github/workflows/main.yml | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

index ff2f90d122..cdd57b4ee1 100644
@@ -156,6 +156,11 @@ jobs:
       with:
         name: windows-artifacts
         path: artifacts
+    - name: upload Cargo.lock
+      uses: actions/upload-artifact@v4
+      with:
+        name: cargo-lock-windows
+        path: rust/Cargo.lock
   windows-test:
     name: win test
     runs-on: windows-latest

@@ -317,6 +322,11 @@ jobs:
       with:
         name: windows-meson-artifacts
         path: build
+    - name: Upload Cargo.lock
+      uses: actions/upload-artifact@v4
+      with:
+        name: cargo-lock-windows-meson
+        path: rust/Cargo.lock
   windows-meson-test:
     name: win+Meson test
     runs-on: windows-latest

@@ -399,6 +409,11 @@ jobs:
       with:
         name: failed-tests-${{matrix.vector.jobname}}
         path: ${{env.FAILED_TEST_ARTIFACTS}}
+    - name: Upload Cargo.lock
+      uses: actions/upload-artifact@v4
+      with:
+        name: cargo-lock-${{matrix.vector.jobname}}
+        path: rust/Cargo.lock
   fuzz-smoke-test:
     name: fuzz smoke test
     needs: ci-config

@@ -509,6 +524,11 @@ jobs:
       with:
         name: failed-tests-${{matrix.vector.jobname}}
         path: ${{env.FAILED_TEST_ARTIFACTS}}
+    - name: Upload Cargo.lock
+      uses: actions/upload-artifact@v4
+      with:
+        name: cargo-lock-${{matrix.vector.jobname}}
+        path: rust/Cargo.lock
   static-analysis:
     needs: ci-config
     if: needs.ci-config.outputs.enabled == 'yes'

gitgitgadget

From: Ezekiel Newren <[email protected]>

Trying to use Rust's Vec in C, or git's ALLOC_GROW() macros (via wrapper functions) in Rust is painful because:

* C doing vector things the Rust way would require wrapper functions, and Rust doing vector things the C way would require wrapper functions, so ivec was created to ensure a consistent contract between the 2 languages for how to manipulate a vector.
* Currently, Rust defines its own 'Vec' type that is generic, but its memory allocator and struct layout weren't designed for interoperability with C (or any language for that matter), meaning that the C side cannot push to or expand a 'Vec' without defining wrapper functions in Rust that C can call. Without special care, the two languages might use different allocators (malloc/free on the C side, and possibly something else in Rust), which would make it difficult for a function in one language to free elements allocated by a call from a function in the other language.
* Similarly, git defines ALLOC_GROW() and related macros in git-compat-util.h. While we could add functions allowing Rust to invoke something similar to those macros, passing three variables (pointer, length, allocated_size) instead of a single variable (vector) across the language boundary requires more cognitive overhead for readers to keep track of and makes it easier to make mistakes. Further, for low-level components that we want to eventually convert to pure Rust, such triplets would feel very out of place.

To address these issue, introduce a new type, ivec -- short for interoperable vector. (We refer to it as 'ivec' generally, though on the Rust side the struct is called IVec to match Rust style.) This new type is specifically designed for FFI purposes, so that both languages handle the vector in the same way, though it could be used on either side independently. This type is designed such that it can easily be replaced by a standard Rust 'Vec' once interoperability is no longer a concern.

One particular item to note is that Git's macros to handle vec operations infer the amount that a vec needs to grow from the size of a pointer, but that makes it somewhat specific to the macros used in C. To avoid defining every ivec function as a macro I opted to also include an element_size field that allows concrete functions like push() to know how much to grow the memory. This element_size also helps in verifying that the ivec is correct when passing from C to Rust.

---
 Makefile                 |  14 +-
 interop/ivec.c           | 180 ++++++++++++++
 interop/ivec.h           |  60 +++++
 meson.build              |   1 +
 rust/interop/src/ivec.rs | 516 +++++++++++++++++++++++++++++++++++++++
 rust/interop/src/lib.rs  |  10 +
 6 files changed, 778 insertions(+), 3 deletions(-)
 create mode 100644 interop/ivec.c
 create mode 100644 interop/ivec.h
 create mode 100644 rust/interop/src/ivec.rs

index 1ec0c1ee66..46e940c96d 100644
@@ -672,6 +672,7 @@ BUILTIN_OBJS =
 BUILT_INS =
 COMPAT_CFLAGS =
 COMPAT_OBJS =
+INTEROP_OBJS =
 XDIFF_OBJS =
 GENERATED_H =
 EXTRA_CPPFLAGS =

@@ -918,6 +919,7 @@ export PYTHON_PATH
 TEST_SHELL_PATH = $(SHELL_PATH)
 
 LIB_FILE = libgit.a
+INTEROP_LIB = interop/lib.a
 XDIFF_LIB = xdiff/lib.a
 
 EXTLIBS =

@@ -1412,7 +1414,7 @@ UNIT_TEST_OBJS += $(UNIT_TEST_DIR)/test-lib.o
 UNIT_TEST_OBJS += $(UNIT_TEST_DIR)/lib-reftable.o
 
 # xdiff and reftable libs may in turn depend on what is in libgit.a
-GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) $(LIB_FILE)
+GITLIBS = common-main.o $(LIB_FILE) $(INTEROP_LIB) $(XDIFF_LIB) $(REFTABLE_LIB) $(LIB_FILE)
 
 
 GIT_USER_AGENT = git/$(GIT_VERSION)

@@ -2747,6 +2749,8 @@ reconfigure config.mak.autogen: config.status
 .PHONY: reconfigure # This is a convenience target.
 endif
 
+INTEROP_OBJS += interop/ivec.o
+
 XDIFF_OBJS += xdiff/xdiffi.o
 XDIFF_OBJS += xdiff/xemit.o
 XDIFF_OBJS += xdiff/xhistogram.o

@@ -2791,6 +2795,7 @@ OBJECTS += $(GIT_OBJS)
 OBJECTS += $(SCALAR_OBJS)
 OBJECTS += $(PROGRAM_OBJS)
 OBJECTS += $(TEST_OBJS)
+OBJECTS += $(INTEROP_OBJS)
 OBJECTS += $(XDIFF_OBJS)
 OBJECTS += $(FUZZ_OBJS)
 OBJECTS += $(REFTABLE_OBJS) $(REFTABLE_TEST_OBJS)

@@ -2945,7 +2950,10 @@ scalar$X: scalar.o GIT-LDFLAGS $(GITLIBS) compile_rust
 $(LIB_FILE): $(LIB_OBJS)
 	$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^
 
-$(XDIFF_LIB): $(XDIFF_OBJS)
+$(INTEROP_LIB): $(INTEROP_OBJS)
+	$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^
+
+$(XDIFF_LIB): $(INTEROP_OBJS) $(XDIFF_OBJS)
 	$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^
 
 

@@ -3790,7 +3798,7 @@ clean: profile-clean coverage-clean cocciclean rustclean
 	$(RM) git.rc git.res
 	$(RM) $(OBJECTS)
 	$(RM) headless-git.o
-	$(RM) $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB)
+	$(RM) $(LIB_FILE) $(INTEROP_LIB) $(XDIFF_LIB) $(REFTABLE_LIB)
 	$(RM) $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) $(OTHER_PROGRAMS)
 	$(RM) $(TEST_PROGRAMS)
 	$(RM) $(FUZZ_PROGRAMS)

index 5aa9901bfc..fc7c133f79 100644
@@ -395,6 +395,7 @@ libgit_sources = [
   'hex-ll.c',
   'hook.c',
   'ident.c',
+  'interop/ivec.c',
   'json-writer.c',
   'kwset.c',
   'levenshtein.c',

index e69de29bb2..4850f66e5b 100644
@@ -0,0 +1,10 @@
+pub mod ivec;
+
+use std::ffi::c_void;
+
+extern "C" {
+    pub fn malloc(size: usize) -> *mut c_void;
+    pub fn calloc(nmemb: usize, size: usize) -> *mut c_void;
+    pub fn realloc(ptr: *mut c_void, size: usize) -> *mut c_void;
+    pub fn free(ptr: *mut c_void);
+}

gitgitgadget

From: Ezekiel Newren <[email protected]>

Move xdl_prepare_env() later in the file to avoid the need for forward declarations.

---
 xdiff/xprepare.c | 116 ++++++++++++++++++++---------------------------
 1 file changed, 50 insertions(+), 66 deletions(-)

index e1d4017b2d..a45c5ee208 100644
@@ -53,21 +53,6 @@ typedef struct s_xdlclassifier {
 
 
 
-static int xdl_init_classifier(xdlclassifier_t *cf, long size, long flags);
-static void xdl_free_classifier(xdlclassifier_t *cf);
-static int xdl_classify_record(unsigned int pass, xdlclassifier_t *cf, xrecord_t **rhash,
-			       unsigned int hbits, xrecord_t *rec);
-static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_t const *xpp,
-			   xdlclassifier_t *cf, xdfile_t *xdf);
-static void xdl_free_ctx(xdfile_t *xdf);
-static int xdl_clean_mmatch(char const *dis, long i, long s, long e);
-static int xdl_cleanup_records(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xdf2);
-static int xdl_trim_ends(xdfile_t *xdf1, xdfile_t *xdf2);
-static int xdl_optimize_ctxs(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xdf2);
-
-
-
-
 static int xdl_init_classifier(xdlclassifier_t *cf, long size, long flags) {
 	cf->flags = flags;
 

@@ -242,57 +227,6 @@ static void xdl_free_ctx(xdfile_t *xdf) {
 }
 
 
-int xdl_prepare_env(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp,
-		    xdfenv_t *xe) {
-	long enl1, enl2, sample;
-	xdlclassifier_t cf;
-
-	memset(&cf, 0, sizeof(cf));
-
-	/*
-	 * For histogram diff, we can afford a smaller sample size and
-	 * thus a poorer estimate of the number of lines, as the hash
-	 * table (rhash) won't be filled up/grown. The number of lines
-	 * (nrecs) will be updated correctly anyway by
-	 * xdl_prepare_ctx().
-	 */
-	sample = (XDF_DIFF_ALG(xpp->flags) == XDF_HISTOGRAM_DIFF
-		  ? XDL_GUESS_NLINES2 : XDL_GUESS_NLINES1);
-
-	enl1 = xdl_guess_lines(mf1, sample) + 1;
-	enl2 = xdl_guess_lines(mf2, sample) + 1;
-
-	if (xdl_init_classifier(&cf, enl1 + enl2 + 1, xpp->flags) < 0)
-		return -1;
-
-	if (xdl_prepare_ctx(1, mf1, enl1, xpp, &cf, &xe->xdf1) < 0) {
-
-		xdl_free_classifier(&cf);
-		return -1;
-	}
-	if (xdl_prepare_ctx(2, mf2, enl2, xpp, &cf, &xe->xdf2) < 0) {
-
-		xdl_free_ctx(&xe->xdf1);
-		xdl_free_classifier(&cf);
-		return -1;
-	}
-
-	if ((XDF_DIFF_ALG(xpp->flags) != XDF_PATIENCE_DIFF) &&
-	    (XDF_DIFF_ALG(xpp->flags) != XDF_HISTOGRAM_DIFF) &&
-	    xdl_optimize_ctxs(&cf, &xe->xdf1, &xe->xdf2) < 0) {
-
-		xdl_free_ctx(&xe->xdf2);
-		xdl_free_ctx(&xe->xdf1);
-		xdl_free_classifier(&cf);
-		return -1;
-	}
-
-	xdl_free_classifier(&cf);
-
-	return 0;
-}
-
-
 void xdl_free_env(xdfenv_t *xe) {
 
 	xdl_free_ctx(&xe->xdf2);

@@ -460,3 +394,53 @@ static int xdl_optimize_ctxs(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xdf2
 
 	return 0;
 }
+
+int xdl_prepare_env(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp,
+		    xdfenv_t *xe) {
+	long enl1, enl2, sample;
+	xdlclassifier_t cf;
+
+	memset(&cf, 0, sizeof(cf));
+
+	/*
+	 * For histogram diff, we can afford a smaller sample size and
+	 * thus a poorer estimate of the number of lines, as the hash
+	 * table (rhash) won't be filled up/grown. The number of lines
+	 * (nrecs) will be updated correctly anyway by
+	 * xdl_prepare_ctx().
+	 */
+	sample = (XDF_DIFF_ALG(xpp->flags) == XDF_HISTOGRAM_DIFF
+		  ? XDL_GUESS_NLINES2 : XDL_GUESS_NLINES1);
+
+	enl1 = xdl_guess_lines(mf1, sample) + 1;
+	enl2 = xdl_guess_lines(mf2, sample) + 1;
+
+	if (xdl_init_classifier(&cf, enl1 + enl2 + 1, xpp->flags) < 0)
+		return -1;
+
+	if (xdl_prepare_ctx(1, mf1, enl1, xpp, &cf, &xe->xdf1) < 0) {
+
+		xdl_free_classifier(&cf);
+		return -1;
+	}
+	if (xdl_prepare_ctx(2, mf2, enl2, xpp, &cf, &xe->xdf2) < 0) {
+
+		xdl_free_ctx(&xe->xdf1);
+		xdl_free_classifier(&cf);
+		return -1;
+	}
+
+	if ((XDF_DIFF_ALG(xpp->flags) != XDF_PATIENCE_DIFF) &&
+	    (XDF_DIFF_ALG(xpp->flags) != XDF_HISTOGRAM_DIFF) &&
+	    xdl_optimize_ctxs(&cf, &xe->xdf1, &xe->xdf2) < 0) {
+
+		xdl_free_ctx(&xe->xdf2);
+		xdl_free_ctx(&xe->xdf1);
+		xdl_free_classifier(&cf);
+		return -1;
+	    }
+
+	xdl_free_classifier(&cf);
+
+	return 0;
+}

gitgitgadget

From: Ezekiel Newren <[email protected]>

xrecord_t.next, xdfile_t.hbits, xdfile_t.rhash are initialized, but never used for anything by the code. Remove them.

---
 xdiff/xprepare.c | 24 +++---------------------
 xdiff/xtypes.h   |  3 ---
 2 files changed, 3 insertions(+), 24 deletions(-)

index a45c5ee208..ad356281f9 100644
@@ -91,8 +91,7 @@ static void xdl_free_classifier(xdlclassifier_t *cf) {
 }
 
 
-static int xdl_classify_record(unsigned int pass, xdlclassifier_t *cf, xrecord_t **rhash,
-			       unsigned int hbits, xrecord_t *rec) {
+static int xdl_classify_record(unsigned int pass, xdlclassifier_t *cf, xrecord_t *rec) {
 	long hi;
 	char const *line;
 	xdlclass_t *rcrec;

@@ -126,23 +125,17 @@ static int xdl_classify_record(unsigned int pass, xdlclassifier_t *cf, xrecord_t
 
 	rec->ha = (unsigned long) rcrec->idx;
 
-	hi = (long) XDL_HASHLONG(rec->ha, hbits);
-	rec->next = rhash[hi];
-	rhash[hi] = rec;
-
 	return 0;
 }
 
 
 static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_t const *xpp,
 			   xdlclassifier_t *cf, xdfile_t *xdf) {
-	unsigned int hbits;
-	long nrec, hsize, bsize;
+	long nrec, bsize;
 	unsigned long hav;
 	char const *blk, *cur, *top, *prev;
 	xrecord_t *crec;
 	xrecord_t **recs;
-	xrecord_t **rhash;
 	unsigned long *ha;
 	char *rchg;
 	long *rindex;

@@ -150,7 +143,6 @@ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_
 	ha = NULL;
 	rindex = NULL;
 	rchg = NULL;
-	rhash = NULL;
 	recs = NULL;
 
 	if (xdl_cha_init(&xdf->rcha, sizeof(xrecord_t), narec / 4 + 1) < 0)

@@ -158,11 +150,6 @@ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_
 	if (!XDL_ALLOC_ARRAY(recs, narec))
 		goto abort;
 
-	hbits = xdl_hashbits((unsigned int) narec);
-	hsize = 1 << hbits;
-	if (!XDL_CALLOC_ARRAY(rhash, hsize))
-		goto abort;
-
 	nrec = 0;
 	if ((cur = blk = xdl_mmfile_first(mf, &bsize))) {
 		for (top = blk + bsize; cur < top; ) {

@@ -176,7 +163,7 @@ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_
 			crec->size = (long) (cur - prev);
 			crec->ha = hav;
 			recs[nrec++] = crec;
-			if (xdl_classify_record(pass, cf, rhash, hbits, crec) < 0)
+			if (xdl_classify_record(pass, cf, crec) < 0)
 				goto abort;
 		}
 	}

@@ -194,8 +181,6 @@ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_
 
 	xdf->nrec = nrec;
 	xdf->recs = recs;
-	xdf->hbits = hbits;
-	xdf->rhash = rhash;
 	xdf->rchg = rchg + 1;
 	xdf->rindex = rindex;
 	xdf->nreff = 0;

@@ -209,7 +194,6 @@ abort:
 	xdl_free(ha);
 	xdl_free(rindex);
 	xdl_free(rchg);
-	xdl_free(rhash);
 	xdl_free(recs);
 	xdl_cha_free(&xdf->rcha);
 	return -1;

@@ -217,8 +201,6 @@ abort:
 
 
 static void xdl_free_ctx(xdfile_t *xdf) {
-
-	xdl_free(xdf->rhash);
 	xdl_free(xdf->rindex);
 	xdl_free(xdf->rchg - 1);
 	xdl_free(xdf->ha);

index 8442bd436e..8b8467360e 100644
@@ -39,7 +39,6 @@ typedef struct s_chastore {
 } chastore_t;
 
 typedef struct s_xrecord {
-	struct s_xrecord *next;
 	char const *ptr;
 	long size;
 	unsigned long ha;

@@ -48,8 +47,6 @@ typedef struct s_xrecord {
 typedef struct s_xdfile {
 	chastore_t rcha;
 	long nrec;
-	unsigned int hbits;
-	xrecord_t **rhash;
 	long dstart, dend;
 	xrecord_t **recs;
 	char *rchg;

gitgitgadget

From: Ezekiel Newren <[email protected]>

A few commits ago, we added definitions for Rust primitive types, to facilitate interoperability between C and Rust. Switch a few variables to use these types. Which, for now, will require adding some casts.

Also change xdlclass_t::ha to be u64 to match xrecord_t::ha, as pointed out by Johannes.

---
 xdiff/xdiffi.c    |  8 ++++----
 xdiff/xemit.c     |  2 +-
 xdiff/xmerge.c    | 14 +++++++-------
 xdiff/xpatience.c |  2 +-
 xdiff/xprepare.c  |  8 ++++----
 xdiff/xtypes.h    |  6 +++---
 xdiff/xutils.c    |  4 ++--
 7 files changed, 22 insertions(+), 22 deletions(-)

index 5a96e36dfb..3b364c61f6 100644
@@ -418,7 +418,7 @@ static int get_indent(xrecord_t *rec)
 	long i;
 	int ret = 0;
 
-	for (i = 0; i < rec->size; i++) {
+	for (i = 0; i < (long) rec->size; i++) {
 		char c = rec->ptr[i];
 
 		if (!XDL_ISSPACE(c))

@@ -1005,11 +1005,11 @@ static void xdl_mark_ignorable_lines(xdchange_t *xscr, xdfenv_t *xe, long flags)
 
 		rec = &xe->xdf1.recs[xch->i1];
 		for (i = 0; i < xch->chg1 && ignore; i++)
-			ignore = xdl_blankline(rec[i]->ptr, rec[i]->size, flags);
+			ignore = xdl_blankline((const char*) rec[i]->ptr, rec[i]->size, flags);
 
 		rec = &xe->xdf2.recs[xch->i2];
 		for (i = 0; i < xch->chg2 && ignore; i++)
-			ignore = xdl_blankline(rec[i]->ptr, rec[i]->size, flags);
+			ignore = xdl_blankline((const char*)rec[i]->ptr, rec[i]->size, flags);
 
 		xch->ignore = ignore;
 	}

@@ -1020,7 +1020,7 @@ static int record_matches_regex(xrecord_t *rec, xpparam_t const *xpp) {
 	size_t i;
 
 	for (i = 0; i < xpp->ignore_regex_nr; i++)
-		if (!regexec_buf(xpp->ignore_regex[i], rec->ptr, rec->size, 1,
+		if (!regexec_buf(xpp->ignore_regex[i], (const char*) rec->ptr, rec->size, 1,
 				 &regmatch, 0))
 			return 1;
 

index 1d40c9cb40..bbf7b7f8c8 100644
@@ -24,7 +24,7 @@
 
 static long xdl_get_rec(xdfile_t *xdf, long ri, char const **rec) {
 
-	*rec = xdf->recs[ri]->ptr;
+	*rec = (char const*) xdf->recs[ri]->ptr;
 
 	return xdf->recs[ri]->size;
 }

index af40c88a5b..6fa6ea61a2 100644
@@ -101,8 +101,8 @@ static int xdl_merge_cmp_lines(xdfenv_t *xe1, int i1, xdfenv_t *xe2, int i2,
 	xrecord_t **rec2 = xe2->xdf2.recs + i2;
 
 	for (i = 0; i < line_count; i++) {
-		int result = xdl_recmatch(rec1[i]->ptr, rec1[i]->size,
-			rec2[i]->ptr, rec2[i]->size, flags);
+		int result = xdl_recmatch((const char*) rec1[i]->ptr, rec1[i]->size,
+			(const char*) rec2[i]->ptr, rec2[i]->size, flags);
 		if (!result)
 			return -1;
 	}

@@ -324,8 +324,8 @@ static int xdl_fill_merge_buffer(xdfenv_t *xe1, const char *name1,
 
 static int recmatch(xrecord_t *rec1, xrecord_t *rec2, unsigned long flags)
 {
-	return xdl_recmatch(rec1->ptr, rec1->size,
-			    rec2->ptr, rec2->size, flags);
+	return xdl_recmatch((char const*) rec1->ptr, rec1->size,
+			    (char const*) rec2->ptr, rec2->size, flags);
 }
 
 /*

@@ -383,10 +383,10 @@ static int xdl_refine_conflicts(xdfenv_t *xe1, xdfenv_t *xe2, xdmerge_t *m,
 		 */
 		t1.ptr = (char *)xe1->xdf2.recs[m->i1]->ptr;
 		t1.size = xe1->xdf2.recs[m->i1 + m->chg1 - 1]->ptr
-			+ xe1->xdf2.recs[m->i1 + m->chg1 - 1]->size - t1.ptr;
+			+ xe1->xdf2.recs[m->i1 + m->chg1 - 1]->size - (u8 const*) t1.ptr;
 		t2.ptr = (char *)xe2->xdf2.recs[m->i2]->ptr;
 		t2.size = xe2->xdf2.recs[m->i2 + m->chg2 - 1]->ptr
-			+ xe2->xdf2.recs[m->i2 + m->chg2 - 1]->size - t2.ptr;
+			+ xe2->xdf2.recs[m->i2 + m->chg2 - 1]->size - (u8 const*) t2.ptr;
 		if (xdl_do_diff(&t1, &t2, xpp, &xe) < 0)
 			return -1;
 		if (xdl_change_compact(&xe.xdf1, &xe.xdf2, xpp->flags) < 0 ||

@@ -440,7 +440,7 @@ static int line_contains_alnum(const char *ptr, long size)
 static int lines_contain_alnum(xdfenv_t *xe, int i, int chg)
 {
 	for (; chg; chg--, i++)
-		if (line_contains_alnum(xe->xdf2.recs[i]->ptr,
+		if (line_contains_alnum((char const*) xe->xdf2.recs[i]->ptr,
 				xe->xdf2.recs[i]->size))
 			return 1;
 	return 0;

index 77dc411d19..986a3a3f74 100644
@@ -121,7 +121,7 @@ static void insert_record(xpparam_t const *xpp, int line, struct hashmap *map,
 		return;
 	map->entries[index].line1 = line;
 	map->entries[index].hash = record->ha;
-	map->entries[index].anchor = is_anchor(xpp, map->env->xdf1.recs[line - 1]->ptr);
+	map->entries[index].anchor = is_anchor(xpp, (const char*) map->env->xdf1.recs[line - 1]->ptr);
 	if (!map->first)
 		map->first = map->entries + index;
 	if (map->last) {

index ad356281f9..00cdf7d8a0 100644
@@ -32,7 +32,7 @@
 
 typedef struct s_xdlclass {
 	struct s_xdlclass *next;
-	unsigned long ha;
+	u64 ha;
 	char const *line;
 	long size;
 	long idx;

@@ -96,12 +96,12 @@ static int xdl_classify_record(unsigned int pass, xdlclassifier_t *cf, xrecord_t
 	char const *line;
 	xdlclass_t *rcrec;
 
-	line = rec->ptr;
+	line = (char const*) rec->ptr;
 	hi = (long) XDL_HASHLONG(rec->ha, cf->hbits);
 	for (rcrec = cf->rchash[hi]; rcrec; rcrec = rcrec->next)
 		if (rcrec->ha == rec->ha &&
 				xdl_recmatch(rcrec->line, rcrec->size,
-					rec->ptr, rec->size, cf->flags))
+					(const char*) rec->ptr, rec->size, cf->flags))
 			break;
 
 	if (!rcrec) {

@@ -159,7 +159,7 @@ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_
 				goto abort;
 			if (!(crec = xdl_cha_alloc(&xdf->rcha)))
 				goto abort;
-			crec->ptr = prev;
+			crec->ptr = (u8 const*) prev;
 			crec->size = (long) (cur - prev);
 			crec->ha = hav;
 			recs[nrec++] = crec;

index 8b8467360e..6e5f67ebf3 100644
@@ -39,9 +39,9 @@ typedef struct s_chastore {
 } chastore_t;
 
 typedef struct s_xrecord {
-	char const *ptr;
-	long size;
-	unsigned long ha;
+	u8 const* ptr;
+	usize size;
+	u64 ha;
 } xrecord_t;
 
 typedef struct s_xdfile {

index 444a108f87..10e4f20b7c 100644
@@ -418,10 +418,10 @@ int xdl_fall_back_diff(xdfenv_t *diff_env, xpparam_t const *xpp,
 
 	subfile1.ptr = (char *)diff_env->xdf1.recs[line1 - 1]->ptr;
 	subfile1.size = diff_env->xdf1.recs[line1 + count1 - 2]->ptr +
-		diff_env->xdf1.recs[line1 + count1 - 2]->size - subfile1.ptr;
+		diff_env->xdf1.recs[line1 + count1 - 2]->size - (u8 const*) subfile1.ptr;
 	subfile2.ptr = (char *)diff_env->xdf2.recs[line2 - 1]->ptr;
 	subfile2.size = diff_env->xdf2.recs[line2 + count2 - 2]->ptr +
-		diff_env->xdf2.recs[line2 + count2 - 2]->size - subfile2.ptr;
+		diff_env->xdf2.recs[line2 + count2 - 2]->size - (u8 const*) subfile2.ptr;
 	if (xdl_do_diff(&subfile1, &subfile2, xpp, &env) < 0)
 		return -1;
 

gitgitgadget

From: Ezekiel Newren <[email protected]>

Simplify xdl_prepare_ctx() by using xdl_free_ctx() instead of using local variables with hand rolled memory management.

---
 xdiff/xprepare.c | 60 +++++++++++++++++++-----------------------------
 1 file changed, 24 insertions(+), 36 deletions(-)

index 00cdf7d8a0..55e1cc3087 100644
@@ -129,86 +129,74 @@ static int xdl_classify_record(unsigned int pass, xdlclassifier_t *cf, xrecord_t
 }
 
 
+static void xdl_free_ctx(xdfile_t *xdf) {
+	xdl_free(xdf->rindex);
+	xdl_free(xdf->rchg - 1);
+	xdl_free(xdf->ha);
+	xdl_free(xdf->recs);
+	xdl_cha_free(&xdf->rcha);
+}
+
+
 static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_t const *xpp,
 			   xdlclassifier_t *cf, xdfile_t *xdf) {
-	long nrec, bsize;
+	long bsize;
 	unsigned long hav;
 	char const *blk, *cur, *top, *prev;
 	xrecord_t *crec;
-	xrecord_t **recs;
-	unsigned long *ha;
-	char *rchg;
-	long *rindex;
 
-	ha = NULL;
-	rindex = NULL;
-	rchg = NULL;
-	recs = NULL;
+	xdf->ha = NULL;
+	xdf->rindex = NULL;
+	xdf->rchg = NULL;
+	xdf->recs = NULL;
+	xdf->nrec = 0;
 
 	if (xdl_cha_init(&xdf->rcha, sizeof(xrecord_t), narec / 4 + 1) < 0)
 		goto abort;
-	if (!XDL_ALLOC_ARRAY(recs, narec))
+	if (!XDL_ALLOC_ARRAY(xdf->recs, narec))
 		goto abort;
 
-	nrec = 0;
 	if ((cur = blk = xdl_mmfile_first(mf, &bsize))) {
 		for (top = blk + bsize; cur < top; ) {
 			prev = cur;
 			hav = xdl_hash_record(&cur, top, xpp->flags);
-			if (XDL_ALLOC_GROW(recs, nrec + 1, narec))
+			if (XDL_ALLOC_GROW(xdf->recs, xdf->nrec + 1, narec))
 				goto abort;
 			if (!(crec = xdl_cha_alloc(&xdf->rcha)))
 				goto abort;
 			crec->ptr = (u8 const*) prev;
 			crec->size = (long) (cur - prev);
 			crec->ha = hav;
-			recs[nrec++] = crec;
+			xdf->recs[xdf->nrec++] = crec;
 			if (xdl_classify_record(pass, cf, crec) < 0)
 				goto abort;
 		}
 	}
 
-	if (!XDL_CALLOC_ARRAY(rchg, nrec + 2))
+	if (!XDL_CALLOC_ARRAY(xdf->rchg, xdf->nrec + 2))
 		goto abort;
 
 	if ((XDF_DIFF_ALG(xpp->flags) != XDF_PATIENCE_DIFF) &&
 	    (XDF_DIFF_ALG(xpp->flags) != XDF_HISTOGRAM_DIFF)) {
-		if (!XDL_ALLOC_ARRAY(rindex, nrec + 1))
+		if (!XDL_ALLOC_ARRAY(xdf->rindex, xdf->nrec + 1))
 			goto abort;
-		if (!XDL_ALLOC_ARRAY(ha, nrec + 1))
+		if (!XDL_ALLOC_ARRAY(xdf->ha, xdf->nrec + 1))
 			goto abort;
 	}
 
-	xdf->nrec = nrec;
-	xdf->recs = recs;
-	xdf->rchg = rchg + 1;
-	xdf->rindex = rindex;
+	xdf->rchg += 1;
 	xdf->nreff = 0;
-	xdf->ha = ha;
 	xdf->dstart = 0;
-	xdf->dend = nrec - 1;
+	xdf->dend = xdf->nrec - 1;
 
 	return 0;
 
 abort:
-	xdl_free(ha);
-	xdl_free(rindex);
-	xdl_free(rchg);
-	xdl_free(recs);
-	xdl_cha_free(&xdf->rcha);
+	xdl_free_ctx(xdf);
 	return -1;
 }
 
 
-static void xdl_free_ctx(xdfile_t *xdf) {
-	xdl_free(xdf->rindex);
-	xdl_free(xdf->rchg - 1);
-	xdl_free(xdf->ha);
-	xdl_free(xdf->recs);
-	xdl_cha_free(&xdf->rcha);
-}
-
-
 void xdl_free_env(xdfenv_t *xe) {
 
 	xdl_free_ctx(&xe->xdf2);

gitgitgadget

From: Ezekiel Newren <[email protected]>

xdfile_t currently uses a chastore which functions as a memory pool and a vector which maps to the allocations created by the chastore. It seems like xrecord_t used to be a linked list until the recs and nrec fields were added. I think that xrecord_t.next was meant to be removed, but was overlooked. This dual data structure setup make the code somewhat confusing.

Additionally the C type chastore_t isn't FFI friendly. While it could be implemented in Rust, since the data structure is confusing anyway, replace it with an ivec whose purpose is to be interoperable. This makes the fields nrec and recs in xdfile_t redundant, which will be removed in the next 2 commits.

---
 xdiff/xprepare.c | 34 +++++++++++++++++-----------------
 xdiff/xtypes.h   |  6 ++++--
 2 files changed, 21 insertions(+), 19 deletions(-)

index 55e1cc3087..3b33186c15 100644
@@ -130,11 +130,11 @@ static int xdl_classify_record(unsigned int pass, xdlclassifier_t *cf, xrecord_t
 
 
 static void xdl_free_ctx(xdfile_t *xdf) {
+	ivec_free(&xdf->record);
 	xdl_free(xdf->rindex);
 	xdl_free(xdf->rchg - 1);
 	xdl_free(xdf->ha);
 	xdl_free(xdf->recs);
-	xdl_cha_free(&xdf->rcha);
 }
 
 

@@ -143,35 +143,35 @@ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_
 	long bsize;
 	unsigned long hav;
 	char const *blk, *cur, *top, *prev;
-	xrecord_t *crec;
 
 	xdf->ha = NULL;
 	xdf->rindex = NULL;
 	xdf->rchg = NULL;
 	xdf->recs = NULL;
 	xdf->nrec = 0;
-
-	if (xdl_cha_init(&xdf->rcha, sizeof(xrecord_t), narec / 4 + 1) < 0)
-		goto abort;
-	if (!XDL_ALLOC_ARRAY(xdf->recs, narec))
-		goto abort;
+	IVEC_INIT(xdf->record);
 
 	if ((cur = blk = xdl_mmfile_first(mf, &bsize))) {
 		for (top = blk + bsize; cur < top; ) {
+			xrecord_t crec;
 			prev = cur;
 			hav = xdl_hash_record(&cur, top, xpp->flags);
-			if (XDL_ALLOC_GROW(xdf->recs, xdf->nrec + 1, narec))
-				goto abort;
-			if (!(crec = xdl_cha_alloc(&xdf->rcha)))
-				goto abort;
-			crec->ptr = (u8 const*) prev;
-			crec->size = (long) (cur - prev);
-			crec->ha = hav;
-			xdf->recs[xdf->nrec++] = crec;
-			if (xdl_classify_record(pass, cf, crec) < 0)
-				goto abort;
+			crec.ptr = (u8 const*) prev;
+			crec.size = cur - prev;
+			crec.ha = hav;
+			ivec_push(&xdf->record, &crec);
 		}
 	}
+	ivec_shrink_to_fit(&xdf->record);
+
+	xdf->nrec = (long) xdf->record.length;
+	if (!XDL_ALLOC_ARRAY(xdf->recs, xdf->record.length))
+		goto abort;
+	for (usize i = 0; i < xdf->record.length; i++) {
+		if (xdl_classify_record(pass, cf, &xdf->record.ptr[i]) < 0)
+			goto abort;
+		xdf->recs[i] = &xdf->record.ptr[i];
+	}
 
 	if (!XDL_CALLOC_ARRAY(xdf->rchg, xdf->nrec + 2))
 		goto abort;

index 6e5f67ebf3..5028a70b26 100644
@@ -23,7 +23,7 @@
 #if !defined(XTYPES_H)
 #define XTYPES_H
 
-
+#include "../interop/ivec.h"
 
 typedef struct s_chanode {
 	struct s_chanode *next;

@@ -44,8 +44,10 @@ typedef struct s_xrecord {
 	u64 ha;
 } xrecord_t;
 
+DEFINE_IVEC_TYPE(xrecord_t, xrecord);
+
 typedef struct s_xdfile {
-	chastore_t rcha;
+	struct ivec_xrecord record;
 	long nrec;
 	long dstart, dend;
 	xrecord_t **recs;

gitgitgadget

From: Ezekiel Newren <[email protected]>

Because of the data structure cleanup in the previous commit, the nrec field is no longer necessary. Use record.length in place of nrec.

This commit is best viewed with --color-words.

---
 xdiff/xdiffi.c    | 10 +++++-----
 xdiff/xemit.c     | 20 ++++++++++----------
 xdiff/xmerge.c    | 10 +++++-----
 xdiff/xpatience.c |  2 +-
 xdiff/xprepare.c  | 34 ++++++++++++++++------------------
 xdiff/xtypes.h    |  1 -
 6 files changed, 37 insertions(+), 40 deletions(-)

index 3b364c61f6..bcab2d7ae5 100644
@@ -496,7 +496,7 @@ static void measure_split(const xdfile_t *xdf, long split,
 {
 	long i;
 
-	if (split >= xdf->nrec) {
+	if (split >= (long) xdf->record.length) {
 		m->end_of_file = 1;
 		m->indent = -1;
 	} else {

@@ -519,7 +519,7 @@ static void measure_split(const xdfile_t *xdf, long split,
 
 	m->post_blank = 0;
 	m->post_indent = -1;
-	for (i = split + 1; i < xdf->nrec; i++) {
+	for (i = split + 1; i < (long) xdf->record.length; i++) {
 		m->post_indent = get_indent(xdf->recs[i]);
 		if (m->post_indent != -1)
 			break;

@@ -730,7 +730,7 @@ static void group_init(xdfile_t *xdf, struct xdlgroup *g)
  */
 static inline int group_next(xdfile_t *xdf, struct xdlgroup *g)
 {
-	if (g->end == xdf->nrec)
+	if (g->end == (long) xdf->record.length)
 		return -1;
 
 	g->start = g->end + 1;

@@ -763,7 +763,7 @@ static inline int group_previous(xdfile_t *xdf, struct xdlgroup *g)
  */
 static int group_slide_down(xdfile_t *xdf, struct xdlgroup *g)
 {
-	if (g->end < xdf->nrec &&
+	if (g->end < (long) xdf->record.length &&
 	    recs_match(xdf->recs[g->start], xdf->recs[g->end])) {
 		xdf->rchg[g->start++] = 0;
 		xdf->rchg[g->end++] = 1;

@@ -950,7 +950,7 @@ int xdl_build_script(xdfenv_t *xe, xdchange_t **xscr) {
 	/*
 	 * Trivial. Collects "groups" of changes and creates an edit script.
 	 */
-	for (i1 = xe->xdf1.nrec, i2 = xe->xdf2.nrec; i1 >= 0 || i2 >= 0; i1--, i2--)
+	for (i1 = xe->xdf1.record.length, i2 = xe->xdf2.record.length; i1 >= 0 || i2 >= 0; i1--, i2--)
 		if (rchg1[i1 - 1] || rchg2[i2 - 1]) {
 			for (l1 = i1; rchg1[i1 - 1]; i1--);
 			for (l2 = i2; rchg2[i2 - 1]; i2--);

index bbf7b7f8c8..11c2823eca 100644
@@ -147,7 +147,7 @@ static long get_func_line(xdfenv_t *xe, xdemitconf_t const *xecfg,
 	buf = func_line ? func_line->buf : dummy;
 	size = func_line ? sizeof(func_line->buf) : sizeof(dummy);
 
-	for (l = start; l != limit && 0 <= l && l < xe->xdf1.nrec; l += step) {
+	for (l = start; l != limit && 0 <= l && l < (long) xe->xdf1.record.length; l += step) {
 		long len = match_func_rec(&xe->xdf1, xecfg, l, buf, size);
 		if (len >= 0) {
 			if (func_line)

@@ -191,14 +191,14 @@ pre_context_calculation:
 			long fs1, i1 = xch->i1;
 
 			/* Appended chunk? */
-			if (i1 >= xe->xdf1.nrec) {
+			if (i1 >= (long) xe->xdf1.record.length) {
 				long i2 = xch->i2;
 
 				/*
 				 * We don't need additional context if
 				 * a whole function was added.
 				 */
-				while (i2 < xe->xdf2.nrec) {
+				while (i2 < (long) xe->xdf2.record.length) {
 					if (is_func_rec(&xe->xdf2, xecfg, i2))
 						goto post_context_calculation;
 					i2++;

@@ -208,7 +208,7 @@ pre_context_calculation:
 				 * Otherwise get more context from the
 				 * pre-image.
 				 */
-				i1 = xe->xdf1.nrec - 1;
+				i1 = xe->xdf1.record.length - 1;
 			}
 
 			fs1 = get_func_line(xe, xecfg, NULL, i1, -1);

@@ -240,8 +240,8 @@ pre_context_calculation:
 
  post_context_calculation:
 		lctx = xecfg->ctxlen;
-		lctx = XDL_MIN(lctx, xe->xdf1.nrec - (xche->i1 + xche->chg1));
-		lctx = XDL_MIN(lctx, xe->xdf2.nrec - (xche->i2 + xche->chg2));
+		lctx = XDL_MIN(lctx, (long) xe->xdf1.record.length - (xche->i1 + xche->chg1));
+		lctx = XDL_MIN(lctx, (long) xe->xdf2.record.length - (xche->i2 + xche->chg2));
 
 		e1 = xche->i1 + xche->chg1 + lctx;
 		e2 = xche->i2 + xche->chg2 + lctx;

@@ -249,13 +249,13 @@ pre_context_calculation:
 		if (xecfg->flags & XDL_EMIT_FUNCCONTEXT) {
 			long fe1 = get_func_line(xe, xecfg, NULL,
 						 xche->i1 + xche->chg1,
-						 xe->xdf1.nrec);
+						 xe->xdf1.record.length);
 			while (fe1 > 0 && is_empty_rec(&xe->xdf1, fe1 - 1))
 				fe1--;
 			if (fe1 < 0)
-				fe1 = xe->xdf1.nrec;
+				fe1 = xe->xdf1.record.length;
 			if (fe1 > e1) {
-				e2 = XDL_MIN(e2 + (fe1 - e1), xe->xdf2.nrec);
+				e2 = XDL_MIN(e2 + (fe1 - e1), (long) xe->xdf2.record.length);
 				e1 = fe1;
 			}
 

@@ -266,7 +266,7 @@ pre_context_calculation:
 			 */
 			if (xche->next) {
 				long l = XDL_MIN(xche->next->i1,
-						 xe->xdf1.nrec - 1);
+						 (long) xe->xdf1.record.length - 1);
 				if (l - xecfg->ctxlen <= e1 ||
 				    get_func_line(xe, xecfg, NULL, l, e1) < 0) {
 					xche = xche->next;

index 6fa6ea61a2..f48549605d 100644
@@ -158,11 +158,11 @@ static int is_eol_crlf(xdfile_t *file, int i)
 {
 	long size;
 
-	if (i < file->nrec - 1)
+	if (i < (isize) file->record.length - 1)
 		/* All lines before the last *must* end in LF */
 		return (size = file->recs[i]->size) > 1 &&
 			file->recs[i]->ptr[size - 2] == '\r';
-	if (!file->nrec)
+	if (!file->record.length)
 		/* Cannot determine eol style from empty file */
 		return -1;
 	if ((size = file->recs[i]->size) &&

@@ -317,7 +317,7 @@ static int xdl_fill_merge_buffer(xdfenv_t *xe1, const char *name1,
 			continue;
 		i = m->i1 + m->chg1;
 	}
-	size += xdl_recs_copy(xe1, i, xe1->xdf2.nrec - i, 0, 0,
+	size += xdl_recs_copy(xe1, i, (int) xe1->xdf2.record.length - i, 0, 0,
 			      dest ? dest + size : NULL);
 	return size;
 }

@@ -622,7 +622,7 @@ static int xdl_do_merge(xdfenv_t *xe1, xdchange_t *xscr1,
 			changes = c;
 		i0 = xscr1->i1;
 		i1 = xscr1->i2;
-		i2 = xscr1->i1 + xe2->xdf2.nrec - xe2->xdf1.nrec;
+		i2 = xscr1->i1 + xe2->xdf2.record.length - xe2->xdf1.record.length;
 		chg0 = xscr1->chg1;
 		chg1 = xscr1->chg2;
 		chg2 = xscr1->chg1;

@@ -637,7 +637,7 @@ static int xdl_do_merge(xdfenv_t *xe1, xdchange_t *xscr1,
 		if (!changes)
 			changes = c;
 		i0 = xscr2->i1;
-		i1 = xscr2->i1 + xe1->xdf2.nrec - xe1->xdf1.nrec;
+		i1 = xscr2->i1 + xe1->xdf2.record.length - xe1->xdf1.record.length;
 		i2 = xscr2->i2;
 		chg0 = xscr2->chg1;
 		chg1 = xscr2->chg1;

index 986a3a3f74..e1ce9a399f 100644
@@ -370,5 +370,5 @@ static int patience_diff(xpparam_t const *xpp, xdfenv_t *env,
 
 int xdl_do_patience_diff(xpparam_t const *xpp, xdfenv_t *env)
 {
-	return patience_diff(xpp, env, 1, env->xdf1.nrec, 1, env->xdf2.nrec);
+	return patience_diff(xpp, env, 1, env->xdf1.record.length, 1, env->xdf2.record.length);
 }

index 3b33186c15..9b46523afe 100644
@@ -138,7 +138,7 @@ static void xdl_free_ctx(xdfile_t *xdf) {
 }
 
 
-static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_t const *xpp,
+static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, xpparam_t const *xpp,
 			   xdlclassifier_t *cf, xdfile_t *xdf) {
 	long bsize;
 	unsigned long hav;

@@ -148,7 +148,6 @@ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_
 	xdf->rindex = NULL;
 	xdf->rchg = NULL;
 	xdf->recs = NULL;
-	xdf->nrec = 0;
 	IVEC_INIT(xdf->record);
 
 	if ((cur = blk = xdl_mmfile_first(mf, &bsize))) {

@@ -164,7 +163,6 @@ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_
 	}
 	ivec_shrink_to_fit(&xdf->record);
 
-	xdf->nrec = (long) xdf->record.length;
 	if (!XDL_ALLOC_ARRAY(xdf->recs, xdf->record.length))
 		goto abort;
 	for (usize i = 0; i < xdf->record.length; i++) {

@@ -173,21 +171,21 @@ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_
 		xdf->recs[i] = &xdf->record.ptr[i];
 	}
 
-	if (!XDL_CALLOC_ARRAY(xdf->rchg, xdf->nrec + 2))
+	if (!XDL_CALLOC_ARRAY(xdf->rchg, xdf->record.length + 2))
 		goto abort;
 
 	if ((XDF_DIFF_ALG(xpp->flags) != XDF_PATIENCE_DIFF) &&
 	    (XDF_DIFF_ALG(xpp->flags) != XDF_HISTOGRAM_DIFF)) {
-		if (!XDL_ALLOC_ARRAY(xdf->rindex, xdf->nrec + 1))
+		if (!XDL_ALLOC_ARRAY(xdf->rindex, xdf->record.length + 1))
 			goto abort;
-		if (!XDL_ALLOC_ARRAY(xdf->ha, xdf->nrec + 1))
+		if (!XDL_ALLOC_ARRAY(xdf->ha, xdf->record.length + 1))
 			goto abort;
 	}
 
 	xdf->rchg += 1;
 	xdf->nreff = 0;
 	xdf->dstart = 0;
-	xdf->dend = xdf->nrec - 1;
+	xdf->dend = xdf->record.length - 1;
 
 	return 0;
 

@@ -274,12 +272,12 @@ static int xdl_cleanup_records(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xd
 	char *dis, *dis1, *dis2;
 	int need_min = !!(cf->flags & XDF_NEED_MINIMAL);
 
-	if (!XDL_CALLOC_ARRAY(dis, xdf1->nrec + xdf2->nrec + 2))
+	if (!XDL_CALLOC_ARRAY(dis, xdf1->record.length + xdf2->record.length + 2))
 		return -1;
 	dis1 = dis;
-	dis2 = dis1 + xdf1->nrec + 1;
+	dis2 = dis1 + xdf1->record.length + 1;
 
-	if ((mlim = xdl_bogosqrt(xdf1->nrec)) > XDL_MAX_EQLIMIT)
+	if ((mlim = xdl_bogosqrt(xdf1->record.length)) > XDL_MAX_EQLIMIT)
 		mlim = XDL_MAX_EQLIMIT;
 	for (i = xdf1->dstart, recs = &xdf1->recs[xdf1->dstart]; i <= xdf1->dend; i++, recs++) {
 		rcrec = cf->rcrecs[(*recs)->ha];

@@ -287,7 +285,7 @@ static int xdl_cleanup_records(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xd
 		dis1[i] = (nm == 0) ? 0: (nm >= mlim && !need_min) ? 2: 1;
 	}
 
-	if ((mlim = xdl_bogosqrt(xdf2->nrec)) > XDL_MAX_EQLIMIT)
+	if ((mlim = xdl_bogosqrt(xdf2->record.length)) > XDL_MAX_EQLIMIT)
 		mlim = XDL_MAX_EQLIMIT;
 	for (i = xdf2->dstart, recs = &xdf2->recs[xdf2->dstart]; i <= xdf2->dend; i++, recs++) {
 		rcrec = cf->rcrecs[(*recs)->ha];

@@ -334,21 +332,21 @@ static int xdl_trim_ends(xdfile_t *xdf1, xdfile_t *xdf2) {
 
 	recs1 = xdf1->recs;
 	recs2 = xdf2->recs;
-	for (i = 0, lim = XDL_MIN(xdf1->nrec, xdf2->nrec); i < lim;
+	for (i = 0, lim = XDL_MIN(xdf1->record.length, xdf2->record.length); i < lim;
 	     i++, recs1++, recs2++)
 		if ((*recs1)->ha != (*recs2)->ha)
 			break;
 
 	xdf1->dstart = xdf2->dstart = i;
 
-	recs1 = xdf1->recs + xdf1->nrec - 1;
-	recs2 = xdf2->recs + xdf2->nrec - 1;
+	recs1 = xdf1->recs + xdf1->record.length - 1;
+	recs2 = xdf2->recs + xdf2->record.length - 1;
 	for (lim -= i, i = 0; i < lim; i++, recs1--, recs2--)
 		if ((*recs1)->ha != (*recs2)->ha)
 			break;
 
-	xdf1->dend = xdf1->nrec - i - 1;
-	xdf2->dend = xdf2->nrec - i - 1;
+	xdf1->dend = xdf1->record.length - i - 1;
+	xdf2->dend = xdf2->record.length - i - 1;
 
 	return 0;
 }

@@ -388,12 +386,12 @@ int xdl_prepare_env(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp,
 	if (xdl_init_classifier(&cf, enl1 + enl2 + 1, xpp->flags) < 0)
 		return -1;
 
-	if (xdl_prepare_ctx(1, mf1, enl1, xpp, &cf, &xe->xdf1) < 0) {
+	if (xdl_prepare_ctx(1, mf1, xpp, &cf, &xe->xdf1) < 0) {
 
 		xdl_free_classifier(&cf);
 		return -1;
 	}
-	if (xdl_prepare_ctx(2, mf2, enl2, xpp, &cf, &xe->xdf2) < 0) {
+	if (xdl_prepare_ctx(2, mf2, xpp, &cf, &xe->xdf2) < 0) {
 
 		xdl_free_ctx(&xe->xdf1);
 		xdl_free_classifier(&cf);

index 5028a70b26..c322e62fbf 100644
@@ -48,7 +48,6 @@ DEFINE_IVEC_TYPE(xrecord_t, xrecord);
 
 typedef struct s_xdfile {
 	struct ivec_xrecord record;
-	long nrec;
 	long dstart, dend;
 	xrecord_t **recs;
 	char *rchg;

gitgitgadget

From: Ezekiel Newren <[email protected]>

Because of the change from chastore to ivec a few commits ago, recs now points to record's elements in a 1:1 mapping. Since both recs and record are vectors, this additional mapping is superfluous. Remove recs.

This commit is best viewed with --color-words.

---
 xdiff/xdiffi.c     | 30 ++++++++++++------------
 xdiff/xemit.c      |  4 ++--
 xdiff/xhistogram.c |  2 +-
 xdiff/xmerge.c     | 58 +++++++++++++++++++++++-----------------------
 xdiff/xpatience.c  | 14 +++++------
 xdiff/xprepare.c   | 37 +++++++++++++----------------
 xdiff/xtypes.h     |  1 -
 xdiff/xutils.c     | 12 +++++-----
 8 files changed, 76 insertions(+), 82 deletions(-)

index bcab2d7ae5..ebdb724322 100644
@@ -501,13 +501,13 @@ static void measure_split(const xdfile_t *xdf, long split,
 		m->indent = -1;
 	} else {
 		m->end_of_file = 0;
-		m->indent = get_indent(xdf->recs[split]);
+		m->indent = get_indent(&xdf->record.ptr[split]);
 	}
 
 	m->pre_blank = 0;
 	m->pre_indent = -1;
 	for (i = split - 1; i >= 0; i--) {
-		m->pre_indent = get_indent(xdf->recs[i]);
+		m->pre_indent = get_indent(&xdf->record.ptr[i]);
 		if (m->pre_indent != -1)
 			break;
 		m->pre_blank += 1;

@@ -520,7 +520,7 @@ static void measure_split(const xdfile_t *xdf, long split,
 	m->post_blank = 0;
 	m->post_indent = -1;
 	for (i = split + 1; i < (long) xdf->record.length; i++) {
-		m->post_indent = get_indent(xdf->recs[i]);
+		m->post_indent = get_indent(&xdf->record.ptr[i]);
 		if (m->post_indent != -1)
 			break;
 		m->post_blank += 1;

@@ -764,7 +764,7 @@ static inline int group_previous(xdfile_t *xdf, struct xdlgroup *g)
 static int group_slide_down(xdfile_t *xdf, struct xdlgroup *g)
 {
 	if (g->end < (long) xdf->record.length &&
-	    recs_match(xdf->recs[g->start], xdf->recs[g->end])) {
+	    recs_match(&xdf->record.ptr[g->start], &xdf->record.ptr[g->end])) {
 		xdf->rchg[g->start++] = 0;
 		xdf->rchg[g->end++] = 1;
 

@@ -785,7 +785,7 @@ static int group_slide_down(xdfile_t *xdf, struct xdlgroup *g)
 static int group_slide_up(xdfile_t *xdf, struct xdlgroup *g)
 {
 	if (g->start > 0 &&
-	    recs_match(xdf->recs[g->start - 1], xdf->recs[g->end - 1])) {
+	    recs_match(&xdf->record.ptr[g->start - 1], &xdf->record.ptr[g->end - 1])) {
 		xdf->rchg[--g->start] = 1;
 		xdf->rchg[--g->end] = 0;
 

@@ -1000,16 +1000,16 @@ static void xdl_mark_ignorable_lines(xdchange_t *xscr, xdfenv_t *xe, long flags)
 
 	for (xch = xscr; xch; xch = xch->next) {
 		int ignore = 1;
-		xrecord_t **rec;
+		xrecord_t *rec;
 		long i;
 
-		rec = &xe->xdf1.recs[xch->i1];
+		rec = &xe->xdf1.record.ptr[xch->i1];
 		for (i = 0; i < xch->chg1 && ignore; i++)
-			ignore = xdl_blankline((const char*) rec[i]->ptr, rec[i]->size, flags);
+			ignore = xdl_blankline((const char*) rec[i].ptr, rec[i].size, flags);
 
-		rec = &xe->xdf2.recs[xch->i2];
+		rec = &xe->xdf2.record.ptr[xch->i2];
 		for (i = 0; i < xch->chg2 && ignore; i++)
-			ignore = xdl_blankline((const char*)rec[i]->ptr, rec[i]->size, flags);
+			ignore = xdl_blankline((const char*)rec[i].ptr, rec[i].size, flags);
 
 		xch->ignore = ignore;
 	}

@@ -1033,7 +1033,7 @@ static void xdl_mark_ignorable_regex(xdchange_t *xscr, const xdfenv_t *xe,
 	xdchange_t *xch;
 
 	for (xch = xscr; xch; xch = xch->next) {
-		xrecord_t **rec;
+		xrecord_t *rec;
 		int ignore = 1;
 		long i;
 

@@ -1043,13 +1043,13 @@ static void xdl_mark_ignorable_regex(xdchange_t *xscr, const xdfenv_t *xe,
 		if (xch->ignore)
 			continue;
 
-		rec = &xe->xdf1.recs[xch->i1];
+		rec = &xe->xdf1.record.ptr[xch->i1];
 		for (i = 0; i < xch->chg1 && ignore; i++)
-			ignore = record_matches_regex(rec[i], xpp);
+			ignore = record_matches_regex(&rec[i], xpp);
 
-		rec = &xe->xdf2.recs[xch->i2];
+		rec = &xe->xdf2.record.ptr[xch->i2];
 		for (i = 0; i < xch->chg2 && ignore; i++)
-			ignore = record_matches_regex(rec[i], xpp);
+			ignore = record_matches_regex(&rec[i], xpp);
 
 		xch->ignore = ignore;
 	}

index 11c2823eca..0c9a12a5e8 100644
@@ -24,9 +24,9 @@
 
 static long xdl_get_rec(xdfile_t *xdf, long ri, char const **rec) {
 
-	*rec = (char const*) xdf->recs[ri]->ptr;
+	*rec = (char const*) xdf->record.ptr[ri].ptr;
 
-	return xdf->recs[ri]->size;
+	return xdf->record.ptr[ri].size;
 }
 
 

index 040d81e0bc..643d1c8b70 100644
@@ -86,7 +86,7 @@ struct region {
 	((LINE_MAP(index, ptr))->cnt)
 
 #define REC(env, s, l) \
-	(env->xdf##s.recs[l - 1])
+	(&env->xdf##s.record.ptr[l - 1])
 
 static int cmp_recs(xrecord_t *r1, xrecord_t *r2)
 {

index e1ce9a399f..31b819ec58 100644
@@ -88,9 +88,9 @@ static int is_anchor(xpparam_t const *xpp, const char *line)
 static void insert_record(xpparam_t const *xpp, int line, struct hashmap *map,
 			  int pass)
 {
-	xrecord_t **records = pass == 1 ?
-		map->env->xdf1.recs : map->env->xdf2.recs;
-	xrecord_t *record = records[line - 1];
+	xrecord_t *records = pass == 1 ?
+		map->env->xdf1.record.ptr : map->env->xdf2.record.ptr;
+	xrecord_t *record = &records[line - 1];
 	/*
 	 * After xdl_prepare_env() (or more precisely, due to
 	 * xdl_classify_record()), the "ha" member of the records (AKA lines)

@@ -121,7 +121,7 @@ static void insert_record(xpparam_t const *xpp, int line, struct hashmap *map,
 		return;
 	map->entries[index].line1 = line;
 	map->entries[index].hash = record->ha;
-	map->entries[index].anchor = is_anchor(xpp, (const char*) map->env->xdf1.recs[line - 1]->ptr);
+	map->entries[index].anchor = is_anchor(xpp, (const char*) map->env->xdf1.record.ptr[line - 1].ptr);
 	if (!map->first)
 		map->first = map->entries + index;
 	if (map->last) {

@@ -246,9 +246,9 @@ static int find_longest_common_sequence(struct hashmap *map, struct entry **res)
 
 static int match(struct hashmap *map, int line1, int line2)
 {
-	xrecord_t *record1 = map->env->xdf1.recs[line1 - 1];
-	xrecord_t *record2 = map->env->xdf2.recs[line2 - 1];
-	return record1->ha == record2->ha;
+	u64 mph1 = map->env->xdf1.record.ptr[line1 - 1].ha;
+	u64 mph2 = map->env->xdf2.record.ptr[line2 - 1].ha;
+	return mph1 == mph2;
 }
 
 static int patience_diff(xpparam_t const *xpp, xdfenv_t *env,

index 9b46523afe..93370f1c6d 100644
@@ -134,7 +134,6 @@ static void xdl_free_ctx(xdfile_t *xdf) {
 	xdl_free(xdf->rindex);
 	xdl_free(xdf->rchg - 1);
 	xdl_free(xdf->ha);
-	xdl_free(xdf->recs);
 }
 
 

@@ -147,7 +146,6 @@ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, xpparam_t const *xpp
 	xdf->ha = NULL;
 	xdf->rindex = NULL;
 	xdf->rchg = NULL;
-	xdf->recs = NULL;
 	IVEC_INIT(xdf->record);
 
 	if ((cur = blk = xdl_mmfile_first(mf, &bsize))) {

@@ -163,12 +161,9 @@ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, xpparam_t const *xpp
 	}
 	ivec_shrink_to_fit(&xdf->record);
 
-	if (!XDL_ALLOC_ARRAY(xdf->recs, xdf->record.length))
-		goto abort;
 	for (usize i = 0; i < xdf->record.length; i++) {
 		if (xdl_classify_record(pass, cf, &xdf->record.ptr[i]) < 0)
 			goto abort;
-		xdf->recs[i] = &xdf->record.ptr[i];
 	}
 
 	if (!XDL_CALLOC_ARRAY(xdf->rchg, xdf->record.length + 2))

@@ -267,7 +262,7 @@ static int xdl_clean_mmatch(char const *dis, long i, long s, long e) {
  */
 static int xdl_cleanup_records(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xdf2) {
 	long i, nm, nreff, mlim;
-	xrecord_t **recs;
+	xrecord_t *recs;
 	xdlclass_t *rcrec;
 	char *dis, *dis1, *dis2;
 	int need_min = !!(cf->flags & XDF_NEED_MINIMAL);

@@ -279,38 +274,38 @@ static int xdl_cleanup_records(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xd
 
 	if ((mlim = xdl_bogosqrt(xdf1->record.length)) > XDL_MAX_EQLIMIT)
 		mlim = XDL_MAX_EQLIMIT;
-	for (i = xdf1->dstart, recs = &xdf1->recs[xdf1->dstart]; i <= xdf1->dend; i++, recs++) {
-		rcrec = cf->rcrecs[(*recs)->ha];
+	for (i = xdf1->dstart, recs = &xdf1->record.ptr[xdf1->dstart]; i <= xdf1->dend; i++, recs++) {
+		rcrec = cf->rcrecs[recs->ha];
 		nm = rcrec ? rcrec->len2 : 0;
 		dis1[i] = (nm == 0) ? 0: (nm >= mlim && !need_min) ? 2: 1;
 	}
 
 	if ((mlim = xdl_bogosqrt(xdf2->record.length)) > XDL_MAX_EQLIMIT)
 		mlim = XDL_MAX_EQLIMIT;
-	for (i = xdf2->dstart, recs = &xdf2->recs[xdf2->dstart]; i <= xdf2->dend; i++, recs++) {
-		rcrec = cf->rcrecs[(*recs)->ha];
+	for (i = xdf2->dstart, recs = &xdf2->record.ptr[xdf2->dstart]; i <= xdf2->dend; i++, recs++) {
+		rcrec = cf->rcrecs[recs->ha];
 		nm = rcrec ? rcrec->len1 : 0;
 		dis2[i] = (nm == 0) ? 0: (nm >= mlim && !need_min) ? 2: 1;
 	}
 
-	for (nreff = 0, i = xdf1->dstart, recs = &xdf1->recs[xdf1->dstart];
+	for (nreff = 0, i = xdf1->dstart, recs = &xdf1->record.ptr[xdf1->dstart];
 	     i <= xdf1->dend; i++, recs++) {
 		if (dis1[i] == 1 ||
 		    (dis1[i] == 2 && !xdl_clean_mmatch(dis1, i, xdf1->dstart, xdf1->dend))) {
 			xdf1->rindex[nreff] = i;
-			xdf1->ha[nreff] = (*recs)->ha;
+			xdf1->ha[nreff] = recs->ha;
 			nreff++;
 		} else
 			xdf1->rchg[i] = 1;
 	}
 	xdf1->nreff = nreff;
 
-	for (nreff = 0, i = xdf2->dstart, recs = &xdf2->recs[xdf2->dstart];
+	for (nreff = 0, i = xdf2->dstart, recs = &xdf2->record.ptr[xdf2->dstart];
 	     i <= xdf2->dend; i++, recs++) {
 		if (dis2[i] == 1 ||
 		    (dis2[i] == 2 && !xdl_clean_mmatch(dis2, i, xdf2->dstart, xdf2->dend))) {
 			xdf2->rindex[nreff] = i;
-			xdf2->ha[nreff] = (*recs)->ha;
+			xdf2->ha[nreff] = recs->ha;
 			nreff++;
 		} else
 			xdf2->rchg[i] = 1;

@@ -328,21 +323,21 @@ static int xdl_cleanup_records(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xd
  */
 static int xdl_trim_ends(xdfile_t *xdf1, xdfile_t *xdf2) {
 	long i, lim;
-	xrecord_t **recs1, **recs2;
+	xrecord_t *recs1, *recs2;
 
-	recs1 = xdf1->recs;
-	recs2 = xdf2->recs;
+	recs1 = xdf1->record.ptr;
+	recs2 = xdf2->record.ptr;
 	for (i = 0, lim = XDL_MIN(xdf1->record.length, xdf2->record.length); i < lim;
 	     i++, recs1++, recs2++)
-		if ((*recs1)->ha != (*recs2)->ha)
+		if (recs1->ha != recs2->ha)
 			break;
 
 	xdf1->dstart = xdf2->dstart = i;
 
-	recs1 = xdf1->recs + xdf1->record.length - 1;
-	recs2 = xdf2->recs + xdf2->record.length - 1;
+	recs1 = xdf1->record.ptr + xdf1->record.length - 1;
+	recs2 = xdf2->record.ptr + xdf2->record.length - 1;
 	for (lim -= i, i = 0; i < lim; i++, recs1--, recs2--)
-		if ((*recs1)->ha != (*recs2)->ha)
+		if (recs1->ha != recs2->ha)
 			break;
 
 	xdf1->dend = xdf1->record.length - i - 1;

index c322e62fbf..849f218b32 100644
@@ -49,7 +49,6 @@ DEFINE_IVEC_TYPE(xrecord_t, xrecord);
 typedef struct s_xdfile {
 	struct ivec_xrecord record;
 	long dstart, dend;
-	xrecord_t **recs;
 	char *rchg;
 	long *rindex;
 	long nreff;

index 10e4f20b7c..eed88ee6cb 100644
@@ -416,12 +416,12 @@ int xdl_fall_back_diff(xdfenv_t *diff_env, xpparam_t const *xpp,
 	mmfile_t subfile1, subfile2;
 	xdfenv_t env;
 
-	subfile1.ptr = (char *)diff_env->xdf1.recs[line1 - 1]->ptr;
-	subfile1.size = diff_env->xdf1.recs[line1 + count1 - 2]->ptr +
-		diff_env->xdf1.recs[line1 + count1 - 2]->size - (u8 const*) subfile1.ptr;
-	subfile2.ptr = (char *)diff_env->xdf2.recs[line2 - 1]->ptr;
-	subfile2.size = diff_env->xdf2.recs[line2 + count2 - 2]->ptr +
-		diff_env->xdf2.recs[line2 + count2 - 2]->size - (u8 const*) subfile2.ptr;
+	subfile1.ptr = (char *)diff_env->xdf1.record.ptr[line1 - 1].ptr;
+	subfile1.size = diff_env->xdf1.record.ptr[line1 + count1 - 2].ptr +
+		diff_env->xdf1.record.ptr[line1 + count1 - 2].size - (u8 const*) subfile1.ptr;
+	subfile2.ptr = (char *)diff_env->xdf2.record.ptr[line2 - 1].ptr;
+	subfile2.size = diff_env->xdf2.record.ptr[line2 + count2 - 2].ptr +
+		diff_env->xdf2.record.ptr[line2 + count2 - 2].size - (u8 const*) subfile2.ptr;
 	if (xdl_do_diff(&subfile1, &subfile2, xpp, &env) < 0)
 		return -1;
 

gitgitgadget

From: Ezekiel Newren <[email protected]>

Convert the remaining ambiguous fields in xdfile_t from C types to Rust types for interoperability.

---
 xdiff/xdiffi.c | 16 ++++++++--------
 xdiff/xdiffi.h |  8 ++++----
 xdiff/xtypes.h | 10 +++++-----
 3 files changed, 17 insertions(+), 17 deletions(-)

index ebdb724322..0509b48759 100644
@@ -42,8 +42,8 @@ typedef struct s_xdpsplit {
  * using this algorithm, so a little bit of heuristic is needed to cut the
  * search and to return a suboptimal point.
  */
-static long xdl_split(unsigned long const *ha1, long off1, long lim1,
-		      unsigned long const *ha2, long off2, long lim2,
+static long xdl_split(u64 const *ha1, long off1, long lim1,
+		      u64 const *ha2, long off2, long lim2,
 		      long *kvdf, long *kvdb, int need_min, xdpsplit_t *spl,
 		      xdalgoenv_t *xenv) {
 	long dmin = off1 - lim2, dmax = lim1 - off2;

@@ -260,7 +260,7 @@ static long xdl_split(unsigned long const *ha1, long off1, long lim1,
 int xdl_recs_cmp(diffdata_t *dd1, long off1, long lim1,
 		 diffdata_t *dd2, long off2, long lim2,
 		 long *kvdf, long *kvdb, int need_min, xdalgoenv_t *xenv) {
-	unsigned long const *ha1 = dd1->ha, *ha2 = dd2->ha;
+	u64 const *ha1 = dd1->ha, *ha2 = dd2->ha;
 
 	/*
 	 * Shrink the box by walking through each diagonal snake (SW and NE).

@@ -273,14 +273,14 @@ int xdl_recs_cmp(diffdata_t *dd1, long off1, long lim1,
 	 * be obviously changed.
 	 */
 	if (off1 == lim1) {
-		char *rchg2 = dd2->rchg;
-		long *rindex2 = dd2->rindex;
+		u8 *rchg2 = dd2->rchg;
+		usize *rindex2 = dd2->rindex;
 
 		for (; off2 < lim2; off2++)
 			rchg2[rindex2[off2]] = 1;
 	} else if (off2 == lim2) {
-		char *rchg1 = dd1->rchg;
-		long *rindex1 = dd1->rindex;
+		u8 *rchg1 = dd1->rchg;
+		usize *rindex1 = dd1->rindex;
 
 		for (; off1 < lim1; off1++)
 			rchg1[rindex1[off1]] = 1;

@@ -944,7 +944,7 @@ int xdl_change_compact(xdfile_t *xdf, xdfile_t *xdfo, long flags) {
 
 int xdl_build_script(xdfenv_t *xe, xdchange_t **xscr) {
 	xdchange_t *cscr = NULL, *xch;
-	char *rchg1 = xe->xdf1.rchg, *rchg2 = xe->xdf2.rchg;
+	u8 *rchg1 = xe->xdf1.rchg, *rchg2 = xe->xdf2.rchg;
 	long i1, i2, l1, l2;
 
 	/*

index 126c9d8ff4..c766ee115c 100644
@@ -25,10 +25,10 @@
 
 
 typedef struct s_diffdata {
-	long nrec;
-	unsigned long const *ha;
-	long *rindex;
-	char *rchg;
+	usize nrec;
+	u64 const *ha;
+	usize *rindex;
+	u8 *rchg;
 } diffdata_t;
 
 typedef struct s_xdalgoenv {

index 849f218b32..66b3dfae8b 100644
@@ -48,11 +48,11 @@ DEFINE_IVEC_TYPE(xrecord_t, xrecord);
 
 typedef struct s_xdfile {
 	struct ivec_xrecord record;
-	long dstart, dend;
-	char *rchg;
-	long *rindex;
-	long nreff;
-	unsigned long *ha;
+	isize dstart, dend;
+	u8 *rchg;
+	usize *rindex;
+	usize nreff;
+	u64 *ha;
 } xdfile_t;
 
 typedef struct s_xdfenv {

gitgitgadget

From: Ezekiel Newren <[email protected]>

Replace the C implementation of xdl_trim_ends() with a Rust implementation.

---
 rust/xdiff/src/lib.rs      | 15 +++++++++++++++
 rust/xdiff/src/xprepare.rs | 27 +++++++++++++++++++++++++++
 rust/xdiff/src/xtypes.rs   | 19 +++++++++++++++++++
 xdiff/xprepare.c           | 28 +---------------------------
 4 files changed, 62 insertions(+), 27 deletions(-)
 create mode 100644 rust/xdiff/src/xprepare.rs
 create mode 100644 rust/xdiff/src/xtypes.rs

index e69de29bb2..4cc05a7e6b 100644
@@ -0,0 +1,15 @@
+pub mod xprepare;
+pub mod xtypes;
+
+use crate::xprepare::trim_ends;
+use crate::xtypes::xdfile;
+
+#[no_mangle]
+unsafe extern "C" fn xdl_trim_ends(xdf1: *mut xdfile, xdf2: *mut xdfile) -> i32 {
+    let xdf1 = xdf1.as_mut().expect("null pointer");
+    let xdf2 = xdf2.as_mut().expect("null pointer");
+
+    trim_ends(xdf1, xdf2);
+
+    0
+}

new file mode 100644
index 0000000000..f64f60c099
@@ -0,0 +1,27 @@
+use crate::xtypes::xdfile;
+
+///
+/// Early trim initial and terminal matching records.
+///
+pub(crate) fn trim_ends(xdf1: &mut xdfile, xdf2: &mut xdfile) {
+    let mut lim = std::cmp::min(xdf1.record.len(), xdf2.record.len());
+
+    for i in 0..lim {
+        if xdf1.record[i].ha != xdf2.record[i].ha {
+            xdf1.dstart = i as isize;
+            xdf2.dstart = i as isize;
+            lim -= i;
+            break;
+        }
+    }
+
+    for i in 0..lim {
+        let f1i = xdf1.record.len() - 1 - i;
+        let f2i = xdf2.record.len() - 1 - i;
+        if xdf1.record[f1i].ha != xdf2.record[f2i].ha {
+            xdf1.dend = f1i as isize;
+            xdf2.dend = f2i as isize;
+            break;
+        }
+    }
+}

new file mode 100644
index 0000000000..3d1ce9742f
@@ -0,0 +1,19 @@
+use interop::ivec::IVec;
+
+#[repr(C)]
+pub(crate) struct xrecord {
+    pub(crate) ptr: *const u8,
+    pub(crate) size: usize,
+    pub(crate) ha: u64,
+}
+
+#[repr(C)]
+pub(crate) struct xdfile {
+    pub(crate) record: IVec<xrecord>,
+    pub(crate) dstart: isize,
+    pub(crate) dend: isize,
+    pub(crate) rchg: *mut u8,
+    pub(crate) rindex: *mut usize,
+    pub(crate) nreff: usize,
+    pub(crate) ha: *mut u64,
+}

index 93370f1c6d..2c7480875f 100644
@@ -318,33 +318,7 @@ static int xdl_cleanup_records(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xd
 }
 
 
-/*
- * Early trim initial and terminal matching records.
- */
-static int xdl_trim_ends(xdfile_t *xdf1, xdfile_t *xdf2) {
-	long i, lim;
-	xrecord_t *recs1, *recs2;
-
-	recs1 = xdf1->record.ptr;
-	recs2 = xdf2->record.ptr;
-	for (i = 0, lim = XDL_MIN(xdf1->record.length, xdf2->record.length); i < lim;
-	     i++, recs1++, recs2++)
-		if (recs1->ha != recs2->ha)
-			break;
-
-	xdf1->dstart = xdf2->dstart = i;
-
-	recs1 = xdf1->record.ptr + xdf1->record.length - 1;
-	recs2 = xdf2->record.ptr + xdf2->record.length - 1;
-	for (lim -= i, i = 0; i < lim; i++, recs1--, recs2--)
-		if (recs1->ha != recs2->ha)
-			break;
-
-	xdf1->dend = xdf1->record.length - i - 1;
-	xdf2->dend = xdf2->record.length - i - 1;
-
-	return 0;
-}
+extern i32 xdl_trim_ends(xdfile_t *xdf1, xdfile_t *xdf2);
 
 
 static int xdl_optimize_ctxs(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xdf2) {

gitgitgadget

This is a continuation of
https://lore.kernel.org/git/[email protected]/,
but I am removing the RFC label.

Suggestions on changes that I could make to this series is appreciated.

Changes in v2: High level overview:

* patch 1: cleanup: rename variables that collide with Rust primitive type names
* patches 2-4: Makefile now produces libgit.a correctly
* patches 5,6: Documentation from Brian and Patrick
* patches 7,8: Introduce Rust
* patches 9-11: github workflows
* patches 12-14: introduce crates build-helper, and cbindgen
* patches 15-18: varint test balloon

I would like feed back in two categories: big changes, and little changes. Is my patch series even going in the right direction, and what are the little details that I've missed? I know Brian asked for cbindgen and it made the series several commits longer, but I think it's a great idea to have. I am not happy with every choice that I made, but I currently don't see a better way or at least an easier alternative to my approach in bringing Rust to Git.

Build results for these changes:
https://github.com/git/git/actions/runs/17783386212?pr=2043. Some of these are failing.

Changes in v1: Changes since the last RFC patch series (range-diff added below):

* Addressed feedback:
* Use self instead of this
* Update ivec to use Git code style
* Remove environment variable CI_IS_DOCKER from github workflows
* Remove PHONY target interop-objs
* Removed left over asserts in Rust and fix near by code
* Other changes:
* Avoid name mangling of ivec if c++ compiler used
* Fixed a typo in patch 11 commit message

High level overview:

* patch 1: add a policy for using Rust (brian's patch, with a small tweak)
* patch 2: introduce Rust to the codebase
* patches 3-5: adapt CI (github workflows) to build Git with Rust
* patch 6: introduce the ivec type
* patches 7-14: xdiff code cleanup in preparation for translating to Rust
* patch 15: translate a C function into Rust and call it from C

Ezekiel Newren (12):
cleanup: rename variables that collide with Rust primitive type names
make: add -fPIE flag
make: merge xdiff lib into libgit.a
make: merge reftable lib into libgit.a
build: introduce rust
github workflows: install rust
github workflows: upload Cargo.lock
build: new crate, build-helper
build-helper: link against libgit.a and any other required C libraries
build-helper: cbindgen, let crates generate a header file
build: new crate, misc
misc: use BuildHelper

Johannes Schindelin (1):
win+Meson: do allow linking with the Rust-built xdiff

Patrick Steinhardt (4):
BreakingChanges: announce Rust becoming mandatory
help: report on whether or not Rust is enabled
varint: use explicit width for integers misc::varint: reimplement as test balloon for Rust

brian m. carlson (1):
doc: add a policy for using Rust

 .github/workflows/main.yml                    |  82 +++++++++-
 .gitignore                                    |   4 +
 Documentation/BreakingChanges.adoc            |  35 +++++
 Documentation/Makefile                        |   1 +
 Documentation/technical/platform-support.adoc |   2 +
 Documentation/technical/rust-support.adoc     | 142 +++++++++++++++++
 Makefile                                      | 148 +++++++++++-------
 build_rust.sh                                 |  60 +++++++
 ci/install-dependencies.sh                    |  14 +-
 ci/install-rust-toolchain.sh                  |  30 ++++
 ci/install-rustup.sh                          |  25 +++
 ci/lib.sh                                     |   1 +
 ci/make-test-artifacts.sh                     |   9 ++
 ci/run-build-and-tests.sh                     |  13 ++
 config.mak.uname                              |   4 +
 dir.c                                         |  18 ++-
 help.c                                        |   6 +
 meson.build                                   |  79 ++++++++--
 meson_options.txt                             |   2 +
 odb.c                                         |   8 +-
 read-cache.c                                  |   6 +-
 reftable/record.c                             |   6 +-
 rust/Cargo.toml                               |   6 +
 rust/build-helper/Cargo.toml                  |  11 ++
 rust/build-helper/src/lib.rs                  | 126 +++++++++++++++
 rust/cbindgen-template.toml                   |  16 ++
 rust/misc/Cargo.toml                          |  17 ++
 rust/misc/build.rs                            |   6 +
 rust/misc/src/lib.rs                          |   1 +
 rust/misc/src/varint.rs                       |  92 +++++++++++
 t/helper/test-parse-options.c                 |  12 +-
 varint.c                                      |   6 +-
 varint.h                                      |   4 +-
 xdiff/xtypes.h                                |   2 +-
 xdiff/xutils.c                                |   8 +-
 xdiff/xutils.h                                |   2 +-
 36 files changed, 899 insertions(+), 105 deletions(-)
 create mode 100644 Documentation/technical/rust-support.adoc
 create mode 100755 build_rust.sh
 create mode 100755 ci/install-rust-toolchain.sh
 create mode 100755 ci/install-rustup.sh
 create mode 100644 rust/Cargo.toml
 create mode 100644 rust/build-helper/Cargo.toml
 create mode 100644 rust/build-helper/src/lib.rs
 create mode 100644 rust/cbindgen-template.toml
 create mode 100644 rust/misc/Cargo.toml
 create mode 100644 rust/misc/build.rs
 create mode 100644 rust/misc/src/lib.rs
 create mode 100644 rust/misc/src/varint.rs

base-commit: c44beea485f0f2feaf460e2ac87fdd5608d63cf0
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-git-2043%2Fezekielnewren%2Fintroduce_rust-v2
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-git-2043/ezekielnewren/introduce_rust-v2
Pull-Request: https://github.com/git/git/pull/2043

Range-diff vs v1:

  -:  ---------- >  1:  5f77f1bd5d cleanup: rename variables that collide with Rust primitive type names
  -:  ---------- >  2:  7082e32c59 make: add -fPIE flag
  -:  ---------- >  3:  7fe85f0eac make: merge xdiff lib into libgit.a
 10:  0cd2168512 !  4:  3061cc46c1 xdiff: use one definition for freeing xdfile_t
     @@ Metadata
      Author: Ezekiel Newren <[email protected]>
      
       ## Commit message ##
     -    xdiff: use one definition for freeing xdfile_t
     -
     -    Simplify xdl_prepare_ctx() by using xdl_free_ctx() instead of using
     -    local variables with hand rolled memory management.
     +    make: merge reftable lib into libgit.a
      
          Signed-off-by: Ezekiel Newren <[email protected]>
      
     - ## xdiff/xprepare.c ##
     -@@ xdiff/xprepare.c: static int xdl_classify_record(unsigned int pass, xdlclassifier_t *cf, xrecord_t
     - }
     - 
     + ## Makefile ##
     +@@ Makefile: export PYTHON_PATH
     + TEST_SHELL_PATH = $(SHELL_PATH)
       
     -+static void xdl_free_ctx(xdfile_t *xdf) {
     -+	xdl_free(xdf->rindex);
     -+	xdl_free(xdf->rchg - 1);
     -+	xdl_free(xdf->ha);
     -+	xdl_free(xdf->recs);
     -+	xdl_cha_free(&xdf->rcha);
     -+}
     -+
     -+
     - static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_t const *xpp,
     - 			   xdlclassifier_t *cf, xdfile_t *xdf) {
     --	long nrec, bsize;
     -+	long bsize;
     - 	unsigned long hav;
     - 	char const *blk, *cur, *top, *prev;
     - 	xrecord_t *crec;
     --	xrecord_t **recs;
     --	unsigned long *ha;
     --	char *rchg;
     --	long *rindex;
     + LIB_FILE = libgit.a
     +-REFTABLE_LIB = reftable/libreftable.a
       
     --	ha = NULL;
     --	rindex = NULL;
     --	rchg = NULL;
     --	recs = NULL;
     -+	xdf->ha = NULL;
     -+	xdf->rindex = NULL;
     -+	xdf->rchg = NULL;
     -+	xdf->recs = NULL;
     -+	xdf->nrec = 0;
     + GENERATED_H += command-list.h
     + GENERATED_H += config-list.h
     +@@ Makefile: LIB_OBJS += rebase.o
     + LIB_OBJS += ref-filter.o
     + LIB_OBJS += reflog-walk.o
     + LIB_OBJS += reflog.o
     ++LIB_OBJS += reftable/basics.o
     ++LIB_OBJS += reftable/error.o
     ++LIB_OBJS += reftable/block.o
     ++LIB_OBJS += reftable/blocksource.o
     ++LIB_OBJS += reftable/iter.o
     ++LIB_OBJS += reftable/merged.o
     ++LIB_OBJS += reftable/pq.o
     ++LIB_OBJS += reftable/record.o
     ++LIB_OBJS += reftable/stack.o
     ++LIB_OBJS += reftable/system.o
     ++LIB_OBJS += reftable/table.o
     ++LIB_OBJS += reftable/tree.o
     ++LIB_OBJS += reftable/writer.o
     + LIB_OBJS += refs.o
     + LIB_OBJS += refs/debug.o
     + LIB_OBJS += refs/files-backend.o
     +@@ Makefile: CLAR_TEST_OBJS += $(UNIT_TEST_DIR)/unit-test.o
       
     - 	if (xdl_cha_init(&xdf->rcha, sizeof(xrecord_t), narec / 4 + 1) < 0)
     - 		goto abort;
     --	if (!XDL_ALLOC_ARRAY(recs, narec))
     -+	if (!XDL_ALLOC_ARRAY(xdf->recs, narec))
     - 		goto abort;
     + UNIT_TEST_OBJS += $(UNIT_TEST_DIR)/test-lib.o
       
     --	nrec = 0;
     - 	if ((cur = blk = xdl_mmfile_first(mf, &bsize))) {
     - 		for (top = blk + bsize; cur < top; ) {
     - 			prev = cur;
     - 			hav = xdl_hash_record(&cur, top, xpp->flags);
     --			if (XDL_ALLOC_GROW(recs, nrec + 1, narec))
     -+			if (XDL_ALLOC_GROW(xdf->recs, xdf->nrec + 1, narec))
     - 				goto abort;
     - 			if (!(crec = xdl_cha_alloc(&xdf->rcha)))
     - 				goto abort;
     - 			crec->ptr = (u8 const*) prev;
     - 			crec->size = (long) (cur - prev);
     - 			crec->ha = hav;
     --			recs[nrec++] = crec;
     -+			xdf->recs[xdf->nrec++] = crec;
     - 			if (xdl_classify_record(pass, cf, crec) < 0)
     - 				goto abort;
     - 		}
     - 	}
     +-# reftable lib may depend on what is in libgit.a
     +-GITLIBS = common-main.o $(LIB_FILE) $(REFTABLE_LIB) $(LIB_FILE)
     ++GITLIBS = common-main.o $(LIB_FILE)
     + EXTLIBS =
       
     --	if (!XDL_CALLOC_ARRAY(rchg, nrec + 2))
     -+	if (!XDL_CALLOC_ARRAY(xdf->rchg, xdf->nrec + 2))
     - 		goto abort;
     + GIT_USER_AGENT = git/$(GIT_VERSION)
     +@@ Makefile: reconfigure config.mak.autogen: config.status
     + .PHONY: reconfigure # This is a convenience target.
     + endif
       
     - 	if ((XDF_DIFF_ALG(xpp->flags) != XDF_PATIENCE_DIFF) &&
     - 	    (XDF_DIFF_ALG(xpp->flags) != XDF_HISTOGRAM_DIFF)) {
     --		if (!XDL_ALLOC_ARRAY(rindex, nrec + 1))
     -+		if (!XDL_ALLOC_ARRAY(xdf->rindex, xdf->nrec + 1))
     - 			goto abort;
     --		if (!XDL_ALLOC_ARRAY(ha, nrec + 1))
     -+		if (!XDL_ALLOC_ARRAY(xdf->ha, xdf->nrec + 1))
     - 			goto abort;
     - 	}
     +-REFTABLE_OBJS += reftable/basics.o
     +-REFTABLE_OBJS += reftable/error.o
     +-REFTABLE_OBJS += reftable/block.o
     +-REFTABLE_OBJS += reftable/blocksource.o
     +-REFTABLE_OBJS += reftable/iter.o
     +-REFTABLE_OBJS += reftable/merged.o
     +-REFTABLE_OBJS += reftable/pq.o
     +-REFTABLE_OBJS += reftable/record.o
     +-REFTABLE_OBJS += reftable/stack.o
     +-REFTABLE_OBJS += reftable/system.o
     +-REFTABLE_OBJS += reftable/table.o
     +-REFTABLE_OBJS += reftable/tree.o
     +-REFTABLE_OBJS += reftable/writer.o
     +-
     + TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS)) $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS))
       
     --	xdf->nrec = nrec;
     --	xdf->recs = recs;
     --	xdf->rchg = rchg + 1;
     --	xdf->rindex = rindex;
     -+	xdf->rchg += 1;
     - 	xdf->nreff = 0;
     --	xdf->ha = ha;
     - 	xdf->dstart = 0;
     --	xdf->dend = nrec - 1;
     -+	xdf->dend = xdf->nrec - 1;
     + .PHONY: test-objs
     +@@ Makefile: OBJECTS += $(SCALAR_OBJS)
     + OBJECTS += $(PROGRAM_OBJS)
     + OBJECTS += $(TEST_OBJS)
     + OBJECTS += $(FUZZ_OBJS)
     +-OBJECTS += $(REFTABLE_OBJS) $(REFTABLE_TEST_OBJS)
     ++OBJECTS += $(REFTABLE_TEST_OBJS)
     + OBJECTS += $(UNIT_TEST_OBJS)
     + OBJECTS += $(CLAR_TEST_OBJS)
     + OBJECTS += $(patsubst %,$(UNIT_TEST_DIR)/%.o,$(UNIT_TEST_PROGRAMS))
     +@@ Makefile: scalar$X: scalar.o GIT-LDFLAGS $(GITLIBS)
     + $(LIB_FILE): $(LIB_OBJS)
     + 	$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^
       
     - 	return 0;
     +-$(REFTABLE_LIB): $(REFTABLE_OBJS)
     +-	$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^
     +-
     + export DEFAULT_EDITOR DEFAULT_PAGER
       
     - abort:
     --	xdl_free(ha);
     --	xdl_free(rindex);
     --	xdl_free(rchg);
     --	xdl_free(recs);
     --	xdl_cha_free(&xdf->rcha);
     -+	xdl_free_ctx(xdf);
     - 	return -1;
     - }
     + Documentation/GIT-EXCLUDED-PROGRAMS: FORCE
     +@@ Makefile: clean: profile-clean coverage-clean cocciclean
     + 	$(RM) git.rc git.res
     + 	$(RM) $(OBJECTS)
     + 	$(RM) headless-git.o
     +-	$(RM) $(LIB_FILE) $(REFTABLE_LIB)
     ++	$(RM) $(LIB_FILE)
     + 	$(RM) $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) $(OTHER_PROGRAMS)
     + 	$(RM) $(TEST_PROGRAMS)
     + 	$(RM) $(FUZZ_PROGRAMS)
     +@@ Makefile: endif
       
     + LIBGIT_PUB_OBJS += contrib/libgit-sys/public_symbol_export.o
     + LIBGIT_PUB_OBJS += libgit.a
     +-LIBGIT_PUB_OBJS += reftable/libreftable.a
       
     --static void xdl_free_ctx(xdfile_t *xdf) {
     --	xdl_free(xdf->rindex);
     --	xdl_free(xdf->rchg - 1);
     --	xdl_free(xdf->ha);
     --	xdl_free(xdf->recs);
     --	xdl_cha_free(&xdf->rcha);
     --}
     --
     --
     - void xdl_free_env(xdfenv_t *xe) {
     + LIBGIT_PARTIAL_EXPORT = contrib/libgit-sys/partial_symbol_export.o
       
     - 	xdl_free_ctx(&xe->xdf2);
  1:  6d065f550f !  5:  0fc89c3ec3 doc: add a policy for using Rust
     @@ Documentation/technical/rust-support.adoc (new)
      +these crates, but for now we omit them.
      +
      +Tools like bindgen and cbindgen create C-styled unsafe Rust code rather than
     -+idiomatic Rust; where possible, we prefer to switch to idiomatic Rust.  Any
     -+standard C library functions that are needed can be manually wrapped on the
     -+Rust side.
     ++idiomatic Rust; where possible, we prefer to switch to idiomatic Rust.
     ++However, we may use bindgen and cbindgen to share existing Git types as an
     ++interim step.
      +
      +Rust upstream releases every six weeks and only supports the latest stable
      +release.  While it is nice that upstream is active, we would like our software
  -:  ---------- >  6:  8e030170dd BreakingChanges: announce Rust becoming mandatory
  2:  cc9cf87775 !  7:  6032a8740c xdiff: introduce rust
     @@ Metadata
      Author: Ezekiel Newren <[email protected]>
      
       ## Commit message ##
     -    xdiff: introduce rust
     +    build: introduce rust
      
          Upcoming patches will simplify xdiff, while also porting parts of it to
          Rust. In preparation, add some stubs and setup the Rust build. For now,
          it is easier to let cargo build rust and have make or meson merely link
          against the static library that cargo builds. In line with ongoing
          libification efforts, use multiple crates to allow more modularity on
     -    the Rust side. xdiff is the crate that this series will focus on, but
     -    we also introduce the interop crate for future patch series.
     +    the Rust side. The xdiff and ivec crates are the focal point for this
     +    series.
      
     -    In order to facilitate interoperability between C and Rust, introduce C
     -    definitions for Rust primitive types in git-compat-util.h.
     +    cbindgen will be used to generate header files for each Rust crate for
     +    C interoperability.
      
          Signed-off-by: Ezekiel Newren <[email protected]>
      
     @@ .gitignore: Release/
       /contrib/buildsystems/out
       /contrib/libgit-rs/target
       /contrib/libgit-sys/target
     -+/.idea/
     ++.idea/
      +/rust/target/
      +/rust/Cargo.lock
      
       ## Makefile ##
     +@@ Makefile: include shared.mak
     + # Define LIBPCREDIR=/foo/bar if your PCRE header and library files are
     + # in /foo/bar/include and /foo/bar/lib directories.
     + #
     ++# == Optional Rust support ==
     ++#
     ++# Define WITH_RUST=false if you don't want to include features and subsystems
     ++# written in Rust into Git. For now, Rust is still an optional feature of the
     ++# build process. With Git 3.0 though, Rust will be mandatory.
     ++#
     ++# Building Rust code requires Cargo.
     ++#
     + # == SHA-1 and SHA-256 defines ==
     + #
     + # === SHA-1 backend ===
      @@ Makefile: TEST_SHELL_PATH = $(SHELL_PATH)
       
       LIB_FILE = libgit.a
     - XDIFF_LIB = xdiff/lib.a
     -+
     + 
      +EXTLIBS =
      +
     ++GIT_BUILD_DIR := $(CURDIR)
     ++export GIT_BUILD_DIR
     ++
     ++RUST_CRATES :=
     ++.PHONY: compile_rust rustclean
     ++
     ++WITH_RUST ?= true
     ++ifeq ($(WITH_RUST),true)
     ++
      +ifeq ($(DEBUG), 1)
     -+  RUST_BUILD_MODE = debug
     ++	RUST_BUILD_MODE := debug
      +else
     -+  RUST_BUILD_MODE = release
     ++	RUST_BUILD_MODE := release
      +endif
      +
     -+RUST_TARGET_DIR = rust/target/$(RUST_BUILD_MODE)
     -+RUST_FLAGS_FOR_C = -L$(RUST_TARGET_DIR)
     ++RUST_TARGET_DIR := $(GIT_BUILD_DIR)/$(RUST_BUILD_MODE)
      +
     -+.PHONY: compile_rust
     -+compile_rust:
     -+	./build_rust.sh . $(RUST_BUILD_MODE) xdiff
     ++RUST_LIBS := $(foreach c,$(RUST_CRATES),$(GIT_BUILD_DIR)/lib$(c).a)
     ++
     ++EXTLIBS += $(RUST_LIBS)
      +
     -+EXTLIBS += ./$(RUST_TARGET_DIR)/libxdiff.a
     ++compile_rust: $(RUST_LIBS)
     ++rustclean:
     ++	$(RM) $(RUST_LIBS)
     ++	cd rust && cargo clean
     ++
     ++$(GIT_BUILD_DIR)/lib%.a:
     ++	./build_rust.sh $(GIT_BUILD_DIR) $(RUST_BUILD_MODE) $*
      +
      +UNAME_S := $(shell uname -s)
      +ifeq ($(UNAME_S),Linux)
     -+  EXTLIBS += -ldl
     ++	EXTLIBS += -ldl
     ++endif
     ++
     ++else ifeq ($(WITH_RUST),false)
     ++compile_rust:
     ++	:
     ++rustclean:
     ++	:
     ++else
     ++$(error 'WITH_RUST' must be true or false)
      +endif
      +
     - REFTABLE_LIB = reftable/libreftable.a
     - 
       GENERATED_H += command-list.h
     -@@ Makefile: UNIT_TEST_OBJS += $(UNIT_TEST_DIR)/lib-reftable.o
     + GENERATED_H += config-list.h
     + GENERATED_H += hook-list.h
     +@@ Makefile: CLAR_TEST_OBJS += $(UNIT_TEST_DIR)/unit-test.o
     + UNIT_TEST_OBJS += $(UNIT_TEST_DIR)/test-lib.o
       
     - # xdiff and reftable libs may in turn depend on what is in libgit.a
     - GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) $(LIB_FILE)
     + GITLIBS = common-main.o $(LIB_FILE)
      -EXTLIBS =
     -+
       
       GIT_USER_AGENT = git/$(GIT_VERSION)
       
     +@@ Makefile: CC_LD_DYNPATH = -Wl,-rpath,
     + BASIC_CFLAGS = -I. -fPIE
     + BASIC_LDFLAGS =
     + 
     ++ifeq ($(WITH_RUST),true)
     ++BASIC_CFLAGS += -DWITH_RUST
     ++endif
     + # library flags
     + ARFLAGS = rcs
     + PTHREAD_CFLAGS =
      @@ Makefile: git.sp git.s git.o: EXTRA_CPPFLAGS = \
       	'-DGIT_MAN_PATH="$(mandir_relative_SQ)"' \
       	'-DGIT_INFO_PATH="$(infodir_relative_SQ)"'
     @@ Makefile: $(REMOTE_CURL_ALIASES): $(REMOTE_CURL_PRIMARY)
       	$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) \
       		$(filter %.o,$^) $(LIBS)
       
     -@@ Makefile: $(LIB_FILE): $(LIB_OBJS)
     - $(XDIFF_LIB): $(XDIFF_OBJS)
     - 	$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^
     - 
     -+
     - $(REFTABLE_LIB): $(REFTABLE_OBJS)
     - 	$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^
     - 
      @@ Makefile: perf: all
       
       t/helper/test-tool$X: $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS)) $(UNIT_TEST_DIR)/test-lib.o
     @@ Makefile: cocciclean:
       	$(RM) contrib/coccinelle/*.cocci.patch
       
      -clean: profile-clean coverage-clean cocciclean
     -+rustclean:
     -+	cd rust && cargo clean
     -+
      +clean: profile-clean coverage-clean cocciclean rustclean
       	$(RM) -r .build $(UNIT_TEST_BIN)
       	$(RM) GIT-TEST-SUITES
     @@ build_rust.sh (new)
      @@
      +#!/bin/sh
      +
     -+
      +rustc -vV || exit $?
      +cargo --version || exit $?
      +
     @@ build_rust.sh (new)
      +  exit 1
      +fi
      +
     -+cd $dir_rust && cargo clean && pwd && cargo build -p $crate $rust_args; cd $dir_git_root
     -+
      +libfile="lib${crate}.a"
      +if rustup show active-toolchain | grep windows-msvc; then
      +  libfile="${crate}.lib"
     ++  PATH="$(echo $PATH | tr ':' '\n' | grep -Ev "^(/mingw64/bin|/usr/bin)$" | paste -sd: -):/mingw64/bin:/usr/bin"
      +fi
     -+dst=$dir_build/$libfile
      +
     -+if [ "$dir_git_root" != "$dir_build" ]; then
     -+  src=$dir_rust/target/$rust_build_profile/$libfile
     -+  if [ ! -f $src ]; then
     -+    echo >&2 "::error:: cannot find path of static library $src is not a file or does not exist"
     -+    exit 5
     -+  fi
     ++CARGO_TARGET_DIR=$dir_git_root/.build/rust/$crate
     ++export CARGO_TARGET_DIR
     ++
     ++cd $dir_rust && cargo clean && pwd && USE_LINKING="false" cargo build -p $crate $rust_args; cd $dir_git_root
     ++
     ++src=$CARGO_TARGET_DIR/$rust_build_profile/$libfile
     ++dst=$dir_build/$libfile
      +
     -+  rm $dst 2>/dev/null
     -+  mv $src $dst
     ++if [ ! -f $src ]; then
     ++  echo >&2 "::error:: cannot find path of static library $src is not a file or does not exist"
     ++  exit 5
      +fi
     -
     - ## git-compat-util.h ##
     -@@ git-compat-util.h: static inline int is_xplatform_dir_sep(int c)
     - #include "compat/msvc.h"
     - #endif
     - 
     -+/* rust types */
     -+typedef uint8_t   u8;
     -+typedef uint16_t  u16;
     -+typedef uint32_t  u32;
     -+typedef uint64_t  u64;
     -+
     -+typedef int8_t    i8;
     -+typedef int16_t   i16;
     -+typedef int32_t   i32;
     -+typedef int64_t   i64;
     -+
     -+typedef float     f32;
     -+typedef double    f64;
     -+
     -+typedef size_t    usize;
     -+typedef ptrdiff_t isize;
     -+
     - /* used on Mac OS X */
     - #ifdef PRECOMPOSE_UNICODE
     - #include "compat/precompose_utf8.h"
     ++
     ++rm $dst 2>/dev/null
     ++echo mv $src $dst
     ++mv $src $dst
      
       ## meson.build ##
      @@ meson.build: version_gen_environment.set('GIT_DATE', get_option('build_date'))
       version_gen_environment.set('GIT_USER_AGENT', get_option('user_agent'))
       version_gen_environment.set('GIT_VERSION', get_option('version'))
       
     -+if get_option('optimization') in ['2', '3', 's', 'z']
     -+  rust_build_profile = 'release'
     -+else
     -+  rust_build_profile = 'debug'
     -+endif
     ++rust_crates = []
      +
     -+# Run `rustup show active-toolchain` and capture output
     -+rustup_out = run_command('rustup', 'show', 'active-toolchain',
     -+                         check: true).stdout().strip()
     -+
     -+rust_crates = ['xdiff']
      +rust_builds = []
     ++with_rust = get_option('with_rust')
     ++if with_rust
     ++  rustc = find_program('rustc', required : false)
     ++  cargo = find_program('cargo', required : false)
      +
     -+foreach crate : rust_crates
     -+  if rustup_out.contains('windows-msvc')
     -+    libfile = crate + '.lib'
     ++  if not rustc.found() or not cargo.found()
     ++    error('Rust toolchain not found. Reconfigure with -Dwith_rust=false')
     ++  endif
     ++
     ++  if get_option('optimization') in ['2', '3', 's', 'z']
     ++    rust_build_profile = 'release'
      +  else
     -+    libfile = 'lib' + crate + '.a'
     ++    rust_build_profile = 'debug'
      +  endif
      +
     -+  rust_builds += custom_target(
     -+    'rust_build_'+crate,
     -+    output: libfile,
     -+    build_by_default: true,
     -+    build_always_stale: true,
     -+    command: [
     -+      meson.project_source_root() / 'build_rust.sh',
     -+      meson.current_build_dir(), rust_build_profile, crate,
     -+    ],
     -+    install: false,
     -+  )
     -+endforeach
     ++  # Run `rustup show active-toolchain` and capture output
     ++  rustup_out = run_command('rustup', 'show', 'active-toolchain',
     ++                           check: true).stdout().strip()
     ++
     ++  rust_environment = script_environment
     ++
     ++  foreach crate : rust_crates
     ++    if rustup_out.contains('windows-msvc')
     ++      libfile = crate + '.lib'
     ++    else
     ++      libfile = 'lib' + crate + '.a'
     ++    endif
     ++
     ++    rust_builds += custom_target(
     ++      'rust_build_'+crate,
     ++      output: libfile,
     ++      build_by_default: true,
     ++      build_always_stale: true,
     ++      command: [
     ++        meson.project_source_root() / 'build_rust.sh',
     ++        meson.current_build_dir(), rust_build_profile, crate,
     ++      ],
     ++      install: false,
     ++      env: rust_environment,
     ++    )
     ++  endforeach
     ++else
     ++  message('Rust components disabled (-Dwith_rust=false)')
     ++endif
      +
      +
       compiler = meson.get_compiler('c')
       
       libgit_sources = [
     +@@ meson.build: libgit_c_args = [
     +   '-DSHELL_PATH="' + fs.as_posix(target_shell.full_path()) + '"',
     + ]
     + 
     ++if with_rust
     ++  libgit_c_args += '-DWITH_RUST'
     ++endif
     ++
     + system_attributes = get_option('gitattributes')
     + if system_attributes != ''
     +   libgit_c_args += '-DETC_GITATTRIBUTES="' + system_attributes + '"'
     +@@ meson.build: elif host_machine.system() == 'windows'
     +   ]
     + 
     +   libgit_dependencies += compiler.find_library('ntdll')
     ++  libgit_dependencies += compiler.find_library('userenv')
     +   libgit_include_directories += 'compat/win32'
     +   if compiler.get_id() == 'msvc'
     +     libgit_include_directories += 'compat/vcbuild/include'
      @@ meson.build: version_def_h = custom_target(
       libgit_sources += version_def_h
       
     @@ meson.build: version_def_h = custom_target(
         dependencies: libgit_dependencies,
         include_directories: libgit_include_directories,
      
     + ## meson_options.txt ##
     +@@ meson_options.txt: option('zlib_backend', type: 'combo', choices: ['auto', 'zlib', 'zlib-ng'], valu
     +   description: 'The backend used for compressing objects and other data.')
     + 
     + # Build tweaks.
     ++option('with_rust', type: 'boolean', value: true,
     ++       description: 'Enable building with Rust, true by default.')
     + option('breaking_changes', type: 'boolean', value: false,
     +   description: 'Enable upcoming breaking changes.')
     + option('macos_use_homebrew_gettext', type: 'boolean', value: true,
     +
       ## rust/Cargo.toml (new) ##
      @@
      +[workspace]
     -+members = [
     -+    "xdiff",
     -+    "interop",
     -+]
     ++members = []
      +resolver = "2"
     -
     - ## rust/interop/Cargo.toml (new) ##
     -@@
     -+[package]
     -+name = "interop"
     -+version = "0.1.0"
     -+edition = "2021"
     -+
     -+[lib]
     -+name = "interop"
     -+path = "src/lib.rs"
     -+## staticlib to generate xdiff.a for use by gcc
     -+## cdylib (optional) to generate xdiff.so for use by gcc
     -+## rlib is required by the rust unit tests
     -+crate-type = ["staticlib", "rlib"]
     -+
     -+[dependencies]
     -
     - ## rust/interop/src/lib.rs (new) ##
     -
     - ## rust/xdiff/Cargo.toml (new) ##
     -@@
     -+[package]
     -+name = "xdiff"
     -+version = "0.1.0"
     -+edition = "2021"
     -+
     -+[lib]
     -+name = "xdiff"
     -+path = "src/lib.rs"
     -+## staticlib to generate xdiff.a for use by gcc
     -+## cdylib (optional) to generate xdiff.so for use by gcc
     -+## rlib is required by the rust unit tests
     -+crate-type = ["staticlib", "rlib"]
     -+
     -+[dependencies]
     -+interop = { path = "../interop" }
     -
     - ## rust/xdiff/src/lib.rs (new) ##
  -:  ---------- >  8:  cd04ed5aa7 help: report on whether or not Rust is enabled
  3:  6c47401ba0 !  9:  fcdfc55fb7 github workflows: install rust

@@ Commit message
the Rust toolchain, the Rust target triple must be set to x86_64-pc-windows-gnu for make (win build), and
x86_64-pc-windows-msvc for meson (win+Meson build).
+ * MSVC builds: Rearrange PATH to look in /mingw64/bin and /usr/bin
+ last.
* On musl: libc differences, such as ftruncate64 vs ftruncate, were
not accounted for until Rust version 1.72.0. No older version of
Rust will work on musl for our needs.
@@ .github/workflows/main.yml: jobs:
outputs:
enabled: ${{ steps.check-ref.outputs.enabled }}${{ steps.skip-if-redundant.outputs.enabled }}
skip_concurrent: ${{ steps.check-ref.outputs.skip_concurrent }}
-+ rust_version_minimum: 1.61.0
++ rust_version_minimum: 1.63.0
+ rust_version_windows: 1.78.0
+ rust_version_musl: 1.72.0
+ ## the rust target is inferred by rustup unless specified
@@ .github/workflows/main.yml: jobs:
steps:
- uses: actions/checkout@v4
- uses: git-for-windows/setup-git-for-windows-sdk@v1
++ with:
++ flavor: full
+ - name: Install rustup via github actions
+ uses: actions-rs/toolchain@v1
+ with:
4: 8e350700bc ! 10: 6eeef4081c win+Meson: do allow linking with the Rust-built xdiff
@@ Commit message
handling from an earlier patch]

- ## .github/workflows/main.yml ##
-@@ .github/workflows/main.yml: jobs:
- steps:
- - uses: actions/checkout@v4
- - uses: git-for-windows/setup-git-for-windows-sdk@v1
-+ - name: ensure that libuserenv.a is present
-+ shell: bash
-+ run: |
-+ cd /mingw64/lib && {
-+ test -f libuserenv.a ||
-+ /c/Program\ Files/Git/mingw64/bin/curl -Lo libuserenv.a \
-+ https://github.com/git-for-windows/git-sdk-64/raw/HEAD/mingw64/lib/libuserenv.a
-+ }
- - name: Install rustup via github actions
- uses: actions-rs/toolchain@v1
- with:
-
## config.mak.uname ##
@@ config.mak.uname: ifeq ($(uname_S),MINGW)
COMPAT_CFLAGS += -D_USE_32BIT_TIME_T
@@ config.mak.uname: ifeq ($(uname_S),MINGW)
CC = gcc
COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 -DDETECT_MSYS_TTY \
-fstack-protector-strong
-
- ## meson.build ##
-@@ meson.build: elif host_machine.system() == 'windows'
- ]
-
- libgit_dependencies += compiler.find_library('ntdll')
-+ libgit_dependencies += compiler.find_library('userenv')
- libgit_include_directories += 'compat/win32'
- if compiler.get_id() == 'msvc'
- libgit_include_directories += 'compat/vcbuild/include'

  5:  a00cbf7bcb = 11:  1e13d6c45f github workflows: upload Cargo.lock
  6:  922d506ed6 <  -:  ---------- ivec: create a vector type that is interoperable between C and Rust
  -:  ---------- > 12:  1c923a71f2 build: new crate, build-helper
  7:  3a6f822e2a ! 13:  6a27e07e63 xdiff/xprepare: remove superfluous forward declarations
     @@ Metadata
      Author: Ezekiel Newren <[email protected]>
      
       ## Commit message ##
     -    xdiff/xprepare: remove superfluous forward declarations
     +    build-helper: link against libgit.a and any other required C libraries
      
     -    Move xdl_prepare_env() later in the file to avoid the need
     -    for forward declarations.
     +    Don't link against the C libraries when building with Make or Meson.
     +    Run cargo tests like this:
     +    cd rust && cargo clean && USE_LINKING=true cargo test
      
          Signed-off-by: Ezekiel Newren <[email protected]>
      
     - ## xdiff/xprepare.c ##
     -@@ xdiff/xprepare.c: typedef struct s_xdlclassifier {
     + ## rust/build-helper/Cargo.toml ##
     +@@ rust/build-helper/Cargo.toml: version = "0.1.0"
     + edition = "2021"
       
     - 
     - 
     --static int xdl_init_classifier(xdlclassifier_t *cf, long size, long flags);
     --static void xdl_free_classifier(xdlclassifier_t *cf);
     --static int xdl_classify_record(unsigned int pass, xdlclassifier_t *cf, xrecord_t **rhash,
     --			       unsigned int hbits, xrecord_t *rec);
     --static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_t const *xpp,
     --			   xdlclassifier_t *cf, xdfile_t *xdf);
     --static void xdl_free_ctx(xdfile_t *xdf);
     --static int xdl_clean_mmatch(char const *dis, long i, long s, long e);
     --static int xdl_cleanup_records(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xdf2);
     --static int xdl_trim_ends(xdfile_t *xdf1, xdfile_t *xdf2);
     --static int xdl_optimize_ctxs(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xdf2);
     --
     --
     --
     --
     - static int xdl_init_classifier(xdlclassifier_t *cf, long size, long flags) {
     - 	cf->flags = flags;
     - 
     -@@ xdiff/xprepare.c: static void xdl_free_ctx(xdfile_t *xdf) {
     - }
     - 
     - 
     --int xdl_prepare_env(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp,
     --		    xdfenv_t *xe) {
     --	long enl1, enl2, sample;
     --	xdlclassifier_t cf;
     --
     --	memset(&cf, 0, sizeof(cf));
     --
     --	/*
     --	 * For histogram diff, we can afford a smaller sample size and
     --	 * thus a poorer estimate of the number of lines, as the hash
     --	 * table (rhash) won't be filled up/grown. The number of lines
     --	 * (nrecs) will be updated correctly anyway by
     --	 * xdl_prepare_ctx().
     --	 */
     --	sample = (XDF_DIFF_ALG(xpp->flags) == XDF_HISTOGRAM_DIFF
     --		  ? XDL_GUESS_NLINES2 : XDL_GUESS_NLINES1);
     --
     --	enl1 = xdl_guess_lines(mf1, sample) + 1;
     --	enl2 = xdl_guess_lines(mf2, sample) + 1;
     --
     --	if (xdl_init_classifier(&cf, enl1 + enl2 + 1, xpp->flags) < 0)
     --		return -1;
     --
     --	if (xdl_prepare_ctx(1, mf1, enl1, xpp, &cf, &xe->xdf1) < 0) {
     --
     --		xdl_free_classifier(&cf);
     --		return -1;
     --	}
     --	if (xdl_prepare_ctx(2, mf2, enl2, xpp, &cf, &xe->xdf2) < 0) {
     --
     --		xdl_free_ctx(&xe->xdf1);
     --		xdl_free_classifier(&cf);
     --		return -1;
     --	}
     + [dependencies]
      -
     --	if ((XDF_DIFF_ALG(xpp->flags) != XDF_PATIENCE_DIFF) &&
     --	    (XDF_DIFF_ALG(xpp->flags) != XDF_HISTOGRAM_DIFF) &&
     --	    xdl_optimize_ctxs(&cf, &xe->xdf1, &xe->xdf2) < 0) {
     --
     --		xdl_free_ctx(&xe->xdf2);
     --		xdl_free_ctx(&xe->xdf1);
     --		xdl_free_classifier(&cf);
     --		return -1;
     --	}
     --
     --	xdl_free_classifier(&cf);
     --
     --	return 0;
     --}
     --
     --
     - void xdl_free_env(xdfenv_t *xe) {
     - 
     - 	xdl_free_ctx(&xe->xdf2);
     -@@ xdiff/xprepare.c: static int xdl_optimize_ctxs(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xdf2
     - 
     - 	return 0;
     - }
     +
     + ## rust/build-helper/src/lib.rs ##
     +@@
     ++use std::collections::HashMap;
     ++use std::path::PathBuf;
      +
     -+int xdl_prepare_env(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp,
     -+		    xdfenv_t *xe) {
     -+	long enl1, enl2, sample;
     -+	xdlclassifier_t cf;
      +
     -+	memset(&cf, 0, sizeof(cf));
     ++fn parse_bool_from_str(value: &str) -> bool {
     ++    match value {
     ++        "1" | "true"  | "yes" | "on"  => true,
     ++        "0" | "false" | "no"  | "off" => false,
     ++        _ => false
     ++    }
     ++}
     ++
     ++fn parse_bool_from_option(value: Option<&String>, default: bool) -> bool {
     ++    match value {
     ++        Some(v) => {
     ++            parse_bool_from_str(v.as_str())
     ++        },
     ++        None => default,
     ++    }
     ++}
      +
     -+	/*
     -+	 * For histogram diff, we can afford a smaller sample size and
     -+	 * thus a poorer estimate of the number of lines, as the hash
     -+	 * table (rhash) won't be filled up/grown. The number of lines
     -+	 * (nrecs) will be updated correctly anyway by
     -+	 * xdl_prepare_ctx().
     -+	 */
     -+	sample = (XDF_DIFF_ALG(xpp->flags) == XDF_HISTOGRAM_DIFF
     -+		  ? XDL_GUESS_NLINES2 : XDL_GUESS_NLINES1);
     ++/// To build without linking against C libraries run `USE_LINKING=false cargo build`
     ++/// To run tests set GIT_BUILD_DIR and run `USE_LINKING=true cargo test`
     ++pub struct BuildHelper {
     ++    crate_env: HashMap<String, String>,
     ++}
      +
     -+	enl1 = xdl_guess_lines(mf1, sample) + 1;
     -+	enl2 = xdl_guess_lines(mf2, sample) + 1;
      +
     -+	if (xdl_init_classifier(&cf, enl1 + enl2 + 1, xpp->flags) < 0)
     -+		return -1;
     ++impl BuildHelper {
     ++    pub fn new(crate_env: HashMap<String, String>) -> Self {
     ++        let it = Self {crate_env};
      +
     -+	if (xdl_prepare_ctx(1, mf1, enl1, xpp, &cf, &xe->xdf1) < 0) {
     ++        let dir_crate = it.dir_crate();
     ++        let dir_workspace = dir_crate.parent().unwrap();
     ++        let dir_git = dir_workspace.parent().unwrap();
     ++        let dir_interop = dir_git.join("interop");
     ++        if !dir_interop.exists() {
     ++            std::fs::create_dir(dir_interop.clone()).unwrap();
     ++        }
      +
     -+		xdl_free_classifier(&cf);
     -+		return -1;
     -+	}
     -+	if (xdl_prepare_ctx(2, mf2, enl2, xpp, &cf, &xe->xdf2) < 0) {
     ++        it
     ++    }
      +
     -+		xdl_free_ctx(&xe->xdf1);
     -+		xdl_free_classifier(&cf);
     -+		return -1;
     -+	}
     ++    pub fn crate_name(&self) -> String {
     ++        self.crate_env["CARGO_PKG_NAME"].clone()
     ++    }
      +
     -+	if ((XDF_DIFF_ALG(xpp->flags) != XDF_PATIENCE_DIFF) &&
     -+	    (XDF_DIFF_ALG(xpp->flags) != XDF_HISTOGRAM_DIFF) &&
     -+	    xdl_optimize_ctxs(&cf, &xe->xdf1, &xe->xdf2) < 0) {
     ++    pub fn dir_crate(&self) -> PathBuf {
     ++        PathBuf::from(self.crate_env["CARGO_MANIFEST_DIR"].clone())
     ++    }
      +
     -+		xdl_free_ctx(&xe->xdf2);
     -+		xdl_free_ctx(&xe->xdf1);
     -+		xdl_free_classifier(&cf);
     -+		return -1;
     -+	    }
     ++    pub fn build(self) {
     ++        let use_linking = parse_bool_from_option(self.crate_env.get("USE_LINKING"), self.crate_env.get("CARGO_TARGET_DIR").is_none());
     ++        let dir_crate = self.dir_crate();
     ++        let dir_git = dir_crate.parent().unwrap().parent().unwrap();
      +
     -+	xdl_free_classifier(&cf);
     ++        println!("cargo:rerun-if-changed={}", dir_git.display());
      +
     -+	return 0;
     ++        if use_linking {
     ++            if let Some(git_build_dir) = self.crate_env.get("GIT_BUILD_DIR") {
     ++                let mut path_git_build_dir = PathBuf::from(git_build_dir);
     ++                path_git_build_dir = path_git_build_dir.canonicalize().unwrap();
     ++                if !path_git_build_dir.is_dir() {
     ++                    panic!("'GIT_BUILD_DIR' is not a directory: {}", path_git_build_dir.display());
     ++                }
     ++                println!("cargo:rustc-link-search=native={}", git_build_dir);
     ++            } else {
     ++                panic!("environment variable 'GIT_BUILD_DIR' is not set");
     ++            }
     ++
     ++            println!("cargo:rustc-link-lib=static=git");
     ++            println!("cargo:rustc-link-lib=pcre2-8");
     ++            if self.crate_env.get("ZLIB_NG").is_some() {
     ++                println!("cargo:rustc-link-lib=z-ng");
     ++            } else {
     ++                println!("cargo:rustc-link-lib=z");
     ++            }
     ++        } else {
     ++            println!("cargo:warning={} is not linking against C objects, `USE_LINKING=true cargo test`", self.crate_env["CARGO_PKG_NAME"]);
     ++        }
     ++    }
      +}
     ++
     ++
  8:  c7cd71dae0 <  -:  ---------- xdiff: delete unnecessary fields from xrecord_t and xdfile_t
  9:  1096c8f0a4 <  -:  ---------- xdiff: make fields of xrecord_t Rust friendly
 11:  7ea2dccd71 <  -:  ---------- xdiff: replace chastore with an ivec in xdfile_t
 12:  d8f561e173 <  -:  ---------- xdiff: delete nrec field from xdfile_t
 13:  15c2854a32 <  -:  ---------- xdiff: delete recs field from xdfile_t
 14:  3dacb2c09d <  -:  ---------- xdiff: make xdfile_t more rust friendly
 15:  9fdd23acf6 <  -:  ---------- xdiff: implement xdl_trim_ends() in Rust
  -:  ---------- > 14:  fa33440568 build-helper: cbindgen, let crates generate a header file
  -:  ---------- > 15:  df837fdb4d varint: use explicit width for integers
  -:  ---------- > 16:  17143ced6f build: new crate, misc
  -:  ---------- > 17:  89bb7c3613 misc: use BuildHelper
  -:  ---------- > 18:  3c521fa2e4 misc::varint: reimplement as test balloon for Rust

gitgitgadget

From: Ezekiel Newren <[email protected]>

Use a regex to find and rename variables that collide with Rust primitive integer and float type names:

    git grep -n -E -e '\<([ui](8|16|32|64|size)|(f(32|64)))\>'

Matches were reviewed and renamed. The remaining matches don't count because:
- Rust source files:
contrib/libgit-rs/src/config.rs contrib/libgit-sys/src/lib.rs
t/t4018/rust-impl
t/t4018/rust-trait
- Intentional references:
t/helper/test-parse-options.c (prints Rust int names) t/t0040-parse-options.sh (tests the above)

View with --color-words to highlight the variable renames.

---
 odb.c                         |  8 ++++----
 reftable/record.c             |  6 +++---
 t/helper/test-parse-options.c | 12 ++++++------
 xdiff/xtypes.h                |  2 +-
 xdiff/xutils.c                |  8 ++++----
 xdiff/xutils.h                |  2 +-
 6 files changed, 19 insertions(+), 19 deletions(-)

index 2a92a018c4..f0255b4b36 100644
@@ -913,7 +913,7 @@ void *odb_read_object_peeled(struct object_database *odb,
 {
 	enum object_type type;
 	void *buffer;
-	unsigned long isize;
+	unsigned long isize_;
 	struct object_id actual_oid;
 
 	oidcpy(&actual_oid, oid);

@@ -921,11 +921,11 @@ void *odb_read_object_peeled(struct object_database *odb,
 		int ref_length = -1;
 		const char *ref_type = NULL;
 
-		buffer = odb_read_object(odb, &actual_oid, &type, &isize);
+		buffer = odb_read_object(odb, &actual_oid, &type, &isize_);
 		if (!buffer)
 			return NULL;
 		if (type == required_type) {
-			*size = isize;
+			*size = isize_;
 			if (actual_oid_return)
 				oidcpy(actual_oid_return, &actual_oid);
 			return buffer;

@@ -941,7 +941,7 @@ void *odb_read_object_peeled(struct object_database *odb,
 		}
 		ref_length = strlen(ref_type);
 
-		if (ref_length + odb->repo->hash_algo->hexsz > isize ||
+		if (ref_length + odb->repo->hash_algo->hexsz > isize_ ||
 		    memcmp(buffer, ref_type, ref_length) ||
 		    get_oid_hex_algop((char *) buffer + ref_length, &actual_oid,
 				      odb->repo->hash_algo)) {

index fcd387ba5d..e8fc854561 100644
@@ -675,7 +675,7 @@ static int reftable_log_record_key(const void *r, struct reftable_buf *dest)
 	const struct reftable_log_record *rec =
 		(const struct reftable_log_record *)r;
 	int len = strlen(rec->refname), err;
-	uint8_t i64[8];
+	uint8_t i64_[8];
 	uint64_t ts = 0;
 
 	reftable_buf_reset(dest);

@@ -684,9 +684,9 @@ static int reftable_log_record_key(const void *r, struct reftable_buf *dest)
 		return err;
 
 	ts = (~ts) - rec->update_index;
-	reftable_put_be64(&i64[0], ts);
+	reftable_put_be64(&i64_[0], ts);
 
-	err = reftable_buf_add(dest, i64, sizeof(i64));
+	err = reftable_buf_add(dest, i64_, sizeof(i64_));
 	if (err < 0)
 		return err;
 

index 68579d83f3..aa1cb693a2 100644
@@ -120,8 +120,8 @@ int cmd__parse_options(int argc, const char **argv)
 	};
 	struct string_list expect = STRING_LIST_INIT_NODUP;
 	struct string_list list = STRING_LIST_INIT_NODUP;
-	uint16_t u16 = 0;
-	int16_t i16 = 0;
+	uint16_t u16_ = 0;
+	int16_t i16_ = 0;
 
 	struct option options[] = {
 		OPT_BOOL(0, "yes", &boolean, "get a boolean"),

@@ -142,10 +142,10 @@ int cmd__parse_options(int argc, const char **argv)
 		OPT_NEGBIT(0, "neg-or4", &boolean, "same as --no-or4", 4),
 		OPT_GROUP(""),
 		OPT_INTEGER('i', "integer", &integer, "get a integer"),
-		OPT_INTEGER(0, "i16", &i16, "get a 16 bit integer"),
+		OPT_INTEGER(0, "i16", &i16_, "get a 16 bit integer"),
 		OPT_INTEGER('j', NULL, &integer, "get a integer, too"),
 		OPT_UNSIGNED('u', "unsigned", &unsigned_integer, "get an unsigned integer"),
-		OPT_UNSIGNED(0, "u16", &u16, "get a 16 bit unsigned integer"),
+		OPT_UNSIGNED(0, "u16", &u16_, "get a 16 bit unsigned integer"),
 		OPT_SET_INT(0, "set23", &integer, "set integer to 23", 23),
 		OPT_CMDMODE(0, "mode1", &integer, "set integer to 1 (cmdmode option)", 1),
 		OPT_CMDMODE(0, "mode2", &integer, "set integer to 2 (cmdmode option)", 2),

@@ -225,9 +225,9 @@ int cmd__parse_options(int argc, const char **argv)
 	}
 	show(&expect, &ret, "boolean: %d", boolean);
 	show(&expect, &ret, "integer: %d", integer);
-	show(&expect, &ret, "i16: %"PRIdMAX, (intmax_t) i16);
+	show(&expect, &ret, "i16: %"PRIdMAX, (intmax_t) i16_);
 	show(&expect, &ret, "unsigned: %lu", unsigned_integer);
-	show(&expect, &ret, "u16: %"PRIuMAX, (uintmax_t) u16);
+	show(&expect, &ret, "u16: %"PRIuMAX, (uintmax_t) u16_);
 	show(&expect, &ret, "timestamp: %"PRItime, timestamp);
 	show(&expect, &ret, "string: %s", string ? string : "(not set)");
 	show(&expect, &ret, "abbrev: %d", abbrev);

index 8442bd436e..ec1dc7548a 100644
@@ -32,7 +32,7 @@ typedef struct s_chanode {
 
 typedef struct s_chastore {
 	chanode_t *head, *tail;
-	long isize, nsize;
+	long isize_, nsize;
 	chanode_t *ancur;
 	chanode_t *sncur;
 	long scurr;

index 444a108f87..20dee53dbd 100644
@@ -71,11 +71,11 @@ long xdl_mmfile_size(mmfile_t *mmf)
 }
 
 
-int xdl_cha_init(chastore_t *cha, long isize, long icount) {
+int xdl_cha_init(chastore_t *cha, long isize_, long icount) {
 
 	cha->head = cha->tail = NULL;
-	cha->isize = isize;
-	cha->nsize = icount * isize;
+	cha->isize_ = isize_;
+	cha->nsize = icount * isize_;
 	cha->ancur = cha->sncur = NULL;
 	cha->scurr = 0;
 

@@ -113,7 +113,7 @@ void *xdl_cha_alloc(chastore_t *cha) {
 	}
 
 	data = (char *) ancur + sizeof(chanode_t) + ancur->icurr;
-	ancur->icurr += cha->isize;
+	ancur->icurr += cha->isize_;
 
 	return data;
 }

index fd0bba94e8..b531d2ebff 100644
@@ -28,7 +28,7 @@
 long xdl_bogosqrt(long n);
 int xdl_emit_diffrec(char const *rec, long size, char const *pre, long psize,
 		     xdemitcb_t *ecb);
-int xdl_cha_init(chastore_t *cha, long isize, long icount);
+int xdl_cha_init(chastore_t *cha, long isize_, long icount);
 void xdl_cha_free(chastore_t *cha);
 void *xdl_cha_alloc(chastore_t *cha);
 long xdl_guess_lines(mmfile_t *mf, long sample);

gitgitgadget

This explains what the patch is doing but doesn't explain why we want this change. Please update the commit message to describe the problem/issue the patch is trying to address, and then (if necessary) explain what the patch is doing.

On top of that, just like a name that begins with an underscore is reserved by C standard, a name taht ends with an underscore is used for specific purposes by convention in this project.

Typically 'foo_' appears as a parameter to a function, whose type is specified with our interhal API, and when that type is cumbersome to work with, we have a local equivalent 'foo' of a more appropriate type whose value is populated from 'foo_' before being used, and use of 'foo' thanks to its better type is more ergonomic.

    static int compare_pt(const void *a_, const void *b_)
    {
            struct possible_tag *a = (struct possible_tag *)a_;
            struct possible_tag *b = (struct possible_tag *)b_;
            if (a->depth != b->depth)
                    return a->depth - b->depth;
            if (a->found_order != b->found_order)
                    return a->found_order - b->found_order;
            return 0;
    }

    static int path_is_beyond_symlink(struct apply_state *state, const char *name_)
    {
            int ret;
            struct strbuf name = STRBUF_INIT;

            assert(*name_ != '\0');
            strbuf_addstr(&name, name_);
            ret = path_is_beyond_symlink_1(state, &name);
            strbuf_release(&name);

            return ret;
    }

are examples.

There are existing crappy code that uses foo_ without corresponding foo; we should clean them up, not emulating or spreading the pattern.

By the way, in this partcular case, why not use "uint16_t u16"?

Isn't the true cause of the trouble the (I might say "misguided") desire to use "u16" as a type in C code? As long as we all agree that the data that can be passed across the ffi barrier should be of the types of known size, and let C side use uint(8|16|32|64)_t and Rust side use u(8|16|32|64) consistently, we do not need to have this "cleanup", do we?

Thanks.

It appears that I am the only one in favor of using Rust type names in C. Going forward I'll use [ui]int(8|16|32|64)_t types going forward. I'll drop this commit.

From: Ezekiel Newren <[email protected]>

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

index e11340c1ae..d3e034f3be 100644
@@ -1402,7 +1402,7 @@ endif
 CFLAGS = -g -O2 -Wall
 LDFLAGS =
 CC_LD_DYNPATH = -Wl,-rpath,
-BASIC_CFLAGS = -I.
+BASIC_CFLAGS = -I. -fPIE
 BASIC_LDFLAGS =
 
 # library flags

gitgitgadget

I was trying to get the rust compiler to link against libgit.a. Both Meson and Makefile produce libgit.a. No changes were to meson.build were necessary for that to work, but but with Makefile I would get errors like:
relocation R_X86_64_32 against `.rodata' can not be used when making a PIE object; recompile with -fPIE
...

I should have added this to the commit message.

This sounds like a workaround that is somewhat platform and compiler specific limitation, at least to me. Does this need to be conditional and if so on what?

Thanks.

I don't know, but I have another question to add to yours. Why does Make need -fPIE in order for the Rust compiler to link against libgit.a created by Make, when Meson doesn't seem to specify PIE anywhere and yet the Rust compiler can work with Meson's libgit.a output?

I do not know what the build procedure using meson exactly does to create a library archive. On the Make side, we do not compile for position independence, and don't do anything other than bog standard "ar rcs". Patrick, any hint?

Not too familiar with Rust, but looks like the answer is here [1]. Specificially, this part:

pic - fully relocatable position independent code, machine instructions need to use relative addressing modes. Equivalent to the "uppercase" -fPIC or -fPIE options in other compilers, depending on the produced crate types. This is the default model for majority of supported targets.

Collin

[1] https://doc.rust-lang.org/rustc/codegen-options/index.html#relocation-model

Well, thanks for joining the conversation, but that much I think is already shared by those who are involved in this thread.

The background was that Ezekiel did not have to do anything special as far as he knows to get things compiled in a relocatable way with meson based build, and with make based build, (which I know has never used nor required any PIE-ness) he of course needs to add it in a way that is portable across platforms and compilers we care about if we wanted to have PIE objects.

And the question was what, if anything, does meson do specially to do so in a portable way to enable PIE. We'd want to see if it meets our needs (it could be that Ezekiel is getting lucky and meson based build covered the platform+compiler combinations that he happened to be working with) and if so mimick that on the Makefile side.

[snip]

Yes, meson defaults the compiler flags to -fPIC (see e.g. near the end of the original cover letter of say [1]). You can change the compiler flags that meson uses (can't remember off-hand how you do that, but it is documented), you can even set -fPIE, or remove -fPIC etc,. ;)

So, I don't know why rust (seems to) require -fPIC/-fPIE to be able to link to libgit.a; maybe we need to add some flags to rustc to _not_ use the equivalent of -fPIC/-fPIE. (I haven't looked, but I will note that the GNU nm program complains about the rust built library!)

[1] https://lore.kernel.org/git/[email protected]/

ATB,
Ramsay Jones

From: Ezekiel Newren <[email protected]>

---
 Makefile | 30 ++++++++++--------------------
 1 file changed, 10 insertions(+), 20 deletions(-)

index d3e034f3be..94f38eb4c9 100644
@@ -670,7 +670,6 @@ BUILTIN_OBJS =
 BUILT_INS =
 COMPAT_CFLAGS =
 COMPAT_OBJS =
-XDIFF_OBJS =
 GENERATED_H =
 EXTRA_CPPFLAGS =
 FUZZ_OBJS =

@@ -916,7 +915,6 @@ export PYTHON_PATH
 TEST_SHELL_PATH = $(SHELL_PATH)
 
 LIB_FILE = libgit.a
-XDIFF_LIB = xdiff/lib.a
 REFTABLE_LIB = reftable/libreftable.a
 
 GENERATED_H += command-list.h

@@ -1207,6 +1205,13 @@ LIB_OBJS += write-or-die.o
 LIB_OBJS += ws.o
 LIB_OBJS += wt-status.o
 LIB_OBJS += xdiff-interface.o
+LIB_OBJS += xdiff/xdiffi.o
+LIB_OBJS += xdiff/xemit.o
+LIB_OBJS += xdiff/xhistogram.o
+LIB_OBJS += xdiff/xmerge.o
+LIB_OBJS += xdiff/xpatience.o
+LIB_OBJS += xdiff/xprepare.o
+LIB_OBJS += xdiff/xutils.o
 
 BUILTIN_OBJS += builtin/add.o
 BUILTIN_OBJS += builtin/am.o

@@ -1385,8 +1390,8 @@ CLAR_TEST_OBJS += $(UNIT_TEST_DIR)/unit-test.o
 
 UNIT_TEST_OBJS += $(UNIT_TEST_DIR)/test-lib.o
 
-# xdiff and reftable libs may in turn depend on what is in libgit.a
-GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) $(LIB_FILE)
+# reftable lib may depend on what is in libgit.a
+GITLIBS = common-main.o $(LIB_FILE) $(REFTABLE_LIB) $(LIB_FILE)
 EXTLIBS =
 
 GIT_USER_AGENT = git/$(GIT_VERSION)

@@ -2718,16 +2723,6 @@ reconfigure config.mak.autogen: config.status
 .PHONY: reconfigure # This is a convenience target.
 endif
 
-XDIFF_OBJS += xdiff/xdiffi.o
-XDIFF_OBJS += xdiff/xemit.o
-XDIFF_OBJS += xdiff/xhistogram.o
-XDIFF_OBJS += xdiff/xmerge.o
-XDIFF_OBJS += xdiff/xpatience.o
-XDIFF_OBJS += xdiff/xprepare.o
-XDIFF_OBJS += xdiff/xutils.o
-.PHONY: xdiff-objs
-xdiff-objs: $(XDIFF_OBJS)
-
 REFTABLE_OBJS += reftable/basics.o
 REFTABLE_OBJS += reftable/error.o
 REFTABLE_OBJS += reftable/block.o

@@ -2762,7 +2757,6 @@ OBJECTS += $(GIT_OBJS)
 OBJECTS += $(SCALAR_OBJS)
 OBJECTS += $(PROGRAM_OBJS)
 OBJECTS += $(TEST_OBJS)
-OBJECTS += $(XDIFF_OBJS)
 OBJECTS += $(FUZZ_OBJS)
 OBJECTS += $(REFTABLE_OBJS) $(REFTABLE_TEST_OBJS)
 OBJECTS += $(UNIT_TEST_OBJS)

@@ -2916,9 +2910,6 @@ scalar$X: scalar.o GIT-LDFLAGS $(GITLIBS)
 $(LIB_FILE): $(LIB_OBJS)
 	$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^
 
-$(XDIFF_LIB): $(XDIFF_OBJS)
-	$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^
-
 $(REFTABLE_LIB): $(REFTABLE_OBJS)
 	$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^
 

@@ -3760,7 +3751,7 @@ clean: profile-clean coverage-clean cocciclean
 	$(RM) git.rc git.res
 	$(RM) $(OBJECTS)
 	$(RM) headless-git.o
-	$(RM) $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB)
+	$(RM) $(LIB_FILE) $(REFTABLE_LIB)
 	$(RM) $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) $(OTHER_PROGRAMS)
 	$(RM) $(TEST_PROGRAMS)
 	$(RM) $(FUZZ_PROGRAMS)

@@ -3955,7 +3946,6 @@ endif
 LIBGIT_PUB_OBJS += contrib/libgit-sys/public_symbol_export.o
 LIBGIT_PUB_OBJS += libgit.a
 LIBGIT_PUB_OBJS += reftable/libreftable.a
-LIBGIT_PUB_OBJS += xdiff/lib.a
 
 LIBGIT_PARTIAL_EXPORT = contrib/libgit-sys/partial_symbol_export.o
 

gitgitgadget

Please extend the commit message to explain why this change is desirable.

This commit continues the effort of making the Rust compiler link against libgit.a. Meson already includes xdiff when it builds libgit.a, but Makefile does not. Rather than telling the Rust compiler to also link against xdiff/lib.a, if Makefile was used as opposed to Meson, seemed like a clunkier solution.

I should have asked for a better way to accomplish this as part of my commit message or as a question on the mailing list.

From: Ezekiel Newren <[email protected]>

---
 Makefile | 39 ++++++++++++++++-----------------------
 1 file changed, 16 insertions(+), 23 deletions(-)

index 94f38eb4c9..ffb898b611 100644
@@ -915,7 +915,6 @@ export PYTHON_PATH
 TEST_SHELL_PATH = $(SHELL_PATH)
 
 LIB_FILE = libgit.a
-REFTABLE_LIB = reftable/libreftable.a
 
 GENERATED_H += command-list.h
 GENERATED_H += config-list.h

@@ -1125,6 +1124,19 @@ LIB_OBJS += rebase.o
 LIB_OBJS += ref-filter.o
 LIB_OBJS += reflog-walk.o
 LIB_OBJS += reflog.o
+LIB_OBJS += reftable/basics.o
+LIB_OBJS += reftable/error.o
+LIB_OBJS += reftable/block.o
+LIB_OBJS += reftable/blocksource.o
+LIB_OBJS += reftable/iter.o
+LIB_OBJS += reftable/merged.o
+LIB_OBJS += reftable/pq.o
+LIB_OBJS += reftable/record.o
+LIB_OBJS += reftable/stack.o
+LIB_OBJS += reftable/system.o
+LIB_OBJS += reftable/table.o
+LIB_OBJS += reftable/tree.o
+LIB_OBJS += reftable/writer.o
 LIB_OBJS += refs.o
 LIB_OBJS += refs/debug.o
 LIB_OBJS += refs/files-backend.o

@@ -1390,8 +1402,7 @@ CLAR_TEST_OBJS += $(UNIT_TEST_DIR)/unit-test.o
 
 UNIT_TEST_OBJS += $(UNIT_TEST_DIR)/test-lib.o
 
-# reftable lib may depend on what is in libgit.a
-GITLIBS = common-main.o $(LIB_FILE) $(REFTABLE_LIB) $(LIB_FILE)
+GITLIBS = common-main.o $(LIB_FILE)
 EXTLIBS =
 
 GIT_USER_AGENT = git/$(GIT_VERSION)

@@ -2723,20 +2734,6 @@ reconfigure config.mak.autogen: config.status
 .PHONY: reconfigure # This is a convenience target.
 endif
 
-REFTABLE_OBJS += reftable/basics.o
-REFTABLE_OBJS += reftable/error.o
-REFTABLE_OBJS += reftable/block.o
-REFTABLE_OBJS += reftable/blocksource.o
-REFTABLE_OBJS += reftable/iter.o
-REFTABLE_OBJS += reftable/merged.o
-REFTABLE_OBJS += reftable/pq.o
-REFTABLE_OBJS += reftable/record.o
-REFTABLE_OBJS += reftable/stack.o
-REFTABLE_OBJS += reftable/system.o
-REFTABLE_OBJS += reftable/table.o
-REFTABLE_OBJS += reftable/tree.o
-REFTABLE_OBJS += reftable/writer.o
-
 TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS)) $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS))
 
 .PHONY: test-objs

@@ -2758,7 +2755,7 @@ OBJECTS += $(SCALAR_OBJS)
 OBJECTS += $(PROGRAM_OBJS)
 OBJECTS += $(TEST_OBJS)
 OBJECTS += $(FUZZ_OBJS)
-OBJECTS += $(REFTABLE_OBJS) $(REFTABLE_TEST_OBJS)
+OBJECTS += $(REFTABLE_TEST_OBJS)
 OBJECTS += $(UNIT_TEST_OBJS)
 OBJECTS += $(CLAR_TEST_OBJS)
 OBJECTS += $(patsubst %,$(UNIT_TEST_DIR)/%.o,$(UNIT_TEST_PROGRAMS))

@@ -2910,9 +2907,6 @@ scalar$X: scalar.o GIT-LDFLAGS $(GITLIBS)
 $(LIB_FILE): $(LIB_OBJS)
 	$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^
 
-$(REFTABLE_LIB): $(REFTABLE_OBJS)
-	$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^
-
 export DEFAULT_EDITOR DEFAULT_PAGER
 
 Documentation/GIT-EXCLUDED-PROGRAMS: FORCE

@@ -3751,7 +3745,7 @@ clean: profile-clean coverage-clean cocciclean
 	$(RM) git.rc git.res
 	$(RM) $(OBJECTS)
 	$(RM) headless-git.o
-	$(RM) $(LIB_FILE) $(REFTABLE_LIB)
+	$(RM) $(LIB_FILE)
 	$(RM) $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) $(OTHER_PROGRAMS)
 	$(RM) $(TEST_PROGRAMS)
 	$(RM) $(FUZZ_PROGRAMS)

@@ -3945,7 +3939,6 @@ endif
 
 LIBGIT_PUB_OBJS += contrib/libgit-sys/public_symbol_export.o
 LIBGIT_PUB_OBJS += libgit.a
-LIBGIT_PUB_OBJS += reftable/libreftable.a
 
 LIBGIT_PARTIAL_EXPORT = contrib/libgit-sys/partial_symbol_export.o
 

gitgitgadget

Same comment as previous patch... please extend the commit message to explain why this change is desirable.

Aside from the comment already given about the fact that the proposed log message does not explain any reason why these change are necessary, this step and the previous step are fairly hostile to merging the topic to play well with other topics, especially given that there would be topics in flight that may want to add, remove, or reorder these two existing lists.

I wonder if these could have been arranged like the following instead?

* Drop "REFTABLE_LIB = reftable/libreftable.a" and the target that runs "ar" to mantain that archive.

* Leave "REFTABLE_OBJS += $objects.o" lines alone.

* Add them into LIB_OBJS so that they are included in libgit.a, perhaps a single line like this:

	LIB_OBJS += $(REFTABLE_OBJS)

Wouldn't that have worked equally well for the (unstated) purpose of these two patches without incurring unnecessary risk of mismerges?

Similar arrangement for xdiff.

Thanks.

Like the previous two commits; This one continues the effort to get the Rust compiler to link against libgit.a. Meson already includes the reftable in its libgit.a, but Makefile does not.

The reason why I was trying to get the Rust compiler to link against libgit.a is because I wanted to get Rusts unit testing to work. If the Rust code calls a C function from Git then 'cargo test' needs to know about it.

However I think I'll drop these 3 commits since 'cargo test' doesn't need to be part of the introduction of Rust. It would be nice for make to be able to run Rust unit tests at some point though.

Neither answers my main question, though.

Instead of rolling everything into LIB_OBJS directly, wouldn't it have been much easier to work with if reftable-related ones are left in REFTABLE_OBJS and then RERFTABLE_OBJS gets added to LIB_OBJS? Wouldn't it have been less prone to mismerges to do it that way?

As we can always extend things more, getting something close to the minimally viable set with some tests for sanity checking would be a good first goal. If you pare down way too much, however, we may end up to be pretty close to what Patrick sent out originally with the varint conversion, so let's make sure we do not drop below the minimum that still demonstrates that we have Rust integration that is viable going forward.

Thanks.

I didn't do it that way because I didn't think of that. I prefer your way because it's much cleaner.

I'm torn between A and B from [1]:
I'm thinking of choosing A because my approach and Patrick's approach are incompatible. In order to use multiple crates and ensure that Rust is built and tested the same way with Make and Meson I'd have to rip out lots of things from Patrick's patch series.

But maybe I should go with B because there's a lot of prep work that should be done with Git before I can start doing what I'd like to do in Rust. So let Patrick's stuff merge while I work on cleaning up xdiff and updating Makefile's libgit.a build process and then deal with ugly merging later.

I can't decide between the two.

[1] https://lore.kernel.org/git/CABPp-BHJUkSERQon6xx=sHrhN7i=6ekv+Hz1+P+2mh0=Xw15Mg@mail.gmail.com/

From: "brian m. carlson" <[email protected]>

Git has historically been written primarily in C, with some shell and Perl. However, C is not memory safe, which makes it more likely that security vulnerabilities or other bugs will be introduced, and it is also more verbose and less ergonomic than other, more modern languages.

One of the most common modern compiled languages which is easily interoperable with C is Rust. It is popular (the most admired language on the 2024 Stack Overflow Developer Survey), efficient, portable, and robust.

Introduce a document laying out the incremental introduction of Rust to Git and provide a detailed rationale for doing so, including the points above. Propose a design for this approach that addresses the needs of downstreams and distributors, as well as contributors.

Since we don't want to carry both a C and Rust version of code and want to be able to add new features only in Rust, mention that Rust is a required part of our platform support policy.

It should be noted that a recent discussion at the Berlin Git Merge Contributor Summit found widespread support for the addition of Rust to Git. While of course not all contributors were represented, the proposal appeared to have the support of a majority of active contributors.

[en: Added some comments about types, and changed the recommondations
about cbindgen, bindgen, rustix, libc.]

---
 Documentation/Makefile                        |   1 +
 Documentation/technical/platform-support.adoc |   2 +
 Documentation/technical/rust-support.adoc     | 142 ++++++++++++++++++
 3 files changed, 145 insertions(+)
 create mode 100644 Documentation/technical/rust-support.adoc

index df2ce187eb..5e7821db3c 100644
@@ -127,6 +127,7 @@ TECH_DOCS += technical/parallel-checkout
 TECH_DOCS += technical/partial-clone
 TECH_DOCS += technical/platform-support
 TECH_DOCS += technical/racy-git
+TECH_DOCS += technical/rust-support
 TECH_DOCS += technical/reftable
 TECH_DOCS += technical/scalar
 TECH_DOCS += technical/send-pack-pipeline

index 0a2fb28d62..dc71672dcb 100644
@@ -33,6 +33,8 @@ meet the following minimum requirements:
 
 * Has active security support (taking security releases of dependencies, etc)
 
+* Supports Rust and the toolchain version specified in link:rust-support.adoc[].
+
 These requirements are a starting point, and not sufficient on their own for the
 Git community to be enthusiastic about supporting your platform. Maintainers of
 platforms which do meet these requirements can follow the steps below to make it

gitgitgadget

From: Patrick Steinhardt <[email protected]>

Over the last couple of years the appetite for bringing Rust into the codebase has grown significantly across the developer base. Introducing Rust is a major change though and has ramifications for the whole ecosystem:

- Some platforms have a Rust toolchain available, but have not yet integrated it into their build infrastructure.

- Some platforms don't have any support for Rust at all.

- Some platforms may have to figure out how to fit Rust into their bootstrapping sequence.

Due to this, and given that Git is a critical piece of infrastructure for the whole industry, we cannot just introduce such a heavyweight dependency without doing our due diligence.

Instead, preceding commits have introduced a test balloon into our build infrastructure that convert one tiny subsystem to use Rust. For now, using Rust to build that subsystem is entirely optional -- if no Rust support is available, we continue to use the C implementation. This test balloon has the intention to give distributions time and let them ease into our adoption of Rust.

Having multiple implementations of the same subsystem is not sustainable though, and the plan is to eventually be able to use Rust freely all across our codebase. As such, there is the intent to make Rust become a mandatory part of our build process.

Add an announcement to our breaking changes that Rust will become mandatory in Git 3.0. A (very careful and non-binding) estimate might be that this major release might be released in the second half of next year, which should give distributors enough time to prepare for the change.

---
 Documentation/BreakingChanges.adoc | 35 ++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

index f8d2eba061..56bbd5699e 100644
@@ -165,6 +165,41 @@ A prerequisite for this change is that the ecosystem is ready to support the
 "reftable" format. Most importantly, alternative implementations of Git like
 JGit, libgit2 and Gitoxide need to support it.
 
+* Git will require Rust as a mandatory part of the build process. While Git
+  already started to adopt Rust in Git 2.49, all parts written in Rust are
+  optional for the time being. This includes:
++
+  ** Subsystems that have an alternative implementation in Rust to test
+     interoperability between our C and Rust codebase.
+  ** Newly written features that are not mission critical for a fully functional
+     Git client.
++
+These changes are meant as test balloons to allow distributors of Git to prepare
+for Rust becoming a mandatory part of the build process. There will be multiple
+milestones for the introduction of Rust:
++
+--
+1. In Git 2.52, both build systems will default-enable support for Rust.
+   Consequently, builds will break by default if Rust is not available on the
+   build host. The use of Rust can still be explicitly disabled via build
+   flags.
+2. In Git 3.0, the build options will be removed and support for Rust is
+   mandatory.
+--
++
+Disable building with Rust:
+Meson: `meson configure -Dwith_rust=false`.
+Makefile: `make WITH_RUST=false`,
++
+The Git project will declare the last version before Git 3.0 to be a long-term
+support release. This long-term release will receive important bug fixes for at
+least four release cycles and security fixes for six release cycles. The Git
+project will hand over maintainership of the long-term release to distributors
+in case they need to extend the life of that long-term release even further. In
+that case, the backporting process will be handled by these distributors, but
+the backported patches will be reviewed on the mailing list and pulled in by the
+Git maintainer.
+
 === Removals
 
 * Support for grafting commits has long been superseded by git-replace(1).

gitgitgadget

From: Ezekiel Newren <[email protected]>

Upcoming patches will simplify xdiff, while also porting parts of it to Rust. In preparation, add some stubs and setup the Rust build. For now, it is easier to let cargo build rust and have make or meson merely link against the static library that cargo builds. In line with ongoing libification efforts, use multiple crates to allow more modularity on the Rust side. The xdiff and ivec crates are the focal point for this series.

cbindgen will be used to generate header files for each Rust crate for C interoperability.

---
 .gitignore        |  3 ++
 Makefile          | 81 +++++++++++++++++++++++++++++++++++++++--------
 build_rust.sh     | 60 +++++++++++++++++++++++++++++++++++
 meson.build       | 72 ++++++++++++++++++++++++++++++++++++-----
 meson_options.txt |  2 ++
 rust/Cargo.toml   |  3 ++
 6 files changed, 200 insertions(+), 21 deletions(-)
 create mode 100755 build_rust.sh
 create mode 100644 rust/Cargo.toml

index 04c444404e..24292eb473 100644
@@ -254,3 +254,6 @@ Release/
 /contrib/buildsystems/out
 /contrib/libgit-rs/target
 /contrib/libgit-sys/target
+.idea/
+/rust/target/
+/rust/Cargo.lock

index 1668f260a1..362e928c79 100644
@@ -69,6 +69,8 @@ option('zlib_backend', type: 'combo', choices: ['auto', 'zlib', 'zlib-ng'], valu
   description: 'The backend used for compressing objects and other data.')
 
 # Build tweaks.
+option('with_rust', type: 'boolean', value: true,
+       description: 'Enable building with Rust, true by default.')
 option('breaking_changes', type: 'boolean', value: false,
   description: 'Enable upcoming breaking changes.')
 option('macos_use_homebrew_gettext', type: 'boolean', value: true,

new file mode 100644
index 0000000000..9bc2f769fb
@@ -0,0 +1,3 @@
+[workspace]
+members = []
+resolver = "2"

gitgitgadget

I knew that the above was our official stance but somehow thought that we loosened at some point to add common ones like *~ and *.sw[op] to make life simpler for majority of developers. But I was wrong. We do not even have *~, which means we haven't officially loosened.

But there are oddballs that violate this policy like ".cache" introduced by a5c01603 (gitignore: ignore clangd .cache directory, 2023-08-04). Three are many other that are *not* our droppings, between /.vscode/ to /contrib/buildsystems/out in .gitignore file.

    /.vscode/
    /tags
    /TAGS
    /cscope*
    /compile_commands.json
    /.cache/
    *.hcc
    *.obj
    *.lib
    *.sln
    *.sp
    *.suo
    *.ncb
    *.vcproj
    *.user
    *.idb
    *.pdb
    *.ilk
    *.iobj
    *.ipdb
    *.dll
    .vs/
    Debug/
    Release/
    /UpgradeLog*.htm
    /git.VC.VC.opendb
    /git.VC.db
    *.dSYM

Some (like TAGS and *.hcc) are our droppings (in other words, what "make" with some build targets may produce), but most of these are tool specific and according to our original official stance, they should never have been added, but there they are.

I actually do not mind having common ones to the project .gitignore as long as it does not get bloated too much with droppings from esoteric tools that majority of us have never heard of. It seems that we have been punishing needlessly Emacs and vim users while being sloppy about others' droppings. A #leftoverbit may be to have a brief discussion to gain consensus and add a few common ones and/or remove too esoteric ones? I dunno.

Good suggestion. This reminds me of my favorite gripe, which is that "make clean" makes no effort to be quiet by default; perhaps leaving a #leftover bit marker here would help?

Thanks.

It's been a while since I had to build code with Visual Studio (the proprietary Microsoft product, not the open-source VScode), but if I recall correctly, all the entries from ".obj" through "Release/" are build detritus from compiling with that tool. Assuming we still support building Git with Visual Studio (which I believe is the case), then those entries all fall within the same categorization as "our droppings" similar to the make case, so having these in ".gitignore" is probably in line with the project's official stance.

Others, such as "/.vscode/", on the other hand, fall into the other category of "someone's favorite editor or handy tool", which has thus far been frowned upon.

I don't have a strong opinion aside from avoiding bloat; I've seen (and inherited at $DAYJOBS) far too many projects which grab some overly bloated .gitignore template from somewhere in which most of the entries are meaningless for the project at hand, yet which is used as-is rather than pruning out the unneeded entries (typically >95% of them). As with dead code, those unneeded .gitignore entries tend to be a source of potential confusion, which is rarely or never the case with a well-curated .gitignore (or with well-curated code).

That said, I also probably do not mind having the common ones in Git's .gitignore. A few which come to mind include:

* editor-specific droppings (Emacs, vi / vim, VScode)
* .DS_Store files which macOS's Finder drops into *every* directory it visits, thus which litter the filesystem

Yes. I use the Jetbrains IDE's CLion and RustRover for C and Rust respectively. Jetbrains has an IDE for MANY languages and all of them use .idea/ as the folder for IDE specific configuration. I'm fine with keeping it out of .gitignore, but I wanted to know what the community thought. Which I should have made it more clear that this patch series was more of an RFC than proposed code to be merged.

I'll try to keep these in mind for the future.

I will amend the commit with something like:
On windows when building with msvc using shell scripts it looks for link in /mingw64/bin|/usr/bin when it actually needs to look somewhere else for the msvc linker program. Since removing these from PATH would break everything else in the shell; move them to be at the end of PATH.

pwd was for debugging and I forgot to take it out. I'll clean up my script.

There is a bit of discussion about this later in this same email thread. If the project does ultimately decide to accept these .gitignore entries, adding them would be done via a patch or series of patches specifically aimed at that goal. Hence, I'd recommend omitting the ".idea" entry from this particular patch series.

I had to read and reread this several times but I think I get what it is saying. To paraphrase your explanation...

When building with cargo (I presume), and it comes time to link the program, the build process is looking for the Microsoft linker named LINK.exe but, due to PATH order, is instead finding the Unix command link (which is a specialized invocation of the more common ln command). As such, the build process incorrectly invokes the Unix link rather than the Microsoft LINK.exe and fails. To work around this problem, you move the standard Unix command-containing paths to the end of PATH so that the Microsoft LINK.exe is found first.

...does that sound correct?

Your understanding is correct.

From: Patrick Steinhardt <[email protected]>

We're about to introduce support for Rust into the core of Git, where some (trivial) subsystems are converted to Rust. These subsystems will also retain a C implementation though as Rust is not yet mandatory. Consequently, it now becomes possible for a Git version to have bugs that are specific to whether or not it is built with Rust support overall.

Expose information about whether or not Git was built with Rust via our build info. This means that both git version --build-options, but also git bugreport will now expose that bit of information. Hopefully, this should make it easier for us to discover any Rust-specific issues.

---
 help.c | 6 ++++++
 1 file changed, 6 insertions(+)

index bb20498cfd..5854dd4a7e 100644
@@ -791,6 +791,12 @@ void get_version_info(struct strbuf *buf, int show_build_options)
 		strbuf_addf(buf, "shell-path: %s\n", SHELL_PATH);
 		/* NEEDSWORK: also save and output GIT-BUILD_OPTIONS? */
 
+#if defined WITH_RUST
+		strbuf_addstr(buf, "rust: enabled\n");
+#else
+		strbuf_addstr(buf, "rust: disabled\n");
+#endif
+
 		if (fsmonitor_ipc__is_supported())
 			strbuf_addstr(buf, "feature: fsmonitor--daemon\n");
 #if defined LIBCURL_VERSION

gitgitgadget

From: Ezekiel Newren <[email protected]>

Prefer using actions-rs/toolchain@v1 where possible to install rustup, but for docker targets use a script to install rustup. Consolidate the Rust toolchain definitions in main.yaml. Use install-rust-toolchain.sh to ensure the correct toolchain is used. Five overrides are used in main.yaml:

* On Windows: Rust didn't resolve the bcrypt library on Windows correctly until version 1.78.0. Also since rustup mis-identifies the Rust toolchain, the Rust target triple must be set to x86_64-pc-windows-gnu for make (win build), and x86_64-pc-windows-msvc for meson (win+Meson build).
* MSVC builds: Rearrange PATH to look in /mingw64/bin and /usr/bin last.
* On musl: libc differences, such as ftruncate64 vs ftruncate, were not accounted for until Rust version 1.72.0. No older version of Rust will work on musl for our needs.
* In a 32-bit docker container running on a 64-bit host, we need to override the Rust target triple. This is because rustup asks the kernel for the bitness of the system and it says 64, even though the container is 32-bit. This also allows us to remove the BITNESS environment variable in ci/lib.sh.

The logic for selecting library names was initially provided in a patch from Johannes, but was reworked and squashed into this commit.

---
 .github/workflows/main.yml   | 62 +++++++++++++++++++++++++++++++++++-
 ci/install-dependencies.sh   | 14 ++++----
 ci/install-rust-toolchain.sh | 30 +++++++++++++++++
 ci/install-rustup.sh         | 25 +++++++++++++++
 ci/lib.sh                    |  1 +
 ci/make-test-artifacts.sh    |  9 ++++++
 ci/run-build-and-tests.sh    | 13 ++++++++
 7 files changed, 146 insertions(+), 8 deletions(-)
 create mode 100755 ci/install-rust-toolchain.sh
 create mode 100755 ci/install-rustup.sh

index d122e79415..e986684a2f 100644
@@ -26,6 +26,13 @@ jobs:
     outputs:
       enabled: ${{ steps.check-ref.outputs.enabled }}${{ steps.skip-if-redundant.outputs.enabled }}
       skip_concurrent: ${{ steps.check-ref.outputs.skip_concurrent }}
+      rust_version_minimum: 1.63.0
+      rust_version_windows: 1.78.0
+      rust_version_musl: 1.72.0
+      ## the rust target is inferred by rustup unless specified
+      rust_target_windows_make: x86_64-pc-windows-gnu
+      rust_target_windows_meson: x86_64-pc-windows-msvc
+      rust_target_32bit_linux: i686-unknown-linux-gnu
     steps:
       - name: try to clone ci-config branch
         run: |

@@ -108,12 +115,28 @@ jobs:
     needs: ci-config
     if: needs.ci-config.outputs.enabled == 'yes'
     runs-on: windows-latest
+    env:
+      CARGO_HOME: "/c/Users/runneradmin/.cargo"
     concurrency:
       group: windows-build-${{ github.ref }}
       cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }}
     steps:
     - uses: actions/checkout@v4
     - uses: git-for-windows/setup-git-for-windows-sdk@v1
+      with:
+        flavor: full
+    - name: Install rustup via github actions
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        profile: minimal
+        override: false
+    - name: Install Rust toolchain
+      shell: bash
+      env:
+        RUST_VERSION: ${{ needs.ci-config.outputs.rust_version_windows }}
+        RUST_TARGET: ${{ needs.ci-config.outputs.rust_target_windows_make }}
+      run: ci/install-rust-toolchain.sh
     - name: build
       shell: bash
       env:

@@ -254,12 +277,26 @@ jobs:
     needs: ci-config
     if: needs.ci-config.outputs.enabled == 'yes'
     runs-on: windows-latest
+    env:
+      CARGO_HOME: "/c/Users/runneradmin/.cargo"
     concurrency:
       group: windows-meson-build-${{ github.ref }}
       cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }}
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
+    - name: Install rustup via github actions
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        profile: minimal
+        override: false
+    - name: Install Rust toolchain
+      shell: bash
+      env:
+        RUST_VERSION: ${{ needs.ci-config.outputs.rust_version_windows }}
+        RUST_TARGET: ${{ needs.ci-config.outputs.rust_target_windows_meson }}
+      run: ci/install-rust-toolchain.sh
     - name: Set up dependencies
       shell: pwsh
       run: pip install meson ninja

@@ -329,11 +366,24 @@ jobs:
       jobname: ${{matrix.vector.jobname}}
       CI_JOB_IMAGE: ${{matrix.vector.pool}}
       TEST_OUTPUT_DIRECTORY: ${{github.workspace}}/t
+      CARGO_HOME: "/Users/runner/.cargo"
     runs-on: ${{matrix.vector.pool}}
     steps:
     - uses: actions/checkout@v4
     - run: ci/install-dependencies.sh
-    - run: ci/run-build-and-tests.sh
+    - name: Install rustup via github actions
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        profile: minimal
+        override: false
+    - name: Install Rust toolchain
+      shell: bash
+      env:
+        RUST_VERSION: ${{ needs.ci-config.outputs.rust_version_minimum }}
+      run: ci/install-rust-toolchain.sh
+    - name: Run build and tests
+      run: ci/run-build-and-tests.sh
     - name: print test failures
       if: failure() && env.FAILED_TEST_ARTIFACTS != ''
       run: ci/print-test-failures.sh

@@ -393,9 +443,11 @@ jobs:
           cc: gcc
         - jobname: linux-musl-meson
           image: alpine:latest
+          rust_version_override: ${{ needs.ci-config.outputs.rust_version_musl }}
         # Supported until 2025-04-02.
         - jobname: linux32
           image: i386/ubuntu:focal
+          rust_target_override: ${{ needs.ci-config.outputs.rust_target_32bit_linux }}
         - jobname: pedantic
           image: fedora:latest
         # A RHEL 8 compatible distro.  Supported until 2029-05-31.

@@ -409,6 +461,7 @@ jobs:
       CC: ${{matrix.vector.cc}}
       CI_JOB_IMAGE: ${{matrix.vector.image}}
       CUSTOM_PATH: /custom
+      CARGO_HOME: /home/builder/.cargo
     runs-on: ubuntu-latest
     container: ${{matrix.vector.image}}
     steps:

@@ -433,6 +486,13 @@ jobs:
     - run: ci/install-dependencies.sh
     - run: useradd builder --create-home
     - run: chown -R builder .
+    - name: Install rustup via script
+      run: sudo --preserve-env --set-home --user=builder ci/install-rustup.sh
+    - name: Install Rust toolchain
+      env:
+        RUST_VERSION: ${{ matrix.vector.rust_version_override || needs.ci-config.outputs.rust_version_minimum }}
+        RUST_TARGET: ${{ matrix.vector.rust_target_override || '' }}
+      run: sudo --preserve-env --set-home --user=builder ci/install-rust-toolchain.sh
     - run: sudo --preserve-env --set-home --user=builder ci/run-build-and-tests.sh
     - name: print test failures
       if: failure() && env.FAILED_TEST_ARTIFACTS != ''

index d061a47293..7801075821 100755
@@ -24,14 +24,14 @@ fi
 
 case "$distro" in
 alpine-*)
-	apk add --update shadow sudo meson ninja-build gcc libc-dev curl-dev openssl-dev expat-dev gettext \
+	apk add --update shadow sudo meson ninja-build gcc libc-dev curl curl-dev openssl-dev expat-dev gettext \
 		zlib-ng-dev pcre2-dev python3 musl-libintl perl-utils ncurses \
 		apache2 apache2-http2 apache2-proxy apache2-ssl apache2-webdav apr-util-dbd_sqlite3 \
 		bash cvs gnupg perl-cgi perl-dbd-sqlite perl-io-tty >/dev/null
 	;;
 fedora-*|almalinux-*)
 	dnf -yq update >/dev/null &&
-	dnf -yq install shadow-utils sudo make gcc findutils diffutils perl python3 gawk gettext zlib-devel expat-devel openssl-devel curl-devel pcre2-devel >/dev/null
+	dnf -yq install shadow-utils sudo make gcc findutils diffutils perl python3 gawk gettext zlib-devel expat-devel openssl-devel curl curl-devel pcre2-devel >/dev/null
 	;;
 ubuntu-*|i386/ubuntu-*|debian-*)
 	# Required so that apt doesn't wait for user input on certain packages.

@@ -55,8 +55,8 @@ ubuntu-*|i386/ubuntu-*|debian-*)
 	sudo apt-get -q update
 	sudo apt-get -q -y install \
 		$LANGUAGES apache2 cvs cvsps git gnupg $SVN \
-		make libssl-dev libcurl4-openssl-dev libexpat-dev wget sudo default-jre \
-		tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl \
+		make libssl-dev curl libcurl4-openssl-dev libexpat-dev wget sudo default-jre \
+		tcl tk gettext zlib1g zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl \
 		libemail-valid-perl libio-pty-perl libio-socket-ssl-perl libnet-smtp-ssl-perl libdbd-sqlite3-perl libcgi-pm-perl \
 		libsecret-1-dev libpcre2-dev meson ninja-build pkg-config \
 		${CC_PACKAGE:-${CC:-gcc}} $PYTHON_PACKAGE

@@ -121,13 +121,13 @@ ClangFormat)
 	;;
 StaticAnalysis)
 	sudo apt-get -q update
-	sudo apt-get -q -y install coccinelle libcurl4-openssl-dev libssl-dev \
+	sudo apt-get -q -y install coccinelle curl libcurl4-openssl-dev libssl-dev \
 		libexpat-dev gettext make
 	;;
 sparse)
 	sudo apt-get -q update -q
-	sudo apt-get -q -y install libssl-dev libcurl4-openssl-dev \
-		libexpat-dev gettext zlib1g-dev sparse
+	sudo apt-get -q -y install libssl-dev curl libcurl4-openssl-dev \
+		libexpat-dev gettext zlib1g zlib1g-dev sparse
 	;;
 Documentation)
 	sudo apt-get -q update

new file mode 100755
index 0000000000..06a29c4cfa
@@ -0,0 +1,30 @@
+#!/bin/sh
+
+if [ "$CARGO_HOME" = "" ]; then
+  echo >&2 "::error:: CARGO_HOME is not set"
+  exit 2
+fi
+export PATH="$CARGO_HOME/bin:$PATH"
+rustup -vV || exit $?
+
+## Enforce the correct Rust toolchain
+rustup override unset || true
+
+## install a specific version of rust
+if [ "$RUST_TARGET" != "" ]; then
+  rustup default --force-non-host "$RUST_VERSION-$RUST_TARGET" || exit $?
+else
+  rustup default "$RUST_VERSION" || exit $?
+fi
+
+rustc -vV || exit $?
+
+RE_RUST_TARGET="$RUST_TARGET"
+if [ "$RUST_TARGET" = "" ]; then
+  RE_RUST_TARGET="[^ ]+"
+fi
+
+if ! rustup show active-toolchain | grep -E "^$RUST_VERSION-$RE_RUST_TARGET \(default\)$"; then
+  echo >&2 "::error:: wrong Rust toolchain, active-toolchain: $(rustup show active-toolchain)"
+  exit 3
+fi

new file mode 100755
index 0000000000..0036231aee
@@ -0,0 +1,25 @@
+#!/bin/sh
+
+## github workflows actions-rs/toolchain@v1 doesn't work for docker
+## targets. This script should only be used if the ci pipeline
+## doesn't support installing rust on a particular target.
+
+if [ "$(id -u)" -eq 0 ]; then
+  echo >&2 "::warning:: installing rust as root"
+fi
+
+if [ "$CARGO_HOME" = "" ]; then
+  echo >&2 "::error:: CARGO_HOME is not set"
+  exit 2
+fi
+
+export RUSTUP_HOME=$CARGO_HOME
+
+## install rustup
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain none -y
+if [ ! -f $CARGO_HOME/env ]; then
+  echo "PATH=$CARGO_HOME/bin:\$PATH" > $CARGO_HOME/env
+fi
+. $CARGO_HOME/env
+
+rustup -vV

index f561884d40..a7992b22fd 100755
@@ -1,5 +1,6 @@
 # Library of functions shared by all CI scripts
 
+
 if test true = "$GITHUB_ACTIONS"
 then
 	begin_group () {