Merge pull request #7117 from pks-gitlab/pks/reftables-support

Reftables support

.github/workflows/main.yml

@@ -122,6 +122,43 @@ jobs:
             SKIP_SSH_TESTS: true
             SKIP_NEGOTIATE_TESTS: true
 
+        # All builds: reftable
+        - name: "Linux (Noble, GCC, OpenSSL, libssh2, reftable)"
+          id: noble-gcc-openssl-reftable
+          os: ubuntu-latest
+          container:
+            name: noble
+          env:
+            CC: gcc
+            CLAR_REF_FORMAT: reftable
+            CMAKE_GENERATOR: Ninja
+            CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON
+        - name: "macOS (reftable)"
+          id: macos-reftable
+          os: macos-14
+          setup-script: osx
+          env:
+            CC: clang
+            CLAR_REF_FORMAT: reftable
+            CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON
+            CMAKE_GENERATOR: Ninja
+            PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig
+            SKIP_SSH_TESTS: true
+            SKIP_NEGOTIATE_TESTS: true
+        - name: "Windows (amd64, Visual Studio, Schannel, reftable)"
+          id: windows-amd64-vs-reftable
+          os: windows-2022
+          setup-script: win32
+          env:
+            ARCH: amd64
+            CLAR_REF_FORMAT: reftable
+            CMAKE_GENERATOR: Visual Studio 17 2022
+            CMAKE_OPTIONS: -A x64 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2
+            BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin
+            BUILD_TEMP: D:\Temp
+            SKIP_SSH_TESTS: true
+            SKIP_NEGOTIATE_TESTS: true
+
         # All builds: sanitizers
         - name: "Sanitizer (Memory)"
           id: sanitizer-memory

deps/clar/clar_libgit2.c

@@ -172,13 +172,42 @@ static git_repository *_cl_repo = NULL;
 
 git_repository *cl_git_sandbox_init(const char *sandbox)
 {
-	/* Get the name of the sandbox folder which will be created */
-	const char *basename = cl_fixture_basename(sandbox);
+	const char *basename;
+	char *ref_format;
 
-	/* Copy the whole sandbox folder from our fixtures to our test sandbox
-	 * area.  After this it can be accessed with `./sandbox`
+	/* Get the name of the sandbox folder which will be created */
+	basename = cl_fixture_basename(sandbox);
+
+	ref_format = cl_getenv("CLAR_REF_FORMAT");
+	if (!ref_format || !strcmp(ref_format, "files")) {
+		/*
+		 * Copy the whole sandbox folder from our fixtures to our
+		 * test sandbox area. After this it can be accessed with
+		 * `./sandbox`
 		 */
 		cl_fixture_sandbox(sandbox);
+	} else if (!strcmp(ref_format, "reftable")) {
+		struct git_str reftable_sandbox = GIT_STR_INIT;
+
+		cl_git_pass(git_str_joinpath(&reftable_sandbox, "reftable", sandbox));
+		if (!git_fs_path_isdir(cl_fixture(reftable_sandbox.ptr))) {
+			git_str_dispose(&reftable_sandbox);
+			_cl_sandbox = NULL;
+			_cl_repo = NULL;
+			cl_fail("no seed for reftable repository");
+		}
+
+		/*
+		 * Copy over the sandbox and rename it so that the basename
+		 * matches the originally requested basename.
+		 */
+		cl_fixture_sandbox(reftable_sandbox.ptr);
+		git_str_dispose(&reftable_sandbox);
+	} else {
+		cl_fail("Unexpected ref format");
+	}
+
+	git__free(ref_format);
 	_cl_sandbox = sandbox;
 
 	cl_git_pass(p_chdir(basename));

deps/reftable/CMakeLists.txt

@@ -0,0 +1,29 @@
+add_library(reftable OBJECT ${SRC_REFTABLE})
+
+disable_warnings(implicit-fallthrough)
+
+file(GLOB SRC_REFTABLE "*.c" "*.h")
+list(SORT SRC_REFTABLE)
+
+set_property(TARGET reftable PROPERTY C_STANDARD 99)
+target_sources(reftable PRIVATE ${SRC_REFTABLE})
+target_include_directories(reftable PRIVATE
+	include
+	"${PROJECT_BINARY_DIR}/src/util"
+	"${PROJECT_BINARY_DIR}/include"
+	"${PROJECT_SOURCE_DIR}/src/util"
+	"${PROJECT_SOURCE_DIR}/include"
+	${LIBGIT2_DEPENDENCY_INCLUDES}
+	${LIBGIT2_SYSTEM_INCLUDES}
+)
+
+# The reftable library is not warning-free, so we disable turning warnings into
+# errors.
+if(MSVC)
+        set_source_files_properties(block.c PROPERTIES COMPILE_FLAGS -WX-)
+        set_source_files_properties(blocksource.c PROPERTIES COMPILE_FLAGS -WX-)
+        set_source_files_properties(record.c PROPERTIES COMPILE_FLAGS -WX-)
+        set_source_files_properties(stack.c PROPERTIES COMPILE_FLAGS -WX-)
+        set_source_files_properties(table.c PROPERTIES COMPILE_FLAGS -WX-)
+        set_source_files_properties(writer.c PROPERTIES COMPILE_FLAGS -WX-)
+endif()

deps/reftable/LICENSE

@@ -0,0 +1,31 @@
+BSD License
+
+Copyright (c) 2020, Google LLC
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+* Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+* Neither the name of Google LLC nor the names of its contributors may
+be used to endorse or promote products derived from this software
+without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

deps/reftable/basics.c

@@ -0,0 +1,278 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#define REFTABLE_ALLOW_BANNED_ALLOCATORS
+#include "basics.h"
+#include "reftable-basics.h"
+#include "reftable-error.h"
+
+static void *(*reftable_malloc_ptr)(size_t sz);
+static void *(*reftable_realloc_ptr)(void *, size_t);
+static void (*reftable_free_ptr)(void *);
+
+void *reftable_malloc(size_t sz)
+{
+	if (!sz)
+		return NULL;
+	if (reftable_malloc_ptr)
+		return (*reftable_malloc_ptr)(sz);
+	return malloc(sz);
+}
+
+void *reftable_realloc(void *p, size_t sz)
+{
+	if (!sz) {
+		reftable_free(p);
+		return NULL;
+	}
+
+	if (reftable_realloc_ptr)
+		return (*reftable_realloc_ptr)(p, sz);
+	return realloc(p, sz);
+}
+
+void reftable_free(void *p)
+{
+	if (reftable_free_ptr)
+		reftable_free_ptr(p);
+	else
+		free(p);
+}
+
+void *reftable_calloc(size_t nelem, size_t elsize)
+{
+	void *p;
+
+	if (nelem && elsize > SIZE_MAX / nelem)
+		return NULL;
+
+	p = reftable_malloc(nelem * elsize);
+	if (!p)
+		return NULL;
+
+	memset(p, 0, nelem * elsize);
+	return p;
+}
+
+char *reftable_strdup(const char *str)
+{
+	size_t len = strlen(str);
+	char *result = reftable_malloc(len + 1);
+	if (!result)
+		return NULL;
+	memcpy(result, str, len + 1);
+	return result;
+}
+
+void reftable_set_alloc(void *(*malloc)(size_t),
+			void *(*realloc)(void *, size_t), void (*free)(void *))
+{
+	reftable_malloc_ptr = malloc;
+	reftable_realloc_ptr = realloc;
+	reftable_free_ptr = free;
+}
+
+void reftable_buf_init(struct reftable_buf *buf)
+{
+	struct reftable_buf empty = REFTABLE_BUF_INIT;
+	*buf = empty;
+}
+
+void reftable_buf_release(struct reftable_buf *buf)
+{
+	reftable_free(buf->buf);
+	reftable_buf_init(buf);
+}
+
+void reftable_buf_reset(struct reftable_buf *buf)
+{
+	if (buf->alloc) {
+		buf->len = 0;
+		buf->buf[0] = '\0';
+	}
+}
+
+int reftable_buf_setlen(struct reftable_buf *buf, size_t len)
+{
+	if (len > buf->len)
+		return -1;
+	if (len == buf->len)
+		return 0;
+	buf->buf[len] = '\0';
+	buf->len = len;
+	return 0;
+}
+
+int reftable_buf_cmp(const struct reftable_buf *a, const struct reftable_buf *b)
+{
+	size_t len = a->len < b->len ? a->len : b->len;
+	if (len) {
+		int cmp = memcmp(a->buf, b->buf, len);
+		if (cmp)
+			return cmp;
+	}
+	return a->len < b->len ? -1 : a->len != b->len;
+}
+
+int reftable_buf_add(struct reftable_buf *buf, const void *data, size_t len)
+{
+	size_t newlen = buf->len + len;
+
+	if (newlen + 1 > buf->alloc) {
+		if (REFTABLE_ALLOC_GROW(buf->buf, newlen + 1, buf->alloc))
+			return REFTABLE_OUT_OF_MEMORY_ERROR;
+	}
+
+	memcpy(buf->buf + buf->len, data, len);
+	buf->buf[newlen] = '\0';
+	buf->len = newlen;
+
+	return 0;
+}
+
+int reftable_buf_addstr(struct reftable_buf *buf, const char *s)
+{
+	return reftable_buf_add(buf, s, strlen(s));
+}
+
+char *reftable_buf_detach(struct reftable_buf *buf)
+{
+	char *result = buf->buf;
+	reftable_buf_init(buf);
+	return result;
+}
+
+size_t binsearch(size_t sz, int (*f)(size_t k, void *args), void *args)
+{
+	size_t lo = 0;
+	size_t hi = sz;
+
+	/* Invariants:
+	 *
+	 *  (hi == sz) || f(hi) == true
+	 *  (lo == 0 && f(0) == true) || fi(lo) == false
+	 */
+	while (hi - lo > 1) {
+		size_t mid = lo + (hi - lo) / 2;
+		int ret = f(mid, args);
+		if (ret < 0)
+			return sz;
+
+		if (ret > 0)
+			hi = mid;
+		else
+			lo = mid;
+	}
+
+	if (lo)
+		return hi;
+
+	return f(0, args) ? 0 : 1;
+}
+
+void free_names(char **a)
+{
+	char **p;
+	if (!a) {
+		return;
+	}
+	for (p = a; *p; p++) {
+		reftable_free(*p);
+	}
+	reftable_free(a);
+}
+
+size_t names_length(const char **names)
+{
+	const char **p = names;
+	while (*p)
+		p++;
+	return p - names;
+}
+
+int parse_names(char *buf, int size, char ***out)
+{
+	char **names = NULL;
+	size_t names_cap = 0;
+	size_t names_len = 0;
+	char *p = buf;
+	char *end = buf + size;
+	int err = 0;
+
+	while (p < end) {
+		char *next = strchr(p, '\n');
+		if (!next) {
+			err = REFTABLE_FORMAT_ERROR;
+			goto done;
+		} else if (next < end) {
+			*next = '\0';
+		} else {
+			next = end;
+		}
+
+		if (p < next) {
+			if (REFTABLE_ALLOC_GROW(names, names_len + 1,
+						names_cap)) {
+				err = REFTABLE_OUT_OF_MEMORY_ERROR;
+				goto done;
+			}
+
+			names[names_len] = reftable_strdup(p);
+			if (!names[names_len++]) {
+				err = REFTABLE_OUT_OF_MEMORY_ERROR;
+				goto done;
+			}
+		}
+		p = next + 1;
+	}
+
+	if (REFTABLE_ALLOC_GROW(names, names_len + 1, names_cap)) {
+		err = REFTABLE_OUT_OF_MEMORY_ERROR;
+		goto done;
+	}
+	names[names_len] = NULL;
+
+	*out = names;
+	return 0;
+done:
+	for (size_t i = 0; i < names_len; i++)
+		reftable_free(names[i]);
+	reftable_free(names);
+	return err;
+}
+
+int names_equal(const char **a, const char **b)
+{
+	size_t i = 0;
+	for (; a[i] && b[i]; i++)
+		if (strcmp(a[i], b[i]))
+			return 0;
+	return a[i] == b[i];
+}
+
+size_t common_prefix_size(struct reftable_buf *a, struct reftable_buf *b)
+{
+	size_t p = 0;
+	for (; p < a->len && p < b->len; p++)
+		if (a->buf[p] != b->buf[p])
+			break;
+	return p;
+}
+
+uint32_t hash_size(enum reftable_hash id)
+{
+	if (!id)
+		return REFTABLE_HASH_SIZE_SHA1;
+	switch (id) {
+	case REFTABLE_HASH_SHA1:
+		return REFTABLE_HASH_SIZE_SHA1;
+	case REFTABLE_HASH_SHA256:
+		return REFTABLE_HASH_SIZE_SHA256;
+	}
+	abort();
+}

deps/reftable/basics.h

@@ -0,0 +1,291 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef BASICS_H
+#define BASICS_H
+
+/*
+ * miscellaneous utilities that are not provided by Git.
+ */
+
+#include "system.h"
+#include "reftable-basics.h"
+
+#ifdef __GNUC__
+#define REFTABLE_UNUSED __attribute__((__unused__))
+#else
+#define REFTABLE_UNUSED
+#endif
+
+/*
+ * Initialize the buffer such that it is ready for use. This is equivalent to
+ * using REFTABLE_BUF_INIT for stack-allocated variables.
+ */
+void reftable_buf_init(struct reftable_buf *buf);
+
+/*
+ * Release memory associated with the buffer. The buffer is reinitialized such
+ * that it can be reused for subsequent operations.
+ */
+void reftable_buf_release(struct reftable_buf *buf);
+
+/*
+ * Reset the buffer such that it is effectively empty, without releasing the
+ * memory that this structure holds on to. This is equivalent to calling
+ * `reftable_buf_setlen(buf, 0)`.
+ */
+void reftable_buf_reset(struct reftable_buf *buf);
+
+/*
+ * Trim the buffer to a shorter length by updating the `len` member and writing
+ * a NUL byte to `buf[len]`. Returns 0 on success, -1 when `len` points outside
+ * of the array.
+ */
+int reftable_buf_setlen(struct reftable_buf *buf, size_t len);
+
+/*
+ * Lexicographically compare the two buffers. Returns 0 when both buffers have
+ * the same contents, -1 when `a` is lexicographically smaller than `b`, and 1
+ * otherwise.
+ */
+int reftable_buf_cmp(const struct reftable_buf *a, const struct reftable_buf *b);
+
+/*
+ * Append `len` bytes from `data` to the buffer. This function works with
+ * arbitrary byte sequences, including ones that contain embedded NUL
+ * characters. As such, we use `void *` as input type. Returns 0 on success,
+ * REFTABLE_OUT_OF_MEMORY_ERROR on allocation failure.
+ */
+int reftable_buf_add(struct reftable_buf *buf, const void *data, size_t len);
+
+/* Equivalent to `reftable_buf_add(buf, s, strlen(s))`. */
+int reftable_buf_addstr(struct reftable_buf *buf, const char *s);
+
+/*
+ * Detach the buffer from the structure such that the underlying memory is now
+ * owned by the caller. The buffer is reinitialized such that it can be reused
+ * for subsequent operations.
+ */
+char *reftable_buf_detach(struct reftable_buf *buf);
+
+/* Bigendian en/decoding of integers */
+
+static inline void reftable_put_be16(void *out, uint16_t i)
+{
+	unsigned char *p = out;
+	p[0] = (uint8_t)((i >> 8) & 0xff);
+	p[1] = (uint8_t)((i >> 0) & 0xff);
+}
+
+static inline void reftable_put_be24(void *out, uint32_t i)
+{
+	unsigned char *p = out;
+	p[0] = (uint8_t)((i >> 16) & 0xff);
+	p[1] = (uint8_t)((i >>  8) & 0xff);
+	p[2] = (uint8_t)((i >>  0) & 0xff);
+}
+
+static inline void reftable_put_be32(void *out, uint32_t i)
+{
+	unsigned char *p = out;
+	p[0] = (uint8_t)((i >> 24) & 0xff);
+	p[1] = (uint8_t)((i >> 16) & 0xff);
+	p[2] = (uint8_t)((i >>  8) & 0xff);
+	p[3] = (uint8_t)((i >>  0) & 0xff);
+}
+
+static inline void reftable_put_be64(void *out, uint64_t i)
+{
+	unsigned char *p = out;
+	p[0] = (uint8_t)((i >> 56) & 0xff);
+	p[1] = (uint8_t)((i >> 48) & 0xff);
+	p[2] = (uint8_t)((i >> 40) & 0xff);
+	p[3] = (uint8_t)((i >> 32) & 0xff);
+	p[4] = (uint8_t)((i >> 24) & 0xff);
+	p[5] = (uint8_t)((i >> 16) & 0xff);
+	p[6] = (uint8_t)((i >>  8) & 0xff);
+	p[7] = (uint8_t)((i >>  0) & 0xff);
+}
+
+static inline uint16_t reftable_get_be16(const void *in)
+{
+	const unsigned char *p = in;
+	return (uint16_t)(p[0]) << 8 |
+	       (uint16_t)(p[1]) << 0;
+}
+
+static inline uint32_t reftable_get_be24(const void *in)
+{
+	const unsigned char *p = in;
+	return (uint32_t)(p[0]) << 16 |
+	       (uint32_t)(p[1]) << 8 |
+	       (uint32_t)(p[2]) << 0;
+}
+
+static inline uint32_t reftable_get_be32(const void *in)
+{
+	const unsigned char *p = in;
+	return (uint32_t)(p[0]) << 24 |
+	       (uint32_t)(p[1]) << 16 |
+	       (uint32_t)(p[2]) <<  8|
+	       (uint32_t)(p[3]) <<  0;
+}
+
+static inline uint64_t reftable_get_be64(const void *in)
+{
+	const unsigned char *p = in;
+	return (uint64_t)(p[0]) << 56 |
+	       (uint64_t)(p[1]) << 48 |
+	       (uint64_t)(p[2]) << 40 |
+	       (uint64_t)(p[3]) << 32 |
+	       (uint64_t)(p[4]) << 24 |
+	       (uint64_t)(p[5]) << 16 |
+	       (uint64_t)(p[6]) <<  8 |
+	       (uint64_t)(p[7]) <<  0;
+}
+
+/*
+ * find smallest index i in [0, sz) at which `f(i) > 0`, assuming that f is
+ * ascending. Return sz if `f(i) == 0` for all indices. The search is aborted
+ * and `sz` is returned in case `f(i) < 0`.
+ *
+ * Contrary to bsearch(3), this returns something useful if the argument is not
+ * found.
+ */
+size_t binsearch(size_t sz, int (*f)(size_t k, void *args), void *args);
+
+/*
+ * Frees a NULL terminated array of malloced strings. The array itself is also
+ * freed.
+ */
+void free_names(char **a);
+
+/*
+ * Parse a newline separated list of names. `size` is the length of the buffer,
+ * without terminating '\0'. Empty names are discarded.
+ *
+ * Returns 0 on success, a reftable error code on error.
+ */
+int parse_names(char *buf, int size, char ***out);
+
+/* compares two NULL-terminated arrays of strings. */
+int names_equal(const char **a, const char **b);
+
+/* returns the array size of a NULL-terminated array of strings. */
+size_t names_length(const char **names);
+
+/* Allocation routines; they invoke the functions set through
+ * reftable_set_alloc() */
+void *reftable_malloc(size_t sz);
+void *reftable_realloc(void *p, size_t sz);
+void reftable_free(void *p);
+void *reftable_calloc(size_t nelem, size_t elsize);
+char *reftable_strdup(const char *str);
+
+static inline int reftable_alloc_size(size_t nelem, size_t elsize, size_t *out)
+{
+	if (nelem && elsize > SIZE_MAX / nelem)
+		return -1;
+	*out = nelem * elsize;
+	return 0;
+}
+
+#define REFTABLE_ALLOC_ARRAY(x, alloc) do { \
+		size_t alloc_size; \
+		if (reftable_alloc_size(sizeof(*(x)), (alloc), &alloc_size) < 0) { \
+			errno = ENOMEM; \
+			(x) = NULL; \
+		} else { \
+			(x) = reftable_malloc(alloc_size); \
+		} \
+	} while (0)
+#define REFTABLE_CALLOC_ARRAY(x, alloc) (x) = reftable_calloc((alloc), sizeof(*(x)))
+#define REFTABLE_REALLOC_ARRAY(x, alloc) do { \
+		size_t alloc_size; \
+		if (reftable_alloc_size(sizeof(*(x)), (alloc), &alloc_size) < 0) { \
+			errno = ENOMEM; \
+			(x) = NULL; \
+		} else { \
+			(x) = reftable_realloc((x), alloc_size); \
+		} \
+	} while (0)
+
+static inline void *reftable_alloc_grow(void *p, size_t nelem, size_t elsize,
+					size_t *allocp)
+{
+	void *new_p;
+	size_t alloc = *allocp * 2 + 1, alloc_bytes;
+	if (alloc < nelem)
+		alloc = nelem;
+	if (reftable_alloc_size(elsize, alloc, &alloc_bytes) < 0) {
+		errno = ENOMEM;
+		return p;
+	}
+	new_p = reftable_realloc(p, alloc_bytes);
+	if (!new_p)
+		return p;
+	*allocp = alloc;
+	return new_p;
+}
+
+#define REFTABLE_ALLOC_GROW(x, nr, alloc) ( \
+	(nr) > (alloc) && ( \
+		(x) = reftable_alloc_grow((x), (nr), sizeof(*(x)), &(alloc)), \
+		(nr) > (alloc) \
+	) \
+)
+
+#define REFTABLE_ALLOC_GROW_OR_NULL(x, nr, alloc) do { \
+	size_t reftable_alloc_grow_or_null_alloc = alloc; \
+	if (REFTABLE_ALLOC_GROW((x), (nr), reftable_alloc_grow_or_null_alloc)) { \
+		REFTABLE_FREE_AND_NULL(x); \
+		alloc = 0; \
+	} else { \
+		alloc = reftable_alloc_grow_or_null_alloc; \
+	} \
+} while (0)
+
+#define REFTABLE_FREE_AND_NULL(p) do { reftable_free(p); (p) = NULL; } while (0)
+
+#ifndef REFTABLE_ALLOW_BANNED_ALLOCATORS
+# define REFTABLE_BANNED(func) use_reftable_##func##_instead
+# undef malloc
+# define malloc(sz) REFTABLE_BANNED(malloc)
+# undef realloc
+# define realloc(ptr, sz) REFTABLE_BANNED(realloc)
+# undef free
+# define free(ptr) REFTABLE_BANNED(free)
+# undef calloc
+# define calloc(nelem, elsize) REFTABLE_BANNED(calloc)
+# undef strdup
+# define strdup(str) REFTABLE_BANNED(strdup)
+#endif
+
+#define REFTABLE_SWAP(a, b) do {								\
+	void *_swap_a_ptr = &(a);								\
+	void *_swap_b_ptr = &(b);								\
+	unsigned char _swap_buffer[sizeof(a) - 2 * sizeof(a) * (sizeof(a) != sizeof(b))];	\
+	memcpy(_swap_buffer, _swap_a_ptr, sizeof(a));						\
+	memcpy(_swap_a_ptr, _swap_b_ptr, sizeof(a));						\
+	memcpy(_swap_b_ptr, _swap_buffer, sizeof(a));						\
+} while (0)
+
+/* Find the longest shared prefix size of `a` and `b` */
+size_t common_prefix_size(struct reftable_buf *a, struct reftable_buf *b);
+
+uint32_t hash_size(enum reftable_hash id);
+
+/*
+ * Format IDs that identify the hash function used by a reftable. Note that
+ * these constants end up on disk and thus mustn't change. The format IDs are
+ * "sha1" and "s256" in big endian, respectively.
+ */
+#define REFTABLE_FORMAT_ID_SHA1   ((uint32_t) 0x73686131)
+#define REFTABLE_FORMAT_ID_SHA256 ((uint32_t) 0x73323536)
+
+#endif

deps/reftable/block.c

@@ -0,0 +1,655 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#include "block.h"
+
+#include "blocksource.h"
+#include "constants.h"
+#include "iter.h"
+#include "record.h"
+#include "reftable-error.h"
+#include "system.h"
+
+size_t header_size(int version)
+{
+	switch (version) {
+	case 1:
+		return 24;
+	case 2:
+		return 28;
+	}
+	abort();
+}
+
+size_t footer_size(int version)
+{
+	switch (version) {
+	case 1:
+		return 68;
+	case 2:
+		return 72;
+	}
+	abort();
+}
+
+static int block_writer_register_restart(struct block_writer *w, int n,
+					 int is_restart, struct reftable_buf *key)
+{
+	uint32_t rlen;
+	int err;
+
+	rlen = w->restart_len;
+	if (rlen >= MAX_RESTARTS)
+		is_restart = 0;
+
+	if (is_restart)
+		rlen++;
+	if (2 + 3 * rlen + n > w->block_size - w->next)
+		return REFTABLE_ENTRY_TOO_BIG_ERROR;
+	if (is_restart) {
+		REFTABLE_ALLOC_GROW_OR_NULL(w->restarts, w->restart_len + 1,
+					    w->restart_cap);
+		if (!w->restarts)
+			return REFTABLE_OUT_OF_MEMORY_ERROR;
+		w->restarts[w->restart_len++] = w->next;
+	}
+
+	w->next += n;
+
+	reftable_buf_reset(&w->last_key);
+	err = reftable_buf_add(&w->last_key, key->buf, key->len);
+	if (err < 0)
+		return err;
+
+	w->entries++;
+	return 0;
+}
+
+int block_writer_init(struct block_writer *bw, uint8_t typ, uint8_t *block,
+		      uint32_t block_size, uint32_t header_off, uint32_t hash_size)
+{
+	bw->block = block;
+	bw->hash_size = hash_size;
+	bw->block_size = block_size;
+	bw->header_off = header_off;
+	bw->block[header_off] = typ;
+	bw->next = header_off + 4;
+	bw->restart_interval = 16;
+	bw->entries = 0;
+	bw->restart_len = 0;
+	bw->last_key.len = 0;
+	if (!bw->zstream) {
+		REFTABLE_CALLOC_ARRAY(bw->zstream, 1);
+		if (!bw->zstream)
+			return REFTABLE_OUT_OF_MEMORY_ERROR;
+		deflateInit(bw->zstream, 9);
+	}
+
+	return 0;
+}
+
+uint8_t block_writer_type(struct block_writer *bw)
+{
+	return bw->block[bw->header_off];
+}
+
+/*
+ * Adds the reftable_record to the block. Returns 0 on success and
+ * appropriate error codes on failure.
+ */
+int block_writer_add(struct block_writer *w, struct reftable_record *rec)
+{
+	struct reftable_buf empty = REFTABLE_BUF_INIT;
+	struct reftable_buf last =
+		w->entries % w->restart_interval == 0 ? empty : w->last_key;
+	struct string_view out = {
+		.buf = w->block + w->next,
+		.len = w->block_size - w->next,
+	};
+	struct string_view start = out;
+	int is_restart = 0;
+	int n = 0;
+	int err;
+
+	err = reftable_record_key(rec, &w->scratch);
+	if (err < 0)
+		goto done;
+
+	if (!w->scratch.len) {
+		err = REFTABLE_API_ERROR;
+		goto done;
+	}
+
+	n = reftable_encode_key(&is_restart, out, last, w->scratch,
+				reftable_record_val_type(rec));
+	if (n < 0) {
+		err = n;
+		goto done;
+	}
+	string_view_consume(&out, n);
+
+	n = reftable_record_encode(rec, out, w->hash_size);
+	if (n < 0) {
+		err = n;
+		goto done;
+	}
+	string_view_consume(&out, n);
+
+	err = block_writer_register_restart(w, start.len - out.len, is_restart,
+					    &w->scratch);
+done:
+	return err;
+}
+
+int block_writer_finish(struct block_writer *w)
+{
+	for (uint32_t i = 0; i < w->restart_len; i++) {
+		reftable_put_be24(w->block + w->next, w->restarts[i]);
+		w->next += 3;
+	}
+
+	reftable_put_be16(w->block + w->next, w->restart_len);
+	w->next += 2;
+	reftable_put_be24(w->block + 1 + w->header_off, w->next);
+
+	/*
+	 * Log records are stored zlib-compressed. Note that the compression
+	 * also spans over the restart points we have just written.
+	 */
+	if (block_writer_type(w) == REFTABLE_BLOCK_TYPE_LOG) {
+		int block_header_skip = 4 + w->header_off;
+		uLongf src_len = w->next - block_header_skip, compressed_len;
+		int ret;
+
+		ret = deflateReset(w->zstream);
+		if (ret != Z_OK)
+			return REFTABLE_ZLIB_ERROR;
+
+		/*
+		 * Precompute the upper bound of how many bytes the compressed
+		 * data may end up with. Combined with `Z_FINISH`, `deflate()`
+		 * is guaranteed to return `Z_STREAM_END`.
+		 */
+		compressed_len = deflateBound(w->zstream, src_len);
+		REFTABLE_ALLOC_GROW_OR_NULL(w->compressed, compressed_len,
+					    w->compressed_cap);
+		if (!w->compressed) {
+			ret = REFTABLE_OUT_OF_MEMORY_ERROR;
+			return ret;
+		}
+
+		w->zstream->next_out = w->compressed;
+		w->zstream->avail_out = compressed_len;
+		w->zstream->next_in = w->block + block_header_skip;
+		w->zstream->avail_in = src_len;
+
+		/*
+		 * We want to perform all decompression in a single step, which
+		 * is why we can pass Z_FINISH here. As we have precomputed the
+		 * deflated buffer's size via `deflateBound()` this function is
+		 * guaranteed to succeed according to the zlib documentation.
+		 */
+		ret = deflate(w->zstream, Z_FINISH);
+		if (ret != Z_STREAM_END)
+			return REFTABLE_ZLIB_ERROR;
+
+		/*
+		 * Overwrite the uncompressed data we have already written and
+		 * adjust the `next` pointer to point right after the
+		 * compressed data.
+		 */
+		memcpy(w->block + block_header_skip, w->compressed,
+		       w->zstream->total_out);
+		w->next = w->zstream->total_out + block_header_skip;
+	}
+
+	return w->next;
+}
+
+static int read_block(struct reftable_block_source *source,
+		      struct reftable_block_data *dest, uint64_t off,
+		      uint32_t sz)
+{
+	size_t size = block_source_size(source);
+	block_source_release_data(dest);
+	if (off >= size)
+		return 0;
+	if (off + sz > size)
+		sz = size - off;
+	return block_source_read_data(source, dest, off, sz);
+}
+
+int reftable_block_init(struct reftable_block *block,
+			struct reftable_block_source *source,
+			uint32_t offset, uint32_t header_size,
+			uint32_t table_block_size, uint32_t hash_size,
+			uint8_t want_type)
+{
+	uint32_t guess_block_size = table_block_size ?
+		table_block_size : DEFAULT_BLOCK_SIZE;
+	uint32_t full_block_size = table_block_size;
+	uint16_t restart_count;
+	uint32_t restart_off;
+	uint32_t block_size;
+	uint8_t block_type;
+	int err;
+
+	err = read_block(source, &block->block_data, offset, guess_block_size);
+	if (err < 0)
+		goto done;
+
+	block_type = block->block_data.data[header_size];
+	if (!reftable_is_block_type(block_type)) {
+		err = REFTABLE_FORMAT_ERROR;
+		goto done;
+	}
+	if (want_type != REFTABLE_BLOCK_TYPE_ANY && block_type != want_type) {
+		err = 1;
+		goto done;
+	}
+
+	block_size = reftable_get_be24(block->block_data.data + header_size + 1);
+	if (block_size > guess_block_size) {
+		err = read_block(source, &block->block_data, offset, block_size);
+		if (err < 0)
+			goto done;
+	}
+
+	if (block_type == REFTABLE_BLOCK_TYPE_LOG) {
+		uint32_t block_header_skip = 4 + header_size;
+		uLong dst_len = block_size - block_header_skip;
+		uLong src_len = block->block_data.len - block_header_skip;
+
+		/* Log blocks specify the *uncompressed* size in their header. */
+		REFTABLE_ALLOC_GROW_OR_NULL(block->uncompressed_data, block_size,
+					    block->uncompressed_cap);
+		if (!block->uncompressed_data) {
+			err = REFTABLE_OUT_OF_MEMORY_ERROR;
+			goto done;
+		}
+
+		/* Copy over the block header verbatim. It's not compressed. */
+		memcpy(block->uncompressed_data, block->block_data.data, block_header_skip);
+
+		if (!block->zstream) {
+			REFTABLE_CALLOC_ARRAY(block->zstream, 1);
+			if (!block->zstream) {
+				err = REFTABLE_OUT_OF_MEMORY_ERROR;
+				goto done;
+			}
+
+			err = inflateInit(block->zstream);
+		} else {
+			err = inflateReset(block->zstream);
+		}
+		if (err != Z_OK) {
+			err = REFTABLE_ZLIB_ERROR;
+			goto done;
+		}
+
+		block->zstream->next_in = block->block_data.data + block_header_skip;
+		block->zstream->avail_in = src_len;
+		block->zstream->next_out = block->uncompressed_data + block_header_skip;
+		block->zstream->avail_out = dst_len;
+
+		/*
+		 * We know both input as well as output size, and we know that
+		 * the sizes should never be bigger than `uInt_MAX` because
+		 * blocks can at most be 16MB large. We can thus use `Z_FINISH`
+		 * here to instruct zlib to inflate the data in one go, which
+		 * is more efficient than using `Z_NO_FLUSH`.
+		 */
+		err = inflate(block->zstream, Z_FINISH);
+		if (err != Z_STREAM_END) {
+			err = REFTABLE_ZLIB_ERROR;
+			goto done;
+		}
+		err = 0;
+
+		if (block->zstream->total_out + block_header_skip != block_size) {
+			err = REFTABLE_FORMAT_ERROR;
+			goto done;
+		}
+
+		/* We're done with the input data. */
+		block_source_release_data(&block->block_data);
+		block->block_data.data = block->uncompressed_data;
+		block->block_data.len = block_size;
+		full_block_size = src_len + block_header_skip - block->zstream->avail_in;
+	} else if (full_block_size == 0) {
+		full_block_size = block_size;
+	} else if (block_size < full_block_size && block_size < block->block_data.len &&
+		   block->block_data.data[block_size] != 0) {
+		/* If the block is smaller than the full block size, it is
+		   padded (data followed by '\0') or the next block is
+		   unaligned. */
+		full_block_size = block_size;
+	}
+
+	restart_count = reftable_get_be16(block->block_data.data + block_size - 2);
+	restart_off = block_size - 2 - 3 * restart_count;
+
+	block->block_type = block_type;
+	block->hash_size = hash_size;
+	block->restart_off = restart_off;
+	block->full_block_size = full_block_size;
+	block->header_off = header_size;
+	block->restart_count = restart_count;
+
+	err = 0;
+
+done:
+	if (err < 0)
+		reftable_block_release(block);
+	return err;
+}
+
+void reftable_block_release(struct reftable_block *block)
+{
+	inflateEnd(block->zstream);
+	reftable_free(block->zstream);
+	reftable_free(block->uncompressed_data);
+	block_source_release_data(&block->block_data);
+	memset(block, 0, sizeof(*block));
+}
+
+uint8_t reftable_block_type(const struct reftable_block *b)
+{
+	return b->block_data.data[b->header_off];
+}
+
+int reftable_block_first_key(const struct reftable_block *block, struct reftable_buf *key)
+{
+	int off = block->header_off + 4, n;
+	struct string_view in = {
+		.buf = block->block_data.data + off,
+		.len = block->restart_off - off,
+	};
+	uint8_t extra = 0;
+
+	reftable_buf_reset(key);
+
+	n = reftable_decode_key(key, &extra, in);
+	if (n < 0)
+		return n;
+	if (!key->len)
+		return REFTABLE_FORMAT_ERROR;
+
+	return 0;
+}
+
+static uint32_t block_restart_offset(const struct reftable_block *b, size_t idx)
+{
+	return reftable_get_be24(b->block_data.data + b->restart_off + 3 * idx);
+}
+
+void block_iter_init(struct block_iter *it, const struct reftable_block *block)
+{
+	it->block = block;
+	block_iter_seek_start(it);
+}
+
+void block_iter_seek_start(struct block_iter *it)
+{
+	reftable_buf_reset(&it->last_key);
+	it->next_off = it->block->header_off + 4;
+}
+
+struct restart_needle_less_args {
+	int error;
+	struct reftable_buf needle;
+	const struct reftable_block *block;
+};
+
+static int restart_needle_less(size_t idx, void *_args)
+{
+	struct restart_needle_less_args *args = _args;
+	uint32_t off = block_restart_offset(args->block, idx);
+	struct string_view in = {
+		.buf = args->block->block_data.data + off,
+		.len = args->block->restart_off - off,
+	};
+	uint64_t prefix_len, suffix_len;
+	uint8_t extra;
+	int n;
+
+	/*
+	 * Records at restart points are stored without prefix compression, so
+	 * there is no need to fully decode the record key here. This removes
+	 * the need for allocating memory.
+	 */
+	n = reftable_decode_keylen(in, &prefix_len, &suffix_len, &extra);
+	if (n < 0 || prefix_len) {
+		args->error = 1;
+		return -1;
+	}
+
+	string_view_consume(&in, n);
+	if (suffix_len > in.len) {
+		args->error = 1;
+		return -1;
+	}
+
+	n = memcmp(args->needle.buf, in.buf,
+		   args->needle.len < suffix_len ? args->needle.len : suffix_len);
+	if (n)
+		return n < 0;
+	return args->needle.len < suffix_len;
+}
+
+int block_iter_next(struct block_iter *it, struct reftable_record *rec)
+{
+	struct string_view in = {
+		.buf = (unsigned char *) it->block->block_data.data + it->next_off,
+		.len = it->block->restart_off - it->next_off,
+	};
+	struct string_view start = in;
+	uint8_t extra = 0;
+	int n = 0;
+
+	if (it->next_off >= it->block->restart_off)
+		return 1;
+
+	n = reftable_decode_key(&it->last_key, &extra, in);
+	if (n < 0)
+		return -1;
+	if (!it->last_key.len)
+		return REFTABLE_FORMAT_ERROR;
+
+	string_view_consume(&in, n);
+	n = reftable_record_decode(rec, it->last_key, extra, in, it->block->hash_size,
+				   &it->scratch);
+	if (n < 0)
+		return -1;
+	string_view_consume(&in, n);
+
+	it->next_off += start.len - in.len;
+	return 0;
+}
+
+void block_iter_reset(struct block_iter *it)
+{
+	reftable_buf_reset(&it->last_key);
+	it->next_off = 0;
+	it->block = NULL;
+}
+
+void block_iter_close(struct block_iter *it)
+{
+	reftable_buf_release(&it->last_key);
+	reftable_buf_release(&it->scratch);
+}
+
+int block_iter_seek_key(struct block_iter *it, struct reftable_buf *want)
+{
+	struct restart_needle_less_args args = {
+		.needle = *want,
+		.block = it->block,
+	};
+	struct reftable_record rec;
+	int err = 0;
+	size_t i;
+
+	/*
+	 * Perform a binary search over the block's restart points, which
+	 * avoids doing a linear scan over the whole block. Like this, we
+	 * identify the section of the block that should contain our key.
+	 *
+	 * Note that we explicitly search for the first restart point _greater_
+	 * than the sought-after record, not _greater or equal_ to it. In case
+	 * the sought-after record is located directly at the restart point we
+	 * would otherwise start doing the linear search at the preceding
+	 * restart point. While that works alright, we would end up scanning
+	 * too many record.
+	 */
+	i = binsearch(it->block->restart_count, &restart_needle_less, &args);
+	if (args.error) {
+		err = REFTABLE_FORMAT_ERROR;
+		goto done;
+	}
+
+	/*
+	 * Now there are multiple cases:
+	 *
+	 *   - `i == 0`: The wanted record is smaller than the record found at
+	 *     the first restart point. As the first restart point is the first
+	 *     record in the block, our wanted record cannot be located in this
+	 *     block at all. We still need to position the iterator so that the
+	 *     next call to `block_iter_next()` will yield an end-of-iterator
+	 *     signal.
+	 *
+	 *   - `i == restart_count`: The wanted record was not found at any of
+	 *     the restart points. As there is no restart point at the end of
+	 *     the section the record may thus be contained in the last block.
+	 *
+	 *   - `i > 0`: The wanted record must be contained in the section
+	 *     before the found restart point. We thus do a linear search
+	 *     starting from the preceding restart point.
+	 */
+	if (i > 0)
+		it->next_off = block_restart_offset(it->block, i - 1);
+	else
+		it->next_off = it->block->header_off + 4;
+
+	err = reftable_record_init(&rec, reftable_block_type(it->block));
+	if (err < 0)
+		goto done;
+
+	/*
+	 * We're looking for the last entry less than the wanted key so that
+	 * the next call to `block_reader_next()` would yield the wanted
+	 * record. We thus don't want to position our iterator at the sought
+	 * after record, but one before. To do so, we have to go one entry too
+	 * far and then back up.
+	 */
+	while (1) {
+		size_t prev_off = it->next_off;
+
+		err = block_iter_next(it, &rec);
+		if (err < 0)
+			goto done;
+		if (err > 0) {
+			it->next_off = prev_off;
+			err = 0;
+			goto done;
+		}
+
+		err = reftable_record_key(&rec, &it->last_key);
+		if (err < 0)
+			goto done;
+
+		/*
+		 * Check whether the current key is greater or equal to the
+		 * sought-after key. In case it is greater we know that the
+		 * record does not exist in the block and can thus abort early.
+		 * In case it is equal to the sought-after key we have found
+		 * the desired record.
+		 *
+		 * Note that we store the next record's key record directly in
+		 * `last_key` without restoring the key of the preceding record
+		 * in case we need to go one record back. This is safe to do as
+		 * `block_iter_next()` would return the ref whose key is equal
+		 * to `last_key` now, and naturally all keys share a prefix
+		 * with themselves.
+		 */
+		if (reftable_buf_cmp(&it->last_key, want) >= 0) {
+			it->next_off = prev_off;
+			goto done;
+		}
+	}
+
+done:
+	reftable_record_release(&rec);
+	return err;
+}
+
+static int block_iter_seek_void(void *it, struct reftable_record *want)
+{
+	struct reftable_buf buf = REFTABLE_BUF_INIT;
+	struct block_iter *bi = it;
+	int err;
+
+	if (bi->block->block_type != want->type)
+		return REFTABLE_API_ERROR;
+
+	err = reftable_record_key(want, &buf);
+	if (err < 0)
+		goto out;
+
+	err = block_iter_seek_key(it, &buf);
+	if (err < 0)
+		goto out;
+
+	err = 0;
+
+out:
+	reftable_buf_release(&buf);
+	return err;
+}
+
+static int block_iter_next_void(void *it, struct reftable_record *rec)
+{
+	return block_iter_next(it, rec);
+}
+
+static void block_iter_close_void(void *it)
+{
+	block_iter_close(it);
+}
+
+static struct reftable_iterator_vtable block_iter_vtable = {
+	.seek = &block_iter_seek_void,
+	.next = &block_iter_next_void,
+	.close = &block_iter_close_void,
+};
+
+int reftable_block_init_iterator(const struct reftable_block *b,
+				 struct reftable_iterator *it)
+{
+	struct block_iter *bi;
+
+	REFTABLE_CALLOC_ARRAY(bi, 1);
+	block_iter_init(bi, b);
+
+	assert(!it->ops);
+	it->iter_arg = bi;
+	it->ops = &block_iter_vtable;
+
+	return 0;
+}
+
+void block_writer_release(struct block_writer *bw)
+{
+	deflateEnd(bw->zstream);
+	REFTABLE_FREE_AND_NULL(bw->zstream);
+	REFTABLE_FREE_AND_NULL(bw->restarts);
+	REFTABLE_FREE_AND_NULL(bw->compressed);
+	reftable_buf_release(&bw->scratch);
+	reftable_buf_release(&bw->last_key);
+	/* the block is not owned. */
+}

deps/reftable/block.h

@@ -0,0 +1,115 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef BLOCK_H
+#define BLOCK_H
+
+#include "basics.h"
+#include "record.h"
+#include "reftable-block.h"
+#include "reftable-blocksource.h"
+
+/*
+ * Writes reftable blocks. The block_writer is reused across blocks to minimize
+ * allocation overhead.
+ */
+struct block_writer {
+	struct z_stream_s *zstream;
+	unsigned char *compressed;
+	size_t compressed_cap;
+
+	uint8_t *block;
+	uint32_t block_size;
+
+	/* Offset of the global header. Nonzero in the first block only. */
+	uint32_t header_off;
+
+	/* How often to restart keys. */
+	uint16_t restart_interval;
+	uint32_t hash_size;
+
+	/* Offset of next uint8_t to write. */
+	uint32_t next;
+	uint32_t *restarts;
+	uint32_t restart_len;
+	uint32_t restart_cap;
+
+	struct reftable_buf last_key;
+	/* Scratch buffer used to avoid allocations. */
+	struct reftable_buf scratch;
+	int entries;
+};
+
+/*
+ * initializes the blockwriter to write `typ` entries, using `block` as temporary
+ * storage. `block` is not owned by the block_writer. */
+int block_writer_init(struct block_writer *bw, uint8_t typ, uint8_t *block,
+		      uint32_t block_size, uint32_t header_off, uint32_t hash_size);
+
+/* returns the block type (eg. 'r' for ref records. */
+uint8_t block_writer_type(struct block_writer *bw);
+
+/* Attempts to append the record. Returns 0 on success or error code on failure. */
+int block_writer_add(struct block_writer *w, struct reftable_record *rec);
+
+/* appends the key restarts, and compress the block if necessary. */
+int block_writer_finish(struct block_writer *w);
+
+/* clears out internally allocated block_writer members. */
+void block_writer_release(struct block_writer *bw);
+
+/* Iterator for records contained in a single block. */
+struct block_iter {
+	/* offset within the block of the next entry to read. */
+	uint32_t next_off;
+	const struct reftable_block *block;
+
+	/* key for last entry we read. */
+	struct reftable_buf last_key;
+	struct reftable_buf scratch;
+};
+
+#define BLOCK_ITER_INIT { \
+	.last_key = REFTABLE_BUF_INIT, \
+	.scratch = REFTABLE_BUF_INIT, \
+}
+
+/*
+ * Initialize the block iterator with the given block. The iterator will be
+ * positioned at the first record contained in the block. The block must remain
+ * valid until the end of the iterator's lifetime. It is valid to re-initialize
+ * iterators multiple times.
+ */
+void block_iter_init(struct block_iter *it, const struct reftable_block *block);
+
+/* Position the initialized iterator at the first record of its block. */
+void block_iter_seek_start(struct block_iter *it);
+
+/*
+ * Position the initialized iterator at the desired record key. It is not an
+ * error in case the record cannot be found. If so, a subsequent call to
+ * `block_iter_next()` will indicate that the iterator is exhausted.
+ */
+int block_iter_seek_key(struct block_iter *it, struct reftable_buf *want);
+
+/* return < 0 for error, 0 for OK, > 0 for EOF. */
+int block_iter_next(struct block_iter *it, struct reftable_record *rec);
+
+/* Reset the block iterator to pristine state without releasing its memory. */
+void block_iter_reset(struct block_iter *it);
+
+/* deallocate memory for `it`. The block reader and its block is left intact. */
+void block_iter_close(struct block_iter *it);
+
+/* size of file header, depending on format version */
+size_t header_size(int version);
+
+/* size of file footer, depending on format version */
+size_t footer_size(int version);
+
+#endif

deps/reftable/blocksource.c

@@ -0,0 +1,174 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#include "system.h"
+
+#include "basics.h"
+#include "blocksource.h"
+#include "reftable-blocksource.h"
+#include "reftable-error.h"
+
+void block_source_release_data(struct reftable_block_data *data)
+{
+	struct reftable_block_source source = data->source;
+	if (data && source.ops)
+		source.ops->release_data(source.arg, data);
+	data->data = NULL;
+	data->len = 0;
+	data->source.ops = NULL;
+	data->source.arg = NULL;
+}
+
+void block_source_close(struct reftable_block_source *source)
+{
+	if (!source->ops) {
+		return;
+	}
+
+	source->ops->close(source->arg);
+	source->ops = NULL;
+}
+
+ssize_t block_source_read_data(struct reftable_block_source *source,
+			       struct reftable_block_data *dest, uint64_t off,
+			       uint32_t size)
+{
+	ssize_t result = source->ops->read_data(source->arg, dest, off, size);
+	dest->source = *source;
+	return result;
+}
+
+uint64_t block_source_size(struct reftable_block_source *source)
+{
+	return source->ops->size(source->arg);
+}
+
+static void reftable_buf_release_data(void *b REFTABLE_UNUSED, struct reftable_block_data *dest)
+{
+	if (dest->len)
+		memset(dest->data, 0xff, dest->len);
+	reftable_free(dest->data);
+}
+
+static void reftable_buf_close(void *b REFTABLE_UNUSED)
+{
+}
+
+static ssize_t reftable_buf_read_data(void *v, struct reftable_block_data *dest,
+				      uint64_t off, uint32_t size)
+{
+	struct reftable_buf *b = v;
+	assert(off + size <= b->len);
+	REFTABLE_CALLOC_ARRAY(dest->data, size);
+	if (!dest->data)
+		return -1;
+	memcpy(dest->data, b->buf + off, size);
+	dest->len = size;
+	return size;
+}
+
+static uint64_t reftable_buf_size(void *b)
+{
+	return ((struct reftable_buf *)b)->len;
+}
+
+static struct reftable_block_source_vtable reftable_buf_vtable = {
+	.size = &reftable_buf_size,
+	.read_data = &reftable_buf_read_data,
+	.release_data = &reftable_buf_release_data,
+	.close = &reftable_buf_close,
+};
+
+void block_source_from_buf(struct reftable_block_source *bs,
+			   struct reftable_buf *buf)
+{
+	assert(!bs->ops);
+	bs->ops = &reftable_buf_vtable;
+	bs->arg = buf;
+}
+
+struct file_block_source {
+	struct reftable_mmap mmap;
+};
+
+static uint64_t file_size(void *b)
+{
+	return ((struct file_block_source *)b)->mmap.size;
+}
+
+static void file_release_data(void *b REFTABLE_UNUSED, struct reftable_block_data *dest REFTABLE_UNUSED)
+{
+}
+
+static void file_close(void *v)
+{
+	struct file_block_source *b = v;
+	reftable_munmap(&b->mmap);
+	reftable_free(b);
+}
+
+static ssize_t file_read_data(void *v, struct reftable_block_data *dest, uint64_t off,
+			      uint32_t size)
+{
+	struct file_block_source *b = v;
+	assert(off + size <= b->mmap.size);
+	dest->data = (unsigned char *) b->mmap.data + off;
+	dest->len = size;
+	return size;
+}
+
+static struct reftable_block_source_vtable file_vtable = {
+	.size = &file_size,
+	.read_data = &file_read_data,
+	.release_data = &file_release_data,
+	.close = &file_close,
+};
+
+int reftable_block_source_from_file(struct reftable_block_source *bs,
+				    const char *name)
+{
+	struct file_block_source *p = NULL;
+	struct stat st;
+	int fd, err;
+
+	fd = open(name, O_RDONLY);
+	if (fd < 0) {
+		if (errno == ENOENT)
+			return REFTABLE_NOT_EXIST_ERROR;
+		err = -1;
+		goto out;
+	}
+
+	if (fstat(fd, &st) < 0) {
+		err = REFTABLE_IO_ERROR;
+		goto out;
+	}
+
+	REFTABLE_CALLOC_ARRAY(p, 1);
+	if (!p) {
+		err = REFTABLE_OUT_OF_MEMORY_ERROR;
+		goto out;
+	}
+
+	err = reftable_mmap(&p->mmap, fd, st.st_size);
+	if (err < 0)
+		goto out;
+
+	assert(!bs->ops);
+	bs->ops = &file_vtable;
+	bs->arg = p;
+
+	err = 0;
+
+out:
+	if (fd >= 0)
+		close(fd);
+	if (err < 0)
+		reftable_free(p);
+	return err;
+}

deps/reftable/blocksource.h

@@ -0,0 +1,46 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef BLOCKSOURCE_H
+#define BLOCKSOURCE_H
+
+#include "system.h"
+
+struct reftable_block_source;
+struct reftable_block_data;
+struct reftable_buf;
+
+/*
+ * Close the block source and the underlying resource. This is a no-op in case
+ * the block source is zero-initialized.
+ */
+void block_source_close(struct reftable_block_source *source);
+
+/*
+ * Read a block of length `size` from the source at the given `off`.
+ */
+ssize_t block_source_read_data(struct reftable_block_source *source,
+			       struct reftable_block_data *dest, uint64_t off,
+			       uint32_t size);
+
+/*
+ * Return the total length of the underlying resource.
+ */
+uint64_t block_source_size(struct reftable_block_source *source);
+
+/*
+ * Return a block to its original source, releasing any resources associated
+ * with it.
+ */
+void block_source_release_data(struct reftable_block_data *data);
+
+/* Create an in-memory block source for reading reftables. */
+void block_source_from_buf(struct reftable_block_source *bs,
+			   struct reftable_buf *buf);
+
+#endif

deps/reftable/constants.h

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef CONSTANTS_H
+#define CONSTANTS_H
+
+#include "reftable-constants.h"
+
+#define MAX_RESTARTS ((1 << 16) - 1)
+#define DEFAULT_BLOCK_SIZE 4096
+#define DEFAULT_GEOMETRIC_FACTOR 2
+
+#endif

deps/reftable/error.c

@@ -0,0 +1,46 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#include "system.h"
+#include "reftable-error.h"
+
+#include <stdio.h>
+
+const char *reftable_error_str(int err)
+{
+	static char buf[250];
+	switch (err) {
+	case REFTABLE_IO_ERROR:
+		return "I/O error";
+	case REFTABLE_FORMAT_ERROR:
+		return "corrupt reftable file";
+	case REFTABLE_NOT_EXIST_ERROR:
+		return "file does not exist";
+	case REFTABLE_LOCK_ERROR:
+		return "data is locked";
+	case REFTABLE_API_ERROR:
+		return "misuse of the reftable API";
+	case REFTABLE_ZLIB_ERROR:
+		return "zlib failure";
+	case REFTABLE_EMPTY_TABLE_ERROR:
+		return "wrote empty table";
+	case REFTABLE_REFNAME_ERROR:
+		return "invalid refname";
+	case REFTABLE_ENTRY_TOO_BIG_ERROR:
+		return "entry too large";
+	case REFTABLE_OUTDATED_ERROR:
+		return "data concurrently modified";
+	case REFTABLE_OUT_OF_MEMORY_ERROR:
+		return "out of memory";
+	case -1:
+		return "general error";
+	default:
+		snprintf(buf, sizeof(buf), "unknown error code %d", err);
+		return buf;
+	}
+}

deps/reftable/fsck.c

@@ -0,0 +1,100 @@
+#include "basics.h"
+#include "reftable-fsck.h"
+#include "reftable-table.h"
+#include "stack.h"
+
+static bool table_has_valid_name(const char *name)
+{
+	const char *ptr = name;
+	char *endptr;
+
+	/* strtoull doesn't set errno on success */
+	errno = 0;
+
+	strtoull(ptr, &endptr, 16);
+	if (errno)
+		return false;
+	ptr = endptr;
+
+	if (*ptr != '-')
+		return false;
+	ptr++;
+
+	strtoull(ptr, &endptr, 16);
+	if (errno)
+		return false;
+	ptr = endptr;
+
+	if (*ptr != '-')
+		return false;
+	ptr++;
+
+	strtoul(ptr, &endptr, 16);
+	if (errno)
+		return false;
+	ptr = endptr;
+
+	if (strcmp(ptr, ".ref") && strcmp(ptr, ".log"))
+		return false;
+
+	return true;
+}
+
+typedef int (*table_check_fn)(struct reftable_table *table,
+			      reftable_fsck_report_fn report_fn,
+			      void *cb_data);
+
+static int table_check_name(struct reftable_table *table,
+			    reftable_fsck_report_fn report_fn,
+			    void *cb_data)
+{
+	if (!table_has_valid_name(table->name)) {
+		struct reftable_fsck_info info;
+
+		info.error = REFTABLE_FSCK_ERROR_TABLE_NAME;
+		info.msg = "invalid reftable table name";
+		info.path = table->name;
+
+		return report_fn(&info, cb_data);
+	}
+
+	return 0;
+}
+
+static int table_checks(struct reftable_table *table,
+			reftable_fsck_report_fn report_fn,
+			reftable_fsck_verbose_fn verbose_fn REFTABLE_UNUSED,
+			void *cb_data)
+{
+	table_check_fn table_check_fns[] = {
+		table_check_name,
+		NULL,
+	};
+	int err = 0;
+
+	for (size_t i = 0; table_check_fns[i]; i++)
+		err |= table_check_fns[i](table, report_fn, cb_data);
+
+	return err;
+}
+
+int reftable_fsck_check(struct reftable_stack *stack,
+			reftable_fsck_report_fn report_fn,
+			reftable_fsck_verbose_fn verbose_fn,
+			void *cb_data)
+{
+	struct reftable_buf msg = REFTABLE_BUF_INIT;
+	int err = 0;
+
+	for (size_t i = 0; i < stack->tables_len; i++) {
+		reftable_buf_reset(&msg);
+		reftable_buf_addstr(&msg, "Checking table: ");
+		reftable_buf_addstr(&msg, stack->tables[i]->name);
+		verbose_fn(msg.buf, cb_data);
+
+		err |= table_checks(stack->tables[i], report_fn, verbose_fn, cb_data);
+	}
+
+	reftable_buf_release(&msg);
+	return err;
+}

deps/reftable/iter.c

@@ -0,0 +1,302 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#include "iter.h"
+
+#include "system.h"
+
+#include "block.h"
+#include "blocksource.h"
+#include "constants.h"
+#include "reftable-error.h"
+#include "table.h"
+
+int iterator_seek(struct reftable_iterator *it, struct reftable_record *want)
+{
+	return it->ops->seek(it->iter_arg, want);
+}
+
+int iterator_next(struct reftable_iterator *it, struct reftable_record *rec)
+{
+	return it->ops->next(it->iter_arg, rec);
+}
+
+static int empty_iterator_seek(void *arg REFTABLE_UNUSED, struct reftable_record *want REFTABLE_UNUSED)
+{
+	return 0;
+}
+
+static int empty_iterator_next(void *arg REFTABLE_UNUSED, struct reftable_record *rec REFTABLE_UNUSED)
+{
+	return 1;
+}
+
+static void empty_iterator_close(void *arg REFTABLE_UNUSED)
+{
+}
+
+static struct reftable_iterator_vtable empty_vtable = {
+	.seek = &empty_iterator_seek,
+	.next = &empty_iterator_next,
+	.close = &empty_iterator_close,
+};
+
+void iterator_set_empty(struct reftable_iterator *it)
+{
+	assert(!it->ops);
+	it->iter_arg = NULL;
+	it->ops = &empty_vtable;
+}
+
+static void filtering_ref_iterator_close(void *iter_arg)
+{
+	struct filtering_ref_iterator *fri = iter_arg;
+	reftable_buf_release(&fri->oid);
+	reftable_iterator_destroy(&fri->it);
+}
+
+static int filtering_ref_iterator_seek(void *iter_arg,
+				       struct reftable_record *want)
+{
+	struct filtering_ref_iterator *fri = iter_arg;
+	return iterator_seek(&fri->it, want);
+}
+
+static int filtering_ref_iterator_next(void *iter_arg,
+				       struct reftable_record *rec)
+{
+	struct filtering_ref_iterator *fri = iter_arg;
+	struct reftable_ref_record *ref = &rec->u.ref;
+	int err = 0;
+	while (1) {
+		err = reftable_iterator_next_ref(&fri->it, ref);
+		if (err != 0) {
+			break;
+		}
+
+		if (ref->value_type == REFTABLE_REF_VAL2 &&
+		    (!memcmp(fri->oid.buf, ref->value.val2.target_value,
+			     fri->oid.len) ||
+		     !memcmp(fri->oid.buf, ref->value.val2.value,
+			     fri->oid.len)))
+			return 0;
+
+		if (ref->value_type == REFTABLE_REF_VAL1 &&
+		    !memcmp(fri->oid.buf, ref->value.val1, fri->oid.len)) {
+			return 0;
+		}
+	}
+
+	reftable_ref_record_release(ref);
+	return err;
+}
+
+static struct reftable_iterator_vtable filtering_ref_iterator_vtable = {
+	.seek = &filtering_ref_iterator_seek,
+	.next = &filtering_ref_iterator_next,
+	.close = &filtering_ref_iterator_close,
+};
+
+void iterator_from_filtering_ref_iterator(struct reftable_iterator *it,
+					  struct filtering_ref_iterator *fri)
+{
+	assert(!it->ops);
+	it->iter_arg = fri;
+	it->ops = &filtering_ref_iterator_vtable;
+}
+
+static void indexed_table_ref_iter_close(void *p)
+{
+	struct indexed_table_ref_iter *it = p;
+	block_iter_close(&it->cur);
+	block_source_release_data(&it->block.block_data);
+	reftable_free(it->offsets);
+	reftable_buf_release(&it->oid);
+}
+
+static int indexed_table_ref_iter_next_block(struct indexed_table_ref_iter *it)
+{
+	uint64_t off;
+	int err = 0;
+	if (it->offset_idx == it->offset_len) {
+		it->is_finished = 1;
+		return 1;
+	}
+
+	block_source_release_data(&it->block.block_data);
+
+	off = it->offsets[it->offset_idx++];
+	err = table_init_block(it->table, &it->block, off, REFTABLE_BLOCK_TYPE_REF);
+	if (err < 0) {
+		return err;
+	}
+	if (err > 0) {
+		/* indexed block does not exist. */
+		return REFTABLE_FORMAT_ERROR;
+	}
+	block_iter_init(&it->cur, &it->block);
+	return 0;
+}
+
+static int indexed_table_ref_iter_seek(void *p REFTABLE_UNUSED,
+				       struct reftable_record *want REFTABLE_UNUSED)
+{
+	return REFTABLE_API_ERROR;
+}
+
+static int indexed_table_ref_iter_next(void *p, struct reftable_record *rec)
+{
+	struct indexed_table_ref_iter *it = p;
+	struct reftable_ref_record *ref = &rec->u.ref;
+
+	while (1) {
+		int err = block_iter_next(&it->cur, rec);
+		if (err < 0) {
+			return err;
+		}
+
+		if (err > 0) {
+			err = indexed_table_ref_iter_next_block(it);
+			if (err < 0) {
+				return err;
+			}
+
+			if (it->is_finished) {
+				return 1;
+			}
+			continue;
+		}
+		/* BUG */
+		if (!memcmp(it->oid.buf, ref->value.val2.target_value,
+			    it->oid.len) ||
+		    !memcmp(it->oid.buf, ref->value.val2.value, it->oid.len)) {
+			return 0;
+		}
+	}
+}
+
+int indexed_table_ref_iter_new(struct indexed_table_ref_iter **dest,
+			       struct reftable_table *t, uint8_t *oid,
+			       int oid_len, uint64_t *offsets, int offset_len)
+{
+	struct indexed_table_ref_iter empty = INDEXED_TABLE_REF_ITER_INIT;
+	struct indexed_table_ref_iter *itr;
+	int err = 0;
+
+	itr = reftable_calloc(1, sizeof(*itr));
+	if (!itr) {
+		err = REFTABLE_OUT_OF_MEMORY_ERROR;
+		goto out;
+	}
+
+	*itr = empty;
+	itr->table = t;
+
+	err = reftable_buf_add(&itr->oid, oid, oid_len);
+	if (err < 0)
+		goto out;
+
+	itr->offsets = offsets;
+	itr->offset_len = offset_len;
+
+	err = indexed_table_ref_iter_next_block(itr);
+	if (err < 0)
+		goto out;
+
+	*dest = itr;
+	err = 0;
+
+out:
+	if (err < 0) {
+		*dest = NULL;
+		reftable_free(itr);
+	}
+	return err;
+}
+
+static struct reftable_iterator_vtable indexed_table_ref_iter_vtable = {
+	.seek = &indexed_table_ref_iter_seek,
+	.next = &indexed_table_ref_iter_next,
+	.close = &indexed_table_ref_iter_close,
+};
+
+void iterator_from_indexed_table_ref_iter(struct reftable_iterator *it,
+					  struct indexed_table_ref_iter *itr)
+{
+	assert(!it->ops);
+	it->iter_arg = itr;
+	it->ops = &indexed_table_ref_iter_vtable;
+}
+
+void reftable_iterator_destroy(struct reftable_iterator *it)
+{
+	if (!it->ops)
+		return;
+	it->ops->close(it->iter_arg);
+	it->ops = NULL;
+	REFTABLE_FREE_AND_NULL(it->iter_arg);
+}
+
+int reftable_iterator_seek_ref(struct reftable_iterator *it,
+			       const char *name)
+{
+	struct reftable_record want = {
+		.type = REFTABLE_BLOCK_TYPE_REF,
+		.u.ref = {
+			.refname = (char *)name,
+		},
+	};
+	return it->ops->seek(it->iter_arg, &want);
+}
+
+int reftable_iterator_next_ref(struct reftable_iterator *it,
+			       struct reftable_ref_record *ref)
+{
+	struct reftable_record rec = {
+		.type = REFTABLE_BLOCK_TYPE_REF,
+		.u = {
+			.ref = *ref
+		},
+	};
+	int err = iterator_next(it, &rec);
+	*ref = rec.u.ref;
+	return err;
+}
+
+int reftable_iterator_seek_log_at(struct reftable_iterator *it,
+				  const char *name, uint64_t update_index)
+{
+	struct reftable_record want = {
+		.type = REFTABLE_BLOCK_TYPE_LOG,
+		.u.log = {
+			.refname = (char *)name,
+			.update_index = update_index,
+		},
+	};
+	return it->ops->seek(it->iter_arg, &want);
+}
+
+int reftable_iterator_seek_log(struct reftable_iterator *it,
+			       const char *name)
+{
+	return reftable_iterator_seek_log_at(it, name, ~((uint64_t) 0));
+}
+
+int reftable_iterator_next_log(struct reftable_iterator *it,
+			       struct reftable_log_record *log)
+{
+	struct reftable_record rec = {
+		.type = REFTABLE_BLOCK_TYPE_LOG,
+		.u = {
+			.log = *log,
+		},
+	};
+	int err = iterator_next(it, &rec);
+	*log = rec.u.log;
+	return err;
+}

deps/reftable/iter.h

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef ITER_H
+#define ITER_H
+
+#include "system.h"
+#include "block.h"
+#include "record.h"
+
+#include "reftable-iterator.h"
+
+/*
+ * The virtual function table for implementing generic reftable iterators.
+ */
+struct reftable_iterator_vtable {
+	int (*seek)(void *iter_arg, struct reftable_record *want);
+	int (*next)(void *iter_arg, struct reftable_record *rec);
+	void (*close)(void *iter_arg);
+};
+
+/*
+ * Position the iterator at the wanted record such that a call to
+ * `iterator_next()` would return that record, if it exists.
+ */
+int iterator_seek(struct reftable_iterator *it, struct reftable_record *want);
+
+/*
+ * Yield the next record and advance the iterator. Returns <0 on error, 0 when
+ * a record was yielded, and >0 when the iterator hit an error.
+ */
+int iterator_next(struct reftable_iterator *it, struct reftable_record *rec);
+
+/*
+ * Set up the iterator such that it behaves the same as an iterator with no
+ * entries.
+ */
+void iterator_set_empty(struct reftable_iterator *it);
+
+/* iterator that produces only ref records that point to `oid` */
+struct filtering_ref_iterator {
+	struct reftable_buf oid;
+	struct reftable_iterator it;
+};
+#define FILTERING_REF_ITERATOR_INIT \
+	{                           \
+		.oid = REFTABLE_BUF_INIT  \
+	}
+
+void iterator_from_filtering_ref_iterator(struct reftable_iterator *,
+					  struct filtering_ref_iterator *);
+
+/* iterator that produces only ref records that point to `oid`,
+ * but using the object index.
+ */
+struct indexed_table_ref_iter {
+	struct reftable_table *table;
+	struct reftable_buf oid;
+
+	/* mutable */
+	uint64_t *offsets;
+
+	/* Points to the next offset to read. */
+	int offset_idx;
+	int offset_len;
+	struct reftable_block block;
+	struct block_iter cur;
+	int is_finished;
+};
+
+#define INDEXED_TABLE_REF_ITER_INIT { \
+	.cur = BLOCK_ITER_INIT, \
+	.oid = REFTABLE_BUF_INIT, \
+}
+
+void iterator_from_indexed_table_ref_iter(struct reftable_iterator *it,
+					  struct indexed_table_ref_iter *itr);
+
+/* Takes ownership of `offsets` */
+int indexed_table_ref_iter_new(struct indexed_table_ref_iter **dest,
+			       struct reftable_table *t, uint8_t *oid,
+			       int oid_len, uint64_t *offsets, int offset_len);
+
+#endif

deps/reftable/merged.c

@@ -0,0 +1,316 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#include "merged.h"
+
+#include "constants.h"
+#include "iter.h"
+#include "pq.h"
+#include "record.h"
+#include "reftable-merged.h"
+#include "reftable-error.h"
+#include "system.h"
+#include "table.h"
+
+struct merged_subiter {
+	struct reftable_iterator iter;
+	struct reftable_record rec;
+};
+
+struct merged_iter {
+	struct merged_subiter *subiters;
+	struct merged_iter_pqueue pq;
+	size_t subiters_len;
+	int suppress_deletions;
+	ssize_t advance_index;
+};
+
+static void merged_iter_close(void *p)
+{
+	struct merged_iter *mi = p;
+
+	merged_iter_pqueue_release(&mi->pq);
+	for (size_t i = 0; i < mi->subiters_len; i++) {
+		reftable_iterator_destroy(&mi->subiters[i].iter);
+		reftable_record_release(&mi->subiters[i].rec);
+	}
+	reftable_free(mi->subiters);
+}
+
+static int merged_iter_advance_subiter(struct merged_iter *mi, size_t idx)
+{
+	struct pq_entry e = {
+		.index = idx,
+		.rec = &mi->subiters[idx].rec,
+	};
+	int err;
+
+	err = iterator_next(&mi->subiters[idx].iter, &mi->subiters[idx].rec);
+	if (err)
+		return err;
+
+	err = merged_iter_pqueue_add(&mi->pq, &e);
+	if (err)
+		return err;
+
+	return 0;
+}
+
+static int merged_iter_seek(struct merged_iter *mi, struct reftable_record *want)
+{
+	int err;
+
+	mi->advance_index = -1;
+	while (!merged_iter_pqueue_is_empty(mi->pq)) {
+		err = merged_iter_pqueue_remove(&mi->pq, NULL);
+		if (err < 0)
+			return err;
+	}
+
+	for (size_t i = 0; i < mi->subiters_len; i++) {
+		err = iterator_seek(&mi->subiters[i].iter, want);
+		if (err < 0)
+			return err;
+		if (err > 0)
+			continue;
+
+		err = merged_iter_advance_subiter(mi, i);
+		if (err < 0)
+			return err;
+	}
+
+	return 0;
+}
+
+static int merged_iter_next_entry(struct merged_iter *mi,
+				  struct reftable_record *rec)
+{
+	struct pq_entry entry = { 0 };
+	int err = 0, empty;
+
+	empty = merged_iter_pqueue_is_empty(mi->pq);
+
+	if (mi->advance_index >= 0) {
+		/*
+		 * When there are no pqueue entries then we only have a single
+		 * subiter left. There is no need to use the pqueue in that
+		 * case anymore as we know that the subiter will return entries
+		 * in the correct order already.
+		 *
+		 * While this may sound like a very specific edge case, it may
+		 * happen more frequently than you think. Most repositories
+		 * will end up having a single large base table that contains
+		 * most of the refs. It's thus likely that we exhaust all
+		 * subiters but the one from that base ref.
+		 */
+		if (empty)
+			return iterator_next(&mi->subiters[mi->advance_index].iter,
+					     rec);
+
+		err = merged_iter_advance_subiter(mi, mi->advance_index);
+		if (err < 0)
+			return err;
+		if (!err)
+			empty = 0;
+		mi->advance_index = -1;
+	}
+
+	if (empty)
+		return 1;
+
+	err = merged_iter_pqueue_remove(&mi->pq, &entry);
+	if (err < 0)
+		return err;
+
+	/*
+	  One can also use reftable as datacenter-local storage, where the ref
+	  database is maintained in globally consistent database (eg.
+	  CockroachDB or Spanner). In this scenario, replication delays together
+	  with compaction may cause newer tables to contain older entries. In
+	  such a deployment, the loop below must be changed to collect all
+	  entries for the same key, and return new the newest one.
+	*/
+	while (!merged_iter_pqueue_is_empty(mi->pq)) {
+		struct pq_entry top = merged_iter_pqueue_top(mi->pq);
+		int cmp;
+
+		err = reftable_record_cmp(top.rec, entry.rec, &cmp);
+		if (err < 0)
+			return err;
+		if (cmp > 0)
+			break;
+
+		err = merged_iter_pqueue_remove(&mi->pq, NULL);
+		if (err < 0)
+			return err;
+
+		err = merged_iter_advance_subiter(mi, top.index);
+		if (err < 0)
+			return err;
+	}
+
+	mi->advance_index = entry.index;
+	REFTABLE_SWAP(*rec, *entry.rec);
+	return 0;
+}
+
+static int merged_iter_seek_void(void *it, struct reftable_record *want)
+{
+	return merged_iter_seek(it, want);
+}
+
+static int merged_iter_next_void(void *p, struct reftable_record *rec)
+{
+	struct merged_iter *mi = p;
+	while (1) {
+		int err = merged_iter_next_entry(mi, rec);
+		if (err)
+			return err;
+		if (mi->suppress_deletions && reftable_record_is_deletion(rec))
+			continue;
+		return 0;
+	}
+}
+
+static struct reftable_iterator_vtable merged_iter_vtable = {
+	.seek = merged_iter_seek_void,
+	.next = &merged_iter_next_void,
+	.close = &merged_iter_close,
+};
+
+static void iterator_from_merged_iter(struct reftable_iterator *it,
+				      struct merged_iter *mi)
+{
+	assert(!it->ops);
+	it->iter_arg = mi;
+	it->ops = &merged_iter_vtable;
+}
+
+int reftable_merged_table_new(struct reftable_merged_table **dest,
+			      struct reftable_table **tables, size_t n,
+			      enum reftable_hash hash_id)
+{
+	struct reftable_merged_table *m = NULL;
+	uint64_t last_max = 0;
+	uint64_t first_min = 0;
+
+	for (size_t i = 0; i < n; i++) {
+		uint64_t min = reftable_table_min_update_index(tables[i]);
+		uint64_t max = reftable_table_max_update_index(tables[i]);
+
+		if (reftable_table_hash_id(tables[i]) != hash_id) {
+			return REFTABLE_FORMAT_ERROR;
+		}
+		if (i == 0 || min < first_min) {
+			first_min = min;
+		}
+		if (i == 0 || max > last_max) {
+			last_max = max;
+		}
+	}
+
+	REFTABLE_CALLOC_ARRAY(m, 1);
+	if (!m)
+		return REFTABLE_OUT_OF_MEMORY_ERROR;
+
+	m->tables = tables;
+	m->tables_len = n;
+	m->min = first_min;
+	m->max = last_max;
+	m->hash_id = hash_id;
+	*dest = m;
+	return 0;
+}
+
+void reftable_merged_table_free(struct reftable_merged_table *mt)
+{
+	if (!mt)
+		return;
+	reftable_free(mt);
+}
+
+uint64_t
+reftable_merged_table_max_update_index(struct reftable_merged_table *mt)
+{
+	return mt->max;
+}
+
+uint64_t
+reftable_merged_table_min_update_index(struct reftable_merged_table *mt)
+{
+	return mt->min;
+}
+
+int merged_table_init_iter(struct reftable_merged_table *mt,
+			   struct reftable_iterator *it,
+			   uint8_t typ)
+{
+	struct merged_subiter *subiters = NULL;
+	struct merged_iter *mi = NULL;
+	int ret;
+
+	if (mt->tables_len) {
+		REFTABLE_CALLOC_ARRAY(subiters, mt->tables_len);
+		if (!subiters) {
+			ret = REFTABLE_OUT_OF_MEMORY_ERROR;
+			goto out;
+		}
+	}
+
+	for (size_t i = 0; i < mt->tables_len; i++) {
+		ret = reftable_record_init(&subiters[i].rec, typ);
+		if (ret < 0)
+			goto out;
+
+		ret = table_init_iter(mt->tables[i], &subiters[i].iter, typ);
+		if (ret < 0)
+			goto out;
+	}
+
+	REFTABLE_CALLOC_ARRAY(mi, 1);
+	if (!mi) {
+		ret = REFTABLE_OUT_OF_MEMORY_ERROR;
+		goto out;
+	}
+	mi->advance_index = -1;
+	mi->suppress_deletions = mt->suppress_deletions;
+	mi->subiters = subiters;
+	mi->subiters_len = mt->tables_len;
+
+	iterator_from_merged_iter(it, mi);
+	ret = 0;
+
+out:
+	if (ret < 0) {
+		for (size_t i = 0; subiters && i < mt->tables_len; i++) {
+			reftable_iterator_destroy(&subiters[i].iter);
+			reftable_record_release(&subiters[i].rec);
+		}
+		reftable_free(subiters);
+		reftable_free(mi);
+	}
+
+	return ret;
+}
+
+int reftable_merged_table_init_ref_iterator(struct reftable_merged_table *mt,
+					    struct reftable_iterator *it)
+{
+	return merged_table_init_iter(mt, it, REFTABLE_BLOCK_TYPE_REF);
+}
+
+int reftable_merged_table_init_log_iterator(struct reftable_merged_table *mt,
+					    struct reftable_iterator *it)
+{
+	return merged_table_init_iter(mt, it, REFTABLE_BLOCK_TYPE_LOG);
+}
+
+enum reftable_hash reftable_merged_table_hash_id(struct reftable_merged_table *mt)
+{
+	return mt->hash_id;
+}

deps/reftable/merged.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef MERGED_H
+#define MERGED_H
+
+#include "system.h"
+#include "reftable-basics.h"
+
+struct reftable_merged_table {
+	struct reftable_table **tables;
+	size_t tables_len;
+	enum reftable_hash hash_id;
+
+	/* If unset, produce deletions. This is useful for compaction. For the
+	 * full stack, deletions should be produced. */
+	int suppress_deletions;
+
+	uint64_t min;
+	uint64_t max;
+};
+
+struct reftable_iterator;
+
+int merged_table_init_iter(struct reftable_merged_table *mt,
+			   struct reftable_iterator *it,
+			   uint8_t typ);
+
+#endif

deps/reftable/pq.c

@@ -0,0 +1,95 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#include "pq.h"
+
+#include "reftable-error.h"
+#include "reftable-record.h"
+#include "system.h"
+#include "basics.h"
+
+int pq_less(struct pq_entry *a, struct pq_entry *b)
+{
+	int cmp, err;
+
+	err = reftable_record_cmp(a->rec, b->rec, &cmp);
+	if (err < 0)
+		return err;
+
+	if (cmp == 0)
+		return a->index > b->index;
+	return cmp < 0;
+}
+
+int merged_iter_pqueue_remove(struct merged_iter_pqueue *pq, struct pq_entry *out)
+{
+	size_t i = 0;
+	struct pq_entry e = pq->heap[0];
+	pq->heap[0] = pq->heap[pq->len - 1];
+	pq->len--;
+
+	while (i < pq->len) {
+		size_t min = i;
+		size_t j = 2 * i + 1;
+		size_t k = 2 * i + 2;
+		int cmp;
+
+		if (j < pq->len) {
+			cmp = pq_less(&pq->heap[j], &pq->heap[i]);
+			if (cmp < 0)
+				return -1;
+			else if (cmp)
+				min = j;
+		}
+
+		if (k < pq->len) {
+			cmp = pq_less(&pq->heap[k], &pq->heap[min]);
+			if (cmp < 0)
+				return -1;
+			else if (cmp)
+				min = k;
+		}
+
+		if (min == i)
+			break;
+		REFTABLE_SWAP(pq->heap[i], pq->heap[min]);
+		i = min;
+	}
+
+	if (out)
+		*out = e;
+
+	return 0;
+}
+
+int merged_iter_pqueue_add(struct merged_iter_pqueue *pq, const struct pq_entry *e)
+{
+	size_t i = 0;
+
+	REFTABLE_ALLOC_GROW_OR_NULL(pq->heap, pq->len + 1, pq->cap);
+	if (!pq->heap)
+		return REFTABLE_OUT_OF_MEMORY_ERROR;
+	pq->heap[pq->len++] = *e;
+
+	i = pq->len - 1;
+	while (i > 0) {
+		size_t j = (i - 1) / 2;
+		if (pq_less(&pq->heap[j], &pq->heap[i]))
+			break;
+		REFTABLE_SWAP(pq->heap[j], pq->heap[i]);
+		i = j;
+	}
+
+	return 0;
+}
+
+void merged_iter_pqueue_release(struct merged_iter_pqueue *pq)
+{
+	REFTABLE_FREE_AND_NULL(pq->heap);
+	memset(pq, 0, sizeof(*pq));
+}

deps/reftable/pq.h

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef PQ_H
+#define PQ_H
+
+#include "record.h"
+
+struct pq_entry {
+	size_t index;
+	struct reftable_record *rec;
+};
+
+struct merged_iter_pqueue {
+	struct pq_entry *heap;
+	size_t len;
+	size_t cap;
+};
+
+int merged_iter_pqueue_remove(struct merged_iter_pqueue *pq, struct pq_entry *out);
+int merged_iter_pqueue_add(struct merged_iter_pqueue *pq, const struct pq_entry *e);
+void merged_iter_pqueue_release(struct merged_iter_pqueue *pq);
+int pq_less(struct pq_entry *a, struct pq_entry *b);
+
+static inline struct pq_entry merged_iter_pqueue_top(struct merged_iter_pqueue pq)
+{
+	return pq.heap[0];
+}
+
+static inline int merged_iter_pqueue_is_empty(struct merged_iter_pqueue pq)
+{
+	return pq.len == 0;
+}
+
+#endif

deps/reftable/record.h

@@ -0,0 +1,164 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef RECORD_H
+#define RECORD_H
+
+#include "basics.h"
+#include "system.h"
+
+#include <stdint.h>
+
+#include "reftable-record.h"
+
+/*
+ * A substring of existing string data. This structure takes no responsibility
+ * for the lifetime of the data it points to.
+ */
+struct string_view {
+	uint8_t *buf;
+	size_t len;
+};
+
+/* Advance `s.buf` by `n`, and decrease length. */
+static inline void string_view_consume(struct string_view *s, int n)
+{
+	s->buf += n;
+	s->len -= n;
+}
+
+/*
+ * Decode and encode a varint. Returns the number of bytes read/written, or a
+ * negative value in case encoding/decoding the varint has failed.
+ */
+int get_var_int(uint64_t *dest, struct string_view *in);
+int put_var_int(struct string_view *dest, uint64_t val);
+
+/* Methods for records. */
+struct reftable_record_vtable {
+	/* encode the key of to a uint8_t reftable_buf. */
+	int (*key)(const void *rec, struct reftable_buf *dest);
+
+	/* The record type of ('r' for ref). */
+	uint8_t type;
+
+	int (*copy_from)(void *dest, const void *src, uint32_t hash_size);
+
+	/* a value of [0..7], indicating record subvariants (eg. ref vs. symref
+	 * vs ref deletion) */
+	uint8_t (*val_type)(const void *rec);
+
+	/* encodes rec into dest, returning how much space was used. */
+	int (*encode)(const void *rec, struct string_view dest, uint32_t hash_size);
+
+	/* decode data from `src` into the record. */
+	int (*decode)(void *rec, struct reftable_buf key, uint8_t extra,
+		      struct string_view src, uint32_t hash_size,
+		      struct reftable_buf *scratch);
+
+	/* deallocate and null the record. */
+	void (*release)(void *rec);
+
+	/* is this a tombstone? */
+	int (*is_deletion)(const void *rec);
+
+	/* Are two records equal? This assumes they have the same type. Returns 0 for non-equal. */
+	int (*equal)(const void *a, const void *b, uint32_t hash_size);
+
+	/*
+	 * Compare keys of two records with each other. The records must have
+	 * the same type.
+	 */
+	int (*cmp)(const void *a, const void *b);
+};
+
+/* returns true for recognized block types. Block start with the block type. */
+int reftable_is_block_type(uint8_t typ);
+
+/* Encode `key` into `dest`. Sets `is_restart` to indicate a restart. Returns
+ * number of bytes written. */
+int reftable_encode_key(int *is_restart, struct string_view dest,
+			struct reftable_buf prev_key, struct reftable_buf key,
+			uint8_t extra);
+
+/* Decode a record's key lengths. */
+int reftable_decode_keylen(struct string_view in,
+			   uint64_t *prefix_len,
+			   uint64_t *suffix_len,
+			   uint8_t *extra);
+
+/*
+ * Decode into `last_key` and `extra` from `in`. `last_key` is expected to
+ * contain the decoded key of the preceding record, if any.
+ */
+int reftable_decode_key(struct reftable_buf *last_key, uint8_t *extra,
+			struct string_view in);
+
+/* reftable_index_record are used internally to speed up lookups. */
+struct reftable_index_record {
+	uint64_t offset; /* Offset of block */
+	struct reftable_buf last_key; /* Last key of the block. */
+};
+
+/* reftable_obj_record stores an object ID => ref mapping. */
+struct reftable_obj_record {
+	uint8_t *hash_prefix; /* leading bytes of the object ID */
+	int hash_prefix_len; /* number of leading bytes. Constant
+			      * across a single table. */
+	uint64_t *offsets; /* a vector of file offsets. */
+	int offset_len;
+};
+
+/* record is a generic wrapper for different types of records. It is normally
+ * created on the stack, or embedded within another struct. If the type is
+ * known, a fresh instance can be initialized explicitly. Otherwise, use
+ * `reftable_record_init()` to initialize generically (as the index_record is
+ * not valid as 0-initialized structure)
+ */
+struct reftable_record {
+	uint8_t type;
+	union {
+		struct reftable_ref_record ref;
+		struct reftable_log_record log;
+		struct reftable_obj_record obj;
+		struct reftable_index_record idx;
+	} u;
+};
+
+/* Initialize the reftable record for the given type. */
+int reftable_record_init(struct reftable_record *rec, uint8_t typ);
+
+/* see struct record_vtable */
+int reftable_record_cmp(struct reftable_record *a, struct reftable_record *b, int *cmp);
+int reftable_record_equal(struct reftable_record *a, struct reftable_record *b, uint32_t hash_size);
+int reftable_record_key(struct reftable_record *rec, struct reftable_buf *dest);
+int reftable_record_copy_from(struct reftable_record *rec,
+			      struct reftable_record *src, uint32_t hash_size);
+uint8_t reftable_record_val_type(struct reftable_record *rec);
+int reftable_record_encode(struct reftable_record *rec, struct string_view dest,
+			   uint32_t hash_size);
+int reftable_record_decode(struct reftable_record *rec, struct reftable_buf key,
+			   uint8_t extra, struct string_view src,
+			   uint32_t hash_size, struct reftable_buf *scratch);
+int reftable_record_is_deletion(struct reftable_record *rec);
+
+static inline uint8_t reftable_record_type(struct reftable_record *rec)
+{
+	return rec->type;
+}
+
+/* frees and zeroes out the embedded record */
+void reftable_record_release(struct reftable_record *rec);
+
+/* for qsort. */
+int reftable_ref_record_compare_name(const void *a, const void *b);
+
+/* for qsort. */
+int reftable_log_record_compare_key(const void *a, const void *b);
+
+#endif

deps/reftable/reftable-basics.h

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef REFTABLE_BASICS_H
+#define REFTABLE_BASICS_H
+
+#include "reftable-system.h"
+
+/* A buffer that contains arbitrary byte slices. */
+struct reftable_buf {
+	size_t alloc;
+	size_t len;
+	char *buf;
+};
+#define REFTABLE_BUF_INIT { 0 }
+
+/*
+ * Hash functions understood by the reftable library. Note that the values are
+ * arbitrary and somewhat random such that we can easily detect cases where the
+ * hash hasn't been properly set up.
+ */
+enum reftable_hash {
+	REFTABLE_HASH_SHA1   = 89,
+	REFTABLE_HASH_SHA256 = 247,
+};
+#define REFTABLE_HASH_SIZE_SHA1   20
+#define REFTABLE_HASH_SIZE_SHA256 32
+#define REFTABLE_HASH_SIZE_MAX    REFTABLE_HASH_SIZE_SHA256
+
+/* Overrides the functions to use for memory management. */
+void reftable_set_alloc(void *(*malloc)(size_t),
+			void *(*realloc)(void *, size_t), void (*free)(void *));
+
+#endif

deps/reftable/reftable-block.h

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef REFTABLE_BLOCK_H
+#define REFTABLE_BLOCK_H
+
+#include "reftable-system.h"
+#include "reftable-basics.h"
+#include "reftable-blocksource.h"
+#include "reftable-iterator.h"
+
+struct z_stream_s;
+
+/*
+ * A block part of a reftable. Contains records as well as some metadata
+ * describing them.
+ */
+struct reftable_block {
+	/*
+	 * Offset of the block header; nonzero for the first block in a
+	 * reftable.
+	 */
+	uint32_t header_off;
+
+	/* The memory block. */
+	struct reftable_block_data block_data;
+	uint32_t hash_size;
+
+	/* Uncompressed data for log entries. */
+	struct z_stream_s *zstream;
+	unsigned char *uncompressed_data;
+	size_t uncompressed_cap;
+
+	/*
+	 * Restart point data. Restart points are located after the block's
+	 * record data.
+	 */
+	uint16_t restart_count;
+	uint32_t restart_off;
+
+	/*
+	 * Size of the data in the file. For log blocks, this is the compressed
+	 * size.
+	 */
+	uint32_t full_block_size;
+	uint8_t block_type;
+};
+
+/* Initialize a reftable block from the given block source. */
+int reftable_block_init(struct reftable_block *b,
+			struct reftable_block_source *source,
+			uint32_t offset, uint32_t header_size,
+			uint32_t table_block_size, uint32_t hash_size,
+			uint8_t want_type);
+
+/* Release resources allocated by the block. */
+void reftable_block_release(struct reftable_block *b);
+
+/* Initialize a generic record iterator from the given block. */
+int reftable_block_init_iterator(const struct reftable_block *b,
+				 struct reftable_iterator *it);
+
+/* Returns the block type (eg. 'r' for refs). */
+uint8_t reftable_block_type(const struct reftable_block *b);
+
+/* Decodes the first key in the block. */
+int reftable_block_first_key(const struct reftable_block *b, struct reftable_buf *key);
+
+#endif /* REFTABLE_BLOCK_H */

deps/reftable/reftable-blocksource.h

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef REFTABLE_BLOCKSOURCE_H
+#define REFTABLE_BLOCKSOURCE_H
+
+#include "reftable-system.h"
+
+/*
+ * Generic wrapper for a seekable readable file.
+ */
+struct reftable_block_source {
+	struct reftable_block_source_vtable *ops;
+	void *arg;
+};
+
+/* a contiguous segment of bytes. It keeps track of its generating block_source
+ * so it can return itself into the pool. */
+struct reftable_block_data {
+	uint8_t *data;
+	size_t len;
+	struct reftable_block_source source;
+};
+
+/* block_source_vtable are the operations that make up block_source */
+struct reftable_block_source_vtable {
+	/* Returns the size of a block source. */
+	uint64_t (*size)(void *source);
+
+	/*
+	 * Reads a segment from the block source. It is an error to read beyond
+	 * the end of the block.
+	 */
+	ssize_t (*read_data)(void *source, struct reftable_block_data *dest,
+			uint64_t off, uint32_t size);
+
+	/* Mark the block as read; may release the data. */
+	void (*release_data)(void *source, struct reftable_block_data *data);
+
+	/* Release all resources associated with the block source. */
+	void (*close)(void *source);
+};
+
+/* opens a file on the file system as a block_source */
+int reftable_block_source_from_file(struct reftable_block_source *block_src,
+				    const char *name);
+
+#endif

deps/reftable/reftable-constants.h

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef REFTABLE_CONSTANTS_H
+#define REFTABLE_CONSTANTS_H
+
+#define REFTABLE_BLOCK_TYPE_LOG 'g'
+#define REFTABLE_BLOCK_TYPE_INDEX 'i'
+#define REFTABLE_BLOCK_TYPE_REF 'r'
+#define REFTABLE_BLOCK_TYPE_OBJ 'o'
+#define REFTABLE_BLOCK_TYPE_ANY 0
+
+#endif /* REFTABLE_CONSTANTS_H */

deps/reftable/reftable-error.h

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef REFTABLE_ERROR_H
+#define REFTABLE_ERROR_H
+
+#include "reftable-system.h"
+
+/*
+ * Errors in reftable calls are signaled with negative integer return values. 0
+ * means success.
+ */
+enum reftable_error {
+	/* Unexpected file system behavior */
+	REFTABLE_IO_ERROR = -2,
+
+	/* Format inconsistency on reading data */
+	REFTABLE_FORMAT_ERROR = -3,
+
+	/* File does not exist. Returned from block_source_from_file(), because
+	 * it needs special handling in stack.
+	 */
+	REFTABLE_NOT_EXIST_ERROR = -4,
+
+	/* Trying to access locked data. */
+	REFTABLE_LOCK_ERROR = -5,
+
+	/* Misuse of the API:
+	 *  - on writing a record with NULL refname.
+	 *  - on writing a record before setting the writer limits.
+	 *  - on writing a reftable_ref_record outside the table limits
+	 *  - on writing a ref or log record before the stack's
+	 * next_update_inde*x
+	 *  - on writing a log record with multiline message with
+	 *  exact_log_message unset
+	 *  - on reading a reftable_ref_record from log iterator, or vice versa.
+	 *
+	 * When a call misuses the API, the internal state of the library is
+	 * kept unchanged.
+	 */
+	REFTABLE_API_ERROR = -6,
+
+	/* Decompression error */
+	REFTABLE_ZLIB_ERROR = -7,
+
+	/* Wrote a table without blocks. */
+	REFTABLE_EMPTY_TABLE_ERROR = -8,
+
+	/* Invalid ref name. */
+	REFTABLE_REFNAME_ERROR = -10,
+
+	/* Entry does not fit. This can happen when writing outsize reflog
+	   messages. */
+	REFTABLE_ENTRY_TOO_BIG_ERROR = -11,
+
+	/* Trying to write out-of-date data. */
+	REFTABLE_OUTDATED_ERROR = -12,
+
+	/* An allocation has failed due to an out-of-memory situation. */
+	REFTABLE_OUT_OF_MEMORY_ERROR = -13,
+};
+
+/* convert the numeric error code to a string. The string should not be
+ * deallocated. */
+const char *reftable_error_str(int err);
+
+#endif

deps/reftable/reftable-fsck.h

@@ -0,0 +1,41 @@
+#ifndef REFTABLE_FSCK_H
+#define REFTABLE_FSCK_H
+
+#include "reftable-system.h"
+#include "reftable-stack.h"
+
+enum reftable_fsck_error {
+	/* Invalid table name */
+	REFTABLE_FSCK_ERROR_TABLE_NAME = 0,
+	/* Used for bounds checking, must be last */
+	REFTABLE_FSCK_MAX_VALUE,
+};
+
+/* Represents an individual error encountered during the FSCK checks. */
+struct reftable_fsck_info {
+	enum reftable_fsck_error error;
+	const char *msg;
+	const char *path;
+};
+
+typedef int reftable_fsck_report_fn(struct reftable_fsck_info *info,
+				    void *cb_data);
+typedef void reftable_fsck_verbose_fn(const char *msg, void *cb_data);
+
+/*
+ * Given a reftable stack, perform consistency checks on the stack.
+ *
+ * If an issue is encountered, the issue is reported to the callee via the
+ * provided 'report_fn'. If the issue is non-recoverable the flow will not
+ * continue. If it is recoverable, the flow will continue and further issues
+ * will be reported as identified.
+ *
+ * The 'verbose_fn' will be invoked to provide verbose information about
+ * the progress and state of the consistency checks.
+ */
+int reftable_fsck_check(struct reftable_stack *stack,
+			reftable_fsck_report_fn report_fn,
+			reftable_fsck_verbose_fn verbose_fn,
+			void *cb_data);
+
+#endif /* REFTABLE_FSCK_H */

deps/reftable/reftable-iterator.h

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef REFTABLE_ITERATOR_H
+#define REFTABLE_ITERATOR_H
+
+#include "reftable-system.h"
+#include "reftable-record.h"
+
+struct reftable_iterator_vtable;
+
+/* iterator is the generic interface for walking over data stored in a
+ * reftable.
+ */
+struct reftable_iterator {
+	struct reftable_iterator_vtable *ops;
+	void *iter_arg;
+};
+
+/*
+ * Position the iterator at the ref record with given name such that the next
+ * call to `next_ref()` would yield the record.
+ */
+int reftable_iterator_seek_ref(struct reftable_iterator *it,
+			       const char *name);
+
+/* reads the next reftable_ref_record. Returns < 0 for error, 0 for OK and > 0:
+ * end of iteration.
+ */
+int reftable_iterator_next_ref(struct reftable_iterator *it,
+			       struct reftable_ref_record *ref);
+
+/*
+ * Position the iterator at the log record with given name and update index
+ * such that the next call to `next_log()` would yield the record.
+ */
+int reftable_iterator_seek_log_at(struct reftable_iterator *it,
+				  const char *name, uint64_t update_index);
+
+/*
+ * Position the iterator at the newest log record with given name such that the
+ * next call to `next_log()` would yield the record.
+ */
+int reftable_iterator_seek_log(struct reftable_iterator *it,
+			       const char *name);
+
+/* reads the next reftable_log_record. Returns < 0 for error, 0 for OK and > 0:
+ * end of iteration.
+ */
+int reftable_iterator_next_log(struct reftable_iterator *it,
+			       struct reftable_log_record *log);
+
+/* releases resources associated with an iterator. */
+void reftable_iterator_destroy(struct reftable_iterator *it);
+
+#endif

deps/reftable/reftable-merged.h

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef REFTABLE_MERGED_H
+#define REFTABLE_MERGED_H
+
+#include "reftable-system.h"
+#include "reftable-iterator.h"
+
+/*
+ * Merged tables
+ *
+ * A ref database kept in a sequence of table files. The merged_table presents a
+ * unified view to reading (seeking, iterating) a sequence of immutable tables.
+ *
+ * The merged tables are on purpose kept disconnected from their actual storage
+ * (eg. files on disk), because it is useful to merge tables aren't files. For
+ * example, the per-workspace and global ref namespace can be implemented as a
+ * merged table of two stacks of file-backed reftables.
+ */
+
+/* A merged table is implements seeking/iterating over a stack of tables. */
+struct reftable_merged_table;
+
+struct reftable_table;
+
+/*
+ * reftable_merged_table_new creates a new merged table. The tables must be
+ * kept alive as long as the merged table is still in use.
+ */
+int reftable_merged_table_new(struct reftable_merged_table **dest,
+			      struct reftable_table **tables, size_t n,
+			      enum reftable_hash hash_id);
+
+/* Initialize a merged table iterator for reading refs. */
+int reftable_merged_table_init_ref_iterator(struct reftable_merged_table *mt,
+					    struct reftable_iterator *it);
+
+/* Initialize a merged table iterator for reading logs. */
+int reftable_merged_table_init_log_iterator(struct reftable_merged_table *mt,
+					    struct reftable_iterator *it);
+
+/* returns the max update_index covered by this merged table. */
+uint64_t
+reftable_merged_table_max_update_index(struct reftable_merged_table *mt);
+
+/* returns the min update_index covered by this merged table. */
+uint64_t
+reftable_merged_table_min_update_index(struct reftable_merged_table *mt);
+
+/* releases memory for the merged_table */
+void reftable_merged_table_free(struct reftable_merged_table *m);
+
+/* return the hash ID of the merged table. */
+enum reftable_hash reftable_merged_table_hash_id(struct reftable_merged_table *m);
+
+#endif

deps/reftable/reftable-record.h

@@ -0,0 +1,110 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef REFTABLE_RECORD_H
+#define REFTABLE_RECORD_H
+
+#include "reftable-system.h"
+#include "reftable-basics.h"
+
+/*
+ * Basic data types
+ *
+ * Reftables store the state of each ref in struct reftable_ref_record, and they
+ * store a sequence of reflog updates in struct reftable_log_record.
+ */
+
+/* reftable_ref_record holds a ref database entry target_value */
+struct reftable_ref_record {
+	char *refname; /* Name of the ref, malloced. */
+	size_t refname_cap;
+	uint64_t update_index; /* Logical timestamp at which this value is
+				* written */
+
+	enum {
+		/* tombstone to hide deletions from earlier tables */
+		REFTABLE_REF_DELETION = 0x0,
+
+		/* a simple ref */
+		REFTABLE_REF_VAL1 = 0x1,
+		/* a tag, plus its peeled hash */
+		REFTABLE_REF_VAL2 = 0x2,
+
+		/* a symbolic reference */
+		REFTABLE_REF_SYMREF = 0x3,
+#define REFTABLE_NR_REF_VALUETYPES 4
+	} value_type;
+	union {
+		unsigned char val1[REFTABLE_HASH_SIZE_MAX];
+		struct {
+			unsigned char value[REFTABLE_HASH_SIZE_MAX]; /* first hash  */
+			unsigned char target_value[REFTABLE_HASH_SIZE_MAX]; /* second hash */
+		} val2;
+		char *symref; /* referent, malloced 0-terminated string */
+	} value;
+};
+
+/* Returns the first hash, or NULL if `rec` is not of type
+ * REFTABLE_REF_VAL1 or REFTABLE_REF_VAL2. */
+const unsigned char *reftable_ref_record_val1(const struct reftable_ref_record *rec);
+
+/* Returns the second hash, or NULL if `rec` is not of type
+ * REFTABLE_REF_VAL2. */
+const unsigned char *reftable_ref_record_val2(const struct reftable_ref_record *rec);
+
+/* returns whether 'ref' represents a deletion */
+int reftable_ref_record_is_deletion(const struct reftable_ref_record *ref);
+
+/* frees and nulls all pointer values inside `ref`. */
+void reftable_ref_record_release(struct reftable_ref_record *ref);
+
+/* returns whether two reftable_ref_records are the same. Useful for testing. */
+int reftable_ref_record_equal(const struct reftable_ref_record *a,
+			      const struct reftable_ref_record *b, uint32_t hash_size);
+
+/* reftable_log_record holds a reflog entry */
+struct reftable_log_record {
+	char *refname;
+	size_t refname_cap;
+	uint64_t update_index; /* logical timestamp of a transactional update.
+				*/
+
+	enum {
+		/* tombstone to hide deletions from earlier tables */
+		REFTABLE_LOG_DELETION = 0x0,
+
+		/* a simple update */
+		REFTABLE_LOG_UPDATE = 0x1,
+#define REFTABLE_NR_LOG_VALUETYPES 2
+	} value_type;
+
+	union {
+		struct {
+			unsigned char new_hash[REFTABLE_HASH_SIZE_MAX];
+			unsigned char old_hash[REFTABLE_HASH_SIZE_MAX];
+			char *name;
+			char *email;
+			uint64_t time;
+			int16_t tz_offset;
+			char *message;
+			size_t message_cap;
+		} update;
+	} value;
+};
+
+/* returns whether 'ref' represents the deletion of a log record. */
+int reftable_log_record_is_deletion(const struct reftable_log_record *log);
+
+/* frees and nulls all pointer values. */
+void reftable_log_record_release(struct reftable_log_record *log);
+
+/* returns whether two records are equal. Useful for testing. */
+int reftable_log_record_equal(const struct reftable_log_record *a,
+			      const struct reftable_log_record *b, uint32_t hash_size);
+
+#endif

deps/reftable/reftable-stack.h

@@ -0,0 +1,170 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef REFTABLE_STACK_H
+#define REFTABLE_STACK_H
+
+#include "reftable-system.h"
+#include "reftable-writer.h"
+
+/*
+ * The stack presents an interface to a mutable sequence of reftables.
+
+ * A stack can be mutated by pushing a table to the top of the stack.
+
+ * The reftable_stack automatically compacts files on disk to ensure good
+ * amortized performance.
+ *
+ * For windows and other platforms that cannot have open files as rename
+ * destinations, concurrent access from multiple processes needs the rand()
+ * random seed to be randomized.
+ */
+struct reftable_stack;
+
+/* open a new reftable stack. The tables along with the table list will be
+ *  stored in 'dir'. Typically, this should be .git/reftables.
+ */
+int reftable_new_stack(struct reftable_stack **dest, const char *dir,
+		       const struct reftable_write_options *opts);
+
+/* returns the update_index at which a next table should be written. */
+uint64_t reftable_stack_next_update_index(struct reftable_stack *st);
+
+/* holds a transaction to add tables at the top of a stack. */
+struct reftable_addition;
+
+enum {
+	/*
+	 * Reload the stack when the stack is out-of-date after locking it.
+	 */
+	REFTABLE_STACK_NEW_ADDITION_RELOAD = (1 << 0),
+};
+
+/*
+ * returns a new transaction to add reftables to the given stack. As a side
+ * effect, the ref database is locked. Accepts REFTABLE_STACK_NEW_ADDITION_*
+ * flags.
+ */
+int reftable_stack_new_addition(struct reftable_addition **dest,
+				struct reftable_stack *st,
+				unsigned int flags);
+
+/* Adds a reftable to transaction. */
+int reftable_addition_add(struct reftable_addition *add,
+			  int (*write_table)(struct reftable_writer *wr,
+					     void *arg),
+			  void *arg);
+
+/* Commits the transaction, releasing the lock. After calling this,
+ * reftable_addition_destroy should still be called.
+ */
+int reftable_addition_commit(struct reftable_addition *add);
+
+/* Release all non-committed data from the transaction, and deallocate the
+ * transaction. Releases the lock if held. */
+void reftable_addition_destroy(struct reftable_addition *add);
+
+/*
+ * Add a new table to the stack. The write_table function must call
+ * reftable_writer_set_limits, add refs and return an error value.
+ * The flags are passed through to `reftable_stack_new_addition()`.
+ */
+int reftable_stack_add(struct reftable_stack *st,
+		       int (*write_table)(struct reftable_writer *wr,
+					  void *write_arg),
+		       void *write_arg, unsigned flags);
+
+struct reftable_iterator;
+
+/*
+ * Initialize an iterator for the merged tables contained in the stack that can
+ * be used to iterate through refs. The iterator is valid until the next reload
+ * or write.
+ */
+int reftable_stack_init_ref_iterator(struct reftable_stack *st,
+				     struct reftable_iterator *it);
+
+/*
+ * Initialize an iterator for the merged tables contained in the stack that can
+ * be used to iterate through logs. The iterator is valid until the next reload
+ * or write.
+ */
+int reftable_stack_init_log_iterator(struct reftable_stack *st,
+				     struct reftable_iterator *it);
+
+/* returns the merged_table for seeking. This table is valid until the
+ * next write or reload, and should not be closed or deleted.
+ */
+struct reftable_merged_table *
+reftable_stack_merged_table(struct reftable_stack *st);
+
+/* frees all resources associated with the stack. */
+void reftable_stack_destroy(struct reftable_stack *st);
+
+/* Reloads the stack if necessary. This is very cheap to run if the stack was up
+ * to date */
+int reftable_stack_reload(struct reftable_stack *st);
+
+/* Policy for expiring reflog entries. */
+struct reftable_log_expiry_config {
+	/* Drop entries older than this timestamp */
+	uint64_t time;
+
+	/* Drop older entries */
+	uint64_t min_update_index;
+};
+
+/* compacts all reftables into a giant table. Expire reflog entries if config is
+ * non-NULL */
+int reftable_stack_compact_all(struct reftable_stack *st,
+			       struct reftable_log_expiry_config *config);
+
+/*
+ * Check if compaction is required.
+ *
+ * When `use_heuristics` is false, check if all tables can be compacted to a
+ * single table. If true, use heuristics to determine if the tables need to be
+ * compacted to maintain geometric progression.
+ */
+int reftable_stack_compaction_required(struct reftable_stack *st,
+				       bool use_heuristics,
+				       bool *required);
+
+/* heuristically compact unbalanced table stack. */
+int reftable_stack_auto_compact(struct reftable_stack *st);
+
+/* delete stale .ref tables. */
+int reftable_stack_clean(struct reftable_stack *st);
+
+/* convenience function to read a single ref. Returns < 0 for error, 0 for
+ * success, and 1 if ref not found. */
+int reftable_stack_read_ref(struct reftable_stack *st, const char *refname,
+			    struct reftable_ref_record *ref);
+
+/* convenience function to read a single log. Returns < 0 for error, 0 for
+ * success, and 1 if ref not found. */
+int reftable_stack_read_log(struct reftable_stack *st, const char *refname,
+			    struct reftable_log_record *log);
+
+/* statistics on past compactions. */
+struct reftable_compaction_stats {
+	uint64_t bytes; /* total number of bytes written */
+	uint64_t entries_written; /* total number of entries written, including
+				     failures. */
+	int attempts; /* how often we tried to compact */
+	int failures; /* failures happen on concurrent updates */
+};
+
+/* return statistics for compaction up till now. */
+struct reftable_compaction_stats *
+reftable_stack_compaction_stats(struct reftable_stack *st);
+
+/* Return the hash of the stack. */
+enum reftable_hash reftable_stack_hash_id(struct reftable_stack *st);
+
+#endif

deps/reftable/reftable-system.h

@@ -0,0 +1,33 @@
+#ifndef REFTABLE_SYSTEM_H
+#define REFTABLE_SYSTEM_H
+
+/*
+ * This header defines the platform-specific bits required to compile the
+ * reftable library. It should provide an environment that bridges over the
+ * gaps between POSIX and your system, as well as the zlib interfaces. This
+ * header is expected to be changed by the individual project.
+ */
+
+#include "map.h"
+#include "posix.h"
+#include "util.h"
+#include <zlib.h>
+
+/*
+ * We only need to redefine on Windows as we expect stat(3p) et al to be
+ * available on Unix platforms. Furthermore, we only need to redefine fstat(3p)
+ * because we already redefine stat(3p) "win32-compat.h".
+ */
+#ifdef GIT_WIN32
+#  define fstat(fd, st) p_fstat(fd, st)
+#endif
+
+#define fsync(fd) p_fsync(fd)
+
+#define poll(fds, fds_len, timeout) p_poll(fds, fds_len, timeout)
+
+#define REFTABLE_ALLOW_BANNED_ALLOCATORS
+
+#define inline GIT_INLINE_KEYWORD
+
+#endif

deps/reftable/reftable-table.h

@@ -0,0 +1,116 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef REFTABLE_TABLE_H
+#define REFTABLE_TABLE_H
+
+#include "reftable-system.h"
+#include "reftable-iterator.h"
+#include "reftable-block.h"
+#include "reftable-blocksource.h"
+
+/*
+ * Reading single tables
+ *
+ * The follow routines are for reading single files. For an
+ * application-level interface, skip ahead to struct
+ * reftable_merged_table and struct reftable_stack.
+ */
+
+/* Metadata for a block type. */
+struct reftable_table_offsets {
+	int is_present;
+	uint64_t offset;
+	uint64_t index_offset;
+};
+
+/* The table struct is a handle to an open reftable file. */
+struct reftable_table {
+	/* for convenience, associate a name with the instance. */
+	char *name;
+	struct reftable_block_source source;
+
+	/* Size of the file, excluding the footer. */
+	uint64_t size;
+
+	/* The hash function used for ref records. */
+	enum reftable_hash hash_id;
+
+	uint32_t block_size;
+	uint64_t min_update_index;
+	uint64_t max_update_index;
+	/* Length of the OID keys in the 'o' section */
+	int object_id_len;
+	int version;
+
+	struct reftable_table_offsets ref_offsets;
+	struct reftable_table_offsets obj_offsets;
+	struct reftable_table_offsets log_offsets;
+
+	uint64_t refcount;
+};
+
+/* reftable_table_new opens a reftable for reading. If successful,
+ * returns 0 code and sets pp. The name is used for creating a
+ * stack. Typically, it is the basename of the file. The block source
+ * `src` is owned by the table, and is closed on calling
+ * reftable_table_destroy(). On error, the block source `src` is
+ * closed as well.
+ */
+int reftable_table_new(struct reftable_table **out,
+		       struct reftable_block_source *src, const char *name);
+
+/*
+ * Manage the reference count of the reftable table. A newly initialized
+ * table starts with a refcount of 1 and will be deleted once the refcount has
+ * reached 0.
+ *
+ * This is required because tables may have longer lifetimes than the stack
+ * they belong to. The stack may for example be reloaded while the old tables
+ * are still being accessed by an iterator.
+ */
+void reftable_table_incref(struct reftable_table *table);
+void reftable_table_decref(struct reftable_table *table);
+
+/* Initialize a reftable iterator for reading refs. */
+int reftable_table_init_ref_iterator(struct reftable_table *t,
+				     struct reftable_iterator *it);
+
+/* Initialize a reftable iterator for reading logs. */
+int reftable_table_init_log_iterator(struct reftable_table *t,
+				     struct reftable_iterator *it);
+
+/* returns the hash ID used in this table. */
+enum reftable_hash reftable_table_hash_id(struct reftable_table *t);
+
+/* return an iterator for the refs pointing to `oid`. */
+int reftable_table_refs_for(struct reftable_table *t,
+			    struct reftable_iterator *it, uint8_t *oid);
+
+/* return the max_update_index for a table */
+uint64_t reftable_table_max_update_index(struct reftable_table *t);
+
+/* return the min_update_index for a table */
+uint64_t reftable_table_min_update_index(struct reftable_table *t);
+
+/*
+ * An iterator that iterates through the blocks contained in a given table.
+ */
+struct reftable_table_iterator {
+	void *iter_arg;
+};
+
+int reftable_table_iterator_init(struct reftable_table_iterator *it,
+				 struct reftable_table *t);
+
+void reftable_table_iterator_release(struct reftable_table_iterator *it);
+
+int reftable_table_iterator_next(struct reftable_table_iterator *it,
+				 const struct reftable_block **out);
+
+#endif

deps/reftable/reftable-writer.h

@@ -0,0 +1,181 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef REFTABLE_WRITER_H
+#define REFTABLE_WRITER_H
+
+#include "reftable-system.h"
+#include "reftable-record.h"
+
+/* Writing single reftables */
+
+/* reftable_write_options sets options for writing a single reftable. */
+struct reftable_write_options {
+	/* boolean: do not pad out blocks to block size. */
+	unsigned unpadded : 1;
+
+	/* the blocksize. Should be less than 2^24. */
+	uint32_t block_size;
+
+	/* boolean: do not generate a SHA1 => ref index. */
+	unsigned skip_index_objects : 1;
+
+	/* how often to write complete keys in each block. */
+	uint16_t restart_interval;
+
+	/* 4-byte identifier ("sha1", "s256") of the hash.
+	 * Defaults to SHA1 if unset
+	 */
+	enum reftable_hash hash_id;
+
+	/* Default mode for creating files. If unset, use 0666 (+umask) */
+	unsigned int default_permissions;
+
+	/* boolean: copy log messages exactly. If unset, check that the message
+	 *   is a single line, and add '\n' if missing.
+	 */
+	unsigned exact_log_message : 1;
+
+	/* boolean: Prevent auto-compaction of tables. */
+	unsigned disable_auto_compact : 1;
+
+	/*
+	 * Geometric sequence factor used by auto-compaction to decide which
+	 * tables to compact. Defaults to 2 if unset.
+	 */
+	uint8_t auto_compaction_factor;
+
+	/*
+	 * The number of milliseconds to wait when trying to lock "tables.list".
+	 * Note that this does not apply to locking individual tables, as these
+	 * should only ever be locked when already holding the "tables.list"
+	 * lock.
+	 *
+	 * Passing 0 will fail immediately when the file is locked, passing a
+	 * negative value will cause us to block indefinitely.
+	 */
+	long lock_timeout_ms;
+
+	/*
+	 * Callback function to execute whenever the stack is being reloaded.
+	 * This can be used e.g. to discard cached information that relies on
+	 * the old stack's data. The payload data will be passed as argument to
+	 * the callback.
+	 */
+	void (*on_reload)(void *payload);
+	void *on_reload_payload;
+};
+
+/* reftable_block_stats holds statistics for a single block type */
+struct reftable_block_stats {
+	/* total number of entries written */
+	int entries;
+	/* total number of key restarts */
+	uint32_t restarts;
+	/* total number of blocks */
+	int blocks;
+	/* total number of index blocks */
+	int index_blocks;
+	/* depth of the index */
+	int max_index_level;
+
+	/* offset of the first block for this type */
+	uint64_t offset;
+	/* offset of the top level index block for this type, or 0 if not
+	 * present */
+	uint64_t index_offset;
+};
+
+/* stats holds overall statistics for a single reftable */
+struct reftable_stats {
+	/* total number of blocks written. */
+	int blocks;
+	/* stats for ref data */
+	struct reftable_block_stats ref_stats;
+	/* stats for the SHA1 to ref map. */
+	struct reftable_block_stats obj_stats;
+	/* stats for index blocks */
+	struct reftable_block_stats idx_stats;
+	/* stats for log blocks */
+	struct reftable_block_stats log_stats;
+
+	/* disambiguation length of shortened object IDs. */
+	int object_id_len;
+};
+
+struct reftable_writer;
+
+/* Create a new writer. */
+int reftable_writer_new(struct reftable_writer **out,
+			ssize_t (*writer_func)(void *, const void *, size_t),
+			int (*flush_func)(void *),
+			void *writer_arg, const struct reftable_write_options *opts);
+
+/*
+ * Set the range of update indices for the records we will add. When writing a
+ * table into a stack, the min should be at least
+ * reftable_stack_next_update_index(), or REFTABLE_API_ERROR is returned.
+ *
+ * For transactional updates to a stack, typically min==max, and the
+ * update_index can be obtained by inspeciting the stack. When converting an
+ * existing ref database into a single reftable, this would be a range of
+ * update-index timestamps.
+ *
+ * The function should be called before adding any records to the writer. If not
+ * it will fail with REFTABLE_API_ERROR.
+ */
+int reftable_writer_set_limits(struct reftable_writer *w, uint64_t min,
+			       uint64_t max);
+
+/*
+  Add a reftable_ref_record. The record should have names that come after
+  already added records.
+
+  The update_index must be within the limits set by
+  reftable_writer_set_limits(), or REFTABLE_API_ERROR is returned. It is an
+  REFTABLE_API_ERROR error to write a ref record after a log record.
+*/
+int reftable_writer_add_ref(struct reftable_writer *w,
+			    struct reftable_ref_record *ref);
+
+/*
+  Convenience function to add multiple reftable_ref_records; the function sorts
+  the records before adding them, reordering the records array passed in.
+*/
+int reftable_writer_add_refs(struct reftable_writer *w,
+			     struct reftable_ref_record *refs, size_t n);
+
+/*
+  adds reftable_log_records. Log records are keyed by (refname, decreasing
+  update_index). The key for the record added must come after the already added
+  log records.
+*/
+int reftable_writer_add_log(struct reftable_writer *w,
+			    struct reftable_log_record *log);
+
+/*
+  Convenience function to add multiple reftable_log_records; the function sorts
+  the records before adding them, reordering records array passed in.
+*/
+int reftable_writer_add_logs(struct reftable_writer *w,
+			     struct reftable_log_record *logs, size_t n);
+
+/* reftable_writer_close finalizes the reftable. The writer is retained so
+ * statistics can be inspected. */
+int reftable_writer_close(struct reftable_writer *w);
+
+/* writer_stats returns the statistics on the reftable being written.
+
+   This struct becomes invalid when the writer is freed.
+ */
+const struct reftable_stats *reftable_writer_stats(struct reftable_writer *w);
+
+/* reftable_writer_free deallocates memory for the writer */
+void reftable_writer_free(struct reftable_writer *w);
+
+#endif

deps/reftable/stack.h

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef STACK_H
+#define STACK_H
+
+#include "system.h"
+#include "reftable-writer.h"
+#include "reftable-stack.h"
+
+struct reftable_stack {
+	struct stat list_st;
+	char *list_file;
+	int list_fd;
+
+	char *reftable_dir;
+
+	struct reftable_write_options opts;
+
+	struct reftable_table **tables;
+	size_t tables_len;
+	struct reftable_merged_table *merged;
+	struct reftable_compaction_stats stats;
+};
+
+int read_lines(const char *filename, char ***lines);
+
+struct segment {
+	size_t start, end;
+	uint64_t bytes;
+};
+
+struct segment suggest_compaction_segment(uint64_t *sizes, size_t n,
+					  uint8_t factor);
+
+#endif

deps/reftable/system.c

@@ -0,0 +1,259 @@
+#include "system.h"
+#include "basics.h"
+#include "rand.h"
+#include "reftable-error.h"
+
+uint32_t reftable_rand(void)
+{
+	return (uint32_t) git_rand_next();
+}
+
+int tmpfile_from_pattern(struct reftable_tmpfile *out, const char *pattern)
+{
+	git_str path = GIT_STR_INIT;
+	unsigned tries = 32;
+
+	if (git__suffixcmp(pattern, ".XXXXXX"))
+		return REFTABLE_API_ERROR;
+
+	while (tries--) {
+		uint64_t rand = git_rand_next();
+		int fd;
+
+		git_str_sets(&path, pattern);
+		git_str_shorten(&path, 6);
+		git_str_encode_hexstr(&path, (void *)&rand, 6);
+
+		if (git_str_oom(&path))
+			return REFTABLE_OUT_OF_MEMORY_ERROR;
+
+		if ((fd = p_open(path.ptr, O_CREAT | O_RDWR | O_EXCL | O_CLOEXEC, 0666)) < 0)
+			continue;
+
+		out->path = git_str_detach(&path);
+		out->fd = fd;
+		return 0;
+	}
+
+	git_str_dispose(&path);
+	return REFTABLE_IO_ERROR;
+}
+
+int tmpfile_close(struct reftable_tmpfile *t)
+{
+	int ret;
+
+	if (t->fd < 0)
+		return 0;
+	ret = close(t->fd);
+	t->fd = -1;
+
+	if (ret < 0)
+		return REFTABLE_IO_ERROR;
+	return 0;
+}
+
+int tmpfile_delete(struct reftable_tmpfile *t)
+{
+	int ret;
+
+	if (!t->path)
+		return 0;
+
+	tmpfile_close(t);
+	if ((ret = unlink(t->path)) < 0)
+		return REFTABLE_IO_ERROR;
+
+	reftable_free((char *) t->path);
+	t->path = NULL;
+
+	return 0;
+}
+
+int tmpfile_rename(struct reftable_tmpfile *t, const char *path)
+{
+	int ret;
+
+	if (!t->path)
+		return REFTABLE_API_ERROR;
+
+	tmpfile_close(t);
+
+	if ((ret = p_rename(t->path, path)) < 0)
+		return REFTABLE_IO_ERROR;
+
+	reftable_free((char *) t->path);
+	t->path = NULL;
+
+	return 0;
+}
+
+struct libgit2_flock {
+	char *lock_path;
+	char *target_path;
+};
+
+int flock_acquire(struct reftable_flock *l, const char *target_path,
+		  long timeout_ms)
+{
+	struct libgit2_flock *flock;
+	unsigned multiplier = 1, n = 1;
+	size_t lock_path_len;
+	uint64_t deadline;
+	int fd = -1, error;
+
+	lock_path_len = strlen(target_path) + strlen(".lock") + 1;
+	if ((flock = reftable_calloc(sizeof(*flock), 1)) == NULL ||
+	    (flock->target_path = reftable_strdup(target_path)) == NULL ||
+	    (flock->lock_path = reftable_malloc(lock_path_len)) == NULL) {
+		error = REFTABLE_OUT_OF_MEMORY_ERROR;
+		goto out;
+	}
+
+	snprintf(flock->lock_path, lock_path_len, "%s.lock", target_path);
+
+	deadline = git_time_monotonic() + timeout_ms;
+
+	while (1) {
+		uint64_t now, wait_ms;
+
+		if ((fd = p_open(flock->lock_path, O_WRONLY | O_EXCL | O_CREAT, 0666)) >= 0)
+			break;
+		if (errno != EEXIST) {
+			error = REFTABLE_IO_ERROR;
+			goto out;
+		}
+
+		now = git_time_monotonic();
+		if (now > deadline) {
+			error = REFTABLE_LOCK_ERROR;
+			goto out;
+		}
+
+		wait_ms = (750 + rand() % 500) * multiplier / 1000;
+		multiplier += 2 * n + 1;
+
+		if (multiplier > 1000)
+			multiplier = 1000;
+		else
+			n++;
+
+		p_poll(NULL, 0, (int) wait_ms);
+	}
+
+	l->priv = flock;
+	l->path = flock->lock_path;
+	l->fd = fd;
+
+	error = 0;
+
+out:
+	if (error) {
+		if (flock) {
+			reftable_free(flock->target_path);
+			reftable_free(flock->lock_path);
+		}
+		reftable_free(flock);
+	}
+
+	return error;
+}
+
+int flock_close(struct reftable_flock *l)
+{
+	int ret;
+
+	if (l->fd < 0)
+		return 0;
+
+	ret = p_close(l->fd);
+	l->fd = -1;
+
+	if (ret < 0)
+		return REFTABLE_IO_ERROR;
+	return 0;
+}
+
+static void libgit2_flock_release(struct reftable_flock *l)
+{
+	struct libgit2_flock *flock = l->priv;
+	reftable_free(flock->lock_path);
+	reftable_free(flock->target_path);
+	reftable_free(flock);
+	l->priv = NULL;
+	l->path = NULL;
+}
+
+int flock_release(struct reftable_flock *l)
+{
+	struct libgit2_flock *flock = l->priv;
+	int ret;
+
+	if (!flock)
+		return 0;
+
+	flock_close(l);
+
+	if ((ret = p_unlink(flock->lock_path)) < 0)
+		goto out;
+
+out:
+	libgit2_flock_release(l);
+	if (ret < 0)
+		return REFTABLE_IO_ERROR;
+	return 0;
+}
+
+int flock_commit(struct reftable_flock *l)
+{
+	struct libgit2_flock *flock = l->priv;
+	int ret;
+
+	flock_close(l);
+
+	if (!flock)
+		return REFTABLE_API_ERROR;
+
+	if ((ret = p_rename(flock->lock_path, flock->target_path)) < 0)
+		goto out;
+
+out:
+	libgit2_flock_release(l);
+	if (ret < 0)
+		return REFTABLE_IO_ERROR;
+	return 0;
+}
+
+uint64_t reftable_time_ms(void)
+{
+	return git_time_monotonic();
+}
+
+int reftable_mmap(struct reftable_mmap *out, int fd, size_t len)
+{
+	git_map *mmap;
+
+	mmap = git__malloc(sizeof(*mmap));
+	if (!mmap)
+		return REFTABLE_OUT_OF_MEMORY_ERROR;
+
+	if (p_mmap(mmap, len, GIT_PROT_READ, GIT_MAP_PRIVATE, fd, 0) < 0)
+		return REFTABLE_IO_ERROR;
+
+	out->priv = mmap;
+	out->data = mmap->data;
+	out->size = mmap->len;
+
+	return 0;
+}
+
+int reftable_munmap(struct reftable_mmap *mmap)
+{
+	git_map *map = mmap->priv;
+
+	if (p_munmap(map) < 0)
+		return REFTABLE_IO_ERROR;
+	git__free(map);
+	memset(mmap, 0, sizeof(*mmap));
+	return 0;
+}

deps/reftable/system.h

@@ -0,0 +1,135 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef SYSTEM_H
+#define SYSTEM_H
+
+/*
+ * This header defines the platform-agnostic interface that is to be
+ * implemented by the project to make it work on their respective supported
+ * systems, and to integrate it into the project itself. This header is not
+ * expected to be changed by the individual project.
+ */
+
+#include "reftable-system.h"
+
+/*
+ * Return a random 32 bit integer. This function is expected to return
+ * pre-seeded data.
+ */
+uint32_t reftable_rand(void);
+
+/*
+ * An implementation-specific temporary file. By making this specific to the
+ * implementation it becomes possible to tie temporary files into any kind of
+ * signal or atexit handlers for cleanup on abnormal situations.
+ */
+struct reftable_tmpfile {
+	const char *path;
+	int fd;
+	void *priv;
+};
+#define REFTABLE_TMPFILE_INIT ((struct reftable_tmpfile) { .fd = -1, })
+
+/*
+ * Create a temporary file from a pattern similar to how mkstemp(3p) would.
+ * The `pattern` shall not be modified. On success, the structure at `out` has
+ * been initialized such that it is ready for use. Returns 0 on success, a
+ * reftable error code on error.
+ */
+int tmpfile_from_pattern(struct reftable_tmpfile *out, const char *pattern);
+
+/*
+ * Close the temporary file's file descriptor without removing the file itself.
+ * This is a no-op in case the file has already been closed beforehand. Returns
+ * 0 on success, a reftable error code on error.
+ */
+int tmpfile_close(struct reftable_tmpfile *t);
+
+/*
+ * Close the temporary file and delete it. This is a no-op in case the file has
+ * already been deleted or renamed beforehand. Returns 0 on success, a reftable
+ * error code on error.
+ */
+int tmpfile_delete(struct reftable_tmpfile *t);
+
+/*
+ * Rename the temporary file to the provided path. The temporary file must be
+ * active. Return 0 on success, a reftable error code on error. Deactivates the
+ * temporary file.
+ */
+int tmpfile_rename(struct reftable_tmpfile *t, const char *path);
+
+/*
+ * An implementation-specific file lock. Same as with `reftable_tmpfile`,
+ * making this specific to the implementation makes it possible to tie this
+ * into signal or atexit handlers such that we know to clean up stale locks on
+ * abnormal exits.
+ */
+struct reftable_flock {
+	const char *path;
+	int fd;
+	void *priv;
+};
+#define REFTABLE_FLOCK_INIT ((struct reftable_flock){ .fd = -1, })
+
+/*
+ * Acquire the lock for the given target path by exclusively creating a file
+ * with ".lock" appended to it. If that lock exists, we wait up to `timeout_ms`
+ * to acquire the lock. If `timeout_ms` is 0 we don't wait, if it is negative
+ * we block indefinitely.
+ *
+ * Retrun 0 on success, a reftable error code on error. Specifically,
+ * `REFTABLE_LOCK_ERROR` should be returned in case the target path is already
+ * locked.
+ */
+int flock_acquire(struct reftable_flock *l, const char *target_path,
+		  long timeout_ms);
+
+/*
+ * Close the lockfile's file descriptor without removing the lock itself. This
+ * is a no-op in case the lockfile has already been closed beforehand. Returns
+ * 0 on success, a reftable error code on error.
+ */
+int flock_close(struct reftable_flock *l);
+
+/*
+ * Release the lock by unlinking the lockfile. This is a no-op in case the
+ * lockfile has already been released or committed beforehand. Returns 0 on
+ * success, a reftable error code on error.
+ */
+int flock_release(struct reftable_flock *l);
+
+/*
+ * Commit the lock by renaming the lockfile into place. Returns 0 on success, a
+ * reftable error code on error.
+ */
+int flock_commit(struct reftable_flock *l);
+
+/* Report the time in milliseconds. */
+uint64_t reftable_time_ms(void);
+
+struct reftable_mmap {
+       void *data;
+       size_t size;
+       void *priv;
+};
+
+/*
+ * Map the file into memory. Returns 0 on success, a reftable error code on
+ * error.
+ */
+int reftable_mmap(struct reftable_mmap *out, int fd, size_t len);
+
+/*
+ * Unmap the file from memory. Returns 0 on success, a reftable error code on
+ * error.
+ */
+int reftable_munmap(struct reftable_mmap *mmap);
+
+#endif

deps/reftable/table.c

@@ -0,0 +1,779 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#include "table.h"
+
+#include "system.h"
+#include "block.h"
+#include "blocksource.h"
+#include "constants.h"
+#include "iter.h"
+#include "record.h"
+#include "reftable-error.h"
+
+static struct reftable_table_offsets *
+table_offsets_for(struct reftable_table *t, uint8_t typ)
+{
+	switch (typ) {
+	case REFTABLE_BLOCK_TYPE_REF:
+		return &t->ref_offsets;
+	case REFTABLE_BLOCK_TYPE_LOG:
+		return &t->log_offsets;
+	case REFTABLE_BLOCK_TYPE_OBJ:
+		return &t->obj_offsets;
+	}
+	abort();
+}
+
+enum reftable_hash reftable_table_hash_id(struct reftable_table *t)
+{
+	return t->hash_id;
+}
+
+const char *reftable_table_name(struct reftable_table *t)
+{
+	return t->name;
+}
+
+static int parse_footer(struct reftable_table *t, uint8_t *footer,
+			uint8_t *header)
+{
+	uint8_t *f = footer;
+	uint8_t first_block_typ;
+	int err = 0;
+	uint32_t computed_crc;
+	uint32_t file_crc;
+
+	if (memcmp(f, "REFT", 4)) {
+		err = REFTABLE_FORMAT_ERROR;
+		goto done;
+	}
+	f += 4;
+
+	if (memcmp(footer, header, header_size(t->version))) {
+		err = REFTABLE_FORMAT_ERROR;
+		goto done;
+	}
+
+	f++;
+	t->block_size = reftable_get_be24(f);
+
+	f += 3;
+	t->min_update_index = reftable_get_be64(f);
+	f += 8;
+	t->max_update_index = reftable_get_be64(f);
+	f += 8;
+
+	if (t->version == 1) {
+		t->hash_id = REFTABLE_HASH_SHA1;
+	} else {
+		switch (reftable_get_be32(f)) {
+		case REFTABLE_FORMAT_ID_SHA1:
+			t->hash_id = REFTABLE_HASH_SHA1;
+			break;
+		case REFTABLE_FORMAT_ID_SHA256:
+			t->hash_id = REFTABLE_HASH_SHA256;
+			break;
+		default:
+			err = REFTABLE_FORMAT_ERROR;
+			goto done;
+		}
+
+		f += 4;
+	}
+
+	t->ref_offsets.index_offset = reftable_get_be64(f);
+	f += 8;
+
+	t->obj_offsets.offset = reftable_get_be64(f);
+	f += 8;
+
+	t->object_id_len = t->obj_offsets.offset & ((1 << 5) - 1);
+	t->obj_offsets.offset >>= 5;
+
+	t->obj_offsets.index_offset = reftable_get_be64(f);
+	f += 8;
+	t->log_offsets.offset = reftable_get_be64(f);
+	f += 8;
+	t->log_offsets.index_offset = reftable_get_be64(f);
+	f += 8;
+
+	computed_crc = crc32(0, footer, f - footer);
+	file_crc = reftable_get_be32(f);
+	f += 4;
+	if (computed_crc != file_crc) {
+		err = REFTABLE_FORMAT_ERROR;
+		goto done;
+	}
+
+	first_block_typ = header[header_size(t->version)];
+	t->ref_offsets.is_present = (first_block_typ == REFTABLE_BLOCK_TYPE_REF);
+	t->ref_offsets.offset = 0;
+	t->log_offsets.is_present = (first_block_typ == REFTABLE_BLOCK_TYPE_LOG ||
+				     t->log_offsets.offset > 0);
+	t->obj_offsets.is_present = t->obj_offsets.offset > 0;
+	if (t->obj_offsets.is_present && !t->object_id_len) {
+		err = REFTABLE_FORMAT_ERROR;
+		goto done;
+	}
+
+	err = 0;
+done:
+	return err;
+}
+
+struct table_iter {
+	struct reftable_table *table;
+	uint8_t typ;
+	uint64_t block_off;
+	struct reftable_block block;
+	struct block_iter bi;
+	int is_finished;
+};
+
+static int table_iter_init(struct table_iter *ti, struct reftable_table *t)
+{
+	struct block_iter bi = BLOCK_ITER_INIT;
+	memset(ti, 0, sizeof(*ti));
+	reftable_table_incref(t);
+	ti->table = t;
+	ti->bi = bi;
+	return 0;
+}
+
+static int table_iter_next_in_block(struct table_iter *ti,
+				    struct reftable_record *rec)
+{
+	int res = block_iter_next(&ti->bi, rec);
+	if (res == 0 && reftable_record_type(rec) == REFTABLE_BLOCK_TYPE_REF) {
+		rec->u.ref.update_index += ti->table->min_update_index;
+	}
+
+	return res;
+}
+
+static void table_iter_block_done(struct table_iter *ti)
+{
+	reftable_block_release(&ti->block);
+	block_iter_reset(&ti->bi);
+}
+
+int table_init_block(struct reftable_table *t, struct reftable_block *block,
+		     uint64_t next_off, uint8_t want_typ)
+{
+	uint32_t header_off = next_off ? 0 : header_size(t->version);
+	int err;
+
+	if (next_off >= t->size)
+		return 1;
+
+	err = reftable_block_init(block, &t->source, next_off, header_off,
+				  t->block_size, hash_size(t->hash_id), want_typ);
+	if (err)
+		reftable_block_release(block);
+	return err;
+}
+
+static void table_iter_close(struct table_iter *ti)
+{
+	table_iter_block_done(ti);
+	block_iter_close(&ti->bi);
+	reftable_table_decref(ti->table);
+}
+
+static int table_iter_next_block(struct table_iter *ti)
+{
+	uint64_t next_block_off = ti->block_off + ti->block.full_block_size;
+	int err;
+
+	err = table_init_block(ti->table, &ti->block, next_block_off, ti->typ);
+	if (err > 0)
+		ti->is_finished = 1;
+	if (err)
+		return err;
+
+	ti->block_off = next_block_off;
+	ti->is_finished = 0;
+	block_iter_init(&ti->bi, &ti->block);
+
+	return 0;
+}
+
+static int table_iter_next(struct table_iter *ti, struct reftable_record *rec)
+{
+	if (reftable_record_type(rec) != ti->typ)
+		return REFTABLE_API_ERROR;
+
+	while (1) {
+		int err;
+
+		if (ti->is_finished)
+			return 1;
+
+		/*
+		 * Check whether the current block still has more records. If
+		 * so, return it. If the iterator returns positive then the
+		 * current block has been exhausted.
+		 */
+		err = table_iter_next_in_block(ti, rec);
+		if (err <= 0)
+			return err;
+
+		/*
+		 * Otherwise, we need to continue to the next block in the
+		 * table and retry. If there are no more blocks then the
+		 * iterator is drained.
+		 */
+		err = table_iter_next_block(ti);
+		if (err) {
+			ti->is_finished = 1;
+			return err;
+		}
+	}
+}
+
+static int table_iter_seek_to(struct table_iter *ti, uint64_t off, uint8_t typ)
+{
+	int err;
+
+	err = table_init_block(ti->table, &ti->block, off, typ);
+	if (err != 0)
+		return err;
+
+	ti->typ = reftable_block_type(&ti->block);
+	ti->block_off = off;
+	block_iter_init(&ti->bi, &ti->block);
+	ti->is_finished = 0;
+	return 0;
+}
+
+static int table_iter_seek_start(struct table_iter *ti, uint8_t typ, int index)
+{
+	struct reftable_table_offsets *offs = table_offsets_for(ti->table, typ);
+	uint64_t off = offs->offset;
+	if (index) {
+		off = offs->index_offset;
+		if (off == 0) {
+			return 1;
+		}
+		typ = REFTABLE_BLOCK_TYPE_INDEX;
+	}
+
+	return table_iter_seek_to(ti, off, typ);
+}
+
+static int table_iter_seek_linear(struct table_iter *ti,
+				  struct reftable_record *want)
+{
+	struct reftable_buf want_key = REFTABLE_BUF_INIT;
+	struct reftable_buf got_key = REFTABLE_BUF_INIT;
+	struct reftable_record rec;
+	int err;
+
+	err = reftable_record_init(&rec, reftable_record_type(want));
+	if (err < 0)
+		goto done;
+
+	err = reftable_record_key(want, &want_key);
+	if (err < 0)
+		goto done;
+
+	/*
+	 * First we need to locate the block that must contain our record. To
+	 * do so we scan through blocks linearly until we find the first block
+	 * whose first key is bigger than our wanted key. Once we have found
+	 * that block we know that the key must be contained in the preceding
+	 * block.
+	 *
+	 * This algorithm is somewhat unfortunate because it means that we
+	 * always have to seek one block too far and then back up. But as we
+	 * can only decode the _first_ key of a block but not its _last_ key we
+	 * have no other way to do this.
+	 */
+	while (1) {
+		struct table_iter next = *ti;
+
+		/*
+		 * We must be careful to not modify underlying data of `ti`
+		 * because we may find that `next` does not contain our desired
+		 * block, but that `ti` does. In that case, we would discard
+		 * `next` and continue with `ti`.
+		 *
+		 * This also means that we cannot reuse allocated memory for
+		 * `next` here. While it would be great if we could, it should
+		 * in practice not be too bad given that we should only ever
+		 * end up doing linear seeks with at most three blocks. As soon
+		 * as we have more than three blocks we would have an index, so
+		 * we would not do a linear search there anymore.
+		 */
+		memset(&next.block.block_data, 0, sizeof(next.block.block_data));
+		next.block.zstream = NULL;
+		next.block.uncompressed_data = NULL;
+		next.block.uncompressed_cap = 0;
+
+		err = table_iter_next_block(&next);
+		if (err < 0)
+			goto done;
+		if (err > 0)
+			break;
+
+		err = reftable_block_first_key(&next.block, &got_key);
+		if (err < 0)
+			goto done;
+
+		if (reftable_buf_cmp(&got_key, &want_key) > 0) {
+			table_iter_block_done(&next);
+			break;
+		}
+
+		table_iter_block_done(ti);
+		*ti = next;
+	}
+
+	/*
+	 * We have located the block that must contain our record, so we seek
+	 * the wanted key inside of it. If the block does not contain our key
+	 * we know that the corresponding record does not exist.
+	 */
+	block_iter_init(&ti->bi, &ti->block);
+	err = block_iter_seek_key(&ti->bi, &want_key);
+	if (err < 0)
+		goto done;
+	err = 0;
+
+done:
+	reftable_record_release(&rec);
+	reftable_buf_release(&want_key);
+	reftable_buf_release(&got_key);
+	return err;
+}
+
+static int table_iter_seek_indexed(struct table_iter *ti,
+				   struct reftable_record *rec)
+{
+	struct reftable_record want_index = {
+		.type = REFTABLE_BLOCK_TYPE_INDEX, .u.idx = { .last_key = REFTABLE_BUF_INIT }
+	};
+	struct reftable_record index_result = {
+		.type = REFTABLE_BLOCK_TYPE_INDEX,
+		.u.idx = { .last_key = REFTABLE_BUF_INIT },
+	};
+	int err;
+
+	err = reftable_record_key(rec, &want_index.u.idx.last_key);
+	if (err < 0)
+		goto done;
+
+	/*
+	 * The index may consist of multiple levels, where each level may have
+	 * multiple index blocks. We start by doing a linear search in the
+	 * highest layer that identifies the relevant index block as well as
+	 * the record inside that block that corresponds to our wanted key.
+	 */
+	err = table_iter_seek_linear(ti, &want_index);
+	if (err < 0)
+		goto done;
+
+	/*
+	 * Traverse down the levels until we find a non-index entry.
+	 */
+	while (1) {
+		/*
+		 * In case we seek a record that does not exist the index iter
+		 * will tell us that the iterator is over. This works because
+		 * the last index entry of the current level will contain the
+		 * last key it knows about. So in case our seeked key is larger
+		 * than the last indexed key we know that it won't exist.
+		 *
+		 * There is one subtlety in the layout of the index section
+		 * that makes this work as expected: the highest-level index is
+		 * at end of the section and will point backwards and thus we
+		 * start reading from the end of the index section, not the
+		 * beginning.
+		 *
+		 * If that wasn't the case and the order was reversed then the
+		 * linear seek would seek into the lower levels and traverse
+		 * all levels of the index only to find out that the key does
+		 * not exist.
+		 */
+		err = table_iter_next(ti, &index_result);
+		if (err != 0)
+			goto done;
+
+		err = table_iter_seek_to(ti, index_result.u.idx.offset, 0);
+		if (err != 0)
+			goto done;
+
+		block_iter_init(&ti->bi, &ti->block);
+
+		err = block_iter_seek_key(&ti->bi, &want_index.u.idx.last_key);
+		if (err < 0)
+			goto done;
+
+		if (ti->typ == reftable_record_type(rec)) {
+			err = 0;
+			break;
+		}
+
+		if (ti->typ != REFTABLE_BLOCK_TYPE_INDEX) {
+			err = REFTABLE_FORMAT_ERROR;
+			goto done;
+		}
+	}
+
+done:
+	reftable_record_release(&want_index);
+	reftable_record_release(&index_result);
+	return err;
+}
+
+static int table_iter_seek(struct table_iter *ti,
+			   struct reftable_record *want)
+{
+	uint8_t typ = reftable_record_type(want);
+	struct reftable_table_offsets *offs = table_offsets_for(ti->table, typ);
+	int err;
+
+	err = table_iter_seek_start(ti, reftable_record_type(want),
+				    !!offs->index_offset);
+	if (err < 0)
+		goto out;
+
+	if (offs->index_offset)
+		err = table_iter_seek_indexed(ti, want);
+	else
+		err = table_iter_seek_linear(ti, want);
+	if (err)
+		goto out;
+
+out:
+	return err;
+}
+
+static int table_iter_seek_void(void *ti, struct reftable_record *want)
+{
+	return table_iter_seek(ti, want);
+}
+
+static int table_iter_next_void(void *ti, struct reftable_record *rec)
+{
+	return table_iter_next(ti, rec);
+}
+
+static void table_iter_close_void(void *ti)
+{
+	table_iter_close(ti);
+}
+
+static struct reftable_iterator_vtable table_iter_vtable = {
+	.seek = &table_iter_seek_void,
+	.next = &table_iter_next_void,
+	.close = &table_iter_close_void,
+};
+
+static void iterator_from_table_iter(struct reftable_iterator *it,
+				     struct table_iter *ti)
+{
+	assert(!it->ops);
+	it->iter_arg = ti;
+	it->ops = &table_iter_vtable;
+}
+
+int table_init_iter(struct reftable_table *t,
+		    struct reftable_iterator *it,
+		    uint8_t typ)
+{
+	struct reftable_table_offsets *offs = table_offsets_for(t, typ);
+
+	if (offs->is_present) {
+		struct table_iter *ti;
+		REFTABLE_ALLOC_ARRAY(ti, 1);
+		if (!ti)
+			return REFTABLE_OUT_OF_MEMORY_ERROR;
+
+		table_iter_init(ti, t);
+		iterator_from_table_iter(it, ti);
+	} else {
+		iterator_set_empty(it);
+	}
+
+	return 0;
+}
+
+int reftable_table_init_ref_iterator(struct reftable_table *t,
+				     struct reftable_iterator *it)
+{
+	return table_init_iter(t, it, REFTABLE_BLOCK_TYPE_REF);
+}
+
+int reftable_table_init_log_iterator(struct reftable_table *t,
+				     struct reftable_iterator *it)
+{
+	return table_init_iter(t, it, REFTABLE_BLOCK_TYPE_LOG);
+}
+
+int reftable_table_new(struct reftable_table **out,
+		       struct reftable_block_source *source, char const *name)
+{
+	struct reftable_block_data footer = { 0 };
+	struct reftable_block_data header = { 0 };
+	struct reftable_table *t;
+	uint64_t file_size = block_source_size(source);
+	uint32_t read_size;
+	ssize_t bytes_read;
+	int err;
+
+	REFTABLE_CALLOC_ARRAY(t, 1);
+	if (!t) {
+		err = REFTABLE_OUT_OF_MEMORY_ERROR;
+		goto done;
+	}
+
+	/*
+	 * We need one extra byte to read the type of first block. We also
+	 * pretend to always be reading v2 of the format because it is larger.
+	 */
+	read_size = header_size(2) + 1;
+	if (read_size > file_size) {
+		err = REFTABLE_FORMAT_ERROR;
+		goto done;
+	}
+
+	bytes_read = block_source_read_data(source, &header, 0, read_size);
+	if (bytes_read < 0 || (size_t)bytes_read != read_size) {
+		err = REFTABLE_IO_ERROR;
+		goto done;
+	}
+
+	if (memcmp(header.data, "REFT", 4)) {
+		err = REFTABLE_FORMAT_ERROR;
+		goto done;
+	}
+	t->version = header.data[4];
+	if (t->version != 1 && t->version != 2) {
+		err = REFTABLE_FORMAT_ERROR;
+		goto done;
+	}
+
+	t->size = file_size - footer_size(t->version);
+	t->source = *source;
+	t->name = reftable_strdup(name);
+	if (!t->name) {
+		err = REFTABLE_OUT_OF_MEMORY_ERROR;
+		goto done;
+	}
+	t->hash_id = 0;
+	t->refcount = 1;
+
+	bytes_read = block_source_read_data(source, &footer, t->size,
+					    footer_size(t->version));
+	if (bytes_read < 0 || (size_t)bytes_read != footer_size(t->version)) {
+		err = REFTABLE_IO_ERROR;
+		goto done;
+	}
+
+	err = parse_footer(t, footer.data, header.data);
+	if (err)
+		goto done;
+
+	*out = t;
+
+done:
+	block_source_release_data(&footer);
+	block_source_release_data(&header);
+	if (err) {
+		if (t)
+			reftable_free(t->name);
+		reftable_free(t);
+		block_source_close(source);
+	}
+	return err;
+}
+
+void reftable_table_incref(struct reftable_table *t)
+{
+	t->refcount++;
+}
+
+void reftable_table_decref(struct reftable_table *t)
+{
+	if (!t)
+		return;
+	if (--t->refcount)
+		return;
+	block_source_close(&t->source);
+	REFTABLE_FREE_AND_NULL(t->name);
+	reftable_free(t);
+}
+
+static int reftable_table_refs_for_indexed(struct reftable_table *t,
+					   struct reftable_iterator *it,
+					   uint8_t *oid)
+{
+	struct reftable_record want = {
+		.type = REFTABLE_BLOCK_TYPE_OBJ,
+		.u.obj = {
+			.hash_prefix = oid,
+			.hash_prefix_len = t->object_id_len,
+		},
+	};
+	struct reftable_iterator oit = { NULL };
+	struct reftable_record got = {
+		.type = REFTABLE_BLOCK_TYPE_OBJ,
+		.u.obj = { 0 },
+	};
+	int err = 0;
+	struct indexed_table_ref_iter *itr = NULL;
+
+	/* Look through the reverse index. */
+	err = table_init_iter(t, &oit, REFTABLE_BLOCK_TYPE_OBJ);
+	if (err < 0)
+		goto done;
+
+	err = iterator_seek(&oit, &want);
+	if (err != 0)
+		goto done;
+
+	/* read out the reftable_obj_record */
+	err = iterator_next(&oit, &got);
+	if (err < 0)
+		goto done;
+
+	if (err > 0 || memcmp(want.u.obj.hash_prefix, got.u.obj.hash_prefix,
+			      t->object_id_len)) {
+		/* didn't find it; return empty iterator */
+		iterator_set_empty(it);
+		err = 0;
+		goto done;
+	}
+
+	err = indexed_table_ref_iter_new(&itr, t, oid, hash_size(t->hash_id),
+					 got.u.obj.offsets,
+					 got.u.obj.offset_len);
+	if (err < 0)
+		goto done;
+	got.u.obj.offsets = NULL;
+	iterator_from_indexed_table_ref_iter(it, itr);
+
+done:
+	reftable_iterator_destroy(&oit);
+	reftable_record_release(&got);
+	return err;
+}
+
+static int reftable_table_refs_for_unindexed(struct reftable_table *t,
+					     struct reftable_iterator *it,
+					     uint8_t *oid)
+{
+	struct table_iter *ti;
+	struct filtering_ref_iterator *filter = NULL;
+	struct filtering_ref_iterator empty = FILTERING_REF_ITERATOR_INIT;
+	uint32_t oid_len = hash_size(t->hash_id);
+	int err;
+
+	REFTABLE_ALLOC_ARRAY(ti, 1);
+	if (!ti) {
+		err = REFTABLE_OUT_OF_MEMORY_ERROR;
+		goto out;
+	}
+
+	table_iter_init(ti, t);
+	err = table_iter_seek_start(ti, REFTABLE_BLOCK_TYPE_REF, 0);
+	if (err < 0)
+		goto out;
+
+	filter = reftable_malloc(sizeof(*filter));
+	if (!filter) {
+		err = REFTABLE_OUT_OF_MEMORY_ERROR;
+		goto out;
+	}
+	*filter = empty;
+
+	err = reftable_buf_add(&filter->oid, oid, oid_len);
+	if (err < 0)
+		goto out;
+
+	iterator_from_table_iter(&filter->it, ti);
+
+	iterator_from_filtering_ref_iterator(it, filter);
+
+	err = 0;
+
+out:
+	if (err < 0) {
+		if (ti)
+			table_iter_close(ti);
+		reftable_free(ti);
+	}
+	return err;
+}
+
+int reftable_table_refs_for(struct reftable_table *t,
+			    struct reftable_iterator *it, uint8_t *oid)
+{
+	if (t->obj_offsets.is_present)
+		return reftable_table_refs_for_indexed(t, it, oid);
+	return reftable_table_refs_for_unindexed(t, it, oid);
+}
+
+uint64_t reftable_table_max_update_index(struct reftable_table *t)
+{
+	return t->max_update_index;
+}
+
+uint64_t reftable_table_min_update_index(struct reftable_table *t)
+{
+	return t->min_update_index;
+}
+
+int reftable_table_iterator_init(struct reftable_table_iterator *it,
+				 struct reftable_table *t)
+{
+	struct table_iter *ti;
+	int err;
+
+	REFTABLE_ALLOC_ARRAY(ti, 1);
+	if (!ti)
+		return REFTABLE_OUT_OF_MEMORY_ERROR;
+
+	err = table_iter_init(ti, t);
+	if (err < 0)
+		goto out;
+
+	it->iter_arg = ti;
+	err = 0;
+
+out:
+	if (err < 0)
+		reftable_free(ti);
+	return err;
+}
+
+void reftable_table_iterator_release(struct reftable_table_iterator *it)
+{
+	if (!it->iter_arg)
+		return;
+	table_iter_close(it->iter_arg);
+	reftable_free(it->iter_arg);
+	it->iter_arg = NULL;
+}
+
+int reftable_table_iterator_next(struct reftable_table_iterator *it,
+				 const struct reftable_block **out)
+{
+	struct table_iter *ti = it->iter_arg;
+	int err;
+
+	err = table_iter_next_block(ti);
+	if (err)
+		return err;
+
+	*out = &ti->block;
+
+	return 0;
+}

deps/reftable/table.h

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef TABLE_H
+#define TABLE_H
+
+#include "block.h"
+#include "record.h"
+#include "reftable-iterator.h"
+#include "reftable-table.h"
+
+const char *reftable_table_name(struct reftable_table *t);
+
+int table_init_iter(struct reftable_table *t,
+		    struct reftable_iterator *it,
+		    uint8_t typ);
+
+/*
+ * Initialize a block by reading from the given table and offset.
+ */
+int table_init_block(struct reftable_table *t, struct reftable_block *block,
+		     uint64_t next_off, uint8_t want_typ);
+
+#endif

deps/reftable/tree.c

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#include "system.h"
+#include "tree.h"
+
+#include "basics.h"
+
+struct tree_node *tree_search(struct tree_node *tree,
+			      void *key,
+			      int (*compare)(const void *, const void *))
+{
+	int res;
+	if (!tree)
+		return NULL;
+	res = compare(key, tree->key);
+	if (res < 0)
+		return tree_search(tree->left, key, compare);
+	else if (res > 0)
+		return tree_search(tree->right, key, compare);
+	return tree;
+}
+
+struct tree_node *tree_insert(struct tree_node **rootp,
+			      void *key,
+			      int (*compare)(const void *, const void *))
+{
+	int res;
+
+	if (!*rootp) {
+		struct tree_node *n;
+
+		REFTABLE_CALLOC_ARRAY(n, 1);
+		if (!n)
+			return NULL;
+
+		n->key = key;
+		*rootp = n;
+		return *rootp;
+	}
+
+	res = compare(key, (*rootp)->key);
+	if (res < 0)
+		return tree_insert(&(*rootp)->left, key, compare);
+	else if (res > 0)
+		return tree_insert(&(*rootp)->right, key, compare);
+	return *rootp;
+}
+
+void infix_walk(struct tree_node *t, void (*action)(void *arg, void *key),
+		void *arg)
+{
+	if (t->left)
+		infix_walk(t->left, action, arg);
+	action(arg, t->key);
+	if (t->right)
+		infix_walk(t->right, action, arg);
+}
+
+void tree_free(struct tree_node *t)
+{
+	if (!t)
+		return;
+	if (t->left)
+		tree_free(t->left);
+	if (t->right)
+		tree_free(t->right);
+	reftable_free(t);
+}

deps/reftable/tree.h

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef TREE_H
+#define TREE_H
+
+/* tree_node is a generic binary search tree. */
+struct tree_node {
+	void *key;
+	struct tree_node *left, *right;
+};
+
+/*
+ * Search the tree for the node matching the given key using `compare` as
+ * comparison function. Returns the node whose key matches or `NULL` in case
+ * the key does not exist in the tree.
+ */
+struct tree_node *tree_search(struct tree_node *tree,
+			      void *key,
+			      int (*compare)(const void *, const void *));
+
+/*
+ * Insert a node into the tree. Returns the newly inserted node if the key does
+ * not yet exist. Otherwise it returns the preexisting node. Returns `NULL`
+ * when allocating the new node fails.
+ */
+struct tree_node *tree_insert(struct tree_node **rootp,
+			      void *key,
+			      int (*compare)(const void *, const void *));
+
+/* performs an infix walk of the tree. */
+void infix_walk(struct tree_node *t, void (*action)(void *arg, void *key),
+		void *arg);
+
+/*
+ * deallocates the tree nodes recursively. Keys should be deallocated separately
+ * by walking over the tree. */
+void tree_free(struct tree_node *t);
+
+#endif

deps/reftable/writer.c

@@ -0,0 +1,887 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#include "writer.h"
+
+#include "system.h"
+
+#include "block.h"
+#include "constants.h"
+#include "record.h"
+#include "tree.h"
+#include "reftable-error.h"
+
+/* finishes a block, and writes it to storage */
+static int writer_flush_block(struct reftable_writer *w);
+
+/* deallocates memory related to the index */
+static void writer_clear_index(struct reftable_writer *w);
+
+/* finishes writing a 'r' (refs) or 'g' (reflogs) section */
+static int writer_finish_public_section(struct reftable_writer *w);
+
+static struct reftable_block_stats *
+writer_reftable_block_stats(struct reftable_writer *w, uint8_t typ)
+{
+	switch (typ) {
+	case 'r':
+		return &w->stats.ref_stats;
+	case 'o':
+		return &w->stats.obj_stats;
+	case 'i':
+		return &w->stats.idx_stats;
+	case 'g':
+		return &w->stats.log_stats;
+	}
+	abort();
+	return NULL;
+}
+
+/* write data, queuing the padding for the next write. Returns negative for
+ * error. */
+static int padded_write(struct reftable_writer *w, uint8_t *data, size_t len,
+			int padding)
+{
+	int n = 0;
+	if (w->pending_padding > 0) {
+		uint8_t *zeroed;
+		int n;
+
+		zeroed = reftable_calloc(w->pending_padding, sizeof(*zeroed));
+		if (!zeroed)
+			return -1;
+
+		n = w->write(w->write_arg, zeroed, w->pending_padding);
+		if (n < 0) {
+			reftable_free(zeroed);
+			return n;
+		}
+
+		w->pending_padding = 0;
+		reftable_free(zeroed);
+	}
+
+	w->pending_padding = padding;
+	n = w->write(w->write_arg, data, len);
+	if (n < 0)
+		return n;
+	n += padding;
+	return 0;
+}
+
+static void options_set_defaults(struct reftable_write_options *opts)
+{
+	if (opts->restart_interval == 0) {
+		opts->restart_interval = 16;
+	}
+
+	if (opts->hash_id == 0) {
+		opts->hash_id = REFTABLE_HASH_SHA1;
+	}
+	if (opts->block_size == 0) {
+		opts->block_size = DEFAULT_BLOCK_SIZE;
+	}
+}
+
+static int writer_version(struct reftable_writer *w)
+{
+	return (w->opts.hash_id == 0 || w->opts.hash_id == REFTABLE_HASH_SHA1) ?
+			     1 :
+			     2;
+}
+
+static int writer_write_header(struct reftable_writer *w, uint8_t *dest)
+{
+	memcpy(dest, "REFT", 4);
+
+	dest[4] = writer_version(w);
+
+	reftable_put_be24(dest + 5, w->opts.block_size);
+	reftable_put_be64(dest + 8, w->min_update_index);
+	reftable_put_be64(dest + 16, w->max_update_index);
+	if (writer_version(w) == 2) {
+		uint32_t hash_id;
+
+		switch (w->opts.hash_id) {
+		case REFTABLE_HASH_SHA1:
+			hash_id = REFTABLE_FORMAT_ID_SHA1;
+			break;
+		case REFTABLE_HASH_SHA256:
+			hash_id = REFTABLE_FORMAT_ID_SHA256;
+			break;
+		default:
+			return -1;
+		}
+
+		reftable_put_be32(dest + 24, hash_id);
+	}
+
+	return header_size(writer_version(w));
+}
+
+static int writer_reinit_block_writer(struct reftable_writer *w, uint8_t typ)
+{
+	int block_start = 0, ret;
+
+	if (w->next == 0)
+		block_start = header_size(writer_version(w));
+
+	reftable_buf_reset(&w->last_key);
+	ret = block_writer_init(&w->block_writer_data, typ, w->block,
+				w->opts.block_size, block_start,
+				hash_size(w->opts.hash_id));
+	if (ret < 0)
+		return ret;
+
+	w->block_writer = &w->block_writer_data;
+	w->block_writer->restart_interval = w->opts.restart_interval;
+
+	return 0;
+}
+
+int reftable_writer_new(struct reftable_writer **out,
+			ssize_t (*writer_func)(void *, const void *, size_t),
+			int (*flush_func)(void *),
+			void *writer_arg, const struct reftable_write_options *_opts)
+{
+	struct reftable_write_options opts = {0};
+	struct reftable_writer *wp;
+
+	wp = reftable_calloc(1, sizeof(*wp));
+	if (!wp)
+		return REFTABLE_OUT_OF_MEMORY_ERROR;
+
+	if (_opts)
+		opts = *_opts;
+	options_set_defaults(&opts);
+	if (opts.block_size >= (1 << 24))
+		return REFTABLE_API_ERROR;
+
+	reftable_buf_init(&wp->block_writer_data.last_key);
+	reftable_buf_init(&wp->last_key);
+	reftable_buf_init(&wp->scratch);
+	REFTABLE_CALLOC_ARRAY(wp->block, opts.block_size);
+	if (!wp->block) {
+		reftable_free(wp);
+		return REFTABLE_OUT_OF_MEMORY_ERROR;
+	}
+	wp->write = writer_func;
+	wp->write_arg = writer_arg;
+	wp->opts = opts;
+	wp->flush = flush_func;
+	writer_reinit_block_writer(wp, REFTABLE_BLOCK_TYPE_REF);
+
+	*out = wp;
+
+	return 0;
+}
+
+int reftable_writer_set_limits(struct reftable_writer *w, uint64_t min,
+				uint64_t max)
+{
+	/*
+	  * Set the min/max update index limits for the reftable writer.
+	  * This must be called before adding any records, since:
+	  * - The 'next' field gets set after writing the first block.
+	  * - The 'last_key' field updates with each new record (but resets
+	  *   after sections).
+	  * Returns REFTABLE_API_ERROR if called after writing has begun.
+	 */
+	if (w->next || w->last_key.len)
+		return REFTABLE_API_ERROR;
+
+	w->min_update_index = min;
+	w->max_update_index = max;
+
+	return 0;
+}
+
+static void writer_release(struct reftable_writer *w)
+{
+	if (w) {
+		reftable_free(w->block);
+		w->block = NULL;
+		block_writer_release(&w->block_writer_data);
+		w->block_writer = NULL;
+		writer_clear_index(w);
+		reftable_buf_release(&w->last_key);
+		reftable_buf_release(&w->scratch);
+	}
+}
+
+void reftable_writer_free(struct reftable_writer *w)
+{
+	writer_release(w);
+	reftable_free(w);
+}
+
+struct obj_index_tree_node {
+	struct reftable_buf hash;
+	uint64_t *offsets;
+	size_t offset_len;
+	size_t offset_cap;
+};
+
+#define OBJ_INDEX_TREE_NODE_INIT    \
+	{                           \
+		.hash = REFTABLE_BUF_INIT \
+	}
+
+static int obj_index_tree_node_compare(const void *a, const void *b)
+{
+	return reftable_buf_cmp(&((const struct obj_index_tree_node *)a)->hash,
+			  &((const struct obj_index_tree_node *)b)->hash);
+}
+
+static int writer_index_hash(struct reftable_writer *w, struct reftable_buf *hash)
+{
+	uint64_t off = w->next;
+	struct obj_index_tree_node want = { .hash = *hash };
+	struct obj_index_tree_node *key;
+	struct tree_node *node;
+
+	node = tree_search(w->obj_index_tree, &want, &obj_index_tree_node_compare);
+	if (!node) {
+		struct obj_index_tree_node empty = OBJ_INDEX_TREE_NODE_INIT;
+		int err;
+
+		key = reftable_malloc(sizeof(*key));
+		if (!key)
+			return REFTABLE_OUT_OF_MEMORY_ERROR;
+
+		*key = empty;
+
+		reftable_buf_reset(&key->hash);
+		err = reftable_buf_add(&key->hash, hash->buf, hash->len);
+		if (err < 0) {
+			reftable_free(key);
+			return err;
+		}
+		tree_insert(&w->obj_index_tree, key,
+			    &obj_index_tree_node_compare);
+	} else {
+		key = node->key;
+	}
+
+	if (key->offset_len > 0 && key->offsets[key->offset_len - 1] == off)
+		return 0;
+
+	REFTABLE_ALLOC_GROW_OR_NULL(key->offsets, key->offset_len + 1,
+				    key->offset_cap);
+	if (!key->offsets)
+		return REFTABLE_OUT_OF_MEMORY_ERROR;
+	key->offsets[key->offset_len++] = off;
+
+	return 0;
+}
+
+static int writer_add_record(struct reftable_writer *w,
+			     struct reftable_record *rec)
+{
+	int err;
+
+	err = reftable_record_key(rec, &w->scratch);
+	if (err < 0)
+		goto done;
+
+	if (reftable_buf_cmp(&w->last_key, &w->scratch) >= 0) {
+		err = REFTABLE_API_ERROR;
+		goto done;
+	}
+
+	reftable_buf_reset(&w->last_key);
+	err = reftable_buf_add(&w->last_key, w->scratch.buf, w->scratch.len);
+	if (err < 0)
+		goto done;
+
+	if (!w->block_writer) {
+		err = writer_reinit_block_writer(w, reftable_record_type(rec));
+		if (err < 0)
+			goto done;
+	}
+
+	if (block_writer_type(w->block_writer) != reftable_record_type(rec))
+		return REFTABLE_API_ERROR;
+
+	/*
+	 * Try to add the record to the writer. If this succeeds then we're
+	 * done. Otherwise the block writer may have hit the block size limit
+	 * and needs to be flushed.
+	 */
+	err = block_writer_add(w->block_writer, rec);
+	if (err == 0)
+		goto done;
+
+	if (err != REFTABLE_ENTRY_TOO_BIG_ERROR)
+		goto done;
+	/*
+	 * The current block is full, so we need to flush and reinitialize the
+	 * writer to start writing the next block.
+	 */
+	err = writer_flush_block(w);
+	if (err < 0)
+		goto done;
+	err = writer_reinit_block_writer(w, reftable_record_type(rec));
+	if (err < 0)
+		goto done;
+
+	/*
+	 * Try to add the record to the writer again. If this still fails then
+	 * the record does not fit into the block size.
+	 */
+	err = block_writer_add(w->block_writer, rec);
+	if (err)
+		goto done;
+
+done:
+	return err;
+}
+
+int reftable_writer_add_ref(struct reftable_writer *w,
+			    struct reftable_ref_record *ref)
+{
+	struct reftable_record rec = {
+		.type = REFTABLE_BLOCK_TYPE_REF,
+		.u = {
+			.ref = *ref
+		},
+	};
+	int err;
+
+	if (!ref->refname ||
+	    ref->update_index < w->min_update_index ||
+	    ref->update_index > w->max_update_index)
+		return REFTABLE_API_ERROR;
+
+	rec.u.ref.update_index -= w->min_update_index;
+
+	err = writer_add_record(w, &rec);
+	if (err < 0)
+		goto out;
+
+	if (!w->opts.skip_index_objects && reftable_ref_record_val1(ref)) {
+		reftable_buf_reset(&w->scratch);
+		err = reftable_buf_add(&w->scratch, (char *)reftable_ref_record_val1(ref),
+				       hash_size(w->opts.hash_id));
+		if (err < 0)
+			goto out;
+
+		err = writer_index_hash(w, &w->scratch);
+		if (err < 0)
+			goto out;
+	}
+
+	if (!w->opts.skip_index_objects && reftable_ref_record_val2(ref)) {
+		reftable_buf_reset(&w->scratch);
+		err = reftable_buf_add(&w->scratch, reftable_ref_record_val2(ref),
+				       hash_size(w->opts.hash_id));
+		if (err < 0)
+			goto out;
+
+		err = writer_index_hash(w, &w->scratch);
+		if (err < 0)
+			goto out;
+	}
+
+	err = 0;
+
+out:
+	return err;
+}
+
+int reftable_writer_add_refs(struct reftable_writer *w,
+			     struct reftable_ref_record *refs, size_t n)
+{
+	int err = 0;
+
+	if (n)
+		qsort(refs, n, sizeof(*refs), reftable_ref_record_compare_name);
+
+	for (size_t i = 0; err == 0 && i < n; i++)
+		err = reftable_writer_add_ref(w, &refs[i]);
+
+	return err;
+}
+
+static int reftable_writer_add_log_verbatim(struct reftable_writer *w,
+					    struct reftable_log_record *log)
+{
+	struct reftable_record rec = {
+		.type = REFTABLE_BLOCK_TYPE_LOG,
+		.u = {
+			.log = *log,
+		},
+	};
+	if (w->block_writer &&
+	    block_writer_type(w->block_writer) == REFTABLE_BLOCK_TYPE_REF) {
+		int err = writer_finish_public_section(w);
+		if (err < 0)
+			return err;
+	}
+
+	w->next -= w->pending_padding;
+	w->pending_padding = 0;
+	return writer_add_record(w, &rec);
+}
+
+int reftable_writer_add_log(struct reftable_writer *w,
+			    struct reftable_log_record *log)
+{
+	char *input_log_message = NULL;
+	struct reftable_buf cleaned_message = REFTABLE_BUF_INIT;
+	int err = 0;
+
+	if (log->value_type == REFTABLE_LOG_DELETION)
+		return reftable_writer_add_log_verbatim(w, log);
+
+	/*
+	 * Verify only the upper limit of the update_index. Each reflog entry
+	 * is tied to a specific update_index. Entries in the reflog can be
+	 * replaced by adding a new entry with the same update_index,
+	 * effectively canceling the old one.
+	 *
+	 * Consequently, reflog updates may include update_index values lower
+	 * than the writer's min_update_index.
+	 */
+	if (log->update_index > w->max_update_index)
+		return REFTABLE_API_ERROR;
+
+	if (!log->refname)
+		return REFTABLE_API_ERROR;
+
+	input_log_message = log->value.update.message;
+	if (!w->opts.exact_log_message && log->value.update.message) {
+		err = reftable_buf_addstr(&cleaned_message, log->value.update.message);
+		if (err < 0)
+			goto done;
+
+		while (cleaned_message.len &&
+		       cleaned_message.buf[cleaned_message.len - 1] == '\n') {
+			err = reftable_buf_setlen(&cleaned_message,
+						  cleaned_message.len - 1);
+			if (err < 0)
+				goto done;
+		}
+		if (strchr(cleaned_message.buf, '\n')) {
+			/* multiple lines not allowed. */
+			err = REFTABLE_API_ERROR;
+			goto done;
+		}
+
+		err = reftable_buf_addstr(&cleaned_message, "\n");
+		if (err < 0)
+			goto done;
+
+		log->value.update.message = cleaned_message.buf;
+	}
+
+	err = reftable_writer_add_log_verbatim(w, log);
+	log->value.update.message = input_log_message;
+done:
+	reftable_buf_release(&cleaned_message);
+	return err;
+}
+
+int reftable_writer_add_logs(struct reftable_writer *w,
+			     struct reftable_log_record *logs, size_t n)
+{
+	int err = 0;
+
+	if (n)
+		qsort(logs, n, sizeof(*logs), reftable_log_record_compare_key);
+
+	for (size_t i = 0; err == 0 && i < n; i++)
+		err = reftable_writer_add_log(w, &logs[i]);
+
+	return err;
+}
+
+static int writer_finish_section(struct reftable_writer *w)
+{
+	struct reftable_block_stats *bstats = NULL;
+	uint8_t typ = block_writer_type(w->block_writer);
+	uint64_t index_start = 0;
+	int max_level = 0;
+	size_t threshold = w->opts.unpadded ? 1 : 3;
+	int before_blocks = w->stats.idx_stats.blocks;
+	int err;
+
+	err = writer_flush_block(w);
+	if (err < 0)
+		return err;
+
+	/*
+	 * When the section we are about to index has a lot of blocks then the
+	 * index itself may span across multiple blocks, as well. This would
+	 * require a linear scan over index blocks only to find the desired
+	 * indexed block, which is inefficient. Instead, we write a multi-level
+	 * index where index records of level N+1 will refer to index blocks of
+	 * level N. This isn't constant time, either, but at least logarithmic.
+	 *
+	 * This loop handles writing this multi-level index. Note that we write
+	 * the lowest-level index pointing to the indexed blocks first. We then
+	 * continue writing additional index levels until the current level has
+	 * less blocks than the threshold so that the highest level will be at
+	 * the end of the index section.
+	 *
+	 * Readers are thus required to start reading the index section from
+	 * its end, which is why we set `index_start` to the beginning of the
+	 * last index section.
+	 */
+	while (w->index_len > threshold) {
+		struct reftable_index_record *idx = NULL;
+		size_t i, idx_len;
+
+		max_level++;
+		index_start = w->next;
+		err = writer_reinit_block_writer(w, REFTABLE_BLOCK_TYPE_INDEX);
+		if (err < 0)
+			return err;
+
+		idx = w->index;
+		idx_len = w->index_len;
+
+		w->index = NULL;
+		w->index_len = 0;
+		w->index_cap = 0;
+		for (i = 0; i < idx_len; i++) {
+			struct reftable_record rec = {
+				.type = REFTABLE_BLOCK_TYPE_INDEX,
+				.u = {
+					.idx = idx[i],
+				},
+			};
+
+			err = writer_add_record(w, &rec);
+			if (err < 0)
+				return err;
+		}
+
+		err = writer_flush_block(w);
+		if (err < 0)
+			return err;
+
+		for (i = 0; i < idx_len; i++)
+			reftable_buf_release(&idx[i].last_key);
+		reftable_free(idx);
+	}
+
+	/*
+	 * The index may still contain a number of index blocks lower than the
+	 * threshold. Clear it so that these entries don't leak into the next
+	 * index section.
+	 */
+	writer_clear_index(w);
+
+	bstats = writer_reftable_block_stats(w, typ);
+	bstats->index_blocks = w->stats.idx_stats.blocks - before_blocks;
+	bstats->index_offset = index_start;
+	bstats->max_index_level = max_level;
+
+	/* Reinit lastKey, as the next section can start with any key. */
+	reftable_buf_reset(&w->last_key);
+
+	return 0;
+}
+
+struct common_prefix_arg {
+	struct reftable_buf *last;
+	size_t max;
+};
+
+static void update_common(void *void_arg, void *key)
+{
+	struct common_prefix_arg *arg = void_arg;
+	struct obj_index_tree_node *entry = key;
+	if (arg->last) {
+		size_t n = common_prefix_size(&entry->hash, arg->last);
+		if (n > arg->max)
+			arg->max = n;
+	}
+	arg->last = &entry->hash;
+}
+
+struct write_record_arg {
+	struct reftable_writer *w;
+	int err;
+};
+
+static void write_object_record(void *void_arg, void *key)
+{
+	struct write_record_arg *arg = void_arg;
+	struct obj_index_tree_node *entry = key;
+	struct reftable_record
+		rec = { .type = REFTABLE_BLOCK_TYPE_OBJ,
+			.u.obj = {
+				.hash_prefix = (uint8_t *)entry->hash.buf,
+				.hash_prefix_len = arg->w->stats.object_id_len,
+				.offsets = entry->offsets,
+				.offset_len = entry->offset_len,
+			} };
+	if (arg->err < 0)
+		goto done;
+
+	/*
+	 * Try to add the record to the writer. If this succeeds then we're
+	 * done. Otherwise the block writer may have hit the block size limit
+	 * and needs to be flushed.
+	 */
+	arg->err = block_writer_add(arg->w->block_writer, &rec);
+	if (arg->err == 0)
+		goto done;
+
+	if (arg->err != REFTABLE_ENTRY_TOO_BIG_ERROR)
+		goto done;
+
+	/*
+	 * The current block is full, so we need to flush and reinitialize the
+	 * writer to start writing the next block.
+	 */
+	arg->err = writer_flush_block(arg->w);
+	if (arg->err < 0)
+		goto done;
+
+	arg->err = writer_reinit_block_writer(arg->w, REFTABLE_BLOCK_TYPE_OBJ);
+	if (arg->err < 0)
+		goto done;
+
+	/*
+	 * If this still fails then we may need to reset record's offset
+	 * length to reduce the data size to be written.
+	 */
+	arg->err = block_writer_add(arg->w->block_writer, &rec);
+	if (arg->err == 0)
+		goto done;
+
+	if (arg->err != REFTABLE_ENTRY_TOO_BIG_ERROR)
+		goto done;
+
+	rec.u.obj.offset_len = 0;
+	arg->err = block_writer_add(arg->w->block_writer, &rec);
+
+	/* Should be able to write into a fresh block. */
+	assert(arg->err == 0);
+
+done:;
+}
+
+static void object_record_free(void *void_arg REFTABLE_UNUSED, void *key)
+{
+	struct obj_index_tree_node *entry = key;
+
+	REFTABLE_FREE_AND_NULL(entry->offsets);
+	reftable_buf_release(&entry->hash);
+	reftable_free(entry);
+}
+
+static int writer_dump_object_index(struct reftable_writer *w)
+{
+	struct write_record_arg closure = { .w = w };
+	struct common_prefix_arg common = {
+		.max = 1,		/* obj_id_len should be >= 2. */
+	};
+	int err;
+
+	if (w->obj_index_tree)
+		infix_walk(w->obj_index_tree, &update_common, &common);
+	w->stats.object_id_len = common.max + 1;
+
+	err = writer_reinit_block_writer(w, REFTABLE_BLOCK_TYPE_OBJ);
+	if (err < 0)
+		return err;
+
+	if (w->obj_index_tree)
+		infix_walk(w->obj_index_tree, &write_object_record, &closure);
+
+	if (closure.err < 0)
+		return closure.err;
+	return writer_finish_section(w);
+}
+
+static int writer_finish_public_section(struct reftable_writer *w)
+{
+	uint8_t typ = 0;
+	int err = 0;
+
+	if (!w->block_writer)
+		return 0;
+
+	typ = block_writer_type(w->block_writer);
+	err = writer_finish_section(w);
+	if (err < 0)
+		return err;
+	if (typ == REFTABLE_BLOCK_TYPE_REF && !w->opts.skip_index_objects &&
+	    w->stats.ref_stats.index_blocks > 0) {
+		err = writer_dump_object_index(w);
+		if (err < 0)
+			return err;
+	}
+
+	if (w->obj_index_tree) {
+		infix_walk(w->obj_index_tree, &object_record_free, NULL);
+		tree_free(w->obj_index_tree);
+		w->obj_index_tree = NULL;
+	}
+
+	w->block_writer = NULL;
+	return 0;
+}
+
+int reftable_writer_close(struct reftable_writer *w)
+{
+	uint8_t footer[72];
+	uint8_t *p = footer;
+	int err = writer_finish_public_section(w);
+	int empty_table = w->next == 0;
+	if (err != 0)
+		goto done;
+	w->pending_padding = 0;
+	if (empty_table) {
+		/* Empty tables need a header anyway. */
+		uint8_t header[28];
+		int n = writer_write_header(w, header);
+		err = padded_write(w, header, n, 0);
+		if (err < 0)
+			goto done;
+	}
+
+	p += writer_write_header(w, footer);
+	reftable_put_be64(p, w->stats.ref_stats.index_offset);
+	p += 8;
+	reftable_put_be64(p, (w->stats.obj_stats.offset) << 5 | w->stats.object_id_len);
+	p += 8;
+	reftable_put_be64(p, w->stats.obj_stats.index_offset);
+	p += 8;
+
+	reftable_put_be64(p, w->stats.log_stats.offset);
+	p += 8;
+	reftable_put_be64(p, w->stats.log_stats.index_offset);
+	p += 8;
+
+	reftable_put_be32(p, crc32(0, footer, p - footer));
+	p += 4;
+
+	err = w->flush(w->write_arg);
+	if (err < 0) {
+		err = REFTABLE_IO_ERROR;
+		goto done;
+	}
+
+	err = padded_write(w, footer, footer_size(writer_version(w)), 0);
+	if (err < 0)
+		goto done;
+
+	if (empty_table) {
+		err = REFTABLE_EMPTY_TABLE_ERROR;
+		goto done;
+	}
+
+done:
+	writer_release(w);
+	return err;
+}
+
+static void writer_clear_index(struct reftable_writer *w)
+{
+	for (size_t i = 0; w->index && i < w->index_len; i++)
+		reftable_buf_release(&w->index[i].last_key);
+	REFTABLE_FREE_AND_NULL(w->index);
+	w->index_len = 0;
+	w->index_cap = 0;
+}
+
+static int writer_flush_nonempty_block(struct reftable_writer *w)
+{
+	struct reftable_index_record index_record = {
+		.last_key = REFTABLE_BUF_INIT,
+	};
+	uint8_t typ = block_writer_type(w->block_writer);
+	struct reftable_block_stats *bstats;
+	int raw_bytes, padding = 0, err;
+	uint64_t block_typ_off;
+
+	/*
+	 * Finish the current block. This will cause the block writer to emit
+	 * restart points and potentially compress records in case we are
+	 * writing a log block.
+	 *
+	 * Note that this is still happening in memory.
+	 */
+	raw_bytes = block_writer_finish(w->block_writer);
+	if (raw_bytes < 0)
+		return raw_bytes;
+
+	/*
+	 * By default, all records except for log records are padded to the
+	 * block size.
+	 */
+	if (!w->opts.unpadded && typ != REFTABLE_BLOCK_TYPE_LOG)
+		padding = w->opts.block_size - raw_bytes;
+
+	bstats = writer_reftable_block_stats(w, typ);
+	block_typ_off = (bstats->blocks == 0) ? w->next : 0;
+	if (block_typ_off > 0)
+		bstats->offset = block_typ_off;
+	bstats->entries += w->block_writer->entries;
+	bstats->restarts += w->block_writer->restart_len;
+	bstats->blocks++;
+	w->stats.blocks++;
+
+	/*
+	 * If this is the first block we're writing to the table then we need
+	 * to also write the reftable header.
+	 */
+	if (!w->next)
+		writer_write_header(w, w->block);
+
+	err = padded_write(w, w->block, raw_bytes, padding);
+	if (err < 0)
+		return err;
+
+	/*
+	 * Add an index record for every block that we're writing. If we end up
+	 * having more than a threshold of index records we will end up writing
+	 * an index section in `writer_finish_section()`. Each index record
+	 * contains the last record key of the block it is indexing as well as
+	 * the offset of that block.
+	 *
+	 * Note that this also applies when flushing index blocks, in which
+	 * case we will end up with a multi-level index.
+	 */
+	REFTABLE_ALLOC_GROW_OR_NULL(w->index, w->index_len + 1, w->index_cap);
+	if (!w->index)
+		return REFTABLE_OUT_OF_MEMORY_ERROR;
+
+	index_record.offset = w->next;
+	reftable_buf_reset(&index_record.last_key);
+	err = reftable_buf_add(&index_record.last_key, w->block_writer->last_key.buf,
+			       w->block_writer->last_key.len);
+	if (err < 0)
+		return err;
+	w->index[w->index_len] = index_record;
+	w->index_len++;
+
+	w->next += padding + raw_bytes;
+	w->block_writer = NULL;
+
+	return 0;
+}
+
+static int writer_flush_block(struct reftable_writer *w)
+{
+	if (!w->block_writer)
+		return 0;
+	if (w->block_writer->entries == 0)
+		return 0;
+	return writer_flush_nonempty_block(w);
+}
+
+const struct reftable_stats *reftable_writer_stats(struct reftable_writer *w)
+{
+	return &w->stats;
+}

deps/reftable/writer.h

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file or at
+ * https://developers.google.com/open-source/licenses/bsd
+ */
+
+#ifndef WRITER_H
+#define WRITER_H
+
+#include "basics.h"
+#include "block.h"
+#include "tree.h"
+#include "reftable-writer.h"
+
+struct reftable_writer {
+	ssize_t (*write)(void *, const void *, size_t);
+	int (*flush)(void *);
+	void *write_arg;
+	int pending_padding;
+	struct reftable_buf last_key;
+	/* Scratch buffer used to avoid allocations. */
+	struct reftable_buf scratch;
+
+	/* offset of next block to write. */
+	uint64_t next;
+	uint64_t min_update_index, max_update_index;
+	struct reftable_write_options opts;
+
+	/* memory buffer for writing */
+	uint8_t *block;
+
+	/* writer for the current section. NULL or points to
+	 * block_writer_data */
+	struct block_writer *block_writer;
+
+	struct block_writer block_writer_data;
+
+	/* pending index records for the current section */
+	struct reftable_index_record *index;
+	size_t index_len;
+	size_t index_cap;
+
+	/*
+	 * tree for use with tsearch; used to populate the 'o' inverse OID
+	 * map */
+	struct tree_node *obj_index_tree;
+
+	struct reftable_stats stats;
+};
+
+#endif

include/git2/refdb.h

@@ -23,7 +23,8 @@ GIT_BEGIN_DECL
 
 /** The type of the refdb as determined by "extensions.refStorage". */
 typedef enum {
-	GIT_REFDB_FILES = 1 /**< Files backend using loose and packed refs. */
+	GIT_REFDB_FILES = 1, /**< Files backend using loose and packed refs. */
+	GIT_REFDB_REFTABLE = 2 /**< Reftable backend. */
 } git_refdb_t;
 
 /**

include/git2/sys/refdb_backend.h

@@ -383,6 +383,21 @@ GIT_EXTERN(int) git_refdb_backend_fs(
 	git_refdb_backend **backend_out,
 	git_repository *repo);
 
+/**
+ * Constructor for the reftable-based refdb backend
+ *
+ * Under normal usage, this is called for you when the repository is
+ * opened / created that uses the reftable format, but you can use this to
+ * explicitly construct a reftable refdb backend for a repository.
+ *
+ * @param backend_out Output pointer to the git_refdb_backend object
+ * @param repo Git repository to access
+ * @return 0 on success, <0 error code on failure
+ */
+GIT_EXTERN(int) git_refdb_backend_reftable(
+	git_refdb_backend **backend_out,
+	git_repository *repo);
+
 /**
  * Sets the custom backend to an existing reference DB
  *

src/CMakeLists.txt

@@ -149,6 +149,10 @@ set(GIT_BUILD_CPU "${CMAKE_SYSTEM_PROCESSOR}")
 execute_process(COMMAND git rev-parse HEAD
 	OUTPUT_VARIABLE GIT_BUILD_COMMIT OUTPUT_STRIP_TRAILING_WHITESPACE)
 
+add_subdirectory("${PROJECT_SOURCE_DIR}/deps/reftable" "${PROJECT_BINARY_DIR}/deps/reftable")
+list(APPEND LIBGIT2_DEPENDENCY_INCLUDES "${PROJECT_SOURCE_DIR}/deps/reftable")
+list(APPEND LIBGIT2_DEPENDENCY_OBJECTS "$<TARGET_OBJECTS:reftable>")
+
 #
 # Include child projects
 #

src/libgit2/libgit2.c

@@ -16,6 +16,7 @@
 #include "mwindow.h"
 #include "oid.h"
 #include "rand.h"
+#include "refdb_reftable.h"
 #include "runtime.h"
 #include "settings.h"
 #include "sysdir.h"
@@ -53,7 +54,8 @@ int git_libgit2_init(void)
 		git_mbedtls_stream_global_init,
 		git_mwindow_global_init,
 		git_pool_global_init,
-		git_settings_global_init
+		git_settings_global_init,
+		git_reftable_global_init
 	};
 
 	return git_runtime_init(init_fns, ARRAY_SIZE(init_fns));

src/libgit2/refdb.c

@@ -56,6 +56,10 @@ int git_refdb_open(git_refdb **out, git_repository *repo)
 		if ((error = git_refdb_backend_fs(&backend, repo)) < 0)
 			goto out;
 		break;
+	case GIT_REFDB_REFTABLE:
+		if ((error = git_refdb_backend_reftable(&backend, repo)) < 0)
+			goto out;
+		break;
 	default:
 		git_error_set(GIT_ERROR_REFERENCE, "unknown reference storage format");
 		error = GIT_EINVALID;

src/libgit2/refdb.h

@@ -137,6 +137,8 @@ GIT_INLINE(const char *) git_refdb_type_name(git_refdb_t type)
 	switch (type) {
 	case GIT_REFDB_FILES:
 		return "files";
+	case GIT_REFDB_REFTABLE:
+		return "reftable";
 	default:
 		return "unknown";
 	}
@@ -146,6 +148,8 @@ GIT_INLINE(git_refdb_t) git_refdb_type_fromstr(const char *name)
 {
 	if (strcmp(name, "files") == 0)
 		return GIT_REFDB_FILES;
+	if (strcmp(name, "reftable") == 0)
+		return GIT_REFDB_REFTABLE;
 	return 0;
 }
 

src/libgit2/refdb_fs.c

@@ -487,21 +487,6 @@ static const char *loose_parse_symbolic(git_str *file_content)
 	return refname_start;
 }
 
-/*
- * Returns whether a reference is stored per worktree or not.
- * Per-worktree references are:
- *
- * - all pseudorefs, e.g. HEAD and MERGE_HEAD
- * - all references stored inside of "refs/bisect/"
- */
-static bool is_per_worktree_ref(const char *ref_name)
-{
-	return git__prefixcmp(ref_name, "refs/") != 0 ||
-	       git__prefixcmp(ref_name, "refs/bisect/") == 0 ||
-	       git__prefixcmp(ref_name, "refs/worktree/") == 0 ||
-	       git__prefixcmp(ref_name, "refs/rewritten/") == 0;
-}
-
 int git_reference__lookup_loose(
 	git_reference **out,
 	const char *ref_dir,
@@ -551,7 +536,7 @@ static int loose_lookup(
 {
 	const char *ref_dir;
 
-	if (is_per_worktree_ref(ref_name))
+	if (git_reference__is_per_worktree_ref(ref_name))
 		ref_dir = backend->gitpath;
 	else
 		ref_dir = backend->commonpath;
@@ -995,11 +980,11 @@ static int iter_load_paths(
 		git_str_puts(&ctx->ref_name, entry->path);
 
 		if (worktree) {
-			if (!is_per_worktree_ref(ctx->ref_name.ptr))
+			if (!git_reference__is_per_worktree_ref(ctx->ref_name.ptr))
 				continue;
 		} else {
 			if (git_repository_is_worktree(ctx->backend->repo) &&
-			    is_per_worktree_ref(ctx->ref_name.ptr))
+			    git_reference__is_per_worktree_ref(ctx->ref_name.ptr))
 				continue;
 		}
 
@@ -1261,7 +1246,7 @@ static int loose_lock(git_filebuf *file, refdb_fs_backend *backend, const char *
 		return GIT_EINVALIDSPEC;
 	}
 
-	if (is_per_worktree_ref(name))
+	if (git_reference__is_per_worktree_ref(name))
 		basedir = backend->gitpath;
 	else
 		basedir = backend->commonpath;

src/libgit2/refdb_reftable.h

@@ -0,0 +1,14 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_refdb_reftable_h__
+#define INCLUDE_refdb_reftable_h__
+
+#include "common.h"
+
+int git_reftable_global_init(void);
+
+#endif

src/libgit2/refs.c

@@ -1323,6 +1323,25 @@ int git_reference__is_pseudoref(const char *ref_name)
 	return 0;
 }
 
+int git_reference__is_per_worktree_ref(const char *ref_name)
+{
+	const char * const worktree_refs[] = {
+		"refs/bisect/",
+		"refs/worktree/",
+		"refs/rewritten/",
+	};
+	size_t i;
+
+	if (git__prefixcmp(ref_name, "refs/") != 0)
+		return 1;
+
+	for (i = 0; i < ARRAY_SIZE(worktree_refs); i++)
+	       if (git__prefixcmp(ref_name, worktree_refs[i]) == 0)
+		       return 1;
+
+	return 0;
+}
+
 static int peel_error(int error, const git_reference *ref, const char *msg)
 {
 	git_error_set(

src/libgit2/refs.h

@@ -98,6 +98,15 @@ int git_reference__is_note(const char *ref_name);
 int git_reference__is_pseudoref(const char *ref_name);
 const char *git_reference__shorthand(const char *name);
 
+/**
+ * Returns whether a reference is stored per worktree or not.
+ * Per-worktree references are:
+ *
+ * - all pseudorefs, e.g. HEAD and MERGE_HEAD
+ * - all references stored inside of "refs/bisect/"
+ */
+int git_reference__is_per_worktree_ref(const char *ref_name);
+
 /*
  * A `git_reference_cmp` wrapper suitable for passing to generic
  * comparators, like `vector_cmp` / `tsort` / etc.

src/util/git2_util.h

@@ -19,12 +19,13 @@ typedef struct git_str git_str;
 
 /** Declare a function as always inlined. */
 #if defined(_MSC_VER)
-# define GIT_INLINE(type) static __inline type
+# define GIT_INLINE_KEYWORD __inline
 #elif defined(__GNUC__)
-# define GIT_INLINE(type) static __inline__ type
+# define GIT_INLINE_KEYWORD __inline__
 #else
-# define GIT_INLINE(type) static type
+# define GIT_INLINE_KEYWORD
 #endif
+#define GIT_INLINE(type) static GIT_INLINE_KEYWORD type
 
 /** Support for gcc/clang __has_builtin intrinsic */
 #ifndef __has_builtin

tests/libgit2/iterator/workdir.c

@@ -608,7 +608,7 @@ void test_iterator_workdir__filesystem(void)
 void test_iterator_workdir__filesystem2(void)
 {
 	git_iterator *i;
-	static const char *expect_base[] = {
+	static const char *expect_files[] = {
 		"heads/br2",
 		"heads/dir",
 		"heads/executable",
@@ -626,14 +626,21 @@ void test_iterator_workdir__filesystem2(void)
 		"tags/foo/foo/bar",
 		"tags/point_to_blob",
 		"tags/test",
-		NULL,
+	};
+	static const char *expect_reftable[] = {
+		"heads",
 	};
 
 	g_repo = cl_git_sandbox_init("testrepo");
 
 	cl_git_pass(git_iterator_for_filesystem(
 		&i, "testrepo/.git/refs", NULL));
-	expect_iterator_items(i, 17, expect_base, 17, expect_base);
+	if (cl_repo_has_ref_format(g_repo, "files"))
+		expect_iterator_items(i, ARRAY_SIZE(expect_files), expect_files,
+				      ARRAY_SIZE(expect_files), expect_files);
+	else
+		expect_iterator_items(i, ARRAY_SIZE(expect_reftable), expect_reftable,
+				      ARRAY_SIZE(expect_reftable), expect_reftable);
 	git_iterator_free(i);
 }
 

tests/libgit2/refs/basic.c

@@ -76,7 +76,10 @@ void test_refs_basic__longpaths(void)
 
 	/* Adding one more character gives us a path that is too long. */
 	cl_git_pass(git_str_putc(&refname, 'z'));
+	if (cl_repo_has_ref_format(g_repo, "files"))
 		cl_git_fail(git_reference_create(&two, g_repo, refname.ptr, &id, 0, NULL));
+	else
+		cl_git_pass(git_reference_create(&two, g_repo, refname.ptr, &id, 0, NULL));
 
 	git_reference_free(one);
 	git_reference_free(two);

tests/libgit2/refs/create.c

@@ -281,7 +281,10 @@ static void test_win32_name(const char *name)
 	ret = git_reference_create(&new_reference, g_repo, name, &id, 0, NULL);
 
 #ifdef GIT_WIN32
+	if (cl_repo_has_ref_format(g_repo, "files"))
 		cl_assert_equal_i(GIT_EINVALIDSPEC, ret);
+	else
+		cl_git_pass(ret);
 #else
 	cl_git_pass(ret);
 #endif

tests/libgit2/refs/foreachglob.c

@@ -8,10 +8,9 @@ void test_refs_foreachglob__initialize(void)
 {
 	git_oid id;
 
-	cl_fixture_sandbox("testrepo.git");
-	cl_git_pass(git_repository_open(&repo, "testrepo.git"));
+	repo = cl_git_sandbox_init("testrepo.git");
 
-	cl_git_pass(git_oid_from_string(&id, "be3563ae3f795b2b4353bcce3a527ad0a4f7f644", GIT_OID_SHA1));
+	cl_git_pass(git_oid_fromstr(&id, "be3563ae3f795b2b4353bcce3a527ad0a4f7f644"));
 	cl_git_pass(git_reference_create(&fake_remote, repo, "refs/remotes/nulltoken/master", &id, 0, NULL));
 }
 
@@ -19,11 +18,7 @@ void test_refs_foreachglob__cleanup(void)
 {
 	git_reference_free(fake_remote);
 	fake_remote = NULL;
-
-	git_repository_free(repo);
-	repo = NULL;
-
-	cl_fixture_cleanup("testrepo.git");
+	cl_git_sandbox_cleanup();
 }
 
 static int count_cb(const char *reference_name, void *payload)

tests/libgit2/refs/namespaces.c

@@ -27,10 +27,16 @@ void test_refs_namespaces__get_and_set(void)
 
 void test_refs_namespaces__namespace_doesnt_show_normal_refs(void)
 {
-	static git_strarray ref_list;
+	static git_strarray ref_list = { 0 };
 
 	cl_git_pass(git_repository_set_namespace(g_repo, "namespace"));
+
+	if (cl_repo_has_ref_format(g_repo, "reftable")) {
+		cl_git_fail(git_reference_list(&ref_list, g_repo));
+	} else {
 		cl_git_pass(git_reference_list(&ref_list, g_repo));
 		cl_assert_equal_i(0, ref_list.count);
+	}
+
 	git_strarray_dispose(&ref_list);
 }

tests/libgit2/refs/reflog/reflog.c

@@ -285,6 +285,9 @@ void test_refs_reflog_reflog__cannot_write_a_moved_reflog(void)
 	git_str master_log_path = GIT_STR_INIT, moved_log_path = GIT_STR_INIT;
 	git_reflog *reflog;
 
+	if (!cl_repo_has_ref_format(g_repo, "files"))
+		cl_skip();
+
 	cl_git_pass(git_reference_lookup(&master, g_repo, "refs/heads/master"));
 	cl_git_pass(git_reflog_read(&reflog, g_repo, "refs/heads/master"));
 
@@ -332,8 +335,12 @@ void test_refs_reflog_reflog__write_when_explicitly_active(void)
 	git_reference *ref;
 	git_oid id;
 
+	if (!cl_repo_has_ref_format(g_repo, "files"))
+		cl_skip();
+
 	git_oid_from_string(&id, current_master_tip, GIT_OID_SHA1);
-	git_reference_ensure_log(g_repo, "refs/tags/foo");
+
+	cl_git_pass(git_reference_ensure_log(g_repo, "refs/tags/foo"));
 
 	cl_git_pass(git_reference_create(&ref, g_repo, "refs/tags/foo", &id, 1, NULL));
 	git_reference_free(ref);

tests/libgit2/refs/revparse.c

@@ -131,12 +131,12 @@ static void test_invalid_revspec(const char* invalid_spec)
 
 void test_refs_revparse__initialize(void)
 {
-	cl_git_pass(git_repository_open(&g_repo, cl_fixture("testrepo.git")));
+	g_repo = cl_git_sandbox_init("testrepo.git");
 }
 
 void test_refs_revparse__cleanup(void)
 {
-	git_repository_free(g_repo);
+	cl_git_sandbox_cleanup();
 }
 
 void test_refs_revparse__nonexistant_object(void)
@@ -346,6 +346,7 @@ static void create_fake_stash_reference_and_reflog(git_repository *repo)
 	cl_git_pass(git_reference_rename(&new_master, master, "refs/fakestash", 0, NULL));
 	git_reference_free(master);
 
+	if (cl_repo_has_ref_format(g_repo, "files"))
 		cl_assert_equal_i(true, git_fs_path_isfile(git_str_cstr(&log_path)));
 
 	git_str_dispose(&log_path);
@@ -354,11 +355,9 @@ static void create_fake_stash_reference_and_reflog(git_repository *repo)
 
 void test_refs_revparse__reflog_of_a_ref_under_refs(void)
 {
-	git_repository *repo = cl_git_sandbox_init("testrepo.git");
+	test_object_inrepo("refs/fakestash", NULL, g_repo);
 
-	test_object_inrepo("refs/fakestash", NULL, repo);
-
-	create_fake_stash_reference_and_reflog(repo);
+	create_fake_stash_reference_and_reflog(g_repo);
 
 	/*
 	 * $ git reflog -1 refs/fakestash
@@ -373,12 +372,10 @@ void test_refs_revparse__reflog_of_a_ref_under_refs(void)
 	 * $ git reflog -1 fakestash@{0}
 	 * a65fedf fakestash@{0}: commit: checking in
 	 */
-	test_object_inrepo("refs/fakestash", "a65fedf39aefe402d3bb6e24df4d4f5fe4547750", repo);
-	test_object_inrepo("refs/fakestash@{0}", "a65fedf39aefe402d3bb6e24df4d4f5fe4547750", repo);
-	test_object_inrepo("fakestash", "a65fedf39aefe402d3bb6e24df4d4f5fe4547750", repo);
-	test_object_inrepo("fakestash@{0}", "a65fedf39aefe402d3bb6e24df4d4f5fe4547750", repo);
-
-	cl_git_sandbox_cleanup();
+	test_object_inrepo("refs/fakestash", "a65fedf39aefe402d3bb6e24df4d4f5fe4547750", g_repo);
+	test_object_inrepo("refs/fakestash@{0}", "a65fedf39aefe402d3bb6e24df4d4f5fe4547750", g_repo);
+	test_object_inrepo("fakestash", "a65fedf39aefe402d3bb6e24df4d4f5fe4547750", g_repo);
+	test_object_inrepo("fakestash@{0}", "a65fedf39aefe402d3bb6e24df4d4f5fe4547750", g_repo);
 }
 
 void test_refs_revparse__revwalk(void)
@@ -593,6 +590,7 @@ void test_refs_revparse__a_not_precise_enough_objectid_returns_EAMBIGUOUS(void)
 	git_index *index;
 	git_object *obj;
 
+	cl_git_sandbox_cleanup();
 	repo = cl_git_sandbox_init("testrepo");
 
 	cl_git_mkfile("testrepo/one.txt", "aabqhq\n");
@@ -613,37 +611,33 @@ void test_refs_revparse__a_not_precise_enough_objectid_returns_EAMBIGUOUS(void)
 
 void test_refs_revparse__issue_994(void)
 {
-	git_repository *repo;
 	git_reference *head, *with_at;
 	git_object *target;
 
-	repo = cl_git_sandbox_init("testrepo.git");
+	cl_assert_equal_i(GIT_ENOTFOUND,
+		git_revparse_single(&target, g_repo, "origin/bim_with_3d@11296"));
 
 	cl_assert_equal_i(GIT_ENOTFOUND,
-		git_revparse_single(&target, repo, "origin/bim_with_3d@11296"));
-
-	cl_assert_equal_i(GIT_ENOTFOUND,
-		git_revparse_single(&target, repo, "refs/remotes/origin/bim_with_3d@11296"));
+		git_revparse_single(&target, g_repo, "refs/remotes/origin/bim_with_3d@11296"));
 
 
-	cl_git_pass(git_repository_head(&head, repo));
+	cl_git_pass(git_repository_head(&head, g_repo));
 	cl_git_pass(git_reference_create(
 		&with_at,
-		repo,
+		g_repo,
 		"refs/remotes/origin/bim_with_3d@11296",
 		git_reference_target(head),
 		0,
 		NULL));
 
-	cl_git_pass(git_revparse_single(&target, repo, "origin/bim_with_3d@11296"));
+	cl_git_pass(git_revparse_single(&target, g_repo, "origin/bim_with_3d@11296"));
 	git_object_free(target);
 
-	cl_git_pass(git_revparse_single(&target, repo, "refs/remotes/origin/bim_with_3d@11296"));
+	cl_git_pass(git_revparse_single(&target, g_repo, "refs/remotes/origin/bim_with_3d@11296"));
 	git_object_free(target);
 
 	git_reference_free(with_at);
 	git_reference_free(head);
-	cl_git_sandbox_cleanup();
 }
 
 /**
@@ -660,25 +654,21 @@ void test_refs_revparse__issue_994(void)
  */
 void test_refs_revparse__try_to_retrieve_branch_before_described_tag(void)
 {
-	git_repository *repo;
 	git_reference *branch;
 	git_object *target;
 	char sha[GIT_OID_SHA1_HEXSIZE + 1];
 
-	repo = cl_git_sandbox_init("testrepo.git");
+	test_object_inrepo("blah-7-gc47800c", "c47800c7266a2be04c571c04d5a6614691ea99bd", g_repo);
 
-	test_object_inrepo("blah-7-gc47800c", "c47800c7266a2be04c571c04d5a6614691ea99bd", repo);
-
-	cl_git_pass(git_revparse_single(&target, repo, "HEAD~3"));
-	cl_git_pass(git_branch_create(&branch, repo, "blah-7-gc47800c", (git_commit *)target, 0));
+	cl_git_pass(git_revparse_single(&target, g_repo, "HEAD~3"));
+	cl_git_pass(git_branch_create(&branch, g_repo, "blah-7-gc47800c", (git_commit *)target, 0));
 
 	git_oid_tostr(sha, GIT_OID_SHA1_HEXSIZE + 1, git_object_id(target));
 
-	test_object_inrepo("blah-7-gc47800c", sha, repo);
+	test_object_inrepo("blah-7-gc47800c", sha, g_repo);
 
 	git_reference_free(branch);
 	git_object_free(target);
-	cl_git_sandbox_cleanup();
 }
 
 /**
@@ -698,26 +688,22 @@ void test_refs_revparse__try_to_retrieve_branch_before_described_tag(void)
  */
 void test_refs_revparse__try_to_retrieve_sha_before_branch(void)
 {
-	git_repository *repo;
 	git_reference *branch;
 	git_object *target;
 	char sha[GIT_OID_SHA1_HEXSIZE + 1];
 
-	repo = cl_git_sandbox_init("testrepo.git");
+	test_object_inrepo("a65fedf39aefe402d3bb6e24df4d4f5fe4547750", "a65fedf39aefe402d3bb6e24df4d4f5fe4547750", g_repo);
 
-	test_object_inrepo("a65fedf39aefe402d3bb6e24df4d4f5fe4547750", "a65fedf39aefe402d3bb6e24df4d4f5fe4547750", repo);
-
-	cl_git_pass(git_revparse_single(&target, repo, "HEAD~3"));
-	cl_git_pass(git_branch_create(&branch, repo, "a65fedf39aefe402d3bb6e24df4d4f5fe4547750", (git_commit *)target, 0));
+	cl_git_pass(git_revparse_single(&target, g_repo, "HEAD~3"));
+	cl_git_pass(git_branch_create(&branch, g_repo, "a65fedf39aefe402d3bb6e24df4d4f5fe4547750", (git_commit *)target, 0));
 
 	git_oid_tostr(sha, GIT_OID_SHA1_HEXSIZE + 1, git_object_id(target));
 
-	test_object_inrepo("a65fedf39aefe402d3bb6e24df4d4f5fe4547750", "a65fedf39aefe402d3bb6e24df4d4f5fe4547750", repo);
-	test_object_inrepo("heads/a65fedf39aefe402d3bb6e24df4d4f5fe4547750", sha, repo);
+	test_object_inrepo("a65fedf39aefe402d3bb6e24df4d4f5fe4547750", "a65fedf39aefe402d3bb6e24df4d4f5fe4547750", g_repo);
+	test_object_inrepo("heads/a65fedf39aefe402d3bb6e24df4d4f5fe4547750", sha, g_repo);
 
 	git_reference_free(branch);
 	git_object_free(target);
-	cl_git_sandbox_cleanup();
 }
 
 /**
@@ -734,45 +720,37 @@ void test_refs_revparse__try_to_retrieve_sha_before_branch(void)
  */
 void test_refs_revparse__try_to_retrieve_branch_before_abbrev_sha(void)
 {
-	git_repository *repo;
 	git_reference *branch;
 	git_object *target;
 	char sha[GIT_OID_SHA1_HEXSIZE + 1];
 
-	repo = cl_git_sandbox_init("testrepo.git");
+	test_object_inrepo("c47800", "c47800c7266a2be04c571c04d5a6614691ea99bd", g_repo);
 
-	test_object_inrepo("c47800", "c47800c7266a2be04c571c04d5a6614691ea99bd", repo);
-
-	cl_git_pass(git_revparse_single(&target, repo, "HEAD~3"));
-	cl_git_pass(git_branch_create(&branch, repo, "c47800", (git_commit *)target, 0));
+	cl_git_pass(git_revparse_single(&target, g_repo, "HEAD~3"));
+	cl_git_pass(git_branch_create(&branch, g_repo, "c47800", (git_commit *)target, 0));
 
 	git_oid_tostr(sha, GIT_OID_SHA1_HEXSIZE + 1, git_object_id(target));
 
-	test_object_inrepo("c47800", sha, repo);
+	test_object_inrepo("c47800", sha, g_repo);
 
 	git_reference_free(branch);
 	git_object_free(target);
-	cl_git_sandbox_cleanup();
 }
 
 void test_refs_revparse__at_at_end_of_refname(void)
 {
-	git_repository *repo;
 	git_reference *branch;
 	git_object *target;
 
-	repo = cl_git_sandbox_init("testrepo.git");
-
-	cl_git_pass(git_revparse_single(&target, repo, "HEAD"));
-	cl_git_pass(git_branch_create(&branch, repo, "master@", (git_commit *)target, 0));
+	cl_git_pass(git_revparse_single(&target, g_repo, "HEAD"));
+	cl_git_pass(git_branch_create(&branch, g_repo, "master@", (git_commit *)target, 0));
 	git_object_free(target);
 
-	test_id_inrepo("master@", "a65fedf39aefe402d3bb6e24df4d4f5fe4547750", NULL, GIT_REVSPEC_SINGLE, repo);
+	test_id_inrepo("master@", "a65fedf39aefe402d3bb6e24df4d4f5fe4547750", NULL, GIT_REVSPEC_SINGLE, g_repo);
 
-	cl_git_fail_with(GIT_ENOTFOUND, git_revparse_single(&target, repo, "foo@"));
+	cl_git_fail_with(GIT_ENOTFOUND, git_revparse_single(&target, g_repo, "foo@"));
 
 	git_reference_free(branch);
-	cl_git_sandbox_cleanup();
 }
 
 void test_refs_revparse__range(void)

tests/libgit2/refs/shorthand.c

@@ -15,13 +15,12 @@ void test_refs_shorthand__0(void)
 {
 	git_repository *repo;
 
-	cl_git_pass(git_repository_open(&repo, cl_fixture("testrepo.git")));
-
+	repo = cl_git_sandbox_init("testrepo.git");
 
 	assert_shorthand(repo, "refs/heads/master", "master");
 	assert_shorthand(repo, "refs/tags/test", "test");
 	assert_shorthand(repo, "refs/remotes/test/master", "test/master");
 	assert_shorthand(repo, "refs/notes/fanout", "notes/fanout");
 
-	git_repository_free(repo);
+	cl_git_sandbox_cleanup();
 }

tests/libgit2/refs/transactions.c

@@ -8,6 +8,8 @@ void test_refs_transactions__initialize(void)
 {
    g_repo = cl_git_sandbox_init("testrepo");
    cl_git_pass(git_transaction_new(&g_tx, g_repo));
+   if (!cl_repo_has_ref_format(g_repo, "files"))
+      cl_skip();
 }
 
 void test_refs_transactions__cleanup(void)

tests/libgit2/threads/refdb.c

@@ -90,7 +90,7 @@ static void *create_refs(void *arg)
 			cl_git_thread_pass(data, git_repository_refdb(&refdb, repo));
 			do {
 				error = git_refdb_compress(refdb);
-			} while (error == GIT_ELOCKED);
+			} while (error == GIT_ELOCKED || error == GIT_EMODIFIED);
 			cl_git_thread_pass(data, error);
 			git_refdb_free(refdb);
 		}
@@ -136,7 +136,7 @@ static void *delete_refs(void *arg)
 			cl_git_thread_pass(data, git_repository_refdb(&refdb, repo));
 			do {
 				error = git_refdb_compress(refdb);
-			} while (error == GIT_ELOCKED);
+			} while (error == GIT_ELOCKED || error == GIT_EMODIFIED);
 			cl_git_thread_pass(data, error);
 			git_refdb_free(refdb);
 		}
@@ -162,6 +162,20 @@ void test_threads_refdb__edit_while_iterate(void)
 
 	g_repo = cl_git_sandbox_init("testrepo2");
 
+#ifdef GIT_WIN32
+	/*
+	 * On Windows we still have issues with renaming the "table.list" file
+	 * into place. This will require a bit of a rework of how we open files
+	 * so that they are always opened with `FILE_SHARE_DELETE`.
+	 * Furthermore, we'll have to start using POSIX-semantics when renaming
+	 * files.
+	 *
+	 * For now, we just accept that this doesn't yet work.
+	 */
+	if (cl_repo_has_ref_format(g_repo, "reftable"))
+		cl_skip();
+#endif
+
 	cl_git_pass(git_reference_name_to_id(&head, g_repo, "HEAD"));
 
 	/* make a bunch of references */

tests/resources/reftable/attr/.gitted/HEAD

@@ -0,0 +1 @@
+ref: refs/heads/.invalid

tests/resources/reftable/attr/.gitted/config

@@ -0,0 +1,8 @@
+[core]
+	repositoryformatversion = 1
+	filemode = true
+	bare = false
+	logallrefupdates = true
+	ignorecase = true
+[extensions]
+	refstorage = reftable

tests/resources/reftable/attr/.gitted/description

@@ -0,0 +1 @@
+Unnamed repository; edit this file 'description' to name the repository.

tests/resources/reftable/attr/.gitted/info/attributes

@@ -0,0 +1,4 @@
+* repoattr
+a*	foo !bar -baz
+sub/*.txt reposub
+sub/sub/*.txt reposubsub

tests/resources/reftable/attr/.gitted/info/exclude

@@ -0,0 +1,6 @@
+# git ls-files --others --exclude-from=.git/info/exclude
+# Lines that start with '#' are comments.
+# For a project mostly in C, the following would be a good set of
+# exclude patterns (uncomment them if you want to use them):
+# *.[oa]
+# *~

tests/resources/reftable/attr/.gitted/objects/21/7878ab49e1314388ea2e32dc6fdb58a1b969e0

@@ -0,0 +1,4 @@
+x
ťŽQ
+Â0DýÎ)öęn“ŘDÄ#x<>mv<6D>…ÖJ’Ţ߀7đcx0Ľ<30>IŰşÎ
­¨‚óž-ąĚÁń+e"Ľvů‚Á‡śâ<C59B>ťůpŃwŽcJH1x‡Ô%Ś”¦HL>Ddˇ‰
ďíµxîµę˛ŔC—¬®\ʤz˙ᔶődí0Z‘ŕ<E28098>#˘ém˙ŘşţĎÚ°<C39A>
+äyŃ
+óę>ť{Ě–qK˛

tests/resources/reftable/attr/.gitted/objects/2b/40c5aca159b04ea8d20ffe36cdf8b09369b14a

@@ -0,0 +1 @@
+x
mPΡj„0μ«ω<C2AB>=ϊP8Z…ήSΎ¤”c£ρhR6{=Ό―obβ™<CEB2>"<22>Ωafv<66><76><EFBFBD>‚χΧζιδ#3‰Ξ¬=7P<37>%[8<Heμ`&]
@?aFZ<04>®<EFBFBD>η@!…χ.ΚΘω:±ω½§…•ldπΪLG­|K’7~XΓN8¤<38>·IΟdμκ}Ώψ<CE8F>q<EFBFBD>σ2cGΎ<>7lΎ5ΤV_pEϋ<45>®’#lZ΄<5A>GMςt[JΜΘΝΒ½¥&©hΈ±uΜ][‰Φί4‰-3;ΛC<CE9B>g<EFBFBD>4Ώx`Zΐ»YΓ<59>“ι<E2809C>»ϊ€bέ^>οyNlΝ£΅>c―;gΣ<67>•¥kΗYXΔ9b|D<>~VΨ—)…vΏψ
ρΞά•

tests/resources/reftable/attr/.gitted/objects/4d/713dc48e6b1bd75b0d61ad078ba9ca3a56745d

@@ -0,0 +1,2 @@
+x
ÁÁ
€ @ßWŶàÇ
+|ø§k 9n$¡}gŠ«à:<3A>‡îÂ;5°1¥e–4ˆ\k_]‘ÞƒŸÙ­hœD¡k›ý'~

tests/resources/reftable/attr/.gitted/objects/60/5812ab7fe421fdd325a935d35cb06a9234a7d7

@@ -0,0 +1,2 @@
+x
²Nш
б0Ц;Sэ═к╩∙B▄ю≈кU ╔╢JсЩ	╟
?╤lы√y≥Гgcц║U RbaБ░ыcG;╦l╡Ц═²Dqж═Z╘й╚AH■<г▒Ёв3NБ╗ЦД=J2d3[⌠0⌠╒╫=√
+В}ш╓╦I≥╓б≥jM"в²x≥/═╜И[г
▌ьГTwШгжЦЪ╢U║&[┐/ЛkЧ(УtJL

tests/resources/reftable/attr/.gitted/objects/6b/ab5c79cd5140d0f800917f550eb2a3dc32b0da

@@ -0,0 +1,3 @@
+x
<01><>Û	Ã0Eûí)´@‹d'~@(¥#tÅQ¨ÁiÀQö¯¡ôëÂánÞ·­(Pôm"<22>Ř2æh°sL+<2B>d{—"{Zœ“`øÔ÷Þàu‡Ô
+O©«4˜¸µYäñ›[Þ·;<3B>³Ã@>¥®M§ýS»þOmʧhá
+*‡ÂÂÊæ¿<-

tests/resources/reftable/attr/.gitted/objects/8d/0b9df9bd30be7910ddda60548d485bc302b911

@@ -0,0 +1 @@
+x
<01><>Kj1D<><44>)zol<6F>i<EFBFBD><69> _"hi<68>K2<4B>L<EFBFBD><4C>G!7Ȫ<05><><EFBFBD>J<EFBFBD>,<2C><><EFBFBD>E<EFBFBD>PX<50><58>D<EFBFBD><44>S<EFBFBD><53>	] /)<29><>}<7D><>/<2F><>Uw<55>R<EFBFBD><52>.	<09>j<EFBFBD><6A><13><>p<EFBFBD><70><EFBFBD><EFBFBD>#<23><>#:?<3F><>:|<7C><>;<3B><>F9<46>܋<EFBFBD>r=_<0B><>)<29><>ơ<>N/<2F><>A[<5B><>l<EFBFBD><6C>!<21>q<EFBFBD><71>}<7D><><EFBFBD><15><Lfx4<78>H\<5C><>\<5C><>q֏cj<63><6A><EFBFBD><EFBFBD>
<04>T<EFBFBD>