mirror of
https://github.com/libgit2/libgit2.git
synced 2026-06-22 06:26:26 +00:00
packfile: handle sha256 packfiles
Teach the packfile machinery to cope with SHA256.
This commit is contained in:
@@ -24,6 +24,26 @@ GIT_BEGIN_DECL
|
||||
* Constructors for in-box ODB backends.
|
||||
*/
|
||||
|
||||
/** Options for configuring a packfile object backend. */
|
||||
typedef struct {
|
||||
unsigned int version; /**< version for the struct */
|
||||
|
||||
/**
|
||||
* Type of object IDs to use for this object database, or
|
||||
* 0 for default (currently SHA1).
|
||||
*/
|
||||
git_oid_t oid_type;
|
||||
} git_odb_backend_pack_options;
|
||||
|
||||
/* The current version of the diff options structure */
|
||||
#define GIT_ODB_BACKEND_PACK_OPTIONS_VERSION 1
|
||||
|
||||
/* Stack initializer for odb pack backend options. Alternatively use
|
||||
* `git_odb_backend_pack_options_init` programmatic initialization.
|
||||
*/
|
||||
#define GIT_ODB_BACKEND_PACK_OPTIONS_INIT \
|
||||
{ GIT_ODB_BACKEND_PACK_OPTIONS_VERSION }
|
||||
|
||||
/**
|
||||
* Create a backend for the packfiles.
|
||||
*
|
||||
@@ -32,7 +52,38 @@ GIT_BEGIN_DECL
|
||||
*
|
||||
* @return 0 or an error code
|
||||
*/
|
||||
GIT_EXTERN(int) git_odb_backend_pack(git_odb_backend **out, const char *objects_dir);
|
||||
#ifdef GIT_EXPERIMENTAL_SHA256
|
||||
GIT_EXTERN(int) git_odb_backend_pack(
|
||||
git_odb_backend **out,
|
||||
const char *objects_dir,
|
||||
const git_odb_backend_pack_options *opts);
|
||||
#else
|
||||
GIT_EXTERN(int) git_odb_backend_pack(
|
||||
git_odb_backend **out,
|
||||
const char *objects_dir);
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Create a backend out of a single packfile
|
||||
*
|
||||
* This can be useful for inspecting the contents of a single
|
||||
* packfile.
|
||||
*
|
||||
* @param out location to store the odb backend pointer
|
||||
* @param index_file path to the packfile's .idx file
|
||||
*
|
||||
* @return 0 or an error code
|
||||
*/
|
||||
#ifdef GIT_EXPERIMENTAL_SHA256
|
||||
GIT_EXTERN(int) git_odb_backend_one_pack(
|
||||
git_odb_backend **out,
|
||||
const char *index_file,
|
||||
const git_odb_backend_pack_options *opts);
|
||||
#else
|
||||
GIT_EXTERN(int) git_odb_backend_one_pack(
|
||||
git_odb_backend **out,
|
||||
const char *index_file);
|
||||
#endif
|
||||
|
||||
typedef enum {
|
||||
GIT_ODB_BACKEND_LOOSE_FSYNC = (1 << 0)
|
||||
@@ -100,19 +151,6 @@ GIT_EXTERN(int) git_odb_backend_loose(
|
||||
unsigned int file_mode);
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Create a backend out of a single packfile
|
||||
*
|
||||
* This can be useful for inspecting the contents of a single
|
||||
* packfile.
|
||||
*
|
||||
* @param out location to store the odb backend pointer
|
||||
* @param index_file path to the packfile's .idx file
|
||||
*
|
||||
* @return 0 or an error code
|
||||
*/
|
||||
GIT_EXTERN(int) git_odb_backend_one_pack(git_odb_backend **out, const char *index_file);
|
||||
|
||||
/** Streaming mode */
|
||||
typedef enum {
|
||||
GIT_STREAM_RDONLY = (1 << 1),
|
||||
|
||||
@@ -538,7 +538,7 @@ int git_commit_graph_entry_find(
|
||||
hi = ntohl(file->oid_fanout[(int)short_oid->id[0]]);
|
||||
lo = ((short_oid->id[0] == 0x0) ? 0 : ntohl(file->oid_fanout[(int)short_oid->id[0] - 1]));
|
||||
|
||||
pos = git_pack__lookup_sha1(file->oid_lookup, GIT_OID_SHA1_SIZE, lo, hi, short_oid->id);
|
||||
pos = git_pack__lookup_id(file->oid_lookup, GIT_OID_SHA1_SIZE, lo, hi, short_oid->id, GIT_OID_SHA1);
|
||||
|
||||
if (pos >= 0) {
|
||||
/* An object matching exactly the oid was found */
|
||||
@@ -726,7 +726,8 @@ int git_commit_graph_writer_add_index_file(
|
||||
if (error < 0)
|
||||
goto cleanup;
|
||||
|
||||
error = git_mwindow_get_pack(&p, idx_path);
|
||||
/* TODO: SHA256 */
|
||||
error = git_mwindow_get_pack(&p, idx_path, 0);
|
||||
if (error < 0)
|
||||
goto cleanup;
|
||||
|
||||
|
||||
@@ -179,7 +179,8 @@ int git_indexer_new(
|
||||
if (fd < 0)
|
||||
goto cleanup;
|
||||
|
||||
error = git_packfile_alloc(&idx->pack, git_str_cstr(&tmp_path));
|
||||
/* TODO: SHA256 */
|
||||
error = git_packfile_alloc(&idx->pack, git_str_cstr(&tmp_path), 0);
|
||||
git_str_dispose(&tmp_path);
|
||||
|
||||
if (error < 0)
|
||||
@@ -468,16 +469,16 @@ static int store_object(git_indexer *idx)
|
||||
goto on_error;
|
||||
}
|
||||
|
||||
git_oid_cpy(&pentry->sha1, &oid);
|
||||
git_oid_cpy(&pentry->id, &oid);
|
||||
pentry->offset = entry_start;
|
||||
|
||||
if (git_oidmap_exists(idx->pack->idx_cache, &pentry->sha1)) {
|
||||
git_error_set(GIT_ERROR_INDEXER, "duplicate object %s found in pack", git_oid_tostr_s(&pentry->sha1));
|
||||
if (git_oidmap_exists(idx->pack->idx_cache, &pentry->id)) {
|
||||
git_error_set(GIT_ERROR_INDEXER, "duplicate object %s found in pack", git_oid_tostr_s(&pentry->id));
|
||||
git__free(pentry);
|
||||
goto on_error;
|
||||
}
|
||||
|
||||
if ((error = git_oidmap_set(idx->pack->idx_cache, &pentry->sha1, pentry)) < 0) {
|
||||
if ((error = git_oidmap_set(idx->pack->idx_cache, &pentry->id, pentry)) < 0) {
|
||||
git__free(pentry);
|
||||
git_error_set_oom();
|
||||
goto on_error;
|
||||
@@ -522,8 +523,8 @@ static int save_entry(git_indexer *idx, struct entry *entry, struct git_pack_ent
|
||||
|
||||
pentry->offset = entry_start;
|
||||
|
||||
if (git_oidmap_exists(idx->pack->idx_cache, &pentry->sha1) ||
|
||||
git_oidmap_set(idx->pack->idx_cache, &pentry->sha1, pentry) < 0) {
|
||||
if (git_oidmap_exists(idx->pack->idx_cache, &pentry->id) ||
|
||||
git_oidmap_set(idx->pack->idx_cache, &pentry->id, pentry) < 0) {
|
||||
git_error_set(GIT_ERROR_INDEXER, "cannot insert object into pack");
|
||||
return -1;
|
||||
}
|
||||
@@ -557,7 +558,7 @@ static int hash_and_save(git_indexer *idx, git_rawobj *obj, off64_t entry_start)
|
||||
pentry = git__calloc(1, sizeof(struct git_pack_entry));
|
||||
GIT_ERROR_CHECK_ALLOC(pentry);
|
||||
|
||||
git_oid_cpy(&pentry->sha1, &oid);
|
||||
git_oid_cpy(&pentry->id, &oid);
|
||||
git_oid_cpy(&entry->oid, &oid);
|
||||
entry->crc = crc32(0L, Z_NULL, 0);
|
||||
|
||||
@@ -987,7 +988,7 @@ static int inject_object(git_indexer *idx, git_oid *id)
|
||||
pentry = git__calloc(1, sizeof(struct git_pack_entry));
|
||||
GIT_ERROR_CHECK_ALLOC(pentry);
|
||||
|
||||
git_oid_cpy(&pentry->sha1, id);
|
||||
git_oid_cpy(&pentry->id, id);
|
||||
git_oid_cpy(&entry->oid, id);
|
||||
idx->off = entry_start + hdr_len + len;
|
||||
|
||||
|
||||
@@ -392,7 +392,7 @@ int git_midx_entry_find(
|
||||
hi = ntohl(idx->oid_fanout[(int)short_oid->id[0]]);
|
||||
lo = ((short_oid->id[0] == 0x0) ? 0 : ntohl(idx->oid_fanout[(int)short_oid->id[0] - 1]));
|
||||
|
||||
pos = git_pack__lookup_sha1(idx->oid_lookup, GIT_OID_SHA1_SIZE, lo, hi, short_oid->id);
|
||||
pos = git_pack__lookup_id(idx->oid_lookup, GIT_OID_SHA1_SIZE, lo, hi, short_oid->id, GIT_OID_SHA1);
|
||||
|
||||
if (pos >= 0) {
|
||||
/* An object matching exactly the oid was found */
|
||||
@@ -549,7 +549,8 @@ int git_midx_writer_add(
|
||||
if (error < 0)
|
||||
return error;
|
||||
|
||||
error = git_mwindow_get_pack(&p, git_str_cstr(&idx_path_buf));
|
||||
/* TODO: SHA256 */
|
||||
error = git_mwindow_get_pack(&p, git_str_cstr(&idx_path_buf), 0);
|
||||
git_str_dispose(&idx_path_buf);
|
||||
if (error < 0)
|
||||
return error;
|
||||
|
||||
@@ -61,7 +61,10 @@ int git_mwindow_global_init(void)
|
||||
return git_runtime_shutdown_register(git_mwindow_global_shutdown);
|
||||
}
|
||||
|
||||
int git_mwindow_get_pack(struct git_pack_file **out, const char *path)
|
||||
int git_mwindow_get_pack(
|
||||
struct git_pack_file **out,
|
||||
const char *path,
|
||||
git_oid_t oid_type)
|
||||
{
|
||||
struct git_pack_file *pack;
|
||||
char *packname;
|
||||
@@ -86,7 +89,7 @@ int git_mwindow_get_pack(struct git_pack_file **out, const char *path)
|
||||
}
|
||||
|
||||
/* If we didn't find it, we need to create it */
|
||||
if ((error = git_packfile_alloc(&pack, path)) < 0) {
|
||||
if ((error = git_packfile_alloc(&pack, path, oid_type)) < 0) {
|
||||
git_mutex_unlock(&git__mwindow_mutex);
|
||||
return error;
|
||||
}
|
||||
|
||||
@@ -48,7 +48,10 @@ void git_mwindow_close(git_mwindow **w_cursor);
|
||||
extern int git_mwindow_global_init(void);
|
||||
|
||||
struct git_pack_file; /* just declaration to avoid cyclical includes */
|
||||
int git_mwindow_get_pack(struct git_pack_file **out, const char *path);
|
||||
int git_mwindow_get_pack(
|
||||
struct git_pack_file **out,
|
||||
const char *path,
|
||||
git_oid_t oid_type);
|
||||
int git_mwindow_put_pack(struct git_pack_file *pack);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -684,6 +684,7 @@ int git_odb__add_default_backends(
|
||||
ino_t inode;
|
||||
git_odb_backend *loose, *packed;
|
||||
git_odb_backend_loose_options loose_opts = GIT_ODB_BACKEND_LOOSE_OPTIONS_INIT;
|
||||
git_odb_backend_pack_options pack_opts = GIT_ODB_BACKEND_PACK_OPTIONS_INIT;
|
||||
|
||||
/* TODO: inodes are not really relevant on Win32, so we need to find
|
||||
* a cross-platform workaround for this */
|
||||
@@ -722,6 +723,7 @@ int git_odb__add_default_backends(
|
||||
loose_opts.flags |= GIT_ODB_BACKEND_LOOSE_FSYNC;
|
||||
|
||||
loose_opts.oid_type = db->options.oid_type;
|
||||
pack_opts.oid_type = db->options.oid_type;
|
||||
|
||||
/* add the loose object backend */
|
||||
if (git_odb__backend_loose(&loose, objects_dir, &loose_opts) < 0 ||
|
||||
@@ -729,8 +731,17 @@ int git_odb__add_default_backends(
|
||||
return -1;
|
||||
|
||||
/* add the packed file backend */
|
||||
if (git_odb_backend_pack(&packed, objects_dir) < 0 ||
|
||||
add_backend_internal(db, packed, git_odb__packed_priority, as_alternates, inode) < 0)
|
||||
#ifdef GIT_EXPERIMENTAL_SHA256
|
||||
if (git_odb_backend_pack(&packed, objects_dir, &pack_opts) < 0)
|
||||
return -1;
|
||||
#else
|
||||
GIT_UNUSED(pack_opts);
|
||||
|
||||
if (git_odb_backend_pack(&packed, objects_dir) < 0)
|
||||
return -1;
|
||||
#endif
|
||||
|
||||
if (add_backend_internal(db, packed, git_odb__packed_priority, as_alternates, inode) < 0)
|
||||
return -1;
|
||||
|
||||
if (git_mutex_lock(&db->lock) < 0) {
|
||||
|
||||
@@ -26,6 +26,7 @@
|
||||
|
||||
struct pack_backend {
|
||||
git_odb_backend parent;
|
||||
git_odb_backend_pack_options opts;
|
||||
git_midx_file *midx;
|
||||
git_vector midx_packs;
|
||||
git_vector packs;
|
||||
@@ -95,24 +96,24 @@ struct pack_writepack {
|
||||
* --------------------------------------------------
|
||||
*
|
||||
* # pack_backend__exists / pack_backend__exists_prefix
|
||||
* | Check if the given SHA1 oid (or a SHA1 oid prefix) exists in any of the
|
||||
* | Check if the given oid (or an oid prefix) exists in any of the
|
||||
* | packs that have been loaded for our ODB.
|
||||
* |
|
||||
* |-# pack_entry_find / pack_entry_find_prefix
|
||||
* | If there is a multi-pack-index present, search the SHA1 oid in that
|
||||
* | If there is a multi-pack-index present, search the oid in that
|
||||
* | index first. If it is not found there, iterate through all the unindexed
|
||||
* | packs that have been preloaded (starting by the pack where the latest
|
||||
* | object was found) to try to find the OID in one of them.
|
||||
* |
|
||||
* |-# git_midx_entry_find
|
||||
* | Search for the SHA1 oid in the multi-pack-index. See
|
||||
* | Search for the oid in the multi-pack-index. See
|
||||
* | <https://github.com/git/git/blob/master/Documentation/technical/pack-format.txt>
|
||||
* | for specifics on the multi-pack-index format and how do we find
|
||||
* | entries in it.
|
||||
* |
|
||||
* |-# git_pack_entry_find
|
||||
* | Check the index of an individual unindexed pack to see if the SHA1
|
||||
* | OID can be found. If we can find the offset to that SHA1 inside of the
|
||||
* | Check the index of an individual unindexed pack to see if the
|
||||
* | OID can be found. If we can find the offset to that inside of the
|
||||
* | index, that means the object is contained inside of the packfile and
|
||||
* | we can stop searching. Before returning, we verify that the
|
||||
* | packfile behind the index we are searching still exists on disk.
|
||||
@@ -141,13 +142,13 @@ struct pack_writepack {
|
||||
* --------------------------------------------------
|
||||
*
|
||||
* # pack_backend__read / pack_backend__read_prefix
|
||||
* | Check if the given SHA1 oid (or a SHA1 oid prefix) exists in any of the
|
||||
* | Check if the given oid (or an oid prefix) exists in any of the
|
||||
* | packs that have been loaded for our ODB. If it does, open the packfile and
|
||||
* | read from it.
|
||||
* |
|
||||
* |-# git_packfile_unpack
|
||||
* Armed with a packfile and the offset within it, we can finally unpack
|
||||
* the object pointed at by the SHA1 oid. This involves mmapping part of
|
||||
* the object pointed at by the oid. This involves mmapping part of
|
||||
* the `.pack` file, and uncompressing the object within it (if it is
|
||||
* stored in the undelfitied representation), or finding a base object and
|
||||
* applying some deltas to its uncompressed representation (if it is stored
|
||||
@@ -177,7 +178,7 @@ static int pack_entry_find(struct git_pack_entry *e,
|
||||
* a prefix of an identifier.
|
||||
* Sets GIT_EAMBIGUOUS if short oid is ambiguous.
|
||||
* This method assumes that len is between
|
||||
* GIT_OID_MINPREFIXLEN and GIT_OID_SHA1_HEXSIZE.
|
||||
* GIT_OID_MINPREFIXLEN and the hexsize for the hash type.
|
||||
*/
|
||||
static int pack_entry_find_prefix(
|
||||
struct git_pack_entry *e,
|
||||
@@ -251,7 +252,7 @@ static int packfile_load__cb(void *data, git_str *path)
|
||||
if (git_vector_search2(NULL, &backend->packs, packfile_byname_search_cmp, &index_prefix) == 0)
|
||||
return 0;
|
||||
|
||||
error = git_mwindow_get_pack(&pack, path->ptr);
|
||||
error = git_mwindow_get_pack(&pack, path->ptr, backend->opts.oid_type);
|
||||
|
||||
/* ignore missing .pack file as git does */
|
||||
if (error == GIT_ENOTFOUND) {
|
||||
@@ -270,33 +271,34 @@ static int pack_entry_find(struct git_pack_entry *e, struct pack_backend *backen
|
||||
{
|
||||
struct git_pack_file *last_found = backend->last_found, *p;
|
||||
git_midx_entry midx_entry;
|
||||
size_t oid_hexsize = git_oid_hexsize(backend->opts.oid_type);
|
||||
size_t i;
|
||||
|
||||
if (backend->midx &&
|
||||
git_midx_entry_find(&midx_entry, backend->midx, oid, GIT_OID_SHA1_HEXSIZE) == 0 &&
|
||||
git_midx_entry_find(&midx_entry, backend->midx, oid, oid_hexsize) == 0 &&
|
||||
midx_entry.pack_index < git_vector_length(&backend->midx_packs)) {
|
||||
e->offset = midx_entry.offset;
|
||||
git_oid_cpy(&e->sha1, &midx_entry.sha1);
|
||||
git_oid_cpy(&e->id, &midx_entry.sha1);
|
||||
e->p = git_vector_get(&backend->midx_packs, midx_entry.pack_index);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (last_found &&
|
||||
git_pack_entry_find(e, last_found, oid, GIT_OID_SHA1_HEXSIZE) == 0)
|
||||
git_pack_entry_find(e, last_found, oid, oid_hexsize) == 0)
|
||||
return 0;
|
||||
|
||||
git_vector_foreach(&backend->packs, i, p) {
|
||||
if (p == last_found)
|
||||
continue;
|
||||
|
||||
if (git_pack_entry_find(e, p, oid, GIT_OID_SHA1_HEXSIZE) == 0) {
|
||||
if (git_pack_entry_find(e, p, oid, oid_hexsize) == 0) {
|
||||
backend->last_found = p;
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
return git_odb__error_notfound(
|
||||
"failed to find pack entry", oid, GIT_OID_SHA1_HEXSIZE);
|
||||
"failed to find pack entry", oid, oid_hexsize);
|
||||
}
|
||||
|
||||
static int pack_entry_find_prefix(
|
||||
@@ -318,9 +320,9 @@ static int pack_entry_find_prefix(
|
||||
return error;
|
||||
if (!error && midx_entry.pack_index < git_vector_length(&backend->midx_packs)) {
|
||||
e->offset = midx_entry.offset;
|
||||
git_oid_cpy(&e->sha1, &midx_entry.sha1);
|
||||
git_oid_cpy(&e->id, &midx_entry.sha1);
|
||||
e->p = git_vector_get(&backend->midx_packs, midx_entry.pack_index);
|
||||
git_oid_cpy(&found_full_oid, &e->sha1);
|
||||
git_oid_cpy(&found_full_oid, &e->id);
|
||||
found = true;
|
||||
}
|
||||
}
|
||||
@@ -330,9 +332,9 @@ static int pack_entry_find_prefix(
|
||||
if (error == GIT_EAMBIGUOUS)
|
||||
return error;
|
||||
if (!error) {
|
||||
if (found && git_oid_cmp(&e->sha1, &found_full_oid))
|
||||
if (found && git_oid_cmp(&e->id, &found_full_oid))
|
||||
return git_odb__error_ambiguous("found multiple pack entries");
|
||||
git_oid_cpy(&found_full_oid, &e->sha1);
|
||||
git_oid_cpy(&found_full_oid, &e->id);
|
||||
found = true;
|
||||
}
|
||||
}
|
||||
@@ -345,9 +347,9 @@ static int pack_entry_find_prefix(
|
||||
if (error == GIT_EAMBIGUOUS)
|
||||
return error;
|
||||
if (!error) {
|
||||
if (found && git_oid_cmp(&e->sha1, &found_full_oid))
|
||||
if (found && git_oid_cmp(&e->id, &found_full_oid))
|
||||
return git_odb__error_ambiguous("found multiple pack entries");
|
||||
git_oid_cpy(&found_full_oid, &e->sha1);
|
||||
git_oid_cpy(&found_full_oid, &e->id);
|
||||
found = true;
|
||||
backend->last_found = p;
|
||||
}
|
||||
@@ -425,7 +427,10 @@ static int process_multi_pack_index_pack(
|
||||
}
|
||||
|
||||
/* Pack was not found. Allocate a new one. */
|
||||
error = git_mwindow_get_pack(&pack, git_str_cstr(&pack_path));
|
||||
error = git_mwindow_get_pack(
|
||||
&pack,
|
||||
git_str_cstr(&pack_path),
|
||||
backend->opts.oid_type);
|
||||
git_str_dispose(&pack_path);
|
||||
if (error < 0)
|
||||
return error;
|
||||
@@ -596,32 +601,33 @@ static int pack_backend__read_prefix(
|
||||
void **buffer_p,
|
||||
size_t *len_p,
|
||||
git_object_t *type_p,
|
||||
git_odb_backend *backend,
|
||||
git_odb_backend *_backend,
|
||||
const git_oid *short_oid,
|
||||
size_t len)
|
||||
{
|
||||
struct pack_backend *backend = (struct pack_backend *)_backend;
|
||||
int error = 0;
|
||||
|
||||
if (len < GIT_OID_MINPREFIXLEN)
|
||||
error = git_odb__error_ambiguous("prefix length too short");
|
||||
|
||||
else if (len >= GIT_OID_SHA1_HEXSIZE) {
|
||||
else if (len >= git_oid_hexsize(backend->opts.oid_type)) {
|
||||
/* We can fall back to regular read method */
|
||||
error = pack_backend__read(buffer_p, len_p, type_p, backend, short_oid);
|
||||
error = pack_backend__read(buffer_p, len_p, type_p, _backend, short_oid);
|
||||
if (!error)
|
||||
git_oid_cpy(out_oid, short_oid);
|
||||
} else {
|
||||
struct git_pack_entry e;
|
||||
git_rawobj raw = {NULL};
|
||||
|
||||
if ((error = pack_entry_find_prefix(
|
||||
&e, (struct pack_backend *)backend, short_oid, len)) == 0 &&
|
||||
(error = git_packfile_unpack(&raw, e.p, &e.offset)) == 0)
|
||||
if ((error = pack_entry_find_prefix(&e,
|
||||
backend, short_oid, len)) == 0 &&
|
||||
(error = git_packfile_unpack(&raw, e.p, &e.offset)) == 0)
|
||||
{
|
||||
*buffer_p = raw.data;
|
||||
*len_p = raw.len;
|
||||
*type_p = raw.type;
|
||||
git_oid_cpy(out_oid, &e.sha1);
|
||||
git_oid_cpy(out_oid, &e.id);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -642,7 +648,7 @@ static int pack_backend__exists_prefix(
|
||||
struct git_pack_entry e = {0};
|
||||
|
||||
error = pack_entry_find_prefix(&e, pb, short_id, len);
|
||||
git_oid_cpy(out, &e.sha1);
|
||||
git_oid_cpy(out, &e.id);
|
||||
return error;
|
||||
}
|
||||
|
||||
@@ -840,7 +846,10 @@ static void pack_backend__free(git_odb_backend *_backend)
|
||||
git__free(backend);
|
||||
}
|
||||
|
||||
static int pack_backend__alloc(struct pack_backend **out, size_t initial_size)
|
||||
static int pack_backend__alloc(
|
||||
struct pack_backend **out,
|
||||
size_t initial_size,
|
||||
const git_odb_backend_pack_options *opts)
|
||||
{
|
||||
struct pack_backend *backend = git__calloc(1, sizeof(struct pack_backend));
|
||||
GIT_ERROR_CHECK_ALLOC(backend);
|
||||
@@ -849,12 +858,19 @@ static int pack_backend__alloc(struct pack_backend **out, size_t initial_size)
|
||||
git__free(backend);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (git_vector_init(&backend->packs, initial_size, packfile_sort__cb) < 0) {
|
||||
git_vector_free(&backend->midx_packs);
|
||||
git__free(backend);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (opts)
|
||||
memcpy(&backend->opts, opts, sizeof(git_odb_backend_pack_options));
|
||||
|
||||
if (!backend->opts.oid_type)
|
||||
backend->opts.oid_type = GIT_OID_DEFAULT;
|
||||
|
||||
backend->parent.version = GIT_ODB_BACKEND_VERSION;
|
||||
|
||||
backend->parent.read = &pack_backend__read;
|
||||
@@ -873,17 +889,31 @@ static int pack_backend__alloc(struct pack_backend **out, size_t initial_size)
|
||||
return 0;
|
||||
}
|
||||
|
||||
int git_odb_backend_one_pack(git_odb_backend **backend_out, const char *idx)
|
||||
#ifdef GIT_EXPERIMENTAL_SHA256
|
||||
int git_odb_backend_one_pack(
|
||||
git_odb_backend **backend_out,
|
||||
const char *idx,
|
||||
const git_odb_backend_pack_options *opts)
|
||||
#else
|
||||
int git_odb_backend_one_pack(
|
||||
git_odb_backend **backend_out,
|
||||
const char *idx)
|
||||
#endif
|
||||
{
|
||||
struct pack_backend *backend = NULL;
|
||||
struct git_pack_file *packfile = NULL;
|
||||
|
||||
if (pack_backend__alloc(&backend, 1) < 0)
|
||||
#ifndef GIT_EXPERIMENTAL_SHA256
|
||||
git_odb_backend_pack_options *opts = NULL;
|
||||
#endif
|
||||
|
||||
git_oid_t oid_type = opts ? opts->oid_type : 0;
|
||||
|
||||
if (pack_backend__alloc(&backend, 1, opts) < 0)
|
||||
return -1;
|
||||
|
||||
if (git_mwindow_get_pack(&packfile, idx) < 0 ||
|
||||
git_vector_insert(&backend->packs, packfile) < 0)
|
||||
{
|
||||
if (git_mwindow_get_pack(&packfile, idx, oid_type) < 0 ||
|
||||
git_vector_insert(&backend->packs, packfile) < 0) {
|
||||
pack_backend__free((git_odb_backend *)backend);
|
||||
return -1;
|
||||
}
|
||||
@@ -892,18 +922,30 @@ int git_odb_backend_one_pack(git_odb_backend **backend_out, const char *idx)
|
||||
return 0;
|
||||
}
|
||||
|
||||
int git_odb_backend_pack(git_odb_backend **backend_out, const char *objects_dir)
|
||||
#ifdef GIT_EXPERIMENTAL_SHA256
|
||||
int git_odb_backend_pack(
|
||||
git_odb_backend **backend_out,
|
||||
const char *objects_dir,
|
||||
const git_odb_backend_pack_options *opts)
|
||||
#else
|
||||
int git_odb_backend_pack(
|
||||
git_odb_backend **backend_out,
|
||||
const char *objects_dir)
|
||||
#endif
|
||||
{
|
||||
int error = 0;
|
||||
struct pack_backend *backend = NULL;
|
||||
git_str path = GIT_STR_INIT;
|
||||
|
||||
if (pack_backend__alloc(&backend, 8) < 0)
|
||||
#ifndef GIT_EXPERIMENTAL_SHA256
|
||||
git_odb_backend_pack_options *opts = NULL;
|
||||
#endif
|
||||
|
||||
if (pack_backend__alloc(&backend, 8, opts) < 0)
|
||||
return -1;
|
||||
|
||||
if (!(error = git_str_joinpath(&path, objects_dir, "pack")) &&
|
||||
git_fs_path_isdir(git_str_cstr(&path)))
|
||||
{
|
||||
git_fs_path_isdir(git_str_cstr(&path))) {
|
||||
backend->pack_folder = git_str_detach(&path);
|
||||
error = pack_backend__refresh((git_odb_backend *)backend);
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@ static int packfile_unpack_compressed(
|
||||
* Throws GIT_EAMBIGUOUSOIDPREFIX if short oid
|
||||
* is ambiguous within the pack.
|
||||
* This method assumes that len is between
|
||||
* GIT_OID_MINPREFIXLEN and GIT_OID_SHA1_HEXSIZE.
|
||||
* GIT_OID_MINPREFIXLEN and the oid type's hexsize.
|
||||
*/
|
||||
static int pack_entry_find_offset(
|
||||
off64_t *offset_out,
|
||||
@@ -186,9 +186,9 @@ static int cache_add(
|
||||
|
||||
static void pack_index_free(struct git_pack_file *p)
|
||||
{
|
||||
if (p->oids) {
|
||||
git__free(p->oids);
|
||||
p->oids = NULL;
|
||||
if (p->ids) {
|
||||
git__free(p->ids);
|
||||
p->ids = NULL;
|
||||
}
|
||||
if (p->index_map.data) {
|
||||
git_futils_mmap_free(&p->index_map);
|
||||
@@ -205,6 +205,7 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p)
|
||||
size_t idx_size;
|
||||
struct stat st;
|
||||
int error;
|
||||
|
||||
/* TODO: properly open the file without access time using O_NOATIME */
|
||||
git_file fd = git_futils_open_ro(path);
|
||||
if (fd < 0)
|
||||
@@ -218,8 +219,7 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p)
|
||||
|
||||
if (!S_ISREG(st.st_mode) ||
|
||||
!git__is_sizet(st.st_size) ||
|
||||
(idx_size = (size_t)st.st_size) < 4 * 256 + 20 + 20)
|
||||
{
|
||||
(idx_size = (size_t)st.st_size) < (size_t)((4 * 256) + (p->oid_size * 2))) {
|
||||
p_close(fd);
|
||||
git_error_set(GIT_ERROR_ODB, "invalid pack index '%s'", path);
|
||||
return -1;
|
||||
@@ -242,8 +242,9 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p)
|
||||
return packfile_error("unsupported index version");
|
||||
}
|
||||
|
||||
} else
|
||||
} else {
|
||||
version = 1;
|
||||
}
|
||||
|
||||
nr = 0;
|
||||
index = idx_map;
|
||||
@@ -264,11 +265,11 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p)
|
||||
/*
|
||||
* Total size:
|
||||
* - 256 index entries 4 bytes each
|
||||
* - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
|
||||
* - 20-byte SHA1 of the packfile
|
||||
* - 20-byte SHA1 file checksum
|
||||
* - 24/36-byte entries * nr (20/32 byte SHA + 4-byte offset)
|
||||
* - 20/32-byte SHA of the packfile
|
||||
* - 20/32-byte SHA file checksum
|
||||
*/
|
||||
if (idx_size != 4*256 + nr * 24 + 20 + 20) {
|
||||
if (idx_size != (4 * 256 + (nr * (p->oid_size + 4)) + (p->oid_size * 2))) {
|
||||
git_futils_mmap_free(&p->index_map);
|
||||
return packfile_error("index is corrupted");
|
||||
}
|
||||
@@ -277,16 +278,16 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p)
|
||||
* Minimum size:
|
||||
* - 8 bytes of header
|
||||
* - 256 index entries 4 bytes each
|
||||
* - 20-byte sha1 entry * nr
|
||||
* - 20/32-byte SHA entry * nr
|
||||
* - 4-byte crc entry * nr
|
||||
* - 4-byte offset entry * nr
|
||||
* - 20-byte SHA1 of the packfile
|
||||
* - 20-byte SHA1 file checksum
|
||||
* - 20/32-byte SHA of the packfile
|
||||
* - 20/32-byte SHA file checksum
|
||||
* And after the 4-byte offset table might be a
|
||||
* variable sized table containing 8-byte entries
|
||||
* for offsets larger than 2^31.
|
||||
*/
|
||||
unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20;
|
||||
unsigned long min_size = 8 + (4 * 256) + (nr * (p->oid_size + 4 + 4)) + (p->oid_size * 2);
|
||||
unsigned long max_size = min_size;
|
||||
|
||||
if (nr)
|
||||
@@ -365,12 +366,12 @@ static unsigned char *pack_window_open(
|
||||
* Don't allow a negative offset, as that means we've wrapped
|
||||
* around.
|
||||
*/
|
||||
if (offset > (p->mwf.size - 20))
|
||||
if (offset > (p->mwf.size - p->oid_size))
|
||||
goto cleanup;
|
||||
if (offset < 0)
|
||||
goto cleanup;
|
||||
|
||||
pack_data = git_mwindow_open(&p->mwf, w_cursor, offset, 20, left);
|
||||
pack_data = git_mwindow_open(&p->mwf, w_cursor, offset, p->oid_size, left);
|
||||
|
||||
cleanup:
|
||||
git_mutex_unlock(&p->mwf.lock);
|
||||
@@ -473,13 +474,13 @@ int git_packfile_unpack_header(
|
||||
return error;
|
||||
}
|
||||
|
||||
/* pack_window_open() assures us we have [base, base + 20) available
|
||||
* as a range that we can look at at. (Its actually the hash
|
||||
* size that is assured.) With our object header encoding
|
||||
* the maximum deflated object size is 2^137, which is just
|
||||
* insane, so we know won't exceed what we have been given.
|
||||
/* pack_window_open() assures us we have [base, base + oid_size)
|
||||
* available as a range that we can look at at. (It's actually
|
||||
* the hash size that is assured.) With our object header
|
||||
* encoding the maximum deflated object size is 2^137, which is
|
||||
* just insane, so we know won't exceed what we have been given.
|
||||
*/
|
||||
base = git_mwindow_open(&p->mwf, w_curs, *curpos, 20, &left);
|
||||
base = git_mwindow_open(&p->mwf, w_curs, *curpos, p->oid_size, &left);
|
||||
git_mutex_unlock(&p->lock);
|
||||
git_mutex_unlock(&p->mwf.lock);
|
||||
if (base == NULL)
|
||||
@@ -977,11 +978,12 @@ int get_delta_base(
|
||||
/* Assumption: the only reason this would fail is because the file is too small */
|
||||
if (base_info == NULL)
|
||||
return GIT_EBUFS;
|
||||
/* pack_window_open() assured us we have [base_info, base_info + 20)
|
||||
* as a range that we can look at without walking off the
|
||||
* end of the mapped window. Its actually the hash size
|
||||
* that is assured. An OFS_DELTA longer than the hash size
|
||||
* is stupid, as then a REF_DELTA would be smaller to store.
|
||||
/* pack_window_open() assured us we have
|
||||
* [base_info, base_info + oid_size) as a range that we can look
|
||||
* at without walking off the end of the mapped window. Its
|
||||
* actually the hash size that is assured. An OFS_DELTA longer
|
||||
* than the hash size is stupid, as then a REF_DELTA would be
|
||||
* smaller to store.
|
||||
*/
|
||||
if (type == GIT_OBJECT_OFS_DELTA) {
|
||||
unsigned used = 0;
|
||||
@@ -1002,7 +1004,7 @@ int get_delta_base(
|
||||
*curpos += used;
|
||||
} else if (type == GIT_OBJECT_REF_DELTA) {
|
||||
git_oid base_oid;
|
||||
git_oid__fromraw(&base_oid, base_info, GIT_OID_SHA1);
|
||||
git_oid__fromraw(&base_oid, base_info, p->oid_type);
|
||||
|
||||
/* If we have the cooperative cache, search in it first */
|
||||
if (p->has_cache) {
|
||||
@@ -1012,7 +1014,7 @@ int get_delta_base(
|
||||
if (entry->offset == 0)
|
||||
return packfile_error("delta offset is zero");
|
||||
|
||||
*curpos += 20;
|
||||
*curpos += p->oid_size;
|
||||
*delta_base_out = entry->offset;
|
||||
return 0;
|
||||
} else {
|
||||
@@ -1025,9 +1027,9 @@ int get_delta_base(
|
||||
}
|
||||
|
||||
/* The base entry _must_ be in the same pack */
|
||||
if (pack_entry_find_offset(&base_offset, &unused, p, &base_oid, GIT_OID_SHA1_HEXSIZE) < 0)
|
||||
if (pack_entry_find_offset(&base_offset, &unused, p, &base_oid, p->oid_hexsize) < 0)
|
||||
return packfile_error("base entry delta is not in the same pack");
|
||||
*curpos += 20;
|
||||
*curpos += p->oid_size;
|
||||
} else
|
||||
return packfile_error("unknown object type");
|
||||
|
||||
@@ -1070,7 +1072,7 @@ void git_packfile_free(struct git_pack_file *p, bool unlink_packfile)
|
||||
|
||||
pack_index_free(p);
|
||||
|
||||
git__free(p->bad_object_sha1);
|
||||
git__free(p->bad_object_ids);
|
||||
|
||||
git_mutex_free(&p->bases.lock);
|
||||
git_mutex_free(&p->mwf.lock);
|
||||
@@ -1083,8 +1085,8 @@ static int packfile_open_locked(struct git_pack_file *p)
|
||||
{
|
||||
struct stat st;
|
||||
struct git_pack_header hdr;
|
||||
unsigned char sha1[GIT_OID_SHA1_SIZE];
|
||||
unsigned char *idx_sha1;
|
||||
unsigned char checksum[GIT_OID_MAX_SIZE];
|
||||
unsigned char *idx_checksum;
|
||||
|
||||
if (pack_index_open_locked(p) < 0)
|
||||
return git_odb__error_notfound("failed to open packfile", NULL, 0);
|
||||
@@ -1131,12 +1133,13 @@ static int packfile_open_locked(struct git_pack_file *p)
|
||||
|
||||
/* Verify the pack matches its index. */
|
||||
if (p->num_objects != ntohl(hdr.hdr_entries) ||
|
||||
p_pread(p->mwf.fd, sha1, GIT_OID_SHA1_SIZE, p->mwf.size - GIT_OID_SHA1_SIZE) < 0)
|
||||
p_pread(p->mwf.fd, checksum, p->oid_size, p->mwf.size - p->oid_size) < 0)
|
||||
goto cleanup;
|
||||
|
||||
idx_sha1 = ((unsigned char *)p->index_map.data) + p->index_map.len - 40;
|
||||
idx_checksum = ((unsigned char *)p->index_map.data) +
|
||||
p->index_map.len - (p->oid_size * 2);
|
||||
|
||||
if (git_oid_raw_cmp(sha1, idx_sha1, GIT_OID_SHA1_SIZE) != 0)
|
||||
if (git_oid_raw_cmp(checksum, idx_checksum, p->oid_size) != 0)
|
||||
goto cleanup;
|
||||
|
||||
if (git_mwindow_file_register(&p->mwf) < 0)
|
||||
@@ -1171,7 +1174,10 @@ int git_packfile__name(char **out, const char *path)
|
||||
return 0;
|
||||
}
|
||||
|
||||
int git_packfile_alloc(struct git_pack_file **pack_out, const char *path)
|
||||
int git_packfile_alloc(
|
||||
struct git_pack_file **pack_out,
|
||||
const char *path,
|
||||
git_oid_t oid_type)
|
||||
{
|
||||
struct stat st;
|
||||
struct git_pack_file *p;
|
||||
@@ -1219,6 +1225,9 @@ int git_packfile_alloc(struct git_pack_file **pack_out, const char *path)
|
||||
p->pack_local = 1;
|
||||
p->mtime = (git_time_t)st.st_mtime;
|
||||
p->index_version = -1;
|
||||
p->oid_type = oid_type ? oid_type : GIT_OID_DEFAULT;
|
||||
p->oid_size = (unsigned int)git_oid_size(p->oid_type);
|
||||
p->oid_hexsize = (unsigned int)git_oid_hexsize(p->oid_type);
|
||||
|
||||
if (git_mutex_init(&p->lock) < 0) {
|
||||
git_error_set(GIT_ERROR_OS, "failed to initialize packfile mutex");
|
||||
@@ -1260,9 +1269,9 @@ static off64_t nth_packed_object_offset_locked(struct git_pack_file *p, uint32_t
|
||||
end = index + p->index_map.len;
|
||||
index += 4 * 256;
|
||||
if (p->index_version == 1)
|
||||
return ntohl(*((uint32_t *)(index + 24 * n)));
|
||||
return ntohl(*((uint32_t *)(index + (p->oid_size + 4) * n)));
|
||||
|
||||
index += 8 + p->num_objects * (20 + 4);
|
||||
index += 8 + p->num_objects * (p->oid_size + 4);
|
||||
off32 = ntohl(*((uint32_t *)(index + 4 * n)));
|
||||
if (!(off32 & 0x80000000))
|
||||
return off32;
|
||||
@@ -1273,7 +1282,7 @@ static off64_t nth_packed_object_offset_locked(struct git_pack_file *p, uint32_t
|
||||
return -1;
|
||||
|
||||
return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) |
|
||||
ntohl(*((uint32_t *)(index + 4)));
|
||||
ntohl(*((uint32_t *)(index + 4)));
|
||||
}
|
||||
|
||||
static int git__memcmp4(const void *a, const void *b) {
|
||||
@@ -1312,7 +1321,7 @@ int git_pack_foreach_entry(
|
||||
|
||||
index += 4 * 256;
|
||||
|
||||
if (p->oids == NULL) {
|
||||
if (p->ids == NULL) {
|
||||
git_vector offsets, oids;
|
||||
|
||||
if ((error = git_vector_init(&oids, p->num_objects, NULL))) {
|
||||
@@ -1326,22 +1335,25 @@ int git_pack_foreach_entry(
|
||||
}
|
||||
|
||||
if (p->index_version > 1) {
|
||||
const unsigned char *off = index + 24 * p->num_objects;
|
||||
const unsigned char *off = index +
|
||||
(p->oid_size + 4) * p->num_objects;
|
||||
|
||||
for (i = 0; i < p->num_objects; i++)
|
||||
git_vector_insert(&offsets, (void*)&off[4 * i]);
|
||||
|
||||
git_vector_sort(&offsets);
|
||||
git_vector_foreach(&offsets, i, current)
|
||||
git_vector_insert(&oids, (void*)&index[5 * (current - off)]);
|
||||
} else {
|
||||
for (i = 0; i < p->num_objects; i++)
|
||||
git_vector_insert(&offsets, (void*)&index[24 * i]);
|
||||
git_vector_insert(&offsets, (void*)&index[(p->oid_size + 4) * i]);
|
||||
git_vector_sort(&offsets);
|
||||
git_vector_foreach(&offsets, i, current)
|
||||
git_vector_insert(&oids, (void*)¤t[4]);
|
||||
}
|
||||
|
||||
git_vector_free(&offsets);
|
||||
p->oids = (unsigned char **)git_vector_detach(NULL, NULL, &oids);
|
||||
p->ids = (unsigned char **)git_vector_detach(NULL, NULL, &oids);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -1362,7 +1374,7 @@ int git_pack_foreach_entry(
|
||||
git_array_clear(oids);
|
||||
GIT_ERROR_CHECK_ALLOC(oid);
|
||||
}
|
||||
git_oid__fromraw(oid, p->oids[i], GIT_OID_SHA1);
|
||||
git_oid__fromraw(oid, p->ids[i], p->oid_type);
|
||||
}
|
||||
|
||||
git_mutex_unlock(&p->lock);
|
||||
@@ -1412,10 +1424,13 @@ int git_pack_foreach_entry_offset(
|
||||
|
||||
/* all offsets should have been validated by pack_index_check_locked */
|
||||
if (p->index_version > 1) {
|
||||
const unsigned char *offsets = index + 24 * p->num_objects;
|
||||
const unsigned char *offsets = index +
|
||||
(p->oid_size + 4) * p->num_objects;
|
||||
const unsigned char *large_offset_ptr;
|
||||
const unsigned char *large_offsets = index + 28 * p->num_objects;
|
||||
const unsigned char *large_offsets_end = ((const unsigned char *)p->index_map.data) + p->index_map.len - 20;
|
||||
const unsigned char *large_offsets = index +
|
||||
(p->oid_size + 8) * p->num_objects;
|
||||
const unsigned char *large_offsets_end = ((const unsigned char *)p->index_map.data) + p->index_map.len - p->oid_size;
|
||||
|
||||
for (i = 0; i < p->num_objects; i++) {
|
||||
current_offset = ntohl(*(const uint32_t *)(offsets + 4 * i));
|
||||
if (current_offset & 0x80000000) {
|
||||
@@ -1428,7 +1443,7 @@ int git_pack_foreach_entry_offset(
|
||||
ntohl(*((uint32_t *)(large_offset_ptr + 4)));
|
||||
}
|
||||
|
||||
git_oid__fromraw(¤t_oid, (index + 20 * i), GIT_OID_SHA1);
|
||||
git_oid__fromraw(¤t_oid, (index + p->oid_size * i), p->oid_type);
|
||||
if ((error = cb(¤t_oid, current_offset, data)) != 0) {
|
||||
error = git_error_set_after_callback(error);
|
||||
goto cleanup;
|
||||
@@ -1436,8 +1451,8 @@ int git_pack_foreach_entry_offset(
|
||||
}
|
||||
} else {
|
||||
for (i = 0; i < p->num_objects; i++) {
|
||||
current_offset = ntohl(*(const uint32_t *)(index + 24 * i));
|
||||
git_oid__fromraw(¤t_oid, (index + 24 * i + 4), GIT_OID_SHA1);
|
||||
current_offset = ntohl(*(const uint32_t *)(index + (p->oid_size + 4) * i));
|
||||
git_oid__fromraw(¤t_oid, (index + (p->oid_size + 4) * i + 4), p->oid_type);
|
||||
if ((error = cb(¤t_oid, current_offset, data)) != 0) {
|
||||
error = git_error_set_after_callback(error);
|
||||
goto cleanup;
|
||||
@@ -1450,14 +1465,20 @@ cleanup:
|
||||
return error;
|
||||
}
|
||||
|
||||
int git_pack__lookup_sha1(const void *oid_lookup_table, size_t stride, unsigned lo,
|
||||
unsigned hi, const unsigned char *oid_prefix)
|
||||
int git_pack__lookup_id(
|
||||
const void *oid_lookup_table,
|
||||
size_t stride,
|
||||
unsigned lo,
|
||||
unsigned hi,
|
||||
const unsigned char *oid_prefix,
|
||||
const git_oid_t oid_type)
|
||||
{
|
||||
const unsigned char *base = oid_lookup_table;
|
||||
size_t oid_size = git_oid_size(oid_type);
|
||||
|
||||
while (lo < hi) {
|
||||
unsigned mi = (lo + hi) / 2;
|
||||
int cmp = git_oid_raw_cmp(base + mi * stride, oid_prefix, GIT_OID_SHA1_SIZE);
|
||||
int cmp = git_oid_raw_cmp(base + mi * stride, oid_prefix, oid_size);
|
||||
|
||||
if (!cmp)
|
||||
return mi;
|
||||
@@ -1512,9 +1533,9 @@ static int pack_entry_find_offset(
|
||||
lo = ((short_oid->id[0] == 0x0) ? 0 : ntohl(level1_ofs[(int)short_oid->id[0] - 1]));
|
||||
|
||||
if (p->index_version > 1) {
|
||||
stride = 20;
|
||||
stride = p->oid_size;
|
||||
} else {
|
||||
stride = 24;
|
||||
stride = p->oid_size + 4;
|
||||
index += 4;
|
||||
}
|
||||
|
||||
@@ -1523,7 +1544,8 @@ static int pack_entry_find_offset(
|
||||
short_oid->id[0], short_oid->id[1], short_oid->id[2], lo, hi, p->num_objects);
|
||||
#endif
|
||||
|
||||
pos = git_pack__lookup_sha1(index, stride, lo, hi, short_oid->id);
|
||||
pos = git_pack__lookup_id(index, stride, lo, hi,
|
||||
short_oid->id, p->oid_type);
|
||||
|
||||
if (pos >= 0) {
|
||||
/* An object matching exactly the oid was found */
|
||||
@@ -1541,7 +1563,9 @@ static int pack_entry_find_offset(
|
||||
}
|
||||
}
|
||||
|
||||
if (found && len != GIT_OID_SHA1_HEXSIZE && pos + 1 < (int)p->num_objects) {
|
||||
if (found &&
|
||||
len != p->oid_hexsize &&
|
||||
pos + 1 < (int)p->num_objects) {
|
||||
/* Check for ambiguousity */
|
||||
const unsigned char *next = current + stride;
|
||||
|
||||
@@ -1566,13 +1590,13 @@ static int pack_entry_find_offset(
|
||||
}
|
||||
|
||||
*offset_out = offset;
|
||||
git_oid__fromraw(found_oid, current, GIT_OID_SHA1);
|
||||
git_oid__fromraw(found_oid, current, p->oid_type);
|
||||
|
||||
#ifdef INDEX_DEBUG_LOOKUP
|
||||
{
|
||||
unsigned char hex_sha1[GIT_OID_SHA1_HEXSIZE + 1];
|
||||
char hex_sha1[p->oid_hexsize + 1];
|
||||
git_oid_fmt(hex_sha1, found_oid);
|
||||
hex_sha1[GIT_OID_SHA1_HEXSIZE] = '\0';
|
||||
hex_sha1[p->oid_hexsize] = '\0';
|
||||
printf("found lo=%d %s\n", lo, hex_sha1);
|
||||
}
|
||||
#endif
|
||||
@@ -1594,10 +1618,10 @@ int git_pack_entry_find(
|
||||
|
||||
GIT_ASSERT_ARG(p);
|
||||
|
||||
if (len == GIT_OID_SHA1_HEXSIZE && p->num_bad_objects) {
|
||||
if (len == p->oid_hexsize && p->num_bad_objects) {
|
||||
unsigned i;
|
||||
for (i = 0; i < p->num_bad_objects; i++)
|
||||
if (git_oid__cmp(short_oid, &p->bad_object_sha1[i]) == 0)
|
||||
if (git_oid__cmp(short_oid, &p->bad_object_ids[i]) == 0)
|
||||
return packfile_error("bad object found in packfile");
|
||||
}
|
||||
|
||||
@@ -1630,6 +1654,6 @@ int git_pack_entry_find(
|
||||
e->offset = offset;
|
||||
e->p = p;
|
||||
|
||||
git_oid_cpy(&e->sha1, &found_oid);
|
||||
git_oid_cpy(&e->id, &found_oid);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -99,13 +99,19 @@ struct git_pack_file {
|
||||
|
||||
uint32_t num_objects;
|
||||
uint32_t num_bad_objects;
|
||||
git_oid *bad_object_sha1; /* array of git_oid */
|
||||
git_oid *bad_object_ids; /* array of git_oid */
|
||||
|
||||
git_oid_t oid_type;
|
||||
unsigned oid_hexsize:7,
|
||||
oid_size:6,
|
||||
pack_local:1,
|
||||
pack_keep:1,
|
||||
has_cache:1;
|
||||
|
||||
int index_version;
|
||||
git_time_t mtime;
|
||||
unsigned pack_local:1, pack_keep:1, has_cache:1;
|
||||
git_oidmap *idx_cache;
|
||||
unsigned char **oids;
|
||||
unsigned char **ids;
|
||||
|
||||
git_pack_cache bases; /* delta base cache */
|
||||
|
||||
@@ -116,21 +122,26 @@ struct git_pack_file {
|
||||
};
|
||||
|
||||
/**
|
||||
* Return the position where an OID (or a prefix) would be inserted within the
|
||||
* OID Lookup Table of an .idx file. This performs binary search between the lo
|
||||
* and hi indices.
|
||||
* Return the position where an OID (or a prefix) would be inserted within
|
||||
* the OID Lookup Table of an .idx file. This performs binary search
|
||||
* between the lo and hi indices.
|
||||
*
|
||||
* The stride parameter is provided because .idx files version 1 store the OIDs
|
||||
* interleaved with the 4-byte file offsets of the objects within the .pack
|
||||
* file (stride = 24), whereas files with version 2 store them in a contiguous
|
||||
* flat array (stride = 20).
|
||||
* The stride parameter is provided because .idx files version 1 store the
|
||||
* OIDs interleaved with the 4-byte file offsets of the objects within the
|
||||
* .pack file (stride = oid_size + 4), whereas files with version 2 store
|
||||
* them in a contiguous flat array (stride = oid_size).
|
||||
*/
|
||||
int git_pack__lookup_sha1(const void *oid_lookup_table, size_t stride, unsigned lo,
|
||||
unsigned hi, const unsigned char *oid_prefix);
|
||||
int git_pack__lookup_id(
|
||||
const void *id_lookup_table,
|
||||
size_t stride,
|
||||
unsigned lo,
|
||||
unsigned hi,
|
||||
const unsigned char *id_prefix,
|
||||
const git_oid_t oid_type);
|
||||
|
||||
struct git_pack_entry {
|
||||
off64_t offset;
|
||||
git_oid sha1;
|
||||
git_oid id;
|
||||
struct git_pack_file *p;
|
||||
};
|
||||
|
||||
@@ -174,12 +185,15 @@ int get_delta_base(
|
||||
off64_t delta_obj_offset);
|
||||
|
||||
void git_packfile_free(struct git_pack_file *p, bool unlink_packfile);
|
||||
int git_packfile_alloc(struct git_pack_file **pack_out, const char *path);
|
||||
int git_packfile_alloc(
|
||||
struct git_pack_file **pack_out,
|
||||
const char *path,
|
||||
git_oid_t oid_type);
|
||||
|
||||
int git_pack_entry_find(
|
||||
struct git_pack_entry *e,
|
||||
struct git_pack_file *p,
|
||||
const git_oid *short_oid,
|
||||
const git_oid *short_id,
|
||||
size_t len);
|
||||
int git_pack_foreach_entry(
|
||||
struct git_pack_file *p,
|
||||
|
||||
@@ -52,7 +52,16 @@ void test_odb_foreach__one_pack(void)
|
||||
int nobj = 0;
|
||||
|
||||
cl_git_pass(git_odb__new(&_odb, NULL));
|
||||
cl_git_pass(git_odb_backend_one_pack(&backend, cl_fixture("testrepo.git/objects/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx")));
|
||||
|
||||
#ifdef GIT_EXPERIMENTAL_SHA256
|
||||
cl_git_pass(git_odb_backend_one_pack(&backend,
|
||||
cl_fixture("testrepo.git/objects/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx"),
|
||||
NULL));
|
||||
#else
|
||||
cl_git_pass(git_odb_backend_one_pack(&backend,
|
||||
cl_fixture("testrepo.git/objects/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx")));
|
||||
#endif
|
||||
|
||||
cl_git_pass(git_odb_add_backend(_odb, backend, 1));
|
||||
_repo = NULL;
|
||||
|
||||
|
||||
21
tests/libgit2/odb/pack_data_one256.h
Normal file
21
tests/libgit2/odb/pack_data_one256.h
Normal file
@@ -0,0 +1,21 @@
|
||||
/* Just a few to make sure it's working, the rest is tested already */
|
||||
#ifdef GIT_EXPERIMENTAL_SHA256
|
||||
static const char *packed_objects_one256[] = {
|
||||
"ea926306b1bab6d3f25f45609907eb6dff91a1460b25e63bf4a0494c70e7a269",
|
||||
"d048ba2ef4fafa502a44cbc1a50cd58359b9bc243b84a08f541a08ca5f621137",
|
||||
"a66bda0109d2b3c9bc87970da81bd91076b5f871febbc860f09ae997668b6800",
|
||||
"3609a41c0506fe19d01fb8b4729923362675f191fe5f63fab3111ef804c48fdf",
|
||||
"22b6705b86e4aa9120eff203af24709d483698d9f78695e86e82de121784b570",
|
||||
"6f11d93bfb269ee8c7a506178f60c430abfac5d424acfd9c0b0b27b98e6ab49b",
|
||||
"0aefd477d9e5b3f8d708a3cf6d78be6b670dfa2e2ec41244634f3b8f115d8e04",
|
||||
"580474d948cd2ebd2e5ce7a5b81b872d87ba4639c1ac4c0fa7a11a8eddf9827c",
|
||||
"0636b4292bfdd7274a977cb6f8b2ded8f315ea1bcd8dbedfca37964c2ed3d085",
|
||||
"19fb1c78b11f0f8bda658d6fa6cc63c0b573c0f6760ee5a9c2df6ce2cde00c5c",
|
||||
"7f2f7afccb317bb3fdd28555f126846dc4eebe5d9ae7b8d8a1456e8ff85422ce",
|
||||
"4066249c68b0d3c8b39ebe02c9188935900465acad02a49269710f56720fa58e",
|
||||
"a560d1fa1edf114f57b402e16d662c17b1e3b7e8601ff7dcea6615ba7f1e48ef",
|
||||
"58923faa87c7d559d308a114ec2b164e5d6046c889420ed1def6eef2d55106a2",
|
||||
"753ddabab8ae9c1e733cda15e8e3c83dd43f5a2d939ae32cc3b30b0be1e91f96",
|
||||
"46333d32b3801cf11d9f80b557245c9e32b0e05deca61dae968914fde159f0e5"
|
||||
};
|
||||
#endif
|
||||
@@ -11,7 +11,14 @@ void test_odb_packedone__initialize(void)
|
||||
git_odb_backend *backend = NULL;
|
||||
|
||||
cl_git_pass(git_odb__new(&_odb, NULL));
|
||||
cl_git_pass(git_odb_backend_one_pack(&backend, cl_fixture("testrepo.git/objects/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx")));
|
||||
#ifdef GIT_EXPERIMENTAL_SHA256
|
||||
cl_git_pass(git_odb_backend_one_pack(&backend,
|
||||
cl_fixture("testrepo.git/objects/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx"),
|
||||
NULL));
|
||||
#else
|
||||
cl_git_pass(git_odb_backend_one_pack(&backend,
|
||||
cl_fixture("testrepo.git/objects/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx")));
|
||||
#endif
|
||||
cl_git_pass(git_odb_add_backend(_odb, backend, 1));
|
||||
}
|
||||
|
||||
|
||||
78
tests/libgit2/odb/packedone256.c
Normal file
78
tests/libgit2/odb/packedone256.c
Normal file
@@ -0,0 +1,78 @@
|
||||
#include "clar_libgit2.h"
|
||||
#include "git2/odb_backend.h"
|
||||
|
||||
#include "pack_data_one256.h"
|
||||
#include "pack.h"
|
||||
|
||||
#ifdef GIT_EXPERIMENTAL_SHA256
|
||||
static git_odb *_odb;
|
||||
#endif
|
||||
|
||||
void test_odb_packedone256__initialize(void)
|
||||
{
|
||||
#ifdef GIT_EXPERIMENTAL_SHA256
|
||||
git_odb_backend *backend = NULL;
|
||||
git_odb_options odb_opts = GIT_ODB_OPTIONS_INIT;
|
||||
git_odb_backend_pack_options backend_opts = GIT_ODB_BACKEND_PACK_OPTIONS_INIT;
|
||||
|
||||
odb_opts.oid_type = GIT_OID_SHA256;
|
||||
backend_opts.oid_type = GIT_OID_SHA256;
|
||||
|
||||
cl_git_pass(git_odb__new(&_odb, &odb_opts));
|
||||
cl_git_pass(git_odb_backend_one_pack(
|
||||
&backend,
|
||||
cl_fixture("testrepo_256.git/objects/pack/pack-e2f07f30db7e480ea84a0e64ee791b9b270067124b2609019b74f33f256f33fa.idx"),
|
||||
&backend_opts));
|
||||
cl_git_pass(git_odb_add_backend(_odb, backend, 1));
|
||||
#endif
|
||||
}
|
||||
|
||||
void test_odb_packedone256__cleanup(void)
|
||||
{
|
||||
#ifdef GIT_EXPERIMENTAL_SHA256
|
||||
git_odb_free(_odb);
|
||||
_odb = NULL;
|
||||
#endif
|
||||
}
|
||||
|
||||
void test_odb_packedone256__mass_read(void)
|
||||
{
|
||||
#ifdef GIT_EXPERIMENTAL_SHA256
|
||||
unsigned int i;
|
||||
|
||||
for (i = 0; i < ARRAY_SIZE(packed_objects_one256); ++i) {
|
||||
git_oid id;
|
||||
git_odb_object *obj;
|
||||
|
||||
cl_git_pass(git_oid__fromstr(&id, packed_objects_one256[i], GIT_OID_SHA256));
|
||||
cl_assert(git_odb_exists(_odb, &id) == 1);
|
||||
cl_git_pass(git_odb_read(&obj, _odb, &id));
|
||||
|
||||
git_odb_object_free(obj);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
void test_odb_packedone256__read_header_0(void)
|
||||
{
|
||||
#ifdef GIT_EXPERIMENTAL_SHA256
|
||||
unsigned int i;
|
||||
|
||||
for (i = 0; i < ARRAY_SIZE(packed_objects_one256); ++i) {
|
||||
git_oid id;
|
||||
git_odb_object *obj;
|
||||
size_t len;
|
||||
git_object_t type;
|
||||
|
||||
cl_git_pass(git_oid__fromstr(&id, packed_objects_one256[i], GIT_OID_SHA256));
|
||||
|
||||
cl_git_pass(git_odb_read(&obj, _odb, &id));
|
||||
cl_git_pass(git_odb_read_header(&len, &type, _odb, &id));
|
||||
|
||||
cl_assert(obj->cached.size == len);
|
||||
cl_assert(obj->cached.type == type);
|
||||
|
||||
git_odb_object_free(obj);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
@@ -82,7 +82,11 @@ void test_odb_sorting__override_default_backend_priority(void)
|
||||
|
||||
cl_git_pass(git_libgit2_opts(GIT_OPT_SET_ODB_LOOSE_PRIORITY, 5));
|
||||
cl_git_pass(git_libgit2_opts(GIT_OPT_SET_ODB_PACKED_PRIORITY, 3));
|
||||
git_odb_backend_pack(&packed, "./testrepo.git/objects");
|
||||
git_odb_backend_pack(&packed, "./testrepo.git/objects"
|
||||
#ifdef GIT_EXPERIMENTAL_SHA256
|
||||
, NULL
|
||||
#endif
|
||||
);
|
||||
git_odb__backend_loose(&loose, "./testrepo.git/objects", NULL);
|
||||
|
||||
cl_git_pass(git_odb__open(&new_odb, cl_fixture("testrepo.git/objects"), NULL));
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Reference in New Issue
Block a user