Idea-Studios/iKv
2
1
Fork 1
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

perf(writer): streamline v2 binary writes and benchmarks

Browse Source
This commit is contained in:
GigabiteStudios
2026-06-17 11:48:46 -05:00
parent 23428aa378
commit 2a528e2b0d
4 changed files with 404 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

328
demo/unit_test.c
View File

@@ -1,6 +1,7 @@
#include "ikv.h"
#include <stdbool.h>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@@ -22,7 +23,7 @@
typedef struct
{
char message[256];
char benchmark_summary[256];
char benchmark_summary[4096];
} test_context_t;
typedef int (*test_fn_t)(test_context_t *context);
@@ -87,11 +88,13 @@ static double benchmark_now_us(void)
#endif
}
static ikv_node_t *create_benchmark_root(void)
static ikv_node_t *create_benchmark_root(unsigned int scale)
{
ikv_node_t *root = ikv_create_object("benchmark");
ikv_node_t *inventory = NULL;
ikv_node_t *stats = NULL;
unsigned int item_count = 0u;
unsigned int stat_count = 0u;
if (!root)
return NULL;
@@ -109,16 +112,44 @@ static ikv_node_t *create_benchmark_root(void)
return NULL;
}
ikv_array_add_string(inventory, "wrench");
ikv_array_add_string(inventory, "battery");
ikv_array_add_string(inventory, "map");
ikv_array_add_string(inventory, "radio");
ikv_array_add_string(inventory, "flares");
item_count = scale * 8u;
if (item_count < 5u)
item_count = 5u;
for (unsigned int i = 0; i < item_count; ++i)
{
char item_name[64];
snprintf(item_name, sizeof(item_name), "item-%u-payload-%u", scale, i);
ikv_array_add_string(inventory, item_name);
}
stat_count = scale * 16u;
if (stat_count < 4u)
stat_count = 4u;
for (unsigned int i = 0; i < stat_count; ++i)
{
char key[64];
snprintf(key, sizeof(key), "metric_%u", i);
switch (i % 4u)
{
case 0u:
ikv_object_set_int(stats, key, (int64_t)(scale * 100u + i));
break;
case 1u:
ikv_object_set_float(stats, key, (double)scale * 0.5 + (double)i * 0.25);
break;
case 2u:
ikv_object_set_bool(stats, key, (i & 1u) != 0u);
break;
default:
{
char value[64];
snprintf(value, sizeof(value), "value-%u-%u", scale, i);
ikv_object_set_string(stats, key, value);
break;
}
}
}
ikv_object_set_int(stats, "health", 100);
ikv_object_set_int(stats, "armor", 75);
ikv_object_set_float(stats, "speed", 12.75);
ikv_object_set_bool(stats, "indoors", false);
return root;
}
@@ -1524,28 +1555,207 @@ static int test_hook_alloc_fail_internal_v2_memory_copy(test_context_t *context)
}
#endif
typedef struct
{
unsigned int scale;
unsigned int iterations;
double avg_index_us;
double avg_read_us;
double avg_write_us;
} benchmark_sample_t;
typedef enum
{
BENCHMARK_FORMAT_TEXT_V1 = 0,
BENCHMARK_FORMAT_TEXT_V2,
BENCHMARK_FORMAT_BINARY_V1,
BENCHMARK_FORMAT_BINARY_V2
} benchmark_format_t;
static const char *benchmark_big_o_label(double exponent)
{
if (exponent < 0.15)
return "O(1)";
if (exponent < 0.50)
return "O(log n)";
if (exponent < 1.20)
return "O(n)";
if (exponent < 1.60)
return "O(n log n)";
if (exponent < 2.40)
return "O(n^2)";
return "O(n^k)";
}
static double benchmark_fit_exponent(const benchmark_sample_t *samples, size_t count, int metric)
{
double sum_x = 0.0;
double sum_y = 0.0;
double sum_xx = 0.0;
double sum_xy = 0.0;
size_t used = 0u;
for (size_t i = 0; i < count; ++i)
{
double y = 0.0;
double x = 0.0;
if (!samples || samples[i].scale == 0u)
continue;
switch (metric)
{
case 0:
y = samples[i].avg_index_us;
break;
case 1:
y = samples[i].avg_read_us;
break;
default:
y = samples[i].avg_write_us;
break;
}
if (y <= 0.0)
continue;
x = log((double)samples[i].scale);
y = log(y);
sum_x += x;
sum_y += y;
sum_xx += x * x;
sum_xy += x * y;
++used;
}
if (used < 2u)
return 0.0;
if (fabs((double)used * sum_xx - sum_x * sum_x) < 1e-9)
return 0.0;
return (((double)used * sum_xy) - (sum_x * sum_y)) / (((double)used * sum_xx) - (sum_x * sum_x));
}
static const char *benchmark_format_name(benchmark_format_t format)
{
switch (format)
{
case BENCHMARK_FORMAT_TEXT_V1:
return "v1-text";
case BENCHMARK_FORMAT_TEXT_V2:
return "v2-text";
case BENCHMARK_FORMAT_BINARY_V1:
return "v1-bin";
case BENCHMARK_FORMAT_BINARY_V2:
return "v2-bin";
default:
return "unknown";
}
}
static const char *benchmark_format_path(benchmark_format_t format)
{
switch (format)
{
case BENCHMARK_FORMAT_TEXT_V1:
return "demo_benchmark_v1.ikv";
case BENCHMARK_FORMAT_TEXT_V2:
return "demo_benchmark_v2.ikv";
case BENCHMARK_FORMAT_BINARY_V1:
return "demo_benchmark_v1.ikvb";
case BENCHMARK_FORMAT_BINARY_V2:
return "demo_benchmark_v2.ikvb";
default:
return "demo_benchmark.ikv";
}
}
static bool benchmark_write_file(benchmark_format_t format, const char *path, const ikv_node_t *root)
{
switch (format)
{
case BENCHMARK_FORMAT_TEXT_V1:
return ikv_write_file_version(path, root, IKV_VERSION_1);
case BENCHMARK_FORMAT_TEXT_V2:
return ikv_write_file_version(path, root, IKV_VERSION_2);
case BENCHMARK_FORMAT_BINARY_V1:
return ikvb_write_file_version(path, root, IKV_VERSION_1);
case BENCHMARK_FORMAT_BINARY_V2:
return ikvb_write_file_version(path, root, IKV_VERSION_2);
default:
return false;
}
}
static ikv_node_t *benchmark_parse_file(benchmark_format_t format, const char *path)
{
switch (format)
{
case BENCHMARK_FORMAT_TEXT_V1:
return ikv_parse_file_version(path, IKV_VERSION_1);
case BENCHMARK_FORMAT_TEXT_V2:
return ikv_parse_file_version(path, IKV_VERSION_2);
case BENCHMARK_FORMAT_BINARY_V1:
return ikvb_parse_file_version(path, IKV_VERSION_1);
case BENCHMARK_FORMAT_BINARY_V2:
return ikvb_parse_file_version(path, IKV_VERSION_2);
default:
return NULL;
}
}
static int test_binary_v2_file_benchmark(test_context_t *context)
{
enum { benchmark_iterations = 1000 };
const char *path = "demo_benchmark_v2.ikvb";
ikv_node_t *root = NULL;
static const unsigned int benchmark_scales[] = {1u, 4u, 16u, 64u};
static const benchmark_format_t benchmark_formats[] = {
BENCHMARK_FORMAT_TEXT_V1,
BENCHMARK_FORMAT_TEXT_V2,
BENCHMARK_FORMAT_BINARY_V1,
BENCHMARK_FORMAT_BINARY_V2
};
enum { benchmark_scale_count = 4, benchmark_iterations_small = 1000, benchmark_iterations_medium = 400, benchmark_iterations_large = 150 };
benchmark_sample_t samples[4][benchmark_scale_count];
size_t summary_offset = 0u;
int result = 0;
memset(samples, 0, sizeof(samples));
context->benchmark_summary[0] = 0;
for (unsigned int format_index = 0; result == 0 && format_index < 4u; ++format_index)
{
benchmark_format_t format = benchmark_formats[format_index];
const char *path = benchmark_format_path(format);
cleanup_file(path);
for (unsigned int sample_index = 0; result == 0 && sample_index < benchmark_scale_count; ++sample_index)
{
unsigned int scale = benchmark_scales[sample_index];
unsigned int iterations = (scale <= 4u) ? benchmark_iterations_small : ((scale <= 16u) ? benchmark_iterations_medium : benchmark_iterations_large);
unsigned int expected_inventory_size = (scale * 8u < 5u) ? 5u : (scale * 8u);
ikv_node_t *root = create_benchmark_root(scale);
double total_index_us = 0.0;
double total_read_us = 0.0;
double total_write_us = 0.0;
int result = 0;
cleanup_file(path);
root = create_benchmark_root();
if (!root)
return fail(context, "failed to create benchmark root");
{
result = fail(context, "failed to create benchmark root");
break;
}
if (!ikvb_write_file(path, root))
if (!benchmark_write_file(format, path, root))
{
ikv_free(root);
result = fail(context, "failed to prepare benchmark file");
break;
}
for (unsigned int i = 0; result == 0 && i < benchmark_iterations; ++i)
for (unsigned int i = 0; result == 0 && i < iterations; ++i)
{
double start_time = benchmark_now_us();
ikv_node_t *loaded = ikv_parse_file(path);
ikv_node_t *loaded = benchmark_parse_file(format, path);
double end_time = benchmark_now_us();
if (!loaded)
@@ -1556,9 +1766,9 @@ static int test_binary_v2_file_benchmark(test_context_t *context)
ikv_free(loaded);
}
for (unsigned int i = 0; result == 0 && i < benchmark_iterations; ++i)
for (unsigned int i = 0; result == 0 && i < iterations; ++i)
{
ikv_node_t *loaded = ikv_parse_file(path);
ikv_node_t *loaded = benchmark_parse_file(format, path);
ikv_node_t *inventory = NULL;
double start_time = 0.0;
double end_time = 0.0;
@@ -1574,7 +1784,7 @@ static int test_binary_v2_file_benchmark(test_context_t *context)
end_time = benchmark_now_us();
if ((result = expect_true(context, inventory != NULL, "benchmark read inventory lookup failed")) == 0 &&
(result = expect_true(context, ikv_array_size(inventory) == 5u, "benchmark read inventory size mismatch")) == 0)
(result = expect_true(context, ikv_array_size(inventory) == expected_inventory_size, "benchmark read inventory size mismatch")) == 0)
{
total_read_us += end_time - start_time;
}
@@ -1582,10 +1792,10 @@ static int test_binary_v2_file_benchmark(test_context_t *context)
ikv_free(loaded);
}
for (unsigned int i = 0; result == 0 && i < benchmark_iterations; ++i)
for (unsigned int i = 0; result == 0 && i < iterations; ++i)
{
double start_time = benchmark_now_us();
bool ok = ikvb_write_file(path, root);
bool ok = benchmark_write_file(format, path, root);
double end_time = benchmark_now_us();
if (!ok)
@@ -1596,17 +1806,65 @@ static int test_binary_v2_file_benchmark(test_context_t *context)
if (result == 0)
{
snprintf(context->benchmark_summary,
sizeof(context->benchmark_summary),
"avg index %.2fus avg read %.2fus avg write %.2fus over %u iterations",
total_index_us / (double)benchmark_iterations,
total_read_us / (double)benchmark_iterations,
total_write_us / (double)benchmark_iterations,
(unsigned int)benchmark_iterations);
samples[format_index][sample_index].scale = scale;
samples[format_index][sample_index].iterations = iterations;
samples[format_index][sample_index].avg_index_us = total_index_us / (double)iterations;
samples[format_index][sample_index].avg_read_us = total_read_us / (double)iterations;
samples[format_index][sample_index].avg_write_us = total_write_us / (double)iterations;
}
cleanup_file(path);
ikv_free(root);
cleanup_file(path);
}
}
if (result == 0)
{
for (unsigned int format_index = 0; format_index < 4u; ++format_index)
{
const benchmark_sample_t *format_samples = samples[format_index];
double index_exp = benchmark_fit_exponent(format_samples, benchmark_scale_count, 0);
double read_exp = benchmark_fit_exponent(format_samples, benchmark_scale_count, 1);
double write_exp = benchmark_fit_exponent(format_samples, benchmark_scale_count, 2);
int wrote = snprintf(
context->benchmark_summary + summary_offset,
sizeof(context->benchmark_summary) - summary_offset,
"%s index %s^%.2f [n=%u %.2fus, n=%u %.2fus, n=%u %.2fus, n=%u %.2fus] "
"read %s^%.2f [n=%u %.2fus, n=%u %.2fus, n=%u %.2fus, n=%u %.2fus] "
"write %s^%.2f [n=%u %.2fus, n=%u %.2fus, n=%u %.2fus, n=%u %.2fus]%s",
benchmark_format_name(benchmark_formats[format_index]),
benchmark_big_o_label(index_exp), index_exp,
format_samples[0].scale, format_samples[0].avg_index_us,
format_samples[1].scale, format_samples[1].avg_index_us,
format_samples[2].scale, format_samples[2].avg_index_us,
format_samples[3].scale, format_samples[3].avg_index_us,
benchmark_big_o_label(read_exp), read_exp,
format_samples[0].scale, format_samples[0].avg_read_us,
format_samples[1].scale, format_samples[1].avg_read_us,
format_samples[2].scale, format_samples[2].avg_read_us,
format_samples[3].scale, format_samples[3].avg_read_us,
benchmark_big_o_label(write_exp), write_exp,
format_samples[0].scale, format_samples[0].avg_write_us,
format_samples[1].scale, format_samples[1].avg_write_us,
format_samples[2].scale, format_samples[2].avg_write_us,
format_samples[3].scale, format_samples[3].avg_write_us,
(format_index + 1u < 4u) ? "\n" : "");
if (wrote < 0)
{
result = fail(context, "failed to format benchmark summary");
break;
}
if ((size_t)wrote >= sizeof(context->benchmark_summary) - summary_offset)
{
summary_offset = sizeof(context->benchmark_summary) - 1u;
break;
}
summary_offset += (size_t)wrote;
}
}
return result;
}
@@ -1711,7 +1969,7 @@ static void log_case(bool passed, unsigned int index, unsigned int total, long e
putchar('\n');
}
static char benchmark_summary[256];
static char benchmark_summary[4096];
static int run_test_case(unsigned int index, unsigned int total)
{

27
src/ikv.c
View File

@@ -1754,6 +1754,33 @@ bool ikv__write_binary_node_memory(const ikv_node_t *node, uint8_t **out_data, u
return true;
}
bool ikv__write_binary_node_append(const ikv_node_t *node, uint8_t **data, size_t *size, size_t *capacity)
{
bw_mem_t mem = {0};
bw_t w = {0};
size_t start_size = 0u;
if (!node || !data || !size || !capacity)
return false;
mem.data = *data;
mem.size = *size;
mem.cap = *capacity;
start_size = mem.size;
w.ctx = &mem;
w.ok = true;
w.write_bytes_fn = bw_mem_write_bytes;
bw_node(&w, node);
if (!w.ok || mem.size == start_size || mem.size > 0xFFFFFFFFu)
return false;
*data = mem.data;
*size = mem.size;
*capacity = mem.cap;
return true;
}
bool ikv__write_binary_file_version(const char *path, const ikv_node_t *root, uint32_t version)
{
if (!path || !root)

1
src/internal/ikv_internal.h
View File

@@ -89,6 +89,7 @@ bool ikv__read_file_bytes(const char *path, uint8_t **out_data, size_t *out_size
bool ikv__write_binary_file_version(const char *path, const ikv_node_t *root, uint32_t version);
bool ikv__write_binary_memory_version(const ikv_node_t *root, uint8_t **out_data, uint32_t *out_size, uint32_t version);
bool ikv__write_binary_node_append(const ikv_node_t *node, uint8_t **data, size_t *size, size_t *capacity);
ikv_node_t *ikv__parse_binary_file_version(const char *path, uint32_t version);
ikv_node_t *ikv__parse_binary_memory_version(const void *data, size_t size, uint32_t version);

73
src/loaders/ikv2.c
View File

@@ -14,9 +14,10 @@ typedef struct
uint32_t key_length;
uint32_t key_hash;
uint8_t type;
const ikv_node_t *source_node;
uint32_t metadata_offset;
uint32_t payload_offset;
uint32_t payload_size;
uint8_t *payload_data;
uint32_t next_in_bucket;
} ikv2_index_entry_t;
@@ -162,6 +163,16 @@ static void ikv2_buffer_write_u32le(ikv2_buffer_t *buffer, uint32_t value)
ikv2_buffer_write_bytes(buffer, bytes, sizeof(bytes));
}
static void ikv2_patch_u32le(uint8_t *data, uint32_t value)
{
if (!data)
return;
data[0] = (uint8_t)(value & 0xFFu);
data[1] = (uint8_t)((value >> 8) & 0xFFu);
data[2] = (uint8_t)((value >> 16) & 0xFFu);
data[3] = (uint8_t)((value >> 24) & 0xFFu);
}
static void ikv2_buffer_write_varu32(ikv2_buffer_t *buffer, uint32_t value)
{
while (value >= 0x80u)
@@ -273,17 +284,8 @@ static bool ikv2_collect_root_entries(const ikv_node_t *root, ikv2_index_entry_t
{
for (ikv_node_t *node = root->value.object.buckets[bucket]; node; node = node->next)
{
uint8_t *payload = NULL;
uint32_t payload_size = 0u;
if (index >= count || !ikv__write_binary_node_memory(node, &payload, &payload_size))
if (index >= count)
{
if (payload)
IKV_FREE(payload);
for (uint32_t i = 0; i < index; ++i)
{
IKV_FREE(entries[i].payload_data);
}
IKV_FREE(entries);
return false;
}
@@ -292,8 +294,7 @@ static bool ikv2_collect_root_entries(const ikv_node_t *root, ikv2_index_entry_t
entries[index].key_length = (uint32_t)strlen(entries[index].key);
entries[index].key_hash = ikv2_hash_key(entries[index].key);
entries[index].type = (uint8_t)node->type;
entries[index].payload_data = payload;
entries[index].payload_size = payload_size;
entries[index].source_node = node;
entries[index].next_in_bucket = IKV2_INDEX_NONE;
++index;
@@ -311,8 +312,6 @@ static bool ikv2_build_indexed_binary(const ikv_node_t *root, uint8_t **out_data
ikv2_index_entry_t *entries = NULL;
uint32_t entry_count = 0u;
uint32_t header_size = 0u;
uint32_t payload_base = 0u;
uint32_t total_size = 0u;
const char *root_key = NULL;
uint32_t root_key_length = 0u;
ikv2_buffer_t buffer = {0};
@@ -335,17 +334,10 @@ static bool ikv2_build_indexed_binary(const ikv_node_t *root, uint8_t **out_data
for (uint32_t i = 0; i < entry_count; ++i)
header_size += ikv2_varu32_size(entries[i].key_length) + entries[i].key_length;
header_size += entry_count * (1u + 4u + 4u);
payload_base = header_size;
total_size = header_size;
for (uint32_t i = 0; i < entry_count; ++i)
total_size += entries[i].payload_size;
buffer.data = total_size ? (uint8_t *)IKV_MALLOC(total_size) : NULL;
buffer.capacity = total_size;
if (total_size > 0u && !buffer.data)
buffer.capacity = header_size;
buffer.data = header_size ? (uint8_t *)IKV_MALLOC(header_size) : NULL;
if (header_size > 0u && !buffer.data)
{
for (uint32_t i = 0; i < entry_count; ++i)
IKV_FREE(entries[i].payload_data);
IKV_FREE(entries);
return false;
}
@@ -362,21 +354,40 @@ static bool ikv2_build_indexed_binary(const ikv_node_t *root, uint8_t **out_data
for (uint32_t i = 0; i < entry_count; ++i)
{
entries[i].metadata_offset = (uint32_t)buffer.size;
ikv2_buffer_write_u8(&buffer, entries[i].type);
ikv2_buffer_write_u32le(&buffer, payload_base);
ikv2_buffer_write_u32le(&buffer, entries[i].payload_size);
payload_base += entries[i].payload_size;
ikv2_buffer_write_u32le(&buffer, 0u);
ikv2_buffer_write_u32le(&buffer, 0u);
}
for (uint32_t i = 0; i < entry_count; ++i)
{
const uint8_t *payload = entries[i].payload_data;
ikv2_buffer_write_bytes(&buffer, payload, entries[i].payload_size);
size_t payload_start = buffer.size;
if (!ikv__write_binary_node_append(entries[i].source_node, &buffer.data, &buffer.size, &buffer.capacity))
{
buffer.ok = false;
break;
}
if (payload_start > 0xFFFFFFFFu || buffer.size - payload_start > 0xFFFFFFFFu)
{
buffer.ok = false;
break;
}
entries[i].payload_offset = (uint32_t)payload_start;
entries[i].payload_size = (uint32_t)(buffer.size - payload_start);
}
if (buffer.ok)
{
for (uint32_t i = 0; i < entry_count; ++i)
{
IKV_FREE(entries[i].payload_data);
uint8_t *metadata = buffer.data + entries[i].metadata_offset + 1u;
ikv2_patch_u32le(metadata, entries[i].payload_offset);
ikv2_patch_u32le(metadata + 4u, entries[i].payload_size);
}
}
IKV_FREE(entries);