From 2a528e2b0d38cfcf1527c5cfda671387df5fa7e8 Mon Sep 17 00:00:00 2001 From: GigabiteStudios Date: Wed, 17 Jun 2026 11:48:46 -0500 Subject: [PATCH] perf(writer): streamline v2 binary writes and benchmarks --- demo/unit_test.c | 408 +++++++++++++++++++++++++++++------- src/ikv.c | 27 +++ src/internal/ikv_internal.h | 1 + src/loaders/ikv2.c | 75 ++++--- 4 files changed, 404 insertions(+), 107 deletions(-) diff --git a/demo/unit_test.c b/demo/unit_test.c index 4194b79..a9d21e7 100644 --- a/demo/unit_test.c +++ b/demo/unit_test.c @@ -1,6 +1,7 @@ #include "ikv.h" #include +#include #include #include #include @@ -22,7 +23,7 @@ typedef struct { char message[256]; - char benchmark_summary[256]; + char benchmark_summary[4096]; } test_context_t; typedef int (*test_fn_t)(test_context_t *context); @@ -87,11 +88,13 @@ static double benchmark_now_us(void) #endif } -static ikv_node_t *create_benchmark_root(void) +static ikv_node_t *create_benchmark_root(unsigned int scale) { ikv_node_t *root = ikv_create_object("benchmark"); ikv_node_t *inventory = NULL; ikv_node_t *stats = NULL; + unsigned int item_count = 0u; + unsigned int stat_count = 0u; if (!root) return NULL; @@ -109,16 +112,44 @@ static ikv_node_t *create_benchmark_root(void) return NULL; } - ikv_array_add_string(inventory, "wrench"); - ikv_array_add_string(inventory, "battery"); - ikv_array_add_string(inventory, "map"); - ikv_array_add_string(inventory, "radio"); - ikv_array_add_string(inventory, "flares"); + item_count = scale * 8u; + if (item_count < 5u) + item_count = 5u; + for (unsigned int i = 0; i < item_count; ++i) + { + char item_name[64]; + snprintf(item_name, sizeof(item_name), "item-%u-payload-%u", scale, i); + ikv_array_add_string(inventory, item_name); + } + + stat_count = scale * 16u; + if (stat_count < 4u) + stat_count = 4u; + for (unsigned int i = 0; i < stat_count; ++i) + { + char key[64]; + snprintf(key, sizeof(key), "metric_%u", i); + switch (i % 4u) + { + case 0u: + ikv_object_set_int(stats, key, (int64_t)(scale * 100u + i)); + break; + case 1u: + ikv_object_set_float(stats, key, (double)scale * 0.5 + (double)i * 0.25); + break; + case 2u: + ikv_object_set_bool(stats, key, (i & 1u) != 0u); + break; + default: + { + char value[64]; + snprintf(value, sizeof(value), "value-%u-%u", scale, i); + ikv_object_set_string(stats, key, value); + break; + } + } + } - ikv_object_set_int(stats, "health", 100); - ikv_object_set_int(stats, "armor", 75); - ikv_object_set_float(stats, "speed", 12.75); - ikv_object_set_bool(stats, "indoors", false); return root; } @@ -1524,89 +1555,316 @@ static int test_hook_alloc_fail_internal_v2_memory_copy(test_context_t *context) } #endif -static int test_binary_v2_file_benchmark(test_context_t *context) +typedef struct { - enum { benchmark_iterations = 1000 }; - const char *path = "demo_benchmark_v2.ikvb"; - ikv_node_t *root = NULL; - double total_index_us = 0.0; - double total_read_us = 0.0; - double total_write_us = 0.0; - int result = 0; + unsigned int scale; + unsigned int iterations; + double avg_index_us; + double avg_read_us; + double avg_write_us; +} benchmark_sample_t; - cleanup_file(path); - root = create_benchmark_root(); - if (!root) - return fail(context, "failed to create benchmark root"); +typedef enum +{ + BENCHMARK_FORMAT_TEXT_V1 = 0, + BENCHMARK_FORMAT_TEXT_V2, + BENCHMARK_FORMAT_BINARY_V1, + BENCHMARK_FORMAT_BINARY_V2 +} benchmark_format_t; - if (!ikvb_write_file(path, root)) - result = fail(context, "failed to prepare benchmark file"); +static const char *benchmark_big_o_label(double exponent) +{ + if (exponent < 0.15) + return "O(1)"; + if (exponent < 0.50) + return "O(log n)"; + if (exponent < 1.20) + return "O(n)"; + if (exponent < 1.60) + return "O(n log n)"; + if (exponent < 2.40) + return "O(n^2)"; + return "O(n^k)"; +} - for (unsigned int i = 0; result == 0 && i < benchmark_iterations; ++i) +static double benchmark_fit_exponent(const benchmark_sample_t *samples, size_t count, int metric) +{ + double sum_x = 0.0; + double sum_y = 0.0; + double sum_xx = 0.0; + double sum_xy = 0.0; + size_t used = 0u; + + for (size_t i = 0; i < count; ++i) { - double start_time = benchmark_now_us(); - ikv_node_t *loaded = ikv_parse_file(path); - double end_time = benchmark_now_us(); + double y = 0.0; + double x = 0.0; - if (!loaded) - result = fail(context, "benchmark index parse failed"); - else - total_index_us += end_time - start_time; + if (!samples || samples[i].scale == 0u) + continue; - ikv_free(loaded); - } - - for (unsigned int i = 0; result == 0 && i < benchmark_iterations; ++i) - { - ikv_node_t *loaded = ikv_parse_file(path); - ikv_node_t *inventory = NULL; - double start_time = 0.0; - double end_time = 0.0; - - if (!loaded) + switch (metric) { - result = fail(context, "benchmark read parse failed"); + case 0: + y = samples[i].avg_index_us; + break; + case 1: + y = samples[i].avg_read_us; + break; + default: + y = samples[i].avg_write_us; break; } - start_time = benchmark_now_us(); - inventory = ikv_object_get(loaded, "inventory"); - end_time = benchmark_now_us(); + if (y <= 0.0) + continue; - if ((result = expect_true(context, inventory != NULL, "benchmark read inventory lookup failed")) == 0 && - (result = expect_true(context, ikv_array_size(inventory) == 5u, "benchmark read inventory size mismatch")) == 0) - { - total_read_us += end_time - start_time; - } - - ikv_free(loaded); + x = log((double)samples[i].scale); + y = log(y); + sum_x += x; + sum_y += y; + sum_xx += x * x; + sum_xy += x * y; + ++used; } - for (unsigned int i = 0; result == 0 && i < benchmark_iterations; ++i) - { - double start_time = benchmark_now_us(); - bool ok = ikvb_write_file(path, root); - double end_time = benchmark_now_us(); + if (used < 2u) + return 0.0; - if (!ok) - result = fail(context, "benchmark write failed"); - else - total_write_us += end_time - start_time; + if (fabs((double)used * sum_xx - sum_x * sum_x) < 1e-9) + return 0.0; + + return (((double)used * sum_xy) - (sum_x * sum_y)) / (((double)used * sum_xx) - (sum_x * sum_x)); +} + +static const char *benchmark_format_name(benchmark_format_t format) +{ + switch (format) + { + case BENCHMARK_FORMAT_TEXT_V1: + return "v1-text"; + case BENCHMARK_FORMAT_TEXT_V2: + return "v2-text"; + case BENCHMARK_FORMAT_BINARY_V1: + return "v1-bin"; + case BENCHMARK_FORMAT_BINARY_V2: + return "v2-bin"; + default: + return "unknown"; + } +} + +static const char *benchmark_format_path(benchmark_format_t format) +{ + switch (format) + { + case BENCHMARK_FORMAT_TEXT_V1: + return "demo_benchmark_v1.ikv"; + case BENCHMARK_FORMAT_TEXT_V2: + return "demo_benchmark_v2.ikv"; + case BENCHMARK_FORMAT_BINARY_V1: + return "demo_benchmark_v1.ikvb"; + case BENCHMARK_FORMAT_BINARY_V2: + return "demo_benchmark_v2.ikvb"; + default: + return "demo_benchmark.ikv"; + } +} + +static bool benchmark_write_file(benchmark_format_t format, const char *path, const ikv_node_t *root) +{ + switch (format) + { + case BENCHMARK_FORMAT_TEXT_V1: + return ikv_write_file_version(path, root, IKV_VERSION_1); + case BENCHMARK_FORMAT_TEXT_V2: + return ikv_write_file_version(path, root, IKV_VERSION_2); + case BENCHMARK_FORMAT_BINARY_V1: + return ikvb_write_file_version(path, root, IKV_VERSION_1); + case BENCHMARK_FORMAT_BINARY_V2: + return ikvb_write_file_version(path, root, IKV_VERSION_2); + default: + return false; + } +} + +static ikv_node_t *benchmark_parse_file(benchmark_format_t format, const char *path) +{ + switch (format) + { + case BENCHMARK_FORMAT_TEXT_V1: + return ikv_parse_file_version(path, IKV_VERSION_1); + case BENCHMARK_FORMAT_TEXT_V2: + return ikv_parse_file_version(path, IKV_VERSION_2); + case BENCHMARK_FORMAT_BINARY_V1: + return ikvb_parse_file_version(path, IKV_VERSION_1); + case BENCHMARK_FORMAT_BINARY_V2: + return ikvb_parse_file_version(path, IKV_VERSION_2); + default: + return NULL; + } +} + +static int test_binary_v2_file_benchmark(test_context_t *context) +{ + static const unsigned int benchmark_scales[] = {1u, 4u, 16u, 64u}; + static const benchmark_format_t benchmark_formats[] = { + BENCHMARK_FORMAT_TEXT_V1, + BENCHMARK_FORMAT_TEXT_V2, + BENCHMARK_FORMAT_BINARY_V1, + BENCHMARK_FORMAT_BINARY_V2 + }; + enum { benchmark_scale_count = 4, benchmark_iterations_small = 1000, benchmark_iterations_medium = 400, benchmark_iterations_large = 150 }; + benchmark_sample_t samples[4][benchmark_scale_count]; + size_t summary_offset = 0u; + int result = 0; + + memset(samples, 0, sizeof(samples)); + context->benchmark_summary[0] = 0; + + for (unsigned int format_index = 0; result == 0 && format_index < 4u; ++format_index) + { + benchmark_format_t format = benchmark_formats[format_index]; + const char *path = benchmark_format_path(format); + + cleanup_file(path); + + for (unsigned int sample_index = 0; result == 0 && sample_index < benchmark_scale_count; ++sample_index) + { + unsigned int scale = benchmark_scales[sample_index]; + unsigned int iterations = (scale <= 4u) ? benchmark_iterations_small : ((scale <= 16u) ? benchmark_iterations_medium : benchmark_iterations_large); + unsigned int expected_inventory_size = (scale * 8u < 5u) ? 5u : (scale * 8u); + ikv_node_t *root = create_benchmark_root(scale); + double total_index_us = 0.0; + double total_read_us = 0.0; + double total_write_us = 0.0; + + if (!root) + { + result = fail(context, "failed to create benchmark root"); + break; + } + + if (!benchmark_write_file(format, path, root)) + { + ikv_free(root); + result = fail(context, "failed to prepare benchmark file"); + break; + } + + for (unsigned int i = 0; result == 0 && i < iterations; ++i) + { + double start_time = benchmark_now_us(); + ikv_node_t *loaded = benchmark_parse_file(format, path); + double end_time = benchmark_now_us(); + + if (!loaded) + result = fail(context, "benchmark index parse failed"); + else + total_index_us += end_time - start_time; + + ikv_free(loaded); + } + + for (unsigned int i = 0; result == 0 && i < iterations; ++i) + { + ikv_node_t *loaded = benchmark_parse_file(format, path); + ikv_node_t *inventory = NULL; + double start_time = 0.0; + double end_time = 0.0; + + if (!loaded) + { + result = fail(context, "benchmark read parse failed"); + break; + } + + start_time = benchmark_now_us(); + inventory = ikv_object_get(loaded, "inventory"); + end_time = benchmark_now_us(); + + if ((result = expect_true(context, inventory != NULL, "benchmark read inventory lookup failed")) == 0 && + (result = expect_true(context, ikv_array_size(inventory) == expected_inventory_size, "benchmark read inventory size mismatch")) == 0) + { + total_read_us += end_time - start_time; + } + + ikv_free(loaded); + } + + for (unsigned int i = 0; result == 0 && i < iterations; ++i) + { + double start_time = benchmark_now_us(); + bool ok = benchmark_write_file(format, path, root); + double end_time = benchmark_now_us(); + + if (!ok) + result = fail(context, "benchmark write failed"); + else + total_write_us += end_time - start_time; + } + + if (result == 0) + { + samples[format_index][sample_index].scale = scale; + samples[format_index][sample_index].iterations = iterations; + samples[format_index][sample_index].avg_index_us = total_index_us / (double)iterations; + samples[format_index][sample_index].avg_read_us = total_read_us / (double)iterations; + samples[format_index][sample_index].avg_write_us = total_write_us / (double)iterations; + } + + ikv_free(root); + cleanup_file(path); + } } if (result == 0) { - snprintf(context->benchmark_summary, - sizeof(context->benchmark_summary), - "avg index %.2fus avg read %.2fus avg write %.2fus over %u iterations", - total_index_us / (double)benchmark_iterations, - total_read_us / (double)benchmark_iterations, - total_write_us / (double)benchmark_iterations, - (unsigned int)benchmark_iterations); - } + for (unsigned int format_index = 0; format_index < 4u; ++format_index) + { + const benchmark_sample_t *format_samples = samples[format_index]; + double index_exp = benchmark_fit_exponent(format_samples, benchmark_scale_count, 0); + double read_exp = benchmark_fit_exponent(format_samples, benchmark_scale_count, 1); + double write_exp = benchmark_fit_exponent(format_samples, benchmark_scale_count, 2); + int wrote = snprintf( + context->benchmark_summary + summary_offset, + sizeof(context->benchmark_summary) - summary_offset, + "%s index %s^%.2f [n=%u %.2fus, n=%u %.2fus, n=%u %.2fus, n=%u %.2fus] " + "read %s^%.2f [n=%u %.2fus, n=%u %.2fus, n=%u %.2fus, n=%u %.2fus] " + "write %s^%.2f [n=%u %.2fus, n=%u %.2fus, n=%u %.2fus, n=%u %.2fus]%s", + benchmark_format_name(benchmark_formats[format_index]), + benchmark_big_o_label(index_exp), index_exp, + format_samples[0].scale, format_samples[0].avg_index_us, + format_samples[1].scale, format_samples[1].avg_index_us, + format_samples[2].scale, format_samples[2].avg_index_us, + format_samples[3].scale, format_samples[3].avg_index_us, + benchmark_big_o_label(read_exp), read_exp, + format_samples[0].scale, format_samples[0].avg_read_us, + format_samples[1].scale, format_samples[1].avg_read_us, + format_samples[2].scale, format_samples[2].avg_read_us, + format_samples[3].scale, format_samples[3].avg_read_us, + benchmark_big_o_label(write_exp), write_exp, + format_samples[0].scale, format_samples[0].avg_write_us, + format_samples[1].scale, format_samples[1].avg_write_us, + format_samples[2].scale, format_samples[2].avg_write_us, + format_samples[3].scale, format_samples[3].avg_write_us, + (format_index + 1u < 4u) ? "\n" : ""); - cleanup_file(path); - ikv_free(root); + if (wrote < 0) + { + result = fail(context, "failed to format benchmark summary"); + break; + } + + if ((size_t)wrote >= sizeof(context->benchmark_summary) - summary_offset) + { + summary_offset = sizeof(context->benchmark_summary) - 1u; + break; + } + + summary_offset += (size_t)wrote; + } + } return result; } @@ -1711,7 +1969,7 @@ static void log_case(bool passed, unsigned int index, unsigned int total, long e putchar('\n'); } -static char benchmark_summary[256]; +static char benchmark_summary[4096]; static int run_test_case(unsigned int index, unsigned int total) { diff --git a/src/ikv.c b/src/ikv.c index 8c47e37..7bb3628 100644 --- a/src/ikv.c +++ b/src/ikv.c @@ -1754,6 +1754,33 @@ bool ikv__write_binary_node_memory(const ikv_node_t *node, uint8_t **out_data, u return true; } +bool ikv__write_binary_node_append(const ikv_node_t *node, uint8_t **data, size_t *size, size_t *capacity) +{ + bw_mem_t mem = {0}; + bw_t w = {0}; + size_t start_size = 0u; + + if (!node || !data || !size || !capacity) + return false; + + mem.data = *data; + mem.size = *size; + mem.cap = *capacity; + start_size = mem.size; + w.ctx = &mem; + w.ok = true; + w.write_bytes_fn = bw_mem_write_bytes; + bw_node(&w, node); + + if (!w.ok || mem.size == start_size || mem.size > 0xFFFFFFFFu) + return false; + + *data = mem.data; + *size = mem.size; + *capacity = mem.cap; + return true; +} + bool ikv__write_binary_file_version(const char *path, const ikv_node_t *root, uint32_t version) { if (!path || !root) diff --git a/src/internal/ikv_internal.h b/src/internal/ikv_internal.h index 2796fc8..8478962 100644 --- a/src/internal/ikv_internal.h +++ b/src/internal/ikv_internal.h @@ -89,6 +89,7 @@ bool ikv__read_file_bytes(const char *path, uint8_t **out_data, size_t *out_size bool ikv__write_binary_file_version(const char *path, const ikv_node_t *root, uint32_t version); bool ikv__write_binary_memory_version(const ikv_node_t *root, uint8_t **out_data, uint32_t *out_size, uint32_t version); +bool ikv__write_binary_node_append(const ikv_node_t *node, uint8_t **data, size_t *size, size_t *capacity); ikv_node_t *ikv__parse_binary_file_version(const char *path, uint32_t version); ikv_node_t *ikv__parse_binary_memory_version(const void *data, size_t size, uint32_t version); diff --git a/src/loaders/ikv2.c b/src/loaders/ikv2.c index eaa6996..09f6ff1 100644 --- a/src/loaders/ikv2.c +++ b/src/loaders/ikv2.c @@ -14,9 +14,10 @@ typedef struct uint32_t key_length; uint32_t key_hash; uint8_t type; + const ikv_node_t *source_node; + uint32_t metadata_offset; uint32_t payload_offset; uint32_t payload_size; - uint8_t *payload_data; uint32_t next_in_bucket; } ikv2_index_entry_t; @@ -162,6 +163,16 @@ static void ikv2_buffer_write_u32le(ikv2_buffer_t *buffer, uint32_t value) ikv2_buffer_write_bytes(buffer, bytes, sizeof(bytes)); } +static void ikv2_patch_u32le(uint8_t *data, uint32_t value) +{ + if (!data) + return; + data[0] = (uint8_t)(value & 0xFFu); + data[1] = (uint8_t)((value >> 8) & 0xFFu); + data[2] = (uint8_t)((value >> 16) & 0xFFu); + data[3] = (uint8_t)((value >> 24) & 0xFFu); +} + static void ikv2_buffer_write_varu32(ikv2_buffer_t *buffer, uint32_t value) { while (value >= 0x80u) @@ -273,17 +284,8 @@ static bool ikv2_collect_root_entries(const ikv_node_t *root, ikv2_index_entry_t { for (ikv_node_t *node = root->value.object.buckets[bucket]; node; node = node->next) { - uint8_t *payload = NULL; - uint32_t payload_size = 0u; - - if (index >= count || !ikv__write_binary_node_memory(node, &payload, &payload_size)) + if (index >= count) { - if (payload) - IKV_FREE(payload); - for (uint32_t i = 0; i < index; ++i) - { - IKV_FREE(entries[i].payload_data); - } IKV_FREE(entries); return false; } @@ -292,8 +294,7 @@ static bool ikv2_collect_root_entries(const ikv_node_t *root, ikv2_index_entry_t entries[index].key_length = (uint32_t)strlen(entries[index].key); entries[index].key_hash = ikv2_hash_key(entries[index].key); entries[index].type = (uint8_t)node->type; - entries[index].payload_data = payload; - entries[index].payload_size = payload_size; + entries[index].source_node = node; entries[index].next_in_bucket = IKV2_INDEX_NONE; ++index; @@ -311,8 +312,6 @@ static bool ikv2_build_indexed_binary(const ikv_node_t *root, uint8_t **out_data ikv2_index_entry_t *entries = NULL; uint32_t entry_count = 0u; uint32_t header_size = 0u; - uint32_t payload_base = 0u; - uint32_t total_size = 0u; const char *root_key = NULL; uint32_t root_key_length = 0u; ikv2_buffer_t buffer = {0}; @@ -335,17 +334,10 @@ static bool ikv2_build_indexed_binary(const ikv_node_t *root, uint8_t **out_data for (uint32_t i = 0; i < entry_count; ++i) header_size += ikv2_varu32_size(entries[i].key_length) + entries[i].key_length; header_size += entry_count * (1u + 4u + 4u); - payload_base = header_size; - total_size = header_size; - for (uint32_t i = 0; i < entry_count; ++i) - total_size += entries[i].payload_size; - - buffer.data = total_size ? (uint8_t *)IKV_MALLOC(total_size) : NULL; - buffer.capacity = total_size; - if (total_size > 0u && !buffer.data) + buffer.capacity = header_size; + buffer.data = header_size ? (uint8_t *)IKV_MALLOC(header_size) : NULL; + if (header_size > 0u && !buffer.data) { - for (uint32_t i = 0; i < entry_count; ++i) - IKV_FREE(entries[i].payload_data); IKV_FREE(entries); return false; } @@ -362,21 +354,40 @@ static bool ikv2_build_indexed_binary(const ikv_node_t *root, uint8_t **out_data for (uint32_t i = 0; i < entry_count; ++i) { + entries[i].metadata_offset = (uint32_t)buffer.size; ikv2_buffer_write_u8(&buffer, entries[i].type); - ikv2_buffer_write_u32le(&buffer, payload_base); - ikv2_buffer_write_u32le(&buffer, entries[i].payload_size); - payload_base += entries[i].payload_size; + ikv2_buffer_write_u32le(&buffer, 0u); + ikv2_buffer_write_u32le(&buffer, 0u); } for (uint32_t i = 0; i < entry_count; ++i) { - const uint8_t *payload = entries[i].payload_data; - ikv2_buffer_write_bytes(&buffer, payload, entries[i].payload_size); + size_t payload_start = buffer.size; + + if (!ikv__write_binary_node_append(entries[i].source_node, &buffer.data, &buffer.size, &buffer.capacity)) + { + buffer.ok = false; + break; + } + + if (payload_start > 0xFFFFFFFFu || buffer.size - payload_start > 0xFFFFFFFFu) + { + buffer.ok = false; + break; + } + + entries[i].payload_offset = (uint32_t)payload_start; + entries[i].payload_size = (uint32_t)(buffer.size - payload_start); } - for (uint32_t i = 0; i < entry_count; ++i) + if (buffer.ok) { - IKV_FREE(entries[i].payload_data); + for (uint32_t i = 0; i < entry_count; ++i) + { + uint8_t *metadata = buffer.data + entries[i].metadata_offset + 1u; + ikv2_patch_u32le(metadata, entries[i].payload_offset); + ikv2_patch_u32le(metadata + 4u, entries[i].payload_size); + } } IKV_FREE(entries);