rle compression for binary blobs

code/common/CMakeLists.txt

@@ -1,2 +1,2 @@
-file(GLOB_RECURSE SRCS *.h *.c)
+file(GLOB_RECURSE SRCS *.h *.c packets/*.c)
 add_library(eco2d-common STATIC ${SRCS})

code/common/packets/compress.c

@@ -0,0 +1,34 @@
+#include "compress.h"
+
+uint32_t compress_rle(void* data, uint32_t size, uint8_t *dest) {
+    if (size < 1) return 0;
+    uint32_t total_size = 0;
+    uint8_t *buf = (uint8_t*)data;
+    uint8_t byte = buf[0];
+    uint16_t occurences = 1;
+    for (uint32_t i = 1; i <= size; i += 1){
+        if (buf[i] != byte || i == size) {
+            *(uint16_t*)dest = occurences; dest += 2;
+            *dest++ = byte;
+            byte = buf[i];
+            occurences = 1;
+            total_size += 3;
+        }
+        else occurences++;
+    }
+    return total_size;
+}
+
+uint32_t decompress_rle(void* data, uint32_t size, uint8_t *dest) {
+    if (size < 1) return 0;
+    uint32_t total_size = 0;
+    uint8_t *buf = (uint8_t*)data;
+    for (uint32_t i = 0; i < size; i += 3){
+        uint16_t len = *(uint16_t*)&buf[i];
+        for (uint16_t j = 0; j < len; j += 1){
+            *dest++ = buf[i+2];
+            total_size++;
+        }
+    }
+    return total_size;
+}

code/common/packets/compress.h

@@ -0,0 +1,5 @@
+#pragma once
+#include "cwpack/cwpack.h"
+
+uint32_t compress_rle(void* data, uint32_t size, uint8_t *dest);
+uint32_t decompress_rle(void* data, uint32_t size, uint8_t *dest);

code/common/packets/packet.c

@@ -1,5 +1,6 @@
 #include "packet.h"
 #include "packet_utils.h"
+#include "compress.h"
 #include "cwpack/cwpack.h"
 
 #define PKT_HEADER_ELEMENTS 2
@@ -12,7 +13,6 @@ pkt_handler pkt_handlers[] = {
 };
 
 uint8_t pkt_buffer[PKT_BUFSIZ];
-uint8_t pkt_pack_buffer[PKT_BUFSIZ];
 
 int32_t pkt_header_encode(pkt_header *table) {
     return 0;
@@ -52,8 +52,11 @@ int32_t pkt_unpack_struct(cw_unpack_context *uc, pkt_desc *desc, void *raw_blob,
                 zpl_memcopy(blob + field->offset, (uint8_t*)&uc->item.as.u64, field->size);
             }break;
             case CWP_ITEM_BIN: {
-                if (uc->item.as.bin.length != field->size) return -1; // bin size mismatch
+                if (uc->item.as.bin.length >= PKT_BUFSIZ) return -1; // bin blob too big
-                zpl_memcopy(blob + field->offset, (uint8_t*)uc->item.as.bin.start, uc->item.as.bin.length);
+                static uint8_t bin_buf[PKT_BUFSIZ] = {0};
+                uint32_t actual_size = decompress_rle(uc->item.as.bin.start, uc->item.as.bin.length, bin_buf);
+                if (actual_size != field->size) return -1; // bin size mismatch
+                zpl_memcopy(blob + field->offset, bin_buf, actual_size);
             }break;
             default: {
                 zpl_printf("[WARN] unsupported pkt field type %lld !\n", field->type); 
@@ -67,11 +70,13 @@ int32_t pkt_unpack_struct(cw_unpack_context *uc, pkt_desc *desc, void *raw_blob,
 
 int32_t pkt_pack_struct(cw_pack_context *pc, pkt_desc *desc, void *raw_blob, uint32_t blob_size) {
     uint8_t *blob = (uint8_t*)raw_blob;
-    zpl_zero_item(pkt_pack_buffer);
     for (pkt_desc *field = desc; field->type != CWP_NOT_AN_ITEM; ++field) {
         switch (field->type) {
             case CWP_ITEM_BIN: {
-                cw_pack_bin(pc, blob + field->offset, field->size);
+                if (field->size >= PKT_BUFSIZ) return -1; // bin blob too big
+                static uint8_t bin_buf[PKT_BUFSIZ] = {0};
+                uint32_t size = compress_rle(blob + field->offset, field->size, bin_buf);
+                cw_pack_bin(pc, bin_buf, size);
             }break;
             case CWP_ITEM_POSITIVE_INTEGER: {
                 uint64_t num;

code/common/packets/packet.h

@@ -29,6 +29,7 @@ int32_t pkt_header_decode(pkt_header *table, void *data, size_t datalen);
 
 extern pkt_handler pkt_handlers[];
 extern uint8_t pkt_buffer[];
-extern uint8_t pkt_pack_buffer[];
 
-#include "packet_list.h"
+// NOTE(zaklaus): packets
+
+#include "pkt_01_welcome.h"

code/common/packets/packet_list.h

@@ -4,18 +4,3 @@
 
 // NOTE(zaklaus): pkt data
 
-#define PKT_01_NUMBERS_SIZ 32
-
-typedef struct {
-    uint32_t chunk_size;
-    uint32_t chunk_amount;
-    uint16_t numbers[PKT_01_NUMBERS_SIZ];
-} pkt_01_welcome;
-
-size_t pkt_01_welcome_encode(pkt_01_welcome *table);
-extern pkt_desc pkt_01_welcome_desc[];
-
-// NOTE(zaklaus): pkt handlers
-
-PKT_HANDLER_PROC(pkt_01_welcome_handler);
-

code/common/packets/pkt_01_welcome.c

@@ -1,10 +1,10 @@
-#include "packet.h"
+#include "pkt_01_welcome.h"
-#include "packet_utils.h"
 
 pkt_desc pkt_01_welcome_desc[] = {
     { PKT_FIELD(CWP_ITEM_POSITIVE_INTEGER, pkt_01_welcome, chunk_size) },
     { PKT_FIELD(CWP_ITEM_POSITIVE_INTEGER, pkt_01_welcome, chunk_amount) },
     { PKT_ARRAY(pkt_01_welcome, numbers) },
+    { PKT_ARRAY(pkt_01_welcome, structs) },
     { PKT_END },
 };
 

code/common/packets/pkt_01_welcome.h

@@ -1,2 +1,17 @@
 #pragma once
+#include "packet_utils.h"
+
+#define PKT_01_NUMBERS_SIZ 32
+
+typedef struct {
+    uint32_t chunk_size;
+    uint32_t chunk_amount;
+    uint16_t numbers[PKT_01_NUMBERS_SIZ];
+    pkt_header structs[64];
+} pkt_01_welcome;
+
+size_t pkt_01_welcome_encode(pkt_01_welcome *table);
+extern pkt_desc pkt_01_welcome_desc[];
+
+PKT_HANDLER_PROC(pkt_01_welcome_handler);