// ----------------------------------------------------------------------------- // file #if 0 // ifdef _WIN32 #include #if is(tcc) #define CP_UTF8 65001 int WINAPI MultiByteToWideChar(); int WINAPI WideCharToMultiByte(); #endif // widen() ? string1252() ? string16() ? stringw() ? wchar_t *widen(const char *utf8) { // wide strings (win only) int chars = MultiByteToWideChar(CP_UTF8, 0, utf8, -1, NULL, 0); char *buf = va("%.*s", (int)(chars * sizeof(wchar_t)), ""); return MultiByteToWideChar(CP_UTF8, 0, utf8, -1, (void*)buf, chars), (wchar_t *)buf; } #define open8(path,mode) ifdef(win, _wopen(widen(path)) , open(path, mode) ) #define fopen8(path,mode) ifdef(win, _wfopen(widen(path),widen(mode)) , fopen(path,mode) ) #define remove8(path) ifdef(win, _wremove(widen(path)) , remove(path) ) #define rename8(path) ifdef(win, _wrename(widen(path)) , rename(path) ) #define stat8(path,st) ifdef(win, _wstat(widen(path),st) , stat(path,st) ) // _stati64() #define stat8_t ifdef(win, _stat , stat_t ) // struct _stati64 #endif char *file_name(const char *pathfile) { char *s = strrchr(pathfile, '/'), *t = strrchr(pathfile, '\\'); return va("%s", s > t ? s+1 : t ? t+1 : pathfile); } char *file_base(const char *pathfile) { char *s = file_name(pathfile); char *e = file_ext(pathfile); return s[ strlen(s) - strlen(e) ] = '\0', s; } char *file_pathabs( const char *pathfile ) { char *out = va("%*.s", DIR_MAX+1, ""); #if is(win32) _fullpath(out, pathfile, DIR_MAX); #else realpath(pathfile, out); #endif return out; } char *file_path(const char *pathfile) { return va("%.*s", (int)(strlen(pathfile)-strlen(file_name(pathfile))), pathfile); } char *file_load(const char *filename, int *len) { // @todo: 32 counters/thread enough? static __thread array(char) buffer[32] = {0}, *invalid[1]; static __thread unsigned i = 0; i = (i+1) % 32; FILE *fp = filename[0] ? fopen(filename, "rb") : NULL; if( fp ) { fseek(fp, 0L, SEEK_END); size_t sz = ftell(fp); fseek(fp, 0L, SEEK_SET); array_resize(buffer[i], sz+1); sz *= fread(buffer[i],sz,1,fp) == 1; buffer[i][sz] = 0; if(len) *len = (int)sz; fclose(fp); return buffer[i]; // @fixme: return 0 on error instead? } if (len) *len = 0; return 0; } char *file_read(const char *filename) { // @todo: fix leaks return file_load(filename, NULL); } bool file_write(const char *name, const void *ptr, int len) { bool ok = 0; for( FILE *fp = name && ptr && len >= 0 ? fopen(name, "wb") : NULL; fp; fclose(fp), fp = 0) { ok = fwrite(ptr, len,1, fp) == 1; } return ok; } bool file_append(const char *name, const void *ptr, int len) { bool ok = 0; for( FILE *fp = name && ptr && len >= 0 ? fopen(name, "a+b") : NULL; fp; fclose(fp), fp = 0) { ok = fwrite(ptr, len,1, fp) == 1; } return ok; } static // not exposed bool file_stat(const char *fname, struct stat *st) { // remove ending slashes. win32+tcc does not like them. int l = strlen(fname), m = l; while( l && (fname[l-1] == '/' || fname[l-1] == '\\') ) --l; fname = l == m ? fname : va("%.*s", l, fname); return stat(fname, st) >= 0; } uint64_t file_stamp(const char *fname) { struct stat st; return !file_stat(fname, &st) ? 0ULL : st.st_mtime; } uint64_t file_stamp10(const char *fname) { time_t mtime = (time_t)file_stamp(fname); struct tm *ti = localtime(&mtime); return atoi64(va("%04d%02d%02d%02d%02d%02d",ti->tm_year+1900,ti->tm_mon+1,ti->tm_mday,ti->tm_hour,ti->tm_min,ti->tm_sec)); } uint64_t file_size(const char *fname) { struct stat st; return !file_stat(fname, &st) ? 0ULL : st.st_size; } bool file_directory( const char *pathfile ) { struct stat st; return !file_stat(pathfile, &st) ? 0 : S_IFDIR == ( st.st_mode & S_IFMT ); } bool file_exist(const char *fname) { struct stat st; return !file_stat(fname, &st) ? false : true; } char *file_normalize(const char *name) { char *copy = va("%s", name), *s = copy, c; #if is(win32) for( int i = 0; copy[i]; ++i ) { if(copy[i] == '/') copy[i] = '\\'; if(copy[i] == '\'') copy[i] = '\"'; } #else for( int i = 0; copy[i]; ++i ) { if(copy[i] == '\\') copy[i] = '/'; if(copy[i] == '\"') copy[i] = '\''; } #endif return copy; } #if 0 char *file_normalize(const char *name) { char *copy = va("%s", name), *s = copy, c; // lowercases+digits+underscores+slashes only. anything else is truncated. for( ; *name ; ++name ) { /**/ if( *name >= 'a' && *name <= 'z' ) *s++ = *name; else if( *name >= 'A' && *name <= 'Z' ) *s++ = *name - 'A' + 'a'; else if( *name >= '0' && *name <= '9' ) *s++ = *name; else if( *name == '/' || *name == '\\') *s++ = '/'; else if( *name <= ' ' || *name == '.' ) *s++ = '_'; } *s++ = 0; // remove dupe slashes for( name = s = copy, c = '/'; *name ; ) { while( *name && *name != c ) *s++ = *name++; if( *name ) *s++ = c; while( *name && *name == c ) name++; } *s++ = 0; // remove dupe underlines for( name = s = copy, c = '_'; *name ; ) { while( *name && *name != c ) *s++ = *name++; if( *name ) *s++ = c; while( *name && *name == c ) name++; } *s++ = 0; return copy; } char *file_normalize_with_folder(const char *name) { char *s = file_normalize(name); char *slash = strrchr(s, '/'); if(slash) *slash = 0; char *penultimate = strrchr(s, '/'); if(slash) *slash = '/'; return penultimate ? penultimate+1 : /*slash ? slash+1 :*/ s; } #endif char *file_ext(const char *name) { char *b = file_name(name), *s = strchr(b, '.'); //strrchr(name, '.'); return va("%s", s ? s : "" ); // , name ); } char *file_id(const char *pathfile) { char *dir = file_path(pathfile); for(int i=0;dir[i];++i) dir[i]=tolower(dir[i]); char *base = file_name(pathfile); for(int i=0;base[i];++i) base[i]=tolower(base[i]); #if 0 // extensionless, larry.mid and larry.txt will collide, diffuse.tga and diffuse.png will match char *ext = strchr(base, '.'); if (ext) ext[0] = '\0'; // remove all extensions #else // extensionless for audio/images only (materials: diffuse.tga and diffuse.png will match) char *ext = strrchr(base, '.'); //if (ext) ext[0] = '\0'; // remove all extensions if(ext) if( strstr(".jpg.png.bmp.tga"".", ext) || strstr(".ogg.mp3.wav.mod.xm.flac"".", ext) || strstr(".mp4.ogv.avi.mkv.wmv.mpg.mpeg"".", ext) ) { ext = strchr(base, '.'); ext[0] = '\0'; //strcpy(ext, "_xxx"); } #endif // if (!dir[0]) return base; char *stem = va("%s/%s", dir, base); // file_name(dir); // /path2/path1/file2_file1 -> file1_file2/path1/path2 int len = 0; int ids_count = 0; char ids[64][64] = { 0 }; // split path stems for each_substring(stem, "/\\@", key) { int tokens_count = 0; char* tokens[64] = { 0 }; // split tokens for each_substring(key, "[]()_ ", it) { tokens[tokens_count++] = va("%s", it); } // sort alphabetically if( tokens_count > 1 ) qsort(tokens, tokens_count, sizeof(char *), strcmp_qsort); // concat sorted token1_token2_... char built[256]; *built = 0; for( int i = 0; i < tokens_count; ++i ) { strlcat(built, "_", 256); strlcat(built, tokens[i], 256); } strncpy( ids[ ids_count ], &built[1], 64 ); len += strlen( ids[ ids_count++ ] ); } // concat in inverse order: file/path1/path2/... char buffer[DIR_MAX]; buffer[0] = 0; for( int it = ids_count; --it >= 0; ) { strcat(buffer, ids[it]); strcat(buffer, "/"); } return va("%s", buffer); } array(char*) file_list(const char *pathmasks) { static __thread array(char*) list = 0; // @fixme: add 16 slots for( int i = 0; i < array_count(list); ++i ) { FREE(list[i]); } array_resize(list, 0); for each_substring(pathmasks,";",pathmask) { char *cwd = 0, *masks = 0; char *slash = strrchr(pathmask, '/'); if( !slash ) cwd = "./", masks = pathmask; else { masks = va("%s", slash+1); cwd = pathmask, slash[1] = '\0'; } if( !masks[0] ) masks = "*"; ASSERT(strend(cwd, "/"), "Error: dirs like '%s' must end with slash", cwd); dir *d = dir_open(cwd, strstr(masks,"**") ? "r" : ""); if( d ) { for( int i = 0; i < dir_count(d); ++i ) { if( dir_file(d,i) ) { // dir_name() should return full normalized paths "C:/prj/v4k/demos/art/fx/fxBloom.fs". should exclude system dirs as well char *entry = dir_name(d,i); char *fname = file_name(entry); int allowed = 0; for each_substring(masks,";",mask) { allowed |= strmatch(fname, mask); } if( !allowed ) continue; // if( strstr(fname, "/.") ) continue; // @fixme: still needed? useful? // insert copy char *copy = STRDUP(entry); array_push(list, copy); } } dir_close(d); } } array_sort(list, strcmp); return list; } bool file_move(const char *src, const char *dst) { bool ok = file_exist(src) && !file_exist(dst) && 0 == rename(src, dst); return ok; } bool file_delete(const char *pathfile) { if( file_exist(pathfile) ) { for( int i = 0; i < 10; ++i ) { bool ok = 0 == unlink(pathfile); if( ok ) return true; sleep_ms(10); } return false; } return true; } bool file_copy(const char *src, const char *dst) { int ok = 0, BUFSIZE = 1 << 20; // 1 MiB static __thread char *buffer = 0; do_once buffer = REALLOC(0, BUFSIZE); // @leak for( FILE *in = fopen(src, "rb"); in; fclose(in), in = 0) { for( FILE *out = fopen(dst, "wb"); out; fclose(out), out = 0, ok = 1) { for( int n; !!(n = fread( buffer, 1, BUFSIZE, in )); ){ if(fwrite( buffer, 1, n, out ) != n) return fclose(in), fclose(out), false; } } } return ok; } char* file_tempname() { static __thread int id; return va("%s/v4k-temp.%s.%p.%d", app_temp(), getenv(ifdef(win32, "username", "USER")), &id, rand()); } FILE *file_temp(void) { const char *fname = file_tempname(); FILE *fp = fopen(fname, "w+b"); if( fp ) unlink(fname); return fp; } char *file_counter(const char *name) { static __thread char outfile[DIR_MAX], init = 0; static __thread map(char*, int) ext_counters; if(!init) map_init(ext_counters, less_str, hash_str), init = '\1'; char *base = va("%s",name), *ext = file_ext(name); if(ext && ext[0]) *strstr(base, ext) = '\0'; int *counter = map_find_or_add(ext_counters, ext, 0); while( *counter >= 0 ) { *counter = *counter + 1; sprintf(outfile, "%s(%03d)%s", base, *counter, ext); if( !file_exist(outfile) ) { return va("%s", outfile); } } return 0; } enum { MD5_HASHLEN = 16 }; enum { SHA1_HASHLEN = 20 }; enum { CRC32_HASHLEN = 4 }; void* file_sha1(const char *file) { // 20bytes hash_state hs = {0}; sha1_init(&hs); for( FILE *fp = fopen(file, "rb"); fp; fclose(fp), fp = 0) { char buf[8192]; for( int inlen; (inlen = fread(buf, 1, sizeof(buf), fp)) > 0; ) { sha1_process(&hs, (const unsigned char *)buf, inlen); } } unsigned char *hash = va("%.*s", SHA1_HASHLEN, ""); sha1_done(&hs, hash); return hash; } void* file_md5(const char *file) { // 16bytes hash_state hs = {0}; md5_init(&hs); for( FILE *fp = fopen(file, "rb"); fp; fclose(fp), fp = 0) { char buf[8192]; for( int inlen; (inlen = fread(buf, 1, sizeof(buf), fp)) > 0; ) { md5_process(&hs, (const unsigned char *)buf, inlen); } } unsigned char *hash = va("%.*s", MD5_HASHLEN, ""); md5_done(&hs, hash); return hash; } void* file_crc32(const char *file) { // 4bytes unsigned crc = 0; for( FILE *fp = fopen(file, "rb"); fp; fclose(fp), fp = 0) { char buf[8192]; for( int inlen; (inlen = fread(buf, 1, sizeof(buf), fp)) > 0; ) { crc = zip__crc32(crc, buf, inlen); // unsigned int stbiw__crc32(unsigned char *buffer, int len) } } unsigned char *hash = va("%.*s", (int)sizeof(crc), ""); memcpy(hash, &crc, sizeof(crc)); return hash; } #if 0 void* crc32_mem(const void *ptr, int inlen) { // 4bytes unsigned hash = 0; hash = zip__crc32(hash, ptr, inlen); // unsigned int stbiw__crc32(unsigned char *buffer, int len) return hash; } void* md5_mem(const void *ptr, int inlen) { // 16bytes hash_state hs = {0}; md5_init(&hs); md5_process(&hs, (const unsigned char *)ptr, inlen); unsigned char *hash = va("%.*s", MD5_HASHLEN, ""); md5_done(&hs, hash); return hash; } void* sha1_mem(const void *ptr, int inlen) { // 20bytes hash_state hs = {0}; sha1_init(&hs); sha1_process(&hs, (const unsigned char *)ptr, inlen); unsigned char *hash = va("%.*s", SHA1_HASHLEN, ""); sha1_done(&hs, hash); return hash; } unsigned crc32_mem(unsigned h, const void *ptr_, unsigned len) { // based on public domain code by Karl Malbrain const uint8_t *ptr = (const uint8_t *)ptr_; if (!ptr) return 0; const unsigned tbl[16] = { 0x00000000, 0x1db71064, 0x3b6e20c8, 0x26d930ac, 0x76dc4190, 0x6b6b51f4, 0x4db26158, 0x5005713c, 0xedb88320, 0xf00f9344, 0xd6d6a3e8, 0xcb61b38c, 0x9b64c2b0, 0x86d3d2d4, 0xa00ae278, 0xbdbdf21c }; for(h = ~h; len--; ) { uint8_t b = *ptr++; h = (h >> 4) ^ tbl[(h & 15) ^ (b & 15)]; h = (h >> 4) ^ tbl[(h & 15) ^ (b >> 4)]; } return ~h; } uint64_t crc64_mem(uint64_t h, const void *ptr, uint64_t len) { // based on public domain code by Lasse Collin // also, use poly64 0xC96C5795D7870F42 for crc64-ecma static uint64_t crc64_table[256]; static uint64_t poly64 = UINT64_C(0x95AC9329AC4BC9B5); if( poly64 ) { for( int b = 0; b < 256; ++b ) { uint64_t r = b; for( int i = 0; i < 8; ++i ) { r = r & 1 ? (r >> 1) ^ poly64 : r >> 1; } crc64_table[ b ] = r; //printf("%016llx\n", crc64_table[b]); } poly64 = 0; } const uint8_t *buf = (const uint8_t *)ptr; uint64_t crc = ~h; // ~crc; while( len != 0 ) { crc = crc64_table[(uint8_t)crc ^ *buf++] ^ (crc >> 8); --len; } return ~crc; } // https://en.wikipedia.org/wiki/MurmurHash static inline uint32_t murmur3_scramble(uint32_t k) { return k *= 0xcc9e2d51, k = (k << 15) | (k >> 17), k *= 0x1b873593; } uint32_t murmur3_mem(const uint8_t* key, size_t len, uint32_t seed) { uint32_t h = seed; uint32_t k; /* Read in groups of 4. */ for (size_t i = len >> 2; i; i--) { // Here is a source of differing results across endiannesses. // A swap here has no effects on hash properties though. k = *((uint32_t*)key); key += sizeof(uint32_t); h ^= murmur3_scramble(k); h = (h << 13) | (h >> 19); h = h * 5 + 0xe6546b64; } /* Read the rest. */ k = 0; for (size_t i = len & 3; i; i--) { k <<= 8; k |= key[i - 1]; } // A swap is *not* necessary here because the preceeding loop already // places the low bytes in the low places according to whatever endianness // we use. Swaps only apply when the memory is copied in a chunk. h ^= murmur3_scramble(k); /* Finalize. */ h ^= len; h ^= h >> 16; h *= 0x85ebca6b; h ^= h >> 13; h *= 0xc2b2ae35; h ^= h >> 16; return h; } #endif // ----------------------------------------------------------------------------- // storage (emscripten only) void storage_mount(const char* folder) { #if is(ems) emscripten_run_script(va("FS.mkdir('%s'); FS.mount(IDBFS, {}, '%s');", folder, folder)); #else (void)folder; #endif } void storage_read() { #if is(ems) EM_ASM( /* sync from persisted state into memory */ FS.syncfs(true, function (err) { assert(!err); }); ); #endif } void storage_flush() { #if is(ems) EM_ASM( FS.syncfs(false, function (err) { assert(!err); }); ); #endif } // ----------------------------------------------------------------------------- // compressed archives // return list of files inside zipfile array(char*) file_zip_list(const char *zipfile) { static __thread array(char*) list[16] = {0}; static __thread int count = 0; count = (count+1) % 16; array_resize(list[count], 0); for( zip *z = zip_open(zipfile, "rb"); z; zip_close(z), z = 0) { for( unsigned i = 0; i < zip_count(z); ++i ) { array_push( list[count], zip_name(z, i) ); } } return list[count]; } // extract single file content from zipfile array(char) file_zip_extract(const char *zipfile, const char *filename) { static __thread array(char) list[16] = {0}; static __thread int count = 0; array(char) out = list[count = (count+1) % 16]; array_resize(out, 0); for( zip *z = zip_open(zipfile, "rb"); z; zip_close(z), z = 0) { int index = zip_find(z, filename); // convert entry to index. returns <0 if not found. if( index < 0 ) return zip_close(z), out; unsigned outlen = zip_size(z, index); unsigned excess = zip_excess(z, index); array_resize(out, outlen + 1 + excess); unsigned ret = zip_extract_inplace(z, index, out, array_count(out)); if(ret) { out[outlen] = '\0'; array_resize(out, outlen); } else { array_resize(out, 0); } } return out; } // append single file into zipfile. compress with DEFLATE|6. Other compressors are also supported (try LZMA|5, ULZ|9, LZ4X|3, etc.) bool file_zip_append(const char *zipfile, const char *filename, int clevel) { bool ok = false; for( zip *z = zip_open(zipfile, "a+b"); z; zip_close(z), z = 0) { for( FILE *fp = fopen(filename, "rb"); fp; fclose(fp), fp = 0) { ok = zip_append_file(z, filename, "", fp, clevel); } } return ok; } // append mem blob into zipfile. compress with DEFLATE|6. Other compressors are also supported (try LZMA|5, ULZ|9, LZ4X|3, etc.) // @fixme: implement zip_append_mem() and use that instead bool file_zip_appendmem(const char *zipfile, const char *entryname, const void *ptr, unsigned len, int clevel) { bool ok = false; if( ptr ) for( zip *z = zip_open(zipfile, "a+b"); z; zip_close(z), z = 0) { ok = zip_append_mem(z, entryname, "", ptr, len, clevel); } return ok; } // ----------------------------------------------------------------------------- // archives enum { is_zip, is_tar, is_pak, is_dir }; typedef struct archive_dir { char* path; union { int type; int size; // for cache only }; union { void *archive; void *data; // for cache only zip* zip_archive; tar* tar_archive; pak* pak_archive; }; struct archive_dir *next; } archive_dir; static archive_dir *dir_mount; static archive_dir *dir_cache; #ifndef MAX_CACHED_FILES // @todo: should this be MAX_CACHED_SIZE (in MiB) instead? #define MAX_CACHED_FILES 32 // @todo: should we cache the cooked contents instead? ie, stbi() result instead of file.png? #endif struct vfs_entry { const char *name; const char *id; unsigned size; }; static array(struct vfs_entry) vfs_hints; // mounted raw assets static array(struct vfs_entry) vfs_entries; // mounted cooked assets static bool vfs_mount_hints(const char *path); static void vfs_reload() { const char *app = app_name(); array_resize(vfs_hints, 0); // @leak array_resize(vfs_entries, 0); // @leak // mount virtual filesystems later (mounting order matters: low -> to -> high priority) #if defined(EMSCRIPTEN) vfs_mount("index.zip"); #else // mount fused executables vfs_mount(va("%s%s%s", app_path(), app_name(), ifdef(win32, ".exe", ""))); // mount all zipfiles for each_array( file_list("*.zip"), char*, file ) vfs_mount(file); #endif // vfs_resolve() will use these art_folder locations as hints when cook-on-demand is in progress. // cook-on-demand will not be able to resolve a virtual pathfile if there are no cooked assets on disk, // unless there is a record of what's actually on disk somewhere, and that's where the hints belong to. if( COOK_ON_DEMAND ) for each_substring(ART,",",art_folder) { vfs_mount_hints(art_folder); } } #define ARK1 0x41724B31 // 'ArK1' in le, 0x314B7241 41 72 4B 31 otherwise #define ARK1_PADDING (512 - 40) // 472 #define ARK_PRINTF(f,...) 0 // printf(f,__VA_ARGS__) #define ARK_SWAP32(x) (x) #define ARK_SWAP64(x) (x) #define ARK_REALLOC REALLOC static uint64_t ark_fget64( FILE *in ) { uint64_t v; fread( &v, 1, 8, in ); return ARK_SWAP64(v); } void ark_list( const char *infile, zip **z ) { for( FILE *in = fopen(infile, "rb"); in; fclose(in), in = 0 ) while(!feof(in)) { if( 0 != (ftell(in) % ARK1_PADDING) ) fseek(in, ARK1_PADDING - (ftell(in) % ARK1_PADDING), SEEK_CUR); ARK_PRINTF("Reading at #%d\n", (int)ftell(in)); uint64_t mark = ark_fget64(in); if( mark != ARK1 ) continue; uint64_t stamp = ark_fget64(in); uint64_t datalen = ark_fget64(in); uint64_t datahash = ark_fget64(in); uint64_t namelen = ark_fget64(in); *z = zip_open_handle(in, "rb"); return; } } static bool vfs_mount_(const char *path, array(struct vfs_entry) *entries) { const char *path_bak = path; zip *z = NULL; tar *t = NULL; pak *p = NULL; dir *d = NULL; int is_folder = ('/' == path[strlen(path)-1]); if( is_folder ) d = dir_open(path, "rb"); if( is_folder && !d ) return 0; if( !is_folder ) z = zip_open(path, "rb"); if( !is_folder && !z ) t = tar_open(path, "rb"); if( !is_folder && !z && !t ) p = pak_open(path, "rb"); if( !is_folder && !z && !t && !p ) ark_list(path, &z); // last resort. try as .ark if( !is_folder && !z && !t && !p ) return 0; // normalize input -> "././" to "" while (path[0] == '.' && path[1] == '/') path += 2; path = STRDUP(path); if( z || t || p ) { // save local path for archives, so we can subtract those from upcoming requests if(strrchr(path,'/')) strrchr(path,'/')[1] = '\0'; } else if(d) 0[(char*)path] = 0; // append to mounted archives archive_dir *prev = dir_mount, zero = {0}; *(dir_mount = REALLOC(0, sizeof(archive_dir))) = zero; dir_mount->next = prev; dir_mount->path = (char*)path; dir_mount->archive = z ? (void*)z : t ? (void*)t : p ? (void*)p : (void*)d; dir_mount->type = is_folder ? is_dir : z ? is_zip : t ? is_tar : p ? is_pak : -1; ASSERT(dir_mount->type >= 0 && dir_mount->type < 4); // append list of files to internal listing for( archive_dir *dir = dir_mount; dir ; dir = 0 ) { // for(archive_dir *dir = dir_mount; dir; dir = dir->next) { assert(dir->type >= 0 && dir->type < 4); unsigned (*fn_count[4])(void*) = {(void*)zip_count, (void*)tar_count, (void*)pak_count, (void*)dir_count}; char* (*fn_name[4])(void*, unsigned index) = {(void*)zip_name, (void*)tar_name, (void*)pak_name, (void*)dir_name}; unsigned (*fn_size[4])(void*, unsigned index) = {(void*)zip_size, (void*)tar_size, (void*)pak_size, (void*)dir_size}; for( unsigned idx = 0, end = fn_count[dir->type](dir->archive); idx < end; ++idx ) { assert(idx < end); const char *filename = STRDUP( fn_name[dir->type](dir->archive, idx) ); const char *fileid = STRDUP( file_id(filename) ); unsigned filesize = fn_size[dir->type](dir->archive, idx); // printf("%u) %s %u [%s]\n", idx, filename, filesize, fileid); // append to list array_push(*entries, (struct vfs_entry){filename, fileid, filesize}); } PRINTF("Mounted VFS volume '%s' (%u entries)\n", path_bak, fn_count[dir->type](dir->archive) ); } return 1; } static bool vfs_mount_hints(const char *path) { return vfs_mount_(path, &vfs_hints); } bool vfs_mount(const char *path) { return vfs_mount_(path, &vfs_entries); } array(char*) vfs_list(const char *masks) { static __thread array(char*) list = 0; // @fixme: add 16 slots for( int i = 0; i < array_count(list); ++i ) { FREE(list[i]); } array_resize(list, 0); for each_substring(masks,";",it) { if( COOK_ON_DEMAND ) // edge case: any game using only vfs api + cook-on-demand flag will never find any file for each_array(file_list(it), char*, item) { // insert copy char *copy = STRDUP(item); array_push(list, copy); } it = va("*/%s", it); // int recurse = !!strstr(it, "**"); // @fixme: support non-recursive for( unsigned i = 0; i < array_count(vfs_entries); ++i ) { const char *name = vfs_entries[i].name; if( strmatch(name, it) ) { // insert copy char *copy = STRDUP(name); array_push(list, copy); } } } // sort alphabetically then remove dupes array_sort(list, strcmp); array_unique(list, strcmp_qsort); return list; } static char *vfs_unpack(const char *pathfile, int *size) { // must FREE() after use // @todo: add cache here char *data = NULL; for(archive_dir *dir = dir_mount; dir && !data; dir = dir->next) { if( dir->type == is_dir ) { #if 0 // sandboxed char buf[DIR_MAX]; snprintf(buf, sizeof(buf), "%s%s", dir->path, pathfile); data = file_load(buf, size); #endif } else { int (*fn_find[3])(void *, const char *) = {(void*)zip_find, (void*)tar_find, (void*)pak_find}; void* (*fn_unpack[3])(void *, unsigned) = {(void*)zip_extract, (void*)tar_extract, (void*)pak_extract}; unsigned (*fn_size[3])(void *, unsigned) = {(void*)zip_size, (void*)tar_size, (void*)pak_size}; #if 0 const char* cleanup = pathfile + strbegi(pathfile, dir->path) * strlen(dir->path); while (cleanup[0] == '/') ++cleanup; #else const char *cleanup = pathfile; #endif int index = fn_find[dir->type](dir->archive, cleanup); data = fn_unpack[dir->type](dir->archive, index); if( size ) *size = fn_size[dir->type](dir->archive, index); } // printf("%c trying %s in %s ...\n", data ? 'Y':'N', pathfile, dir->path); } //wait_ms(1000); // <-- simulate slow hdd return data; } const char *vfs_resolve(const char *pathfile) { // we dont resolve absolute paths. they dont belong to the vfs // if( pathfile[0] == '/' || pathfile[0] == '\\' || pathfile[1] == ':' ) return pathfile; char* id = file_id(pathfile); // find best match (vfs_entries first) for (int i = array_count(vfs_entries); --i >= 0; ) { if (strbegi(vfs_entries[i].id, id) ) { return vfs_entries[i].name; } } // find best match (vfs_hints later) for (int i = array_count(vfs_hints); --i >= 0; ) { if (strbegi(vfs_hints[i].id, id) ) { return vfs_hints[i].name; } } return pathfile; } char* vfs_load(const char *pathfile, int *size_out) { // @todo: fix leaks, vfs_unpack() // @fixme: handle \\?\ absolute path (win) if (!pathfile[0]) return file_load(pathfile, size_out); while( pathfile[0] == '.' && (pathfile[1] == '/' || pathfile[1] == '\\') ) pathfile += 2; // if (pathfile[0] == '/' || pathfile[1] == ':') return file_load(pathfile, size_out); // @fixme: handle current cooked /home/V4K or C:/V4K path cases within zipfiles if( size_out ) *size_out = 0; if( strend(pathfile, "/") ) return 0; // it's a dir static __thread map(char*,int) misses = 0, *init = 0; if(!init) init = misses, map_init(misses, less_str, hash_str); int *found = map_find_or_add_allocated_key(misses, STRDUP(pathfile), -1); // [-1]non-init,[false]could not cook,[true]cooked if( found && *found == 0 ) { return 0; } //{ // exclude garbage from material names // @todo: exclude double slashs in paths char *base = file_name(pathfile); if(strchr(base,'+')) base = strchr(base, '+')+1; if(base[0] == '\0') return 0; // it's a dir char *folder = file_path(pathfile); pathfile = va("%s%s", folder, base); // solve virtual path pathfile = va("%s", vfs_resolve(pathfile)); base = file_name(pathfile); if(base[0] == '\0') return 0; // it's a dir folder = file_path(pathfile); // ease folders reading by shortening them: /home/rlyeh/prj/v4k/art/demos/audio/coin.wav -> demos/audio/coin.wav // or C:/prj/v4k/engine/art/fonts/B612-BoldItalic.ttf -> fonts/B612-BoldItalic.ttf static __thread array(char*) art_paths = 0; if(!art_paths) for each_substring(ART,",",stem) array_push(art_paths, STRDUP(stem)); char* pretty_folder = ""; if( folder ) for( int i = 0; i < array_count(art_paths); ++i ) { if( strbeg(folder, art_paths[i]) ) { pretty_folder = folder + strlen(art_paths[i]); break; } } //} // PRINTF("VFS: %s\n", pathfile); int size = 0; void *ptr = 0; #if 0 // clean pathfile while (pathfile[0] == '.' && pathfile[1] == '/') pathfile += 2; while (pathfile[0] == '/') ++pathfile; #endif const char *lookup_id = /*file_normalize_with_folder*/(pathfile); // search (last item) static __thread char last_item[256] = { 0 }; static __thread void *last_ptr = 0; static __thread int last_size = 0; if( !strcmpi(lookup_id, last_item)) { ptr = last_ptr; size = last_size; } // search (cache) if( !ptr && ! is(osx) ) { ptr = cache_lookup(lookup_id, &size); } if( ptr ) { PRINTF("Loading VFS (%s)%s (cached)\n", pretty_folder, base); } else { PRINTF("Loading VFS (%s)%s\n", pretty_folder, base); } // read cooked asset from mounted disks if( !ptr ) { ptr = vfs_unpack(pathfile, &size); // asset not found? maybe it has not been cooked yet at this point (see --cook-on-demand) if( !ptr && COOK_ON_DEMAND ) { static thread_mutex_t mutex, *init = 0; if(!init) thread_mutex_init(init = &mutex); thread_mutex_lock(&mutex); // this block saves some boot time (editor --cook-on-demand: boot 1.50s -> 0.90s) #if 1 // EXPERIMENTAL_DONT_COOK_NON_EXISTING_ASSETS static set(char*) disk = 0; if(!disk) { set_init_str(disk); for each_substring(ART,",",art_folder) for each_array(file_list(va("%s**", art_folder)), char*, item) set_insert(disk, STRDUP(item)); } // art_folder ends with '/' int found = !!set_find(disk, (char*)pathfile); if( found ) #endif { // technically, we should only cook `base` asset at this point. however, cooks on demand can be very // expensive. not only because we're invoking an external tool/cook app in here, which is scanning all the // cook folders at every call, but also because there is this expensive vfs_reload() call at end of current scope. // so, in order to minimize the number of upcoming cook requests, we cook more stuff than needed at this point; // just in anticipation of what's likely going to happen in the next frames. // so, when asked to cook "models/model.fbx" we actually: // - do cook "models/model.* (so, "model.tga","model.png","model.jpg","model.mtl",etc. are also cooked) // - do cook "models/*.fbx" as well char *dir = file_path(pathfile + ART_SKIP_ROOT); char *group1 = dir[0] ? va("\"*/%s%s.*\"", dir, file_base(pathfile)) : base; // -> "*/models/model.*" char *group2 = dir[0] ? va("\"*/%s*%s\"", dir, file_ext(pathfile)) : ""; // -> "*/models/*.fbx" char *cmd = va("%scook" ifdef(osx,".osx",ifdef(linux,".linux",".exe"))" %s %s --cook-ini=%s --cook-additive --cook-jobs=1 --quiet", TOOLS, group1, group2, COOK_INI); // cook groups int rc = system(cmd); if(rc < 0) PANIC("cannot invoke `%scook` (return code %d)", TOOLS, rc); vfs_reload(); // @todo: optimize me. it is waaay inefficent to reload the whole VFS layout after cooking a single asset } thread_mutex_unlock(&mutex); // finally, try again pathfile = va("%s", vfs_resolve(pathfile)); ptr = vfs_unpack(pathfile, &size); } if( ptr ) { cache_insert(lookup_id, ptr, size); } } if( ptr && size ) if( ptr != last_ptr) { snprintf(last_item, 256, "%s", lookup_id); last_ptr = ptr; last_size = size; } // yet another last resort: redirect vfs_load() calls to file_load() // (for environments without tools or cooked assets) if(!ptr) { if( !have_tools() ) { ptr = file_load(pathfile, size_out); } } if(!ptr) { PRINTF("Loading %s (not found)\n", pathfile); } *found = ptr ? true : false; if( size_out ) *size_out = ptr ? size : 0; return ptr; } char* vfs_read(const char *pathfile) { return vfs_load(pathfile, NULL); } int vfs_size(const char *pathfile) { int sz; return vfs_load(pathfile, &sz), sz; } FILE* vfs_handle(const char *pathfile) { // @fixme: non-compressible assets (adpcm wavs,mp3,ogg,mp4,avi,...) are taking a suboptimal code path here. // no need to unzip them. just seek within the zipfile and return the *fp at that position int sz; char *buf = vfs_load(pathfile, &sz); FILE *fp = fmemopen(buf ? buf : "", buf ? sz : 0, "r+b"); ASSERT( fp, "cannot create tempfile" ); return fp; } #if 0 const char *vfs_extract(const char *pathfile) { // extract a vfs file into the local (temp) filesystem #if 0 FILE* fp = vfs_handle(pathfile); return fp ? pathfile_from_handle(fp) : ""; #else int sz; char *buf = vfs_load(pathfile, &sz); if( !buf ) return ""; // pool of temp files. recycles after every loop enum { MAX_TEMP_FILES = 16 }; static __thread char temps[MAX_TEMP_FILES][DIR_MAX] = {0}; static __thread int i = 0; if( temps[i][0] ) unlink(temps[i]); i = (i + 1) % MAX_TEMP_FILES; if(!temps[i][0] ) snprintf(temps[i], DIR_MAX, "%s", file_tempname()); char *name = temps[i]; FILE *tmp = fopen(name, "wb"); //unlink(name); ASSERT( tmp, "cannot create tempfile %s", name ); fwrite(buf ? buf : "", 1, buf ? sz : 0, tmp); fclose(tmp); return name; #endif } #endif // ----------------------------------------------------------------------------- // cache static thread_mutex_t cache_mutex; AUTORUN{ thread_mutex_init(&cache_mutex); } void* cache_lookup(const char *pathfile, int *size) { // find key->value if( !MAX_CACHED_FILES ) return 0; void* data = 0; thread_mutex_lock(&cache_mutex); for(archive_dir *dir = dir_cache; dir; dir = dir->next) { if( !strcmp(dir->path, pathfile) ) { if(size) *size = dir->size; data = dir->data; break; } } thread_mutex_unlock(&cache_mutex); return data; } void* cache_insert(const char *pathfile, void *ptr, int size) { // append key/value; return LRU or NULL if( !MAX_CACHED_FILES ) return 0; if( !ptr || !size ) return 0; // keep cached files within limits thread_mutex_lock(&cache_mutex); // append to cache archive_dir zero = {0}, *old = dir_cache; *(dir_cache = REALLOC(0, sizeof(archive_dir))) = zero; dir_cache->next = old; dir_cache->path = STRDUP(pathfile); dir_cache->size = size; dir_cache->data = REALLOC(0, size+1); memcpy(dir_cache->data, ptr, size); size[(char*)dir_cache->data] = 0; // copy+terminator void *found = 0; static int added = 0; if( added < MAX_CACHED_FILES ) { ++added; } else { // remove oldest cache entry for( archive_dir *prev = dir_cache, *dir = prev; dir ; prev = dir, dir = dir->next ) { if( !dir->next ) { prev->next = 0; // break link found = dir->data; dir->path = REALLOC(dir->path, 0); dir->data = REALLOC(dir->data, 0); dir = REALLOC(dir, 0); break; } } } thread_mutex_unlock(&cache_mutex); return found; } // ---------------------------------------------------------------------------- // ini /* ini+, extended ini format // - rlyeh, public domain // // # spec // // ; line comment // [user] ; map section name (optional) // name=john ; key and value (mapped here as user.name=john) // +surname=doe jr. ; sub-key and value (mapped here as user.name.surname=doe jr.) // age=30 ; numbers // color=240 ; key and value \ // color=253 ; key and value |> array: color[0], color[1] and color[2] // color=255 ; key and value / // color= ; remove key/value(s) // color=white ; recreate key; color[1] and color[2] no longer exist // [] ; unmap section // -note=keys may start with symbols (except plus and semicolon) // -note=linefeeds are either \r, \n or \r\n. // -note=utf8 everywhere. */ static char *ini_parse( const char *s ) { char *map = 0; int mapcap = 0, maplen = 0; enum { DEL, REM, TAG, KEY, SUB, VAL } fsm = DEL; const char *cut[6] = {0}, *end[6] = {0}; while( *s ) { while( *s && (*s == ' ' || *s == '\t' || *s == '\r' || *s == '\n') ) ++s; /**/ if( *s == ';' ) cut[fsm = REM] = ++s; else if( *s == '[' ) cut[fsm = TAG] = ++s; else if( *s == '+' ) cut[fsm = SUB] = ++s; else if( *s == '=' ) cut[fsm = VAL] = ++s; else if( *s > ' ' && *s <= 'z' && *s != ']' ) cut[fsm = KEY] = cut[SUB] = end[SUB] = s; else { if( *s ) ++s; continue; } /**/ if( fsm == REM ) { while(*s && *s != '\r'&& *s != '\n') ++s; } else if( fsm == TAG ) { while(*s && *s != '\r'&& *s != '\n'&& *s != ']') ++s; end[TAG] = s; } else if( fsm == KEY ) { while(*s && *s > ' ' && *s <= 'z' && *s != '=') ++s; end[KEY] = s; } else if( fsm == SUB ) { while(*s && *s > ' ' && *s <= 'z' && *s != '=') ++s; end[SUB] = s; } else if( fsm == VAL ) { while(*s && *s >= ' ' && *s <= 127 && *s != ';') ++s; end[VAL] = s; while( end[VAL][-1] <= ' ' ) { --end[VAL]; } char buf[256] = {0}, *key = buf; if( end[TAG] - cut[TAG] ) key += sprintf(key, "%.*s.", (int)(end[TAG] - cut[TAG]), cut[TAG] ); if( end[KEY] - cut[KEY] ) key += sprintf(key, "%.*s", (int)(end[KEY] - cut[KEY]), cut[KEY] ); if( end[SUB] - cut[SUB] ) key += sprintf(key, ".%.*s", (int)(end[SUB] - cut[SUB]), cut[SUB] ); int reqlen = (key - buf) + 1 + (end[VAL] - cut[VAL]) + 1 + 1; if( (reqlen + maplen) >= mapcap ) map = REALLOC( map, mapcap += reqlen + 512 ); sprintf( map + maplen, "%.*s%c%.*s%c%c", (int)(key - buf), buf, 0, (int)(end[VAL] - cut[VAL]), cut[VAL], 0, 0 ); maplen += reqlen - 1; } } return map; } // @todo: evaluate alt api // int count = ini_count(filename); // char *key = ini_key(filename, id); // char *val = ini_val(filename, id); void ini_destroy(ini_t x) { for each_map(x, char*, k, char*, v) { FREE(k); FREE(v); } map_free(x); } ini_t ini_from_mem(const char *data) { if( !data || !data[0] ) return 0; char *kv = ini_parse(data); if( !kv ) return 0; ini_t map = 0; map_init(map, less_str, hash_str); for( char *iter = kv; iter[0]; ) { char *key = iter; while( *iter++ ); char *val = iter; while( *iter++ ); char **found = map_find(map, key); if( !found ) map_insert(map, STRDUP(key), STRDUP(val)); assert( map_find(map,key) ); } FREE( kv ); return map; } ini_t ini(const char *filename) { return ini_from_mem(file_read(filename)); } bool ini_write(const char *filename, const char *section, const char *key, const char *value) { // this is a little hacky { char *data = file_read(filename); if( data && data[0] ) { char *begin = strrchr(data, '['); char *end = strrchr(data, ']'); if( begin && end && begin < end ) { char *last_section = va("%.*s", (int)(end - begin - 1), begin + 1); if( !strcmpi(last_section, section) ) section = 0; } } // } char *s = va("%s%s=%s\r\n", section ? va("[%s]\r\n", section) : "", key, value); return file_append(filename, s, strlen(s)); }