// data pipeline // - rlyeh, public domain. // ---------------------------------------------------------------------------- // @todo: threads should steal workloads from job queue // @todo: restore errno/errorlevel checks // @todo: +=, -=, that_asset.ini // @todo: @dae FLAGS+=-U // @todo: SF2_SOUNDBANK=TOOLS/soundbank.sf2 // @fixme: leaks (worth?) // ----------------------------------------------------------------------------- #ifndef COOK_INI_PATHFILE #define COOK_INI_PATHFILE "tools/cook.ini" #endif const char *ART = "art/"; const char *TOOLS = "tools/bin/"; const char *EDITOR = "tools/"; const char *COOK_INI = COOK_INI_PATHFILE; static unsigned ART_SKIP_ROOT; // number of chars to skip the base root in ART folder static unsigned ART_LEN; // dupe typedef struct cook_subscript_t { char *infile; char *outfile; // can be either infile, or a totally different file char *script; char *outname; int compress_level; uint64_t pass_ns, gen_ns, exe_ns, zip_ns; } cook_subscript_t; typedef struct cook_script_t { cook_subscript_t cs[8]; int num_passes; uint64_t pass_ns, gen_ns, exe_ns, zip_ns; } cook_script_t; static cook_script_t cook_script(const char *rules, const char *infile, const char *outfile) { cook_script_t mcs = { 0 }; // pass loop: some asset rules may require multiple cook passes for( int pass = 0; pass < countof(mcs.cs); ++pass ) { // by default, assume: // - no script is going to be generated (empty script) // - if no script is going to be generated, output is in fact input file. // - no compression is going to be required. cook_subscript_t cs = { 0 }; cs.gen_ns -= time_ns(); // reuse script heap from last call if possible (optimization) static __thread char *script = 0; if(script) script[0] = 0; // reuse parsing maps if possible (optimization) static __thread map(char*, char*) symbols = 0; if(!symbols) map_init_str(symbols); static __thread map(char*, char*) groups = 0; if(!groups) map_init_str(groups); static __thread set(char*) passes = 0; if(!passes) set_init_str(passes); map_clear(symbols); map_clear(groups); map_find_or_add(symbols, "INFILE", STRDUP(infile)); map_find_or_add(symbols, "INPUT", STRDUP(infile)); map_find_or_add(symbols, "PRETTY", STRDUP(infile + ART_SKIP_ROOT)); // pretty (truncated) input (C:/prj/V4K/art/file.wav -> file.wav) map_find_or_add(symbols, "OUTPUT", STRDUP(outfile)); map_find_or_add(symbols, "TOOLS", STRDUP(TOOLS)); map_find_or_add(symbols, "EDITOR", STRDUP(EDITOR)); map_find_or_add(symbols, "PROGRESS", STRDUP(va("%03d", cook_progress()))); // clear pass counter set_clear(passes); // start parsing. parsing is enabled by default int enabled = 1; array(char*)lines = strsplit(rules, "\r\n"); for( int i = 0, end = array_count(lines); i < end; ++i ) { // skip blanks int blanks = strspn(lines[i], " \t"); char *line = lines[i] + blanks; // discard full comments if( line[0] == ';' ) continue; // truncate inline comments if( strstr(line, ";") ) *strstr(line, ";") = 0; // trim ending spaces char *eos = line + strlen(line); while(eos > line && eos[-1] == ' ' ) *--eos = 0; // discard non-specific lines if( line[0] == '@' ) { int with_wine = flag("--cook-wine") && !!strstr(line, "@win"); int parse = 0 | ifdef(win32, (!!strstr(line, "@win")), 0) | ifdef(linux, (!!strstr(line, "@lin") ? 1 : with_wine), 0) | ifdef(osx, (!!strstr(line, "@osx") ? 1 : with_wine), 0); if( !parse ) continue; line = strchr(line+1, ' '); if(!line) continue; line += strspn(line, " \t"); } // execute `shell` commands if( line[0] == '`' ) { char *eos = strrchr(++line, '`'); if( eos ) *eos = 0; // replace all symbols char* nl = STRDUP(line); // @leak for each_map(symbols, char*, key, char*, val) { strrepl(&nl, key, val); } lines[i] = line = nl; #if 0 static thread_mutex_t lock, *init = 0; if(!init) thread_mutex_init(init = &lock); thread_mutex_lock( &lock ); system(line); // strcatf(&script, "%s\n", line); thread_mutex_unlock( &lock ); #else // append line strcatf(&script, "%s\n", line); #endif continue; } // process [sections] if( line[0] == '[' ) { enabled = 1; int is_cook = !!strstr(line, "[cook]"); int is_compress = !!strstr(line, "[compress]"); if( !is_cook && !is_compress ) { // if not a special section... // remove hint cook tag if present. that's informative only. if(strbegi(line, "[cook ") ) memcpy(line+1, " ", 4); // line += 6; // start parsing expressions like `[media && !avi && mp3]` array(char*) tags = strsplit(line, " []&"); // let's check whether INPUT belongs to tags above char **INPUT = map_find(symbols, "INPUT"); bool found_in_set = true; for( int i = 0, end = array_count(tags); i < end; ++i) { bool negate = false; char *tag = tags[i]; while(*tag == '!') negate ^= 1, ++tag; // find tag in groups map // either a group or an extension char **is_group = map_find(groups, tag); if( is_group ) { char *list = *is_group; char *INPUT_EXT = file_ext(infile); INPUT_EXT = strrchr(INPUT_EXT, '.'); // .ext1.ext -> .ext char *ext = INPUT_EXT; ext += ext[0] == '.'; // dotless bool in_list = strbegi(list, ext) || strendi(list, va(",%s",ext)) || strstri(list, va(",%s,",ext)); if( !in_list ^ negate ) { found_in_set = false; break; } } else { char *ext = va(".%s", tag); bool found = !!strendi(*INPUT, ext); if( !found ^ negate ) { found_in_set = false; break; } } } if( found_in_set ) { // inc pass set_find_or_add(passes, STRDUP(*tags)); // @leak // check whether we keep searching int num_passes = set_count(passes); found_in_set = ( pass == (num_passes-1) ); } // enabled = found_in_set ? 1 : 0; } } // either SYMBOL=, group=, or regular script line if( enabled && line[0] != '[' ) { enum { group, symbol, regular } type = regular; int tokenlen = strspn(line, "-+_.|0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"); char *token = va("%.*s", tokenlen, line); char *equal = strchr(line, '='); if( equal ) { if( equal == &line[tokenlen] ) { // if key=value expression found // discriminate: symbols are uppercase and never begin with digits. groups are [0-9]+[|][a-z]. type = strcmp(strupper(token), token) || isdigit(token[0]) ? group : symbol; } } if( type == group ) map_find_or_add(groups, token, STRDUP(equal+1)); if( type == symbol ) { // @todo: perform the replacement/union/intersection on set here bool is_add = strendi(token, "+"); bool is_del = strendi(token, "-"); // if present, remove last sign from token -> (FLAGS1+)=, (FLAGS1-)= if(is_add || is_del) token[strlen(token) - 1] = 0; map_find_or_add(symbols, token, STRDUP(equal+1)); } // for each_map(symbols, char*, key, char*, val) printf("%s=%s,", key, val); puts(""); // for each_map(groups, char*, key, char*, val) printf("%s=%s,", key, val); puts(""); // if( type != regular ) printf("%s found >> %s\n", type == group ? "group" : "symbol", line); if( type == regular ) { char** INPUT = map_find(symbols, "INPUT"); char** OUTPUT = map_find(symbols, "OUTPUT"); // parse return code char *has_errorlevel = strstr(line, "=="); //==N form int errorlevel = has_errorlevel ? atoi(has_errorlevel + 2) : 0; if( has_errorlevel ) memcpy(has_errorlevel, " ", 3); // detect if newer extension or filename is present, and thus update OUTPUT if needed char *newer_extension = strstr(line, "->"); if(newer_extension) { *newer_extension = 0; newer_extension += 2 + strspn(newer_extension + 2, " "); if( strchr(newer_extension, '.') ) { // newer filename cs.outname = stringf("%s@%s", cs.outname ? cs.outname : infile, newer_extension); // @leak // special char (multi-pass cooks) newer_extension = NULL; } else { strcatf(&*OUTPUT, ".%s", newer_extension); } } // replace all symbols char* nl = STRDUP(line); // @leak for each_map(symbols, char*, key, char*, val) { strrepl(&nl, key, val); } lines[i] = line = nl; // convert slashes ifdef(win32, strswap(line, "/", "\\") , // else strswap(line, "\\", "/") ); // append line strcatf(&script, "%s\n", line); // handle return code here // if(has_errorlevel) // strcatf(&script, "IF NOT '%%ERRORLEVEL%%'=='%d' echo ERROR!\n", errorlevel); // rename output->input for further chaining, in case it is needed if( newer_extension ) { *INPUT[0] = 0; strcatf(&*INPUT, "%s", *OUTPUT); } } } } char** OUTPUT = map_find(symbols, "OUTPUT"); int ext_num_groups = 0; // compression if( 1 ) { char* ext = file_ext(infile); ext = strrchr(ext, '.'); ext += ext[0] == '.'; // dotless INPUT_EXT char* belongs_to = 0; for each_map(groups, char*, key, char*, val) { if( !isdigit(key[0]) ) { char *comma = va(",%s,", ext); if( !strcmpi(val,ext) || strbegi(val, comma+1) || strstri(val, comma) || strendi(val, va(",%s", ext))) { belongs_to = key; ext_num_groups++; } } } char *compression = 0; for each_map_ptr_sorted(groups, char*, key, char*, val) { // sorted iteration, so hopefully '0' no compression gets evaluated first if( !compression && isdigit((*key)[0]) ) { char *comma = va(",%s,", ext); if( !strcmpi(*val,ext) || strbegi(*val, comma+1) || strstri(*val, comma) || strendi(*val, va(",%s", ext))) { compression = (*key); } comma = va(",%s,", belongs_to); if( !strcmpi(*val,ext) || strbegi(*val, comma+1) || strstri(*val, comma) || strendi(*val, va(",%s", ext))) { compression = (*key); } } } cs.compress_level = 0; if( compression ) { // last chance to optionally override the compressor at command-line level static const char *compressor_override; do_once compressor_override = option("--cook-compressor", ""); if( compressor_override[0] ) compression = (char*)compressor_override; /**/ if(strstri(compression, "PPP")) cs.compress_level = atoi(compression) | PPP; else if(strstri(compression, "ULZ")) cs.compress_level = atoi(compression) | ULZ; else if(strstri(compression, "LZ4")) cs.compress_level = atoi(compression) | LZ4X; else if(strstri(compression, "CRSH")) cs.compress_level = atoi(compression) | CRSH; else if(strstri(compression, "DEFL")) cs.compress_level = isdigit(compression[0]) ? atoi(compression) : 6 /*| DEFL*/; //else if(strstri(compression, "LZP")) cs.compress_level = atoi(compression) | LZP1; // not supported else if(strstri(compression, "LZMA")) cs.compress_level = atoi(compression) | LZMA; else if(strstri(compression, "BALZ")) cs.compress_level = atoi(compression) | BALZ; else if(strstri(compression, "LZW")) cs.compress_level = atoi(compression) | LZW3; else if(strstri(compression, "LZSS")) cs.compress_level = atoi(compression) | LZSS; else if(strstri(compression, "BCM")) cs.compress_level = atoi(compression) | BCM; else cs.compress_level = isdigit(compression[0]) ? atoi(compression) : 6 /*| DEFL*/; } } // if script was generated... if( script && script[0] && strstr(script, ifdef(win32, file_normalize(va("%s",infile)), infile )) ) { // update outfile cs.outfile = *OUTPUT; // amalgamate script array(char*) lines = strsplit(script, "\r\n"); #if is(win32) char *joint = strjoin(lines, " && "); cs.script = joint; #else if( flag("--cook-wine") ) { // dear linux/osx/bsd users: // tools going wrong for any reason? cant compile them maybe? // small hack to use win32 pipeline tools instead char *joint = strjoin(lines, " && wine " ); cs.script = va("wine %s", /*TOOLS,*/ joint); } else { char *joint = strjoin(lines, " && " ); cs.script = va("export LD_LIBRARY_PATH=%s && %s", TOOLS, joint); } #endif } else { // if( script && script[0] ) system(script); //< @todo: un-comment this line if we want to get the shell command prints invoked per entry // ... else bypass infile->outfile char** INFILE = map_find(symbols, "INFILE"); cs.outfile = *INFILE; // and return an empty script cs.script = ""; } cs.outname = cs.outname ? cs.outname : (char*)infile; cs.gen_ns += time_ns(); ASSERT(mcs.num_passes < countof(mcs.cs)); mcs.cs[mcs.num_passes++] = cs; bool next_pass_required = mcs.num_passes < ext_num_groups; if( !next_pass_required ) break; } return mcs; } // ---------------------------------------------------------------------------- struct fs { char *fname, status; uint64_t stamp; uint64_t bytes; }; static array(struct fs) fs_now; static __thread array(char*) added; static __thread array(char*) changed; static __thread array(char*) deleted; static __thread array(char*) uncooked; static array(struct fs) zipscan_filter(int threadid, int numthreads) { // iterate all previously scanned files array(struct fs) fs = 0; for( int i = 0, end = array_count(fs_now); i < end; ++i ) { // during workload distribution, we assign random files to specific thread buckets. // we achieve this by hashing the basename of the file. we used to hash also the path // long time ago but that is less resilient to file relocations across the repository. // excluding the file extension from the hash also helps from external file conversions. char *fname = file_name(fs_now[i].fname); char *sign = strrchr(fname, '@'); if(sign) *sign = '\0'; // special char (multi-pass cooks) char *dot = strrchr(fname, '.'); if(dot) *dot = '\0'; // skip if list item does not belong to this thread bucket uint64_t hash = hash_str(fname); unsigned bucket = (hash /*>> 32*/) % numthreads; if(bucket != threadid) continue; array_push(fs, fs_now[i]); } return fs; } static int zipscan_diff( zip* old, array(struct fs) now ) { array_free(added); array_free(changed); array_free(deleted); array_free(uncooked); // if not zipfile is present, all files are new and must be added if( !old ) { for( int i = 0; i < array_count(now); ++i ) { array_push(uncooked, STRDUP(now[i].fname)); } return 1; } // compare for new & changed files for( int i = 0; i < array_count(now); ++i ) { int found = zip_find(old, now[i].fname); if( found < 0 ) { array_push(added, STRDUP(now[i].fname)); array_push(uncooked, STRDUP(now[i].fname)); } else { uint64_t oldsize = atoi64(zip_comment(old,found)); // zip_size(old, found); returns sizeof processed asset. return original size of unprocessed asset, which we store in comment section uint64_t oldstamp = atoi64(zip_modt(old,found)+20); // format is "YYYY/MM/DD hh:mm:ss", then +20 chars later a hidden epoch timestamp in base10 can be found int64_t diffstamp = oldstamp < now[i].stamp ? now[i].stamp - oldstamp : oldstamp - now[i].stamp; if( oldsize != now[i].bytes || diffstamp > 1 ) { // @fixme: should use hash instead. hashof(tool) ^ hashof(args used) ^ hashof(rawsize) ^ hashof(rawdate) printf("%s:\t%u vs %u, %llu vs %llu\n", now[i].fname, (unsigned)oldsize,(unsigned)now[i].bytes, (long long unsigned)oldstamp, (long long unsigned)now[i].stamp); array_push(changed, STRDUP(now[i].fname)); array_push(uncooked, STRDUP(now[i].fname)); } } } // compare for deleted files for( int i = 0; i < zip_count(old); ++i ) { char *oldname = zip_name(old, i); //if( strchr(oldname, '@') ) oldname = va("%*.s", (int)(strchr(oldname, '@') - oldname), oldname ); // special char (multi-pass cooks) int idx = zip_find(old, oldname); // find latest versioned file in zip unsigned oldsize = zip_size(old, idx); if (!oldsize) continue; struct fs *found = 0; // zipscan_locate(now, oldname); for(int j = 0; j < array_count(now); ++j) { if( !strcmp(now[j].fname,oldname)) { found = &now[j]; break; } } if( !found ) { array_push(deleted, STRDUP(oldname)); } } return 1; } // ---------------------------------------------------------------------------- typedef struct cook_worker { const char **files; const char *rules; int threadid, numthreads; thread_ptr_t self; volatile int progress; thread_mutex_t *lock; } cook_worker; enum { JOBS_MAX = 256 }; static cook_worker jobs[JOBS_MAX] = {0}; static volatile bool cook_cancelable = false, cook_cancelling = false, cook_debug = false; #ifndef COOK_ON_DEMAND #define COOK_ON_DEMAND ifdef(cook, optioni("--cook-on-demand", 1), false) #endif static int cook(void *userdata) { cook_worker *job = (cook_worker*)userdata; // start progress volatile int *progress = &job->progress; *progress = 0; // preload a few large binaries // dll("tools/furnace.exe", 0); // dll("tools/assimp-vc143-mt.dll", 0); // dll("tools/ffmpeg.exe", 0); // scan disk from fs_now snapshot array(struct fs) filtered = zipscan_filter(job->threadid, job->numthreads); //printf("Scanned: %d items found\n", array_count(now)); // prepare out tempname char COOK_TMPFILE[64]; snprintf(COOK_TMPFILE, 64, "temp_%02d", job->threadid); // prepare zip char zipfile[64]; snprintf(zipfile, 64, ".art[%02x].zip", job->threadid); if( file_size(zipfile) == 0 ) unlink(zipfile); // populate added/deleted/changed arrays by examining current disk vs last cache zip *z; { z = zip_open(zipfile, "r+b"); zipscan_diff(z, filtered); if( z ) zip_close(z); fflush(0); z = zip_open(zipfile, "a+b"); if( !z ) { unlink(zipfile); z = zip_open(zipfile, "a+b"); // try again if(!z) PANIC("cannot open file for updating: %s", zipfile); } } // deleted files. --cook-additive runs are append-only, so they skip this block if( !flag("--cook-additive") ) for( int i = 0, end = array_count(deleted); i < end; ++i ) { printf("Deleting %03d%% %s\n", (i+1) == end ? 100 : (i * 100) / end, deleted[i]); FILE* out = fopen(COOK_TMPFILE, "wb"); fclose(out); FILE* in = fopen(COOK_TMPFILE, "rb"); char *comment = "0"; zip_append_file/*_timeinfo*/(z, deleted[i], comment, in, 0/*, tm_now*/); fclose(in); } // if(array_count(uncooked)) // PRINTF("cook_jobs[%d]=%d\n", job->threadid, array_count(uncooked)); // generate cook metrics. you usually do `game.exe --cook-stats && (type *.csv | sort /R > cook.csv)` static __thread FILE *statsfile = 0; if(flag("--cook-stats")) fseek(statsfile = fopen(va("cook%d.csv",job->threadid), "a+t"), 0L, SEEK_END); if(statsfile && !job->threadid && ftell(statsfile) == 0) fprintf(statsfile,"%10s,%10s,%10s,%10s,%10s, %s\n","+total_ms","gen_ms","exe_ms","zip_ms","pass","file"); // added or changed files for( int i = 0, end = array_count(uncooked); i < end && !cook_cancelling; ++i ) { *progress = ((i+1) == end ? 90 : (i * 90) / end); // (i+i>0) * 100.f / end; // start cook const char *infile = uncooked[i]; //job->files[j]; int inlen = file_size(infile); // generate a cooking script for this asset cook_script_t mcs = cook_script(job->rules, infile, COOK_TMPFILE); // puts(cs.script); for(int pass = 0; pass < mcs.num_passes; ++pass) { cook_subscript_t cs = mcs.cs[pass]; // log to batch file for forensic purposes, if explicitly requested static __thread int logging = -1; if(logging < 0) logging = !!flag("--cook-debug") || cook_debug; if( logging ) { static __thread FILE *logfile = 0; if(!logfile) fseek(logfile = fopen(va("cook%d.cmd",job->threadid), "a+t"), 0L, SEEK_END); if( logfile ) { fprintf(logfile, "@rem %s\n%s\n", cs.outname, cs.script); fprintf(logfile, "for %%%%i in (\"%s\") do md _cook\\%%%%~pi\\%%%%~ni%%%%~xi 1>nul 2>nul\n", infile); fprintf(logfile, "for %%%%i in (\"%s\") do xcopy /y %s _cook\\%%%%~pi\\%%%%~ni%%%%~xi\n\n", infile, file_normalize(cs.outfile)); } } // invoke cooking script mcs.cs[pass].exe_ns -= time_ns(); // invoke cooking script const char *rc_output = app_exec(cs.script); // recap status int rc = atoi(rc_output); // int outlen = file_size(cs.outfile); int failed = rc; // cs.script[0] ? rc || !outlen : 0; // print errors if( failed ) { PRINTF("Import failed: %s while executing:\n%s\nReturned:\n%s\n", cs.outname, cs.script, rc_output); continue; } if( pass > 0 ) { // (multi-pass cook) // newly generated file: refresh values // ensure newly created files by cook are also present on repo/disc for further cook passes file_delete(cs.outname); file_move(cs.outfile, cs.outname); inlen = file_size(infile = cs.outfile = cs.outname); } mcs.cs[pass].exe_ns += time_ns(); // process only if included. may include optional compression. mcs.cs[pass].zip_ns -= time_ns(); if( cs.compress_level >= 0 ) { FILE *in = fopen(cs.outfile ? cs.outfile : infile, "rb"); if(!in) in = fopen(infile, "rb"); char *comment = va("%d", inlen); if( !zip_append_file(z, infile, comment, in, cs.compress_level) ) { PANIC("failed to add processed file into %s: %s(%s)", zipfile, cs.outname, infile); } fclose(in); } mcs.cs[pass].zip_ns += time_ns(); // stats per subscript mcs.cs[pass].pass_ns = mcs.cs[pass].gen_ns + mcs.cs[pass].exe_ns + mcs.cs[pass].zip_ns; if(statsfile) fprintf(statsfile, "%10.f,%10.f,%10.f,%10.f,%10d, \"%s\"\n", mcs.cs[pass].pass_ns/1e6, mcs.cs[pass].gen_ns/1e6, mcs.cs[pass].exe_ns/1e6, mcs.cs[pass].zip_ns/1e6, pass+1, infile); } } zip_close(z); // end progress if( file_size(zipfile) == 0 ) unlink(zipfile); *progress = 100; return 1; } static int cook_async( void *userdata ) { #if COOK_FROM_TERMINAL // nothing to do... #else while(!window_handle()) sleep_ms(100); // wait for window handle to be created #endif // boost cook thread #0, which happens to be the only spawn thread when num_jobs=1 (tcc case, cook-sync case). // also in multi-threaded scenarios, it is not bad at all to have one high priority thread... // in any case, game view is not going to look bad because the game will be displaying a progress bar at that time. cook_worker *job = (cook_worker*)userdata; if( job->threadid == 0 ) thread_set_high_priority(); // tcc: only a single running thread shall pass, because of racing shared state due to missing thread_local support at compiler level ifdef(tcc, thread_mutex_lock( job->lock )); ifdef(osx, thread_mutex_lock( job->lock )); // @todo: remove silicon mac M1 hack int ret = cook(userdata); // tcc: only a single running thread shall pass, because of racing shared state due to missing thread_local support at compiler level ifdef(osx, thread_mutex_unlock( job->lock )); // @todo: remove silicon mac M1 hack ifdef(tcc, thread_mutex_unlock( job->lock )); thread_exit( ret ); return ret; } bool cook_start( const char *cook_ini, const char *masks, int flags ) { cook_ini = cook_ini ? cook_ini : COOK_INI; char *rules_ = file_read(cook_ini); if(!rules_ || rules_[0] == 0) return false; static char *rules; do_once rules = STRDUP(rules_); do_once { #if 0 const char *HOME = file_pathabs(cook_ini); // ../tools/cook.ini -> c:/prj/v4k/tools/cook.ini if( strbeg(HOME, app_path() ) ) HOME = STRDUP( file_path( HOME += strlen(app_path()) ) ); // -> tools/ @leak #else char *HOME = STRDUP(file_pathabs(cook_ini)); // ../tools/cook.ini -> c:/prj/v4k/tools/cook.ini HOME[ strlen(HOME) - strlen(file_name(cook_ini)) ] = '\0'; // -> tools/ @leak #endif ART_LEN = 0; //strlen(app_path()); /* = MAX_PATH; for each_substring(ART, ",", art_folder) { ART_LEN = mini(ART_LEN, strlen(art_folder)); }*/ if( strstr(rules, "ART=") ) { ART = va( "%s", strstr(rules, "ART=") + 4 ); char *r = strchr( ART, '\r' ); if(r) *r = 0; char *n = strchr( ART, '\n' ); if(n) *n = 0; char *s = strchr( ART, ';' ); if(s) *s = 0; char *w = strchr( ART, ' ' ); if(w) *w = 0; char *out = 0; const char *sep = ""; for each_substring(ART, ",", t) { char *tmp = file_pathabs(va("%s%s", HOME, t)) + ART_LEN; for(int i = 0; tmp[i]; ++i) if(tmp[i]=='\\') tmp[i] = '/'; strcatf(&out, "%s%s%s", sep, tmp, strendi(tmp, "/") ? "" : "/"); assert( out[strlen(out) - 1] == '/' ); sep = ","; } ART = out; // @leak } if( strstr(rules, "TOOLS=") ) { TOOLS = va( "%s", strstr(rules, "TOOLS=") + 6 ); char *r = strchr( TOOLS, '\r' ); if(r) *r = 0; char *n = strchr( TOOLS, '\n' ); if(n) *n = 0; char *s = strchr( TOOLS, ';' ); if(s) *s = 0; char *w = strchr( TOOLS, ' ' ); if(w) *w = 0; char *cat = va("%s%s", HOME, TOOLS), *out = 0; for(int i = 0; cat[i]; ++i) if(cat[i]=='\\') cat[i] = '/'; strcatf(&out, "%s%s", cat, strend(cat, "/") ? "" : "/"); TOOLS = out; // @leak assert( TOOLS[strlen(TOOLS) - 1] == '/' ); // last chance to autodetect tools folder (from cook.ini path) if( !file_directory(TOOLS) ) { out = STRDUP(cook_ini); for(int i = 0; out[i]; ++i) if(out[i]=='\\') out[i] = '/'; TOOLS = out; // @leak } } if( strstr(rules, "EDITOR=") ) { EDITOR = va( "%s", strstr(rules, "EDITOR=") + 7 ); char *r = strchr( EDITOR, '\r' ); if(r) *r = 0; char *n = strchr( EDITOR, '\n' ); if(n) *n = 0; char *s = strchr( EDITOR, ';' ); if(s) *s = 0; char *w = strchr( EDITOR, ' ' ); if(w) *w = 0; char *cat = va("%s%s", HOME, EDITOR), *out = 0; for(int i = 0; cat[i]; ++i) if(cat[i]=='\\') cat[i] = '/'; strcatf(&out, "%s%s", cat, strend(cat, "/") ? "" : "/"); EDITOR = out; // @leak assert( EDITOR[strlen(EDITOR) - 1] == '/' ); } // small optimization for upcoming parser: remove whole comments from file array(char*) lines = strsplit(rules, "\r\n"); for( int i = 0; i < array_count(lines); ) { if( lines[i][0] == ';' ) array_erase_slow(lines, i); else ++i; } rules = STRDUP( strjoin(lines, "\n") ); } if( !masks ) { return true; // nothing to do } // estimate ART_SKIP_ROOT (C:/prj/v4k/demos/assets/file.png -> strlen(C:/prj/v4k/) -> 11) { array(char*) dirs = 0; for each_substring(ART, ",", art_folder) { array_push(dirs, file_pathabs(art_folder)); } if( array_count(dirs) > 1 ) { for( int ok = 1, ch = dirs[0][ART_SKIP_ROOT]; ch && ok; ch = dirs[0][++ART_SKIP_ROOT] ) { for( int i = 1; i < array_count(dirs) && ok; ++i ) { ok = dirs[i][ART_SKIP_ROOT] == ch; } } } while( ART_SKIP_ROOT > 0 && !strchr("\\/", dirs[0][ART_SKIP_ROOT-1]) ) --ART_SKIP_ROOT; array_free(dirs); } if( COOK_ON_DEMAND ) { return true; // cooking is deferred } // scan disk: all subfolders in ART (comma-separated) static array(char *) list = 0; // @leak for each_substring(ART, ",", art_folder) { array(char *) glob = file_list(va("%s**",art_folder)); // art_folder ends with '/' for( unsigned i = 0, end = array_count(glob); i < end; ++i ) { const char *fname = glob[i]; if( !strmatchi(fname, masks)) continue; // skip special files, folders and internal files like .art.zip const char *dir = file_path(fname); if( dir[0] == '.' ) continue; // discard system dirs and hidden files if( strbegi(dir, TOOLS) ) continue; // discard tools folder if( !file_ext(fname)[0] ) continue; // discard extensionless entries if( !file_size(fname)) continue; // skip dirs and empty files // exclude vc c/c++ .obj files. they're not 3d wavefront .obj files if( strend(fname, ".obj") ) { char header[4] = {0}; for( FILE *in = fopen(fname, "rb"); in; fclose(in), in = NULL) { fread(header, 2, 1, in); } if( !memcmp(header, "\x64\x86", 2) ) continue; if( !memcmp(header, "\x00\x00", 2) ) continue; } char *dot = strrchr(fname, '.'); if( dot ) { char extdot[32]; snprintf(extdot, 32, "%s.", dot); // .png -> .png. // exclude vc/gcc/clang files if( strstr(fname, ".a.o.pdb.lib.ilk.exp.dSYM.") ) // must end with dot continue; } // @todo: normalize path & rebase here (absolute to local) // [...] // fi.normalized = ; tolower->to_underscore([]();:+ )->remove_extra_underscores if (file_name(fname)[0] == '.') continue; // skip system files if (file_name(fname)[0] == ';') continue; // skip comment files array_push(list, STRDUP(fname)); } } // inspect disk for( int i = 0, end = array_count(list); i < end; ++i ) { char *fname = list[i]; struct fs fi = {0}; fi.fname = fname; // STRDUP(fname); fi.bytes = file_size(fname); fi.stamp = file_stamp10(fname); // timestamp in base10(yyyymmddhhmmss) array_push(fs_now, fi); } cook_debug = !!( flags & COOK_DEBUGLOG ); cook_cancelable = !!( flags & COOK_CANCELABLE ); // spawn all the threads int num_jobs = cook_jobs(); for( int i = 0; i < num_jobs; ++i ) { jobs[i].self = 0; jobs[i].threadid = i; jobs[i].numthreads = flags & COOK_ASYNC ? num_jobs : 1; jobs[i].files = (const char **)list; jobs[i].rules = rules; jobs[i].progress = -1; static thread_mutex_t lock; do_once thread_mutex_init(&lock); jobs[i].lock = &lock; } for( int i = 0; i < num_jobs; ++i ) { if( flags & COOK_ASYNC ) { jobs[i].self = thread_init(cook_async, &jobs[i], "cook_async()", 0/*STACK_SIZE*/); continue; } if(!cook(&jobs[i])) return false; } return true; } void cook_stop() { // join all threads int num_jobs = cook_jobs(); for( int i = 0; i < num_jobs; ++i ) { if(jobs[i].self) thread_join(jobs[i].self); } // remove all temporary outfiles for each_array(file_list("temp_*"), char*, tempfile) unlink(tempfile); } int cook_progress() { int count = 0, sum = 0; for( int i = 0, end = cook_jobs(); i < end; ++i ) { sum += jobs[i].progress; ++count; } return cook_jobs() ? sum / (count+!count) : 100; } void cook_cancel() { if( cook_cancelable ) cook_cancelling = true; } int cook_jobs() { int num_jobs = optioni("--cook-jobs", maxf(1.15,app_cores()) * 1.75), max_jobs = countof(jobs); ifdef(ems, num_jobs = 0); ifdef(retail, num_jobs = 0); ifdef(nocook, num_jobs = 0); return clampi(num_jobs, 0, max_jobs); } void cook_config( const char *pathfile_to_cook_ini ) { // @todo: test run-from-"bin/" case on Linux. COOK_INI = pathfile_to_cook_ini; ASSERT( file_exist(COOK_INI) ); } bool have_tools() { static bool found; do_once found = file_exist(COOK_INI); return ifdef(retail, false, found); }