diff --git a/engine/joint/v4k.h b/engine/joint/v4k.h index 2f53c89..4638df8 100644 --- a/engine/joint/v4k.h +++ b/engine/joint/v4k.h @@ -15418,8 +15418,8 @@ API const char * xml_string(char *key); API unsigned xml_count(char *key); API array(char) xml_blob(char *key); #define xml_string(...) xml_string(va(__VA_ARGS__)) // syntax sugar: string -#define xml_int(...) atoi(xml_string(va(__VA_ARGS__))) // syntax sugar: int -#define xml_float(...) atof(xml_string(va(__VA_ARGS__))) // syntax sugar: float +#define xml_int(...) atoi(xml_string(__VA_ARGS__)) // syntax sugar: int +#define xml_float(...) atof(xml_string(__VA_ARGS__)) // syntax sugar: float #define xml_blob(...) xml_blob(va(__VA_ARGS__)) // syntax sugar: base64 blob #define xml_count(...) xml_count(va(__VA_ARGS__)) // syntax sugar: count nodes API void xml_pop(); @@ -331929,288 +331929,322 @@ const char *COOK_INI = "tools/cook.ini"; static unsigned ART_SKIP_ROOT; // number of chars to skip the base root in ART folder static unsigned ART_LEN; // dupe -typedef struct cook_script_t { - char *infile; // free after use - char *finalfile; // free after use. can be either infile or a totally different file +typedef struct cook_subscript_t { + char *infile; + char *outfile; // can be either infile, or a totally different file char *script; + char *outname; int compress_level; +} cook_subscript_t; + +typedef struct cook_script_t { + cook_subscript_t cs[8]; + + int num_passes; } cook_script_t; static cook_script_t cook_script(const char *rules, const char *infile, const char *outfile) { - // by default, assume: - // - no script is going to be generated (empty script) - // - if no script is going to be generated, output is in fact input file. - // - no compression is going to be required. - cook_script_t cs = { 0 }; + cook_script_t mcs = { 0 }; - // reuse script heap from last call if possible (optimization) - static __thread char *script = 0; - if(script) script[0] = 0; + // pass loop: some asset rules may require multiple cook passes + for( int pass = 0; pass < countof(mcs.cs); ++pass ) { + // by default, assume: + // - no script is going to be generated (empty script) + // - if no script is going to be generated, output is in fact input file. + // - no compression is going to be required. + cook_subscript_t cs = { 0 }; - // reuse parsing maps if possible (optimization) - static __thread map(char*, char*) symbols = 0; - static __thread map(char*, char*) groups = 0; + // reuse script heap from last call if possible (optimization) + static __thread char *script = 0; + if(script) script[0] = 0; - if(!symbols) map_init(symbols, less_str, hash_str); - if(!groups) map_init(groups, less_str, hash_str); + // reuse parsing maps if possible (optimization) + static __thread map(char*, char*) symbols = 0; if(!symbols) map_init_str(symbols); + static __thread map(char*, char*) groups = 0; if(!groups) map_init_str(groups); + static __thread set(char*) passes = 0; if(!passes) set_init_str(passes); + map_clear(symbols); + map_clear(groups); - map_find_or_add(symbols, "INFILE", STRDUP(infile)); - map_find_or_add(symbols, "INPUT", STRDUP(infile)); - map_find_or_add(symbols, "PRETTY", STRDUP(infile + ART_SKIP_ROOT)); // pretty (truncated) input (C:/prj/V4K/art/file.wav -> file.wav) - map_find_or_add(symbols, "OUTPUT", STRDUP(outfile)); - map_find_or_add(symbols, "TOOLS", STRDUP(TOOLS)); - map_find_or_add(symbols, "EDITOR", STRDUP(EDITOR)); - map_find_or_add(symbols, "PROGRESS", STRDUP(va("%03d", cook_progress()))); + map_find_or_add(symbols, "INFILE", STRDUP(infile)); + map_find_or_add(symbols, "INPUT", STRDUP(infile)); + map_find_or_add(symbols, "PRETTY", STRDUP(infile + ART_SKIP_ROOT)); // pretty (truncated) input (C:/prj/V4K/art/file.wav -> file.wav) + map_find_or_add(symbols, "OUTPUT", STRDUP(outfile)); + map_find_or_add(symbols, "TOOLS", STRDUP(TOOLS)); + map_find_or_add(symbols, "EDITOR", STRDUP(EDITOR)); + map_find_or_add(symbols, "PROGRESS", STRDUP(va("%03d", cook_progress()))); - // start parsing. parsing is enabled by default - int enabled = 1; - array(char*)lines = strsplit(rules, "\r\n"); - for( int i = 0, end = array_count(lines); i < end; ++i ) { - // skip blanks - int blanks = strspn(lines[i], " \t"); - char *line = lines[i] + blanks; + // clear pass counter + set_clear(passes); - // discard full comments - if( line[0] == ';' ) continue; - // truncate inline comments - if( strstr(line, ";") ) *strstr(line, ";") = 0; - // trim ending spaces - char *eos = line + strlen(line); while(eos > line && eos[-1] == ' ' ) *--eos = 0; - // discard non-specific lines - if( line[0] == '@' ) { - int with_wine = flag("--cook-wine") && !!strstr(line, "@win"); - int parse = 0 - | ifdef(win32, (!!strstr(line, "@win")), 0) - | ifdef(linux, (!!strstr(line, "@lin") ? 1 : with_wine), 0) - | ifdef(osx, (!!strstr(line, "@osx") ? 1 : with_wine), 0); + // start parsing. parsing is enabled by default + int enabled = 1; + array(char*)lines = strsplit(rules, "\r\n"); + for( int i = 0, end = array_count(lines); i < end; ++i ) { + // skip blanks + int blanks = strspn(lines[i], " \t"); + char *line = lines[i] + blanks; - if( !parse ) continue; + // discard full comments + if( line[0] == ';' ) continue; + // truncate inline comments + if( strstr(line, ";") ) *strstr(line, ";") = 0; + // trim ending spaces + char *eos = line + strlen(line); while(eos > line && eos[-1] == ' ' ) *--eos = 0; + // discard non-specific lines + if( line[0] == '@' ) { + int with_wine = flag("--cook-wine") && !!strstr(line, "@win"); + int parse = 0 + | ifdef(win32, (!!strstr(line, "@win")), 0) + | ifdef(linux, (!!strstr(line, "@lin") ? 1 : with_wine), 0) + | ifdef(osx, (!!strstr(line, "@osx") ? 1 : with_wine), 0); - line = strchr(line+1, ' '); - if(!line) continue; - line += strspn(line, " \t"); - } - // execute `shell` commands - if( line[0] == '`' ) { - char *eos = strrchr(++line, '`'); - if( eos ) *eos = 0; + if( !parse ) continue; - // replace all symbols - char* nl = STRDUP(line); - for each_map(symbols, char*, key, char*, val) { - strrepl(&nl, key, val); + line = strchr(line+1, ' '); + if(!line) continue; + line += strspn(line, " \t"); + } + // execute `shell` commands + if( line[0] == '`' ) { + char *eos = strrchr(++line, '`'); + if( eos ) *eos = 0; + + // replace all symbols + char* nl = STRDUP(line); // @leak + for each_map(symbols, char*, key, char*, val) { + strrepl(&nl, key, val); + } + lines[i] = line = nl; + + static thread_mutex_t lock, *init = 0; if(!init) thread_mutex_init(init = &lock); + thread_mutex_lock( &lock ); + system(line); // strcatf(&script, "%s\n", line); + thread_mutex_unlock( &lock ); + + continue; + } + // process [sections] + if( line[0] == '[' ) { + enabled = 1; + int is_cook = !!strstr(line, "[cook]"); + int is_compress = !!strstr(line, "[compress]"); + if( !is_cook && !is_compress ) { // if not a special section... + // remove hint cook tag if present. that's informative only. + if(strbegi(line, "[cook ") ) memcpy(line+1, " ", 4); // line += 6; + + // start parsing expressions like `[media && !avi && mp3]` + array(char*) tags = strsplit(line, " []&"); + + // let's check whether INPUT belongs to tags above + char **INPUT = map_find(symbols, "INPUT"); + bool found_in_set = true; + + for( int i = 0, end = array_count(tags); i < end; ++i) { + bool negate = false; + char *tag = tags[i]; + while(*tag == '!') negate ^= 1, ++tag; + + // find tag in groups map + // either a group or an extension + char **is_group = map_find(groups, tag); + if( is_group ) { + char *list = *is_group; + char *INPUT_EXT = file_ext(infile); INPUT_EXT = strrchr(INPUT_EXT, '.'); // .ext1.ext -> .ext + char *ext = INPUT_EXT; ext += ext[0] == '.'; // dotless + bool in_list = strbegi(list, ext) || strendi(list, va(",%s",ext)) || strstri(list, va(",%s,",ext)); + if( !in_list ^ negate ) { found_in_set = false; break; } + } else { + char *ext = va(".%s", tag); + bool found = !!strendi(*INPUT, ext); + if( !found ^ negate ) { found_in_set = false; break; } + } + } + if( found_in_set ) { + // inc pass + set_find_or_add(passes, STRDUP(*tags)); // @leak + // check whether we keep searching + int num_passes = set_count(passes); + found_in_set = ( pass == (num_passes-1) ); + } + // + enabled = found_in_set ? 1 : 0; } - lines[i] = line = nl; // @fixme:leak - - static thread_mutex_t lock, *init = 0; if(!init) thread_mutex_init(init = &lock); - thread_mutex_lock( &lock ); - system(line); // strcatf(&script, "%s\n", line); - thread_mutex_unlock( &lock ); - - continue; - } - // process [sections] - if( line[0] == '[' ) { - enabled = 1; - int is_cook = !!strstr(line, "[cook]"); - int is_compress = !!strstr(line, "[compress]"); - if( !is_cook && !is_compress ) { - // remove hint cook tag if present. that's informative only. - if(strbegi(line, "[cook ") ) memcpy(line+1, " ", 4); - - // start parsing expressions like `[media && !avi && mp3]` - array(char*) tags = strsplit(line, " []&"); - - // let's check whether INPUT belongs to tags above - char **INPUT = map_find(symbols, "INPUT"); - bool found_in_set = true; - - for( int i = 0, end = array_count(tags); i < end; ++i) { char *tag = tags[i]; - bool negate = false; - while(*tag == '!') negate ^= 1, ++tag; - - // find tag in groups map - // either a group or an extension - char **is_group = map_find(groups, tag); - if( is_group ) { - char *list = *is_group; - char *INPUT_EXT = file_ext(infile); INPUT_EXT = strrchr(INPUT_EXT, '.'); // .ext1.ext -> .ext - char *ext = INPUT_EXT; ext += ext[0] == '.'; // dotless - bool in_list = strbegi(list, ext) || strstri(list, va(",%s,",ext)) || strendi(list, va(",%s",ext)); - if( !in_list ^ negate ) { found_in_set = false; break; } - } else { - char *ext = va(".%s", tag); - bool found = !!strendi(*INPUT, ext); - if( !found ^ negate ) { found_in_set = false; break; } + } + // either SYMBOL=, group=, or regular script line + if( enabled && line[0] != '[' ) { + enum { group, symbol, regular } type = regular; + int tokenlen = strspn(line, "-+_.|0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"); + char *token = va("%.*s", tokenlen, line); + char *equal = strchr(line, '='); + if( equal ) { + if( equal == &line[tokenlen] ) { // if key=value expression found + // discriminate: symbols are uppercase and never begin with digits. groups are [0-9]+[|][a-z]. + type = strcmp(strupper(token), token) || isdigit(token[0]) ? group : symbol; } } - // - enabled = found_in_set ? 1 : 0; - } - } - // either SYMBOL=, group=, or regular script line - if( enabled && line[0] != '[' ) { - enum { group, symbol, regular } type = regular; - int tokenlen = strspn(line, "-+_.|0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"); - char *token = va("%.*s", tokenlen, line); - char *equal = strchr(line, '='); - if( equal ) { - if( equal == &line[tokenlen] ) { // if key=value expression found - // discriminate: symbols are uppercase and never begin with digits. groups are [0-9]+[|][a-z]. - type = strcmp(strupper(token), token) || isdigit(token[0]) ? group : symbol; + if( type == group ) map_find_or_add(groups, token, STRDUP(equal+1)); + if( type == symbol ) { + // @todo: perform the replacement/union/intersection on set here + bool is_add = strendi(token, "+"); + bool is_del = strendi(token, "-"); + + // if present, remove last sign from token -> (FLAGS1+)=, (FLAGS1-)= + if(is_add || is_del) token[strlen(token) - 1] = 0; + + map_find_or_add(symbols, token, STRDUP(equal+1)); } - } - if( type == group ) map_find_or_add(groups, token, STRDUP(equal+1)); - if( type == symbol ) { - // @todo: perform the replacement/union/intersection on set here - bool is_add = strendi(token, "+"); - bool is_del = strendi(token, "-"); + // for each_map(symbols, char*, key, char*, val) printf("%s=%s,", key, val); puts(""); + // for each_map(groups, char*, key, char*, val) printf("%s=%s,", key, val); puts(""); + // if( type != regular ) printf("%s found >> %s\n", type == group ? "group" : "symbol", line); - // if present, remove last sign from token -> (FLAGS1+)=, (FLAGS1-)= - if(is_add || is_del) token[strlen(token) - 1] = 0; + if( type == regular ) { + char** INPUT = map_find(symbols, "INPUT"); + char** OUTPUT = map_find(symbols, "OUTPUT"); - map_find_or_add(symbols, token, STRDUP(equal+1)); - } - // for each_map(symbols, char*, key, char*, val) printf("%s=%s,", key, val); puts(""); - // for each_map(groups, char*, key, char*, val) printf("%s=%s,", key, val); puts(""); - // if( type != regular ) printf("%s found >> %s\n", type == group ? "group" : "symbol", line); + // parse return code + char *has_errorlevel = strstr(line, "=="); //==N form + int errorlevel = has_errorlevel ? atoi(has_errorlevel + 2) : 0; + if( has_errorlevel ) memcpy(has_errorlevel, " ", 3); - if( type == regular ) { - char** INPUT = map_find(symbols, "INPUT"); - char** OUTPUT = map_find(symbols, "OUTPUT"); + // detect if newer extension or filename is present, and thus update OUTPUT if needed + char *newer_extension = strstr(line, "->"); if(newer_extension) { + *newer_extension = 0; + newer_extension += 2 + strspn(newer_extension + 2, " "); - // parse return code - char *has_errorlevel = strstr(line, "=="); //==N form - int errorlevel = has_errorlevel ? atoi(has_errorlevel + 2) : 0; - if( has_errorlevel ) memcpy(has_errorlevel, " ", 3); + if( strchr(newer_extension, '.') ) { + // newer filename + cs.outname = stringf("%s@%s", cs.outname ? cs.outname : infile, newer_extension); // @leak + newer_extension = NULL; + } else { + strcatf(&*OUTPUT, ".%s", newer_extension); + } + } - // detect if newer extension is present, and thus update OUTPUT if needed - char *newer_extension = strstr(line, "->"); if(newer_extension) { - *newer_extension = 0; - newer_extension += 2 + strspn(newer_extension + 2, " "); + // replace all symbols + char* nl = STRDUP(line); // @leak + for each_map(symbols, char*, key, char*, val) { + strrepl(&nl, key, val); + } + lines[i] = line = nl; - strcatf(&*OUTPUT, ".%s", newer_extension); - } + // convert slashes + ifdef(win32, + strswap(line, "/", "\\") + , // else + strswap(line, "\\", "/") + ); - // replace all symbols - char* nl = STRDUP(line); - for each_map(symbols, char*, key, char*, val) { - strrepl(&nl, key, val); - } - lines[i] = line = nl; // @fixme:leak + // append line + strcatf(&script, "%s\n", line); - // convert slashes - ifdef(win32, - strswap(line, "/", "\\") - , // else - strswap(line, "\\", "/") - ); + // handle return code here + // if(has_errorlevel) + // strcatf(&script, "IF NOT '%%ERRORLEVEL%%'=='%d' echo ERROR!\n", errorlevel); - // append line - strcatf(&script, "%s\n", line); - - // handle return code here - // if(has_errorlevel) - // strcatf(&script, "IF NOT '%%ERRORLEVEL%%'=='%d' echo ERROR!\n", errorlevel); - - // rename output->input for further chaining, in case it is needed - if( newer_extension ) { - *INPUT[0] = 0; - strcatf(&*INPUT, "%s", *OUTPUT); + // rename output->input for further chaining, in case it is needed + if( newer_extension ) { + *INPUT[0] = 0; + strcatf(&*INPUT, "%s", *OUTPUT); + } } } } - } - // compression - char* ext = file_ext(infile); ext = strrchr(ext, '.'); ext += ext[0] == '.'; // dotless INPUT_EXT + char** OUTPUT = map_find(symbols, "OUTPUT"); + int ext_num_groups = 0; - char** OUTPUT = map_find(symbols, "OUTPUT"); - char* belongs_to = 0; - for each_map(groups, char*, key, char*, val) { - if( !isdigit(key[0]) ) { - char *comma = va(",%s,", ext); - if( strbegi(val, comma+1) || strstri(val, comma) || strendi(val, va(",%s", ext))) { - belongs_to = key; - //goto break1; // each_map() macro is made of multiple for(;;)s. goto needed; you cant escape with single break. + // compression + if( 1 ) { + char* ext = file_ext(infile); ext = strrchr(ext, '.'); ext += ext[0] == '.'; // dotless INPUT_EXT + char* belongs_to = 0; + for each_map(groups, char*, key, char*, val) { + if( !isdigit(key[0]) ) { + char *comma = va(",%s,", ext); + if( !strcmpi(val,ext) || strbegi(val, comma+1) || strstri(val, comma) || strendi(val, va(",%s", ext))) { + belongs_to = key; + ext_num_groups++; + } + } + } + char *compression = 0; + for each_map(groups, char*, key, char*, val) { + if( isdigit(key[0]) ) { + char *comma = va(",%s,", ext); + if( !strcmpi(val,ext) || strbegi(val, comma+1) || strstri(val, comma) || strendi(val, va(",%s", ext))) { + compression = key; + } + comma = va(",%s,", belongs_to); + if( !strcmpi(val,ext) || strbegi(val, comma+1) || strstri(val, comma) || strendi(val, va(",%s", ext))) { + compression = key; + } + } + } + + cs.compress_level = 0; + if( compression ) { + // last chance to optionally override the compressor at command-line level + static const char *compressor_override; + do_once compressor_override = option("--cook-compressor", ""); + if( compressor_override[0] ) compression = (char*)compressor_override; + + /**/ if(strstri(compression, "PPP")) cs.compress_level = atoi(compression) | PPP; + else if(strstri(compression, "ULZ")) cs.compress_level = atoi(compression) | ULZ; + else if(strstri(compression, "LZ4")) cs.compress_level = atoi(compression) | LZ4X; + else if(strstri(compression, "CRSH")) cs.compress_level = atoi(compression) | CRSH; + else if(strstri(compression, "DEFL")) cs.compress_level = isdigit(compression[0]) ? atoi(compression) : 6 /*| DEFL*/; + //else if(strstri(compression, "LZP")) cs.compress_level = atoi(compression) | LZP1; // not supported + else if(strstri(compression, "LZMA")) cs.compress_level = atoi(compression) | LZMA; + else if(strstri(compression, "BALZ")) cs.compress_level = atoi(compression) | BALZ; + else if(strstri(compression, "LZW")) cs.compress_level = atoi(compression) | LZW3; + else if(strstri(compression, "LZSS")) cs.compress_level = atoi(compression) | LZSS; + else if(strstri(compression, "BCM")) cs.compress_level = atoi(compression) | BCM; + else cs.compress_level = isdigit(compression[0]) ? atoi(compression) : 6 /*| DEFL*/; } } - } - break1:; - char *compression = 0; - for each_map(groups, char*, key, char*, val) { - if( isdigit(key[0]) ) { - char *comma = va(",%s,", ext); - if( strbegi(val, comma+1) || strstri(val, comma) || strendi(val, va(",%s", ext))) { - compression = key; - //goto break2; // each_map() macro is made of multiple for(;;)s. goto needed; you cant escape with single break. - } - comma = va(",%s,", belongs_to); - if( strbegi(val, comma+1) || strstri(val, comma) || strendi(val, va(",%s", ext))) { - compression = key; - //goto break2; // each_map() macro is made of multiple for(;;)s. goto needed; you cant escape with single break. - } + + // if script was generated... + if( script && script[0]) { + // update outfile + cs.outfile = *OUTPUT; + + // amalgamate script + array(char*) lines = strsplit(script, "\r\n"); + + #if is(win32) + char *joint = strjoin(lines, " && "); + cs.script = joint; + #else + if( flag("--cook-wine") ) { + // dear linux/osx/bsd users: + // tools going wrong for any reason? cant compile them maybe? + // small hack to use win32 pipeline tools instead + char *joint = strjoin(lines, " && wine " ); + cs.script = va("wine %s", /*TOOLS,*/ joint); + } else { + char *joint = strjoin(lines, " && " ); + cs.script = va("export LD_LIBRARY_PATH=%s && %s", TOOLS, joint); + } + #endif + } else { + // ... else bypass infile->outfile + char** INFILE = map_find(symbols, "INFILE"); + cs.outfile = *INFILE; + + // and return an empty script + cs.script = ""; } - } - break2:; - cs.compress_level = 0; - if( compression ) { - // last chance to optionally override the compressor at command-line level - static const char *compressor_override, **init = 0; - if( !init ) *(init = &compressor_override) = option("--cook-compressor", ""); - if( compressor_override[0] ) compression = (char*)compressor_override; + cs.outname = cs.outname ? cs.outname : (char*)infile; - /**/ if(strstri(compression, "PPP")) cs.compress_level = atoi(compression) | PPP; - else if(strstri(compression, "ULZ")) cs.compress_level = atoi(compression) | ULZ; - else if(strstri(compression, "LZ4")) cs.compress_level = atoi(compression) | LZ4X; - else if(strstri(compression, "CRSH")) cs.compress_level = atoi(compression) | CRSH; - else if(strstri(compression, "DEFL")) cs.compress_level = isdigit(compression[0]) ? atoi(compression) : 6 /*| DEFL*/; - //else if(strstri(compression, "LZP")) cs.compress_level = atoi(compression) | LZP1; // not supported - else if(strstri(compression, "LZMA")) cs.compress_level = atoi(compression) | LZMA; - else if(strstri(compression, "BALZ")) cs.compress_level = atoi(compression) | BALZ; - else if(strstri(compression, "LZW")) cs.compress_level = atoi(compression) | LZW3; - else if(strstri(compression, "LZSS")) cs.compress_level = atoi(compression) | LZSS; - else if(strstri(compression, "BCM")) cs.compress_level = atoi(compression) | BCM; - else cs.compress_level = isdigit(compression[0]) ? atoi(compression) : 6 /*| DEFL*/; + ASSERT(mcs.num_passes < countof(mcs.cs)); + mcs.cs[mcs.num_passes++] = cs; + + bool next_pass_required = mcs.num_passes < ext_num_groups; + if( !next_pass_required ) break; } - // if script was generated... - if( script && script[0]) { - // update outfile - cs.finalfile = *OUTPUT; - - // amalgamate script - array(char*) lines = strsplit(script, "\r\n"); - - #if is(win32) - char *joint = strjoin(lines, " && "); - cs.script = joint; - #else - if( flag("--cook-wine") ) { - // dear linux/osx/bsd users: - // tools going wrong for any reason? cant compile them maybe? - // small hack to use win32 pipeline tools instead - char *joint = strjoin(lines, " && wine " ); - cs.script = va("wine %s", /*TOOLS,*/ joint); - } else { - char *joint = strjoin(lines, " && " ); - cs.script = va("export LD_LIBRARY_PATH=%s && %s", TOOLS, joint); - } - #endif - } else { - // ... else bypass infile->outfile - char** INFILE = map_find(symbols, "INFILE"); - cs.finalfile = *INFILE; - - // and return an empty script - cs.script = ""; - } - - map_clear(symbols); - map_clear(groups); - return cs; + return mcs; } // ---------------------------------------------------------------------------- @@ -332372,55 +332406,52 @@ int cook(void *userdata) { *progress = ((i+1) == end ? 90 : (i * 90) / end); // (i+i>0) * 100.f / end; // start cook - const char *fname = uncooked[i]; //job->files[j]; - int inlen = file_size(fname); + const char *infile = uncooked[i]; //job->files[j]; + int inlen = file_size(infile); // generate a cooking script for this asset - cook_script_t cs = cook_script(job->rules, fname, COOK_TMPFILE); + cook_script_t mcs = cook_script(job->rules, infile, COOK_TMPFILE); // puts(cs.script); - // log to batch file for forensic purposes, if explicitly requested - static __thread bool logging = 0, *init = 0; if(!init) *(init = &logging) = !!flag("--cook-debug") || cook_debug; - if( logging ) { - FILE *logfile = fopen(va("cook%d.cmd",job->threadid), "a+t"); - if( logfile ) { fprintf(logfile, "@rem %s\n%s\n", fname, cs.script); fclose(logfile); } - // maybe log fprintf(logfile, "@rem %*.s\n", 4096, app_exec_output()); ? - } + for(int pass = 0; pass < mcs.num_passes; ++pass) { + cook_subscript_t cs = mcs.cs[pass]; - // invoke cooking script and recap status - const char *rcout = app_exec(cs.script); - int rc = atoi(rcout); - int outlen = file_size(cs.finalfile); - int failed = cs.script[0] ? rc || !outlen : 0; + // log to batch file for forensic purposes, if explicitly requested + static __thread bool logging = 0; do_once logging = !!flag("--cook-debug") || cook_debug; + if( logging ) { + FILE *logfile = fopen(va("cook%d.cmd",job->threadid), "a+t"); + if( logfile ) { fprintf(logfile, "@rem %s\n%s\n", cs.outname, cs.script); fclose(logfile); } + fprintf(stderr, "%s\n", cs.script); + } - // print errors, or... - if( failed ) { - PRINTF("Import failed: %s while executing:\n%s\nReturned:\n%s\n", fname, cs.script, rcout); - } - // ...process only if included. may include optional compression. - else if( cs.compress_level >= 0 ) { - FILE *in = fopen(cs.finalfile, "rb"); + // invoke cooking script and recap status + const char *rc_output = app_exec(cs.script); + int rc = atoi(rc_output); + int outlen = file_size(cs.outfile); + int failed = cs.script[0] ? rc || !outlen : 0; -#if 0 - struct stat st; stat(fname, &st); - struct tm *timeinfo = localtime(&st.st_mtime); - ASSERT(timeinfo); + // print errors, or... + if( failed ) { + PRINTF("Import failed: %s while executing:\n%s\nReturned:\n%s\n", cs.outname, cs.script, rc_output); + } + // ...process only if included. may include optional compression. + else if( cs.compress_level >= 0 ) { + FILE *in = fopen(cs.outfile, "rb"); - // pretty (truncated) input (C:/prj/V4K/art/file.wav -> file.wav) - static __thread int artlen = 0; if(!artlen) artlen = strlen(ART); - const char *pretty = fname; - if( !strncmp(pretty, ART, artlen) ) pretty += artlen; - while(pretty[0] == '/') ++pretty; - fname = pretty; - //puts(fname); -#endif + #if 0 + struct stat st; stat(infile, &st); + struct tm *timeinfo = localtime(&st.st_mtime); + ASSERT(timeinfo); + #endif - char *comment = va("%d", inlen); - if( !zip_append_file/*_timeinfo*/(z, fname, comment, in, cs.compress_level/*, timeinfo*/) ) { - PANIC("failed to add processed file into %s: %s", zipfile, fname); - } + char *comment = va("%d", inlen); + if( !zip_append_file/*_timeinfo*/(z, cs.outname, comment, in, cs.compress_level/*, timeinfo*/) ) { + PANIC("failed to add processed file into %s: %s(%s)", zipfile, cs.outname, infile); + } + + fclose(in); + } - fclose(in); } } diff --git a/engine/split/v4k_cooker.c b/engine/split/v4k_cooker.c index dd70999..79761ac 100644 --- a/engine/split/v4k_cooker.c +++ b/engine/split/v4k_cooker.c @@ -17,288 +17,322 @@ const char *COOK_INI = "tools/cook.ini"; static unsigned ART_SKIP_ROOT; // number of chars to skip the base root in ART folder static unsigned ART_LEN; // dupe -typedef struct cook_script_t { - char *infile; // free after use - char *finalfile; // free after use. can be either infile or a totally different file +typedef struct cook_subscript_t { + char *infile; + char *outfile; // can be either infile, or a totally different file char *script; + char *outname; int compress_level; +} cook_subscript_t; + +typedef struct cook_script_t { + cook_subscript_t cs[8]; + + int num_passes; } cook_script_t; static cook_script_t cook_script(const char *rules, const char *infile, const char *outfile) { - // by default, assume: - // - no script is going to be generated (empty script) - // - if no script is going to be generated, output is in fact input file. - // - no compression is going to be required. - cook_script_t cs = { 0 }; + cook_script_t mcs = { 0 }; - // reuse script heap from last call if possible (optimization) - static __thread char *script = 0; - if(script) script[0] = 0; + // pass loop: some asset rules may require multiple cook passes + for( int pass = 0; pass < countof(mcs.cs); ++pass ) { + // by default, assume: + // - no script is going to be generated (empty script) + // - if no script is going to be generated, output is in fact input file. + // - no compression is going to be required. + cook_subscript_t cs = { 0 }; - // reuse parsing maps if possible (optimization) - static __thread map(char*, char*) symbols = 0; - static __thread map(char*, char*) groups = 0; + // reuse script heap from last call if possible (optimization) + static __thread char *script = 0; + if(script) script[0] = 0; - if(!symbols) map_init(symbols, less_str, hash_str); - if(!groups) map_init(groups, less_str, hash_str); + // reuse parsing maps if possible (optimization) + static __thread map(char*, char*) symbols = 0; if(!symbols) map_init_str(symbols); + static __thread map(char*, char*) groups = 0; if(!groups) map_init_str(groups); + static __thread set(char*) passes = 0; if(!passes) set_init_str(passes); + map_clear(symbols); + map_clear(groups); - map_find_or_add(symbols, "INFILE", STRDUP(infile)); - map_find_or_add(symbols, "INPUT", STRDUP(infile)); - map_find_or_add(symbols, "PRETTY", STRDUP(infile + ART_SKIP_ROOT)); // pretty (truncated) input (C:/prj/V4K/art/file.wav -> file.wav) - map_find_or_add(symbols, "OUTPUT", STRDUP(outfile)); - map_find_or_add(symbols, "TOOLS", STRDUP(TOOLS)); - map_find_or_add(symbols, "EDITOR", STRDUP(EDITOR)); - map_find_or_add(symbols, "PROGRESS", STRDUP(va("%03d", cook_progress()))); + map_find_or_add(symbols, "INFILE", STRDUP(infile)); + map_find_or_add(symbols, "INPUT", STRDUP(infile)); + map_find_or_add(symbols, "PRETTY", STRDUP(infile + ART_SKIP_ROOT)); // pretty (truncated) input (C:/prj/V4K/art/file.wav -> file.wav) + map_find_or_add(symbols, "OUTPUT", STRDUP(outfile)); + map_find_or_add(symbols, "TOOLS", STRDUP(TOOLS)); + map_find_or_add(symbols, "EDITOR", STRDUP(EDITOR)); + map_find_or_add(symbols, "PROGRESS", STRDUP(va("%03d", cook_progress()))); - // start parsing. parsing is enabled by default - int enabled = 1; - array(char*)lines = strsplit(rules, "\r\n"); - for( int i = 0, end = array_count(lines); i < end; ++i ) { - // skip blanks - int blanks = strspn(lines[i], " \t"); - char *line = lines[i] + blanks; + // clear pass counter + set_clear(passes); - // discard full comments - if( line[0] == ';' ) continue; - // truncate inline comments - if( strstr(line, ";") ) *strstr(line, ";") = 0; - // trim ending spaces - char *eos = line + strlen(line); while(eos > line && eos[-1] == ' ' ) *--eos = 0; - // discard non-specific lines - if( line[0] == '@' ) { - int with_wine = flag("--cook-wine") && !!strstr(line, "@win"); - int parse = 0 - | ifdef(win32, (!!strstr(line, "@win")), 0) - | ifdef(linux, (!!strstr(line, "@lin") ? 1 : with_wine), 0) - | ifdef(osx, (!!strstr(line, "@osx") ? 1 : with_wine), 0); + // start parsing. parsing is enabled by default + int enabled = 1; + array(char*)lines = strsplit(rules, "\r\n"); + for( int i = 0, end = array_count(lines); i < end; ++i ) { + // skip blanks + int blanks = strspn(lines[i], " \t"); + char *line = lines[i] + blanks; - if( !parse ) continue; + // discard full comments + if( line[0] == ';' ) continue; + // truncate inline comments + if( strstr(line, ";") ) *strstr(line, ";") = 0; + // trim ending spaces + char *eos = line + strlen(line); while(eos > line && eos[-1] == ' ' ) *--eos = 0; + // discard non-specific lines + if( line[0] == '@' ) { + int with_wine = flag("--cook-wine") && !!strstr(line, "@win"); + int parse = 0 + | ifdef(win32, (!!strstr(line, "@win")), 0) + | ifdef(linux, (!!strstr(line, "@lin") ? 1 : with_wine), 0) + | ifdef(osx, (!!strstr(line, "@osx") ? 1 : with_wine), 0); - line = strchr(line+1, ' '); - if(!line) continue; - line += strspn(line, " \t"); - } - // execute `shell` commands - if( line[0] == '`' ) { - char *eos = strrchr(++line, '`'); - if( eos ) *eos = 0; + if( !parse ) continue; - // replace all symbols - char* nl = STRDUP(line); - for each_map(symbols, char*, key, char*, val) { - strrepl(&nl, key, val); + line = strchr(line+1, ' '); + if(!line) continue; + line += strspn(line, " \t"); + } + // execute `shell` commands + if( line[0] == '`' ) { + char *eos = strrchr(++line, '`'); + if( eos ) *eos = 0; + + // replace all symbols + char* nl = STRDUP(line); // @leak + for each_map(symbols, char*, key, char*, val) { + strrepl(&nl, key, val); + } + lines[i] = line = nl; + + static thread_mutex_t lock, *init = 0; if(!init) thread_mutex_init(init = &lock); + thread_mutex_lock( &lock ); + system(line); // strcatf(&script, "%s\n", line); + thread_mutex_unlock( &lock ); + + continue; + } + // process [sections] + if( line[0] == '[' ) { + enabled = 1; + int is_cook = !!strstr(line, "[cook]"); + int is_compress = !!strstr(line, "[compress]"); + if( !is_cook && !is_compress ) { // if not a special section... + // remove hint cook tag if present. that's informative only. + if(strbegi(line, "[cook ") ) memcpy(line+1, " ", 4); // line += 6; + + // start parsing expressions like `[media && !avi && mp3]` + array(char*) tags = strsplit(line, " []&"); + + // let's check whether INPUT belongs to tags above + char **INPUT = map_find(symbols, "INPUT"); + bool found_in_set = true; + + for( int i = 0, end = array_count(tags); i < end; ++i) { + bool negate = false; + char *tag = tags[i]; + while(*tag == '!') negate ^= 1, ++tag; + + // find tag in groups map + // either a group or an extension + char **is_group = map_find(groups, tag); + if( is_group ) { + char *list = *is_group; + char *INPUT_EXT = file_ext(infile); INPUT_EXT = strrchr(INPUT_EXT, '.'); // .ext1.ext -> .ext + char *ext = INPUT_EXT; ext += ext[0] == '.'; // dotless + bool in_list = strbegi(list, ext) || strendi(list, va(",%s",ext)) || strstri(list, va(",%s,",ext)); + if( !in_list ^ negate ) { found_in_set = false; break; } + } else { + char *ext = va(".%s", tag); + bool found = !!strendi(*INPUT, ext); + if( !found ^ negate ) { found_in_set = false; break; } + } + } + if( found_in_set ) { + // inc pass + set_find_or_add(passes, STRDUP(*tags)); // @leak + // check whether we keep searching + int num_passes = set_count(passes); + found_in_set = ( pass == (num_passes-1) ); + } + // + enabled = found_in_set ? 1 : 0; } - lines[i] = line = nl; // @fixme:leak - - static thread_mutex_t lock, *init = 0; if(!init) thread_mutex_init(init = &lock); - thread_mutex_lock( &lock ); - system(line); // strcatf(&script, "%s\n", line); - thread_mutex_unlock( &lock ); - - continue; - } - // process [sections] - if( line[0] == '[' ) { - enabled = 1; - int is_cook = !!strstr(line, "[cook]"); - int is_compress = !!strstr(line, "[compress]"); - if( !is_cook && !is_compress ) { - // remove hint cook tag if present. that's informative only. - if(strbegi(line, "[cook ") ) memcpy(line+1, " ", 4); - - // start parsing expressions like `[media && !avi && mp3]` - array(char*) tags = strsplit(line, " []&"); - - // let's check whether INPUT belongs to tags above - char **INPUT = map_find(symbols, "INPUT"); - bool found_in_set = true; - - for( int i = 0, end = array_count(tags); i < end; ++i) { char *tag = tags[i]; - bool negate = false; - while(*tag == '!') negate ^= 1, ++tag; - - // find tag in groups map - // either a group or an extension - char **is_group = map_find(groups, tag); - if( is_group ) { - char *list = *is_group; - char *INPUT_EXT = file_ext(infile); INPUT_EXT = strrchr(INPUT_EXT, '.'); // .ext1.ext -> .ext - char *ext = INPUT_EXT; ext += ext[0] == '.'; // dotless - bool in_list = strbegi(list, ext) || strstri(list, va(",%s,",ext)) || strendi(list, va(",%s",ext)); - if( !in_list ^ negate ) { found_in_set = false; break; } - } else { - char *ext = va(".%s", tag); - bool found = !!strendi(*INPUT, ext); - if( !found ^ negate ) { found_in_set = false; break; } + } + // either SYMBOL=, group=, or regular script line + if( enabled && line[0] != '[' ) { + enum { group, symbol, regular } type = regular; + int tokenlen = strspn(line, "-+_.|0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"); + char *token = va("%.*s", tokenlen, line); + char *equal = strchr(line, '='); + if( equal ) { + if( equal == &line[tokenlen] ) { // if key=value expression found + // discriminate: symbols are uppercase and never begin with digits. groups are [0-9]+[|][a-z]. + type = strcmp(strupper(token), token) || isdigit(token[0]) ? group : symbol; } } - // - enabled = found_in_set ? 1 : 0; - } - } - // either SYMBOL=, group=, or regular script line - if( enabled && line[0] != '[' ) { - enum { group, symbol, regular } type = regular; - int tokenlen = strspn(line, "-+_.|0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"); - char *token = va("%.*s", tokenlen, line); - char *equal = strchr(line, '='); - if( equal ) { - if( equal == &line[tokenlen] ) { // if key=value expression found - // discriminate: symbols are uppercase and never begin with digits. groups are [0-9]+[|][a-z]. - type = strcmp(strupper(token), token) || isdigit(token[0]) ? group : symbol; + if( type == group ) map_find_or_add(groups, token, STRDUP(equal+1)); + if( type == symbol ) { + // @todo: perform the replacement/union/intersection on set here + bool is_add = strendi(token, "+"); + bool is_del = strendi(token, "-"); + + // if present, remove last sign from token -> (FLAGS1+)=, (FLAGS1-)= + if(is_add || is_del) token[strlen(token) - 1] = 0; + + map_find_or_add(symbols, token, STRDUP(equal+1)); } - } - if( type == group ) map_find_or_add(groups, token, STRDUP(equal+1)); - if( type == symbol ) { - // @todo: perform the replacement/union/intersection on set here - bool is_add = strendi(token, "+"); - bool is_del = strendi(token, "-"); + // for each_map(symbols, char*, key, char*, val) printf("%s=%s,", key, val); puts(""); + // for each_map(groups, char*, key, char*, val) printf("%s=%s,", key, val); puts(""); + // if( type != regular ) printf("%s found >> %s\n", type == group ? "group" : "symbol", line); - // if present, remove last sign from token -> (FLAGS1+)=, (FLAGS1-)= - if(is_add || is_del) token[strlen(token) - 1] = 0; + if( type == regular ) { + char** INPUT = map_find(symbols, "INPUT"); + char** OUTPUT = map_find(symbols, "OUTPUT"); - map_find_or_add(symbols, token, STRDUP(equal+1)); - } - // for each_map(symbols, char*, key, char*, val) printf("%s=%s,", key, val); puts(""); - // for each_map(groups, char*, key, char*, val) printf("%s=%s,", key, val); puts(""); - // if( type != regular ) printf("%s found >> %s\n", type == group ? "group" : "symbol", line); + // parse return code + char *has_errorlevel = strstr(line, "=="); //==N form + int errorlevel = has_errorlevel ? atoi(has_errorlevel + 2) : 0; + if( has_errorlevel ) memcpy(has_errorlevel, " ", 3); - if( type == regular ) { - char** INPUT = map_find(symbols, "INPUT"); - char** OUTPUT = map_find(symbols, "OUTPUT"); + // detect if newer extension or filename is present, and thus update OUTPUT if needed + char *newer_extension = strstr(line, "->"); if(newer_extension) { + *newer_extension = 0; + newer_extension += 2 + strspn(newer_extension + 2, " "); - // parse return code - char *has_errorlevel = strstr(line, "=="); //==N form - int errorlevel = has_errorlevel ? atoi(has_errorlevel + 2) : 0; - if( has_errorlevel ) memcpy(has_errorlevel, " ", 3); + if( strchr(newer_extension, '.') ) { + // newer filename + cs.outname = stringf("%s@%s", cs.outname ? cs.outname : infile, newer_extension); // @leak + newer_extension = NULL; + } else { + strcatf(&*OUTPUT, ".%s", newer_extension); + } + } - // detect if newer extension is present, and thus update OUTPUT if needed - char *newer_extension = strstr(line, "->"); if(newer_extension) { - *newer_extension = 0; - newer_extension += 2 + strspn(newer_extension + 2, " "); + // replace all symbols + char* nl = STRDUP(line); // @leak + for each_map(symbols, char*, key, char*, val) { + strrepl(&nl, key, val); + } + lines[i] = line = nl; - strcatf(&*OUTPUT, ".%s", newer_extension); - } + // convert slashes + ifdef(win32, + strswap(line, "/", "\\") + , // else + strswap(line, "\\", "/") + ); - // replace all symbols - char* nl = STRDUP(line); - for each_map(symbols, char*, key, char*, val) { - strrepl(&nl, key, val); - } - lines[i] = line = nl; // @fixme:leak + // append line + strcatf(&script, "%s\n", line); - // convert slashes - ifdef(win32, - strswap(line, "/", "\\") - , // else - strswap(line, "\\", "/") - ); + // handle return code here + // if(has_errorlevel) + // strcatf(&script, "IF NOT '%%ERRORLEVEL%%'=='%d' echo ERROR!\n", errorlevel); - // append line - strcatf(&script, "%s\n", line); - - // handle return code here - // if(has_errorlevel) - // strcatf(&script, "IF NOT '%%ERRORLEVEL%%'=='%d' echo ERROR!\n", errorlevel); - - // rename output->input for further chaining, in case it is needed - if( newer_extension ) { - *INPUT[0] = 0; - strcatf(&*INPUT, "%s", *OUTPUT); + // rename output->input for further chaining, in case it is needed + if( newer_extension ) { + *INPUT[0] = 0; + strcatf(&*INPUT, "%s", *OUTPUT); + } } } } - } - // compression - char* ext = file_ext(infile); ext = strrchr(ext, '.'); ext += ext[0] == '.'; // dotless INPUT_EXT + char** OUTPUT = map_find(symbols, "OUTPUT"); + int ext_num_groups = 0; - char** OUTPUT = map_find(symbols, "OUTPUT"); - char* belongs_to = 0; - for each_map(groups, char*, key, char*, val) { - if( !isdigit(key[0]) ) { - char *comma = va(",%s,", ext); - if( strbegi(val, comma+1) || strstri(val, comma) || strendi(val, va(",%s", ext))) { - belongs_to = key; - //goto break1; // each_map() macro is made of multiple for(;;)s. goto needed; you cant escape with single break. + // compression + if( 1 ) { + char* ext = file_ext(infile); ext = strrchr(ext, '.'); ext += ext[0] == '.'; // dotless INPUT_EXT + char* belongs_to = 0; + for each_map(groups, char*, key, char*, val) { + if( !isdigit(key[0]) ) { + char *comma = va(",%s,", ext); + if( !strcmpi(val,ext) || strbegi(val, comma+1) || strstri(val, comma) || strendi(val, va(",%s", ext))) { + belongs_to = key; + ext_num_groups++; + } + } + } + char *compression = 0; + for each_map(groups, char*, key, char*, val) { + if( isdigit(key[0]) ) { + char *comma = va(",%s,", ext); + if( !strcmpi(val,ext) || strbegi(val, comma+1) || strstri(val, comma) || strendi(val, va(",%s", ext))) { + compression = key; + } + comma = va(",%s,", belongs_to); + if( !strcmpi(val,ext) || strbegi(val, comma+1) || strstri(val, comma) || strendi(val, va(",%s", ext))) { + compression = key; + } + } + } + + cs.compress_level = 0; + if( compression ) { + // last chance to optionally override the compressor at command-line level + static const char *compressor_override; + do_once compressor_override = option("--cook-compressor", ""); + if( compressor_override[0] ) compression = (char*)compressor_override; + + /**/ if(strstri(compression, "PPP")) cs.compress_level = atoi(compression) | PPP; + else if(strstri(compression, "ULZ")) cs.compress_level = atoi(compression) | ULZ; + else if(strstri(compression, "LZ4")) cs.compress_level = atoi(compression) | LZ4X; + else if(strstri(compression, "CRSH")) cs.compress_level = atoi(compression) | CRSH; + else if(strstri(compression, "DEFL")) cs.compress_level = isdigit(compression[0]) ? atoi(compression) : 6 /*| DEFL*/; + //else if(strstri(compression, "LZP")) cs.compress_level = atoi(compression) | LZP1; // not supported + else if(strstri(compression, "LZMA")) cs.compress_level = atoi(compression) | LZMA; + else if(strstri(compression, "BALZ")) cs.compress_level = atoi(compression) | BALZ; + else if(strstri(compression, "LZW")) cs.compress_level = atoi(compression) | LZW3; + else if(strstri(compression, "LZSS")) cs.compress_level = atoi(compression) | LZSS; + else if(strstri(compression, "BCM")) cs.compress_level = atoi(compression) | BCM; + else cs.compress_level = isdigit(compression[0]) ? atoi(compression) : 6 /*| DEFL*/; } } - } - break1:; - char *compression = 0; - for each_map(groups, char*, key, char*, val) { - if( isdigit(key[0]) ) { - char *comma = va(",%s,", ext); - if( strbegi(val, comma+1) || strstri(val, comma) || strendi(val, va(",%s", ext))) { - compression = key; - //goto break2; // each_map() macro is made of multiple for(;;)s. goto needed; you cant escape with single break. - } - comma = va(",%s,", belongs_to); - if( strbegi(val, comma+1) || strstri(val, comma) || strendi(val, va(",%s", ext))) { - compression = key; - //goto break2; // each_map() macro is made of multiple for(;;)s. goto needed; you cant escape with single break. - } + + // if script was generated... + if( script && script[0]) { + // update outfile + cs.outfile = *OUTPUT; + + // amalgamate script + array(char*) lines = strsplit(script, "\r\n"); + + #if is(win32) + char *joint = strjoin(lines, " && "); + cs.script = joint; + #else + if( flag("--cook-wine") ) { + // dear linux/osx/bsd users: + // tools going wrong for any reason? cant compile them maybe? + // small hack to use win32 pipeline tools instead + char *joint = strjoin(lines, " && wine " ); + cs.script = va("wine %s", /*TOOLS,*/ joint); + } else { + char *joint = strjoin(lines, " && " ); + cs.script = va("export LD_LIBRARY_PATH=%s && %s", TOOLS, joint); + } + #endif + } else { + // ... else bypass infile->outfile + char** INFILE = map_find(symbols, "INFILE"); + cs.outfile = *INFILE; + + // and return an empty script + cs.script = ""; } - } - break2:; - cs.compress_level = 0; - if( compression ) { - // last chance to optionally override the compressor at command-line level - static const char *compressor_override, **init = 0; - if( !init ) *(init = &compressor_override) = option("--cook-compressor", ""); - if( compressor_override[0] ) compression = (char*)compressor_override; + cs.outname = cs.outname ? cs.outname : (char*)infile; - /**/ if(strstri(compression, "PPP")) cs.compress_level = atoi(compression) | PPP; - else if(strstri(compression, "ULZ")) cs.compress_level = atoi(compression) | ULZ; - else if(strstri(compression, "LZ4")) cs.compress_level = atoi(compression) | LZ4X; - else if(strstri(compression, "CRSH")) cs.compress_level = atoi(compression) | CRSH; - else if(strstri(compression, "DEFL")) cs.compress_level = isdigit(compression[0]) ? atoi(compression) : 6 /*| DEFL*/; - //else if(strstri(compression, "LZP")) cs.compress_level = atoi(compression) | LZP1; // not supported - else if(strstri(compression, "LZMA")) cs.compress_level = atoi(compression) | LZMA; - else if(strstri(compression, "BALZ")) cs.compress_level = atoi(compression) | BALZ; - else if(strstri(compression, "LZW")) cs.compress_level = atoi(compression) | LZW3; - else if(strstri(compression, "LZSS")) cs.compress_level = atoi(compression) | LZSS; - else if(strstri(compression, "BCM")) cs.compress_level = atoi(compression) | BCM; - else cs.compress_level = isdigit(compression[0]) ? atoi(compression) : 6 /*| DEFL*/; + ASSERT(mcs.num_passes < countof(mcs.cs)); + mcs.cs[mcs.num_passes++] = cs; + + bool next_pass_required = mcs.num_passes < ext_num_groups; + if( !next_pass_required ) break; } - // if script was generated... - if( script && script[0]) { - // update outfile - cs.finalfile = *OUTPUT; - - // amalgamate script - array(char*) lines = strsplit(script, "\r\n"); - - #if is(win32) - char *joint = strjoin(lines, " && "); - cs.script = joint; - #else - if( flag("--cook-wine") ) { - // dear linux/osx/bsd users: - // tools going wrong for any reason? cant compile them maybe? - // small hack to use win32 pipeline tools instead - char *joint = strjoin(lines, " && wine " ); - cs.script = va("wine %s", /*TOOLS,*/ joint); - } else { - char *joint = strjoin(lines, " && " ); - cs.script = va("export LD_LIBRARY_PATH=%s && %s", TOOLS, joint); - } - #endif - } else { - // ... else bypass infile->outfile - char** INFILE = map_find(symbols, "INFILE"); - cs.finalfile = *INFILE; - - // and return an empty script - cs.script = ""; - } - - map_clear(symbols); - map_clear(groups); - return cs; + return mcs; } // ---------------------------------------------------------------------------- @@ -460,55 +494,52 @@ int cook(void *userdata) { *progress = ((i+1) == end ? 90 : (i * 90) / end); // (i+i>0) * 100.f / end; // start cook - const char *fname = uncooked[i]; //job->files[j]; - int inlen = file_size(fname); + const char *infile = uncooked[i]; //job->files[j]; + int inlen = file_size(infile); // generate a cooking script for this asset - cook_script_t cs = cook_script(job->rules, fname, COOK_TMPFILE); + cook_script_t mcs = cook_script(job->rules, infile, COOK_TMPFILE); // puts(cs.script); - // log to batch file for forensic purposes, if explicitly requested - static __thread bool logging = 0, *init = 0; if(!init) *(init = &logging) = !!flag("--cook-debug") || cook_debug; - if( logging ) { - FILE *logfile = fopen(va("cook%d.cmd",job->threadid), "a+t"); - if( logfile ) { fprintf(logfile, "@rem %s\n%s\n", fname, cs.script); fclose(logfile); } - // maybe log fprintf(logfile, "@rem %*.s\n", 4096, app_exec_output()); ? - } + for(int pass = 0; pass < mcs.num_passes; ++pass) { + cook_subscript_t cs = mcs.cs[pass]; - // invoke cooking script and recap status - const char *rcout = app_exec(cs.script); - int rc = atoi(rcout); - int outlen = file_size(cs.finalfile); - int failed = cs.script[0] ? rc || !outlen : 0; + // log to batch file for forensic purposes, if explicitly requested + static __thread bool logging = 0; do_once logging = !!flag("--cook-debug") || cook_debug; + if( logging ) { + FILE *logfile = fopen(va("cook%d.cmd",job->threadid), "a+t"); + if( logfile ) { fprintf(logfile, "@rem %s\n%s\n", cs.outname, cs.script); fclose(logfile); } + fprintf(stderr, "%s\n", cs.script); + } - // print errors, or... - if( failed ) { - PRINTF("Import failed: %s while executing:\n%s\nReturned:\n%s\n", fname, cs.script, rcout); - } - // ...process only if included. may include optional compression. - else if( cs.compress_level >= 0 ) { - FILE *in = fopen(cs.finalfile, "rb"); + // invoke cooking script and recap status + const char *rc_output = app_exec(cs.script); + int rc = atoi(rc_output); + int outlen = file_size(cs.outfile); + int failed = cs.script[0] ? rc || !outlen : 0; -#if 0 - struct stat st; stat(fname, &st); - struct tm *timeinfo = localtime(&st.st_mtime); - ASSERT(timeinfo); + // print errors, or... + if( failed ) { + PRINTF("Import failed: %s while executing:\n%s\nReturned:\n%s\n", cs.outname, cs.script, rc_output); + } + // ...process only if included. may include optional compression. + else if( cs.compress_level >= 0 ) { + FILE *in = fopen(cs.outfile, "rb"); - // pretty (truncated) input (C:/prj/V4K/art/file.wav -> file.wav) - static __thread int artlen = 0; if(!artlen) artlen = strlen(ART); - const char *pretty = fname; - if( !strncmp(pretty, ART, artlen) ) pretty += artlen; - while(pretty[0] == '/') ++pretty; - fname = pretty; - //puts(fname); -#endif + #if 0 + struct stat st; stat(infile, &st); + struct tm *timeinfo = localtime(&st.st_mtime); + ASSERT(timeinfo); + #endif - char *comment = va("%d", inlen); - if( !zip_append_file/*_timeinfo*/(z, fname, comment, in, cs.compress_level/*, timeinfo*/) ) { - PANIC("failed to add processed file into %s: %s", zipfile, fname); - } + char *comment = va("%d", inlen); + if( !zip_append_file/*_timeinfo*/(z, cs.outname, comment, in, cs.compress_level/*, timeinfo*/) ) { + PANIC("failed to add processed file into %s: %s(%s)", zipfile, cs.outname, infile); + } + + fclose(in); + } - fclose(in); } } diff --git a/engine/split/v4k_data.h b/engine/split/v4k_data.h index 0a080b9..e168791 100644 --- a/engine/split/v4k_data.h +++ b/engine/split/v4k_data.h @@ -27,8 +27,8 @@ API const char * xml_string(char *key); API unsigned xml_count(char *key); API array(char) xml_blob(char *key); #define xml_string(...) xml_string(va(__VA_ARGS__)) // syntax sugar: string -#define xml_int(...) atoi(xml_string(va(__VA_ARGS__))) // syntax sugar: int -#define xml_float(...) atof(xml_string(va(__VA_ARGS__))) // syntax sugar: float +#define xml_int(...) atoi(xml_string(__VA_ARGS__)) // syntax sugar: int +#define xml_float(...) atof(xml_string(__VA_ARGS__)) // syntax sugar: float #define xml_blob(...) xml_blob(va(__VA_ARGS__)) // syntax sugar: base64 blob #define xml_count(...) xml_count(va(__VA_ARGS__)) // syntax sugar: count nodes API void xml_pop(); diff --git a/engine/v4k.c b/engine/v4k.c index 6ed0135..e0d497b 100644 --- a/engine/v4k.c +++ b/engine/v4k.c @@ -2942,288 +2942,322 @@ const char *COOK_INI = "tools/cook.ini"; static unsigned ART_SKIP_ROOT; // number of chars to skip the base root in ART folder static unsigned ART_LEN; // dupe -typedef struct cook_script_t { - char *infile; // free after use - char *finalfile; // free after use. can be either infile or a totally different file +typedef struct cook_subscript_t { + char *infile; + char *outfile; // can be either infile, or a totally different file char *script; + char *outname; int compress_level; +} cook_subscript_t; + +typedef struct cook_script_t { + cook_subscript_t cs[8]; + + int num_passes; } cook_script_t; static cook_script_t cook_script(const char *rules, const char *infile, const char *outfile) { - // by default, assume: - // - no script is going to be generated (empty script) - // - if no script is going to be generated, output is in fact input file. - // - no compression is going to be required. - cook_script_t cs = { 0 }; + cook_script_t mcs = { 0 }; - // reuse script heap from last call if possible (optimization) - static __thread char *script = 0; - if(script) script[0] = 0; + // pass loop: some asset rules may require multiple cook passes + for( int pass = 0; pass < countof(mcs.cs); ++pass ) { + // by default, assume: + // - no script is going to be generated (empty script) + // - if no script is going to be generated, output is in fact input file. + // - no compression is going to be required. + cook_subscript_t cs = { 0 }; - // reuse parsing maps if possible (optimization) - static __thread map(char*, char*) symbols = 0; - static __thread map(char*, char*) groups = 0; + // reuse script heap from last call if possible (optimization) + static __thread char *script = 0; + if(script) script[0] = 0; - if(!symbols) map_init(symbols, less_str, hash_str); - if(!groups) map_init(groups, less_str, hash_str); + // reuse parsing maps if possible (optimization) + static __thread map(char*, char*) symbols = 0; if(!symbols) map_init_str(symbols); + static __thread map(char*, char*) groups = 0; if(!groups) map_init_str(groups); + static __thread set(char*) passes = 0; if(!passes) set_init_str(passes); + map_clear(symbols); + map_clear(groups); - map_find_or_add(symbols, "INFILE", STRDUP(infile)); - map_find_or_add(symbols, "INPUT", STRDUP(infile)); - map_find_or_add(symbols, "PRETTY", STRDUP(infile + ART_SKIP_ROOT)); // pretty (truncated) input (C:/prj/V4K/art/file.wav -> file.wav) - map_find_or_add(symbols, "OUTPUT", STRDUP(outfile)); - map_find_or_add(symbols, "TOOLS", STRDUP(TOOLS)); - map_find_or_add(symbols, "EDITOR", STRDUP(EDITOR)); - map_find_or_add(symbols, "PROGRESS", STRDUP(va("%03d", cook_progress()))); + map_find_or_add(symbols, "INFILE", STRDUP(infile)); + map_find_or_add(symbols, "INPUT", STRDUP(infile)); + map_find_or_add(symbols, "PRETTY", STRDUP(infile + ART_SKIP_ROOT)); // pretty (truncated) input (C:/prj/V4K/art/file.wav -> file.wav) + map_find_or_add(symbols, "OUTPUT", STRDUP(outfile)); + map_find_or_add(symbols, "TOOLS", STRDUP(TOOLS)); + map_find_or_add(symbols, "EDITOR", STRDUP(EDITOR)); + map_find_or_add(symbols, "PROGRESS", STRDUP(va("%03d", cook_progress()))); - // start parsing. parsing is enabled by default - int enabled = 1; - array(char*)lines = strsplit(rules, "\r\n"); - for( int i = 0, end = array_count(lines); i < end; ++i ) { - // skip blanks - int blanks = strspn(lines[i], " \t"); - char *line = lines[i] + blanks; + // clear pass counter + set_clear(passes); - // discard full comments - if( line[0] == ';' ) continue; - // truncate inline comments - if( strstr(line, ";") ) *strstr(line, ";") = 0; - // trim ending spaces - char *eos = line + strlen(line); while(eos > line && eos[-1] == ' ' ) *--eos = 0; - // discard non-specific lines - if( line[0] == '@' ) { - int with_wine = flag("--cook-wine") && !!strstr(line, "@win"); - int parse = 0 - | ifdef(win32, (!!strstr(line, "@win")), 0) - | ifdef(linux, (!!strstr(line, "@lin") ? 1 : with_wine), 0) - | ifdef(osx, (!!strstr(line, "@osx") ? 1 : with_wine), 0); + // start parsing. parsing is enabled by default + int enabled = 1; + array(char*)lines = strsplit(rules, "\r\n"); + for( int i = 0, end = array_count(lines); i < end; ++i ) { + // skip blanks + int blanks = strspn(lines[i], " \t"); + char *line = lines[i] + blanks; - if( !parse ) continue; + // discard full comments + if( line[0] == ';' ) continue; + // truncate inline comments + if( strstr(line, ";") ) *strstr(line, ";") = 0; + // trim ending spaces + char *eos = line + strlen(line); while(eos > line && eos[-1] == ' ' ) *--eos = 0; + // discard non-specific lines + if( line[0] == '@' ) { + int with_wine = flag("--cook-wine") && !!strstr(line, "@win"); + int parse = 0 + | ifdef(win32, (!!strstr(line, "@win")), 0) + | ifdef(linux, (!!strstr(line, "@lin") ? 1 : with_wine), 0) + | ifdef(osx, (!!strstr(line, "@osx") ? 1 : with_wine), 0); - line = strchr(line+1, ' '); - if(!line) continue; - line += strspn(line, " \t"); - } - // execute `shell` commands - if( line[0] == '`' ) { - char *eos = strrchr(++line, '`'); - if( eos ) *eos = 0; + if( !parse ) continue; - // replace all symbols - char* nl = STRDUP(line); - for each_map(symbols, char*, key, char*, val) { - strrepl(&nl, key, val); + line = strchr(line+1, ' '); + if(!line) continue; + line += strspn(line, " \t"); + } + // execute `shell` commands + if( line[0] == '`' ) { + char *eos = strrchr(++line, '`'); + if( eos ) *eos = 0; + + // replace all symbols + char* nl = STRDUP(line); // @leak + for each_map(symbols, char*, key, char*, val) { + strrepl(&nl, key, val); + } + lines[i] = line = nl; + + static thread_mutex_t lock, *init = 0; if(!init) thread_mutex_init(init = &lock); + thread_mutex_lock( &lock ); + system(line); // strcatf(&script, "%s\n", line); + thread_mutex_unlock( &lock ); + + continue; + } + // process [sections] + if( line[0] == '[' ) { + enabled = 1; + int is_cook = !!strstr(line, "[cook]"); + int is_compress = !!strstr(line, "[compress]"); + if( !is_cook && !is_compress ) { // if not a special section... + // remove hint cook tag if present. that's informative only. + if(strbegi(line, "[cook ") ) memcpy(line+1, " ", 4); // line += 6; + + // start parsing expressions like `[media && !avi && mp3]` + array(char*) tags = strsplit(line, " []&"); + + // let's check whether INPUT belongs to tags above + char **INPUT = map_find(symbols, "INPUT"); + bool found_in_set = true; + + for( int i = 0, end = array_count(tags); i < end; ++i) { + bool negate = false; + char *tag = tags[i]; + while(*tag == '!') negate ^= 1, ++tag; + + // find tag in groups map + // either a group or an extension + char **is_group = map_find(groups, tag); + if( is_group ) { + char *list = *is_group; + char *INPUT_EXT = file_ext(infile); INPUT_EXT = strrchr(INPUT_EXT, '.'); // .ext1.ext -> .ext + char *ext = INPUT_EXT; ext += ext[0] == '.'; // dotless + bool in_list = strbegi(list, ext) || strendi(list, va(",%s",ext)) || strstri(list, va(",%s,",ext)); + if( !in_list ^ negate ) { found_in_set = false; break; } + } else { + char *ext = va(".%s", tag); + bool found = !!strendi(*INPUT, ext); + if( !found ^ negate ) { found_in_set = false; break; } + } + } + if( found_in_set ) { + // inc pass + set_find_or_add(passes, STRDUP(*tags)); // @leak + // check whether we keep searching + int num_passes = set_count(passes); + found_in_set = ( pass == (num_passes-1) ); + } + // + enabled = found_in_set ? 1 : 0; } - lines[i] = line = nl; // @fixme:leak - - static thread_mutex_t lock, *init = 0; if(!init) thread_mutex_init(init = &lock); - thread_mutex_lock( &lock ); - system(line); // strcatf(&script, "%s\n", line); - thread_mutex_unlock( &lock ); - - continue; - } - // process [sections] - if( line[0] == '[' ) { - enabled = 1; - int is_cook = !!strstr(line, "[cook]"); - int is_compress = !!strstr(line, "[compress]"); - if( !is_cook && !is_compress ) { - // remove hint cook tag if present. that's informative only. - if(strbegi(line, "[cook ") ) memcpy(line+1, " ", 4); - - // start parsing expressions like `[media && !avi && mp3]` - array(char*) tags = strsplit(line, " []&"); - - // let's check whether INPUT belongs to tags above - char **INPUT = map_find(symbols, "INPUT"); - bool found_in_set = true; - - for( int i = 0, end = array_count(tags); i < end; ++i) { char *tag = tags[i]; - bool negate = false; - while(*tag == '!') negate ^= 1, ++tag; - - // find tag in groups map - // either a group or an extension - char **is_group = map_find(groups, tag); - if( is_group ) { - char *list = *is_group; - char *INPUT_EXT = file_ext(infile); INPUT_EXT = strrchr(INPUT_EXT, '.'); // .ext1.ext -> .ext - char *ext = INPUT_EXT; ext += ext[0] == '.'; // dotless - bool in_list = strbegi(list, ext) || strstri(list, va(",%s,",ext)) || strendi(list, va(",%s",ext)); - if( !in_list ^ negate ) { found_in_set = false; break; } - } else { - char *ext = va(".%s", tag); - bool found = !!strendi(*INPUT, ext); - if( !found ^ negate ) { found_in_set = false; break; } + } + // either SYMBOL=, group=, or regular script line + if( enabled && line[0] != '[' ) { + enum { group, symbol, regular } type = regular; + int tokenlen = strspn(line, "-+_.|0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"); + char *token = va("%.*s", tokenlen, line); + char *equal = strchr(line, '='); + if( equal ) { + if( equal == &line[tokenlen] ) { // if key=value expression found + // discriminate: symbols are uppercase and never begin with digits. groups are [0-9]+[|][a-z]. + type = strcmp(strupper(token), token) || isdigit(token[0]) ? group : symbol; } } - // - enabled = found_in_set ? 1 : 0; - } - } - // either SYMBOL=, group=, or regular script line - if( enabled && line[0] != '[' ) { - enum { group, symbol, regular } type = regular; - int tokenlen = strspn(line, "-+_.|0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"); - char *token = va("%.*s", tokenlen, line); - char *equal = strchr(line, '='); - if( equal ) { - if( equal == &line[tokenlen] ) { // if key=value expression found - // discriminate: symbols are uppercase and never begin with digits. groups are [0-9]+[|][a-z]. - type = strcmp(strupper(token), token) || isdigit(token[0]) ? group : symbol; + if( type == group ) map_find_or_add(groups, token, STRDUP(equal+1)); + if( type == symbol ) { + // @todo: perform the replacement/union/intersection on set here + bool is_add = strendi(token, "+"); + bool is_del = strendi(token, "-"); + + // if present, remove last sign from token -> (FLAGS1+)=, (FLAGS1-)= + if(is_add || is_del) token[strlen(token) - 1] = 0; + + map_find_or_add(symbols, token, STRDUP(equal+1)); } - } - if( type == group ) map_find_or_add(groups, token, STRDUP(equal+1)); - if( type == symbol ) { - // @todo: perform the replacement/union/intersection on set here - bool is_add = strendi(token, "+"); - bool is_del = strendi(token, "-"); + // for each_map(symbols, char*, key, char*, val) printf("%s=%s,", key, val); puts(""); + // for each_map(groups, char*, key, char*, val) printf("%s=%s,", key, val); puts(""); + // if( type != regular ) printf("%s found >> %s\n", type == group ? "group" : "symbol", line); - // if present, remove last sign from token -> (FLAGS1+)=, (FLAGS1-)= - if(is_add || is_del) token[strlen(token) - 1] = 0; + if( type == regular ) { + char** INPUT = map_find(symbols, "INPUT"); + char** OUTPUT = map_find(symbols, "OUTPUT"); - map_find_or_add(symbols, token, STRDUP(equal+1)); - } - // for each_map(symbols, char*, key, char*, val) printf("%s=%s,", key, val); puts(""); - // for each_map(groups, char*, key, char*, val) printf("%s=%s,", key, val); puts(""); - // if( type != regular ) printf("%s found >> %s\n", type == group ? "group" : "symbol", line); + // parse return code + char *has_errorlevel = strstr(line, "=="); //==N form + int errorlevel = has_errorlevel ? atoi(has_errorlevel + 2) : 0; + if( has_errorlevel ) memcpy(has_errorlevel, " ", 3); - if( type == regular ) { - char** INPUT = map_find(symbols, "INPUT"); - char** OUTPUT = map_find(symbols, "OUTPUT"); + // detect if newer extension or filename is present, and thus update OUTPUT if needed + char *newer_extension = strstr(line, "->"); if(newer_extension) { + *newer_extension = 0; + newer_extension += 2 + strspn(newer_extension + 2, " "); - // parse return code - char *has_errorlevel = strstr(line, "=="); //==N form - int errorlevel = has_errorlevel ? atoi(has_errorlevel + 2) : 0; - if( has_errorlevel ) memcpy(has_errorlevel, " ", 3); + if( strchr(newer_extension, '.') ) { + // newer filename + cs.outname = stringf("%s@%s", cs.outname ? cs.outname : infile, newer_extension); // @leak + newer_extension = NULL; + } else { + strcatf(&*OUTPUT, ".%s", newer_extension); + } + } - // detect if newer extension is present, and thus update OUTPUT if needed - char *newer_extension = strstr(line, "->"); if(newer_extension) { - *newer_extension = 0; - newer_extension += 2 + strspn(newer_extension + 2, " "); + // replace all symbols + char* nl = STRDUP(line); // @leak + for each_map(symbols, char*, key, char*, val) { + strrepl(&nl, key, val); + } + lines[i] = line = nl; - strcatf(&*OUTPUT, ".%s", newer_extension); - } + // convert slashes + ifdef(win32, + strswap(line, "/", "\\") + , // else + strswap(line, "\\", "/") + ); - // replace all symbols - char* nl = STRDUP(line); - for each_map(symbols, char*, key, char*, val) { - strrepl(&nl, key, val); - } - lines[i] = line = nl; // @fixme:leak + // append line + strcatf(&script, "%s\n", line); - // convert slashes - ifdef(win32, - strswap(line, "/", "\\") - , // else - strswap(line, "\\", "/") - ); + // handle return code here + // if(has_errorlevel) + // strcatf(&script, "IF NOT '%%ERRORLEVEL%%'=='%d' echo ERROR!\n", errorlevel); - // append line - strcatf(&script, "%s\n", line); - - // handle return code here - // if(has_errorlevel) - // strcatf(&script, "IF NOT '%%ERRORLEVEL%%'=='%d' echo ERROR!\n", errorlevel); - - // rename output->input for further chaining, in case it is needed - if( newer_extension ) { - *INPUT[0] = 0; - strcatf(&*INPUT, "%s", *OUTPUT); + // rename output->input for further chaining, in case it is needed + if( newer_extension ) { + *INPUT[0] = 0; + strcatf(&*INPUT, "%s", *OUTPUT); + } } } } - } - // compression - char* ext = file_ext(infile); ext = strrchr(ext, '.'); ext += ext[0] == '.'; // dotless INPUT_EXT + char** OUTPUT = map_find(symbols, "OUTPUT"); + int ext_num_groups = 0; - char** OUTPUT = map_find(symbols, "OUTPUT"); - char* belongs_to = 0; - for each_map(groups, char*, key, char*, val) { - if( !isdigit(key[0]) ) { - char *comma = va(",%s,", ext); - if( strbegi(val, comma+1) || strstri(val, comma) || strendi(val, va(",%s", ext))) { - belongs_to = key; - //goto break1; // each_map() macro is made of multiple for(;;)s. goto needed; you cant escape with single break. + // compression + if( 1 ) { + char* ext = file_ext(infile); ext = strrchr(ext, '.'); ext += ext[0] == '.'; // dotless INPUT_EXT + char* belongs_to = 0; + for each_map(groups, char*, key, char*, val) { + if( !isdigit(key[0]) ) { + char *comma = va(",%s,", ext); + if( !strcmpi(val,ext) || strbegi(val, comma+1) || strstri(val, comma) || strendi(val, va(",%s", ext))) { + belongs_to = key; + ext_num_groups++; + } + } + } + char *compression = 0; + for each_map(groups, char*, key, char*, val) { + if( isdigit(key[0]) ) { + char *comma = va(",%s,", ext); + if( !strcmpi(val,ext) || strbegi(val, comma+1) || strstri(val, comma) || strendi(val, va(",%s", ext))) { + compression = key; + } + comma = va(",%s,", belongs_to); + if( !strcmpi(val,ext) || strbegi(val, comma+1) || strstri(val, comma) || strendi(val, va(",%s", ext))) { + compression = key; + } + } + } + + cs.compress_level = 0; + if( compression ) { + // last chance to optionally override the compressor at command-line level + static const char *compressor_override; + do_once compressor_override = option("--cook-compressor", ""); + if( compressor_override[0] ) compression = (char*)compressor_override; + + /**/ if(strstri(compression, "PPP")) cs.compress_level = atoi(compression) | PPP; + else if(strstri(compression, "ULZ")) cs.compress_level = atoi(compression) | ULZ; + else if(strstri(compression, "LZ4")) cs.compress_level = atoi(compression) | LZ4X; + else if(strstri(compression, "CRSH")) cs.compress_level = atoi(compression) | CRSH; + else if(strstri(compression, "DEFL")) cs.compress_level = isdigit(compression[0]) ? atoi(compression) : 6 /*| DEFL*/; + //else if(strstri(compression, "LZP")) cs.compress_level = atoi(compression) | LZP1; // not supported + else if(strstri(compression, "LZMA")) cs.compress_level = atoi(compression) | LZMA; + else if(strstri(compression, "BALZ")) cs.compress_level = atoi(compression) | BALZ; + else if(strstri(compression, "LZW")) cs.compress_level = atoi(compression) | LZW3; + else if(strstri(compression, "LZSS")) cs.compress_level = atoi(compression) | LZSS; + else if(strstri(compression, "BCM")) cs.compress_level = atoi(compression) | BCM; + else cs.compress_level = isdigit(compression[0]) ? atoi(compression) : 6 /*| DEFL*/; } } - } - break1:; - char *compression = 0; - for each_map(groups, char*, key, char*, val) { - if( isdigit(key[0]) ) { - char *comma = va(",%s,", ext); - if( strbegi(val, comma+1) || strstri(val, comma) || strendi(val, va(",%s", ext))) { - compression = key; - //goto break2; // each_map() macro is made of multiple for(;;)s. goto needed; you cant escape with single break. - } - comma = va(",%s,", belongs_to); - if( strbegi(val, comma+1) || strstri(val, comma) || strendi(val, va(",%s", ext))) { - compression = key; - //goto break2; // each_map() macro is made of multiple for(;;)s. goto needed; you cant escape with single break. - } + + // if script was generated... + if( script && script[0]) { + // update outfile + cs.outfile = *OUTPUT; + + // amalgamate script + array(char*) lines = strsplit(script, "\r\n"); + + #if is(win32) + char *joint = strjoin(lines, " && "); + cs.script = joint; + #else + if( flag("--cook-wine") ) { + // dear linux/osx/bsd users: + // tools going wrong for any reason? cant compile them maybe? + // small hack to use win32 pipeline tools instead + char *joint = strjoin(lines, " && wine " ); + cs.script = va("wine %s", /*TOOLS,*/ joint); + } else { + char *joint = strjoin(lines, " && " ); + cs.script = va("export LD_LIBRARY_PATH=%s && %s", TOOLS, joint); + } + #endif + } else { + // ... else bypass infile->outfile + char** INFILE = map_find(symbols, "INFILE"); + cs.outfile = *INFILE; + + // and return an empty script + cs.script = ""; } - } - break2:; - cs.compress_level = 0; - if( compression ) { - // last chance to optionally override the compressor at command-line level - static const char *compressor_override, **init = 0; - if( !init ) *(init = &compressor_override) = option("--cook-compressor", ""); - if( compressor_override[0] ) compression = (char*)compressor_override; + cs.outname = cs.outname ? cs.outname : (char*)infile; - /**/ if(strstri(compression, "PPP")) cs.compress_level = atoi(compression) | PPP; - else if(strstri(compression, "ULZ")) cs.compress_level = atoi(compression) | ULZ; - else if(strstri(compression, "LZ4")) cs.compress_level = atoi(compression) | LZ4X; - else if(strstri(compression, "CRSH")) cs.compress_level = atoi(compression) | CRSH; - else if(strstri(compression, "DEFL")) cs.compress_level = isdigit(compression[0]) ? atoi(compression) : 6 /*| DEFL*/; - //else if(strstri(compression, "LZP")) cs.compress_level = atoi(compression) | LZP1; // not supported - else if(strstri(compression, "LZMA")) cs.compress_level = atoi(compression) | LZMA; - else if(strstri(compression, "BALZ")) cs.compress_level = atoi(compression) | BALZ; - else if(strstri(compression, "LZW")) cs.compress_level = atoi(compression) | LZW3; - else if(strstri(compression, "LZSS")) cs.compress_level = atoi(compression) | LZSS; - else if(strstri(compression, "BCM")) cs.compress_level = atoi(compression) | BCM; - else cs.compress_level = isdigit(compression[0]) ? atoi(compression) : 6 /*| DEFL*/; + ASSERT(mcs.num_passes < countof(mcs.cs)); + mcs.cs[mcs.num_passes++] = cs; + + bool next_pass_required = mcs.num_passes < ext_num_groups; + if( !next_pass_required ) break; } - // if script was generated... - if( script && script[0]) { - // update outfile - cs.finalfile = *OUTPUT; - - // amalgamate script - array(char*) lines = strsplit(script, "\r\n"); - - #if is(win32) - char *joint = strjoin(lines, " && "); - cs.script = joint; - #else - if( flag("--cook-wine") ) { - // dear linux/osx/bsd users: - // tools going wrong for any reason? cant compile them maybe? - // small hack to use win32 pipeline tools instead - char *joint = strjoin(lines, " && wine " ); - cs.script = va("wine %s", /*TOOLS,*/ joint); - } else { - char *joint = strjoin(lines, " && " ); - cs.script = va("export LD_LIBRARY_PATH=%s && %s", TOOLS, joint); - } - #endif - } else { - // ... else bypass infile->outfile - char** INFILE = map_find(symbols, "INFILE"); - cs.finalfile = *INFILE; - - // and return an empty script - cs.script = ""; - } - - map_clear(symbols); - map_clear(groups); - return cs; + return mcs; } // ---------------------------------------------------------------------------- @@ -3385,55 +3419,52 @@ int cook(void *userdata) { *progress = ((i+1) == end ? 90 : (i * 90) / end); // (i+i>0) * 100.f / end; // start cook - const char *fname = uncooked[i]; //job->files[j]; - int inlen = file_size(fname); + const char *infile = uncooked[i]; //job->files[j]; + int inlen = file_size(infile); // generate a cooking script for this asset - cook_script_t cs = cook_script(job->rules, fname, COOK_TMPFILE); + cook_script_t mcs = cook_script(job->rules, infile, COOK_TMPFILE); // puts(cs.script); - // log to batch file for forensic purposes, if explicitly requested - static __thread bool logging = 0, *init = 0; if(!init) *(init = &logging) = !!flag("--cook-debug") || cook_debug; - if( logging ) { - FILE *logfile = fopen(va("cook%d.cmd",job->threadid), "a+t"); - if( logfile ) { fprintf(logfile, "@rem %s\n%s\n", fname, cs.script); fclose(logfile); } - // maybe log fprintf(logfile, "@rem %*.s\n", 4096, app_exec_output()); ? - } + for(int pass = 0; pass < mcs.num_passes; ++pass) { + cook_subscript_t cs = mcs.cs[pass]; - // invoke cooking script and recap status - const char *rcout = app_exec(cs.script); - int rc = atoi(rcout); - int outlen = file_size(cs.finalfile); - int failed = cs.script[0] ? rc || !outlen : 0; + // log to batch file for forensic purposes, if explicitly requested + static __thread bool logging = 0; do_once logging = !!flag("--cook-debug") || cook_debug; + if( logging ) { + FILE *logfile = fopen(va("cook%d.cmd",job->threadid), "a+t"); + if( logfile ) { fprintf(logfile, "@rem %s\n%s\n", cs.outname, cs.script); fclose(logfile); } + fprintf(stderr, "%s\n", cs.script); + } - // print errors, or... - if( failed ) { - PRINTF("Import failed: %s while executing:\n%s\nReturned:\n%s\n", fname, cs.script, rcout); - } - // ...process only if included. may include optional compression. - else if( cs.compress_level >= 0 ) { - FILE *in = fopen(cs.finalfile, "rb"); + // invoke cooking script and recap status + const char *rc_output = app_exec(cs.script); + int rc = atoi(rc_output); + int outlen = file_size(cs.outfile); + int failed = cs.script[0] ? rc || !outlen : 0; -#if 0 - struct stat st; stat(fname, &st); - struct tm *timeinfo = localtime(&st.st_mtime); - ASSERT(timeinfo); + // print errors, or... + if( failed ) { + PRINTF("Import failed: %s while executing:\n%s\nReturned:\n%s\n", cs.outname, cs.script, rc_output); + } + // ...process only if included. may include optional compression. + else if( cs.compress_level >= 0 ) { + FILE *in = fopen(cs.outfile, "rb"); - // pretty (truncated) input (C:/prj/V4K/art/file.wav -> file.wav) - static __thread int artlen = 0; if(!artlen) artlen = strlen(ART); - const char *pretty = fname; - if( !strncmp(pretty, ART, artlen) ) pretty += artlen; - while(pretty[0] == '/') ++pretty; - fname = pretty; - //puts(fname); -#endif + #if 0 + struct stat st; stat(infile, &st); + struct tm *timeinfo = localtime(&st.st_mtime); + ASSERT(timeinfo); + #endif - char *comment = va("%d", inlen); - if( !zip_append_file/*_timeinfo*/(z, fname, comment, in, cs.compress_level/*, timeinfo*/) ) { - PANIC("failed to add processed file into %s: %s", zipfile, fname); - } + char *comment = va("%d", inlen); + if( !zip_append_file/*_timeinfo*/(z, cs.outname, comment, in, cs.compress_level/*, timeinfo*/) ) { + PANIC("failed to add processed file into %s: %s(%s)", zipfile, cs.outname, infile); + } + + fclose(in); + } - fclose(in); } } diff --git a/engine/v4k.h b/engine/v4k.h index 9c0e0b9..503f290 100644 --- a/engine/v4k.h +++ b/engine/v4k.h @@ -1501,8 +1501,8 @@ API const char * xml_string(char *key); API unsigned xml_count(char *key); API array(char) xml_blob(char *key); #define xml_string(...) xml_string(va(__VA_ARGS__)) // syntax sugar: string -#define xml_int(...) atoi(xml_string(va(__VA_ARGS__))) // syntax sugar: int -#define xml_float(...) atof(xml_string(va(__VA_ARGS__))) // syntax sugar: float +#define xml_int(...) atoi(xml_string(__VA_ARGS__)) // syntax sugar: int +#define xml_float(...) atof(xml_string(__VA_ARGS__)) // syntax sugar: float #define xml_blob(...) xml_blob(va(__VA_ARGS__)) // syntax sugar: base64 blob #define xml_count(...) xml_count(va(__VA_ARGS__)) // syntax sugar: count nodes API void xml_pop(); diff --git a/engine/v4k.html b/engine/v4k.html index 3caf7bc..db2ce88 100644 --- a/engine/v4k.html +++ b/engine/v4k.html @@ -596,7 +596,7 @@ details > summary::-webkit-details-marker { |Version: | 2023.7 | |:--------------|:------------| |Branch: | main | -|Commit: | 62 | +|Commit: | 63 | # [V·4·K 2023.7 ](https://dev.v4.games/zaklaus/v4k) @@ -8667,7 +8667,7 @@ Other documentation examples: [dll](#dll), [strsplit](#strsplit), [strjoin](#str ## shaders - +
extern const char* const fs_0_0_shadowmap_lit; Under construction. Yet to be documented. @@ -8676,7 +8676,7 @@ Other documentation examples: [dll](#dll), [strsplit](#strsplit), [strjoin](#str
- +
extern const char* const fs_0_0_shadowmap_unlit; Under construction. Yet to be documented. @@ -8685,7 +8685,7 @@ Other documentation examples: [dll](#dll), [strsplit](#strsplit), [strjoin](#str
- +
extern const char* const fs_24_4_sprite; Under construction. Yet to be documented. @@ -8694,7 +8694,7 @@ Other documentation examples: [dll](#dll), [strsplit](#strsplit), [strjoin](#str
- +
extern const char* const fs_2_4_preamble; Under construction. Yet to be documented. @@ -8703,7 +8703,7 @@ Other documentation examples: [dll](#dll), [strsplit](#strsplit), [strjoin](#str
- +
extern const char* const fs_2_4_texel_inv_gamma; Under construction. Yet to be documented. @@ -8712,7 +8712,7 @@ Other documentation examples: [dll](#dll), [strsplit](#strsplit), [strjoin](#str
- +
extern const char* const fs_2_4_texel_ycbr_gamma_saturation; Under construction. Yet to be documented. @@ -8721,7 +8721,7 @@ Other documentation examples: [dll](#dll), [strsplit](#strsplit), [strjoin](#str
- +
extern const char* const fs_32_4_model; Under construction. Yet to be documented. @@ -8730,7 +8730,7 @@ Other documentation examples: [dll](#dll), [strsplit](#strsplit), [strjoin](#str
- +
extern const char* const fs_32_4_model_basic; Under construction. Yet to be documented. @@ -8739,7 +8739,7 @@ Other documentation examples: [dll](#dll), [strsplit](#strsplit), [strjoin](#str
- +
extern const char* const fs_3_4_skybox; Under construction. Yet to be documented. @@ -8748,7 +8748,7 @@ Other documentation examples: [dll](#dll), [strsplit](#strsplit), [strjoin](#str
- +
extern const char* const fs_3_4_skybox_rayleigh; Under construction. Yet to be documented. @@ -8757,7 +8757,7 @@ Other documentation examples: [dll](#dll), [strsplit](#strsplit), [strjoin](#str
- +
extern const char* const fs_main_shadertoy; Under construction. Yet to be documented. @@ -8766,7 +8766,7 @@ Other documentation examples: [dll](#dll), [strsplit](#strsplit), [strjoin](#str
- +
extern const char* const vs_0_2_fullscreen_quad_A; Under construction. Yet to be documented. @@ -8775,7 +8775,7 @@ Other documentation examples: [dll](#dll), [strsplit](#strsplit), [strjoin](#str
- +
extern const char* const vs_0_2_fullscreen_quad_B; Under construction. Yet to be documented. @@ -8784,7 +8784,7 @@ Other documentation examples: [dll](#dll), [strsplit](#strsplit), [strjoin](#str
- +
extern const char* const vs_0_2_fullscreen_quad_B_flipped; Under construction. Yet to be documented. @@ -8793,7 +8793,7 @@ Other documentation examples: [dll](#dll), [strsplit](#strsplit), [strjoin](#str
- +
extern const char* const vs_323444143_16_332_model; Under construction. Yet to be documented. @@ -8802,7 +8802,7 @@ Other documentation examples: [dll](#dll), [strsplit](#strsplit), [strjoin](#str
- +
extern const char* const vs_324_24_sprite; Under construction. Yet to be documented. @@ -8811,7 +8811,7 @@ Other documentation examples: [dll](#dll), [strsplit](#strsplit), [strjoin](#str
- +
extern const char* const vs_332_32; Under construction. Yet to be documented. @@ -8820,7 +8820,7 @@ Other documentation examples: [dll](#dll), [strsplit](#strsplit), [strjoin](#str
- +
extern const char* const vs_3_3_skybox; Under construction. Yet to be documented. diff --git a/tools/cook.exe b/tools/cook.exe index 90115e4..ec1e9d6 100644 Binary files a/tools/cook.exe and b/tools/cook.exe differ diff --git a/tools/cook.ini b/tools/cook.ini index 97a8e88..501682a 100644 --- a/tools/cook.ini +++ b/tools/cook.ini @@ -1,4 +1,4 @@ -; this is where you specify and configure the FWK pipeline. +; this is where you specify and configure the V4K pipeline. ; tweak the pipeline and add new importers just by editing this file. ; there is no flow control in this script file: lines are parsed and evaluated, from top to bottom. @@ -6,11 +6,11 @@ ; let's create a symbol. symbols are uppercase words always. ; syntax: symbols are defined in KEY=value form, as seen below. +TOOLS=./ ; folder where our pipeline tools are located ART=../demos/art/,../engine/art/ ; comma-separated folder(s) that store all our asset files -TOOLS=./ ; where our pipeline tools are located ; lines starting with @windows, @linux or @osx will be processed only where OS matches. -; we are defining here some symbols differently on each platform. +; we are defining here some symbols differently for each platform. ; syntax: lines starting with @keyword. valid keywords are win/dows, lin/ux, and osx. @linux NUL=/dev/null @@ -23,7 +23,7 @@ TOOLS=./ ; where our pipeline tools are located ; you can invoke shell commands directly with `command` at anytime. ; also, once a symbol is found, it is replaced by its value always. -; PROGRESS (percent), INPUT (input filename), OUTPUT (output filename) and PRETTY (clean input filename) are some predefined symbols. +; some predefined symbols: INPUT (input filename), OUTPUT (output filename), PRETTY (clean input filename), PROGRESS (cook progress). @windows `echo Cooking PROGRESS% PRETTY...` @linux `echo "Cooking PROGRESS% PRETTY..."` @@ -35,16 +35,16 @@ TOOLS=./ ; where our pipeline tools are located ; syntax: group=ext1,ext2[...] [cook] -icons=ico -image=jpg,png,bmp,psd,pic,pnm,hdr +icon=ico +image=jpg,jpeg,png,bmp,psd,pic,pnm,hdr texture=pvr,ktx,ktx2,dds,astc,basis,tga +anim=fbx model=iqm,iqe,gltf,gltf2,glb,fbx,obj,dae,blend,md3,md5,ms3d,smd,x,3ds,bvh,dxf,lwo -anims=anim audio=wav,flac,ogg,mp1,mp3,mid,sfxr ; ,mod,xm -audio-modules=mod,xm,s3m,it +audio-module=mod,xm,s3m,it audio-furnace=fur font=ttf,ttc,otf -text=json,xml,csv,ini,cfg,doc,txt,md,c,h,lua,inl,cpp,hpp,htm,html +text=json,xml,csv,ini,cfg,doc,txt,md,c,h,inl,cpp,hpp,htm,html shader=hlsl,fx,dxil,dxbc,glsl,vert,frag,geom,tese,tesc,comp,vs,fs,gs,ts,cs,spirv,spv,slang script=lua,tl video=mp4,ogv,avi,mkv,wmv,mpg,mpeg @@ -58,7 +58,7 @@ tiled=tmx,tsx ; hint: the ->ogg and ->wav parts below do signal the pipeline that the commands we are about ; to execute are performing a data conversion (from flac to ogg for example). -[cook audio-modules] +[cook audio-module] TOOLS/mod2wav.EXE INPUT OUTPUT -> wav TOOLS/ffmpeg.EXE -hide_banner -nostdin -loglevel fatal -y -i INPUT -f ogg -b:a 192k OUTPUT -> ogg ; -stats @@ -164,16 +164,12 @@ TOOLS/cuttlefish.EXE -q -m -i INPUT -o OUTPUT -f BC1_RGBA -> ktx ; ------------------------------------------------------------------------------ ; finally, let's cook all models. the logic here is: -; 1. export animation list from models using ass2iqe -L -; 2. cook all models into iqe (ass2iqe), then into iqm (iqe2iqm): any -> iqe -> iqm -; 3. unless input is iqe. these models will run iqe2iqm only (no ass2iqe): iqe -> iqm. -; 4. unless input is iqm. these models will not run any conversion at all: iqm. -; 5. also, dae models need to flip their UVs coordinates (see -U flag below). +; 1. cook all models into iqe (ass2iqe), then into iqm (iqe2iqm): any -> iqe -> iqm +; 2. unless input is iqe. these models will run iqe2iqm only (no ass2iqe): iqe -> iqm. +; 3. unless input is iqm. these models will not run any conversion at all: iqm. +; 4. also, dae models need to flip their UVs coordinates (see -U flag below). -[cook anims] ; process all models to extract animlist -TOOLS/ass2iqe.EXE -L -o OUTPUT INPUT 2> NUL - -[cook model && dae &&] ; pass dae, reject iqm,iqe or any other model +[cook model && dae] ; pass dae, reject iqm,iqe or any other model FLAGS= TOOLS/ass2iqe.EXE FLAGS -o OUTPUT INPUT -U 2> NUL -> iqe @@ -184,6 +180,10 @@ TOOLS/ass2iqe.EXE FLAGS -o OUTPUT INPUT 2> NUL -> iqe [cook model && !iqm] TOOLS/iqe2iqm.EXE OUTPUT INPUT > NUL -> iqm +[cook anim] +FLAGS= +TOOLS/ass2iqe.EXE FLAGS -L -o OUTPUT INPUT 2> NUL -> animlist.txt + ; ------------------------------------------------------------------------------ ; cook localization files @@ -204,5 +204,5 @@ TOOLS/iqe2iqm.EXE OUTPUT INPUT > NUL -> iqm ; hint: use plain `0` to exclude those files we would like to directly stream within the final zipfile (flac,mp3,adpcm wav,...) [compress] -0|ULZ=texture,image,model,audio,font,text,shader,script,animlist -0=video,flac,ogg,wav,mp1,mp3,jpg,png \ No newline at end of file +0|ULZ=texture,image,model,audio,font,text,shader,script +0=video,flac,ogg,wav,mp1,mp3,jpg,png